Line data Source code
1 : use std::collections::HashMap;
2 : use std::num::NonZeroUsize;
3 : use std::os::fd::RawFd;
4 : use std::sync::atomic::AtomicU64;
5 : use std::sync::{Arc, Mutex};
6 : use std::time::{Duration, Instant};
7 :
8 : use enum_map::{Enum as _, EnumMap};
9 : use futures::Future;
10 : use metrics::{
11 : Counter, CounterVec, GaugeVec, Histogram, HistogramVec, IntCounter, IntCounterPair,
12 : IntCounterPairVec, IntCounterVec, IntGauge, IntGaugeVec, UIntGauge, UIntGaugeVec,
13 : register_counter_vec, register_gauge_vec, register_histogram, register_histogram_vec,
14 : register_int_counter, register_int_counter_pair_vec, register_int_counter_vec,
15 : register_int_gauge, register_int_gauge_vec, register_uint_gauge, register_uint_gauge_vec,
16 : };
17 : use once_cell::sync::Lazy;
18 : use pageserver_api::config::{
19 : PageServicePipeliningConfig, PageServicePipeliningConfigPipelined,
20 : PageServiceProtocolPipelinedExecutionStrategy,
21 : };
22 : use pageserver_api::models::InMemoryLayerInfo;
23 : use pageserver_api::shard::TenantShardId;
24 : use postgres_backend::{QueryError, is_expected_io_error};
25 : use pq_proto::framed::ConnectionError;
26 : use strum::{EnumCount, IntoEnumIterator as _, VariantNames};
27 : use strum_macros::{IntoStaticStr, VariantNames};
28 : use utils::id::TimelineId;
29 :
30 : use crate::config;
31 : use crate::config::PageServerConf;
32 : use crate::context::{PageContentKind, RequestContext};
33 : use crate::pgdatadir_mapping::DatadirModificationStats;
34 : use crate::task_mgr::TaskKind;
35 : use crate::tenant::Timeline;
36 : use crate::tenant::layer_map::LayerMap;
37 : use crate::tenant::mgr::TenantSlot;
38 : use crate::tenant::storage_layer::{InMemoryLayer, PersistentLayerDesc};
39 : use crate::tenant::tasks::BackgroundLoopKind;
40 : use crate::tenant::throttle::ThrottleResult;
41 :
42 : /// Prometheus histogram buckets (in seconds) for operations in the critical
43 : /// path. In other words, operations that directly affect that latency of user
44 : /// queries.
45 : ///
46 : /// The buckets capture the majority of latencies in the microsecond and
47 : /// millisecond range but also extend far enough up to distinguish "bad" from
48 : /// "really bad".
49 : const CRITICAL_OP_BUCKETS: &[f64] = &[
50 : 0.000_001, 0.000_010, 0.000_100, // 1 us, 10 us, 100 us
51 : 0.001_000, 0.010_000, 0.100_000, // 1 ms, 10 ms, 100 ms
52 : 1.0, 10.0, 100.0, // 1 s, 10 s, 100 s
53 : ];
54 :
55 : // Metrics collected on operations on the storage repository.
56 : #[derive(Debug, VariantNames, IntoStaticStr)]
57 : #[strum(serialize_all = "kebab_case")]
58 : pub(crate) enum StorageTimeOperation {
59 : #[strum(serialize = "layer flush")]
60 : LayerFlush,
61 :
62 : #[strum(serialize = "layer flush delay")]
63 : LayerFlushDelay,
64 :
65 : #[strum(serialize = "compact")]
66 : Compact,
67 :
68 : #[strum(serialize = "create images")]
69 : CreateImages,
70 :
71 : #[strum(serialize = "logical size")]
72 : LogicalSize,
73 :
74 : #[strum(serialize = "imitate logical size")]
75 : ImitateLogicalSize,
76 :
77 : #[strum(serialize = "load layer map")]
78 : LoadLayerMap,
79 :
80 : #[strum(serialize = "gc")]
81 : Gc,
82 :
83 : #[strum(serialize = "find gc cutoffs")]
84 : FindGcCutoffs,
85 : }
86 :
87 420 : pub(crate) static STORAGE_TIME_SUM_PER_TIMELINE: Lazy<CounterVec> = Lazy::new(|| {
88 420 : register_counter_vec!(
89 420 : "pageserver_storage_operations_seconds_sum",
90 420 : "Total time spent on storage operations with operation, tenant and timeline dimensions",
91 420 : &["operation", "tenant_id", "shard_id", "timeline_id"],
92 420 : )
93 420 : .expect("failed to define a metric")
94 420 : });
95 :
96 420 : pub(crate) static STORAGE_TIME_COUNT_PER_TIMELINE: Lazy<IntCounterVec> = Lazy::new(|| {
97 420 : register_int_counter_vec!(
98 420 : "pageserver_storage_operations_seconds_count",
99 420 : "Count of storage operations with operation, tenant and timeline dimensions",
100 420 : &["operation", "tenant_id", "shard_id", "timeline_id"],
101 420 : )
102 420 : .expect("failed to define a metric")
103 420 : });
104 :
105 : // Buckets for background operation duration in seconds, like compaction, GC, size calculation.
106 : const STORAGE_OP_BUCKETS: &[f64] = &[0.010, 0.100, 1.0, 10.0, 100.0, 1000.0];
107 :
108 420 : pub(crate) static STORAGE_TIME_GLOBAL: Lazy<HistogramVec> = Lazy::new(|| {
109 420 : register_histogram_vec!(
110 420 : "pageserver_storage_operations_seconds_global",
111 420 : "Time spent on storage operations",
112 420 : &["operation"],
113 420 : STORAGE_OP_BUCKETS.into(),
114 420 : )
115 420 : .expect("failed to define a metric")
116 420 : });
117 :
118 : /// Measures layers visited per read (i.e. read amplification).
119 : ///
120 : /// NB: for a batch, we count all visited layers towards each read. While the cost of layer visits
121 : /// are amortized across the batch, and some layers may not intersect with a given key, each visited
122 : /// layer contributes directly to the observed latency for every read in the batch, which is what we
123 : /// care about.
124 420 : pub(crate) static LAYERS_PER_READ: Lazy<HistogramVec> = Lazy::new(|| {
125 420 : register_histogram_vec!(
126 420 : "pageserver_layers_per_read",
127 420 : "Layers visited to serve a single read (read amplification). In a batch, all visited layers count towards every read.",
128 420 : &["tenant_id", "shard_id", "timeline_id"],
129 420 : // Low resolution to reduce cardinality.
130 420 : vec![4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0],
131 420 : )
132 420 : .expect("failed to define a metric")
133 420 : });
134 :
135 412 : pub(crate) static LAYERS_PER_READ_GLOBAL: Lazy<Histogram> = Lazy::new(|| {
136 412 : register_histogram!(
137 412 : "pageserver_layers_per_read_global",
138 412 : "Layers visited to serve a single read (read amplification). In a batch, all visited layers count towards every read.",
139 412 : vec![1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0],
140 412 : )
141 412 : .expect("failed to define a metric")
142 412 : });
143 :
144 412 : pub(crate) static LAYERS_PER_READ_BATCH_GLOBAL: Lazy<Histogram> = Lazy::new(|| {
145 412 : register_histogram!(
146 412 : "pageserver_layers_per_read_batch_global",
147 412 : "Layers visited to serve a single read batch (read amplification), regardless of number of reads.",
148 412 : vec![
149 412 : 1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0
150 412 : ],
151 412 : )
152 412 : .expect("failed to define a metric")
153 412 : });
154 :
155 412 : pub(crate) static LAYERS_PER_READ_AMORTIZED_GLOBAL: Lazy<Histogram> = Lazy::new(|| {
156 412 : register_histogram!(
157 412 : "pageserver_layers_per_read_amortized_global",
158 412 : "Layers visited to serve a single read (read amplification). Amortized across a batch: \
159 412 : all visited layers are divided by number of reads.",
160 412 : vec![
161 412 : 1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0
162 412 : ],
163 412 : )
164 412 : .expect("failed to define a metric")
165 412 : });
166 :
167 412 : pub(crate) static DELTAS_PER_READ_GLOBAL: Lazy<Histogram> = Lazy::new(|| {
168 412 : // We expect this to be low because of Postgres checkpoints. Let's see if that holds.
169 412 : register_histogram!(
170 412 : "pageserver_deltas_per_read_global",
171 412 : "Number of delta pages applied to image page per read",
172 412 : vec![0.0, 1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0],
173 412 : )
174 412 : .expect("failed to define a metric")
175 412 : });
176 :
177 0 : pub(crate) static CONCURRENT_INITDBS: Lazy<UIntGauge> = Lazy::new(|| {
178 0 : register_uint_gauge!(
179 0 : "pageserver_concurrent_initdb",
180 0 : "Number of initdb processes running"
181 0 : )
182 0 : .expect("failed to define a metric")
183 0 : });
184 :
185 0 : pub(crate) static INITDB_SEMAPHORE_ACQUISITION_TIME: Lazy<Histogram> = Lazy::new(|| {
186 0 : register_histogram!(
187 0 : "pageserver_initdb_semaphore_seconds_global",
188 0 : "Time spent getting a permit from the global initdb semaphore",
189 0 : STORAGE_OP_BUCKETS.into()
190 0 : )
191 0 : .expect("failed to define metric")
192 0 : });
193 :
194 0 : pub(crate) static INITDB_RUN_TIME: Lazy<Histogram> = Lazy::new(|| {
195 0 : register_histogram!(
196 0 : "pageserver_initdb_seconds_global",
197 0 : "Time spent performing initdb",
198 0 : STORAGE_OP_BUCKETS.into()
199 0 : )
200 0 : .expect("failed to define metric")
201 0 : });
202 :
203 : pub(crate) struct GetVectoredLatency {
204 : map: EnumMap<TaskKind, Option<Histogram>>,
205 : }
206 :
207 : #[allow(dead_code)]
208 : pub(crate) struct ScanLatency {
209 : map: EnumMap<TaskKind, Option<Histogram>>,
210 : }
211 :
212 : impl GetVectoredLatency {
213 : // Only these task types perform vectored gets. Filter all other tasks out to reduce total
214 : // cardinality of the metric.
215 : const TRACKED_TASK_KINDS: [TaskKind; 2] = [TaskKind::Compaction, TaskKind::PageRequestHandler];
216 :
217 39480 : pub(crate) fn for_task_kind(&self, task_kind: TaskKind) -> Option<&Histogram> {
218 39480 : self.map[task_kind].as_ref()
219 39480 : }
220 : }
221 :
222 : impl ScanLatency {
223 : // Only these task types perform vectored gets. Filter all other tasks out to reduce total
224 : // cardinality of the metric.
225 : const TRACKED_TASK_KINDS: [TaskKind; 1] = [TaskKind::PageRequestHandler];
226 :
227 24 : pub(crate) fn for_task_kind(&self, task_kind: TaskKind) -> Option<&Histogram> {
228 24 : self.map[task_kind].as_ref()
229 24 : }
230 : }
231 :
232 : pub(crate) struct ScanLatencyOngoingRecording<'a> {
233 : parent: &'a Histogram,
234 : start: std::time::Instant,
235 : }
236 :
237 : impl<'a> ScanLatencyOngoingRecording<'a> {
238 0 : pub(crate) fn start_recording(parent: &'a Histogram) -> ScanLatencyOngoingRecording<'a> {
239 0 : let start = Instant::now();
240 0 : ScanLatencyOngoingRecording { parent, start }
241 0 : }
242 :
243 0 : pub(crate) fn observe(self) {
244 0 : let elapsed = self.start.elapsed();
245 0 : self.parent.observe(elapsed.as_secs_f64());
246 0 : }
247 : }
248 :
249 404 : pub(crate) static GET_VECTORED_LATENCY: Lazy<GetVectoredLatency> = Lazy::new(|| {
250 404 : let inner = register_histogram_vec!(
251 404 : "pageserver_get_vectored_seconds",
252 404 : "Time spent in get_vectored.",
253 404 : &["task_kind"],
254 404 : CRITICAL_OP_BUCKETS.into(),
255 404 : )
256 404 : .expect("failed to define a metric");
257 404 :
258 404 : GetVectoredLatency {
259 12524 : map: EnumMap::from_array(std::array::from_fn(|task_kind_idx| {
260 12524 : let task_kind = TaskKind::from_usize(task_kind_idx);
261 12524 :
262 12524 : if GetVectoredLatency::TRACKED_TASK_KINDS.contains(&task_kind) {
263 808 : let task_kind = task_kind.into();
264 808 : Some(inner.with_label_values(&[task_kind]))
265 : } else {
266 11716 : None
267 : }
268 12524 : })),
269 404 : }
270 404 : });
271 :
272 8 : pub(crate) static SCAN_LATENCY: Lazy<ScanLatency> = Lazy::new(|| {
273 8 : let inner = register_histogram_vec!(
274 8 : "pageserver_scan_seconds",
275 8 : "Time spent in scan.",
276 8 : &["task_kind"],
277 8 : CRITICAL_OP_BUCKETS.into(),
278 8 : )
279 8 : .expect("failed to define a metric");
280 8 :
281 8 : ScanLatency {
282 248 : map: EnumMap::from_array(std::array::from_fn(|task_kind_idx| {
283 248 : let task_kind = TaskKind::from_usize(task_kind_idx);
284 248 :
285 248 : if ScanLatency::TRACKED_TASK_KINDS.contains(&task_kind) {
286 8 : let task_kind = task_kind.into();
287 8 : Some(inner.with_label_values(&[task_kind]))
288 : } else {
289 240 : None
290 : }
291 248 : })),
292 8 : }
293 8 : });
294 :
295 : pub(crate) struct PageCacheMetricsForTaskKind {
296 : pub read_accesses_immutable: IntCounter,
297 : pub read_hits_immutable: IntCounter,
298 : }
299 :
300 : pub(crate) struct PageCacheMetrics {
301 : map: EnumMap<TaskKind, EnumMap<PageContentKind, PageCacheMetricsForTaskKind>>,
302 : }
303 :
304 196 : static PAGE_CACHE_READ_HITS: Lazy<IntCounterVec> = Lazy::new(|| {
305 196 : register_int_counter_vec!(
306 196 : "pageserver_page_cache_read_hits_total",
307 196 : "Number of read accesses to the page cache that hit",
308 196 : &["task_kind", "key_kind", "content_kind", "hit_kind"]
309 196 : )
310 196 : .expect("failed to define a metric")
311 196 : });
312 :
313 196 : static PAGE_CACHE_READ_ACCESSES: Lazy<IntCounterVec> = Lazy::new(|| {
314 196 : register_int_counter_vec!(
315 196 : "pageserver_page_cache_read_accesses_total",
316 196 : "Number of read accesses to the page cache",
317 196 : &["task_kind", "key_kind", "content_kind"]
318 196 : )
319 196 : .expect("failed to define a metric")
320 196 : });
321 :
322 196 : pub(crate) static PAGE_CACHE: Lazy<PageCacheMetrics> = Lazy::new(|| PageCacheMetrics {
323 6076 : map: EnumMap::from_array(std::array::from_fn(|task_kind| {
324 6076 : let task_kind = TaskKind::from_usize(task_kind);
325 6076 : let task_kind: &'static str = task_kind.into();
326 48608 : EnumMap::from_array(std::array::from_fn(|content_kind| {
327 48608 : let content_kind = PageContentKind::from_usize(content_kind);
328 48608 : let content_kind: &'static str = content_kind.into();
329 48608 : PageCacheMetricsForTaskKind {
330 48608 : read_accesses_immutable: {
331 48608 : PAGE_CACHE_READ_ACCESSES
332 48608 : .get_metric_with_label_values(&[task_kind, "immutable", content_kind])
333 48608 : .unwrap()
334 48608 : },
335 48608 :
336 48608 : read_hits_immutable: {
337 48608 : PAGE_CACHE_READ_HITS
338 48608 : .get_metric_with_label_values(&[task_kind, "immutable", content_kind, "-"])
339 48608 : .unwrap()
340 48608 : },
341 48608 : }
342 48608 : }))
343 6076 : })),
344 196 : });
345 :
346 : impl PageCacheMetrics {
347 1941870 : pub(crate) fn for_ctx(&self, ctx: &RequestContext) -> &PageCacheMetricsForTaskKind {
348 1941870 : &self.map[ctx.task_kind()][ctx.page_content_kind()]
349 1941870 : }
350 : }
351 :
352 : pub(crate) struct PageCacheSizeMetrics {
353 : pub max_bytes: UIntGauge,
354 :
355 : pub current_bytes_immutable: UIntGauge,
356 : }
357 :
358 196 : static PAGE_CACHE_SIZE_CURRENT_BYTES: Lazy<UIntGaugeVec> = Lazy::new(|| {
359 196 : register_uint_gauge_vec!(
360 196 : "pageserver_page_cache_size_current_bytes",
361 196 : "Current size of the page cache in bytes, by key kind",
362 196 : &["key_kind"]
363 196 : )
364 196 : .expect("failed to define a metric")
365 196 : });
366 :
367 : pub(crate) static PAGE_CACHE_SIZE: Lazy<PageCacheSizeMetrics> =
368 196 : Lazy::new(|| PageCacheSizeMetrics {
369 196 : max_bytes: {
370 196 : register_uint_gauge!(
371 196 : "pageserver_page_cache_size_max_bytes",
372 196 : "Maximum size of the page cache in bytes"
373 196 : )
374 196 : .expect("failed to define a metric")
375 196 : },
376 196 : current_bytes_immutable: {
377 196 : PAGE_CACHE_SIZE_CURRENT_BYTES
378 196 : .get_metric_with_label_values(&["immutable"])
379 196 : .unwrap()
380 196 : },
381 196 : });
382 :
383 : pub(crate) mod page_cache_eviction_metrics {
384 : use std::num::NonZeroUsize;
385 :
386 : use metrics::{IntCounter, IntCounterVec, register_int_counter_vec};
387 : use once_cell::sync::Lazy;
388 :
389 : #[derive(Clone, Copy)]
390 : pub(crate) enum Outcome {
391 : FoundSlotUnused { iters: NonZeroUsize },
392 : FoundSlotEvicted { iters: NonZeroUsize },
393 : ItersExceeded { iters: NonZeroUsize },
394 : }
395 :
396 196 : static ITERS_TOTAL_VEC: Lazy<IntCounterVec> = Lazy::new(|| {
397 196 : register_int_counter_vec!(
398 196 : "pageserver_page_cache_find_victim_iters_total",
399 196 : "Counter for the number of iterations in the find_victim loop",
400 196 : &["outcome"],
401 196 : )
402 196 : .expect("failed to define a metric")
403 196 : });
404 :
405 196 : static CALLS_VEC: Lazy<IntCounterVec> = Lazy::new(|| {
406 196 : register_int_counter_vec!(
407 196 : "pageserver_page_cache_find_victim_calls",
408 196 : "Incremented at the end of each find_victim() call.\
409 196 : Filter by outcome to get e.g., eviction rate.",
410 196 : &["outcome"]
411 196 : )
412 196 : .unwrap()
413 196 : });
414 :
415 61782 : pub(crate) fn observe(outcome: Outcome) {
416 : macro_rules! dry {
417 : ($label:literal, $iters:expr) => {{
418 : static LABEL: &'static str = $label;
419 : static ITERS_TOTAL: Lazy<IntCounter> =
420 232 : Lazy::new(|| ITERS_TOTAL_VEC.with_label_values(&[LABEL]));
421 : static CALLS: Lazy<IntCounter> =
422 232 : Lazy::new(|| CALLS_VEC.with_label_values(&[LABEL]));
423 : ITERS_TOTAL.inc_by(($iters.get()) as u64);
424 : CALLS.inc();
425 : }};
426 : }
427 61782 : match outcome {
428 3160 : Outcome::FoundSlotUnused { iters } => dry!("found_empty", iters),
429 58622 : Outcome::FoundSlotEvicted { iters } => {
430 58622 : dry!("found_evicted", iters)
431 : }
432 0 : Outcome::ItersExceeded { iters } => {
433 0 : dry!("err_iters_exceeded", iters);
434 0 : super::page_cache_errors_inc(super::PageCacheErrorKind::EvictIterLimit);
435 0 : }
436 : }
437 61782 : }
438 : }
439 :
440 0 : static PAGE_CACHE_ERRORS: Lazy<IntCounterVec> = Lazy::new(|| {
441 0 : register_int_counter_vec!(
442 0 : "page_cache_errors_total",
443 0 : "Number of timeouts while acquiring a pinned slot in the page cache",
444 0 : &["error_kind"]
445 0 : )
446 0 : .expect("failed to define a metric")
447 0 : });
448 :
449 : #[derive(IntoStaticStr)]
450 : #[strum(serialize_all = "kebab_case")]
451 : pub(crate) enum PageCacheErrorKind {
452 : AcquirePinnedSlotTimeout,
453 : EvictIterLimit,
454 : }
455 :
456 0 : pub(crate) fn page_cache_errors_inc(error_kind: PageCacheErrorKind) {
457 0 : PAGE_CACHE_ERRORS
458 0 : .get_metric_with_label_values(&[error_kind.into()])
459 0 : .unwrap()
460 0 : .inc();
461 0 : }
462 :
463 44 : pub(crate) static WAIT_LSN_TIME: Lazy<Histogram> = Lazy::new(|| {
464 44 : register_histogram!(
465 44 : "pageserver_wait_lsn_seconds",
466 44 : "Time spent waiting for WAL to arrive. Updated on completion of the wait_lsn operation.",
467 44 : CRITICAL_OP_BUCKETS.into(),
468 44 : )
469 44 : .expect("failed to define a metric")
470 44 : });
471 :
472 420 : pub(crate) static WAIT_LSN_START_FINISH_COUNTERPAIR: Lazy<IntCounterPairVec> = Lazy::new(|| {
473 420 : register_int_counter_pair_vec!(
474 420 : "pageserver_wait_lsn_started_count",
475 420 : "Number of wait_lsn operations started.",
476 420 : "pageserver_wait_lsn_finished_count",
477 420 : "Number of wait_lsn operations finished.",
478 420 : &["tenant_id", "shard_id", "timeline_id"],
479 420 : )
480 420 : .expect("failed to define a metric")
481 420 : });
482 :
483 420 : pub(crate) static WAIT_LSN_IN_PROGRESS_MICROS: Lazy<IntCounterVec> = Lazy::new(|| {
484 420 : register_int_counter_vec!(
485 420 : "pageserver_wait_lsn_in_progress_micros",
486 420 : "Time spent waiting for WAL to arrive, by timeline_id. Updated periodically while waiting.",
487 420 : &["tenant_id", "shard_id", "timeline_id"],
488 420 : )
489 420 : .expect("failed to define a metric")
490 420 : });
491 :
492 420 : pub(crate) static WAIT_LSN_IN_PROGRESS_GLOBAL_MICROS: Lazy<IntCounter> = Lazy::new(|| {
493 420 : register_int_counter!(
494 420 : "pageserver_wait_lsn_in_progress_micros_global",
495 420 : "Time spent waiting for WAL to arrive, globally. Updated periodically while waiting."
496 420 : )
497 420 : .expect("failed to define a metric")
498 420 : });
499 :
500 : pub(crate) mod wait_ondemand_download_time {
501 : use super::*;
502 : const WAIT_ONDEMAND_DOWNLOAD_TIME_BUCKETS: &[f64] = &[
503 : 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, // 10 ms - 100ms
504 : 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, // 100ms to 1s
505 : 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, // 1s to 10s
506 : 10.0, 20.0, 30.0, 40.0, 50.0, 60.0, // 10s to 1m
507 : ];
508 :
509 : /// The task kinds for which we want to track wait times for on-demand downloads.
510 : /// Other task kinds' wait times are accumulated in label value `unknown`.
511 : pub(crate) const WAIT_ONDEMAND_DOWNLOAD_METRIC_TASK_KINDS: [TaskKind; 2] = [
512 : TaskKind::PageRequestHandler,
513 : TaskKind::WalReceiverConnectionHandler,
514 : ];
515 :
516 0 : pub(crate) static WAIT_ONDEMAND_DOWNLOAD_TIME_GLOBAL: Lazy<Vec<Histogram>> = Lazy::new(|| {
517 0 : let histo = register_histogram_vec!(
518 0 : "pageserver_wait_ondemand_download_seconds_global",
519 0 : "Observations are individual tasks' wait times for on-demand downloads. \
520 0 : If N tasks coalesce on an on-demand download, and it takes 10s, than we observe N * 10s.",
521 0 : &["task_kind"],
522 0 : WAIT_ONDEMAND_DOWNLOAD_TIME_BUCKETS.into(),
523 0 : )
524 0 : .expect("failed to define a metric");
525 0 : WAIT_ONDEMAND_DOWNLOAD_METRIC_TASK_KINDS
526 0 : .iter()
527 0 : .map(|task_kind| histo.with_label_values(&[task_kind.into()]))
528 0 : .collect::<Vec<_>>()
529 0 : });
530 :
531 420 : pub(crate) static WAIT_ONDEMAND_DOWNLOAD_TIME_SUM: Lazy<CounterVec> = Lazy::new(|| {
532 420 : register_counter_vec!(
533 420 : // use a name that _could_ be evolved into a per-timeline histogram later
534 420 : "pageserver_wait_ondemand_download_seconds_sum",
535 420 : "Like `pageserver_wait_ondemand_download_seconds_global` but per timeline",
536 420 : &["tenant_id", "shard_id", "timeline_id", "task_kind"],
537 420 : )
538 420 : .unwrap()
539 420 : });
540 :
541 : pub struct WaitOndemandDownloadTimeSum {
542 : counters: [Counter; WAIT_ONDEMAND_DOWNLOAD_METRIC_TASK_KINDS.len()],
543 : }
544 :
545 : impl WaitOndemandDownloadTimeSum {
546 924 : pub(crate) fn new(tenant_id: &str, shard_id: &str, timeline_id: &str) -> Self {
547 924 : let counters = WAIT_ONDEMAND_DOWNLOAD_METRIC_TASK_KINDS
548 924 : .iter()
549 1848 : .map(|task_kind| {
550 1848 : WAIT_ONDEMAND_DOWNLOAD_TIME_SUM
551 1848 : .get_metric_with_label_values(&[
552 1848 : tenant_id,
553 1848 : shard_id,
554 1848 : timeline_id,
555 1848 : task_kind.into(),
556 1848 : ])
557 1848 : .unwrap()
558 1848 : })
559 924 : .collect::<Vec<_>>();
560 924 : Self {
561 924 : counters: counters.try_into().unwrap(),
562 924 : }
563 924 : }
564 48 : pub(crate) fn observe(&self, task_kind: TaskKind, duration: Duration) {
565 48 : let maybe = WAIT_ONDEMAND_DOWNLOAD_METRIC_TASK_KINDS
566 48 : .iter()
567 48 : .enumerate()
568 96 : .find(|(_, kind)| **kind == task_kind);
569 48 : let Some((idx, _)) = maybe else {
570 48 : return;
571 : };
572 0 : WAIT_ONDEMAND_DOWNLOAD_TIME_GLOBAL[idx].observe(duration.as_secs_f64());
573 0 : let counter = &self.counters[idx];
574 0 : counter.inc_by(duration.as_secs_f64());
575 48 : }
576 : }
577 :
578 20 : pub(crate) fn shutdown_timeline(tenant_id: &str, shard_id: &str, timeline_id: &str) {
579 60 : for task_kind in WAIT_ONDEMAND_DOWNLOAD_METRIC_TASK_KINDS {
580 40 : let _ = WAIT_ONDEMAND_DOWNLOAD_TIME_SUM.remove_label_values(&[
581 40 : tenant_id,
582 40 : shard_id,
583 40 : timeline_id,
584 40 : task_kind.into(),
585 40 : ]);
586 40 : }
587 20 : }
588 :
589 0 : pub(crate) fn preinitialize_global_metrics() {
590 0 : Lazy::force(&WAIT_ONDEMAND_DOWNLOAD_TIME_GLOBAL);
591 0 : }
592 : }
593 :
594 420 : static LAST_RECORD_LSN: Lazy<IntGaugeVec> = Lazy::new(|| {
595 420 : register_int_gauge_vec!(
596 420 : "pageserver_last_record_lsn",
597 420 : "Last record LSN grouped by timeline",
598 420 : &["tenant_id", "shard_id", "timeline_id"]
599 420 : )
600 420 : .expect("failed to define a metric")
601 420 : });
602 :
603 420 : static DISK_CONSISTENT_LSN: Lazy<IntGaugeVec> = Lazy::new(|| {
604 420 : register_int_gauge_vec!(
605 420 : "pageserver_disk_consistent_lsn",
606 420 : "Disk consistent LSN grouped by timeline",
607 420 : &["tenant_id", "shard_id", "timeline_id"]
608 420 : )
609 420 : .expect("failed to define a metric")
610 420 : });
611 :
612 420 : pub(crate) static PROJECTED_REMOTE_CONSISTENT_LSN: Lazy<UIntGaugeVec> = Lazy::new(|| {
613 420 : register_uint_gauge_vec!(
614 420 : "pageserver_projected_remote_consistent_lsn",
615 420 : "Projected remote consistent LSN grouped by timeline",
616 420 : &["tenant_id", "shard_id", "timeline_id"]
617 420 : )
618 420 : .expect("failed to define a metric")
619 420 : });
620 :
621 420 : static PITR_HISTORY_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
622 420 : register_uint_gauge_vec!(
623 420 : "pageserver_pitr_history_size",
624 420 : "Data written since PITR cutoff on this timeline",
625 420 : &["tenant_id", "shard_id", "timeline_id"]
626 420 : )
627 420 : .expect("failed to define a metric")
628 420 : });
629 :
630 : #[derive(
631 240 : strum_macros::EnumIter,
632 0 : strum_macros::EnumString,
633 : strum_macros::Display,
634 : strum_macros::IntoStaticStr,
635 : )]
636 : #[strum(serialize_all = "kebab_case")]
637 : pub(crate) enum LayerKind {
638 : Delta,
639 : Image,
640 : }
641 :
642 : #[derive(
643 100 : strum_macros::EnumIter,
644 0 : strum_macros::EnumString,
645 : strum_macros::Display,
646 : strum_macros::IntoStaticStr,
647 : )]
648 : #[strum(serialize_all = "kebab_case")]
649 : pub(crate) enum LayerLevel {
650 : // We don't track the currently open ephemeral layer, since there's always exactly 1 and its
651 : // size changes. See `TIMELINE_EPHEMERAL_BYTES`.
652 : Frozen,
653 : L0,
654 : L1,
655 : }
656 :
657 412 : static TIMELINE_LAYER_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
658 412 : register_uint_gauge_vec!(
659 412 : "pageserver_layer_bytes",
660 412 : "Sum of frozen, L0, and L1 layer physical sizes in bytes (excluding the open ephemeral layer)",
661 412 : &["tenant_id", "shard_id", "timeline_id", "level", "kind"]
662 412 : )
663 412 : .expect("failed to define a metric")
664 412 : });
665 :
666 412 : static TIMELINE_LAYER_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
667 412 : register_uint_gauge_vec!(
668 412 : "pageserver_layer_count",
669 412 : "Number of frozen, L0, and L1 layers (excluding the open ephemeral layer)",
670 412 : &["tenant_id", "shard_id", "timeline_id", "level", "kind"]
671 412 : )
672 412 : .expect("failed to define a metric")
673 412 : });
674 :
675 420 : static TIMELINE_ARCHIVE_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
676 420 : register_uint_gauge_vec!(
677 420 : "pageserver_archive_size",
678 420 : "Timeline's logical size if it is considered eligible for archival (outside PITR window), else zero",
679 420 : &["tenant_id", "shard_id", "timeline_id"]
680 420 : )
681 420 : .expect("failed to define a metric")
682 420 : });
683 :
684 420 : static STANDBY_HORIZON: Lazy<IntGaugeVec> = Lazy::new(|| {
685 420 : register_int_gauge_vec!(
686 420 : "pageserver_standby_horizon",
687 420 : "Standby apply LSN for which GC is hold off, by timeline.",
688 420 : &["tenant_id", "shard_id", "timeline_id"]
689 420 : )
690 420 : .expect("failed to define a metric")
691 420 : });
692 :
693 420 : static RESIDENT_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
694 420 : register_uint_gauge_vec!(
695 420 : "pageserver_resident_physical_size",
696 420 : "The size of the layer files present in the pageserver's filesystem, for attached locations.",
697 420 : &["tenant_id", "shard_id", "timeline_id"]
698 420 : )
699 420 : .expect("failed to define a metric")
700 420 : });
701 :
702 420 : static VISIBLE_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
703 420 : register_uint_gauge_vec!(
704 420 : "pageserver_visible_physical_size",
705 420 : "The size of the layer files present in the pageserver's filesystem.",
706 420 : &["tenant_id", "shard_id", "timeline_id"]
707 420 : )
708 420 : .expect("failed to define a metric")
709 420 : });
710 :
711 412 : pub(crate) static RESIDENT_PHYSICAL_SIZE_GLOBAL: Lazy<UIntGauge> = Lazy::new(|| {
712 412 : register_uint_gauge!(
713 412 : "pageserver_resident_physical_size_global",
714 412 : "Like `pageserver_resident_physical_size`, but without tenant/timeline dimensions."
715 412 : )
716 412 : .expect("failed to define a metric")
717 412 : });
718 :
719 420 : static REMOTE_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
720 420 : register_uint_gauge_vec!(
721 420 : "pageserver_remote_physical_size",
722 420 : "The size of the layer files present in the remote storage that are listed in the remote index_part.json.",
723 420 : // Corollary: If any files are missing from the index part, they won't be included here.
724 420 : &["tenant_id", "shard_id", "timeline_id"]
725 420 : )
726 420 : .expect("failed to define a metric")
727 420 : });
728 :
729 420 : static REMOTE_PHYSICAL_SIZE_GLOBAL: Lazy<UIntGauge> = Lazy::new(|| {
730 420 : register_uint_gauge!(
731 420 : "pageserver_remote_physical_size_global",
732 420 : "Like `pageserver_remote_physical_size`, but without tenant/timeline dimensions."
733 420 : )
734 420 : .expect("failed to define a metric")
735 420 : });
736 :
737 12 : pub(crate) static REMOTE_ONDEMAND_DOWNLOADED_LAYERS: Lazy<IntCounter> = Lazy::new(|| {
738 12 : register_int_counter!(
739 12 : "pageserver_remote_ondemand_downloaded_layers_total",
740 12 : "Total on-demand downloaded layers"
741 12 : )
742 12 : .unwrap()
743 12 : });
744 :
745 12 : pub(crate) static REMOTE_ONDEMAND_DOWNLOADED_BYTES: Lazy<IntCounter> = Lazy::new(|| {
746 12 : register_int_counter!(
747 12 : "pageserver_remote_ondemand_downloaded_bytes_total",
748 12 : "Total bytes of layers on-demand downloaded",
749 12 : )
750 12 : .unwrap()
751 12 : });
752 :
753 420 : static CURRENT_LOGICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
754 420 : register_uint_gauge_vec!(
755 420 : "pageserver_current_logical_size",
756 420 : "Current logical size grouped by timeline",
757 420 : &["tenant_id", "shard_id", "timeline_id"]
758 420 : )
759 420 : .expect("failed to define current logical size metric")
760 420 : });
761 :
762 420 : static AUX_FILE_SIZE: Lazy<IntGaugeVec> = Lazy::new(|| {
763 420 : register_int_gauge_vec!(
764 420 : "pageserver_aux_file_estimated_size",
765 420 : "The size of all aux files for a timeline in aux file v2 store.",
766 420 : &["tenant_id", "shard_id", "timeline_id"]
767 420 : )
768 420 : .expect("failed to define a metric")
769 420 : });
770 :
771 420 : static VALID_LSN_LEASE_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
772 420 : register_uint_gauge_vec!(
773 420 : "pageserver_valid_lsn_lease_count",
774 420 : "The number of valid leases after refreshing gc info.",
775 420 : &["tenant_id", "shard_id", "timeline_id"],
776 420 : )
777 420 : .expect("failed to define a metric")
778 420 : });
779 :
780 0 : pub(crate) static CIRCUIT_BREAKERS_BROKEN: Lazy<IntCounter> = Lazy::new(|| {
781 0 : register_int_counter!(
782 0 : "pageserver_circuit_breaker_broken",
783 0 : "How many times a circuit breaker has broken"
784 0 : )
785 0 : .expect("failed to define a metric")
786 0 : });
787 :
788 0 : pub(crate) static CIRCUIT_BREAKERS_UNBROKEN: Lazy<IntCounter> = Lazy::new(|| {
789 0 : register_int_counter!(
790 0 : "pageserver_circuit_breaker_unbroken",
791 0 : "How many times a circuit breaker has been un-broken (recovered)"
792 0 : )
793 0 : .expect("failed to define a metric")
794 0 : });
795 :
796 404 : pub(crate) static COMPRESSION_IMAGE_INPUT_BYTES: Lazy<IntCounter> = Lazy::new(|| {
797 404 : register_int_counter!(
798 404 : "pageserver_compression_image_in_bytes_total",
799 404 : "Size of data written into image layers before compression"
800 404 : )
801 404 : .expect("failed to define a metric")
802 404 : });
803 :
804 404 : pub(crate) static COMPRESSION_IMAGE_INPUT_BYTES_CONSIDERED: Lazy<IntCounter> = Lazy::new(|| {
805 404 : register_int_counter!(
806 404 : "pageserver_compression_image_in_bytes_considered",
807 404 : "Size of potentially compressible data written into image layers before compression"
808 404 : )
809 404 : .expect("failed to define a metric")
810 404 : });
811 :
812 404 : pub(crate) static COMPRESSION_IMAGE_INPUT_BYTES_CHOSEN: Lazy<IntCounter> = Lazy::new(|| {
813 404 : register_int_counter!(
814 404 : "pageserver_compression_image_in_bytes_chosen",
815 404 : "Size of data whose compressed form was written into image layers"
816 404 : )
817 404 : .expect("failed to define a metric")
818 404 : });
819 :
820 404 : pub(crate) static COMPRESSION_IMAGE_OUTPUT_BYTES: Lazy<IntCounter> = Lazy::new(|| {
821 404 : register_int_counter!(
822 404 : "pageserver_compression_image_out_bytes_total",
823 404 : "Size of compressed image layer written"
824 404 : )
825 404 : .expect("failed to define a metric")
826 404 : });
827 :
828 20 : pub(crate) static RELSIZE_CACHE_ENTRIES: Lazy<UIntGauge> = Lazy::new(|| {
829 20 : register_uint_gauge!(
830 20 : "pageserver_relsize_cache_entries",
831 20 : "Number of entries in the relation size cache",
832 20 : )
833 20 : .expect("failed to define a metric")
834 20 : });
835 :
836 20 : pub(crate) static RELSIZE_CACHE_HITS: Lazy<IntCounter> = Lazy::new(|| {
837 20 : register_int_counter!("pageserver_relsize_cache_hits", "Relation size cache hits",)
838 20 : .expect("failed to define a metric")
839 20 : });
840 :
841 20 : pub(crate) static RELSIZE_CACHE_MISSES: Lazy<IntCounter> = Lazy::new(|| {
842 20 : register_int_counter!(
843 20 : "pageserver_relsize_cache_misses",
844 20 : "Relation size cache misses",
845 20 : )
846 20 : .expect("failed to define a metric")
847 20 : });
848 :
849 8 : pub(crate) static RELSIZE_CACHE_MISSES_OLD: Lazy<IntCounter> = Lazy::new(|| {
850 8 : register_int_counter!(
851 8 : "pageserver_relsize_cache_misses_old",
852 8 : "Relation size cache misses where the lookup LSN is older than the last relation update"
853 8 : )
854 8 : .expect("failed to define a metric")
855 8 : });
856 :
857 : pub(crate) mod initial_logical_size {
858 : use metrics::{IntCounter, IntCounterVec, register_int_counter, register_int_counter_vec};
859 : use once_cell::sync::Lazy;
860 :
861 : pub(crate) struct StartCalculation(IntCounterVec);
862 420 : pub(crate) static START_CALCULATION: Lazy<StartCalculation> = Lazy::new(|| {
863 420 : StartCalculation(
864 420 : register_int_counter_vec!(
865 420 : "pageserver_initial_logical_size_start_calculation",
866 420 : "Incremented each time we start an initial logical size calculation attempt. \
867 420 : The `circumstances` label provides some additional details.",
868 420 : &["attempt", "circumstances"]
869 420 : )
870 420 : .unwrap(),
871 420 : )
872 420 : });
873 :
874 : struct DropCalculation {
875 : first: IntCounter,
876 : retry: IntCounter,
877 : }
878 :
879 420 : static DROP_CALCULATION: Lazy<DropCalculation> = Lazy::new(|| {
880 420 : let vec = register_int_counter_vec!(
881 420 : "pageserver_initial_logical_size_drop_calculation",
882 420 : "Incremented each time we abort a started size calculation attmpt.",
883 420 : &["attempt"]
884 420 : )
885 420 : .unwrap();
886 420 : DropCalculation {
887 420 : first: vec.with_label_values(&["first"]),
888 420 : retry: vec.with_label_values(&["retry"]),
889 420 : }
890 420 : });
891 :
892 : pub(crate) struct Calculated {
893 : pub(crate) births: IntCounter,
894 : pub(crate) deaths: IntCounter,
895 : }
896 :
897 420 : pub(crate) static CALCULATED: Lazy<Calculated> = Lazy::new(|| Calculated {
898 420 : births: register_int_counter!(
899 420 : "pageserver_initial_logical_size_finish_calculation",
900 420 : "Incremented every time we finish calculation of initial logical size.\
901 420 : If everything is working well, this should happen at most once per Timeline object."
902 420 : )
903 420 : .unwrap(),
904 420 : deaths: register_int_counter!(
905 420 : "pageserver_initial_logical_size_drop_finished_calculation",
906 420 : "Incremented when we drop a finished initial logical size calculation result.\
907 420 : Mainly useful to turn pageserver_initial_logical_size_finish_calculation into a gauge."
908 420 : )
909 420 : .unwrap(),
910 420 : });
911 :
912 : pub(crate) struct OngoingCalculationGuard {
913 : inc_drop_calculation: Option<IntCounter>,
914 : }
915 :
916 : #[derive(strum_macros::IntoStaticStr)]
917 : pub(crate) enum StartCircumstances {
918 : EmptyInitial,
919 : SkippedConcurrencyLimiter,
920 : AfterBackgroundTasksRateLimit,
921 : }
922 :
923 : impl StartCalculation {
924 444 : pub(crate) fn first(&self, circumstances: StartCircumstances) -> OngoingCalculationGuard {
925 444 : let circumstances_label: &'static str = circumstances.into();
926 444 : self.0
927 444 : .with_label_values(&["first", circumstances_label])
928 444 : .inc();
929 444 : OngoingCalculationGuard {
930 444 : inc_drop_calculation: Some(DROP_CALCULATION.first.clone()),
931 444 : }
932 444 : }
933 0 : pub(crate) fn retry(&self, circumstances: StartCircumstances) -> OngoingCalculationGuard {
934 0 : let circumstances_label: &'static str = circumstances.into();
935 0 : self.0
936 0 : .with_label_values(&["retry", circumstances_label])
937 0 : .inc();
938 0 : OngoingCalculationGuard {
939 0 : inc_drop_calculation: Some(DROP_CALCULATION.retry.clone()),
940 0 : }
941 0 : }
942 : }
943 :
944 : impl Drop for OngoingCalculationGuard {
945 444 : fn drop(&mut self) {
946 444 : if let Some(counter) = self.inc_drop_calculation.take() {
947 0 : counter.inc();
948 444 : }
949 444 : }
950 : }
951 :
952 : impl OngoingCalculationGuard {
953 444 : pub(crate) fn calculation_result_saved(mut self) -> FinishedCalculationGuard {
954 444 : drop(self.inc_drop_calculation.take());
955 444 : CALCULATED.births.inc();
956 444 : FinishedCalculationGuard {
957 444 : inc_on_drop: CALCULATED.deaths.clone(),
958 444 : }
959 444 : }
960 : }
961 :
962 : pub(crate) struct FinishedCalculationGuard {
963 : inc_on_drop: IntCounter,
964 : }
965 :
966 : impl Drop for FinishedCalculationGuard {
967 12 : fn drop(&mut self) {
968 12 : self.inc_on_drop.inc();
969 12 : }
970 : }
971 :
972 : // context: https://github.com/neondatabase/neon/issues/5963
973 : pub(crate) static TIMELINES_WHERE_WALRECEIVER_GOT_APPROXIMATE_SIZE: Lazy<IntCounter> =
974 0 : Lazy::new(|| {
975 0 : register_int_counter!(
976 0 : "pageserver_initial_logical_size_timelines_where_walreceiver_got_approximate_size",
977 0 : "Counter for the following event: walreceiver calls\
978 0 : Timeline::get_current_logical_size() and it returns `Approximate` for the first time."
979 0 : )
980 0 : .unwrap()
981 0 : });
982 : }
983 :
984 0 : static DIRECTORY_ENTRIES_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
985 0 : register_uint_gauge_vec!(
986 0 : "pageserver_directory_entries_count",
987 0 : "Sum of the entries in pageserver-stored directory listings",
988 0 : &["tenant_id", "shard_id", "timeline_id"]
989 0 : )
990 0 : .expect("failed to define a metric")
991 0 : });
992 :
993 424 : pub(crate) static TENANT_STATE_METRIC: Lazy<UIntGaugeVec> = Lazy::new(|| {
994 424 : register_uint_gauge_vec!(
995 424 : "pageserver_tenant_states_count",
996 424 : "Count of tenants per state",
997 424 : &["state"]
998 424 : )
999 424 : .expect("Failed to register pageserver_tenant_states_count metric")
1000 424 : });
1001 :
1002 : /// A set of broken tenants.
1003 : ///
1004 : /// These are expected to be so rare that a set is fine. Set as in a new timeseries per each broken
1005 : /// tenant.
1006 20 : pub(crate) static BROKEN_TENANTS_SET: Lazy<UIntGaugeVec> = Lazy::new(|| {
1007 20 : register_uint_gauge_vec!(
1008 20 : "pageserver_broken_tenants_count",
1009 20 : "Set of broken tenants",
1010 20 : &["tenant_id", "shard_id"]
1011 20 : )
1012 20 : .expect("Failed to register pageserver_tenant_states_count metric")
1013 20 : });
1014 :
1015 12 : pub(crate) static TENANT_SYNTHETIC_SIZE_METRIC: Lazy<UIntGaugeVec> = Lazy::new(|| {
1016 12 : register_uint_gauge_vec!(
1017 12 : "pageserver_tenant_synthetic_cached_size_bytes",
1018 12 : "Synthetic size of each tenant in bytes",
1019 12 : &["tenant_id"]
1020 12 : )
1021 12 : .expect("Failed to register pageserver_tenant_synthetic_cached_size_bytes metric")
1022 12 : });
1023 :
1024 0 : pub(crate) static EVICTION_ITERATION_DURATION: Lazy<HistogramVec> = Lazy::new(|| {
1025 0 : register_histogram_vec!(
1026 0 : "pageserver_eviction_iteration_duration_seconds_global",
1027 0 : "Time spent on a single eviction iteration",
1028 0 : &["period_secs", "threshold_secs"],
1029 0 : STORAGE_OP_BUCKETS.into(),
1030 0 : )
1031 0 : .expect("failed to define a metric")
1032 0 : });
1033 :
1034 420 : static EVICTIONS: Lazy<IntCounterVec> = Lazy::new(|| {
1035 420 : register_int_counter_vec!(
1036 420 : "pageserver_evictions",
1037 420 : "Number of layers evicted from the pageserver",
1038 420 : &["tenant_id", "shard_id", "timeline_id"]
1039 420 : )
1040 420 : .expect("failed to define a metric")
1041 420 : });
1042 :
1043 420 : static EVICTIONS_WITH_LOW_RESIDENCE_DURATION: Lazy<IntCounterVec> = Lazy::new(|| {
1044 420 : register_int_counter_vec!(
1045 420 : "pageserver_evictions_with_low_residence_duration",
1046 420 : "If a layer is evicted that was resident for less than `low_threshold`, it is counted to this counter. \
1047 420 : Residence duration is determined using the `residence_duration_data_source`.",
1048 420 : &["tenant_id", "shard_id", "timeline_id", "residence_duration_data_source", "low_threshold_secs"]
1049 420 : )
1050 420 : .expect("failed to define a metric")
1051 420 : });
1052 :
1053 0 : pub(crate) static UNEXPECTED_ONDEMAND_DOWNLOADS: Lazy<IntCounter> = Lazy::new(|| {
1054 0 : register_int_counter!(
1055 0 : "pageserver_unexpected_ondemand_downloads_count",
1056 0 : "Number of unexpected on-demand downloads. \
1057 0 : We log more context for each increment, so, forgo any labels in this metric.",
1058 0 : )
1059 0 : .expect("failed to define a metric")
1060 0 : });
1061 :
1062 : /// How long did we take to start up? Broken down by labels to describe
1063 : /// different phases of startup.
1064 0 : pub static STARTUP_DURATION: Lazy<GaugeVec> = Lazy::new(|| {
1065 0 : register_gauge_vec!(
1066 0 : "pageserver_startup_duration_seconds",
1067 0 : "Time taken by phases of pageserver startup, in seconds",
1068 0 : &["phase"]
1069 0 : )
1070 0 : .expect("Failed to register pageserver_startup_duration_seconds metric")
1071 0 : });
1072 :
1073 0 : pub static STARTUP_IS_LOADING: Lazy<UIntGauge> = Lazy::new(|| {
1074 0 : register_uint_gauge!(
1075 0 : "pageserver_startup_is_loading",
1076 0 : "1 while in initial startup load of tenants, 0 at other times"
1077 0 : )
1078 0 : .expect("Failed to register pageserver_startup_is_loading")
1079 0 : });
1080 :
1081 412 : pub(crate) static TIMELINE_EPHEMERAL_BYTES: Lazy<UIntGauge> = Lazy::new(|| {
1082 412 : register_uint_gauge!(
1083 412 : "pageserver_timeline_ephemeral_bytes",
1084 412 : "Total number of bytes in ephemeral layers, summed for all timelines. Approximate, lazily updated."
1085 412 : )
1086 412 : .expect("Failed to register metric")
1087 412 : });
1088 :
1089 : /// Metrics related to the lifecycle of a [`crate::tenant::Tenant`] object: things
1090 : /// like how long it took to load.
1091 : ///
1092 : /// Note that these are process-global metrics, _not_ per-tenant metrics. Per-tenant
1093 : /// metrics are rather expensive, and usually fine grained stuff makes more sense
1094 : /// at a timeline level than tenant level.
1095 : pub(crate) struct TenantMetrics {
1096 : /// How long did tenants take to go from construction to active state?
1097 : pub(crate) activation: Histogram,
1098 : pub(crate) preload: Histogram,
1099 : pub(crate) attach: Histogram,
1100 :
1101 : /// How many tenants are included in the initial startup of the pagesrever?
1102 : pub(crate) startup_scheduled: IntCounter,
1103 : pub(crate) startup_complete: IntCounter,
1104 : }
1105 :
1106 0 : pub(crate) static TENANT: Lazy<TenantMetrics> = Lazy::new(|| {
1107 0 : TenantMetrics {
1108 0 : activation: register_histogram!(
1109 0 : "pageserver_tenant_activation_seconds",
1110 0 : "Time taken by tenants to activate, in seconds",
1111 0 : CRITICAL_OP_BUCKETS.into()
1112 0 : )
1113 0 : .expect("Failed to register metric"),
1114 0 : preload: register_histogram!(
1115 0 : "pageserver_tenant_preload_seconds",
1116 0 : "Time taken by tenants to load remote metadata on startup/attach, in seconds",
1117 0 : CRITICAL_OP_BUCKETS.into()
1118 0 : )
1119 0 : .expect("Failed to register metric"),
1120 0 : attach: register_histogram!(
1121 0 : "pageserver_tenant_attach_seconds",
1122 0 : "Time taken by tenants to intialize, after remote metadata is already loaded",
1123 0 : CRITICAL_OP_BUCKETS.into()
1124 0 : )
1125 0 : .expect("Failed to register metric"),
1126 0 : startup_scheduled: register_int_counter!(
1127 0 : "pageserver_tenant_startup_scheduled",
1128 0 : "Number of tenants included in pageserver startup (doesn't count tenants attached later)"
1129 0 : ).expect("Failed to register metric"),
1130 0 : startup_complete: register_int_counter!(
1131 0 : "pageserver_tenant_startup_complete",
1132 0 : "Number of tenants that have completed warm-up, or activated on-demand during initial startup: \
1133 0 : should eventually reach `pageserver_tenant_startup_scheduled_total`. Does not include broken \
1134 0 : tenants: such cases will lead to this metric never reaching the scheduled count."
1135 0 : ).expect("Failed to register metric"),
1136 0 : }
1137 0 : });
1138 :
1139 : /// Each `Timeline`'s [`EVICTIONS_WITH_LOW_RESIDENCE_DURATION`] metric.
1140 : #[derive(Debug)]
1141 : pub(crate) struct EvictionsWithLowResidenceDuration {
1142 : data_source: &'static str,
1143 : threshold: Duration,
1144 : counter: Option<IntCounter>,
1145 : }
1146 :
1147 : pub(crate) struct EvictionsWithLowResidenceDurationBuilder {
1148 : data_source: &'static str,
1149 : threshold: Duration,
1150 : }
1151 :
1152 : impl EvictionsWithLowResidenceDurationBuilder {
1153 924 : pub fn new(data_source: &'static str, threshold: Duration) -> Self {
1154 924 : Self {
1155 924 : data_source,
1156 924 : threshold,
1157 924 : }
1158 924 : }
1159 :
1160 924 : fn build(
1161 924 : &self,
1162 924 : tenant_id: &str,
1163 924 : shard_id: &str,
1164 924 : timeline_id: &str,
1165 924 : ) -> EvictionsWithLowResidenceDuration {
1166 924 : let counter = EVICTIONS_WITH_LOW_RESIDENCE_DURATION
1167 924 : .get_metric_with_label_values(&[
1168 924 : tenant_id,
1169 924 : shard_id,
1170 924 : timeline_id,
1171 924 : self.data_source,
1172 924 : &EvictionsWithLowResidenceDuration::threshold_label_value(self.threshold),
1173 924 : ])
1174 924 : .unwrap();
1175 924 : EvictionsWithLowResidenceDuration {
1176 924 : data_source: self.data_source,
1177 924 : threshold: self.threshold,
1178 924 : counter: Some(counter),
1179 924 : }
1180 924 : }
1181 : }
1182 :
1183 : impl EvictionsWithLowResidenceDuration {
1184 944 : fn threshold_label_value(threshold: Duration) -> String {
1185 944 : format!("{}", threshold.as_secs())
1186 944 : }
1187 :
1188 8 : pub fn observe(&self, observed_value: Duration) {
1189 8 : if observed_value < self.threshold {
1190 8 : self.counter
1191 8 : .as_ref()
1192 8 : .expect("nobody calls this function after `remove_from_vec`")
1193 8 : .inc();
1194 8 : }
1195 8 : }
1196 :
1197 0 : pub fn change_threshold(
1198 0 : &mut self,
1199 0 : tenant_id: &str,
1200 0 : shard_id: &str,
1201 0 : timeline_id: &str,
1202 0 : new_threshold: Duration,
1203 0 : ) {
1204 0 : if new_threshold == self.threshold {
1205 0 : return;
1206 0 : }
1207 0 : let mut with_new = EvictionsWithLowResidenceDurationBuilder::new(
1208 0 : self.data_source,
1209 0 : new_threshold,
1210 0 : )
1211 0 : .build(tenant_id, shard_id, timeline_id);
1212 0 : std::mem::swap(self, &mut with_new);
1213 0 : with_new.remove(tenant_id, shard_id, timeline_id);
1214 0 : }
1215 :
1216 : // This could be a `Drop` impl, but, we need the `tenant_id` and `timeline_id`.
1217 20 : fn remove(&mut self, tenant_id: &str, shard_id: &str, timeline_id: &str) {
1218 20 : let Some(_counter) = self.counter.take() else {
1219 0 : return;
1220 : };
1221 :
1222 20 : let threshold = Self::threshold_label_value(self.threshold);
1223 20 :
1224 20 : let removed = EVICTIONS_WITH_LOW_RESIDENCE_DURATION.remove_label_values(&[
1225 20 : tenant_id,
1226 20 : shard_id,
1227 20 : timeline_id,
1228 20 : self.data_source,
1229 20 : &threshold,
1230 20 : ]);
1231 20 :
1232 20 : match removed {
1233 0 : Err(e) => {
1234 0 : // this has been hit in staging as
1235 0 : // <https://neondatabase.sentry.io/issues/4142396994/>, but we don't know how.
1236 0 : // because we can be in the drop path already, don't risk:
1237 0 : // - "double-panic => illegal instruction" or
1238 0 : // - future "drop panick => abort"
1239 0 : //
1240 0 : // so just nag: (the error has the labels)
1241 0 : tracing::warn!(
1242 0 : "failed to remove EvictionsWithLowResidenceDuration, it was already removed? {e:#?}"
1243 : );
1244 : }
1245 : Ok(()) => {
1246 : // to help identify cases where we double-remove the same values, let's log all
1247 : // deletions?
1248 20 : tracing::info!(
1249 0 : "removed EvictionsWithLowResidenceDuration with {tenant_id}, {timeline_id}, {}, {threshold}",
1250 : self.data_source
1251 : );
1252 : }
1253 : }
1254 20 : }
1255 : }
1256 :
1257 : // Metrics collected on disk IO operations
1258 : //
1259 : // Roughly logarithmic scale.
1260 : const STORAGE_IO_TIME_BUCKETS: &[f64] = &[
1261 : 0.000030, // 30 usec
1262 : 0.001000, // 1000 usec
1263 : 0.030, // 30 ms
1264 : 1.000, // 1000 ms
1265 : 30.000, // 30000 ms
1266 : ];
1267 :
1268 : /// VirtualFile fs operation variants.
1269 : ///
1270 : /// Operations:
1271 : /// - open ([`std::fs::OpenOptions::open`])
1272 : /// - close (dropping [`crate::virtual_file::VirtualFile`])
1273 : /// - close-by-replace (close by replacement algorithm)
1274 : /// - read (`read_at`)
1275 : /// - write (`write_at`)
1276 : /// - seek (modify internal position or file length query)
1277 : /// - fsync ([`std::fs::File::sync_all`])
1278 : /// - metadata ([`std::fs::File::metadata`])
1279 : #[derive(
1280 0 : Debug, Clone, Copy, strum_macros::EnumCount, strum_macros::EnumIter, strum_macros::FromRepr,
1281 : )]
1282 : pub(crate) enum StorageIoOperation {
1283 : Open,
1284 : OpenAfterReplace,
1285 : Close,
1286 : CloseByReplace,
1287 : Read,
1288 : Write,
1289 : Seek,
1290 : Fsync,
1291 : Metadata,
1292 : }
1293 :
1294 : impl StorageIoOperation {
1295 4320 : pub fn as_str(&self) -> &'static str {
1296 4320 : match self {
1297 480 : StorageIoOperation::Open => "open",
1298 480 : StorageIoOperation::OpenAfterReplace => "open-after-replace",
1299 480 : StorageIoOperation::Close => "close",
1300 480 : StorageIoOperation::CloseByReplace => "close-by-replace",
1301 480 : StorageIoOperation::Read => "read",
1302 480 : StorageIoOperation::Write => "write",
1303 480 : StorageIoOperation::Seek => "seek",
1304 480 : StorageIoOperation::Fsync => "fsync",
1305 480 : StorageIoOperation::Metadata => "metadata",
1306 : }
1307 4320 : }
1308 : }
1309 :
1310 : /// Tracks time taken by fs operations near VirtualFile.
1311 : #[derive(Debug)]
1312 : pub(crate) struct StorageIoTime {
1313 : metrics: [Histogram; StorageIoOperation::COUNT],
1314 : }
1315 :
1316 : impl StorageIoTime {
1317 480 : fn new() -> Self {
1318 480 : let storage_io_histogram_vec = register_histogram_vec!(
1319 480 : "pageserver_io_operations_seconds",
1320 480 : "Time spent in IO operations",
1321 480 : &["operation"],
1322 480 : STORAGE_IO_TIME_BUCKETS.into()
1323 480 : )
1324 480 : .expect("failed to define a metric");
1325 4320 : let metrics = std::array::from_fn(|i| {
1326 4320 : let op = StorageIoOperation::from_repr(i).unwrap();
1327 4320 : storage_io_histogram_vec
1328 4320 : .get_metric_with_label_values(&[op.as_str()])
1329 4320 : .unwrap()
1330 4320 : });
1331 480 : Self { metrics }
1332 480 : }
1333 :
1334 2055919 : pub(crate) fn get(&self, op: StorageIoOperation) -> &Histogram {
1335 2055919 : &self.metrics[op as usize]
1336 2055919 : }
1337 : }
1338 :
1339 : pub(crate) static STORAGE_IO_TIME_METRIC: Lazy<StorageIoTime> = Lazy::new(StorageIoTime::new);
1340 :
1341 : #[derive(Clone, Copy)]
1342 : #[repr(usize)]
1343 : pub(crate) enum StorageIoSizeOperation {
1344 : Read,
1345 : Write,
1346 : }
1347 :
1348 : impl StorageIoSizeOperation {
1349 : pub(crate) const VARIANTS: &'static [&'static str] = &["read", "write"];
1350 :
1351 2960 : fn as_str(&self) -> &'static str {
1352 2960 : Self::VARIANTS[*self as usize]
1353 2960 : }
1354 : }
1355 :
1356 : // Needed for the https://neonprod.grafana.net/d/5uK9tHL4k/picking-tenant-for-relocation?orgId=1
1357 556 : pub(crate) static STORAGE_IO_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
1358 556 : register_uint_gauge_vec!(
1359 556 : "pageserver_io_operations_bytes_total",
1360 556 : "Total amount of bytes read/written in IO operations",
1361 556 : &["operation", "tenant_id", "shard_id", "timeline_id"]
1362 556 : )
1363 556 : .expect("failed to define a metric")
1364 556 : });
1365 :
1366 : #[derive(Clone, Debug)]
1367 : pub(crate) struct StorageIoSizeMetrics {
1368 : pub read: UIntGauge,
1369 : pub write: UIntGauge,
1370 : }
1371 :
1372 : impl StorageIoSizeMetrics {
1373 1480 : pub(crate) fn new(tenant_id: &str, shard_id: &str, timeline_id: &str) -> Self {
1374 1480 : let read = STORAGE_IO_SIZE
1375 1480 : .get_metric_with_label_values(&[
1376 1480 : StorageIoSizeOperation::Read.as_str(),
1377 1480 : tenant_id,
1378 1480 : shard_id,
1379 1480 : timeline_id,
1380 1480 : ])
1381 1480 : .unwrap();
1382 1480 : let write = STORAGE_IO_SIZE
1383 1480 : .get_metric_with_label_values(&[
1384 1480 : StorageIoSizeOperation::Write.as_str(),
1385 1480 : tenant_id,
1386 1480 : shard_id,
1387 1480 : timeline_id,
1388 1480 : ])
1389 1480 : .unwrap();
1390 1480 : Self { read, write }
1391 1480 : }
1392 : }
1393 :
1394 : #[cfg(not(test))]
1395 : pub(crate) mod virtual_file_descriptor_cache {
1396 : use super::*;
1397 :
1398 0 : pub(crate) static SIZE_MAX: Lazy<UIntGauge> = Lazy::new(|| {
1399 0 : register_uint_gauge!(
1400 0 : "pageserver_virtual_file_descriptor_cache_size_max",
1401 0 : "Maximum number of open file descriptors in the cache."
1402 0 : )
1403 0 : .unwrap()
1404 0 : });
1405 :
1406 : // SIZE_CURRENT: derive it like so:
1407 : // ```
1408 : // sum (pageserver_io_operations_seconds_count{operation=~"^(open|open-after-replace)$")
1409 : // -ignoring(operation)
1410 : // sum(pageserver_io_operations_seconds_count{operation=~"^(close|close-by-replace)$"}
1411 : // ```
1412 : }
1413 :
1414 : #[cfg(not(test))]
1415 : pub(crate) mod virtual_file_io_engine {
1416 : use super::*;
1417 :
1418 0 : pub(crate) static KIND: Lazy<UIntGaugeVec> = Lazy::new(|| {
1419 0 : register_uint_gauge_vec!(
1420 0 : "pageserver_virtual_file_io_engine_kind",
1421 0 : "The configured io engine for VirtualFile",
1422 0 : &["kind"],
1423 0 : )
1424 0 : .unwrap()
1425 0 : });
1426 : }
1427 :
1428 : pub(crate) struct SmgrOpTimer(Option<SmgrOpTimerInner>);
1429 : pub(crate) struct SmgrOpTimerInner {
1430 : global_execution_latency_histo: Histogram,
1431 : per_timeline_execution_latency_histo: Option<Histogram>,
1432 :
1433 : global_batch_wait_time: Histogram,
1434 : per_timeline_batch_wait_time: Histogram,
1435 :
1436 : global_flush_in_progress_micros: IntCounter,
1437 : per_timeline_flush_in_progress_micros: IntCounter,
1438 :
1439 : throttling: Arc<tenant_throttling::Pagestream>,
1440 :
1441 : timings: SmgrOpTimerState,
1442 : }
1443 :
1444 : /// The stages of request processing are represented by the enum variants.
1445 : /// Used as part of [`SmgrOpTimerInner::timings`].
1446 : ///
1447 : /// Request processing calls into the `SmgrOpTimer::observe_*` methods at the
1448 : /// transition points.
1449 : /// These methods bump relevant counters and then update [`SmgrOpTimerInner::timings`]
1450 : /// to the next state.
1451 : ///
1452 : /// Each request goes through every stage, in all configurations.
1453 : ///
1454 : #[derive(Debug)]
1455 : enum SmgrOpTimerState {
1456 : Received {
1457 : // In the future, we may want to track the full time the request spent
1458 : // inside pageserver process (time spent in kernel buffers can't be tracked).
1459 : // `received_at` would be used for that.
1460 : #[allow(dead_code)]
1461 : received_at: Instant,
1462 : },
1463 : Throttling {
1464 : throttle_started_at: Instant,
1465 : },
1466 : Batching {
1467 : throttle_done_at: Instant,
1468 : },
1469 : Executing {
1470 : execution_started_at: Instant,
1471 : },
1472 : Flushing,
1473 : // NB: when adding observation points, remember to update the Drop impl.
1474 : }
1475 :
1476 : // NB: when adding observation points, remember to update the Drop impl.
1477 : impl SmgrOpTimer {
1478 : /// See [`SmgrOpTimerState`] for more context.
1479 0 : pub(crate) fn observe_throttle_start(&mut self, at: Instant) {
1480 0 : let Some(inner) = self.0.as_mut() else {
1481 0 : return;
1482 : };
1483 0 : let SmgrOpTimerState::Received { received_at: _ } = &mut inner.timings else {
1484 0 : return;
1485 : };
1486 0 : inner.throttling.count_accounted_start.inc();
1487 0 : inner.timings = SmgrOpTimerState::Throttling {
1488 0 : throttle_started_at: at,
1489 0 : };
1490 0 : }
1491 :
1492 : /// See [`SmgrOpTimerState`] for more context.
1493 0 : pub(crate) fn observe_throttle_done(&mut self, throttle: ThrottleResult) {
1494 0 : let Some(inner) = self.0.as_mut() else {
1495 0 : return;
1496 : };
1497 : let SmgrOpTimerState::Throttling {
1498 0 : throttle_started_at,
1499 0 : } = &inner.timings
1500 : else {
1501 0 : return;
1502 : };
1503 0 : inner.throttling.count_accounted_finish.inc();
1504 0 : match throttle {
1505 0 : ThrottleResult::NotThrottled { end } => {
1506 0 : inner.timings = SmgrOpTimerState::Batching {
1507 0 : throttle_done_at: end,
1508 0 : };
1509 0 : }
1510 0 : ThrottleResult::Throttled { end } => {
1511 0 : // update metrics
1512 0 : inner.throttling.count_throttled.inc();
1513 0 : inner
1514 0 : .throttling
1515 0 : .wait_time
1516 0 : .inc_by((end - *throttle_started_at).as_micros().try_into().unwrap());
1517 0 : // state transition
1518 0 : inner.timings = SmgrOpTimerState::Batching {
1519 0 : throttle_done_at: end,
1520 0 : };
1521 0 : }
1522 : }
1523 0 : }
1524 :
1525 : /// See [`SmgrOpTimerState`] for more context.
1526 0 : pub(crate) fn observe_execution_start(&mut self, at: Instant) {
1527 0 : let Some(inner) = self.0.as_mut() else {
1528 0 : return;
1529 : };
1530 0 : let SmgrOpTimerState::Batching { throttle_done_at } = &inner.timings else {
1531 0 : return;
1532 : };
1533 : // update metrics
1534 0 : let batch = at - *throttle_done_at;
1535 0 : inner.global_batch_wait_time.observe(batch.as_secs_f64());
1536 0 : inner
1537 0 : .per_timeline_batch_wait_time
1538 0 : .observe(batch.as_secs_f64());
1539 0 : // state transition
1540 0 : inner.timings = SmgrOpTimerState::Executing {
1541 0 : execution_started_at: at,
1542 0 : }
1543 0 : }
1544 :
1545 : /// For all but the first caller, this is a no-op.
1546 : /// The first callers receives Some, subsequent ones None.
1547 : ///
1548 : /// See [`SmgrOpTimerState`] for more context.
1549 0 : pub(crate) fn observe_execution_end(&mut self, at: Instant) -> Option<SmgrOpFlushInProgress> {
1550 : // NB: unlike the other observe_* methods, this one take()s.
1551 : #[allow(clippy::question_mark)] // maintain similar code pattern.
1552 0 : let Some(mut inner) = self.0.take() else {
1553 0 : return None;
1554 : };
1555 : let SmgrOpTimerState::Executing {
1556 0 : execution_started_at,
1557 0 : } = &inner.timings
1558 : else {
1559 0 : return None;
1560 : };
1561 : // update metrics
1562 0 : let execution = at - *execution_started_at;
1563 0 : inner
1564 0 : .global_execution_latency_histo
1565 0 : .observe(execution.as_secs_f64());
1566 0 : if let Some(per_timeline_execution_latency_histo) =
1567 0 : &inner.per_timeline_execution_latency_histo
1568 0 : {
1569 0 : per_timeline_execution_latency_histo.observe(execution.as_secs_f64());
1570 0 : }
1571 :
1572 : // state transition
1573 0 : inner.timings = SmgrOpTimerState::Flushing;
1574 0 :
1575 0 : // return the flush in progress object which
1576 0 : // will do the remaining metrics updates
1577 0 : let SmgrOpTimerInner {
1578 0 : global_flush_in_progress_micros,
1579 0 : per_timeline_flush_in_progress_micros,
1580 0 : ..
1581 0 : } = inner;
1582 0 : Some(SmgrOpFlushInProgress {
1583 0 : global_micros: global_flush_in_progress_micros,
1584 0 : per_timeline_micros: per_timeline_flush_in_progress_micros,
1585 0 : })
1586 0 : }
1587 : }
1588 :
1589 : /// The last stage of request processing is serializing and flushing the request
1590 : /// into the TCP connection. We want to make slow flushes observable
1591 : /// _while they are occuring_, so this struct provides a wrapper method [`Self::measure`]
1592 : /// to periodically bump the metric.
1593 : ///
1594 : /// If in the future we decide that we're not interested in live updates, we can
1595 : /// add another `observe_*` method to [`SmgrOpTimer`], follow the existing pattern there,
1596 : /// and remove this struct from the code base.
1597 : pub(crate) struct SmgrOpFlushInProgress {
1598 : global_micros: IntCounter,
1599 : per_timeline_micros: IntCounter,
1600 : }
1601 :
1602 : impl Drop for SmgrOpTimer {
1603 0 : fn drop(&mut self) {
1604 0 : // In case of early drop, update any of the remaining metrics with
1605 0 : // observations so that (started,finished) counter pairs balance out
1606 0 : // and all counters on the latency path have the the same number of
1607 0 : // observations.
1608 0 : // It's technically lying and it would be better if each metric had
1609 0 : // a separate label or similar for cancelled requests.
1610 0 : // But we don't have that right now and counter pairs balancing
1611 0 : // out is useful when using the metrics in panels and whatnot.
1612 0 : let now = Instant::now();
1613 0 : self.observe_throttle_start(now);
1614 0 : self.observe_throttle_done(ThrottleResult::NotThrottled { end: now });
1615 0 : self.observe_execution_start(now);
1616 0 : let maybe_flush_timer = self.observe_execution_end(now);
1617 0 : drop(maybe_flush_timer);
1618 0 : }
1619 : }
1620 :
1621 : impl SmgrOpFlushInProgress {
1622 : /// The caller must guarantee that `socket_fd`` outlives this function.
1623 0 : pub(crate) async fn measure<Fut, O>(
1624 0 : self,
1625 0 : started_at: Instant,
1626 0 : mut fut: Fut,
1627 0 : socket_fd: RawFd,
1628 0 : ) -> O
1629 0 : where
1630 0 : Fut: std::future::Future<Output = O>,
1631 0 : {
1632 0 : let mut fut = std::pin::pin!(fut);
1633 0 :
1634 0 : let mut logged = false;
1635 0 : let mut last_counter_increment_at = started_at;
1636 0 : let mut observe_guard = scopeguard::guard(
1637 0 : |is_timeout| {
1638 0 : let now = Instant::now();
1639 0 :
1640 0 : // Increment counter
1641 0 : {
1642 0 : let elapsed_since_last_observe = now - last_counter_increment_at;
1643 0 : self.global_micros
1644 0 : .inc_by(u64::try_from(elapsed_since_last_observe.as_micros()).unwrap());
1645 0 : self.per_timeline_micros
1646 0 : .inc_by(u64::try_from(elapsed_since_last_observe.as_micros()).unwrap());
1647 0 : last_counter_increment_at = now;
1648 0 : }
1649 0 :
1650 0 : // Log something on every timeout, and on completion but only if we hit a timeout.
1651 0 : if is_timeout || logged {
1652 0 : logged = true;
1653 0 : let elapsed_total = now - started_at;
1654 0 : let msg = if is_timeout {
1655 0 : "slow flush ongoing"
1656 : } else {
1657 0 : "slow flush completed or cancelled"
1658 : };
1659 :
1660 0 : let (inq, outq) = {
1661 0 : // SAFETY: caller guarantees that `socket_fd` outlives this function.
1662 0 : #[cfg(target_os = "linux")]
1663 0 : unsafe {
1664 0 : (
1665 0 : utils::linux_socket_ioctl::inq(socket_fd).unwrap_or(-2),
1666 0 : utils::linux_socket_ioctl::outq(socket_fd).unwrap_or(-2),
1667 0 : )
1668 0 : }
1669 0 : #[cfg(not(target_os = "linux"))]
1670 0 : {
1671 0 : _ = socket_fd; // appease unused lint on macOS
1672 0 : (-1, -1)
1673 0 : }
1674 0 : };
1675 0 :
1676 0 : let elapsed_total_secs = format!("{:.6}", elapsed_total.as_secs_f64());
1677 0 : tracing::info!(elapsed_total_secs, inq, outq, msg);
1678 0 : }
1679 0 : },
1680 0 : |mut observe| {
1681 0 : observe(false);
1682 0 : },
1683 0 : );
1684 :
1685 : loop {
1686 0 : match tokio::time::timeout(Duration::from_secs(10), &mut fut).await {
1687 0 : Ok(v) => return v,
1688 0 : Err(_timeout) => {
1689 0 : (*observe_guard)(true);
1690 0 : }
1691 : }
1692 : }
1693 0 : }
1694 : }
1695 :
1696 : #[derive(
1697 : Debug,
1698 : Clone,
1699 : Copy,
1700 : IntoStaticStr,
1701 : strum_macros::EnumCount,
1702 0 : strum_macros::EnumIter,
1703 : strum_macros::FromRepr,
1704 : enum_map::Enum,
1705 : )]
1706 : #[strum(serialize_all = "snake_case")]
1707 : pub enum SmgrQueryType {
1708 : GetRelExists,
1709 : GetRelSize,
1710 : GetPageAtLsn,
1711 : GetDbSize,
1712 : GetSlruSegment,
1713 : #[cfg(feature = "testing")]
1714 : Test,
1715 : }
1716 :
1717 : pub(crate) struct SmgrQueryTimePerTimeline {
1718 : global_started: [IntCounter; SmgrQueryType::COUNT],
1719 : global_latency: [Histogram; SmgrQueryType::COUNT],
1720 : per_timeline_getpage_started: IntCounter,
1721 : per_timeline_getpage_latency: Histogram,
1722 : global_batch_size: Histogram,
1723 : per_timeline_batch_size: Histogram,
1724 : global_flush_in_progress_micros: IntCounter,
1725 : per_timeline_flush_in_progress_micros: IntCounter,
1726 : global_batch_wait_time: Histogram,
1727 : per_timeline_batch_wait_time: Histogram,
1728 : throttling: Arc<tenant_throttling::Pagestream>,
1729 : }
1730 :
1731 420 : static SMGR_QUERY_STARTED_GLOBAL: Lazy<IntCounterVec> = Lazy::new(|| {
1732 420 : register_int_counter_vec!(
1733 420 : // it's a counter, but, name is prepared to extend it to a histogram of queue depth
1734 420 : "pageserver_smgr_query_started_global_count",
1735 420 : "Number of smgr queries started, aggregated by query type.",
1736 420 : &["smgr_query_type"],
1737 420 : )
1738 420 : .expect("failed to define a metric")
1739 420 : });
1740 :
1741 420 : static SMGR_QUERY_STARTED_PER_TENANT_TIMELINE: Lazy<IntCounterVec> = Lazy::new(|| {
1742 420 : register_int_counter_vec!(
1743 420 : // it's a counter, but, name is prepared to extend it to a histogram of queue depth
1744 420 : "pageserver_smgr_query_started_count",
1745 420 : "Number of smgr queries started, aggregated by query type and tenant/timeline.",
1746 420 : &["smgr_query_type", "tenant_id", "shard_id", "timeline_id"],
1747 420 : )
1748 420 : .expect("failed to define a metric")
1749 420 : });
1750 :
1751 : // Alias so all histograms recording per-timeline smgr timings use the same buckets.
1752 : static SMGR_QUERY_TIME_PER_TENANT_TIMELINE_BUCKETS: &[f64] = CRITICAL_OP_BUCKETS;
1753 :
1754 420 : static SMGR_QUERY_TIME_PER_TENANT_TIMELINE: Lazy<HistogramVec> = Lazy::new(|| {
1755 420 : register_histogram_vec!(
1756 420 : "pageserver_smgr_query_seconds",
1757 420 : "Time spent _executing_ smgr query handling, excluding batch and throttle delays.",
1758 420 : &["smgr_query_type", "tenant_id", "shard_id", "timeline_id"],
1759 420 : SMGR_QUERY_TIME_PER_TENANT_TIMELINE_BUCKETS.into(),
1760 420 : )
1761 420 : .expect("failed to define a metric")
1762 420 : });
1763 :
1764 420 : static SMGR_QUERY_TIME_GLOBAL_BUCKETS: Lazy<Vec<f64>> = Lazy::new(|| {
1765 420 : [
1766 420 : 1,
1767 420 : 10,
1768 420 : 20,
1769 420 : 40,
1770 420 : 60,
1771 420 : 80,
1772 420 : 100,
1773 420 : 200,
1774 420 : 300,
1775 420 : 400,
1776 420 : 500,
1777 420 : 600,
1778 420 : 700,
1779 420 : 800,
1780 420 : 900,
1781 420 : 1_000, // 1ms
1782 420 : 2_000,
1783 420 : 4_000,
1784 420 : 6_000,
1785 420 : 8_000,
1786 420 : 10_000, // 10ms
1787 420 : 20_000,
1788 420 : 40_000,
1789 420 : 60_000,
1790 420 : 80_000,
1791 420 : 100_000,
1792 420 : 200_000,
1793 420 : 400_000,
1794 420 : 600_000,
1795 420 : 800_000,
1796 420 : 1_000_000, // 1s
1797 420 : 2_000_000,
1798 420 : 4_000_000,
1799 420 : 6_000_000,
1800 420 : 8_000_000,
1801 420 : 10_000_000, // 10s
1802 420 : 20_000_000,
1803 420 : 50_000_000,
1804 420 : 100_000_000,
1805 420 : 200_000_000,
1806 420 : 1_000_000_000, // 1000s
1807 420 : ]
1808 420 : .into_iter()
1809 420 : .map(Duration::from_micros)
1810 17220 : .map(|d| d.as_secs_f64())
1811 420 : .collect()
1812 420 : });
1813 :
1814 420 : static SMGR_QUERY_TIME_GLOBAL: Lazy<HistogramVec> = Lazy::new(|| {
1815 420 : register_histogram_vec!(
1816 420 : "pageserver_smgr_query_seconds_global",
1817 420 : "Like pageserver_smgr_query_seconds, but aggregated to instance level.",
1818 420 : &["smgr_query_type"],
1819 420 : SMGR_QUERY_TIME_GLOBAL_BUCKETS.clone(),
1820 420 : )
1821 420 : .expect("failed to define a metric")
1822 420 : });
1823 :
1824 420 : static PAGE_SERVICE_BATCH_SIZE_BUCKETS_GLOBAL: Lazy<Vec<f64>> = Lazy::new(|| {
1825 420 : (1..=u32::try_from(Timeline::MAX_GET_VECTORED_KEYS).unwrap())
1826 13440 : .map(|v| v.into())
1827 420 : .collect()
1828 420 : });
1829 :
1830 420 : static PAGE_SERVICE_BATCH_SIZE_GLOBAL: Lazy<Histogram> = Lazy::new(|| {
1831 420 : register_histogram!(
1832 420 : "pageserver_page_service_batch_size_global",
1833 420 : "Batch size of pageserver page service requests",
1834 420 : PAGE_SERVICE_BATCH_SIZE_BUCKETS_GLOBAL.clone(),
1835 420 : )
1836 420 : .expect("failed to define a metric")
1837 420 : });
1838 :
1839 420 : static PAGE_SERVICE_BATCH_SIZE_BUCKETS_PER_TIMELINE: Lazy<Vec<f64>> = Lazy::new(|| {
1840 420 : let mut buckets = Vec::new();
1841 2940 : for i in 0.. {
1842 2940 : let bucket = 1 << i;
1843 2940 : if bucket > u32::try_from(Timeline::MAX_GET_VECTORED_KEYS).unwrap() {
1844 420 : break;
1845 2520 : }
1846 2520 : buckets.push(bucket.into());
1847 : }
1848 420 : buckets
1849 420 : });
1850 :
1851 420 : static PAGE_SERVICE_BATCH_SIZE_PER_TENANT_TIMELINE: Lazy<HistogramVec> = Lazy::new(|| {
1852 420 : register_histogram_vec!(
1853 420 : "pageserver_page_service_batch_size",
1854 420 : "Batch size of pageserver page service requests",
1855 420 : &["tenant_id", "shard_id", "timeline_id"],
1856 420 : PAGE_SERVICE_BATCH_SIZE_BUCKETS_PER_TIMELINE.clone()
1857 420 : )
1858 420 : .expect("failed to define a metric")
1859 420 : });
1860 :
1861 0 : pub(crate) static PAGE_SERVICE_CONFIG_MAX_BATCH_SIZE: Lazy<IntGaugeVec> = Lazy::new(|| {
1862 0 : register_int_gauge_vec!(
1863 0 : "pageserver_page_service_config_max_batch_size",
1864 0 : "Configured maximum batch size for the server-side batching functionality of page_service. \
1865 0 : Labels expose more of the configuration parameters.",
1866 0 : &["mode", "execution"]
1867 0 : )
1868 0 : .expect("failed to define a metric")
1869 0 : });
1870 :
1871 0 : fn set_page_service_config_max_batch_size(conf: &PageServicePipeliningConfig) {
1872 0 : PAGE_SERVICE_CONFIG_MAX_BATCH_SIZE.reset();
1873 0 : let (label_values, value) = match conf {
1874 0 : PageServicePipeliningConfig::Serial => (["serial", "-"], 1),
1875 : PageServicePipeliningConfig::Pipelined(PageServicePipeliningConfigPipelined {
1876 0 : max_batch_size,
1877 0 : execution,
1878 0 : }) => {
1879 0 : let mode = "pipelined";
1880 0 : let execution = match execution {
1881 : PageServiceProtocolPipelinedExecutionStrategy::ConcurrentFutures => {
1882 0 : "concurrent-futures"
1883 : }
1884 0 : PageServiceProtocolPipelinedExecutionStrategy::Tasks => "tasks",
1885 : };
1886 0 : ([mode, execution], max_batch_size.get())
1887 : }
1888 : };
1889 0 : PAGE_SERVICE_CONFIG_MAX_BATCH_SIZE
1890 0 : .with_label_values(&label_values)
1891 0 : .set(value.try_into().unwrap());
1892 0 : }
1893 :
1894 420 : static PAGE_SERVICE_SMGR_FLUSH_INPROGRESS_MICROS: Lazy<IntCounterVec> = Lazy::new(|| {
1895 420 : register_int_counter_vec!(
1896 420 : "pageserver_page_service_pagestream_flush_in_progress_micros",
1897 420 : "Counter that sums up the microseconds that a pagestream response was being flushed into the TCP connection. \
1898 420 : If the flush is particularly slow, this counter will be updated periodically to make slow flushes \
1899 420 : easily discoverable in monitoring. \
1900 420 : Hence, this is NOT a completion latency historgram.",
1901 420 : &["tenant_id", "shard_id", "timeline_id"],
1902 420 : )
1903 420 : .expect("failed to define a metric")
1904 420 : });
1905 :
1906 420 : static PAGE_SERVICE_SMGR_FLUSH_INPROGRESS_MICROS_GLOBAL: Lazy<IntCounter> = Lazy::new(|| {
1907 420 : register_int_counter!(
1908 420 : "pageserver_page_service_pagestream_flush_in_progress_micros_global",
1909 420 : "Like pageserver_page_service_pagestream_flush_in_progress_seconds, but instance-wide.",
1910 420 : )
1911 420 : .expect("failed to define a metric")
1912 420 : });
1913 :
1914 420 : static PAGE_SERVICE_SMGR_BATCH_WAIT_TIME: Lazy<HistogramVec> = Lazy::new(|| {
1915 420 : register_histogram_vec!(
1916 420 : "pageserver_page_service_pagestream_batch_wait_time_seconds",
1917 420 : "Time a request spent waiting in its batch until the batch moved to throttle&execution.",
1918 420 : &["tenant_id", "shard_id", "timeline_id"],
1919 420 : SMGR_QUERY_TIME_PER_TENANT_TIMELINE_BUCKETS.into(),
1920 420 : )
1921 420 : .expect("failed to define a metric")
1922 420 : });
1923 :
1924 420 : static PAGE_SERVICE_SMGR_BATCH_WAIT_TIME_GLOBAL: Lazy<Histogram> = Lazy::new(|| {
1925 420 : register_histogram!(
1926 420 : "pageserver_page_service_pagestream_batch_wait_time_seconds_global",
1927 420 : "Like pageserver_page_service_pagestream_batch_wait_time_seconds, but aggregated to instance level.",
1928 420 : SMGR_QUERY_TIME_GLOBAL_BUCKETS.to_vec(),
1929 420 : )
1930 420 : .expect("failed to define a metric")
1931 420 : });
1932 :
1933 : impl SmgrQueryTimePerTimeline {
1934 924 : pub(crate) fn new(
1935 924 : tenant_shard_id: &TenantShardId,
1936 924 : timeline_id: &TimelineId,
1937 924 : pagestream_throttle_metrics: Arc<tenant_throttling::Pagestream>,
1938 924 : ) -> Self {
1939 924 : let tenant_id = tenant_shard_id.tenant_id.to_string();
1940 924 : let shard_slug = format!("{}", tenant_shard_id.shard_slug());
1941 924 : let timeline_id = timeline_id.to_string();
1942 5544 : let global_started = std::array::from_fn(|i| {
1943 5544 : let op = SmgrQueryType::from_repr(i).unwrap();
1944 5544 : SMGR_QUERY_STARTED_GLOBAL
1945 5544 : .get_metric_with_label_values(&[op.into()])
1946 5544 : .unwrap()
1947 5544 : });
1948 5544 : let global_latency = std::array::from_fn(|i| {
1949 5544 : let op = SmgrQueryType::from_repr(i).unwrap();
1950 5544 : SMGR_QUERY_TIME_GLOBAL
1951 5544 : .get_metric_with_label_values(&[op.into()])
1952 5544 : .unwrap()
1953 5544 : });
1954 924 :
1955 924 : let per_timeline_getpage_started = SMGR_QUERY_STARTED_PER_TENANT_TIMELINE
1956 924 : .get_metric_with_label_values(&[
1957 924 : SmgrQueryType::GetPageAtLsn.into(),
1958 924 : &tenant_id,
1959 924 : &shard_slug,
1960 924 : &timeline_id,
1961 924 : ])
1962 924 : .unwrap();
1963 924 : let per_timeline_getpage_latency = SMGR_QUERY_TIME_PER_TENANT_TIMELINE
1964 924 : .get_metric_with_label_values(&[
1965 924 : SmgrQueryType::GetPageAtLsn.into(),
1966 924 : &tenant_id,
1967 924 : &shard_slug,
1968 924 : &timeline_id,
1969 924 : ])
1970 924 : .unwrap();
1971 924 :
1972 924 : let global_batch_size = PAGE_SERVICE_BATCH_SIZE_GLOBAL.clone();
1973 924 : let per_timeline_batch_size = PAGE_SERVICE_BATCH_SIZE_PER_TENANT_TIMELINE
1974 924 : .get_metric_with_label_values(&[&tenant_id, &shard_slug, &timeline_id])
1975 924 : .unwrap();
1976 924 :
1977 924 : let global_batch_wait_time = PAGE_SERVICE_SMGR_BATCH_WAIT_TIME_GLOBAL.clone();
1978 924 : let per_timeline_batch_wait_time = PAGE_SERVICE_SMGR_BATCH_WAIT_TIME
1979 924 : .get_metric_with_label_values(&[&tenant_id, &shard_slug, &timeline_id])
1980 924 : .unwrap();
1981 924 :
1982 924 : let global_flush_in_progress_micros =
1983 924 : PAGE_SERVICE_SMGR_FLUSH_INPROGRESS_MICROS_GLOBAL.clone();
1984 924 : let per_timeline_flush_in_progress_micros = PAGE_SERVICE_SMGR_FLUSH_INPROGRESS_MICROS
1985 924 : .get_metric_with_label_values(&[&tenant_id, &shard_slug, &timeline_id])
1986 924 : .unwrap();
1987 924 :
1988 924 : Self {
1989 924 : global_started,
1990 924 : global_latency,
1991 924 : per_timeline_getpage_latency,
1992 924 : per_timeline_getpage_started,
1993 924 : global_batch_size,
1994 924 : per_timeline_batch_size,
1995 924 : global_flush_in_progress_micros,
1996 924 : per_timeline_flush_in_progress_micros,
1997 924 : global_batch_wait_time,
1998 924 : per_timeline_batch_wait_time,
1999 924 : throttling: pagestream_throttle_metrics,
2000 924 : }
2001 924 : }
2002 0 : pub(crate) fn start_smgr_op(&self, op: SmgrQueryType, received_at: Instant) -> SmgrOpTimer {
2003 0 : self.global_started[op as usize].inc();
2004 :
2005 0 : let per_timeline_latency_histo = if matches!(op, SmgrQueryType::GetPageAtLsn) {
2006 0 : self.per_timeline_getpage_started.inc();
2007 0 : Some(self.per_timeline_getpage_latency.clone())
2008 : } else {
2009 0 : None
2010 : };
2011 :
2012 0 : SmgrOpTimer(Some(SmgrOpTimerInner {
2013 0 : global_execution_latency_histo: self.global_latency[op as usize].clone(),
2014 0 : per_timeline_execution_latency_histo: per_timeline_latency_histo,
2015 0 : global_flush_in_progress_micros: self.global_flush_in_progress_micros.clone(),
2016 0 : per_timeline_flush_in_progress_micros: self
2017 0 : .per_timeline_flush_in_progress_micros
2018 0 : .clone(),
2019 0 : global_batch_wait_time: self.global_batch_wait_time.clone(),
2020 0 : per_timeline_batch_wait_time: self.per_timeline_batch_wait_time.clone(),
2021 0 : throttling: self.throttling.clone(),
2022 0 : timings: SmgrOpTimerState::Received { received_at },
2023 0 : }))
2024 0 : }
2025 :
2026 : /// TODO: do something about this? seems odd, we have a similar call on SmgrOpTimer
2027 0 : pub(crate) fn observe_getpage_batch_start(&self, batch_size: usize) {
2028 0 : self.global_batch_size.observe(batch_size as f64);
2029 0 : self.per_timeline_batch_size.observe(batch_size as f64);
2030 0 : }
2031 : }
2032 :
2033 : // keep in sync with control plane Go code so that we can validate
2034 : // compute's basebackup_ms metric with our perspective in the context of SLI/SLO.
2035 0 : static COMPUTE_STARTUP_BUCKETS: Lazy<[f64; 28]> = Lazy::new(|| {
2036 0 : // Go code uses milliseconds. Variable is called `computeStartupBuckets`
2037 0 : [
2038 0 : 5, 10, 20, 30, 50, 70, 100, 120, 150, 200, 250, 300, 350, 400, 450, 500, 600, 800, 1000,
2039 0 : 1500, 2000, 2500, 3000, 5000, 10000, 20000, 40000, 60000,
2040 0 : ]
2041 0 : .map(|ms| (ms as f64) / 1000.0)
2042 0 : });
2043 :
2044 : pub(crate) struct BasebackupQueryTime {
2045 : ok: Histogram,
2046 : error: Histogram,
2047 : client_error: Histogram,
2048 : }
2049 :
2050 0 : pub(crate) static BASEBACKUP_QUERY_TIME: Lazy<BasebackupQueryTime> = Lazy::new(|| {
2051 0 : let vec = register_histogram_vec!(
2052 0 : "pageserver_basebackup_query_seconds",
2053 0 : "Histogram of basebackup queries durations, by result type",
2054 0 : &["result"],
2055 0 : COMPUTE_STARTUP_BUCKETS.to_vec(),
2056 0 : )
2057 0 : .expect("failed to define a metric");
2058 0 : BasebackupQueryTime {
2059 0 : ok: vec.get_metric_with_label_values(&["ok"]).unwrap(),
2060 0 : error: vec.get_metric_with_label_values(&["error"]).unwrap(),
2061 0 : client_error: vec.get_metric_with_label_values(&["client_error"]).unwrap(),
2062 0 : }
2063 0 : });
2064 :
2065 : pub(crate) struct BasebackupQueryTimeOngoingRecording<'a> {
2066 : parent: &'a BasebackupQueryTime,
2067 : start: std::time::Instant,
2068 : }
2069 :
2070 : impl BasebackupQueryTime {
2071 0 : pub(crate) fn start_recording(&self) -> BasebackupQueryTimeOngoingRecording<'_> {
2072 0 : let start = Instant::now();
2073 0 : BasebackupQueryTimeOngoingRecording {
2074 0 : parent: self,
2075 0 : start,
2076 0 : }
2077 0 : }
2078 : }
2079 :
2080 : impl BasebackupQueryTimeOngoingRecording<'_> {
2081 0 : pub(crate) fn observe<T>(self, res: &Result<T, QueryError>) {
2082 0 : let elapsed = self.start.elapsed().as_secs_f64();
2083 : // If you want to change categorize of a specific error, also change it in `log_query_error`.
2084 0 : let metric = match res {
2085 0 : Ok(_) => &self.parent.ok,
2086 0 : Err(QueryError::Disconnected(ConnectionError::Io(io_error)))
2087 0 : if is_expected_io_error(io_error) =>
2088 0 : {
2089 0 : &self.parent.client_error
2090 : }
2091 0 : Err(_) => &self.parent.error,
2092 : };
2093 0 : metric.observe(elapsed);
2094 0 : }
2095 : }
2096 :
2097 0 : pub(crate) static LIVE_CONNECTIONS: Lazy<IntCounterPairVec> = Lazy::new(|| {
2098 0 : register_int_counter_pair_vec!(
2099 0 : "pageserver_live_connections_started",
2100 0 : "Number of network connections that we started handling",
2101 0 : "pageserver_live_connections_finished",
2102 0 : "Number of network connections that we finished handling",
2103 0 : &["pageserver_connection_kind"]
2104 0 : )
2105 0 : .expect("failed to define a metric")
2106 0 : });
2107 :
2108 : #[derive(Clone, Copy, enum_map::Enum, IntoStaticStr)]
2109 : pub(crate) enum ComputeCommandKind {
2110 : PageStreamV3,
2111 : PageStreamV2,
2112 : Basebackup,
2113 : Fullbackup,
2114 : LeaseLsn,
2115 : }
2116 :
2117 : pub(crate) struct ComputeCommandCounters {
2118 : map: EnumMap<ComputeCommandKind, IntCounter>,
2119 : }
2120 :
2121 0 : pub(crate) static COMPUTE_COMMANDS_COUNTERS: Lazy<ComputeCommandCounters> = Lazy::new(|| {
2122 0 : let inner = register_int_counter_vec!(
2123 0 : "pageserver_compute_commands",
2124 0 : "Number of compute -> pageserver commands processed",
2125 0 : &["command"]
2126 0 : )
2127 0 : .expect("failed to define a metric");
2128 0 :
2129 0 : ComputeCommandCounters {
2130 0 : map: EnumMap::from_array(std::array::from_fn(|i| {
2131 0 : let command = ComputeCommandKind::from_usize(i);
2132 0 : let command_str: &'static str = command.into();
2133 0 : inner.with_label_values(&[command_str])
2134 0 : })),
2135 0 : }
2136 0 : });
2137 :
2138 : impl ComputeCommandCounters {
2139 0 : pub(crate) fn for_command(&self, command: ComputeCommandKind) -> &IntCounter {
2140 0 : &self.map[command]
2141 0 : }
2142 : }
2143 :
2144 : // remote storage metrics
2145 :
2146 412 : static REMOTE_TIMELINE_CLIENT_CALLS: Lazy<IntCounterPairVec> = Lazy::new(|| {
2147 412 : register_int_counter_pair_vec!(
2148 412 : "pageserver_remote_timeline_client_calls_started",
2149 412 : "Number of started calls to remote timeline client.",
2150 412 : "pageserver_remote_timeline_client_calls_finished",
2151 412 : "Number of finshed calls to remote timeline client.",
2152 412 : &[
2153 412 : "tenant_id",
2154 412 : "shard_id",
2155 412 : "timeline_id",
2156 412 : "file_kind",
2157 412 : "op_kind"
2158 412 : ],
2159 412 : )
2160 412 : .unwrap()
2161 412 : });
2162 :
2163 : static REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER: Lazy<IntCounterVec> =
2164 408 : Lazy::new(|| {
2165 408 : register_int_counter_vec!(
2166 408 : "pageserver_remote_timeline_client_bytes_started",
2167 408 : "Incremented by the number of bytes associated with a remote timeline client operation. \
2168 408 : The increment happens when the operation is scheduled.",
2169 408 : &["tenant_id", "shard_id", "timeline_id", "file_kind", "op_kind"],
2170 408 : )
2171 408 : .expect("failed to define a metric")
2172 408 : });
2173 :
2174 408 : static REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER: Lazy<IntCounterVec> = Lazy::new(|| {
2175 408 : register_int_counter_vec!(
2176 408 : "pageserver_remote_timeline_client_bytes_finished",
2177 408 : "Incremented by the number of bytes associated with a remote timeline client operation. \
2178 408 : The increment happens when the operation finishes (regardless of success/failure/shutdown).",
2179 408 : &["tenant_id", "shard_id", "timeline_id", "file_kind", "op_kind"],
2180 408 : )
2181 408 : .expect("failed to define a metric")
2182 408 : });
2183 :
2184 : pub(crate) struct TenantManagerMetrics {
2185 : tenant_slots_attached: UIntGauge,
2186 : tenant_slots_secondary: UIntGauge,
2187 : tenant_slots_inprogress: UIntGauge,
2188 : pub(crate) tenant_slot_writes: IntCounter,
2189 : pub(crate) unexpected_errors: IntCounter,
2190 : }
2191 :
2192 : impl TenantManagerMetrics {
2193 : /// Helpers for tracking slots. Note that these do not track the lifetime of TenantSlot objects
2194 : /// exactly: they track the lifetime of the slots _in the tenant map_.
2195 4 : pub(crate) fn slot_inserted(&self, slot: &TenantSlot) {
2196 4 : match slot {
2197 0 : TenantSlot::Attached(_) => {
2198 0 : self.tenant_slots_attached.inc();
2199 0 : }
2200 0 : TenantSlot::Secondary(_) => {
2201 0 : self.tenant_slots_secondary.inc();
2202 0 : }
2203 4 : TenantSlot::InProgress(_) => {
2204 4 : self.tenant_slots_inprogress.inc();
2205 4 : }
2206 : }
2207 4 : }
2208 :
2209 4 : pub(crate) fn slot_removed(&self, slot: &TenantSlot) {
2210 4 : match slot {
2211 4 : TenantSlot::Attached(_) => {
2212 4 : self.tenant_slots_attached.dec();
2213 4 : }
2214 0 : TenantSlot::Secondary(_) => {
2215 0 : self.tenant_slots_secondary.dec();
2216 0 : }
2217 0 : TenantSlot::InProgress(_) => {
2218 0 : self.tenant_slots_inprogress.dec();
2219 0 : }
2220 : }
2221 4 : }
2222 :
2223 : #[cfg(all(debug_assertions, not(test)))]
2224 0 : pub(crate) fn slots_total(&self) -> u64 {
2225 0 : self.tenant_slots_attached.get()
2226 0 : + self.tenant_slots_secondary.get()
2227 0 : + self.tenant_slots_inprogress.get()
2228 0 : }
2229 : }
2230 :
2231 4 : pub(crate) static TENANT_MANAGER: Lazy<TenantManagerMetrics> = Lazy::new(|| {
2232 4 : let tenant_slots = register_uint_gauge_vec!(
2233 4 : "pageserver_tenant_manager_slots",
2234 4 : "How many slots currently exist, including all attached, secondary and in-progress operations",
2235 4 : &["mode"]
2236 4 : )
2237 4 : .expect("failed to define a metric");
2238 4 : TenantManagerMetrics {
2239 4 : tenant_slots_attached: tenant_slots
2240 4 : .get_metric_with_label_values(&["attached"])
2241 4 : .unwrap(),
2242 4 : tenant_slots_secondary: tenant_slots
2243 4 : .get_metric_with_label_values(&["secondary"])
2244 4 : .unwrap(),
2245 4 : tenant_slots_inprogress: tenant_slots
2246 4 : .get_metric_with_label_values(&["inprogress"])
2247 4 : .unwrap(),
2248 4 : tenant_slot_writes: register_int_counter!(
2249 4 : "pageserver_tenant_manager_slot_writes",
2250 4 : "Writes to a tenant slot, including all of create/attach/detach/delete"
2251 4 : )
2252 4 : .expect("failed to define a metric"),
2253 4 : unexpected_errors: register_int_counter!(
2254 4 : "pageserver_tenant_manager_unexpected_errors_total",
2255 4 : "Number of unexpected conditions encountered: nonzero value indicates a non-fatal bug."
2256 4 : )
2257 4 : .expect("failed to define a metric"),
2258 4 : }
2259 4 : });
2260 :
2261 : pub(crate) struct DeletionQueueMetrics {
2262 : pub(crate) keys_submitted: IntCounter,
2263 : pub(crate) keys_dropped: IntCounter,
2264 : pub(crate) keys_executed: IntCounter,
2265 : pub(crate) keys_validated: IntCounter,
2266 : pub(crate) dropped_lsn_updates: IntCounter,
2267 : pub(crate) unexpected_errors: IntCounter,
2268 : pub(crate) remote_errors: IntCounterVec,
2269 : }
2270 65 : pub(crate) static DELETION_QUEUE: Lazy<DeletionQueueMetrics> = Lazy::new(|| {
2271 65 : DeletionQueueMetrics{
2272 65 :
2273 65 : keys_submitted: register_int_counter!(
2274 65 : "pageserver_deletion_queue_submitted_total",
2275 65 : "Number of objects submitted for deletion"
2276 65 : )
2277 65 : .expect("failed to define a metric"),
2278 65 :
2279 65 : keys_dropped: register_int_counter!(
2280 65 : "pageserver_deletion_queue_dropped_total",
2281 65 : "Number of object deletions dropped due to stale generation."
2282 65 : )
2283 65 : .expect("failed to define a metric"),
2284 65 :
2285 65 : keys_executed: register_int_counter!(
2286 65 : "pageserver_deletion_queue_executed_total",
2287 65 : "Number of objects deleted. Only includes objects that we actually deleted, sum with pageserver_deletion_queue_dropped_total for the total number of keys processed to completion"
2288 65 : )
2289 65 : .expect("failed to define a metric"),
2290 65 :
2291 65 : keys_validated: register_int_counter!(
2292 65 : "pageserver_deletion_queue_validated_total",
2293 65 : "Number of keys validated for deletion. Sum with pageserver_deletion_queue_dropped_total for the total number of keys that have passed through the validation stage."
2294 65 : )
2295 65 : .expect("failed to define a metric"),
2296 65 :
2297 65 : dropped_lsn_updates: register_int_counter!(
2298 65 : "pageserver_deletion_queue_dropped_lsn_updates_total",
2299 65 : "Updates to remote_consistent_lsn dropped due to stale generation number."
2300 65 : )
2301 65 : .expect("failed to define a metric"),
2302 65 : unexpected_errors: register_int_counter!(
2303 65 : "pageserver_deletion_queue_unexpected_errors_total",
2304 65 : "Number of unexpected condiions that may stall the queue: any value above zero is unexpected."
2305 65 : )
2306 65 : .expect("failed to define a metric"),
2307 65 : remote_errors: register_int_counter_vec!(
2308 65 : "pageserver_deletion_queue_remote_errors_total",
2309 65 : "Retryable remote I/O errors while executing deletions, for example 503 responses to DeleteObjects",
2310 65 : &["op_kind"],
2311 65 : )
2312 65 : .expect("failed to define a metric")
2313 65 : }
2314 65 : });
2315 :
2316 : pub(crate) struct SecondaryModeMetrics {
2317 : pub(crate) upload_heatmap: IntCounter,
2318 : pub(crate) upload_heatmap_errors: IntCounter,
2319 : pub(crate) upload_heatmap_duration: Histogram,
2320 : pub(crate) download_heatmap: IntCounter,
2321 : pub(crate) download_layer: IntCounter,
2322 : }
2323 0 : pub(crate) static SECONDARY_MODE: Lazy<SecondaryModeMetrics> = Lazy::new(|| {
2324 0 : SecondaryModeMetrics {
2325 0 : upload_heatmap: register_int_counter!(
2326 0 : "pageserver_secondary_upload_heatmap",
2327 0 : "Number of heatmaps written to remote storage by attached tenants"
2328 0 : )
2329 0 : .expect("failed to define a metric"),
2330 0 : upload_heatmap_errors: register_int_counter!(
2331 0 : "pageserver_secondary_upload_heatmap_errors",
2332 0 : "Failures writing heatmap to remote storage"
2333 0 : )
2334 0 : .expect("failed to define a metric"),
2335 0 : upload_heatmap_duration: register_histogram!(
2336 0 : "pageserver_secondary_upload_heatmap_duration",
2337 0 : "Time to build and upload a heatmap, including any waiting inside the remote storage client"
2338 0 : )
2339 0 : .expect("failed to define a metric"),
2340 0 : download_heatmap: register_int_counter!(
2341 0 : "pageserver_secondary_download_heatmap",
2342 0 : "Number of downloads of heatmaps by secondary mode locations, including when it hasn't changed"
2343 0 : )
2344 0 : .expect("failed to define a metric"),
2345 0 : download_layer: register_int_counter!(
2346 0 : "pageserver_secondary_download_layer",
2347 0 : "Number of downloads of layers by secondary mode locations"
2348 0 : )
2349 0 : .expect("failed to define a metric"),
2350 0 : }
2351 0 : });
2352 :
2353 0 : pub(crate) static SECONDARY_RESIDENT_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
2354 0 : register_uint_gauge_vec!(
2355 0 : "pageserver_secondary_resident_physical_size",
2356 0 : "The size of the layer files present in the pageserver's filesystem, for secondary locations.",
2357 0 : &["tenant_id", "shard_id"]
2358 0 : )
2359 0 : .expect("failed to define a metric")
2360 0 : });
2361 :
2362 0 : pub(crate) static NODE_UTILIZATION_SCORE: Lazy<UIntGauge> = Lazy::new(|| {
2363 0 : register_uint_gauge!(
2364 0 : "pageserver_utilization_score",
2365 0 : "The utilization score we report to the storage controller for scheduling, where 0 is empty, 1000000 is full, and anything above is considered overloaded",
2366 0 : )
2367 0 : .expect("failed to define a metric")
2368 0 : });
2369 :
2370 0 : pub(crate) static SECONDARY_HEATMAP_TOTAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
2371 0 : register_uint_gauge_vec!(
2372 0 : "pageserver_secondary_heatmap_total_size",
2373 0 : "The total size in bytes of all layers in the most recently downloaded heatmap.",
2374 0 : &["tenant_id", "shard_id"]
2375 0 : )
2376 0 : .expect("failed to define a metric")
2377 0 : });
2378 :
2379 : #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
2380 : pub enum RemoteOpKind {
2381 : Upload,
2382 : Download,
2383 : Delete,
2384 : }
2385 : impl RemoteOpKind {
2386 30571 : pub fn as_str(&self) -> &'static str {
2387 30571 : match self {
2388 28788 : Self::Upload => "upload",
2389 136 : Self::Download => "download",
2390 1647 : Self::Delete => "delete",
2391 : }
2392 30571 : }
2393 : }
2394 :
2395 : #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
2396 : pub enum RemoteOpFileKind {
2397 : Layer,
2398 : Index,
2399 : }
2400 : impl RemoteOpFileKind {
2401 30571 : pub fn as_str(&self) -> &'static str {
2402 30571 : match self {
2403 21464 : Self::Layer => "layer",
2404 9107 : Self::Index => "index",
2405 : }
2406 30571 : }
2407 : }
2408 :
2409 405 : pub(crate) static REMOTE_TIMELINE_CLIENT_COMPLETION_LATENCY: Lazy<HistogramVec> = Lazy::new(|| {
2410 405 : register_histogram_vec!(
2411 405 : "pageserver_remote_timeline_client_seconds_global",
2412 405 : "Time spent on remote timeline client operations. \
2413 405 : Grouped by task_kind, file_kind, operation_kind and status. \
2414 405 : The task_kind is \
2415 405 : - for layer downloads, populated from RequestContext (primary objective of having the label) \
2416 405 : - for index downloads, set to 'unknown' \
2417 405 : - for any upload operation, set to 'RemoteUploadTask' \
2418 405 : This keeps dimensionality at bay. \
2419 405 : Does not account for time spent waiting in remote timeline client's queues.",
2420 405 : &["task_kind", "file_kind", "op_kind", "status"]
2421 405 : )
2422 405 : .expect("failed to define a metric")
2423 405 : });
2424 :
2425 0 : pub(crate) static TENANT_TASK_EVENTS: Lazy<IntCounterVec> = Lazy::new(|| {
2426 0 : register_int_counter_vec!(
2427 0 : "pageserver_tenant_task_events",
2428 0 : "Number of task start/stop/fail events.",
2429 0 : &["event"],
2430 0 : )
2431 0 : .expect("Failed to register tenant_task_events metric")
2432 0 : });
2433 :
2434 : pub struct BackgroundLoopSemaphoreMetrics {
2435 : counters: EnumMap<BackgroundLoopKind, IntCounterPair>,
2436 : durations: EnumMap<BackgroundLoopKind, Histogram>,
2437 : waiting_tasks: EnumMap<BackgroundLoopKind, IntGauge>,
2438 : running_tasks: EnumMap<BackgroundLoopKind, IntGauge>,
2439 : }
2440 :
2441 : pub(crate) static BACKGROUND_LOOP_SEMAPHORE: Lazy<BackgroundLoopSemaphoreMetrics> =
2442 40 : Lazy::new(|| {
2443 40 : let counters = register_int_counter_pair_vec!(
2444 40 : "pageserver_background_loop_semaphore_wait_start_count",
2445 40 : "Counter for background loop concurrency-limiting semaphore acquire calls started",
2446 40 : "pageserver_background_loop_semaphore_wait_finish_count",
2447 40 : "Counter for background loop concurrency-limiting semaphore acquire calls finished",
2448 40 : &["task"],
2449 40 : )
2450 40 : .unwrap();
2451 40 :
2452 40 : let durations = register_histogram_vec!(
2453 40 : "pageserver_background_loop_semaphore_wait_seconds",
2454 40 : "Seconds spent waiting on background loop semaphore acquisition",
2455 40 : &["task"],
2456 40 : vec![0.01, 1.0, 5.0, 10.0, 30.0, 60.0, 180.0, 300.0, 600.0],
2457 40 : )
2458 40 : .unwrap();
2459 40 :
2460 40 : let waiting_tasks = register_int_gauge_vec!(
2461 40 : "pageserver_background_loop_semaphore_waiting_tasks",
2462 40 : "Number of background loop tasks waiting for semaphore",
2463 40 : &["task"],
2464 40 : )
2465 40 : .unwrap();
2466 40 :
2467 40 : let running_tasks = register_int_gauge_vec!(
2468 40 : "pageserver_background_loop_semaphore_running_tasks",
2469 40 : "Number of background loop tasks running concurrently",
2470 40 : &["task"],
2471 40 : )
2472 40 : .unwrap();
2473 40 :
2474 40 : BackgroundLoopSemaphoreMetrics {
2475 400 : counters: EnumMap::from_array(std::array::from_fn(|i| {
2476 400 : let kind = BackgroundLoopKind::from_usize(i);
2477 400 : counters.with_label_values(&[kind.into()])
2478 400 : })),
2479 400 : durations: EnumMap::from_array(std::array::from_fn(|i| {
2480 400 : let kind = BackgroundLoopKind::from_usize(i);
2481 400 : durations.with_label_values(&[kind.into()])
2482 400 : })),
2483 400 : waiting_tasks: EnumMap::from_array(std::array::from_fn(|i| {
2484 400 : let kind = BackgroundLoopKind::from_usize(i);
2485 400 : waiting_tasks.with_label_values(&[kind.into()])
2486 400 : })),
2487 400 : running_tasks: EnumMap::from_array(std::array::from_fn(|i| {
2488 400 : let kind = BackgroundLoopKind::from_usize(i);
2489 400 : running_tasks.with_label_values(&[kind.into()])
2490 400 : })),
2491 40 : }
2492 40 : });
2493 :
2494 : impl BackgroundLoopSemaphoreMetrics {
2495 : /// Starts recording semaphore metrics. Call `acquired()` on the returned recorder when the
2496 : /// semaphore is acquired, and drop it when the task completes or is cancelled.
2497 728 : pub(crate) fn record(
2498 728 : &self,
2499 728 : task: BackgroundLoopKind,
2500 728 : ) -> BackgroundLoopSemaphoreMetricsRecorder {
2501 728 : BackgroundLoopSemaphoreMetricsRecorder::start(self, task)
2502 728 : }
2503 : }
2504 :
2505 : /// Records metrics for a background task.
2506 : pub struct BackgroundLoopSemaphoreMetricsRecorder<'a> {
2507 : metrics: &'a BackgroundLoopSemaphoreMetrics,
2508 : task: BackgroundLoopKind,
2509 : start: Instant,
2510 : wait_counter_guard: Option<metrics::IntCounterPairGuard>,
2511 : }
2512 :
2513 : impl<'a> BackgroundLoopSemaphoreMetricsRecorder<'a> {
2514 : /// Starts recording semaphore metrics, by recording wait time and incrementing
2515 : /// `wait_start_count` and `waiting_tasks`.
2516 728 : fn start(metrics: &'a BackgroundLoopSemaphoreMetrics, task: BackgroundLoopKind) -> Self {
2517 728 : metrics.waiting_tasks[task].inc();
2518 728 : Self {
2519 728 : metrics,
2520 728 : task,
2521 728 : start: Instant::now(),
2522 728 : wait_counter_guard: Some(metrics.counters[task].guard()),
2523 728 : }
2524 728 : }
2525 :
2526 : /// Signals that the semaphore has been acquired, and updates relevant metrics.
2527 728 : pub fn acquired(&mut self) -> Duration {
2528 728 : let waited = self.start.elapsed();
2529 728 : self.wait_counter_guard.take().expect("already acquired");
2530 728 : self.metrics.durations[self.task].observe(waited.as_secs_f64());
2531 728 : self.metrics.waiting_tasks[self.task].dec();
2532 728 : self.metrics.running_tasks[self.task].inc();
2533 728 : waited
2534 728 : }
2535 : }
2536 :
2537 : impl Drop for BackgroundLoopSemaphoreMetricsRecorder<'_> {
2538 : /// The task either completed or was cancelled.
2539 728 : fn drop(&mut self) {
2540 728 : if self.wait_counter_guard.take().is_some() {
2541 0 : // Waiting.
2542 0 : self.metrics.durations[self.task].observe(self.start.elapsed().as_secs_f64());
2543 0 : self.metrics.waiting_tasks[self.task].dec();
2544 728 : } else {
2545 728 : // Running.
2546 728 : self.metrics.running_tasks[self.task].dec();
2547 728 : }
2548 728 : }
2549 : }
2550 :
2551 0 : pub(crate) static BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT: Lazy<IntCounterVec> = Lazy::new(|| {
2552 0 : register_int_counter_vec!(
2553 0 : "pageserver_background_loop_period_overrun_count",
2554 0 : "Incremented whenever warn_when_period_overrun() logs a warning.",
2555 0 : &["task", "period"],
2556 0 : )
2557 0 : .expect("failed to define a metric")
2558 0 : });
2559 :
2560 : // walreceiver metrics
2561 :
2562 0 : pub(crate) static WALRECEIVER_STARTED_CONNECTIONS: Lazy<IntCounter> = Lazy::new(|| {
2563 0 : register_int_counter!(
2564 0 : "pageserver_walreceiver_started_connections_total",
2565 0 : "Number of started walreceiver connections"
2566 0 : )
2567 0 : .expect("failed to define a metric")
2568 0 : });
2569 :
2570 0 : pub(crate) static WALRECEIVER_ACTIVE_MANAGERS: Lazy<IntGauge> = Lazy::new(|| {
2571 0 : register_int_gauge!(
2572 0 : "pageserver_walreceiver_active_managers",
2573 0 : "Number of active walreceiver managers"
2574 0 : )
2575 0 : .expect("failed to define a metric")
2576 0 : });
2577 :
2578 0 : pub(crate) static WALRECEIVER_SWITCHES: Lazy<IntCounterVec> = Lazy::new(|| {
2579 0 : register_int_counter_vec!(
2580 0 : "pageserver_walreceiver_switches_total",
2581 0 : "Number of walreceiver manager change_connection calls",
2582 0 : &["reason"]
2583 0 : )
2584 0 : .expect("failed to define a metric")
2585 0 : });
2586 :
2587 0 : pub(crate) static WALRECEIVER_BROKER_UPDATES: Lazy<IntCounter> = Lazy::new(|| {
2588 0 : register_int_counter!(
2589 0 : "pageserver_walreceiver_broker_updates_total",
2590 0 : "Number of received broker updates in walreceiver"
2591 0 : )
2592 0 : .expect("failed to define a metric")
2593 0 : });
2594 :
2595 4 : pub(crate) static WALRECEIVER_CANDIDATES_EVENTS: Lazy<IntCounterVec> = Lazy::new(|| {
2596 4 : register_int_counter_vec!(
2597 4 : "pageserver_walreceiver_candidates_events_total",
2598 4 : "Number of walreceiver candidate events",
2599 4 : &["event"]
2600 4 : )
2601 4 : .expect("failed to define a metric")
2602 4 : });
2603 :
2604 : pub(crate) static WALRECEIVER_CANDIDATES_ADDED: Lazy<IntCounter> =
2605 0 : Lazy::new(|| WALRECEIVER_CANDIDATES_EVENTS.with_label_values(&["add"]));
2606 :
2607 : pub(crate) static WALRECEIVER_CANDIDATES_REMOVED: Lazy<IntCounter> =
2608 4 : Lazy::new(|| WALRECEIVER_CANDIDATES_EVENTS.with_label_values(&["remove"]));
2609 :
2610 : // Metrics collected on WAL redo operations
2611 : //
2612 : // We collect the time spent in actual WAL redo ('redo'), and time waiting
2613 : // for access to the postgres process ('wait') since there is only one for
2614 : // each tenant.
2615 :
2616 : /// Time buckets are small because we want to be able to measure the
2617 : /// smallest redo processing times. These buckets allow us to measure down
2618 : /// to 5us, which equates to 200'000 pages/sec, which equates to 1.6GB/sec.
2619 : /// This is much better than the previous 5ms aka 200 pages/sec aka 1.6MB/sec.
2620 : ///
2621 : /// Values up to 1s are recorded because metrics show that we have redo
2622 : /// durations and lock times larger than 0.250s.
2623 : macro_rules! redo_histogram_time_buckets {
2624 : () => {
2625 : vec![
2626 : 0.000_005, 0.000_010, 0.000_025, 0.000_050, 0.000_100, 0.000_250, 0.000_500, 0.001_000,
2627 : 0.002_500, 0.005_000, 0.010_000, 0.025_000, 0.050_000, 0.100_000, 0.250_000, 0.500_000,
2628 : 1.000_000,
2629 : ]
2630 : };
2631 : }
2632 :
2633 : /// While we're at it, also measure the amount of records replayed in each
2634 : /// operation. We have a global 'total replayed' counter, but that's not
2635 : /// as useful as 'what is the skew for how many records we replay in one
2636 : /// operation'.
2637 : macro_rules! redo_histogram_count_buckets {
2638 : () => {
2639 : vec![0.0, 1.0, 2.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0]
2640 : };
2641 : }
2642 :
2643 : macro_rules! redo_bytes_histogram_count_buckets {
2644 : () => {
2645 : // powers of (2^.5), from 2^4.5 to 2^15 (22 buckets)
2646 : // rounded up to the next multiple of 8 to capture any MAXALIGNed record of that size, too.
2647 : vec![
2648 : 24.0, 32.0, 48.0, 64.0, 96.0, 128.0, 184.0, 256.0, 368.0, 512.0, 728.0, 1024.0, 1456.0,
2649 : 2048.0, 2904.0, 4096.0, 5800.0, 8192.0, 11592.0, 16384.0, 23176.0, 32768.0,
2650 : ]
2651 : };
2652 : }
2653 :
2654 : pub(crate) struct WalIngestMetrics {
2655 : pub(crate) bytes_received: IntCounter,
2656 : pub(crate) records_received: IntCounter,
2657 : pub(crate) records_observed: IntCounter,
2658 : pub(crate) records_committed: IntCounter,
2659 : pub(crate) records_filtered: IntCounter,
2660 : pub(crate) values_committed_metadata_images: IntCounter,
2661 : pub(crate) values_committed_metadata_deltas: IntCounter,
2662 : pub(crate) values_committed_data_images: IntCounter,
2663 : pub(crate) values_committed_data_deltas: IntCounter,
2664 : pub(crate) gap_blocks_zeroed_on_rel_extend: IntCounter,
2665 : }
2666 :
2667 : impl WalIngestMetrics {
2668 0 : pub(crate) fn inc_values_committed(&self, stats: &DatadirModificationStats) {
2669 0 : if stats.metadata_images > 0 {
2670 0 : self.values_committed_metadata_images
2671 0 : .inc_by(stats.metadata_images);
2672 0 : }
2673 0 : if stats.metadata_deltas > 0 {
2674 0 : self.values_committed_metadata_deltas
2675 0 : .inc_by(stats.metadata_deltas);
2676 0 : }
2677 0 : if stats.data_images > 0 {
2678 0 : self.values_committed_data_images.inc_by(stats.data_images);
2679 0 : }
2680 0 : if stats.data_deltas > 0 {
2681 0 : self.values_committed_data_deltas.inc_by(stats.data_deltas);
2682 0 : }
2683 0 : }
2684 : }
2685 :
2686 20 : pub(crate) static WAL_INGEST: Lazy<WalIngestMetrics> = Lazy::new(|| {
2687 20 : let values_committed = register_int_counter_vec!(
2688 20 : "pageserver_wal_ingest_values_committed",
2689 20 : "Number of values committed to pageserver storage from WAL records",
2690 20 : &["class", "kind"],
2691 20 : )
2692 20 : .expect("failed to define a metric");
2693 20 :
2694 20 : WalIngestMetrics {
2695 20 : bytes_received: register_int_counter!(
2696 20 : "pageserver_wal_ingest_bytes_received",
2697 20 : "Bytes of WAL ingested from safekeepers",
2698 20 : )
2699 20 : .unwrap(),
2700 20 : records_received: register_int_counter!(
2701 20 : "pageserver_wal_ingest_records_received",
2702 20 : "Number of WAL records received from safekeepers"
2703 20 : )
2704 20 : .expect("failed to define a metric"),
2705 20 : records_observed: register_int_counter!(
2706 20 : "pageserver_wal_ingest_records_observed",
2707 20 : "Number of WAL records observed from safekeepers. These are metadata only records for shard 0."
2708 20 : )
2709 20 : .expect("failed to define a metric"),
2710 20 : records_committed: register_int_counter!(
2711 20 : "pageserver_wal_ingest_records_committed",
2712 20 : "Number of WAL records which resulted in writes to pageserver storage"
2713 20 : )
2714 20 : .expect("failed to define a metric"),
2715 20 : records_filtered: register_int_counter!(
2716 20 : "pageserver_wal_ingest_records_filtered",
2717 20 : "Number of WAL records filtered out due to sharding"
2718 20 : )
2719 20 : .expect("failed to define a metric"),
2720 20 : values_committed_metadata_images: values_committed.with_label_values(&["metadata", "image"]),
2721 20 : values_committed_metadata_deltas: values_committed.with_label_values(&["metadata", "delta"]),
2722 20 : values_committed_data_images: values_committed.with_label_values(&["data", "image"]),
2723 20 : values_committed_data_deltas: values_committed.with_label_values(&["data", "delta"]),
2724 20 : gap_blocks_zeroed_on_rel_extend: register_int_counter!(
2725 20 : "pageserver_gap_blocks_zeroed_on_rel_extend",
2726 20 : "Total number of zero gap blocks written on relation extends"
2727 20 : )
2728 20 : .expect("failed to define a metric"),
2729 20 : }
2730 20 : });
2731 :
2732 420 : pub(crate) static PAGESERVER_TIMELINE_WAL_RECORDS_RECEIVED: Lazy<IntCounterVec> = Lazy::new(|| {
2733 420 : register_int_counter_vec!(
2734 420 : "pageserver_timeline_wal_records_received",
2735 420 : "Number of WAL records received per shard",
2736 420 : &["tenant_id", "shard_id", "timeline_id"]
2737 420 : )
2738 420 : .expect("failed to define a metric")
2739 420 : });
2740 :
2741 12 : pub(crate) static WAL_REDO_TIME: Lazy<Histogram> = Lazy::new(|| {
2742 12 : register_histogram!(
2743 12 : "pageserver_wal_redo_seconds",
2744 12 : "Time spent on WAL redo",
2745 12 : redo_histogram_time_buckets!()
2746 12 : )
2747 12 : .expect("failed to define a metric")
2748 12 : });
2749 :
2750 12 : pub(crate) static WAL_REDO_RECORDS_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
2751 12 : register_histogram!(
2752 12 : "pageserver_wal_redo_records_histogram",
2753 12 : "Histogram of number of records replayed per redo in the Postgres WAL redo process",
2754 12 : redo_histogram_count_buckets!(),
2755 12 : )
2756 12 : .expect("failed to define a metric")
2757 12 : });
2758 :
2759 12 : pub(crate) static WAL_REDO_BYTES_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
2760 12 : register_histogram!(
2761 12 : "pageserver_wal_redo_bytes_histogram",
2762 12 : "Histogram of number of records replayed per redo sent to Postgres",
2763 12 : redo_bytes_histogram_count_buckets!(),
2764 12 : )
2765 12 : .expect("failed to define a metric")
2766 12 : });
2767 :
2768 : // FIXME: isn't this already included by WAL_REDO_RECORDS_HISTOGRAM which has _count?
2769 12 : pub(crate) static WAL_REDO_RECORD_COUNTER: Lazy<IntCounter> = Lazy::new(|| {
2770 12 : register_int_counter!(
2771 12 : "pageserver_replayed_wal_records_total",
2772 12 : "Number of WAL records replayed in WAL redo process"
2773 12 : )
2774 12 : .unwrap()
2775 12 : });
2776 :
2777 : #[rustfmt::skip]
2778 16 : pub(crate) static WAL_REDO_PROCESS_LAUNCH_DURATION_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
2779 16 : register_histogram!(
2780 16 : "pageserver_wal_redo_process_launch_duration",
2781 16 : "Histogram of the duration of successful WalRedoProcess::launch calls",
2782 16 : vec![
2783 16 : 0.0002, 0.0004, 0.0006, 0.0008, 0.0010,
2784 16 : 0.0020, 0.0040, 0.0060, 0.0080, 0.0100,
2785 16 : 0.0200, 0.0400, 0.0600, 0.0800, 0.1000,
2786 16 : 0.2000, 0.4000, 0.6000, 0.8000, 1.0000,
2787 16 : 1.5000, 2.0000, 2.5000, 3.0000, 4.0000, 10.0000
2788 16 : ],
2789 16 : )
2790 16 : .expect("failed to define a metric")
2791 16 : });
2792 :
2793 : pub(crate) struct WalRedoProcessCounters {
2794 : pub(crate) started: IntCounter,
2795 : pub(crate) killed_by_cause: EnumMap<WalRedoKillCause, IntCounter>,
2796 : pub(crate) active_stderr_logger_tasks_started: IntCounter,
2797 : pub(crate) active_stderr_logger_tasks_finished: IntCounter,
2798 : }
2799 :
2800 : #[derive(Debug, enum_map::Enum, strum_macros::IntoStaticStr)]
2801 : pub(crate) enum WalRedoKillCause {
2802 : WalRedoProcessDrop,
2803 : NoLeakChildDrop,
2804 : Startup,
2805 : }
2806 :
2807 : impl Default for WalRedoProcessCounters {
2808 16 : fn default() -> Self {
2809 16 : let started = register_int_counter!(
2810 16 : "pageserver_wal_redo_process_started_total",
2811 16 : "Number of WAL redo processes started",
2812 16 : )
2813 16 : .unwrap();
2814 16 :
2815 16 : let killed = register_int_counter_vec!(
2816 16 : "pageserver_wal_redo_process_stopped_total",
2817 16 : "Number of WAL redo processes stopped",
2818 16 : &["cause"],
2819 16 : )
2820 16 : .unwrap();
2821 16 :
2822 16 : let active_stderr_logger_tasks_started = register_int_counter!(
2823 16 : "pageserver_walredo_stderr_logger_tasks_started_total",
2824 16 : "Number of active walredo stderr logger tasks that have started",
2825 16 : )
2826 16 : .unwrap();
2827 16 :
2828 16 : let active_stderr_logger_tasks_finished = register_int_counter!(
2829 16 : "pageserver_walredo_stderr_logger_tasks_finished_total",
2830 16 : "Number of active walredo stderr logger tasks that have finished",
2831 16 : )
2832 16 : .unwrap();
2833 16 :
2834 16 : Self {
2835 16 : started,
2836 48 : killed_by_cause: EnumMap::from_array(std::array::from_fn(|i| {
2837 48 : let cause = WalRedoKillCause::from_usize(i);
2838 48 : let cause_str: &'static str = cause.into();
2839 48 : killed.with_label_values(&[cause_str])
2840 48 : })),
2841 16 : active_stderr_logger_tasks_started,
2842 16 : active_stderr_logger_tasks_finished,
2843 16 : }
2844 16 : }
2845 : }
2846 :
2847 : pub(crate) static WAL_REDO_PROCESS_COUNTERS: Lazy<WalRedoProcessCounters> =
2848 : Lazy::new(WalRedoProcessCounters::default);
2849 :
2850 : /// Similar to `prometheus::HistogramTimer` but does not record on drop.
2851 : pub(crate) struct StorageTimeMetricsTimer {
2852 : metrics: StorageTimeMetrics,
2853 : start: Instant,
2854 : }
2855 :
2856 : impl StorageTimeMetricsTimer {
2857 4336 : fn new(metrics: StorageTimeMetrics) -> Self {
2858 4336 : Self {
2859 4336 : metrics,
2860 4336 : start: Instant::now(),
2861 4336 : }
2862 4336 : }
2863 :
2864 : /// Returns the elapsed duration of the timer.
2865 4336 : pub fn elapsed(&self) -> Duration {
2866 4336 : self.start.elapsed()
2867 4336 : }
2868 :
2869 : /// Record the time from creation to now and return it.
2870 4336 : pub fn stop_and_record(self) -> Duration {
2871 4336 : let duration = self.elapsed();
2872 4336 : let seconds = duration.as_secs_f64();
2873 4336 : self.metrics.timeline_sum.inc_by(seconds);
2874 4336 : self.metrics.timeline_count.inc();
2875 4336 : self.metrics.global_histogram.observe(seconds);
2876 4336 : duration
2877 4336 : }
2878 :
2879 : /// Turns this timer into a timer, which will always record -- usually this means recording
2880 : /// regardless an early `?` path was taken in a function.
2881 40 : pub(crate) fn record_on_drop(self) -> AlwaysRecordingStorageTimeMetricsTimer {
2882 40 : AlwaysRecordingStorageTimeMetricsTimer(Some(self))
2883 40 : }
2884 : }
2885 :
2886 : pub(crate) struct AlwaysRecordingStorageTimeMetricsTimer(Option<StorageTimeMetricsTimer>);
2887 :
2888 : impl Drop for AlwaysRecordingStorageTimeMetricsTimer {
2889 40 : fn drop(&mut self) {
2890 40 : if let Some(inner) = self.0.take() {
2891 40 : inner.stop_and_record();
2892 40 : }
2893 40 : }
2894 : }
2895 :
2896 : impl AlwaysRecordingStorageTimeMetricsTimer {
2897 : /// Returns the elapsed duration of the timer.
2898 0 : pub fn elapsed(&self) -> Duration {
2899 0 : self.0.as_ref().expect("not dropped yet").elapsed()
2900 0 : }
2901 : }
2902 :
2903 : /// Timing facilities for an globally histogrammed metric, which is supported by per tenant and
2904 : /// timeline total sum and count.
2905 : #[derive(Clone, Debug)]
2906 : pub(crate) struct StorageTimeMetrics {
2907 : /// Sum of f64 seconds, per operation, tenant_id and timeline_id
2908 : timeline_sum: Counter,
2909 : /// Number of oeprations, per operation, tenant_id and timeline_id
2910 : timeline_count: IntCounter,
2911 : /// Global histogram having only the "operation" label.
2912 : global_histogram: Histogram,
2913 : }
2914 :
2915 : impl StorageTimeMetrics {
2916 8316 : pub fn new(
2917 8316 : operation: StorageTimeOperation,
2918 8316 : tenant_id: &str,
2919 8316 : shard_id: &str,
2920 8316 : timeline_id: &str,
2921 8316 : ) -> Self {
2922 8316 : let operation: &'static str = operation.into();
2923 8316 :
2924 8316 : let timeline_sum = STORAGE_TIME_SUM_PER_TIMELINE
2925 8316 : .get_metric_with_label_values(&[operation, tenant_id, shard_id, timeline_id])
2926 8316 : .unwrap();
2927 8316 : let timeline_count = STORAGE_TIME_COUNT_PER_TIMELINE
2928 8316 : .get_metric_with_label_values(&[operation, tenant_id, shard_id, timeline_id])
2929 8316 : .unwrap();
2930 8316 : let global_histogram = STORAGE_TIME_GLOBAL
2931 8316 : .get_metric_with_label_values(&[operation])
2932 8316 : .unwrap();
2933 8316 :
2934 8316 : StorageTimeMetrics {
2935 8316 : timeline_sum,
2936 8316 : timeline_count,
2937 8316 : global_histogram,
2938 8316 : }
2939 8316 : }
2940 :
2941 : /// Starts timing a new operation.
2942 : ///
2943 : /// Note: unlike `prometheus::HistogramTimer` the returned timer does not record on drop.
2944 4336 : pub fn start_timer(&self) -> StorageTimeMetricsTimer {
2945 4336 : StorageTimeMetricsTimer::new(self.clone())
2946 4336 : }
2947 : }
2948 :
2949 : pub(crate) struct TimelineMetrics {
2950 : tenant_id: String,
2951 : shard_id: String,
2952 : timeline_id: String,
2953 : pub flush_time_histo: StorageTimeMetrics,
2954 : pub flush_delay_histo: StorageTimeMetrics,
2955 : pub compact_time_histo: StorageTimeMetrics,
2956 : pub create_images_time_histo: StorageTimeMetrics,
2957 : pub logical_size_histo: StorageTimeMetrics,
2958 : pub imitate_logical_size_histo: StorageTimeMetrics,
2959 : pub load_layer_map_histo: StorageTimeMetrics,
2960 : pub garbage_collect_histo: StorageTimeMetrics,
2961 : pub find_gc_cutoffs_histo: StorageTimeMetrics,
2962 : pub last_record_lsn_gauge: IntGauge,
2963 : pub disk_consistent_lsn_gauge: IntGauge,
2964 : pub pitr_history_size: UIntGauge,
2965 : pub archival_size: UIntGauge,
2966 : pub layers_per_read: Histogram,
2967 : pub standby_horizon_gauge: IntGauge,
2968 : pub resident_physical_size_gauge: UIntGauge,
2969 : pub visible_physical_size_gauge: UIntGauge,
2970 : /// copy of LayeredTimeline.current_logical_size
2971 : pub current_logical_size_gauge: UIntGauge,
2972 : pub aux_file_size_gauge: IntGauge,
2973 : pub directory_entries_count_gauge: Lazy<UIntGauge, Box<dyn Send + Fn() -> UIntGauge>>,
2974 : pub evictions: IntCounter,
2975 : pub evictions_with_low_residence_duration: std::sync::RwLock<EvictionsWithLowResidenceDuration>,
2976 : /// Number of valid LSN leases.
2977 : pub valid_lsn_lease_count_gauge: UIntGauge,
2978 : pub wal_records_received: IntCounter,
2979 : pub storage_io_size: StorageIoSizeMetrics,
2980 : pub wait_lsn_in_progress_micros: GlobalAndPerTenantIntCounter,
2981 : pub wait_lsn_start_finish_counterpair: IntCounterPair,
2982 : pub wait_ondemand_download_time: wait_ondemand_download_time::WaitOndemandDownloadTimeSum,
2983 : shutdown: std::sync::atomic::AtomicBool,
2984 : }
2985 :
2986 : impl TimelineMetrics {
2987 924 : pub fn new(
2988 924 : tenant_shard_id: &TenantShardId,
2989 924 : timeline_id_raw: &TimelineId,
2990 924 : evictions_with_low_residence_duration_builder: EvictionsWithLowResidenceDurationBuilder,
2991 924 : ) -> Self {
2992 924 : let tenant_id = tenant_shard_id.tenant_id.to_string();
2993 924 : let shard_id = format!("{}", tenant_shard_id.shard_slug());
2994 924 : let timeline_id = timeline_id_raw.to_string();
2995 924 : let flush_time_histo = StorageTimeMetrics::new(
2996 924 : StorageTimeOperation::LayerFlush,
2997 924 : &tenant_id,
2998 924 : &shard_id,
2999 924 : &timeline_id,
3000 924 : );
3001 924 : let flush_delay_histo = StorageTimeMetrics::new(
3002 924 : StorageTimeOperation::LayerFlushDelay,
3003 924 : &tenant_id,
3004 924 : &shard_id,
3005 924 : &timeline_id,
3006 924 : );
3007 924 : let compact_time_histo = StorageTimeMetrics::new(
3008 924 : StorageTimeOperation::Compact,
3009 924 : &tenant_id,
3010 924 : &shard_id,
3011 924 : &timeline_id,
3012 924 : );
3013 924 : let create_images_time_histo = StorageTimeMetrics::new(
3014 924 : StorageTimeOperation::CreateImages,
3015 924 : &tenant_id,
3016 924 : &shard_id,
3017 924 : &timeline_id,
3018 924 : );
3019 924 : let logical_size_histo = StorageTimeMetrics::new(
3020 924 : StorageTimeOperation::LogicalSize,
3021 924 : &tenant_id,
3022 924 : &shard_id,
3023 924 : &timeline_id,
3024 924 : );
3025 924 : let imitate_logical_size_histo = StorageTimeMetrics::new(
3026 924 : StorageTimeOperation::ImitateLogicalSize,
3027 924 : &tenant_id,
3028 924 : &shard_id,
3029 924 : &timeline_id,
3030 924 : );
3031 924 : let load_layer_map_histo = StorageTimeMetrics::new(
3032 924 : StorageTimeOperation::LoadLayerMap,
3033 924 : &tenant_id,
3034 924 : &shard_id,
3035 924 : &timeline_id,
3036 924 : );
3037 924 : let garbage_collect_histo = StorageTimeMetrics::new(
3038 924 : StorageTimeOperation::Gc,
3039 924 : &tenant_id,
3040 924 : &shard_id,
3041 924 : &timeline_id,
3042 924 : );
3043 924 : let find_gc_cutoffs_histo = StorageTimeMetrics::new(
3044 924 : StorageTimeOperation::FindGcCutoffs,
3045 924 : &tenant_id,
3046 924 : &shard_id,
3047 924 : &timeline_id,
3048 924 : );
3049 924 : let last_record_lsn_gauge = LAST_RECORD_LSN
3050 924 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
3051 924 : .unwrap();
3052 924 :
3053 924 : let disk_consistent_lsn_gauge = DISK_CONSISTENT_LSN
3054 924 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
3055 924 : .unwrap();
3056 924 :
3057 924 : let pitr_history_size = PITR_HISTORY_SIZE
3058 924 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
3059 924 : .unwrap();
3060 924 :
3061 924 : let archival_size = TIMELINE_ARCHIVE_SIZE
3062 924 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
3063 924 : .unwrap();
3064 924 :
3065 924 : let layers_per_read = LAYERS_PER_READ
3066 924 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
3067 924 : .unwrap();
3068 924 :
3069 924 : let standby_horizon_gauge = STANDBY_HORIZON
3070 924 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
3071 924 : .unwrap();
3072 924 : let resident_physical_size_gauge = RESIDENT_PHYSICAL_SIZE
3073 924 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
3074 924 : .unwrap();
3075 924 : let visible_physical_size_gauge = VISIBLE_PHYSICAL_SIZE
3076 924 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
3077 924 : .unwrap();
3078 924 : // TODO: we shouldn't expose this metric
3079 924 : let current_logical_size_gauge = CURRENT_LOGICAL_SIZE
3080 924 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
3081 924 : .unwrap();
3082 924 : let aux_file_size_gauge = AUX_FILE_SIZE
3083 924 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
3084 924 : .unwrap();
3085 924 : // TODO use impl Trait syntax here once we have ability to use it: https://github.com/rust-lang/rust/issues/63065
3086 924 : let directory_entries_count_gauge_closure = {
3087 924 : let tenant_shard_id = *tenant_shard_id;
3088 924 : let timeline_id_raw = *timeline_id_raw;
3089 0 : move || {
3090 0 : let tenant_id = tenant_shard_id.tenant_id.to_string();
3091 0 : let shard_id = format!("{}", tenant_shard_id.shard_slug());
3092 0 : let timeline_id = timeline_id_raw.to_string();
3093 0 : let gauge: UIntGauge = DIRECTORY_ENTRIES_COUNT
3094 0 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
3095 0 : .unwrap();
3096 0 : gauge
3097 0 : }
3098 : };
3099 924 : let directory_entries_count_gauge: Lazy<UIntGauge, Box<dyn Send + Fn() -> UIntGauge>> =
3100 924 : Lazy::new(Box::new(directory_entries_count_gauge_closure));
3101 924 : let evictions = EVICTIONS
3102 924 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
3103 924 : .unwrap();
3104 924 : let evictions_with_low_residence_duration = evictions_with_low_residence_duration_builder
3105 924 : .build(&tenant_id, &shard_id, &timeline_id);
3106 924 :
3107 924 : let valid_lsn_lease_count_gauge = VALID_LSN_LEASE_COUNT
3108 924 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
3109 924 : .unwrap();
3110 924 :
3111 924 : let wal_records_received = PAGESERVER_TIMELINE_WAL_RECORDS_RECEIVED
3112 924 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
3113 924 : .unwrap();
3114 924 :
3115 924 : let storage_io_size = StorageIoSizeMetrics::new(&tenant_id, &shard_id, &timeline_id);
3116 924 :
3117 924 : let wait_lsn_in_progress_micros = GlobalAndPerTenantIntCounter {
3118 924 : global: WAIT_LSN_IN_PROGRESS_GLOBAL_MICROS.clone(),
3119 924 : per_tenant: WAIT_LSN_IN_PROGRESS_MICROS
3120 924 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
3121 924 : .unwrap(),
3122 924 : };
3123 924 :
3124 924 : let wait_lsn_start_finish_counterpair = WAIT_LSN_START_FINISH_COUNTERPAIR
3125 924 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
3126 924 : .unwrap();
3127 924 :
3128 924 : let wait_ondemand_download_time =
3129 924 : wait_ondemand_download_time::WaitOndemandDownloadTimeSum::new(
3130 924 : &tenant_id,
3131 924 : &shard_id,
3132 924 : &timeline_id,
3133 924 : );
3134 924 :
3135 924 : TimelineMetrics {
3136 924 : tenant_id,
3137 924 : shard_id,
3138 924 : timeline_id,
3139 924 : flush_time_histo,
3140 924 : flush_delay_histo,
3141 924 : compact_time_histo,
3142 924 : create_images_time_histo,
3143 924 : logical_size_histo,
3144 924 : imitate_logical_size_histo,
3145 924 : garbage_collect_histo,
3146 924 : find_gc_cutoffs_histo,
3147 924 : load_layer_map_histo,
3148 924 : last_record_lsn_gauge,
3149 924 : disk_consistent_lsn_gauge,
3150 924 : pitr_history_size,
3151 924 : archival_size,
3152 924 : layers_per_read,
3153 924 : standby_horizon_gauge,
3154 924 : resident_physical_size_gauge,
3155 924 : visible_physical_size_gauge,
3156 924 : current_logical_size_gauge,
3157 924 : aux_file_size_gauge,
3158 924 : directory_entries_count_gauge,
3159 924 : evictions,
3160 924 : evictions_with_low_residence_duration: std::sync::RwLock::new(
3161 924 : evictions_with_low_residence_duration,
3162 924 : ),
3163 924 : storage_io_size,
3164 924 : valid_lsn_lease_count_gauge,
3165 924 : wal_records_received,
3166 924 : wait_lsn_in_progress_micros,
3167 924 : wait_lsn_start_finish_counterpair,
3168 924 : wait_ondemand_download_time,
3169 924 : shutdown: std::sync::atomic::AtomicBool::default(),
3170 924 : }
3171 924 : }
3172 :
3173 3164 : pub(crate) fn record_new_file_metrics(&self, sz: u64) {
3174 3164 : self.resident_physical_size_add(sz);
3175 3164 : }
3176 :
3177 1084 : pub(crate) fn resident_physical_size_sub(&self, sz: u64) {
3178 1084 : self.resident_physical_size_gauge.sub(sz);
3179 1084 : crate::metrics::RESIDENT_PHYSICAL_SIZE_GLOBAL.sub(sz);
3180 1084 : }
3181 :
3182 3436 : pub(crate) fn resident_physical_size_add(&self, sz: u64) {
3183 3436 : self.resident_physical_size_gauge.add(sz);
3184 3436 : crate::metrics::RESIDENT_PHYSICAL_SIZE_GLOBAL.add(sz);
3185 3436 : }
3186 :
3187 20 : pub(crate) fn resident_physical_size_get(&self) -> u64 {
3188 20 : self.resident_physical_size_gauge.get()
3189 20 : }
3190 :
3191 : /// Generates TIMELINE_LAYER labels for a persistent layer.
3192 5256 : fn make_layer_labels(&self, layer_desc: &PersistentLayerDesc) -> [&str; 5] {
3193 5256 : let level = match LayerMap::is_l0(&layer_desc.key_range, layer_desc.is_delta()) {
3194 2846 : true => LayerLevel::L0,
3195 2410 : false => LayerLevel::L1,
3196 : };
3197 5256 : let kind = match layer_desc.is_delta() {
3198 4362 : true => LayerKind::Delta,
3199 894 : false => LayerKind::Image,
3200 : };
3201 5256 : [
3202 5256 : &self.tenant_id,
3203 5256 : &self.shard_id,
3204 5256 : &self.timeline_id,
3205 5256 : level.into(),
3206 5256 : kind.into(),
3207 5256 : ]
3208 5256 : }
3209 :
3210 : /// Generates TIMELINE_LAYER labels for a frozen ephemeral layer.
3211 4728 : fn make_frozen_layer_labels(&self, _layer: &InMemoryLayer) -> [&str; 5] {
3212 4728 : [
3213 4728 : &self.tenant_id,
3214 4728 : &self.shard_id,
3215 4728 : &self.timeline_id,
3216 4728 : LayerLevel::Frozen.into(),
3217 4728 : LayerKind::Delta.into(), // by definition
3218 4728 : ]
3219 4728 : }
3220 :
3221 : /// Removes a frozen ephemeral layer to TIMELINE_LAYER metrics.
3222 2364 : pub fn dec_frozen_layer(&self, layer: &InMemoryLayer) {
3223 2364 : assert!(matches!(layer.info(), InMemoryLayerInfo::Frozen { .. }));
3224 2364 : let labels = self.make_frozen_layer_labels(layer);
3225 2364 : let size = layer.try_len().expect("frozen layer should have no writer");
3226 2364 : TIMELINE_LAYER_COUNT
3227 2364 : .get_metric_with_label_values(&labels)
3228 2364 : .unwrap()
3229 2364 : .dec();
3230 2364 : TIMELINE_LAYER_SIZE
3231 2364 : .get_metric_with_label_values(&labels)
3232 2364 : .unwrap()
3233 2364 : .sub(size);
3234 2364 : }
3235 :
3236 : /// Adds a frozen ephemeral layer to TIMELINE_LAYER metrics.
3237 2364 : pub fn inc_frozen_layer(&self, layer: &InMemoryLayer) {
3238 2364 : assert!(matches!(layer.info(), InMemoryLayerInfo::Frozen { .. }));
3239 2364 : let labels = self.make_frozen_layer_labels(layer);
3240 2364 : let size = layer.try_len().expect("frozen layer should have no writer");
3241 2364 : TIMELINE_LAYER_COUNT
3242 2364 : .get_metric_with_label_values(&labels)
3243 2364 : .unwrap()
3244 2364 : .inc();
3245 2364 : TIMELINE_LAYER_SIZE
3246 2364 : .get_metric_with_label_values(&labels)
3247 2364 : .unwrap()
3248 2364 : .add(size);
3249 2364 : }
3250 :
3251 : /// Removes a persistent layer from TIMELINE_LAYER metrics.
3252 1380 : pub fn dec_layer(&self, layer_desc: &PersistentLayerDesc) {
3253 1380 : let labels = self.make_layer_labels(layer_desc);
3254 1380 : TIMELINE_LAYER_COUNT
3255 1380 : .get_metric_with_label_values(&labels)
3256 1380 : .unwrap()
3257 1380 : .dec();
3258 1380 : TIMELINE_LAYER_SIZE
3259 1380 : .get_metric_with_label_values(&labels)
3260 1380 : .unwrap()
3261 1380 : .sub(layer_desc.file_size);
3262 1380 : }
3263 :
3264 : /// Adds a persistent layer to TIMELINE_LAYER metrics.
3265 3876 : pub fn inc_layer(&self, layer_desc: &PersistentLayerDesc) {
3266 3876 : let labels = self.make_layer_labels(layer_desc);
3267 3876 : TIMELINE_LAYER_COUNT
3268 3876 : .get_metric_with_label_values(&labels)
3269 3876 : .unwrap()
3270 3876 : .inc();
3271 3876 : TIMELINE_LAYER_SIZE
3272 3876 : .get_metric_with_label_values(&labels)
3273 3876 : .unwrap()
3274 3876 : .add(layer_desc.file_size);
3275 3876 : }
3276 :
3277 20 : pub(crate) fn shutdown(&self) {
3278 20 : let was_shutdown = self
3279 20 : .shutdown
3280 20 : .swap(true, std::sync::atomic::Ordering::Relaxed);
3281 20 :
3282 20 : if was_shutdown {
3283 : // this happens on tenant deletion because tenant first shuts down timelines, then
3284 : // invokes timeline deletion which first shuts down the timeline again.
3285 : // TODO: this can be removed once https://github.com/neondatabase/neon/issues/5080
3286 0 : return;
3287 20 : }
3288 20 :
3289 20 : let tenant_id = &self.tenant_id;
3290 20 : let timeline_id = &self.timeline_id;
3291 20 : let shard_id = &self.shard_id;
3292 20 : let _ = LAST_RECORD_LSN.remove_label_values(&[tenant_id, shard_id, timeline_id]);
3293 20 : let _ = DISK_CONSISTENT_LSN.remove_label_values(&[tenant_id, shard_id, timeline_id]);
3294 20 : let _ = STANDBY_HORIZON.remove_label_values(&[tenant_id, shard_id, timeline_id]);
3295 20 : {
3296 20 : RESIDENT_PHYSICAL_SIZE_GLOBAL.sub(self.resident_physical_size_get());
3297 20 : let _ = RESIDENT_PHYSICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
3298 20 : }
3299 20 : let _ = VISIBLE_PHYSICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
3300 20 : let _ = CURRENT_LOGICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
3301 20 : if let Some(metric) = Lazy::get(&DIRECTORY_ENTRIES_COUNT) {
3302 0 : let _ = metric.remove_label_values(&[tenant_id, shard_id, timeline_id]);
3303 20 : }
3304 :
3305 20 : let _ = TIMELINE_ARCHIVE_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
3306 20 : let _ = PITR_HISTORY_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
3307 :
3308 80 : for ref level in LayerLevel::iter() {
3309 180 : for ref kind in LayerKind::iter() {
3310 120 : let labels: [&str; 5] =
3311 120 : [tenant_id, shard_id, timeline_id, level.into(), kind.into()];
3312 120 : let _ = TIMELINE_LAYER_SIZE.remove_label_values(&labels);
3313 120 : let _ = TIMELINE_LAYER_COUNT.remove_label_values(&labels);
3314 120 : }
3315 : }
3316 :
3317 20 : let _ = LAYERS_PER_READ.remove_label_values(&[tenant_id, shard_id, timeline_id]);
3318 20 :
3319 20 : let _ = EVICTIONS.remove_label_values(&[tenant_id, shard_id, timeline_id]);
3320 20 : let _ = AUX_FILE_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
3321 20 : let _ = VALID_LSN_LEASE_COUNT.remove_label_values(&[tenant_id, shard_id, timeline_id]);
3322 20 :
3323 20 : self.evictions_with_low_residence_duration
3324 20 : .write()
3325 20 : .unwrap()
3326 20 : .remove(tenant_id, shard_id, timeline_id);
3327 :
3328 : // The following metrics are born outside of the TimelineMetrics lifecycle but still
3329 : // removed at the end of it. The idea is to have the metrics outlive the
3330 : // entity during which they're observed, e.g., the smgr metrics shall
3331 : // outlive an individual smgr connection, but not the timeline.
3332 :
3333 200 : for op in StorageTimeOperation::VARIANTS {
3334 180 : let _ = STORAGE_TIME_SUM_PER_TIMELINE.remove_label_values(&[
3335 180 : op,
3336 180 : tenant_id,
3337 180 : shard_id,
3338 180 : timeline_id,
3339 180 : ]);
3340 180 : let _ = STORAGE_TIME_COUNT_PER_TIMELINE.remove_label_values(&[
3341 180 : op,
3342 180 : tenant_id,
3343 180 : shard_id,
3344 180 : timeline_id,
3345 180 : ]);
3346 180 : }
3347 :
3348 60 : for op in StorageIoSizeOperation::VARIANTS {
3349 40 : let _ = STORAGE_IO_SIZE.remove_label_values(&[op, tenant_id, shard_id, timeline_id]);
3350 40 : }
3351 :
3352 : let _ =
3353 20 : WAIT_LSN_IN_PROGRESS_MICROS.remove_label_values(&[tenant_id, shard_id, timeline_id]);
3354 20 :
3355 20 : {
3356 20 : let mut res = [Ok(()), Ok(())];
3357 20 : WAIT_LSN_START_FINISH_COUNTERPAIR
3358 20 : .remove_label_values(&mut res, &[tenant_id, shard_id, timeline_id]);
3359 20 : }
3360 20 :
3361 20 : wait_ondemand_download_time::shutdown_timeline(tenant_id, shard_id, timeline_id);
3362 20 :
3363 20 : let _ = SMGR_QUERY_STARTED_PER_TENANT_TIMELINE.remove_label_values(&[
3364 20 : SmgrQueryType::GetPageAtLsn.into(),
3365 20 : tenant_id,
3366 20 : shard_id,
3367 20 : timeline_id,
3368 20 : ]);
3369 20 : let _ = SMGR_QUERY_TIME_PER_TENANT_TIMELINE.remove_label_values(&[
3370 20 : SmgrQueryType::GetPageAtLsn.into(),
3371 20 : tenant_id,
3372 20 : shard_id,
3373 20 : timeline_id,
3374 20 : ]);
3375 20 : let _ = PAGE_SERVICE_BATCH_SIZE_PER_TENANT_TIMELINE.remove_label_values(&[
3376 20 : tenant_id,
3377 20 : shard_id,
3378 20 : timeline_id,
3379 20 : ]);
3380 20 : let _ = PAGESERVER_TIMELINE_WAL_RECORDS_RECEIVED.remove_label_values(&[
3381 20 : tenant_id,
3382 20 : shard_id,
3383 20 : timeline_id,
3384 20 : ]);
3385 20 : let _ = PAGE_SERVICE_SMGR_FLUSH_INPROGRESS_MICROS.remove_label_values(&[
3386 20 : tenant_id,
3387 20 : shard_id,
3388 20 : timeline_id,
3389 20 : ]);
3390 20 : let _ = PAGE_SERVICE_SMGR_BATCH_WAIT_TIME.remove_label_values(&[
3391 20 : tenant_id,
3392 20 : shard_id,
3393 20 : timeline_id,
3394 20 : ]);
3395 20 : }
3396 : }
3397 :
3398 12 : pub(crate) fn remove_tenant_metrics(tenant_shard_id: &TenantShardId) {
3399 12 : // Only shard zero deals in synthetic sizes
3400 12 : if tenant_shard_id.is_shard_zero() {
3401 12 : let tid = tenant_shard_id.tenant_id.to_string();
3402 12 : let _ = TENANT_SYNTHETIC_SIZE_METRIC.remove_label_values(&[&tid]);
3403 12 : }
3404 :
3405 12 : tenant_throttling::remove_tenant_metrics(tenant_shard_id);
3406 12 :
3407 12 : // we leave the BROKEN_TENANTS_SET entry if any
3408 12 : }
3409 :
3410 : /// Maintain a per timeline gauge in addition to the global gauge.
3411 : pub(crate) struct PerTimelineRemotePhysicalSizeGauge {
3412 : last_set: AtomicU64,
3413 : gauge: UIntGauge,
3414 : }
3415 :
3416 : impl PerTimelineRemotePhysicalSizeGauge {
3417 944 : fn new(per_timeline_gauge: UIntGauge) -> Self {
3418 944 : Self {
3419 944 : last_set: AtomicU64::new(0),
3420 944 : gauge: per_timeline_gauge,
3421 944 : }
3422 944 : }
3423 3878 : pub(crate) fn set(&self, sz: u64) {
3424 3878 : self.gauge.set(sz);
3425 3878 : let prev = self.last_set.swap(sz, std::sync::atomic::Ordering::Relaxed);
3426 3878 : if sz < prev {
3427 69 : REMOTE_PHYSICAL_SIZE_GLOBAL.sub(prev - sz);
3428 3809 : } else {
3429 3809 : REMOTE_PHYSICAL_SIZE_GLOBAL.add(sz - prev);
3430 3809 : };
3431 3878 : }
3432 4 : pub(crate) fn get(&self) -> u64 {
3433 4 : self.gauge.get()
3434 4 : }
3435 : }
3436 :
3437 : impl Drop for PerTimelineRemotePhysicalSizeGauge {
3438 40 : fn drop(&mut self) {
3439 40 : REMOTE_PHYSICAL_SIZE_GLOBAL.sub(self.last_set.load(std::sync::atomic::Ordering::Relaxed));
3440 40 : }
3441 : }
3442 :
3443 : pub(crate) struct RemoteTimelineClientMetrics {
3444 : tenant_id: String,
3445 : shard_id: String,
3446 : timeline_id: String,
3447 : pub(crate) remote_physical_size_gauge: PerTimelineRemotePhysicalSizeGauge,
3448 : calls: Mutex<HashMap<(&'static str, &'static str), IntCounterPair>>,
3449 : bytes_started_counter: Mutex<HashMap<(&'static str, &'static str), IntCounter>>,
3450 : bytes_finished_counter: Mutex<HashMap<(&'static str, &'static str), IntCounter>>,
3451 : pub(crate) projected_remote_consistent_lsn_gauge: UIntGauge,
3452 : }
3453 :
3454 : impl RemoteTimelineClientMetrics {
3455 944 : pub fn new(tenant_shard_id: &TenantShardId, timeline_id: &TimelineId) -> Self {
3456 944 : let tenant_id_str = tenant_shard_id.tenant_id.to_string();
3457 944 : let shard_id_str = format!("{}", tenant_shard_id.shard_slug());
3458 944 : let timeline_id_str = timeline_id.to_string();
3459 944 :
3460 944 : let remote_physical_size_gauge = PerTimelineRemotePhysicalSizeGauge::new(
3461 944 : REMOTE_PHYSICAL_SIZE
3462 944 : .get_metric_with_label_values(&[&tenant_id_str, &shard_id_str, &timeline_id_str])
3463 944 : .unwrap(),
3464 944 : );
3465 944 :
3466 944 : let projected_remote_consistent_lsn_gauge = PROJECTED_REMOTE_CONSISTENT_LSN
3467 944 : .get_metric_with_label_values(&[&tenant_id_str, &shard_id_str, &timeline_id_str])
3468 944 : .unwrap();
3469 944 :
3470 944 : RemoteTimelineClientMetrics {
3471 944 : tenant_id: tenant_id_str,
3472 944 : shard_id: shard_id_str,
3473 944 : timeline_id: timeline_id_str,
3474 944 : calls: Mutex::new(HashMap::default()),
3475 944 : bytes_started_counter: Mutex::new(HashMap::default()),
3476 944 : bytes_finished_counter: Mutex::new(HashMap::default()),
3477 944 : remote_physical_size_gauge,
3478 944 : projected_remote_consistent_lsn_gauge,
3479 944 : }
3480 944 : }
3481 :
3482 6101 : pub fn remote_operation_time(
3483 6101 : &self,
3484 6101 : task_kind: Option<TaskKind>,
3485 6101 : file_kind: &RemoteOpFileKind,
3486 6101 : op_kind: &RemoteOpKind,
3487 6101 : status: &'static str,
3488 6101 : ) -> Histogram {
3489 6101 : REMOTE_TIMELINE_CLIENT_COMPLETION_LATENCY
3490 6101 : .get_metric_with_label_values(&[
3491 6101 : task_kind.as_ref().map(|tk| tk.into()).unwrap_or("unknown"),
3492 6101 : file_kind.as_str(),
3493 6101 : op_kind.as_str(),
3494 6101 : status,
3495 6101 : ])
3496 6101 : .unwrap()
3497 6101 : }
3498 :
3499 14367 : fn calls_counter_pair(
3500 14367 : &self,
3501 14367 : file_kind: &RemoteOpFileKind,
3502 14367 : op_kind: &RemoteOpKind,
3503 14367 : ) -> IntCounterPair {
3504 14367 : let mut guard = self.calls.lock().unwrap();
3505 14367 : let key = (file_kind.as_str(), op_kind.as_str());
3506 14367 : let metric = guard.entry(key).or_insert_with(move || {
3507 1692 : REMOTE_TIMELINE_CLIENT_CALLS
3508 1692 : .get_metric_with_label_values(&[
3509 1692 : &self.tenant_id,
3510 1692 : &self.shard_id,
3511 1692 : &self.timeline_id,
3512 1692 : key.0,
3513 1692 : key.1,
3514 1692 : ])
3515 1692 : .unwrap()
3516 14367 : });
3517 14367 : metric.clone()
3518 14367 : }
3519 :
3520 3500 : fn bytes_started_counter(
3521 3500 : &self,
3522 3500 : file_kind: &RemoteOpFileKind,
3523 3500 : op_kind: &RemoteOpKind,
3524 3500 : ) -> IntCounter {
3525 3500 : let mut guard = self.bytes_started_counter.lock().unwrap();
3526 3500 : let key = (file_kind.as_str(), op_kind.as_str());
3527 3500 : let metric = guard.entry(key).or_insert_with(move || {
3528 664 : REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER
3529 664 : .get_metric_with_label_values(&[
3530 664 : &self.tenant_id,
3531 664 : &self.shard_id,
3532 664 : &self.timeline_id,
3533 664 : key.0,
3534 664 : key.1,
3535 664 : ])
3536 664 : .unwrap()
3537 3500 : });
3538 3500 : metric.clone()
3539 3500 : }
3540 :
3541 6579 : fn bytes_finished_counter(
3542 6579 : &self,
3543 6579 : file_kind: &RemoteOpFileKind,
3544 6579 : op_kind: &RemoteOpKind,
3545 6579 : ) -> IntCounter {
3546 6579 : let mut guard = self.bytes_finished_counter.lock().unwrap();
3547 6579 : let key = (file_kind.as_str(), op_kind.as_str());
3548 6579 : let metric = guard.entry(key).or_insert_with(move || {
3549 664 : REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER
3550 664 : .get_metric_with_label_values(&[
3551 664 : &self.tenant_id,
3552 664 : &self.shard_id,
3553 664 : &self.timeline_id,
3554 664 : key.0,
3555 664 : key.1,
3556 664 : ])
3557 664 : .unwrap()
3558 6579 : });
3559 6579 : metric.clone()
3560 6579 : }
3561 : }
3562 :
3563 : #[cfg(test)]
3564 : impl RemoteTimelineClientMetrics {
3565 12 : pub fn get_bytes_started_counter_value(
3566 12 : &self,
3567 12 : file_kind: &RemoteOpFileKind,
3568 12 : op_kind: &RemoteOpKind,
3569 12 : ) -> Option<u64> {
3570 12 : let guard = self.bytes_started_counter.lock().unwrap();
3571 12 : let key = (file_kind.as_str(), op_kind.as_str());
3572 12 : guard.get(&key).map(|counter| counter.get())
3573 12 : }
3574 :
3575 12 : pub fn get_bytes_finished_counter_value(
3576 12 : &self,
3577 12 : file_kind: &RemoteOpFileKind,
3578 12 : op_kind: &RemoteOpKind,
3579 12 : ) -> Option<u64> {
3580 12 : let guard = self.bytes_finished_counter.lock().unwrap();
3581 12 : let key = (file_kind.as_str(), op_kind.as_str());
3582 12 : guard.get(&key).map(|counter| counter.get())
3583 12 : }
3584 : }
3585 :
3586 : /// See [`RemoteTimelineClientMetrics::call_begin`].
3587 : #[must_use]
3588 : pub(crate) struct RemoteTimelineClientCallMetricGuard {
3589 : /// Decremented on drop.
3590 : calls_counter_pair: Option<IntCounterPair>,
3591 : /// If Some(), this references the bytes_finished metric, and we increment it by the given `u64` on drop.
3592 : bytes_finished: Option<(IntCounter, u64)>,
3593 : }
3594 :
3595 : impl RemoteTimelineClientCallMetricGuard {
3596 : /// Consume this guard object without performing the metric updates it would do on `drop()`.
3597 : /// The caller vouches to do the metric updates manually.
3598 7636 : pub fn will_decrement_manually(mut self) {
3599 7636 : let RemoteTimelineClientCallMetricGuard {
3600 7636 : calls_counter_pair,
3601 7636 : bytes_finished,
3602 7636 : } = &mut self;
3603 7636 : calls_counter_pair.take();
3604 7636 : bytes_finished.take();
3605 7636 : }
3606 : }
3607 :
3608 : impl Drop for RemoteTimelineClientCallMetricGuard {
3609 7704 : fn drop(&mut self) {
3610 7704 : let RemoteTimelineClientCallMetricGuard {
3611 7704 : calls_counter_pair,
3612 7704 : bytes_finished,
3613 7704 : } = self;
3614 7704 : if let Some(guard) = calls_counter_pair.take() {
3615 68 : guard.dec();
3616 7636 : }
3617 7704 : if let Some((bytes_finished_metric, value)) = bytes_finished {
3618 0 : bytes_finished_metric.inc_by(*value);
3619 7704 : }
3620 7704 : }
3621 : }
3622 :
3623 : /// The enum variants communicate to the [`RemoteTimelineClientMetrics`] whether to
3624 : /// track the byte size of this call in applicable metric(s).
3625 : pub(crate) enum RemoteTimelineClientMetricsCallTrackSize {
3626 : /// Do not account for this call's byte size in any metrics.
3627 : /// The `reason` field is there to make the call sites self-documenting
3628 : /// about why they don't need the metric.
3629 : DontTrackSize { reason: &'static str },
3630 : /// Track the byte size of the call in applicable metric(s).
3631 : Bytes(u64),
3632 : }
3633 :
3634 : impl RemoteTimelineClientMetrics {
3635 : /// Update the metrics that change when a call to the remote timeline client instance starts.
3636 : ///
3637 : /// Drop the returned guard object once the operation is finished to updates corresponding metrics that track completions.
3638 : /// Or, use [`RemoteTimelineClientCallMetricGuard::will_decrement_manually`] and [`call_end`](Self::call_end) if that
3639 : /// is more suitable.
3640 : /// Never do both.
3641 7704 : pub(crate) fn call_begin(
3642 7704 : &self,
3643 7704 : file_kind: &RemoteOpFileKind,
3644 7704 : op_kind: &RemoteOpKind,
3645 7704 : size: RemoteTimelineClientMetricsCallTrackSize,
3646 7704 : ) -> RemoteTimelineClientCallMetricGuard {
3647 7704 : let calls_counter_pair = self.calls_counter_pair(file_kind, op_kind);
3648 7704 : calls_counter_pair.inc();
3649 :
3650 7704 : let bytes_finished = match size {
3651 4204 : RemoteTimelineClientMetricsCallTrackSize::DontTrackSize { reason: _reason } => {
3652 4204 : // nothing to do
3653 4204 : None
3654 : }
3655 3500 : RemoteTimelineClientMetricsCallTrackSize::Bytes(size) => {
3656 3500 : self.bytes_started_counter(file_kind, op_kind).inc_by(size);
3657 3500 : let finished_counter = self.bytes_finished_counter(file_kind, op_kind);
3658 3500 : Some((finished_counter, size))
3659 : }
3660 : };
3661 7704 : RemoteTimelineClientCallMetricGuard {
3662 7704 : calls_counter_pair: Some(calls_counter_pair),
3663 7704 : bytes_finished,
3664 7704 : }
3665 7704 : }
3666 :
3667 : /// Manually udpate the metrics that track completions, instead of using the guard object.
3668 : /// Using the guard object is generally preferable.
3669 : /// See [`call_begin`](Self::call_begin) for more context.
3670 6663 : pub(crate) fn call_end(
3671 6663 : &self,
3672 6663 : file_kind: &RemoteOpFileKind,
3673 6663 : op_kind: &RemoteOpKind,
3674 6663 : size: RemoteTimelineClientMetricsCallTrackSize,
3675 6663 : ) {
3676 6663 : let calls_counter_pair = self.calls_counter_pair(file_kind, op_kind);
3677 6663 : calls_counter_pair.dec();
3678 6663 : match size {
3679 3584 : RemoteTimelineClientMetricsCallTrackSize::DontTrackSize { reason: _reason } => {}
3680 3079 : RemoteTimelineClientMetricsCallTrackSize::Bytes(size) => {
3681 3079 : self.bytes_finished_counter(file_kind, op_kind).inc_by(size);
3682 3079 : }
3683 : }
3684 6663 : }
3685 : }
3686 :
3687 : impl Drop for RemoteTimelineClientMetrics {
3688 40 : fn drop(&mut self) {
3689 40 : let RemoteTimelineClientMetrics {
3690 40 : tenant_id,
3691 40 : shard_id,
3692 40 : timeline_id,
3693 40 : remote_physical_size_gauge,
3694 40 : calls,
3695 40 : bytes_started_counter,
3696 40 : bytes_finished_counter,
3697 40 : projected_remote_consistent_lsn_gauge,
3698 40 : } = self;
3699 48 : for ((a, b), _) in calls.get_mut().unwrap().drain() {
3700 48 : let mut res = [Ok(()), Ok(())];
3701 48 : REMOTE_TIMELINE_CLIENT_CALLS
3702 48 : .remove_label_values(&mut res, &[tenant_id, shard_id, timeline_id, a, b]);
3703 48 : // don't care about results
3704 48 : }
3705 40 : for ((a, b), _) in bytes_started_counter.get_mut().unwrap().drain() {
3706 12 : let _ = REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER.remove_label_values(&[
3707 12 : tenant_id,
3708 12 : shard_id,
3709 12 : timeline_id,
3710 12 : a,
3711 12 : b,
3712 12 : ]);
3713 12 : }
3714 40 : for ((a, b), _) in bytes_finished_counter.get_mut().unwrap().drain() {
3715 12 : let _ = REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER.remove_label_values(&[
3716 12 : tenant_id,
3717 12 : shard_id,
3718 12 : timeline_id,
3719 12 : a,
3720 12 : b,
3721 12 : ]);
3722 12 : }
3723 40 : {
3724 40 : let _ = remote_physical_size_gauge; // use to avoid 'unused' warning in desctructuring above
3725 40 : let _ = REMOTE_PHYSICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
3726 40 : }
3727 40 : {
3728 40 : let _ = projected_remote_consistent_lsn_gauge;
3729 40 : let _ = PROJECTED_REMOTE_CONSISTENT_LSN.remove_label_values(&[
3730 40 : tenant_id,
3731 40 : shard_id,
3732 40 : timeline_id,
3733 40 : ]);
3734 40 : }
3735 40 : }
3736 : }
3737 :
3738 : /// Wrapper future that measures the time spent by a remote storage operation,
3739 : /// and records the time and success/failure as a prometheus metric.
3740 : pub(crate) trait MeasureRemoteOp<O, E>: Sized + Future<Output = Result<O, E>> {
3741 6436 : async fn measure_remote_op(
3742 6436 : self,
3743 6436 : task_kind: Option<TaskKind>, // not all caller contexts have a RequestContext / TaskKind handy
3744 6436 : file_kind: RemoteOpFileKind,
3745 6436 : op: RemoteOpKind,
3746 6436 : metrics: Arc<RemoteTimelineClientMetrics>,
3747 6436 : ) -> Result<O, E> {
3748 6436 : let start = Instant::now();
3749 6436 : let res = self.await;
3750 6101 : let duration = start.elapsed();
3751 6101 : let status = if res.is_ok() { &"success" } else { &"failure" };
3752 6101 : metrics
3753 6101 : .remote_operation_time(task_kind, &file_kind, &op, status)
3754 6101 : .observe(duration.as_secs_f64());
3755 6101 : res
3756 6101 : }
3757 : }
3758 :
3759 : impl<Fut, O, E> MeasureRemoteOp<O, E> for Fut where Fut: Sized + Future<Output = Result<O, E>> {}
3760 :
3761 : pub mod tokio_epoll_uring {
3762 : use std::collections::HashMap;
3763 : use std::sync::{Arc, Mutex};
3764 :
3765 : use metrics::{Histogram, LocalHistogram, UIntGauge, register_histogram, register_int_counter};
3766 : use once_cell::sync::Lazy;
3767 :
3768 : /// Shared storage for tokio-epoll-uring thread local metrics.
3769 : pub(crate) static THREAD_LOCAL_METRICS_STORAGE: Lazy<ThreadLocalMetricsStorage> =
3770 242 : Lazy::new(|| {
3771 242 : let slots_submission_queue_depth = register_histogram!(
3772 242 : "pageserver_tokio_epoll_uring_slots_submission_queue_depth",
3773 242 : "The slots waiters queue depth of each tokio_epoll_uring system",
3774 242 : vec![
3775 242 : 1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0
3776 242 : ],
3777 242 : )
3778 242 : .expect("failed to define a metric");
3779 242 : ThreadLocalMetricsStorage {
3780 242 : observers: Mutex::new(HashMap::new()),
3781 242 : slots_submission_queue_depth,
3782 242 : }
3783 242 : });
3784 :
3785 : pub struct ThreadLocalMetricsStorage {
3786 : /// List of thread local metrics observers.
3787 : observers: Mutex<HashMap<u64, Arc<ThreadLocalMetrics>>>,
3788 : /// A histogram shared between all thread local systems
3789 : /// for collecting slots submission queue depth.
3790 : slots_submission_queue_depth: Histogram,
3791 : }
3792 :
3793 : /// Each thread-local [`tokio_epoll_uring::System`] gets one of these as its
3794 : /// [`tokio_epoll_uring::metrics::PerSystemMetrics`] generic.
3795 : ///
3796 : /// The System makes observations into [`Self`] and periodically, the collector
3797 : /// comes along and flushes [`Self`] into the shared storage [`THREAD_LOCAL_METRICS_STORAGE`].
3798 : ///
3799 : /// [`LocalHistogram`] is `!Send`, so, we need to put it behind a [`Mutex`].
3800 : /// But except for the periodic flush, the lock is uncontended so there's no waiting
3801 : /// for cache coherence protocol to get an exclusive cache line.
3802 : pub struct ThreadLocalMetrics {
3803 : /// Local observer of thread local tokio-epoll-uring system's slots waiters queue depth.
3804 : slots_submission_queue_depth: Mutex<LocalHistogram>,
3805 : }
3806 :
3807 : impl ThreadLocalMetricsStorage {
3808 : /// Registers a new thread local system. Returns a thread local metrics observer.
3809 953 : pub fn register_system(&self, id: u64) -> Arc<ThreadLocalMetrics> {
3810 953 : let per_system_metrics = Arc::new(ThreadLocalMetrics::new(
3811 953 : self.slots_submission_queue_depth.local(),
3812 953 : ));
3813 953 : let mut g = self.observers.lock().unwrap();
3814 953 : g.insert(id, Arc::clone(&per_system_metrics));
3815 953 : per_system_metrics
3816 953 : }
3817 :
3818 : /// Removes metrics observer for a thread local system.
3819 : /// This should be called before dropping a thread local system.
3820 242 : pub fn remove_system(&self, id: u64) {
3821 242 : let mut g = self.observers.lock().unwrap();
3822 242 : g.remove(&id);
3823 242 : }
3824 :
3825 : /// Flush all thread local metrics to the shared storage.
3826 0 : pub fn flush_thread_local_metrics(&self) {
3827 0 : let g = self.observers.lock().unwrap();
3828 0 : g.values().for_each(|local| {
3829 0 : local.flush();
3830 0 : });
3831 0 : }
3832 : }
3833 :
3834 : impl ThreadLocalMetrics {
3835 953 : pub fn new(slots_submission_queue_depth: LocalHistogram) -> Self {
3836 953 : ThreadLocalMetrics {
3837 953 : slots_submission_queue_depth: Mutex::new(slots_submission_queue_depth),
3838 953 : }
3839 953 : }
3840 :
3841 : /// Flushes the thread local metrics to shared aggregator.
3842 0 : pub fn flush(&self) {
3843 0 : let Self {
3844 0 : slots_submission_queue_depth,
3845 0 : } = self;
3846 0 : slots_submission_queue_depth.lock().unwrap().flush();
3847 0 : }
3848 : }
3849 :
3850 : impl tokio_epoll_uring::metrics::PerSystemMetrics for ThreadLocalMetrics {
3851 834872 : fn observe_slots_submission_queue_depth(&self, queue_depth: u64) {
3852 834872 : let Self {
3853 834872 : slots_submission_queue_depth,
3854 834872 : } = self;
3855 834872 : slots_submission_queue_depth
3856 834872 : .lock()
3857 834872 : .unwrap()
3858 834872 : .observe(queue_depth as f64);
3859 834872 : }
3860 : }
3861 :
3862 : pub struct Collector {
3863 : descs: Vec<metrics::core::Desc>,
3864 : systems_created: UIntGauge,
3865 : systems_destroyed: UIntGauge,
3866 : thread_local_metrics_storage: &'static ThreadLocalMetricsStorage,
3867 : }
3868 :
3869 : impl metrics::core::Collector for Collector {
3870 0 : fn desc(&self) -> Vec<&metrics::core::Desc> {
3871 0 : self.descs.iter().collect()
3872 0 : }
3873 :
3874 0 : fn collect(&self) -> Vec<metrics::proto::MetricFamily> {
3875 0 : let mut mfs = Vec::with_capacity(Self::NMETRICS);
3876 0 : let tokio_epoll_uring::metrics::GlobalMetrics {
3877 0 : systems_created,
3878 0 : systems_destroyed,
3879 0 : } = tokio_epoll_uring::metrics::global();
3880 0 : self.systems_created.set(systems_created);
3881 0 : mfs.extend(self.systems_created.collect());
3882 0 : self.systems_destroyed.set(systems_destroyed);
3883 0 : mfs.extend(self.systems_destroyed.collect());
3884 0 :
3885 0 : self.thread_local_metrics_storage
3886 0 : .flush_thread_local_metrics();
3887 0 :
3888 0 : mfs.extend(
3889 0 : self.thread_local_metrics_storage
3890 0 : .slots_submission_queue_depth
3891 0 : .collect(),
3892 0 : );
3893 0 : mfs
3894 0 : }
3895 : }
3896 :
3897 : impl Collector {
3898 : const NMETRICS: usize = 3;
3899 :
3900 : #[allow(clippy::new_without_default)]
3901 0 : pub fn new() -> Self {
3902 0 : let mut descs = Vec::new();
3903 0 :
3904 0 : let systems_created = UIntGauge::new(
3905 0 : "pageserver_tokio_epoll_uring_systems_created",
3906 0 : "counter of tokio-epoll-uring systems that were created",
3907 0 : )
3908 0 : .unwrap();
3909 0 : descs.extend(
3910 0 : metrics::core::Collector::desc(&systems_created)
3911 0 : .into_iter()
3912 0 : .cloned(),
3913 0 : );
3914 0 :
3915 0 : let systems_destroyed = UIntGauge::new(
3916 0 : "pageserver_tokio_epoll_uring_systems_destroyed",
3917 0 : "counter of tokio-epoll-uring systems that were destroyed",
3918 0 : )
3919 0 : .unwrap();
3920 0 : descs.extend(
3921 0 : metrics::core::Collector::desc(&systems_destroyed)
3922 0 : .into_iter()
3923 0 : .cloned(),
3924 0 : );
3925 0 :
3926 0 : Self {
3927 0 : descs,
3928 0 : systems_created,
3929 0 : systems_destroyed,
3930 0 : thread_local_metrics_storage: &THREAD_LOCAL_METRICS_STORAGE,
3931 0 : }
3932 0 : }
3933 : }
3934 :
3935 242 : pub(crate) static THREAD_LOCAL_LAUNCH_SUCCESSES: Lazy<metrics::IntCounter> = Lazy::new(|| {
3936 242 : register_int_counter!(
3937 242 : "pageserver_tokio_epoll_uring_pageserver_thread_local_launch_success_count",
3938 242 : "Number of times where thread_local_system creation spanned multiple executor threads",
3939 242 : )
3940 242 : .unwrap()
3941 242 : });
3942 :
3943 0 : pub(crate) static THREAD_LOCAL_LAUNCH_FAILURES: Lazy<metrics::IntCounter> = Lazy::new(|| {
3944 0 : register_int_counter!(
3945 0 : "pageserver_tokio_epoll_uring_pageserver_thread_local_launch_failures_count",
3946 0 : "Number of times thread_local_system creation failed and was retried after back-off.",
3947 0 : )
3948 0 : .unwrap()
3949 0 : });
3950 : }
3951 :
3952 : pub(crate) struct GlobalAndPerTenantIntCounter {
3953 : global: IntCounter,
3954 : per_tenant: IntCounter,
3955 : }
3956 :
3957 : impl GlobalAndPerTenantIntCounter {
3958 : #[inline(always)]
3959 0 : pub(crate) fn inc(&self) {
3960 0 : self.inc_by(1)
3961 0 : }
3962 : #[inline(always)]
3963 451799 : pub(crate) fn inc_by(&self, n: u64) {
3964 451799 : self.global.inc_by(n);
3965 451799 : self.per_tenant.inc_by(n);
3966 451799 : }
3967 : }
3968 :
3969 : pub(crate) mod tenant_throttling {
3970 : use metrics::register_int_counter_vec;
3971 : use once_cell::sync::Lazy;
3972 : use utils::shard::TenantShardId;
3973 :
3974 : use super::GlobalAndPerTenantIntCounter;
3975 :
3976 : pub(crate) struct Metrics<const KIND: usize> {
3977 : pub(super) count_accounted_start: GlobalAndPerTenantIntCounter,
3978 : pub(super) count_accounted_finish: GlobalAndPerTenantIntCounter,
3979 : pub(super) wait_time: GlobalAndPerTenantIntCounter,
3980 : pub(super) count_throttled: GlobalAndPerTenantIntCounter,
3981 : }
3982 :
3983 424 : static COUNT_ACCOUNTED_START: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
3984 424 : register_int_counter_vec!(
3985 424 : "pageserver_tenant_throttling_count_accounted_start_global",
3986 424 : "Count of tenant throttling starts, by kind of throttle.",
3987 424 : &["kind"]
3988 424 : )
3989 424 : .unwrap()
3990 424 : });
3991 424 : static COUNT_ACCOUNTED_START_PER_TENANT: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
3992 424 : register_int_counter_vec!(
3993 424 : "pageserver_tenant_throttling_count_accounted_start",
3994 424 : "Count of tenant throttling starts, by kind of throttle.",
3995 424 : &["kind", "tenant_id", "shard_id"]
3996 424 : )
3997 424 : .unwrap()
3998 424 : });
3999 424 : static COUNT_ACCOUNTED_FINISH: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
4000 424 : register_int_counter_vec!(
4001 424 : "pageserver_tenant_throttling_count_accounted_finish_global",
4002 424 : "Count of tenant throttling finishes, by kind of throttle.",
4003 424 : &["kind"]
4004 424 : )
4005 424 : .unwrap()
4006 424 : });
4007 424 : static COUNT_ACCOUNTED_FINISH_PER_TENANT: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
4008 424 : register_int_counter_vec!(
4009 424 : "pageserver_tenant_throttling_count_accounted_finish",
4010 424 : "Count of tenant throttling finishes, by kind of throttle.",
4011 424 : &["kind", "tenant_id", "shard_id"]
4012 424 : )
4013 424 : .unwrap()
4014 424 : });
4015 424 : static WAIT_USECS: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
4016 424 : register_int_counter_vec!(
4017 424 : "pageserver_tenant_throttling_wait_usecs_sum_global",
4018 424 : "Sum of microseconds that spent waiting throttle by kind of throttle.",
4019 424 : &["kind"]
4020 424 : )
4021 424 : .unwrap()
4022 424 : });
4023 424 : static WAIT_USECS_PER_TENANT: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
4024 424 : register_int_counter_vec!(
4025 424 : "pageserver_tenant_throttling_wait_usecs_sum",
4026 424 : "Sum of microseconds that spent waiting throttle by kind of throttle.",
4027 424 : &["kind", "tenant_id", "shard_id"]
4028 424 : )
4029 424 : .unwrap()
4030 424 : });
4031 :
4032 424 : static WAIT_COUNT: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
4033 424 : register_int_counter_vec!(
4034 424 : "pageserver_tenant_throttling_count_global",
4035 424 : "Count of tenant throttlings, by kind of throttle.",
4036 424 : &["kind"]
4037 424 : )
4038 424 : .unwrap()
4039 424 : });
4040 424 : static WAIT_COUNT_PER_TENANT: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
4041 424 : register_int_counter_vec!(
4042 424 : "pageserver_tenant_throttling_count",
4043 424 : "Count of tenant throttlings, by kind of throttle.",
4044 424 : &["kind", "tenant_id", "shard_id"]
4045 424 : )
4046 424 : .unwrap()
4047 424 : });
4048 :
4049 : const KINDS: &[&str] = &["pagestream"];
4050 : pub type Pagestream = Metrics<0>;
4051 :
4052 : impl<const KIND: usize> Metrics<KIND> {
4053 460 : pub(crate) fn new(tenant_shard_id: &TenantShardId) -> Self {
4054 460 : let per_tenant_label_values = &[
4055 460 : KINDS[KIND],
4056 460 : &tenant_shard_id.tenant_id.to_string(),
4057 460 : &tenant_shard_id.shard_slug().to_string(),
4058 460 : ];
4059 460 : Metrics {
4060 460 : count_accounted_start: {
4061 460 : GlobalAndPerTenantIntCounter {
4062 460 : global: COUNT_ACCOUNTED_START.with_label_values(&[KINDS[KIND]]),
4063 460 : per_tenant: COUNT_ACCOUNTED_START_PER_TENANT
4064 460 : .with_label_values(per_tenant_label_values),
4065 460 : }
4066 460 : },
4067 460 : count_accounted_finish: {
4068 460 : GlobalAndPerTenantIntCounter {
4069 460 : global: COUNT_ACCOUNTED_FINISH.with_label_values(&[KINDS[KIND]]),
4070 460 : per_tenant: COUNT_ACCOUNTED_FINISH_PER_TENANT
4071 460 : .with_label_values(per_tenant_label_values),
4072 460 : }
4073 460 : },
4074 460 : wait_time: {
4075 460 : GlobalAndPerTenantIntCounter {
4076 460 : global: WAIT_USECS.with_label_values(&[KINDS[KIND]]),
4077 460 : per_tenant: WAIT_USECS_PER_TENANT
4078 460 : .with_label_values(per_tenant_label_values),
4079 460 : }
4080 460 : },
4081 460 : count_throttled: {
4082 460 : GlobalAndPerTenantIntCounter {
4083 460 : global: WAIT_COUNT.with_label_values(&[KINDS[KIND]]),
4084 460 : per_tenant: WAIT_COUNT_PER_TENANT
4085 460 : .with_label_values(per_tenant_label_values),
4086 460 : }
4087 460 : },
4088 460 : }
4089 460 : }
4090 : }
4091 :
4092 0 : pub(crate) fn preinitialize_global_metrics() {
4093 0 : Lazy::force(&COUNT_ACCOUNTED_START);
4094 0 : Lazy::force(&COUNT_ACCOUNTED_FINISH);
4095 0 : Lazy::force(&WAIT_USECS);
4096 0 : Lazy::force(&WAIT_COUNT);
4097 0 : }
4098 :
4099 12 : pub(crate) fn remove_tenant_metrics(tenant_shard_id: &TenantShardId) {
4100 48 : for m in &[
4101 12 : &COUNT_ACCOUNTED_START_PER_TENANT,
4102 12 : &COUNT_ACCOUNTED_FINISH_PER_TENANT,
4103 12 : &WAIT_USECS_PER_TENANT,
4104 12 : &WAIT_COUNT_PER_TENANT,
4105 12 : ] {
4106 96 : for kind in KINDS {
4107 48 : let _ = m.remove_label_values(&[
4108 48 : kind,
4109 48 : &tenant_shard_id.tenant_id.to_string(),
4110 48 : &tenant_shard_id.shard_slug().to_string(),
4111 48 : ]);
4112 48 : }
4113 : }
4114 12 : }
4115 : }
4116 :
4117 : pub(crate) mod disk_usage_based_eviction {
4118 : use super::*;
4119 :
4120 : pub(crate) struct Metrics {
4121 : pub(crate) tenant_collection_time: Histogram,
4122 : pub(crate) tenant_layer_count: Histogram,
4123 : pub(crate) layers_collected: IntCounter,
4124 : pub(crate) layers_selected: IntCounter,
4125 : pub(crate) layers_evicted: IntCounter,
4126 : }
4127 :
4128 : impl Default for Metrics {
4129 0 : fn default() -> Self {
4130 0 : let tenant_collection_time = register_histogram!(
4131 0 : "pageserver_disk_usage_based_eviction_tenant_collection_seconds",
4132 0 : "Time spent collecting layers from a tenant -- not normalized by collected layer amount",
4133 0 : vec![0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0, 10.0]
4134 0 : )
4135 0 : .unwrap();
4136 0 :
4137 0 : let tenant_layer_count = register_histogram!(
4138 0 : "pageserver_disk_usage_based_eviction_tenant_collected_layers",
4139 0 : "Amount of layers gathered from a tenant",
4140 0 : vec![5.0, 50.0, 500.0, 5000.0, 50000.0]
4141 0 : )
4142 0 : .unwrap();
4143 0 :
4144 0 : let layers_collected = register_int_counter!(
4145 0 : "pageserver_disk_usage_based_eviction_collected_layers_total",
4146 0 : "Amount of layers collected"
4147 0 : )
4148 0 : .unwrap();
4149 0 :
4150 0 : let layers_selected = register_int_counter!(
4151 0 : "pageserver_disk_usage_based_eviction_select_layers_total",
4152 0 : "Amount of layers selected"
4153 0 : )
4154 0 : .unwrap();
4155 0 :
4156 0 : let layers_evicted = register_int_counter!(
4157 0 : "pageserver_disk_usage_based_eviction_evicted_layers_total",
4158 0 : "Amount of layers successfully evicted"
4159 0 : )
4160 0 : .unwrap();
4161 0 :
4162 0 : Self {
4163 0 : tenant_collection_time,
4164 0 : tenant_layer_count,
4165 0 : layers_collected,
4166 0 : layers_selected,
4167 0 : layers_evicted,
4168 0 : }
4169 0 : }
4170 : }
4171 :
4172 : pub(crate) static METRICS: Lazy<Metrics> = Lazy::new(Metrics::default);
4173 : }
4174 :
4175 412 : static TOKIO_EXECUTOR_THREAD_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
4176 412 : register_uint_gauge_vec!(
4177 412 : "pageserver_tokio_executor_thread_configured_count",
4178 412 : "Total number of configued tokio executor threads in the process.
4179 412 : The `setup` label denotes whether we're running with multiple runtimes or a single runtime.",
4180 412 : &["setup"],
4181 412 : )
4182 412 : .unwrap()
4183 412 : });
4184 :
4185 412 : pub(crate) fn set_tokio_runtime_setup(setup: &str, num_threads: NonZeroUsize) {
4186 : static SERIALIZE: std::sync::Mutex<()> = std::sync::Mutex::new(());
4187 412 : let _guard = SERIALIZE.lock().unwrap();
4188 412 : TOKIO_EXECUTOR_THREAD_COUNT.reset();
4189 412 : TOKIO_EXECUTOR_THREAD_COUNT
4190 412 : .get_metric_with_label_values(&[setup])
4191 412 : .unwrap()
4192 412 : .set(u64::try_from(num_threads.get()).unwrap());
4193 412 : }
4194 :
4195 0 : static PAGESERVER_CONFIG_IGNORED_ITEMS: Lazy<UIntGaugeVec> = Lazy::new(|| {
4196 0 : register_uint_gauge_vec!(
4197 0 : "pageserver_config_ignored_items",
4198 0 : "TOML items present in the on-disk configuration file but ignored by the pageserver config parser.\
4199 0 : The `item` label is the dot-separated path of the ignored item in the on-disk configuration file.\
4200 0 : The value for an unknown config item is always 1.\
4201 0 : There is a special label value \"\", which is 0, so that there is always a metric exposed (simplifies dashboards).",
4202 0 : &["item"]
4203 0 : )
4204 0 : .unwrap()
4205 0 : });
4206 :
4207 0 : pub fn preinitialize_metrics(
4208 0 : conf: &'static PageServerConf,
4209 0 : ignored: config::ignored_fields::Paths,
4210 0 : ) {
4211 0 : set_page_service_config_max_batch_size(&conf.page_service_pipelining);
4212 0 :
4213 0 : PAGESERVER_CONFIG_IGNORED_ITEMS
4214 0 : .with_label_values(&[""])
4215 0 : .set(0);
4216 0 : for path in &ignored.paths {
4217 0 : PAGESERVER_CONFIG_IGNORED_ITEMS
4218 0 : .with_label_values(&[path])
4219 0 : .set(1);
4220 0 : }
4221 :
4222 : // Python tests need these and on some we do alerting.
4223 : //
4224 : // FIXME(4813): make it so that we have no top level metrics as this fn will easily fall out of
4225 : // order:
4226 : // - global metrics reside in a Lazy<PageserverMetrics>
4227 : // - access via crate::metrics::PS_METRICS.some_metric.inc()
4228 : // - could move the statics into TimelineMetrics::new()?
4229 :
4230 : // counters
4231 0 : [
4232 0 : &UNEXPECTED_ONDEMAND_DOWNLOADS,
4233 0 : &WALRECEIVER_STARTED_CONNECTIONS,
4234 0 : &WALRECEIVER_BROKER_UPDATES,
4235 0 : &WALRECEIVER_CANDIDATES_ADDED,
4236 0 : &WALRECEIVER_CANDIDATES_REMOVED,
4237 0 : &tokio_epoll_uring::THREAD_LOCAL_LAUNCH_FAILURES,
4238 0 : &tokio_epoll_uring::THREAD_LOCAL_LAUNCH_SUCCESSES,
4239 0 : &REMOTE_ONDEMAND_DOWNLOADED_LAYERS,
4240 0 : &REMOTE_ONDEMAND_DOWNLOADED_BYTES,
4241 0 : &CIRCUIT_BREAKERS_BROKEN,
4242 0 : &CIRCUIT_BREAKERS_UNBROKEN,
4243 0 : &PAGE_SERVICE_SMGR_FLUSH_INPROGRESS_MICROS_GLOBAL,
4244 0 : &WAIT_LSN_IN_PROGRESS_GLOBAL_MICROS,
4245 0 : ]
4246 0 : .into_iter()
4247 0 : .for_each(|c| {
4248 0 : Lazy::force(c);
4249 0 : });
4250 0 :
4251 0 : // Deletion queue stats
4252 0 : Lazy::force(&DELETION_QUEUE);
4253 0 :
4254 0 : // Tenant stats
4255 0 : Lazy::force(&TENANT);
4256 0 :
4257 0 : // Tenant manager stats
4258 0 : Lazy::force(&TENANT_MANAGER);
4259 0 :
4260 0 : Lazy::force(&crate::tenant::storage_layer::layer::LAYER_IMPL_METRICS);
4261 0 : Lazy::force(&disk_usage_based_eviction::METRICS);
4262 :
4263 0 : for state_name in pageserver_api::models::TenantState::VARIANTS {
4264 0 : // initialize the metric for all gauges, otherwise the time series might seemingly show
4265 0 : // values from last restart.
4266 0 : TENANT_STATE_METRIC.with_label_values(&[state_name]).set(0);
4267 0 : }
4268 :
4269 : // countervecs
4270 0 : [
4271 0 : &BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT,
4272 0 : &SMGR_QUERY_STARTED_GLOBAL,
4273 0 : ]
4274 0 : .into_iter()
4275 0 : .for_each(|c| {
4276 0 : Lazy::force(c);
4277 0 : });
4278 0 :
4279 0 : // gauges
4280 0 : WALRECEIVER_ACTIVE_MANAGERS.get();
4281 0 :
4282 0 : // histograms
4283 0 : [
4284 0 : &LAYERS_PER_READ_GLOBAL,
4285 0 : &LAYERS_PER_READ_BATCH_GLOBAL,
4286 0 : &LAYERS_PER_READ_AMORTIZED_GLOBAL,
4287 0 : &DELTAS_PER_READ_GLOBAL,
4288 0 : &WAIT_LSN_TIME,
4289 0 : &WAL_REDO_TIME,
4290 0 : &WAL_REDO_RECORDS_HISTOGRAM,
4291 0 : &WAL_REDO_BYTES_HISTOGRAM,
4292 0 : &WAL_REDO_PROCESS_LAUNCH_DURATION_HISTOGRAM,
4293 0 : &PAGE_SERVICE_BATCH_SIZE_GLOBAL,
4294 0 : &PAGE_SERVICE_SMGR_BATCH_WAIT_TIME_GLOBAL,
4295 0 : ]
4296 0 : .into_iter()
4297 0 : .for_each(|h| {
4298 0 : Lazy::force(h);
4299 0 : });
4300 0 :
4301 0 : // Custom
4302 0 : Lazy::force(&BASEBACKUP_QUERY_TIME);
4303 0 : Lazy::force(&COMPUTE_COMMANDS_COUNTERS);
4304 0 : Lazy::force(&tokio_epoll_uring::THREAD_LOCAL_METRICS_STORAGE);
4305 0 :
4306 0 : tenant_throttling::preinitialize_global_metrics();
4307 0 : wait_ondemand_download_time::preinitialize_global_metrics();
4308 0 : }
|
