Line data Source code
1 : use enum_map::EnumMap;
2 : use metrics::metric_vec_duration::DurationResultObserver;
3 : use metrics::{
4 : register_counter_vec, register_gauge_vec, register_histogram, register_histogram_vec,
5 : register_int_counter, register_int_counter_pair_vec, register_int_counter_vec,
6 : register_int_gauge, register_int_gauge_vec, register_uint_gauge, register_uint_gauge_vec,
7 : Counter, CounterVec, GaugeVec, Histogram, HistogramVec, IntCounter, IntCounterPairVec,
8 : IntCounterVec, IntGauge, IntGaugeVec, UIntGauge, UIntGaugeVec,
9 : };
10 : use once_cell::sync::Lazy;
11 : use pageserver_api::shard::TenantShardId;
12 : use strum::{EnumCount, IntoEnumIterator, VariantNames};
13 : use strum_macros::{EnumVariantNames, IntoStaticStr};
14 : use utils::id::TimelineId;
15 :
16 : /// Prometheus histogram buckets (in seconds) for operations in the critical
17 : /// path. In other words, operations that directly affect that latency of user
18 : /// queries.
19 : ///
20 : /// The buckets capture the majority of latencies in the microsecond and
21 : /// millisecond range but also extend far enough up to distinguish "bad" from
22 : /// "really bad".
23 : const CRITICAL_OP_BUCKETS: &[f64] = &[
24 : 0.000_001, 0.000_010, 0.000_100, // 1 us, 10 us, 100 us
25 : 0.001_000, 0.010_000, 0.100_000, // 1 ms, 10 ms, 100 ms
26 : 1.0, 10.0, 100.0, // 1 s, 10 s, 100 s
27 : ];
28 :
29 : // Metrics collected on operations on the storage repository.
30 11265 : #[derive(Debug, EnumVariantNames, IntoStaticStr)]
31 : #[strum(serialize_all = "kebab_case")]
32 : pub(crate) enum StorageTimeOperation {
33 : #[strum(serialize = "layer flush")]
34 : LayerFlush,
35 :
36 : #[strum(serialize = "compact")]
37 : Compact,
38 :
39 : #[strum(serialize = "create images")]
40 : CreateImages,
41 :
42 : #[strum(serialize = "logical size")]
43 : LogicalSize,
44 :
45 : #[strum(serialize = "imitate logical size")]
46 : ImitateLogicalSize,
47 :
48 : #[strum(serialize = "load layer map")]
49 : LoadLayerMap,
50 :
51 : #[strum(serialize = "gc")]
52 : Gc,
53 :
54 : #[strum(serialize = "create tenant")]
55 : CreateTenant,
56 : }
57 :
58 656 : pub(crate) static STORAGE_TIME_SUM_PER_TIMELINE: Lazy<CounterVec> = Lazy::new(|| {
59 656 : register_counter_vec!(
60 656 : "pageserver_storage_operations_seconds_sum",
61 656 : "Total time spent on storage operations with operation, tenant and timeline dimensions",
62 656 : &["operation", "tenant_id", "shard_id", "timeline_id"],
63 656 : )
64 656 : .expect("failed to define a metric")
65 656 : });
66 :
67 656 : pub(crate) static STORAGE_TIME_COUNT_PER_TIMELINE: Lazy<IntCounterVec> = Lazy::new(|| {
68 656 : register_int_counter_vec!(
69 656 : "pageserver_storage_operations_seconds_count",
70 656 : "Count of storage operations with operation, tenant and timeline dimensions",
71 656 : &["operation", "tenant_id", "shard_id", "timeline_id"],
72 656 : )
73 656 : .expect("failed to define a metric")
74 656 : });
75 :
76 : // Buckets for background operations like compaction, GC, size calculation
77 : const STORAGE_OP_BUCKETS: &[f64] = &[0.010, 0.100, 1.0, 10.0, 100.0, 1000.0];
78 :
79 657 : pub(crate) static STORAGE_TIME_GLOBAL: Lazy<HistogramVec> = Lazy::new(|| {
80 657 : register_histogram_vec!(
81 657 : "pageserver_storage_operations_seconds_global",
82 657 : "Time spent on storage operations",
83 657 : &["operation"],
84 657 : STORAGE_OP_BUCKETS.into(),
85 657 : )
86 657 : .expect("failed to define a metric")
87 657 : });
88 :
89 699 : pub(crate) static READ_NUM_FS_LAYERS: Lazy<Histogram> = Lazy::new(|| {
90 699 : register_histogram!(
91 699 : "pageserver_read_num_fs_layers",
92 699 : "Number of persistent layers accessed for processing a read request, including those in the cache",
93 699 : vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 10.0, 20.0, 50.0, 100.0],
94 699 : )
95 699 : .expect("failed to define a metric")
96 699 : });
97 :
98 : // Metrics collected on operations on the storage repository.
99 :
100 : pub(crate) struct ReconstructTimeMetrics {
101 : ok: Histogram,
102 : err: Histogram,
103 : }
104 :
105 699 : pub(crate) static RECONSTRUCT_TIME: Lazy<ReconstructTimeMetrics> = Lazy::new(|| {
106 699 : let inner = register_histogram_vec!(
107 699 : "pageserver_getpage_reconstruct_seconds",
108 699 : "Time spent in reconstruct_value (reconstruct a page from deltas)",
109 699 : &["result"],
110 699 : CRITICAL_OP_BUCKETS.into(),
111 699 : )
112 699 : .expect("failed to define a metric");
113 699 : ReconstructTimeMetrics {
114 699 : ok: inner.get_metric_with_label_values(&["ok"]).unwrap(),
115 699 : err: inner.get_metric_with_label_values(&["err"]).unwrap(),
116 699 : }
117 699 : });
118 :
119 : impl ReconstructTimeMetrics {
120 7320961 : pub(crate) fn for_result<T, E>(&self, result: &Result<T, E>) -> &Histogram {
121 7320961 : match result {
122 7320961 : Ok(_) => &self.ok,
123 0 : Err(_) => &self.err,
124 : }
125 7320961 : }
126 : }
127 :
128 625 : pub(crate) static MATERIALIZED_PAGE_CACHE_HIT_DIRECT: Lazy<IntCounter> = Lazy::new(|| {
129 625 : register_int_counter!(
130 625 : "pageserver_materialized_cache_hits_direct_total",
131 625 : "Number of cache hits from materialized page cache without redo",
132 625 : )
133 625 : .expect("failed to define a metric")
134 625 : });
135 :
136 641 : pub(crate) static GET_RECONSTRUCT_DATA_TIME: Lazy<Histogram> = Lazy::new(|| {
137 641 : register_histogram!(
138 641 : "pageserver_getpage_get_reconstruct_data_seconds",
139 641 : "Time spent in get_reconstruct_value_data",
140 641 : CRITICAL_OP_BUCKETS.into(),
141 641 : )
142 641 : .expect("failed to define a metric")
143 641 : });
144 :
145 625 : pub(crate) static MATERIALIZED_PAGE_CACHE_HIT: Lazy<IntCounter> = Lazy::new(|| {
146 625 : register_int_counter!(
147 625 : "pageserver_materialized_cache_hits_total",
148 625 : "Number of cache hits from materialized page cache",
149 625 : )
150 625 : .expect("failed to define a metric")
151 625 : });
152 :
153 : pub(crate) struct GetVectoredLatency {
154 : map: EnumMap<TaskKind, Option<Histogram>>,
155 : }
156 :
157 : impl GetVectoredLatency {
158 : // Only these task types perform vectored gets. Filter all other tasks out to reduce total
159 : // cardinality of the metric.
160 : const TRACKED_TASK_KINDS: [TaskKind; 2] = [TaskKind::Compaction, TaskKind::PageRequestHandler];
161 :
162 78555 : pub(crate) fn for_task_kind(&self, task_kind: TaskKind) -> Option<&Histogram> {
163 78555 : self.map[task_kind].as_ref()
164 78555 : }
165 : }
166 :
167 431 : pub(crate) static GET_VECTORED_LATENCY: Lazy<GetVectoredLatency> = Lazy::new(|| {
168 431 : let inner = register_histogram_vec!(
169 431 : "pageserver_get_vectored_seconds",
170 431 : "Time spent in get_vectored",
171 431 : &["task_kind"],
172 431 : CRITICAL_OP_BUCKETS.into(),
173 431 : )
174 431 : .expect("failed to define a metric");
175 431 :
176 431 : GetVectoredLatency {
177 11276 : map: EnumMap::from_array(std::array::from_fn(|task_kind_idx| {
178 11276 : let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind_idx);
179 11276 :
180 11276 : if GetVectoredLatency::TRACKED_TASK_KINDS.contains(&task_kind) {
181 862 : let task_kind = task_kind.into();
182 862 : Some(inner.with_label_values(&[task_kind]))
183 : } else {
184 10414 : None
185 : }
186 11276 : })),
187 431 : }
188 431 : });
189 :
190 : pub(crate) struct PageCacheMetricsForTaskKind {
191 : pub read_accesses_materialized_page: IntCounter,
192 : pub read_accesses_immutable: IntCounter,
193 :
194 : pub read_hits_immutable: IntCounter,
195 : pub read_hits_materialized_page_exact: IntCounter,
196 : pub read_hits_materialized_page_older_lsn: IntCounter,
197 : }
198 :
199 : pub(crate) struct PageCacheMetrics {
200 : map: EnumMap<TaskKind, EnumMap<PageContentKind, PageCacheMetricsForTaskKind>>,
201 : }
202 :
203 643 : static PAGE_CACHE_READ_HITS: Lazy<IntCounterVec> = Lazy::new(|| {
204 643 : register_int_counter_vec!(
205 643 : "pageserver_page_cache_read_hits_total",
206 643 : "Number of read accesses to the page cache that hit",
207 643 : &["task_kind", "key_kind", "content_kind", "hit_kind"]
208 643 : )
209 643 : .expect("failed to define a metric")
210 643 : });
211 :
212 643 : static PAGE_CACHE_READ_ACCESSES: Lazy<IntCounterVec> = Lazy::new(|| {
213 643 : register_int_counter_vec!(
214 643 : "pageserver_page_cache_read_accesses_total",
215 643 : "Number of read accesses to the page cache",
216 643 : &["task_kind", "key_kind", "content_kind"]
217 643 : )
218 643 : .expect("failed to define a metric")
219 643 : });
220 :
221 643 : pub(crate) static PAGE_CACHE: Lazy<PageCacheMetrics> = Lazy::new(|| PageCacheMetrics {
222 16794 : map: EnumMap::from_array(std::array::from_fn(|task_kind| {
223 16794 : let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind);
224 16794 : let task_kind: &'static str = task_kind.into();
225 100764 : EnumMap::from_array(std::array::from_fn(|content_kind| {
226 100764 : let content_kind = <PageContentKind as enum_map::Enum>::from_usize(content_kind);
227 100764 : let content_kind: &'static str = content_kind.into();
228 100764 : PageCacheMetricsForTaskKind {
229 100764 : read_accesses_materialized_page: {
230 100764 : PAGE_CACHE_READ_ACCESSES
231 100764 : .get_metric_with_label_values(&[
232 100764 : task_kind,
233 100764 : "materialized_page",
234 100764 : content_kind,
235 100764 : ])
236 100764 : .unwrap()
237 100764 : },
238 100764 :
239 100764 : read_accesses_immutable: {
240 100764 : PAGE_CACHE_READ_ACCESSES
241 100764 : .get_metric_with_label_values(&[task_kind, "immutable", content_kind])
242 100764 : .unwrap()
243 100764 : },
244 100764 :
245 100764 : read_hits_immutable: {
246 100764 : PAGE_CACHE_READ_HITS
247 100764 : .get_metric_with_label_values(&[task_kind, "immutable", content_kind, "-"])
248 100764 : .unwrap()
249 100764 : },
250 100764 :
251 100764 : read_hits_materialized_page_exact: {
252 100764 : PAGE_CACHE_READ_HITS
253 100764 : .get_metric_with_label_values(&[
254 100764 : task_kind,
255 100764 : "materialized_page",
256 100764 : content_kind,
257 100764 : "exact",
258 100764 : ])
259 100764 : .unwrap()
260 100764 : },
261 100764 :
262 100764 : read_hits_materialized_page_older_lsn: {
263 100764 : PAGE_CACHE_READ_HITS
264 100764 : .get_metric_with_label_values(&[
265 100764 : task_kind,
266 100764 : "materialized_page",
267 100764 : content_kind,
268 100764 : "older_lsn",
269 100764 : ])
270 100764 : .unwrap()
271 100764 : },
272 100764 : }
273 100764 : }))
274 16794 : })),
275 643 : });
276 :
277 : impl PageCacheMetrics {
278 350200020 : pub(crate) fn for_ctx(&self, ctx: &RequestContext) -> &PageCacheMetricsForTaskKind {
279 350200020 : &self.map[ctx.task_kind()][ctx.page_content_kind()]
280 350200020 : }
281 : }
282 :
283 : pub(crate) struct PageCacheSizeMetrics {
284 : pub max_bytes: UIntGauge,
285 :
286 : pub current_bytes_immutable: UIntGauge,
287 : pub current_bytes_materialized_page: UIntGauge,
288 : }
289 :
290 701 : static PAGE_CACHE_SIZE_CURRENT_BYTES: Lazy<UIntGaugeVec> = Lazy::new(|| {
291 701 : register_uint_gauge_vec!(
292 701 : "pageserver_page_cache_size_current_bytes",
293 701 : "Current size of the page cache in bytes, by key kind",
294 701 : &["key_kind"]
295 701 : )
296 701 : .expect("failed to define a metric")
297 701 : });
298 :
299 : pub(crate) static PAGE_CACHE_SIZE: Lazy<PageCacheSizeMetrics> =
300 701 : Lazy::new(|| PageCacheSizeMetrics {
301 701 : max_bytes: {
302 701 : register_uint_gauge!(
303 701 : "pageserver_page_cache_size_max_bytes",
304 701 : "Maximum size of the page cache in bytes"
305 701 : )
306 701 : .expect("failed to define a metric")
307 701 : },
308 701 : current_bytes_immutable: {
309 701 : PAGE_CACHE_SIZE_CURRENT_BYTES
310 701 : .get_metric_with_label_values(&["immutable"])
311 701 : .unwrap()
312 701 : },
313 701 : current_bytes_materialized_page: {
314 701 : PAGE_CACHE_SIZE_CURRENT_BYTES
315 701 : .get_metric_with_label_values(&["materialized_page"])
316 701 : .unwrap()
317 701 : },
318 701 : });
319 :
320 : pub(crate) mod page_cache_eviction_metrics {
321 : use std::num::NonZeroUsize;
322 :
323 : use metrics::{register_int_counter_vec, IntCounter, IntCounterVec};
324 : use once_cell::sync::Lazy;
325 :
326 0 : #[derive(Clone, Copy)]
327 : pub(crate) enum Outcome {
328 : FoundSlotUnused { iters: NonZeroUsize },
329 : FoundSlotEvicted { iters: NonZeroUsize },
330 : ItersExceeded { iters: NonZeroUsize },
331 : }
332 :
333 595 : static ITERS_TOTAL_VEC: Lazy<IntCounterVec> = Lazy::new(|| {
334 595 : register_int_counter_vec!(
335 595 : "pageserver_page_cache_find_victim_iters_total",
336 595 : "Counter for the number of iterations in the find_victim loop",
337 595 : &["outcome"],
338 595 : )
339 595 : .expect("failed to define a metric")
340 595 : });
341 :
342 595 : static CALLS_VEC: Lazy<IntCounterVec> = Lazy::new(|| {
343 595 : register_int_counter_vec!(
344 595 : "pageserver_page_cache_find_victim_calls",
345 595 : "Incremented at the end of each find_victim() call.\
346 595 : Filter by outcome to get e.g., eviction rate.",
347 595 : &["outcome"]
348 595 : )
349 595 : .unwrap()
350 595 : });
351 :
352 10938947 : pub(crate) fn observe(outcome: Outcome) {
353 10938947 : macro_rules! dry {
354 10938947 : ($label:literal, $iters:expr) => {{
355 10938947 : static LABEL: &'static str = $label;
356 10938947 : static ITERS_TOTAL: Lazy<IntCounter> =
357 10938947 : Lazy::new(|| ITERS_TOTAL_VEC.with_label_values(&[LABEL]));
358 10938947 : static CALLS: Lazy<IntCounter> =
359 10938947 : Lazy::new(|| CALLS_VEC.with_label_values(&[LABEL]));
360 10938947 : ITERS_TOTAL.inc_by(($iters.get()) as u64);
361 10938947 : CALLS.inc();
362 10938947 : }};
363 10938947 : }
364 10938947 : match outcome {
365 2882857 : Outcome::FoundSlotUnused { iters } => dry!("found_empty", iters),
366 8056090 : Outcome::FoundSlotEvicted { iters } => {
367 8056090 : dry!("found_evicted", iters)
368 : }
369 0 : Outcome::ItersExceeded { iters } => {
370 0 : dry!("err_iters_exceeded", iters);
371 0 : super::page_cache_errors_inc(super::PageCacheErrorKind::EvictIterLimit);
372 0 : }
373 : }
374 10938947 : }
375 : }
376 :
377 0 : static PAGE_CACHE_ERRORS: Lazy<IntCounterVec> = Lazy::new(|| {
378 0 : register_int_counter_vec!(
379 0 : "page_cache_errors_total",
380 0 : "Number of timeouts while acquiring a pinned slot in the page cache",
381 0 : &["error_kind"]
382 0 : )
383 0 : .expect("failed to define a metric")
384 0 : });
385 :
386 0 : #[derive(IntoStaticStr)]
387 : #[strum(serialize_all = "kebab_case")]
388 : pub(crate) enum PageCacheErrorKind {
389 : AcquirePinnedSlotTimeout,
390 : EvictIterLimit,
391 : }
392 :
393 0 : pub(crate) fn page_cache_errors_inc(error_kind: PageCacheErrorKind) {
394 0 : PAGE_CACHE_ERRORS
395 0 : .get_metric_with_label_values(&[error_kind.into()])
396 0 : .unwrap()
397 0 : .inc();
398 0 : }
399 :
400 635 : pub(crate) static WAIT_LSN_TIME: Lazy<Histogram> = Lazy::new(|| {
401 635 : register_histogram!(
402 635 : "pageserver_wait_lsn_seconds",
403 635 : "Time spent waiting for WAL to arrive",
404 635 : CRITICAL_OP_BUCKETS.into(),
405 635 : )
406 635 : .expect("failed to define a metric")
407 635 : });
408 :
409 656 : static LAST_RECORD_LSN: Lazy<IntGaugeVec> = Lazy::new(|| {
410 656 : register_int_gauge_vec!(
411 656 : "pageserver_last_record_lsn",
412 656 : "Last record LSN grouped by timeline",
413 656 : &["tenant_id", "shard_id", "timeline_id"]
414 656 : )
415 656 : .expect("failed to define a metric")
416 656 : });
417 :
418 656 : static RESIDENT_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
419 656 : register_uint_gauge_vec!(
420 656 : "pageserver_resident_physical_size",
421 656 : "The size of the layer files present in the pageserver's filesystem.",
422 656 : &["tenant_id", "shard_id", "timeline_id"]
423 656 : )
424 656 : .expect("failed to define a metric")
425 656 : });
426 :
427 654 : pub(crate) static RESIDENT_PHYSICAL_SIZE_GLOBAL: Lazy<UIntGauge> = Lazy::new(|| {
428 654 : register_uint_gauge!(
429 654 : "pageserver_resident_physical_size_global",
430 654 : "Like `pageserver_resident_physical_size`, but without tenant/timeline dimensions."
431 654 : )
432 654 : .expect("failed to define a metric")
433 654 : });
434 :
435 656 : static REMOTE_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
436 656 : register_uint_gauge_vec!(
437 656 : "pageserver_remote_physical_size",
438 656 : "The size of the layer files present in the remote storage that are listed in the the remote index_part.json.",
439 656 : // Corollary: If any files are missing from the index part, they won't be included here.
440 656 : &["tenant_id", "shard_id", "timeline_id"]
441 656 : )
442 656 : .expect("failed to define a metric")
443 656 : });
444 :
445 656 : static REMOTE_PHYSICAL_SIZE_GLOBAL: Lazy<UIntGauge> = Lazy::new(|| {
446 656 : register_uint_gauge!(
447 656 : "pageserver_remote_physical_size_global",
448 656 : "Like `pageserver_remote_physical_size`, but without tenant/timeline dimensions."
449 656 : )
450 656 : .expect("failed to define a metric")
451 656 : });
452 :
453 86 : pub(crate) static REMOTE_ONDEMAND_DOWNLOADED_LAYERS: Lazy<IntCounter> = Lazy::new(|| {
454 86 : register_int_counter!(
455 86 : "pageserver_remote_ondemand_downloaded_layers_total",
456 86 : "Total on-demand downloaded layers"
457 86 : )
458 86 : .unwrap()
459 86 : });
460 :
461 86 : pub(crate) static REMOTE_ONDEMAND_DOWNLOADED_BYTES: Lazy<IntCounter> = Lazy::new(|| {
462 86 : register_int_counter!(
463 86 : "pageserver_remote_ondemand_downloaded_bytes_total",
464 86 : "Total bytes of layers on-demand downloaded",
465 86 : )
466 86 : .unwrap()
467 86 : });
468 :
469 656 : static CURRENT_LOGICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
470 656 : register_uint_gauge_vec!(
471 656 : "pageserver_current_logical_size",
472 656 : "Current logical size grouped by timeline",
473 656 : &["tenant_id", "shard_id", "timeline_id"]
474 656 : )
475 656 : .expect("failed to define current logical size metric")
476 656 : });
477 :
478 : pub(crate) mod initial_logical_size {
479 : use metrics::{register_int_counter, register_int_counter_vec, IntCounter, IntCounterVec};
480 : use once_cell::sync::Lazy;
481 :
482 : pub(crate) struct StartCalculation(IntCounterVec);
483 637 : pub(crate) static START_CALCULATION: Lazy<StartCalculation> = Lazy::new(|| {
484 637 : StartCalculation(
485 637 : register_int_counter_vec!(
486 637 : "pageserver_initial_logical_size_start_calculation",
487 637 : "Incremented each time we start an initial logical size calculation attempt. \
488 637 : The `circumstances` label provides some additional details.",
489 637 : &["attempt", "circumstances"]
490 637 : )
491 637 : .unwrap(),
492 637 : )
493 637 : });
494 :
495 : struct DropCalculation {
496 : first: IntCounter,
497 : retry: IntCounter,
498 : }
499 :
500 637 : static DROP_CALCULATION: Lazy<DropCalculation> = Lazy::new(|| {
501 637 : let vec = register_int_counter_vec!(
502 637 : "pageserver_initial_logical_size_drop_calculation",
503 637 : "Incremented each time we abort a started size calculation attmpt.",
504 637 : &["attempt"]
505 637 : )
506 637 : .unwrap();
507 637 : DropCalculation {
508 637 : first: vec.with_label_values(&["first"]),
509 637 : retry: vec.with_label_values(&["retry"]),
510 637 : }
511 637 : });
512 :
513 : pub(crate) struct Calculated {
514 : pub(crate) births: IntCounter,
515 : pub(crate) deaths: IntCounter,
516 : }
517 :
518 630 : pub(crate) static CALCULATED: Lazy<Calculated> = Lazy::new(|| Calculated {
519 630 : births: register_int_counter!(
520 630 : "pageserver_initial_logical_size_finish_calculation",
521 630 : "Incremented every time we finish calculation of initial logical size.\
522 630 : If everything is working well, this should happen at most once per Timeline object."
523 630 : )
524 630 : .unwrap(),
525 630 : deaths: register_int_counter!(
526 630 : "pageserver_initial_logical_size_drop_finished_calculation",
527 630 : "Incremented when we drop a finished initial logical size calculation result.\
528 630 : Mainly useful to turn pageserver_initial_logical_size_finish_calculation into a gauge."
529 630 : )
530 630 : .unwrap(),
531 630 : });
532 :
533 : pub(crate) struct OngoingCalculationGuard {
534 : inc_drop_calculation: Option<IntCounter>,
535 : }
536 :
537 1302 : #[derive(strum_macros::IntoStaticStr)]
538 : pub(crate) enum StartCircumstances {
539 : EmptyInitial,
540 : SkippedConcurrencyLimiter,
541 : AfterBackgroundTasksRateLimit,
542 : }
543 :
544 : impl StartCalculation {
545 1302 : pub(crate) fn first(&self, circumstances: StartCircumstances) -> OngoingCalculationGuard {
546 1302 : let circumstances_label: &'static str = circumstances.into();
547 1302 : self.0
548 1302 : .with_label_values(&["first", circumstances_label])
549 1302 : .inc();
550 1302 : OngoingCalculationGuard {
551 1302 : inc_drop_calculation: Some(DROP_CALCULATION.first.clone()),
552 1302 : }
553 1302 : }
554 0 : pub(crate) fn retry(&self, circumstances: StartCircumstances) -> OngoingCalculationGuard {
555 0 : let circumstances_label: &'static str = circumstances.into();
556 0 : self.0
557 0 : .with_label_values(&["retry", circumstances_label])
558 0 : .inc();
559 0 : OngoingCalculationGuard {
560 0 : inc_drop_calculation: Some(DROP_CALCULATION.retry.clone()),
561 0 : }
562 0 : }
563 : }
564 :
565 : impl Drop for OngoingCalculationGuard {
566 1296 : fn drop(&mut self) {
567 1296 : if let Some(counter) = self.inc_drop_calculation.take() {
568 34 : counter.inc();
569 1262 : }
570 1296 : }
571 : }
572 :
573 : impl OngoingCalculationGuard {
574 1262 : pub(crate) fn calculation_result_saved(mut self) -> FinishedCalculationGuard {
575 1262 : drop(self.inc_drop_calculation.take());
576 1262 : CALCULATED.births.inc();
577 1262 : FinishedCalculationGuard {
578 1262 : inc_on_drop: CALCULATED.deaths.clone(),
579 1262 : }
580 1262 : }
581 : }
582 :
583 : pub(crate) struct FinishedCalculationGuard {
584 : inc_on_drop: IntCounter,
585 : }
586 :
587 : impl Drop for FinishedCalculationGuard {
588 272 : fn drop(&mut self) {
589 272 : self.inc_on_drop.inc();
590 272 : }
591 : }
592 :
593 : // context: https://github.com/neondatabase/neon/issues/5963
594 : pub(crate) static TIMELINES_WHERE_WALRECEIVER_GOT_APPROXIMATE_SIZE: Lazy<IntCounter> =
595 38 : Lazy::new(|| {
596 38 : register_int_counter!(
597 38 : "pageserver_initial_logical_size_timelines_where_walreceiver_got_approximate_size",
598 38 : "Counter for the following event: walreceiver calls\
599 38 : Timeline::get_current_logical_size() and it returns `Approximate` for the first time."
600 38 : )
601 38 : .unwrap()
602 38 : });
603 : }
604 :
605 0 : static DIRECTORY_ENTRIES_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
606 0 : register_uint_gauge_vec!(
607 0 : "pageserver_directory_entries_count",
608 0 : "Sum of the entries in pageserver-stored directory listings",
609 0 : &["tenant_id", "shard_id", "timeline_id"]
610 0 : )
611 0 : .expect("failed to define a metric")
612 0 : });
613 :
614 678 : pub(crate) static TENANT_STATE_METRIC: Lazy<UIntGaugeVec> = Lazy::new(|| {
615 678 : register_uint_gauge_vec!(
616 678 : "pageserver_tenant_states_count",
617 678 : "Count of tenants per state",
618 678 : &["state"]
619 678 : )
620 678 : .expect("Failed to register pageserver_tenant_states_count metric")
621 678 : });
622 :
623 : /// A set of broken tenants.
624 : ///
625 : /// These are expected to be so rare that a set is fine. Set as in a new timeseries per each broken
626 : /// tenant.
627 185 : pub(crate) static BROKEN_TENANTS_SET: Lazy<UIntGaugeVec> = Lazy::new(|| {
628 185 : register_uint_gauge_vec!(
629 185 : "pageserver_broken_tenants_count",
630 185 : "Set of broken tenants",
631 185 : &["tenant_id", "shard_id"]
632 185 : )
633 185 : .expect("Failed to register pageserver_tenant_states_count metric")
634 185 : });
635 :
636 222 : pub(crate) static TENANT_SYNTHETIC_SIZE_METRIC: Lazy<UIntGaugeVec> = Lazy::new(|| {
637 222 : register_uint_gauge_vec!(
638 222 : "pageserver_tenant_synthetic_cached_size_bytes",
639 222 : "Synthetic size of each tenant in bytes",
640 222 : &["tenant_id"]
641 222 : )
642 222 : .expect("Failed to register pageserver_tenant_synthetic_cached_size_bytes metric")
643 222 : });
644 :
645 : // Metrics for cloud upload. These metrics reflect data uploaded to cloud storage,
646 : // or in testing they estimate how much we would upload if we did.
647 656 : static NUM_PERSISTENT_FILES_CREATED: Lazy<IntCounterVec> = Lazy::new(|| {
648 656 : register_int_counter_vec!(
649 656 : "pageserver_created_persistent_files_total",
650 656 : "Number of files created that are meant to be uploaded to cloud storage",
651 656 : &["tenant_id", "shard_id", "timeline_id"]
652 656 : )
653 656 : .expect("failed to define a metric")
654 656 : });
655 :
656 656 : static PERSISTENT_BYTES_WRITTEN: Lazy<IntCounterVec> = Lazy::new(|| {
657 656 : register_int_counter_vec!(
658 656 : "pageserver_written_persistent_bytes_total",
659 656 : "Total bytes written that are meant to be uploaded to cloud storage",
660 656 : &["tenant_id", "shard_id", "timeline_id"]
661 656 : )
662 656 : .expect("failed to define a metric")
663 656 : });
664 :
665 1 : pub(crate) static EVICTION_ITERATION_DURATION: Lazy<HistogramVec> = Lazy::new(|| {
666 1 : register_histogram_vec!(
667 1 : "pageserver_eviction_iteration_duration_seconds_global",
668 1 : "Time spent on a single eviction iteration",
669 1 : &["period_secs", "threshold_secs"],
670 1 : STORAGE_OP_BUCKETS.into(),
671 1 : )
672 1 : .expect("failed to define a metric")
673 1 : });
674 :
675 656 : static EVICTIONS: Lazy<IntCounterVec> = Lazy::new(|| {
676 656 : register_int_counter_vec!(
677 656 : "pageserver_evictions",
678 656 : "Number of layers evicted from the pageserver",
679 656 : &["tenant_id", "shard_id", "timeline_id"]
680 656 : )
681 656 : .expect("failed to define a metric")
682 656 : });
683 :
684 656 : static EVICTIONS_WITH_LOW_RESIDENCE_DURATION: Lazy<IntCounterVec> = Lazy::new(|| {
685 656 : register_int_counter_vec!(
686 656 : "pageserver_evictions_with_low_residence_duration",
687 656 : "If a layer is evicted that was resident for less than `low_threshold`, it is counted to this counter. \
688 656 : Residence duration is determined using the `residence_duration_data_source`.",
689 656 : &["tenant_id", "shard_id", "timeline_id", "residence_duration_data_source", "low_threshold_secs"]
690 656 : )
691 656 : .expect("failed to define a metric")
692 656 : });
693 :
694 625 : pub(crate) static UNEXPECTED_ONDEMAND_DOWNLOADS: Lazy<IntCounter> = Lazy::new(|| {
695 625 : register_int_counter!(
696 625 : "pageserver_unexpected_ondemand_downloads_count",
697 625 : "Number of unexpected on-demand downloads. \
698 625 : We log more context for each increment, so, forgo any labels in this metric.",
699 625 : )
700 625 : .expect("failed to define a metric")
701 625 : });
702 :
703 : /// How long did we take to start up? Broken down by labels to describe
704 : /// different phases of startup.
705 625 : pub static STARTUP_DURATION: Lazy<GaugeVec> = Lazy::new(|| {
706 625 : register_gauge_vec!(
707 625 : "pageserver_startup_duration_seconds",
708 625 : "Time taken by phases of pageserver startup, in seconds",
709 625 : &["phase"]
710 625 : )
711 625 : .expect("Failed to register pageserver_startup_duration_seconds metric")
712 625 : });
713 :
714 625 : pub static STARTUP_IS_LOADING: Lazy<UIntGauge> = Lazy::new(|| {
715 625 : register_uint_gauge!(
716 625 : "pageserver_startup_is_loading",
717 625 : "1 while in initial startup load of tenants, 0 at other times"
718 625 : )
719 625 : .expect("Failed to register pageserver_startup_is_loading")
720 625 : });
721 :
722 : /// Metrics related to the lifecycle of a [`crate::tenant::Tenant`] object: things
723 : /// like how long it took to load.
724 : ///
725 : /// Note that these are process-global metrics, _not_ per-tenant metrics. Per-tenant
726 : /// metrics are rather expensive, and usually fine grained stuff makes more sense
727 : /// at a timeline level than tenant level.
728 : pub(crate) struct TenantMetrics {
729 : /// How long did tenants take to go from construction to active state?
730 : pub(crate) activation: Histogram,
731 : pub(crate) preload: Histogram,
732 : pub(crate) attach: Histogram,
733 :
734 : /// How many tenants are included in the initial startup of the pagesrever?
735 : pub(crate) startup_scheduled: IntCounter,
736 : pub(crate) startup_complete: IntCounter,
737 : }
738 :
739 625 : pub(crate) static TENANT: Lazy<TenantMetrics> = Lazy::new(|| {
740 625 : TenantMetrics {
741 625 : activation: register_histogram!(
742 625 : "pageserver_tenant_activation_seconds",
743 625 : "Time taken by tenants to activate, in seconds",
744 625 : CRITICAL_OP_BUCKETS.into()
745 625 : )
746 625 : .expect("Failed to register metric"),
747 625 : preload: register_histogram!(
748 625 : "pageserver_tenant_preload_seconds",
749 625 : "Time taken by tenants to load remote metadata on startup/attach, in seconds",
750 625 : CRITICAL_OP_BUCKETS.into()
751 625 : )
752 625 : .expect("Failed to register metric"),
753 625 : attach: register_histogram!(
754 625 : "pageserver_tenant_attach_seconds",
755 625 : "Time taken by tenants to intialize, after remote metadata is already loaded",
756 625 : CRITICAL_OP_BUCKETS.into()
757 625 : )
758 625 : .expect("Failed to register metric"),
759 625 : startup_scheduled: register_int_counter!(
760 625 : "pageserver_tenant_startup_scheduled",
761 625 : "Number of tenants included in pageserver startup (doesn't count tenants attached later)"
762 625 : ).expect("Failed to register metric"),
763 625 : startup_complete: register_int_counter!(
764 625 : "pageserver_tenant_startup_complete",
765 625 : "Number of tenants that have completed warm-up, or activated on-demand during initial startup: \
766 625 : should eventually reach `pageserver_tenant_startup_scheduled_total`. Does not include broken \
767 625 : tenants: such cases will lead to this metric never reaching the scheduled count."
768 625 : ).expect("Failed to register metric"),
769 625 : }
770 625 : });
771 :
772 : /// Each `Timeline`'s [`EVICTIONS_WITH_LOW_RESIDENCE_DURATION`] metric.
773 0 : #[derive(Debug)]
774 : pub(crate) struct EvictionsWithLowResidenceDuration {
775 : data_source: &'static str,
776 : threshold: Duration,
777 : counter: Option<IntCounter>,
778 : }
779 :
780 : pub(crate) struct EvictionsWithLowResidenceDurationBuilder {
781 : data_source: &'static str,
782 : threshold: Duration,
783 : }
784 :
785 : impl EvictionsWithLowResidenceDurationBuilder {
786 1600 : pub fn new(data_source: &'static str, threshold: Duration) -> Self {
787 1600 : Self {
788 1600 : data_source,
789 1600 : threshold,
790 1600 : }
791 1600 : }
792 :
793 1600 : fn build(
794 1600 : &self,
795 1600 : tenant_id: &str,
796 1600 : shard_id: &str,
797 1600 : timeline_id: &str,
798 1600 : ) -> EvictionsWithLowResidenceDuration {
799 1600 : let counter = EVICTIONS_WITH_LOW_RESIDENCE_DURATION
800 1600 : .get_metric_with_label_values(&[
801 1600 : tenant_id,
802 1600 : shard_id,
803 1600 : timeline_id,
804 1600 : self.data_source,
805 1600 : &EvictionsWithLowResidenceDuration::threshold_label_value(self.threshold),
806 1600 : ])
807 1600 : .unwrap();
808 1600 : EvictionsWithLowResidenceDuration {
809 1600 : data_source: self.data_source,
810 1600 : threshold: self.threshold,
811 1600 : counter: Some(counter),
812 1600 : }
813 1600 : }
814 : }
815 :
816 : impl EvictionsWithLowResidenceDuration {
817 1954 : fn threshold_label_value(threshold: Duration) -> String {
818 1954 : format!("{}", threshold.as_secs())
819 1954 : }
820 :
821 2515 : pub fn observe(&self, observed_value: Duration) {
822 2515 : if observed_value < self.threshold {
823 2515 : self.counter
824 2515 : .as_ref()
825 2515 : .expect("nobody calls this function after `remove_from_vec`")
826 2515 : .inc();
827 2515 : }
828 2515 : }
829 :
830 62 : pub fn change_threshold(
831 62 : &mut self,
832 62 : tenant_id: &str,
833 62 : shard_id: &str,
834 62 : timeline_id: &str,
835 62 : new_threshold: Duration,
836 62 : ) {
837 62 : if new_threshold == self.threshold {
838 54 : return;
839 8 : }
840 8 : let mut with_new = EvictionsWithLowResidenceDurationBuilder::new(
841 8 : self.data_source,
842 8 : new_threshold,
843 8 : )
844 8 : .build(tenant_id, shard_id, timeline_id);
845 8 : std::mem::swap(self, &mut with_new);
846 8 : with_new.remove(tenant_id, shard_id, timeline_id);
847 62 : }
848 :
849 : // This could be a `Drop` impl, but, we need the `tenant_id` and `timeline_id`.
850 354 : fn remove(&mut self, tenant_id: &str, shard_id: &str, timeline_id: &str) {
851 354 : let Some(_counter) = self.counter.take() else {
852 0 : return;
853 : };
854 :
855 354 : let threshold = Self::threshold_label_value(self.threshold);
856 354 :
857 354 : let removed = EVICTIONS_WITH_LOW_RESIDENCE_DURATION.remove_label_values(&[
858 354 : tenant_id,
859 354 : shard_id,
860 354 : timeline_id,
861 354 : self.data_source,
862 354 : &threshold,
863 354 : ]);
864 354 :
865 354 : match removed {
866 0 : Err(e) => {
867 0 : // this has been hit in staging as
868 0 : // <https://neondatabase.sentry.io/issues/4142396994/>, but we don't know how.
869 0 : // because we can be in the drop path already, don't risk:
870 0 : // - "double-panic => illegal instruction" or
871 0 : // - future "drop panick => abort"
872 0 : //
873 0 : // so just nag: (the error has the labels)
874 0 : tracing::warn!("failed to remove EvictionsWithLowResidenceDuration, it was already removed? {e:#?}");
875 : }
876 : Ok(()) => {
877 : // to help identify cases where we double-remove the same values, let's log all
878 : // deletions?
879 354 : tracing::info!("removed EvictionsWithLowResidenceDuration with {tenant_id}, {timeline_id}, {}, {threshold}", self.data_source);
880 : }
881 : }
882 354 : }
883 : }
884 :
885 : // Metrics collected on disk IO operations
886 : //
887 : // Roughly logarithmic scale.
888 : const STORAGE_IO_TIME_BUCKETS: &[f64] = &[
889 : 0.000030, // 30 usec
890 : 0.001000, // 1000 usec
891 : 0.030, // 30 ms
892 : 1.000, // 1000 ms
893 : 30.000, // 30000 ms
894 : ];
895 :
896 : /// VirtualFile fs operation variants.
897 : ///
898 : /// Operations:
899 : /// - open ([`std::fs::OpenOptions::open`])
900 : /// - close (dropping [`crate::virtual_file::VirtualFile`])
901 : /// - close-by-replace (close by replacement algorithm)
902 : /// - read (`read_at`)
903 : /// - write (`write_at`)
904 : /// - seek (modify internal position or file length query)
905 : /// - fsync ([`std::fs::File::sync_all`])
906 : /// - metadata ([`std::fs::File::metadata`])
907 : #[derive(
908 5967 : Debug, Clone, Copy, strum_macros::EnumCount, strum_macros::EnumIter, strum_macros::FromRepr,
909 : )]
910 : pub(crate) enum StorageIoOperation {
911 : Open,
912 : OpenAfterReplace,
913 : Close,
914 : CloseByReplace,
915 : Read,
916 : Write,
917 : Seek,
918 : Fsync,
919 : Metadata,
920 : }
921 :
922 : impl StorageIoOperation {
923 5967 : pub fn as_str(&self) -> &'static str {
924 5967 : match self {
925 663 : StorageIoOperation::Open => "open",
926 663 : StorageIoOperation::OpenAfterReplace => "open-after-replace",
927 663 : StorageIoOperation::Close => "close",
928 663 : StorageIoOperation::CloseByReplace => "close-by-replace",
929 663 : StorageIoOperation::Read => "read",
930 663 : StorageIoOperation::Write => "write",
931 663 : StorageIoOperation::Seek => "seek",
932 663 : StorageIoOperation::Fsync => "fsync",
933 663 : StorageIoOperation::Metadata => "metadata",
934 : }
935 5967 : }
936 : }
937 :
938 : /// Tracks time taken by fs operations near VirtualFile.
939 0 : #[derive(Debug)]
940 : pub(crate) struct StorageIoTime {
941 : metrics: [Histogram; StorageIoOperation::COUNT],
942 : }
943 :
944 : impl StorageIoTime {
945 663 : fn new() -> Self {
946 663 : let storage_io_histogram_vec = register_histogram_vec!(
947 663 : "pageserver_io_operations_seconds",
948 663 : "Time spent in IO operations",
949 663 : &["operation"],
950 663 : STORAGE_IO_TIME_BUCKETS.into()
951 663 : )
952 663 : .expect("failed to define a metric");
953 5967 : let metrics = std::array::from_fn(|i| {
954 5967 : let op = StorageIoOperation::from_repr(i).unwrap();
955 5967 : storage_io_histogram_vec
956 5967 : .get_metric_with_label_values(&[op.as_str()])
957 5967 : .unwrap()
958 5967 : });
959 663 : Self { metrics }
960 663 : }
961 :
962 14367855 : pub(crate) fn get(&self, op: StorageIoOperation) -> &Histogram {
963 14367855 : &self.metrics[op as usize]
964 14367855 : }
965 : }
966 :
967 : pub(crate) static STORAGE_IO_TIME_METRIC: Lazy<StorageIoTime> = Lazy::new(StorageIoTime::new);
968 :
969 : const STORAGE_IO_SIZE_OPERATIONS: &[&str] = &["read", "write"];
970 :
971 : // Needed for the https://neonprod.grafana.net/d/5uK9tHL4k/picking-tenant-for-relocation?orgId=1
972 676 : pub(crate) static STORAGE_IO_SIZE: Lazy<IntGaugeVec> = Lazy::new(|| {
973 676 : register_int_gauge_vec!(
974 676 : "pageserver_io_operations_bytes_total",
975 676 : "Total amount of bytes read/written in IO operations",
976 676 : &["operation", "tenant_id", "shard_id", "timeline_id"]
977 676 : )
978 676 : .expect("failed to define a metric")
979 676 : });
980 :
981 : #[cfg(not(test))]
982 : pub(crate) mod virtual_file_descriptor_cache {
983 : use super::*;
984 :
985 625 : pub(crate) static SIZE_MAX: Lazy<UIntGauge> = Lazy::new(|| {
986 625 : register_uint_gauge!(
987 625 : "pageserver_virtual_file_descriptor_cache_size_max",
988 625 : "Maximum number of open file descriptors in the cache."
989 625 : )
990 625 : .unwrap()
991 625 : });
992 :
993 : // SIZE_CURRENT: derive it like so:
994 : // ```
995 : // sum (pageserver_io_operations_seconds_count{operation=~"^(open|open-after-replace)$")
996 : // -ignoring(operation)
997 : // sum(pageserver_io_operations_seconds_count{operation=~"^(close|close-by-replace)$"}
998 : // ```
999 : }
1000 :
1001 : #[cfg(not(test))]
1002 : pub(crate) mod virtual_file_io_engine {
1003 : use super::*;
1004 :
1005 625 : pub(crate) static KIND: Lazy<UIntGaugeVec> = Lazy::new(|| {
1006 625 : register_uint_gauge_vec!(
1007 625 : "pageserver_virtual_file_io_engine_kind",
1008 625 : "The configured io engine for VirtualFile",
1009 625 : &["kind"],
1010 625 : )
1011 625 : .unwrap()
1012 625 : });
1013 : }
1014 :
1015 0 : #[derive(Debug)]
1016 : struct GlobalAndPerTimelineHistogram {
1017 : global: Histogram,
1018 : per_tenant_timeline: Histogram,
1019 : }
1020 :
1021 : impl GlobalAndPerTimelineHistogram {
1022 4467215 : fn observe(&self, value: f64) {
1023 4467215 : self.global.observe(value);
1024 4467215 : self.per_tenant_timeline.observe(value);
1025 4467215 : }
1026 : }
1027 :
1028 : struct GlobalAndPerTimelineHistogramTimer<'a> {
1029 : h: &'a GlobalAndPerTimelineHistogram,
1030 : start: std::time::Instant,
1031 : }
1032 :
1033 : impl<'a> Drop for GlobalAndPerTimelineHistogramTimer<'a> {
1034 4467215 : fn drop(&mut self) {
1035 4467215 : let elapsed = self.start.elapsed();
1036 4467215 : self.h.observe(elapsed.as_secs_f64());
1037 4467215 : }
1038 : }
1039 :
1040 : #[derive(
1041 0 : Debug,
1042 0 : Clone,
1043 : Copy,
1044 17760 : IntoStaticStr,
1045 : strum_macros::EnumCount,
1046 3830 : strum_macros::EnumIter,
1047 8010 : strum_macros::FromRepr,
1048 : )]
1049 : #[strum(serialize_all = "snake_case")]
1050 : pub enum SmgrQueryType {
1051 : GetRelExists,
1052 : GetRelSize,
1053 : GetPageAtLsn,
1054 : GetDbSize,
1055 : GetSlruSegment,
1056 : }
1057 :
1058 0 : #[derive(Debug)]
1059 : pub(crate) struct SmgrQueryTimePerTimeline {
1060 : metrics: [GlobalAndPerTimelineHistogram; SmgrQueryType::COUNT],
1061 : }
1062 :
1063 658 : static SMGR_QUERY_TIME_PER_TENANT_TIMELINE: Lazy<HistogramVec> = Lazy::new(|| {
1064 658 : register_histogram_vec!(
1065 658 : "pageserver_smgr_query_seconds",
1066 658 : "Time spent on smgr query handling, aggegated by query type and tenant/timeline.",
1067 658 : &["smgr_query_type", "tenant_id", "shard_id", "timeline_id"],
1068 658 : CRITICAL_OP_BUCKETS.into(),
1069 658 : )
1070 658 : .expect("failed to define a metric")
1071 658 : });
1072 :
1073 658 : static SMGR_QUERY_TIME_GLOBAL_BUCKETS: Lazy<Vec<f64>> = Lazy::new(|| {
1074 658 : [
1075 658 : 1,
1076 658 : 10,
1077 658 : 20,
1078 658 : 40,
1079 658 : 60,
1080 658 : 80,
1081 658 : 100,
1082 658 : 200,
1083 658 : 300,
1084 658 : 400,
1085 658 : 500,
1086 658 : 600,
1087 658 : 700,
1088 658 : 800,
1089 658 : 900,
1090 658 : 1_000, // 1ms
1091 658 : 2_000,
1092 658 : 4_000,
1093 658 : 6_000,
1094 658 : 8_000,
1095 658 : 10_000, // 10ms
1096 658 : 20_000,
1097 658 : 40_000,
1098 658 : 60_000,
1099 658 : 80_000,
1100 658 : 100_000,
1101 658 : 200_000,
1102 658 : 400_000,
1103 658 : 600_000,
1104 658 : 800_000,
1105 658 : 1_000_000, // 1s
1106 658 : 2_000_000,
1107 658 : 4_000_000,
1108 658 : 6_000_000,
1109 658 : 8_000_000,
1110 658 : 10_000_000, // 10s
1111 658 : 20_000_000,
1112 658 : 50_000_000,
1113 658 : 100_000_000,
1114 658 : 200_000_000,
1115 658 : 1_000_000_000, // 1000s
1116 658 : ]
1117 658 : .into_iter()
1118 658 : .map(Duration::from_micros)
1119 26978 : .map(|d| d.as_secs_f64())
1120 658 : .collect()
1121 658 : });
1122 :
1123 658 : static SMGR_QUERY_TIME_GLOBAL: Lazy<HistogramVec> = Lazy::new(|| {
1124 658 : register_histogram_vec!(
1125 658 : "pageserver_smgr_query_seconds_global",
1126 658 : "Time spent on smgr query handling, aggregated by query type.",
1127 658 : &["smgr_query_type"],
1128 658 : SMGR_QUERY_TIME_GLOBAL_BUCKETS.clone(),
1129 658 : )
1130 658 : .expect("failed to define a metric")
1131 658 : });
1132 :
1133 : impl SmgrQueryTimePerTimeline {
1134 1602 : pub(crate) fn new(tenant_shard_id: &TenantShardId, timeline_id: &TimelineId) -> Self {
1135 1602 : let tenant_id = tenant_shard_id.tenant_id.to_string();
1136 1602 : let shard_slug = format!("{}", tenant_shard_id.shard_slug());
1137 1602 : let timeline_id = timeline_id.to_string();
1138 8010 : let metrics = std::array::from_fn(|i| {
1139 8010 : let op = SmgrQueryType::from_repr(i).unwrap();
1140 8010 : let global = SMGR_QUERY_TIME_GLOBAL
1141 8010 : .get_metric_with_label_values(&[op.into()])
1142 8010 : .unwrap();
1143 8010 : let per_tenant_timeline = SMGR_QUERY_TIME_PER_TENANT_TIMELINE
1144 8010 : .get_metric_with_label_values(&[op.into(), &tenant_id, &shard_slug, &timeline_id])
1145 8010 : .unwrap();
1146 8010 : GlobalAndPerTimelineHistogram {
1147 8010 : global,
1148 8010 : per_tenant_timeline,
1149 8010 : }
1150 8010 : });
1151 1602 : Self { metrics }
1152 1602 : }
1153 4467216 : pub(crate) fn start_timer(&self, op: SmgrQueryType) -> impl Drop + '_ {
1154 4467216 : let metric = &self.metrics[op as usize];
1155 4467216 : GlobalAndPerTimelineHistogramTimer {
1156 4467216 : h: metric,
1157 4467216 : start: std::time::Instant::now(),
1158 4467216 : }
1159 4467216 : }
1160 : }
1161 :
1162 : #[cfg(test)]
1163 : mod smgr_query_time_tests {
1164 : use pageserver_api::shard::TenantShardId;
1165 : use strum::IntoEnumIterator;
1166 : use utils::id::{TenantId, TimelineId};
1167 :
1168 : // Regression test, we used hard-coded string constants before using an enum.
1169 2 : #[test]
1170 2 : fn op_label_name() {
1171 2 : use super::SmgrQueryType::*;
1172 2 : let expect: [(super::SmgrQueryType, &'static str); 5] = [
1173 2 : (GetRelExists, "get_rel_exists"),
1174 2 : (GetRelSize, "get_rel_size"),
1175 2 : (GetPageAtLsn, "get_page_at_lsn"),
1176 2 : (GetDbSize, "get_db_size"),
1177 2 : (GetSlruSegment, "get_slru_segment"),
1178 2 : ];
1179 12 : for (op, expect) in expect {
1180 10 : let actual: &'static str = op.into();
1181 10 : assert_eq!(actual, expect);
1182 : }
1183 2 : }
1184 :
1185 2 : #[test]
1186 2 : fn basic() {
1187 2 : let ops: Vec<_> = super::SmgrQueryType::iter().collect();
1188 :
1189 12 : for op in &ops {
1190 10 : let tenant_id = TenantId::generate();
1191 10 : let timeline_id = TimelineId::generate();
1192 10 : let metrics = super::SmgrQueryTimePerTimeline::new(
1193 10 : &TenantShardId::unsharded(tenant_id),
1194 10 : &timeline_id,
1195 10 : );
1196 10 :
1197 20 : let get_counts = || {
1198 20 : let global: u64 = ops
1199 20 : .iter()
1200 100 : .map(|op| metrics.metrics[*op as usize].global.get_sample_count())
1201 20 : .sum();
1202 20 : let per_tenant_timeline: u64 = ops
1203 20 : .iter()
1204 100 : .map(|op| {
1205 100 : metrics.metrics[*op as usize]
1206 100 : .per_tenant_timeline
1207 100 : .get_sample_count()
1208 100 : })
1209 20 : .sum();
1210 20 : (global, per_tenant_timeline)
1211 20 : };
1212 :
1213 10 : let (pre_global, pre_per_tenant_timeline) = get_counts();
1214 10 : assert_eq!(pre_per_tenant_timeline, 0);
1215 :
1216 10 : let timer = metrics.start_timer(*op);
1217 10 : drop(timer);
1218 10 :
1219 10 : let (post_global, post_per_tenant_timeline) = get_counts();
1220 10 : assert_eq!(post_per_tenant_timeline, 1);
1221 10 : assert!(post_global > pre_global);
1222 : }
1223 2 : }
1224 : }
1225 :
1226 : // keep in sync with control plane Go code so that we can validate
1227 : // compute's basebackup_ms metric with our perspective in the context of SLI/SLO.
1228 358 : static COMPUTE_STARTUP_BUCKETS: Lazy<[f64; 28]> = Lazy::new(|| {
1229 358 : // Go code uses milliseconds. Variable is called `computeStartupBuckets`
1230 358 : [
1231 358 : 5, 10, 20, 30, 50, 70, 100, 120, 150, 200, 250, 300, 350, 400, 450, 500, 600, 800, 1000,
1232 358 : 1500, 2000, 2500, 3000, 5000, 10000, 20000, 40000, 60000,
1233 358 : ]
1234 10024 : .map(|ms| (ms as f64) / 1000.0)
1235 358 : });
1236 :
1237 : pub(crate) struct BasebackupQueryTime(HistogramVec);
1238 358 : pub(crate) static BASEBACKUP_QUERY_TIME: Lazy<BasebackupQueryTime> = Lazy::new(|| {
1239 358 : BasebackupQueryTime({
1240 358 : register_histogram_vec!(
1241 358 : "pageserver_basebackup_query_seconds",
1242 358 : "Histogram of basebackup queries durations, by result type",
1243 358 : &["result"],
1244 358 : COMPUTE_STARTUP_BUCKETS.to_vec(),
1245 358 : )
1246 358 : .expect("failed to define a metric")
1247 358 : })
1248 358 : });
1249 :
1250 : impl DurationResultObserver for BasebackupQueryTime {
1251 628 : fn observe_result<T, E>(&self, res: &Result<T, E>, duration: std::time::Duration) {
1252 628 : let label_value = if res.is_ok() { "ok" } else { "error" };
1253 628 : let metric = self.0.get_metric_with_label_values(&[label_value]).unwrap();
1254 628 : metric.observe(duration.as_secs_f64());
1255 628 : }
1256 : }
1257 :
1258 457 : pub(crate) static LIVE_CONNECTIONS_COUNT: Lazy<IntGaugeVec> = Lazy::new(|| {
1259 457 : register_int_gauge_vec!(
1260 457 : "pageserver_live_connections",
1261 457 : "Number of live network connections",
1262 457 : &["pageserver_connection_kind"]
1263 457 : )
1264 457 : .expect("failed to define a metric")
1265 457 : });
1266 :
1267 : // remote storage metrics
1268 :
1269 : /// NB: increment _after_ recording the current value into [`REMOTE_TIMELINE_CLIENT_CALLS_STARTED_HIST`].
1270 651 : static REMOTE_TIMELINE_CLIENT_CALLS_UNFINISHED_GAUGE: Lazy<IntGaugeVec> = Lazy::new(|| {
1271 651 : register_int_gauge_vec!(
1272 651 : "pageserver_remote_timeline_client_calls_unfinished",
1273 651 : "Number of ongoing calls to remote timeline client. \
1274 651 : Used to populate pageserver_remote_timeline_client_calls_started. \
1275 651 : This metric is not useful for sampling from Prometheus, but useful in tests.",
1276 651 : &[
1277 651 : "tenant_id",
1278 651 : "shard_id",
1279 651 : "timeline_id",
1280 651 : "file_kind",
1281 651 : "op_kind"
1282 651 : ],
1283 651 : )
1284 651 : .expect("failed to define a metric")
1285 651 : });
1286 :
1287 651 : static REMOTE_TIMELINE_CLIENT_CALLS_STARTED_HIST: Lazy<HistogramVec> = Lazy::new(|| {
1288 651 : register_histogram_vec!(
1289 651 : "pageserver_remote_timeline_client_calls_started",
1290 651 : "When calling a remote timeline client method, we record the current value \
1291 651 : of the calls_unfinished gauge in this histogram. Plot the histogram \
1292 651 : over time in a heatmap to visualize how many operations were ongoing \
1293 651 : at a given instant. It gives you a better idea of the queue depth \
1294 651 : than plotting the gauge directly, since operations may complete faster \
1295 651 : than the sampling interval.",
1296 651 : &["file_kind", "op_kind"],
1297 651 : // The calls_unfinished gauge is an integer gauge, hence we have integer buckets.
1298 651 : vec![0.0, 1.0, 2.0, 4.0, 6.0, 8.0, 10.0, 15.0, 20.0, 40.0, 60.0, 80.0, 100.0, 500.0],
1299 651 : )
1300 651 : .expect("failed to define a metric")
1301 651 : });
1302 :
1303 : static REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER: Lazy<IntCounterVec> =
1304 510 : Lazy::new(|| {
1305 510 : register_int_counter_vec!(
1306 510 : "pageserver_remote_timeline_client_bytes_started",
1307 510 : "Incremented by the number of bytes associated with a remote timeline client operation. \
1308 510 : The increment happens when the operation is scheduled.",
1309 510 : &["tenant_id", "shard_id", "timeline_id", "file_kind", "op_kind"],
1310 510 : )
1311 510 : .expect("failed to define a metric")
1312 510 : });
1313 :
1314 510 : static REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER: Lazy<IntCounterVec> = Lazy::new(|| {
1315 510 : register_int_counter_vec!(
1316 510 : "pageserver_remote_timeline_client_bytes_finished",
1317 510 : "Incremented by the number of bytes associated with a remote timeline client operation. \
1318 510 : The increment happens when the operation finishes (regardless of success/failure/shutdown).",
1319 510 : &["tenant_id", "shard_id", "timeline_id", "file_kind", "op_kind"],
1320 510 : )
1321 510 : .expect("failed to define a metric")
1322 510 : });
1323 :
1324 : pub(crate) struct TenantManagerMetrics {
1325 : pub(crate) tenant_slots: UIntGauge,
1326 : pub(crate) tenant_slot_writes: IntCounter,
1327 : pub(crate) unexpected_errors: IntCounter,
1328 : }
1329 :
1330 627 : pub(crate) static TENANT_MANAGER: Lazy<TenantManagerMetrics> = Lazy::new(|| {
1331 627 : TenantManagerMetrics {
1332 627 : tenant_slots: register_uint_gauge!(
1333 627 : "pageserver_tenant_manager_slots",
1334 627 : "How many slots currently exist, including all attached, secondary and in-progress operations",
1335 627 : )
1336 627 : .expect("failed to define a metric"),
1337 627 : tenant_slot_writes: register_int_counter!(
1338 627 : "pageserver_tenant_manager_slot_writes",
1339 627 : "Writes to a tenant slot, including all of create/attach/detach/delete"
1340 627 : )
1341 627 : .expect("failed to define a metric"),
1342 627 : unexpected_errors: register_int_counter!(
1343 627 : "pageserver_tenant_manager_unexpected_errors_total",
1344 627 : "Number of unexpected conditions encountered: nonzero value indicates a non-fatal bug."
1345 627 : )
1346 627 : .expect("failed to define a metric"),
1347 627 : }
1348 627 : });
1349 :
1350 : pub(crate) struct DeletionQueueMetrics {
1351 : pub(crate) keys_submitted: IntCounter,
1352 : pub(crate) keys_dropped: IntCounter,
1353 : pub(crate) keys_executed: IntCounter,
1354 : pub(crate) keys_validated: IntCounter,
1355 : pub(crate) dropped_lsn_updates: IntCounter,
1356 : pub(crate) unexpected_errors: IntCounter,
1357 : pub(crate) remote_errors: IntCounterVec,
1358 : }
1359 639 : pub(crate) static DELETION_QUEUE: Lazy<DeletionQueueMetrics> = Lazy::new(|| {
1360 639 : DeletionQueueMetrics{
1361 639 :
1362 639 : keys_submitted: register_int_counter!(
1363 639 : "pageserver_deletion_queue_submitted_total",
1364 639 : "Number of objects submitted for deletion"
1365 639 : )
1366 639 : .expect("failed to define a metric"),
1367 639 :
1368 639 : keys_dropped: register_int_counter!(
1369 639 : "pageserver_deletion_queue_dropped_total",
1370 639 : "Number of object deletions dropped due to stale generation."
1371 639 : )
1372 639 : .expect("failed to define a metric"),
1373 639 :
1374 639 : keys_executed: register_int_counter!(
1375 639 : "pageserver_deletion_queue_executed_total",
1376 639 : "Number of objects deleted. Only includes objects that we actually deleted, sum with pageserver_deletion_queue_dropped_total for the total number of keys processed to completion"
1377 639 : )
1378 639 : .expect("failed to define a metric"),
1379 639 :
1380 639 : keys_validated: register_int_counter!(
1381 639 : "pageserver_deletion_queue_validated_total",
1382 639 : "Number of keys validated for deletion. Sum with pageserver_deletion_queue_dropped_total for the total number of keys that have passed through the validation stage."
1383 639 : )
1384 639 : .expect("failed to define a metric"),
1385 639 :
1386 639 : dropped_lsn_updates: register_int_counter!(
1387 639 : "pageserver_deletion_queue_dropped_lsn_updates_total",
1388 639 : "Updates to remote_consistent_lsn dropped due to stale generation number."
1389 639 : )
1390 639 : .expect("failed to define a metric"),
1391 639 : unexpected_errors: register_int_counter!(
1392 639 : "pageserver_deletion_queue_unexpected_errors_total",
1393 639 : "Number of unexpected condiions that may stall the queue: any value above zero is unexpected."
1394 639 : )
1395 639 : .expect("failed to define a metric"),
1396 639 : remote_errors: register_int_counter_vec!(
1397 639 : "pageserver_deletion_queue_remote_errors_total",
1398 639 : "Retryable remote I/O errors while executing deletions, for example 503 responses to DeleteObjects",
1399 639 : &["op_kind"],
1400 639 : )
1401 639 : .expect("failed to define a metric")
1402 639 : }
1403 639 : });
1404 :
1405 : pub(crate) struct WalIngestMetrics {
1406 : pub(crate) records_received: IntCounter,
1407 : pub(crate) records_committed: IntCounter,
1408 : pub(crate) records_filtered: IntCounter,
1409 : }
1410 :
1411 496 : pub(crate) static WAL_INGEST: Lazy<WalIngestMetrics> = Lazy::new(|| WalIngestMetrics {
1412 496 : records_received: register_int_counter!(
1413 496 : "pageserver_wal_ingest_records_received",
1414 496 : "Number of WAL records received from safekeepers"
1415 496 : )
1416 496 : .expect("failed to define a metric"),
1417 496 : records_committed: register_int_counter!(
1418 496 : "pageserver_wal_ingest_records_committed",
1419 496 : "Number of WAL records which resulted in writes to pageserver storage"
1420 496 : )
1421 496 : .expect("failed to define a metric"),
1422 496 : records_filtered: register_int_counter!(
1423 496 : "pageserver_wal_ingest_records_filtered",
1424 496 : "Number of WAL records filtered out due to sharding"
1425 496 : )
1426 496 : .expect("failed to define a metric"),
1427 496 : });
1428 : pub(crate) struct SecondaryModeMetrics {
1429 : pub(crate) upload_heatmap: IntCounter,
1430 : pub(crate) upload_heatmap_errors: IntCounter,
1431 : pub(crate) upload_heatmap_duration: Histogram,
1432 : pub(crate) download_heatmap: IntCounter,
1433 : pub(crate) download_layer: IntCounter,
1434 : }
1435 11 : pub(crate) static SECONDARY_MODE: Lazy<SecondaryModeMetrics> = Lazy::new(|| SecondaryModeMetrics {
1436 11 : upload_heatmap: register_int_counter!(
1437 11 : "pageserver_secondary_upload_heatmap",
1438 11 : "Number of heatmaps written to remote storage by attached tenants"
1439 11 : )
1440 11 : .expect("failed to define a metric"),
1441 11 : upload_heatmap_errors: register_int_counter!(
1442 11 : "pageserver_secondary_upload_heatmap_errors",
1443 11 : "Failures writing heatmap to remote storage"
1444 11 : )
1445 11 : .expect("failed to define a metric"),
1446 11 : upload_heatmap_duration: register_histogram!(
1447 11 : "pageserver_secondary_upload_heatmap_duration",
1448 11 : "Time to build and upload a heatmap, including any waiting inside the S3 client"
1449 11 : )
1450 11 : .expect("failed to define a metric"),
1451 11 : download_heatmap: register_int_counter!(
1452 11 : "pageserver_secondary_download_heatmap",
1453 11 : "Number of downloads of heatmaps by secondary mode locations"
1454 11 : )
1455 11 : .expect("failed to define a metric"),
1456 11 : download_layer: register_int_counter!(
1457 11 : "pageserver_secondary_download_layer",
1458 11 : "Number of downloads of layers by secondary mode locations"
1459 11 : )
1460 11 : .expect("failed to define a metric"),
1461 11 : });
1462 :
1463 0 : #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1464 : pub enum RemoteOpKind {
1465 : Upload,
1466 : Download,
1467 : Delete,
1468 : }
1469 : impl RemoteOpKind {
1470 224771 : pub fn as_str(&self) -> &'static str {
1471 224771 : match self {
1472 178292 : Self::Upload => "upload",
1473 29800 : Self::Download => "download",
1474 16679 : Self::Delete => "delete",
1475 : }
1476 224771 : }
1477 : }
1478 :
1479 0 : #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
1480 : pub enum RemoteOpFileKind {
1481 : Layer,
1482 : Index,
1483 : }
1484 : impl RemoteOpFileKind {
1485 224771 : pub fn as_str(&self) -> &'static str {
1486 224771 : match self {
1487 198516 : Self::Layer => "layer",
1488 26255 : Self::Index => "index",
1489 : }
1490 224771 : }
1491 : }
1492 :
1493 651 : pub(crate) static REMOTE_OPERATION_TIME: Lazy<HistogramVec> = Lazy::new(|| {
1494 651 : register_histogram_vec!(
1495 651 : "pageserver_remote_operation_seconds",
1496 651 : "Time spent on remote storage operations. \
1497 651 : Grouped by tenant, timeline, operation_kind and status. \
1498 651 : Does not account for time spent waiting in remote timeline client's queues.",
1499 651 : &["file_kind", "op_kind", "status"]
1500 651 : )
1501 651 : .expect("failed to define a metric")
1502 651 : });
1503 :
1504 573 : pub(crate) static TENANT_TASK_EVENTS: Lazy<IntCounterVec> = Lazy::new(|| {
1505 573 : register_int_counter_vec!(
1506 573 : "pageserver_tenant_task_events",
1507 573 : "Number of task start/stop/fail events.",
1508 573 : &["event"],
1509 573 : )
1510 573 : .expect("Failed to register tenant_task_events metric")
1511 573 : });
1512 :
1513 500 : pub(crate) static BACKGROUND_LOOP_SEMAPHORE_WAIT_GAUGE: Lazy<IntCounterPairVec> = Lazy::new(|| {
1514 1000 : register_int_counter_pair_vec!(
1515 1000 : "pageserver_background_loop_semaphore_wait_start_count",
1516 1000 : "Counter for background loop concurrency-limiting semaphore acquire calls started",
1517 1000 : "pageserver_background_loop_semaphore_wait_finish_count",
1518 1000 : "Counter for background loop concurrency-limiting semaphore acquire calls finished",
1519 1000 : &["task"],
1520 1000 : )
1521 500 : .unwrap()
1522 500 : });
1523 :
1524 625 : pub(crate) static BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT: Lazy<IntCounterVec> = Lazy::new(|| {
1525 625 : register_int_counter_vec!(
1526 625 : "pageserver_background_loop_period_overrun_count",
1527 625 : "Incremented whenever warn_when_period_overrun() logs a warning.",
1528 625 : &["task", "period"],
1529 625 : )
1530 625 : .expect("failed to define a metric")
1531 625 : });
1532 :
1533 : // walreceiver metrics
1534 :
1535 625 : pub(crate) static WALRECEIVER_STARTED_CONNECTIONS: Lazy<IntCounter> = Lazy::new(|| {
1536 625 : register_int_counter!(
1537 625 : "pageserver_walreceiver_started_connections_total",
1538 625 : "Number of started walreceiver connections"
1539 625 : )
1540 625 : .expect("failed to define a metric")
1541 625 : });
1542 :
1543 625 : pub(crate) static WALRECEIVER_ACTIVE_MANAGERS: Lazy<IntGauge> = Lazy::new(|| {
1544 625 : register_int_gauge!(
1545 625 : "pageserver_walreceiver_active_managers",
1546 625 : "Number of active walreceiver managers"
1547 625 : )
1548 625 : .expect("failed to define a metric")
1549 625 : });
1550 :
1551 433 : pub(crate) static WALRECEIVER_SWITCHES: Lazy<IntCounterVec> = Lazy::new(|| {
1552 433 : register_int_counter_vec!(
1553 433 : "pageserver_walreceiver_switches_total",
1554 433 : "Number of walreceiver manager change_connection calls",
1555 433 : &["reason"]
1556 433 : )
1557 433 : .expect("failed to define a metric")
1558 433 : });
1559 :
1560 625 : pub(crate) static WALRECEIVER_BROKER_UPDATES: Lazy<IntCounter> = Lazy::new(|| {
1561 625 : register_int_counter!(
1562 625 : "pageserver_walreceiver_broker_updates_total",
1563 625 : "Number of received broker updates in walreceiver"
1564 625 : )
1565 625 : .expect("failed to define a metric")
1566 625 : });
1567 :
1568 627 : pub(crate) static WALRECEIVER_CANDIDATES_EVENTS: Lazy<IntCounterVec> = Lazy::new(|| {
1569 627 : register_int_counter_vec!(
1570 627 : "pageserver_walreceiver_candidates_events_total",
1571 627 : "Number of walreceiver candidate events",
1572 627 : &["event"]
1573 627 : )
1574 627 : .expect("failed to define a metric")
1575 627 : });
1576 :
1577 : pub(crate) static WALRECEIVER_CANDIDATES_ADDED: Lazy<IntCounter> =
1578 625 : Lazy::new(|| WALRECEIVER_CANDIDATES_EVENTS.with_label_values(&["add"]));
1579 :
1580 : pub(crate) static WALRECEIVER_CANDIDATES_REMOVED: Lazy<IntCounter> =
1581 627 : Lazy::new(|| WALRECEIVER_CANDIDATES_EVENTS.with_label_values(&["remove"]));
1582 :
1583 : // Metrics collected on WAL redo operations
1584 : //
1585 : // We collect the time spent in actual WAL redo ('redo'), and time waiting
1586 : // for access to the postgres process ('wait') since there is only one for
1587 : // each tenant.
1588 :
1589 : /// Time buckets are small because we want to be able to measure the
1590 : /// smallest redo processing times. These buckets allow us to measure down
1591 : /// to 5us, which equates to 200'000 pages/sec, which equates to 1.6GB/sec.
1592 : /// This is much better than the previous 5ms aka 200 pages/sec aka 1.6MB/sec.
1593 : ///
1594 : /// Values up to 1s are recorded because metrics show that we have redo
1595 : /// durations and lock times larger than 0.250s.
1596 : macro_rules! redo_histogram_time_buckets {
1597 : () => {
1598 : vec![
1599 : 0.000_005, 0.000_010, 0.000_025, 0.000_050, 0.000_100, 0.000_250, 0.000_500, 0.001_000,
1600 : 0.002_500, 0.005_000, 0.010_000, 0.025_000, 0.050_000, 0.100_000, 0.250_000, 0.500_000,
1601 : 1.000_000,
1602 : ]
1603 : };
1604 : }
1605 :
1606 : /// While we're at it, also measure the amount of records replayed in each
1607 : /// operation. We have a global 'total replayed' counter, but that's not
1608 : /// as useful as 'what is the skew for how many records we replay in one
1609 : /// operation'.
1610 : macro_rules! redo_histogram_count_buckets {
1611 : () => {
1612 : vec![0.0, 1.0, 2.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0]
1613 : };
1614 : }
1615 :
1616 : macro_rules! redo_bytes_histogram_count_buckets {
1617 : () => {
1618 : // powers of (2^.5), from 2^4.5 to 2^15 (22 buckets)
1619 : // rounded up to the next multiple of 8 to capture any MAXALIGNed record of that size, too.
1620 : vec![
1621 : 24.0, 32.0, 48.0, 64.0, 96.0, 128.0, 184.0, 256.0, 368.0, 512.0, 728.0, 1024.0, 1456.0,
1622 : 2048.0, 2904.0, 4096.0, 5800.0, 8192.0, 11592.0, 16384.0, 23176.0, 32768.0,
1623 : ]
1624 : };
1625 : }
1626 :
1627 631 : pub(crate) static WAL_REDO_TIME: Lazy<Histogram> = Lazy::new(|| {
1628 631 : register_histogram!(
1629 631 : "pageserver_wal_redo_seconds",
1630 631 : "Time spent on WAL redo",
1631 631 : redo_histogram_time_buckets!()
1632 631 : )
1633 631 : .expect("failed to define a metric")
1634 631 : });
1635 :
1636 631 : pub(crate) static WAL_REDO_RECORDS_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
1637 631 : register_histogram!(
1638 631 : "pageserver_wal_redo_records_histogram",
1639 631 : "Histogram of number of records replayed per redo in the Postgres WAL redo process",
1640 631 : redo_histogram_count_buckets!(),
1641 631 : )
1642 631 : .expect("failed to define a metric")
1643 631 : });
1644 :
1645 631 : pub(crate) static WAL_REDO_BYTES_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
1646 631 : register_histogram!(
1647 631 : "pageserver_wal_redo_bytes_histogram",
1648 631 : "Histogram of number of records replayed per redo sent to Postgres",
1649 631 : redo_bytes_histogram_count_buckets!(),
1650 631 : )
1651 631 : .expect("failed to define a metric")
1652 631 : });
1653 :
1654 : // FIXME: isn't this already included by WAL_REDO_RECORDS_HISTOGRAM which has _count?
1655 399 : pub(crate) static WAL_REDO_RECORD_COUNTER: Lazy<IntCounter> = Lazy::new(|| {
1656 399 : register_int_counter!(
1657 399 : "pageserver_replayed_wal_records_total",
1658 399 : "Number of WAL records replayed in WAL redo process"
1659 399 : )
1660 399 : .unwrap()
1661 399 : });
1662 :
1663 : #[rustfmt::skip]
1664 631 : pub(crate) static WAL_REDO_PROCESS_LAUNCH_DURATION_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
1665 631 : register_histogram!(
1666 631 : "pageserver_wal_redo_process_launch_duration",
1667 631 : "Histogram of the duration of successful WalRedoProcess::launch calls",
1668 631 : vec![
1669 631 : 0.0002, 0.0004, 0.0006, 0.0008, 0.0010,
1670 631 : 0.0020, 0.0040, 0.0060, 0.0080, 0.0100,
1671 631 : 0.0200, 0.0400, 0.0600, 0.0800, 0.1000,
1672 631 : 0.2000, 0.4000, 0.6000, 0.8000, 1.0000,
1673 631 : 1.5000, 2.0000, 2.5000, 3.0000, 4.0000, 10.0000
1674 631 : ],
1675 631 : )
1676 631 : .expect("failed to define a metric")
1677 631 : });
1678 :
1679 : pub(crate) struct WalRedoProcessCounters {
1680 : pub(crate) started: IntCounter,
1681 : pub(crate) killed_by_cause: enum_map::EnumMap<WalRedoKillCause, IntCounter>,
1682 : pub(crate) active_stderr_logger_tasks_started: IntCounter,
1683 : pub(crate) active_stderr_logger_tasks_finished: IntCounter,
1684 : }
1685 :
1686 1497 : #[derive(Debug, enum_map::Enum, strum_macros::IntoStaticStr)]
1687 : pub(crate) enum WalRedoKillCause {
1688 : WalRedoProcessDrop,
1689 : NoLeakChildDrop,
1690 : Startup,
1691 : }
1692 :
1693 : impl Default for WalRedoProcessCounters {
1694 399 : fn default() -> Self {
1695 399 : let started = register_int_counter!(
1696 399 : "pageserver_wal_redo_process_started_total",
1697 399 : "Number of WAL redo processes started",
1698 399 : )
1699 399 : .unwrap();
1700 399 :
1701 399 : let killed = register_int_counter_vec!(
1702 399 : "pageserver_wal_redo_process_stopped_total",
1703 399 : "Number of WAL redo processes stopped",
1704 399 : &["cause"],
1705 399 : )
1706 399 : .unwrap();
1707 399 :
1708 399 : let active_stderr_logger_tasks_started = register_int_counter!(
1709 399 : "pageserver_walredo_stderr_logger_tasks_started_total",
1710 399 : "Number of active walredo stderr logger tasks that have started",
1711 399 : )
1712 399 : .unwrap();
1713 399 :
1714 399 : let active_stderr_logger_tasks_finished = register_int_counter!(
1715 399 : "pageserver_walredo_stderr_logger_tasks_finished_total",
1716 399 : "Number of active walredo stderr logger tasks that have finished",
1717 399 : )
1718 399 : .unwrap();
1719 399 :
1720 399 : Self {
1721 399 : started,
1722 1197 : killed_by_cause: EnumMap::from_array(std::array::from_fn(|i| {
1723 1197 : let cause = <WalRedoKillCause as enum_map::Enum>::from_usize(i);
1724 1197 : let cause_str: &'static str = cause.into();
1725 1197 : killed.with_label_values(&[cause_str])
1726 1197 : })),
1727 399 : active_stderr_logger_tasks_started,
1728 399 : active_stderr_logger_tasks_finished,
1729 399 : }
1730 399 : }
1731 : }
1732 :
1733 : pub(crate) static WAL_REDO_PROCESS_COUNTERS: Lazy<WalRedoProcessCounters> =
1734 : Lazy::new(WalRedoProcessCounters::default);
1735 :
1736 : /// Similar to `prometheus::HistogramTimer` but does not record on drop.
1737 : pub(crate) struct StorageTimeMetricsTimer {
1738 : metrics: StorageTimeMetrics,
1739 : start: Instant,
1740 : }
1741 :
1742 : impl StorageTimeMetricsTimer {
1743 9723 : fn new(metrics: StorageTimeMetrics) -> Self {
1744 9723 : Self {
1745 9723 : metrics,
1746 9723 : start: Instant::now(),
1747 9723 : }
1748 9723 : }
1749 :
1750 : /// Record the time from creation to now.
1751 9673 : pub fn stop_and_record(self) {
1752 9673 : let duration = self.start.elapsed().as_secs_f64();
1753 9673 : self.metrics.timeline_sum.inc_by(duration);
1754 9673 : self.metrics.timeline_count.inc();
1755 9673 : self.metrics.global_histogram.observe(duration);
1756 9673 : }
1757 : }
1758 :
1759 : /// Timing facilities for an globally histogrammed metric, which is supported by per tenant and
1760 : /// timeline total sum and count.
1761 9723 : #[derive(Clone, Debug)]
1762 : pub(crate) struct StorageTimeMetrics {
1763 : /// Sum of f64 seconds, per operation, tenant_id and timeline_id
1764 : timeline_sum: Counter,
1765 : /// Number of oeprations, per operation, tenant_id and timeline_id
1766 : timeline_count: IntCounter,
1767 : /// Global histogram having only the "operation" label.
1768 : global_histogram: Histogram,
1769 : }
1770 :
1771 : impl StorageTimeMetrics {
1772 11144 : pub fn new(
1773 11144 : operation: StorageTimeOperation,
1774 11144 : tenant_id: &str,
1775 11144 : shard_id: &str,
1776 11144 : timeline_id: &str,
1777 11144 : ) -> Self {
1778 11144 : let operation: &'static str = operation.into();
1779 11144 :
1780 11144 : let timeline_sum = STORAGE_TIME_SUM_PER_TIMELINE
1781 11144 : .get_metric_with_label_values(&[operation, tenant_id, shard_id, timeline_id])
1782 11144 : .unwrap();
1783 11144 : let timeline_count = STORAGE_TIME_COUNT_PER_TIMELINE
1784 11144 : .get_metric_with_label_values(&[operation, tenant_id, shard_id, timeline_id])
1785 11144 : .unwrap();
1786 11144 : let global_histogram = STORAGE_TIME_GLOBAL
1787 11144 : .get_metric_with_label_values(&[operation])
1788 11144 : .unwrap();
1789 11144 :
1790 11144 : StorageTimeMetrics {
1791 11144 : timeline_sum,
1792 11144 : timeline_count,
1793 11144 : global_histogram,
1794 11144 : }
1795 11144 : }
1796 :
1797 : /// Starts timing a new operation.
1798 : ///
1799 : /// Note: unlike `prometheus::HistogramTimer` the returned timer does not record on drop.
1800 9723 : pub fn start_timer(&self) -> StorageTimeMetricsTimer {
1801 9723 : StorageTimeMetricsTimer::new(self.clone())
1802 9723 : }
1803 : }
1804 :
1805 0 : #[derive(Debug)]
1806 : pub(crate) struct TimelineMetrics {
1807 : tenant_id: String,
1808 : shard_id: String,
1809 : timeline_id: String,
1810 : pub flush_time_histo: StorageTimeMetrics,
1811 : pub compact_time_histo: StorageTimeMetrics,
1812 : pub create_images_time_histo: StorageTimeMetrics,
1813 : pub logical_size_histo: StorageTimeMetrics,
1814 : pub imitate_logical_size_histo: StorageTimeMetrics,
1815 : pub load_layer_map_histo: StorageTimeMetrics,
1816 : pub garbage_collect_histo: StorageTimeMetrics,
1817 : pub last_record_gauge: IntGauge,
1818 : resident_physical_size_gauge: UIntGauge,
1819 : /// copy of LayeredTimeline.current_logical_size
1820 : pub current_logical_size_gauge: UIntGauge,
1821 : pub directory_entries_count_gauge: Lazy<UIntGauge, Box<dyn Send + Fn() -> UIntGauge>>,
1822 : pub num_persistent_files_created: IntCounter,
1823 : pub persistent_bytes_written: IntCounter,
1824 : pub evictions: IntCounter,
1825 : pub evictions_with_low_residence_duration: std::sync::RwLock<EvictionsWithLowResidenceDuration>,
1826 : }
1827 :
1828 : impl TimelineMetrics {
1829 1592 : pub fn new(
1830 1592 : tenant_shard_id: &TenantShardId,
1831 1592 : timeline_id_raw: &TimelineId,
1832 1592 : evictions_with_low_residence_duration_builder: EvictionsWithLowResidenceDurationBuilder,
1833 1592 : ) -> Self {
1834 1592 : let tenant_id = tenant_shard_id.tenant_id.to_string();
1835 1592 : let shard_id = format!("{}", tenant_shard_id.shard_slug());
1836 1592 : let timeline_id = timeline_id_raw.to_string();
1837 1592 : let flush_time_histo = StorageTimeMetrics::new(
1838 1592 : StorageTimeOperation::LayerFlush,
1839 1592 : &tenant_id,
1840 1592 : &shard_id,
1841 1592 : &timeline_id,
1842 1592 : );
1843 1592 : let compact_time_histo = StorageTimeMetrics::new(
1844 1592 : StorageTimeOperation::Compact,
1845 1592 : &tenant_id,
1846 1592 : &shard_id,
1847 1592 : &timeline_id,
1848 1592 : );
1849 1592 : let create_images_time_histo = StorageTimeMetrics::new(
1850 1592 : StorageTimeOperation::CreateImages,
1851 1592 : &tenant_id,
1852 1592 : &shard_id,
1853 1592 : &timeline_id,
1854 1592 : );
1855 1592 : let logical_size_histo = StorageTimeMetrics::new(
1856 1592 : StorageTimeOperation::LogicalSize,
1857 1592 : &tenant_id,
1858 1592 : &shard_id,
1859 1592 : &timeline_id,
1860 1592 : );
1861 1592 : let imitate_logical_size_histo = StorageTimeMetrics::new(
1862 1592 : StorageTimeOperation::ImitateLogicalSize,
1863 1592 : &tenant_id,
1864 1592 : &shard_id,
1865 1592 : &timeline_id,
1866 1592 : );
1867 1592 : let load_layer_map_histo = StorageTimeMetrics::new(
1868 1592 : StorageTimeOperation::LoadLayerMap,
1869 1592 : &tenant_id,
1870 1592 : &shard_id,
1871 1592 : &timeline_id,
1872 1592 : );
1873 1592 : let garbage_collect_histo = StorageTimeMetrics::new(
1874 1592 : StorageTimeOperation::Gc,
1875 1592 : &tenant_id,
1876 1592 : &shard_id,
1877 1592 : &timeline_id,
1878 1592 : );
1879 1592 : let last_record_gauge = LAST_RECORD_LSN
1880 1592 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
1881 1592 : .unwrap();
1882 1592 : let resident_physical_size_gauge = RESIDENT_PHYSICAL_SIZE
1883 1592 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
1884 1592 : .unwrap();
1885 1592 : // TODO: we shouldn't expose this metric
1886 1592 : let current_logical_size_gauge = CURRENT_LOGICAL_SIZE
1887 1592 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
1888 1592 : .unwrap();
1889 1592 : // TODO use impl Trait syntax here once we have ability to use it: https://github.com/rust-lang/rust/issues/63065
1890 1592 : let directory_entries_count_gauge_closure = {
1891 1592 : let tenant_shard_id = *tenant_shard_id;
1892 1592 : let timeline_id_raw = *timeline_id_raw;
1893 0 : move || {
1894 0 : let tenant_id = tenant_shard_id.tenant_id.to_string();
1895 0 : let shard_id = format!("{}", tenant_shard_id.shard_slug());
1896 0 : let timeline_id = timeline_id_raw.to_string();
1897 0 : let gauge: UIntGauge = DIRECTORY_ENTRIES_COUNT
1898 0 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
1899 0 : .unwrap();
1900 0 : gauge
1901 0 : }
1902 : };
1903 1592 : let directory_entries_count_gauge: Lazy<UIntGauge, Box<dyn Send + Fn() -> UIntGauge>> =
1904 1592 : Lazy::new(Box::new(directory_entries_count_gauge_closure));
1905 1592 : let num_persistent_files_created = NUM_PERSISTENT_FILES_CREATED
1906 1592 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
1907 1592 : .unwrap();
1908 1592 : let persistent_bytes_written = PERSISTENT_BYTES_WRITTEN
1909 1592 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
1910 1592 : .unwrap();
1911 1592 : let evictions = EVICTIONS
1912 1592 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
1913 1592 : .unwrap();
1914 1592 : let evictions_with_low_residence_duration = evictions_with_low_residence_duration_builder
1915 1592 : .build(&tenant_id, &shard_id, &timeline_id);
1916 1592 :
1917 1592 : TimelineMetrics {
1918 1592 : tenant_id,
1919 1592 : shard_id,
1920 1592 : timeline_id,
1921 1592 : flush_time_histo,
1922 1592 : compact_time_histo,
1923 1592 : create_images_time_histo,
1924 1592 : logical_size_histo,
1925 1592 : imitate_logical_size_histo,
1926 1592 : garbage_collect_histo,
1927 1592 : load_layer_map_histo,
1928 1592 : last_record_gauge,
1929 1592 : resident_physical_size_gauge,
1930 1592 : current_logical_size_gauge,
1931 1592 : directory_entries_count_gauge,
1932 1592 : num_persistent_files_created,
1933 1592 : persistent_bytes_written,
1934 1592 : evictions,
1935 1592 : evictions_with_low_residence_duration: std::sync::RwLock::new(
1936 1592 : evictions_with_low_residence_duration,
1937 1592 : ),
1938 1592 : }
1939 1592 : }
1940 :
1941 22298 : pub(crate) fn record_new_file_metrics(&self, sz: u64) {
1942 22298 : self.resident_physical_size_add(sz);
1943 22298 : self.num_persistent_files_created.inc_by(1);
1944 22298 : self.persistent_bytes_written.inc_by(sz);
1945 22298 : }
1946 :
1947 7670 : pub(crate) fn resident_physical_size_sub(&self, sz: u64) {
1948 7670 : self.resident_physical_size_gauge.sub(sz);
1949 7670 : crate::metrics::RESIDENT_PHYSICAL_SIZE_GLOBAL.sub(sz);
1950 7670 : }
1951 :
1952 43908 : pub(crate) fn resident_physical_size_add(&self, sz: u64) {
1953 43908 : self.resident_physical_size_gauge.add(sz);
1954 43908 : crate::metrics::RESIDENT_PHYSICAL_SIZE_GLOBAL.add(sz);
1955 43908 : }
1956 :
1957 363 : pub(crate) fn resident_physical_size_get(&self) -> u64 {
1958 363 : self.resident_physical_size_gauge.get()
1959 363 : }
1960 : }
1961 :
1962 : impl Drop for TimelineMetrics {
1963 346 : fn drop(&mut self) {
1964 346 : let tenant_id = &self.tenant_id;
1965 346 : let timeline_id = &self.timeline_id;
1966 346 : let shard_id = &self.shard_id;
1967 346 : let _ = LAST_RECORD_LSN.remove_label_values(&[tenant_id, &shard_id, timeline_id]);
1968 346 : {
1969 346 : RESIDENT_PHYSICAL_SIZE_GLOBAL.sub(self.resident_physical_size_get());
1970 346 : let _ =
1971 346 : RESIDENT_PHYSICAL_SIZE.remove_label_values(&[tenant_id, &shard_id, timeline_id]);
1972 346 : }
1973 346 : let _ = CURRENT_LOGICAL_SIZE.remove_label_values(&[tenant_id, &shard_id, timeline_id]);
1974 346 : if let Some(metric) = Lazy::get(&DIRECTORY_ENTRIES_COUNT) {
1975 0 : let _ = metric.remove_label_values(&[tenant_id, &shard_id, timeline_id]);
1976 346 : }
1977 : let _ =
1978 346 : NUM_PERSISTENT_FILES_CREATED.remove_label_values(&[tenant_id, &shard_id, timeline_id]);
1979 346 : let _ = PERSISTENT_BYTES_WRITTEN.remove_label_values(&[tenant_id, &shard_id, timeline_id]);
1980 346 : let _ = EVICTIONS.remove_label_values(&[tenant_id, &shard_id, timeline_id]);
1981 346 :
1982 346 : self.evictions_with_low_residence_duration
1983 346 : .write()
1984 346 : .unwrap()
1985 346 : .remove(tenant_id, shard_id, timeline_id);
1986 :
1987 : // The following metrics are born outside of the TimelineMetrics lifecycle but still
1988 : // removed at the end of it. The idea is to have the metrics outlive the
1989 : // entity during which they're observed, e.g., the smgr metrics shall
1990 : // outlive an individual smgr connection, but not the timeline.
1991 :
1992 3114 : for op in StorageTimeOperation::VARIANTS {
1993 2768 : let _ = STORAGE_TIME_SUM_PER_TIMELINE.remove_label_values(&[
1994 2768 : op,
1995 2768 : tenant_id,
1996 2768 : shard_id,
1997 2768 : timeline_id,
1998 2768 : ]);
1999 2768 : let _ = STORAGE_TIME_COUNT_PER_TIMELINE.remove_label_values(&[
2000 2768 : op,
2001 2768 : tenant_id,
2002 2768 : shard_id,
2003 2768 : timeline_id,
2004 2768 : ]);
2005 2768 : }
2006 :
2007 1038 : for op in STORAGE_IO_SIZE_OPERATIONS {
2008 692 : let _ = STORAGE_IO_SIZE.remove_label_values(&[op, tenant_id, shard_id, timeline_id]);
2009 692 : }
2010 :
2011 2076 : for op in SmgrQueryType::iter() {
2012 1730 : let _ = SMGR_QUERY_TIME_PER_TENANT_TIMELINE.remove_label_values(&[
2013 1730 : op.into(),
2014 1730 : tenant_id,
2015 1730 : shard_id,
2016 1730 : timeline_id,
2017 1730 : ]);
2018 1730 : }
2019 346 : }
2020 : }
2021 :
2022 318 : pub(crate) fn remove_tenant_metrics(tenant_shard_id: &TenantShardId) {
2023 318 : // Only shard zero deals in synthetic sizes
2024 318 : if tenant_shard_id.is_zero() {
2025 304 : let tid = tenant_shard_id.tenant_id.to_string();
2026 304 : let _ = TENANT_SYNTHETIC_SIZE_METRIC.remove_label_values(&[&tid]);
2027 304 : }
2028 :
2029 : // we leave the BROKEN_TENANTS_SET entry if any
2030 318 : }
2031 :
2032 : use futures::Future;
2033 : use pin_project_lite::pin_project;
2034 : use std::collections::HashMap;
2035 : use std::pin::Pin;
2036 : use std::sync::{Arc, Mutex};
2037 : use std::task::{Context, Poll};
2038 : use std::time::{Duration, Instant};
2039 :
2040 : use crate::context::{PageContentKind, RequestContext};
2041 : use crate::task_mgr::TaskKind;
2042 :
2043 : /// Maintain a per timeline gauge in addition to the global gauge.
2044 : struct PerTimelineRemotePhysicalSizeGauge {
2045 : last_set: u64,
2046 : gauge: UIntGauge,
2047 : }
2048 :
2049 : impl PerTimelineRemotePhysicalSizeGauge {
2050 1592 : fn new(per_timeline_gauge: UIntGauge) -> Self {
2051 1592 : Self {
2052 1592 : last_set: per_timeline_gauge.get(),
2053 1592 : gauge: per_timeline_gauge,
2054 1592 : }
2055 1592 : }
2056 7574 : fn set(&mut self, sz: u64) {
2057 7574 : self.gauge.set(sz);
2058 7574 : if sz < self.last_set {
2059 80 : REMOTE_PHYSICAL_SIZE_GLOBAL.sub(self.last_set - sz);
2060 7494 : } else {
2061 7494 : REMOTE_PHYSICAL_SIZE_GLOBAL.add(sz - self.last_set);
2062 7494 : };
2063 7574 : self.last_set = sz;
2064 7574 : }
2065 17 : fn get(&self) -> u64 {
2066 17 : self.gauge.get()
2067 17 : }
2068 : }
2069 :
2070 : impl Drop for PerTimelineRemotePhysicalSizeGauge {
2071 346 : fn drop(&mut self) {
2072 346 : REMOTE_PHYSICAL_SIZE_GLOBAL.sub(self.last_set);
2073 346 : }
2074 : }
2075 :
2076 : pub(crate) struct RemoteTimelineClientMetrics {
2077 : tenant_id: String,
2078 : shard_id: String,
2079 : timeline_id: String,
2080 : remote_physical_size_gauge: Mutex<Option<PerTimelineRemotePhysicalSizeGauge>>,
2081 : calls_unfinished_gauge: Mutex<HashMap<(&'static str, &'static str), IntGauge>>,
2082 : bytes_started_counter: Mutex<HashMap<(&'static str, &'static str), IntCounter>>,
2083 : bytes_finished_counter: Mutex<HashMap<(&'static str, &'static str), IntCounter>>,
2084 : }
2085 :
2086 : impl RemoteTimelineClientMetrics {
2087 1605 : pub fn new(tenant_shard_id: &TenantShardId, timeline_id: &TimelineId) -> Self {
2088 1605 : RemoteTimelineClientMetrics {
2089 1605 : tenant_id: tenant_shard_id.tenant_id.to_string(),
2090 1605 : shard_id: format!("{}", tenant_shard_id.shard_slug()),
2091 1605 : timeline_id: timeline_id.to_string(),
2092 1605 : calls_unfinished_gauge: Mutex::new(HashMap::default()),
2093 1605 : bytes_started_counter: Mutex::new(HashMap::default()),
2094 1605 : bytes_finished_counter: Mutex::new(HashMap::default()),
2095 1605 : remote_physical_size_gauge: Mutex::new(None),
2096 1605 : }
2097 1605 : }
2098 :
2099 7574 : pub(crate) fn remote_physical_size_set(&self, sz: u64) {
2100 7574 : let mut guard = self.remote_physical_size_gauge.lock().unwrap();
2101 7574 : let gauge = guard.get_or_insert_with(|| {
2102 1592 : PerTimelineRemotePhysicalSizeGauge::new(
2103 1592 : REMOTE_PHYSICAL_SIZE
2104 1592 : .get_metric_with_label_values(&[
2105 1592 : &self.tenant_id,
2106 1592 : &self.shard_id,
2107 1592 : &self.timeline_id,
2108 1592 : ])
2109 1592 : .unwrap(),
2110 1592 : )
2111 7574 : });
2112 7574 : gauge.set(sz);
2113 7574 : }
2114 :
2115 17 : pub(crate) fn remote_physical_size_get(&self) -> u64 {
2116 17 : let guard = self.remote_physical_size_gauge.lock().unwrap();
2117 17 : guard.as_ref().map(|gauge| gauge.get()).unwrap_or(0)
2118 17 : }
2119 :
2120 40186 : pub fn remote_operation_time(
2121 40186 : &self,
2122 40186 : file_kind: &RemoteOpFileKind,
2123 40186 : op_kind: &RemoteOpKind,
2124 40186 : status: &'static str,
2125 40186 : ) -> Histogram {
2126 40186 : let key = (file_kind.as_str(), op_kind.as_str(), status);
2127 40186 : REMOTE_OPERATION_TIME
2128 40186 : .get_metric_with_label_values(&[key.0, key.1, key.2])
2129 40186 : .unwrap()
2130 40186 : }
2131 :
2132 75743 : fn calls_unfinished_gauge(
2133 75743 : &self,
2134 75743 : file_kind: &RemoteOpFileKind,
2135 75743 : op_kind: &RemoteOpKind,
2136 75743 : ) -> IntGauge {
2137 75743 : let mut guard = self.calls_unfinished_gauge.lock().unwrap();
2138 75743 : let key = (file_kind.as_str(), op_kind.as_str());
2139 75743 : let metric = guard.entry(key).or_insert_with(move || {
2140 3381 : REMOTE_TIMELINE_CLIENT_CALLS_UNFINISHED_GAUGE
2141 3381 : .get_metric_with_label_values(&[
2142 3381 : &self.tenant_id,
2143 3381 : &self.shard_id,
2144 3381 : &self.timeline_id,
2145 3381 : key.0,
2146 3381 : key.1,
2147 3381 : ])
2148 3381 : .unwrap()
2149 75743 : });
2150 75743 : metric.clone()
2151 75743 : }
2152 :
2153 43923 : fn calls_started_hist(
2154 43923 : &self,
2155 43923 : file_kind: &RemoteOpFileKind,
2156 43923 : op_kind: &RemoteOpKind,
2157 43923 : ) -> Histogram {
2158 43923 : let key = (file_kind.as_str(), op_kind.as_str());
2159 43923 : REMOTE_TIMELINE_CLIENT_CALLS_STARTED_HIST
2160 43923 : .get_metric_with_label_values(&[key.0, key.1])
2161 43923 : .unwrap()
2162 43923 : }
2163 :
2164 22334 : fn bytes_started_counter(
2165 22334 : &self,
2166 22334 : file_kind: &RemoteOpFileKind,
2167 22334 : op_kind: &RemoteOpKind,
2168 22334 : ) -> IntCounter {
2169 22334 : let mut guard = self.bytes_started_counter.lock().unwrap();
2170 22334 : let key = (file_kind.as_str(), op_kind.as_str());
2171 22334 : let metric = guard.entry(key).or_insert_with(move || {
2172 957 : REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER
2173 957 : .get_metric_with_label_values(&[
2174 957 : &self.tenant_id,
2175 957 : &self.shard_id,
2176 957 : &self.timeline_id,
2177 957 : key.0,
2178 957 : key.1,
2179 957 : ])
2180 957 : .unwrap()
2181 22334 : });
2182 22334 : metric.clone()
2183 22334 : }
2184 :
2185 42573 : fn bytes_finished_counter(
2186 42573 : &self,
2187 42573 : file_kind: &RemoteOpFileKind,
2188 42573 : op_kind: &RemoteOpKind,
2189 42573 : ) -> IntCounter {
2190 42573 : let mut guard = self.bytes_finished_counter.lock().unwrap();
2191 42573 : let key = (file_kind.as_str(), op_kind.as_str());
2192 42573 : let metric = guard.entry(key).or_insert_with(move || {
2193 957 : REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER
2194 957 : .get_metric_with_label_values(&[
2195 957 : &self.tenant_id,
2196 957 : &self.shard_id,
2197 957 : &self.timeline_id,
2198 957 : key.0,
2199 957 : key.1,
2200 957 : ])
2201 957 : .unwrap()
2202 42573 : });
2203 42573 : metric.clone()
2204 42573 : }
2205 : }
2206 :
2207 : #[cfg(test)]
2208 : impl RemoteTimelineClientMetrics {
2209 6 : pub fn get_bytes_started_counter_value(
2210 6 : &self,
2211 6 : file_kind: &RemoteOpFileKind,
2212 6 : op_kind: &RemoteOpKind,
2213 6 : ) -> Option<u64> {
2214 6 : let guard = self.bytes_started_counter.lock().unwrap();
2215 6 : let key = (file_kind.as_str(), op_kind.as_str());
2216 6 : guard.get(&key).map(|counter| counter.get())
2217 6 : }
2218 :
2219 6 : pub fn get_bytes_finished_counter_value(
2220 6 : &self,
2221 6 : file_kind: &RemoteOpFileKind,
2222 6 : op_kind: &RemoteOpKind,
2223 6 : ) -> Option<u64> {
2224 6 : let guard = self.bytes_finished_counter.lock().unwrap();
2225 6 : let key = (file_kind.as_str(), op_kind.as_str());
2226 6 : guard.get(&key).map(|counter| counter.get())
2227 6 : }
2228 : }
2229 :
2230 : /// See [`RemoteTimelineClientMetrics::call_begin`].
2231 : #[must_use]
2232 : pub(crate) struct RemoteTimelineClientCallMetricGuard {
2233 : /// Decremented on drop.
2234 : calls_unfinished_metric: Option<IntGauge>,
2235 : /// If Some(), this references the bytes_finished metric, and we increment it by the given `u64` on drop.
2236 : bytes_finished: Option<(IntCounter, u64)>,
2237 : }
2238 :
2239 : impl RemoteTimelineClientCallMetricGuard {
2240 : /// Consume this guard object without performing the metric updates it would do on `drop()`.
2241 : /// The caller vouches to do the metric updates manually.
2242 33989 : pub fn will_decrement_manually(mut self) {
2243 33989 : let RemoteTimelineClientCallMetricGuard {
2244 33989 : calls_unfinished_metric,
2245 33989 : bytes_finished,
2246 33989 : } = &mut self;
2247 33989 : calls_unfinished_metric.take();
2248 33989 : bytes_finished.take();
2249 33989 : }
2250 : }
2251 :
2252 : impl Drop for RemoteTimelineClientCallMetricGuard {
2253 43921 : fn drop(&mut self) {
2254 43921 : let RemoteTimelineClientCallMetricGuard {
2255 43921 : calls_unfinished_metric,
2256 43921 : bytes_finished,
2257 43921 : } = self;
2258 43921 : if let Some(guard) = calls_unfinished_metric.take() {
2259 9932 : guard.dec();
2260 33989 : }
2261 43921 : if let Some((bytes_finished_metric, value)) = bytes_finished {
2262 0 : bytes_finished_metric.inc_by(*value);
2263 43921 : }
2264 43921 : }
2265 : }
2266 :
2267 : /// The enum variants communicate to the [`RemoteTimelineClientMetrics`] whether to
2268 : /// track the byte size of this call in applicable metric(s).
2269 : pub(crate) enum RemoteTimelineClientMetricsCallTrackSize {
2270 : /// Do not account for this call's byte size in any metrics.
2271 : /// The `reason` field is there to make the call sites self-documenting
2272 : /// about why they don't need the metric.
2273 : DontTrackSize { reason: &'static str },
2274 : /// Track the byte size of the call in applicable metric(s).
2275 : Bytes(u64),
2276 : }
2277 :
2278 : impl RemoteTimelineClientMetrics {
2279 : /// Update the metrics that change when a call to the remote timeline client instance starts.
2280 : ///
2281 : /// Drop the returned guard object once the operation is finished to updates corresponding metrics that track completions.
2282 : /// Or, use [`RemoteTimelineClientCallMetricGuard::will_decrement_manually`] and [`call_end`](Self::call_end) if that
2283 : /// is more suitable.
2284 : /// Never do both.
2285 43923 : pub(crate) fn call_begin(
2286 43923 : &self,
2287 43923 : file_kind: &RemoteOpFileKind,
2288 43923 : op_kind: &RemoteOpKind,
2289 43923 : size: RemoteTimelineClientMetricsCallTrackSize,
2290 43923 : ) -> RemoteTimelineClientCallMetricGuard {
2291 43923 : let calls_unfinished_metric = self.calls_unfinished_gauge(file_kind, op_kind);
2292 43923 : self.calls_started_hist(file_kind, op_kind)
2293 43923 : .observe(calls_unfinished_metric.get() as f64);
2294 43923 : calls_unfinished_metric.inc(); // NB: inc after the histogram, see comment on underlying metric
2295 :
2296 43923 : let bytes_finished = match size {
2297 21589 : RemoteTimelineClientMetricsCallTrackSize::DontTrackSize { reason: _reason } => {
2298 21589 : // nothing to do
2299 21589 : None
2300 : }
2301 22334 : RemoteTimelineClientMetricsCallTrackSize::Bytes(size) => {
2302 22334 : self.bytes_started_counter(file_kind, op_kind).inc_by(size);
2303 22334 : let finished_counter = self.bytes_finished_counter(file_kind, op_kind);
2304 22334 : Some((finished_counter, size))
2305 : }
2306 : };
2307 43923 : RemoteTimelineClientCallMetricGuard {
2308 43923 : calls_unfinished_metric: Some(calls_unfinished_metric),
2309 43923 : bytes_finished,
2310 43923 : }
2311 43923 : }
2312 :
2313 : /// Manually udpate the metrics that track completions, instead of using the guard object.
2314 : /// Using the guard object is generally preferable.
2315 : /// See [`call_begin`](Self::call_begin) for more context.
2316 31820 : pub(crate) fn call_end(
2317 31820 : &self,
2318 31820 : file_kind: &RemoteOpFileKind,
2319 31820 : op_kind: &RemoteOpKind,
2320 31820 : size: RemoteTimelineClientMetricsCallTrackSize,
2321 31820 : ) {
2322 31820 : let calls_unfinished_metric = self.calls_unfinished_gauge(file_kind, op_kind);
2323 : debug_assert!(
2324 31820 : calls_unfinished_metric.get() > 0,
2325 0 : "begin and end should cancel out"
2326 : );
2327 31820 : calls_unfinished_metric.dec();
2328 31820 : match size {
2329 11581 : RemoteTimelineClientMetricsCallTrackSize::DontTrackSize { reason: _reason } => {}
2330 20239 : RemoteTimelineClientMetricsCallTrackSize::Bytes(size) => {
2331 20239 : self.bytes_finished_counter(file_kind, op_kind).inc_by(size);
2332 20239 : }
2333 : }
2334 31820 : }
2335 : }
2336 :
2337 : impl Drop for RemoteTimelineClientMetrics {
2338 359 : fn drop(&mut self) {
2339 359 : let RemoteTimelineClientMetrics {
2340 359 : tenant_id,
2341 359 : shard_id,
2342 359 : timeline_id,
2343 359 : remote_physical_size_gauge,
2344 359 : calls_unfinished_gauge,
2345 359 : bytes_started_counter,
2346 359 : bytes_finished_counter,
2347 359 : } = self;
2348 877 : for ((a, b), _) in calls_unfinished_gauge.get_mut().unwrap().drain() {
2349 877 : let _ = REMOTE_TIMELINE_CLIENT_CALLS_UNFINISHED_GAUGE.remove_label_values(&[
2350 877 : tenant_id,
2351 877 : shard_id,
2352 877 : timeline_id,
2353 877 : a,
2354 877 : b,
2355 877 : ]);
2356 877 : }
2357 359 : for ((a, b), _) in bytes_started_counter.get_mut().unwrap().drain() {
2358 222 : let _ = REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER.remove_label_values(&[
2359 222 : tenant_id,
2360 222 : shard_id,
2361 222 : timeline_id,
2362 222 : a,
2363 222 : b,
2364 222 : ]);
2365 222 : }
2366 359 : for ((a, b), _) in bytes_finished_counter.get_mut().unwrap().drain() {
2367 222 : let _ = REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER.remove_label_values(&[
2368 222 : tenant_id,
2369 222 : shard_id,
2370 222 : timeline_id,
2371 222 : a,
2372 222 : b,
2373 222 : ]);
2374 222 : }
2375 359 : {
2376 359 : let _ = remote_physical_size_gauge; // use to avoid 'unused' warning in desctructuring above
2377 359 : let _ = REMOTE_PHYSICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2378 359 : }
2379 359 : }
2380 : }
2381 :
2382 : /// Wrapper future that measures the time spent by a remote storage operation,
2383 : /// and records the time and success/failure as a prometheus metric.
2384 : pub(crate) trait MeasureRemoteOp: Sized {
2385 40203 : fn measure_remote_op(
2386 40203 : self,
2387 40203 : file_kind: RemoteOpFileKind,
2388 40203 : op: RemoteOpKind,
2389 40203 : metrics: Arc<RemoteTimelineClientMetrics>,
2390 40203 : ) -> MeasuredRemoteOp<Self> {
2391 40203 : let start = Instant::now();
2392 40203 : MeasuredRemoteOp {
2393 40203 : inner: self,
2394 40203 : file_kind,
2395 40203 : op,
2396 40203 : start,
2397 40203 : metrics,
2398 40203 : }
2399 40203 : }
2400 : }
2401 :
2402 : impl<T: Sized> MeasureRemoteOp for T {}
2403 :
2404 : pin_project! {
2405 : pub(crate) struct MeasuredRemoteOp<F>
2406 : {
2407 : #[pin]
2408 : inner: F,
2409 : file_kind: RemoteOpFileKind,
2410 : op: RemoteOpKind,
2411 : start: Instant,
2412 : metrics: Arc<RemoteTimelineClientMetrics>,
2413 : }
2414 : }
2415 :
2416 : impl<F: Future<Output = Result<O, E>>, O, E> Future for MeasuredRemoteOp<F> {
2417 : type Output = Result<O, E>;
2418 :
2419 1529778 : fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
2420 1529778 : let this = self.project();
2421 1529778 : let poll_result = this.inner.poll(cx);
2422 1529778 : if let Poll::Ready(ref res) = poll_result {
2423 40186 : let duration = this.start.elapsed();
2424 40186 : let status = if res.is_ok() { &"success" } else { &"failure" };
2425 40186 : this.metrics
2426 40186 : .remote_operation_time(this.file_kind, this.op, status)
2427 40186 : .observe(duration.as_secs_f64());
2428 1489592 : }
2429 1529778 : poll_result
2430 1529778 : }
2431 : }
2432 :
2433 : pub mod tokio_epoll_uring {
2434 : use metrics::UIntGauge;
2435 :
2436 : pub struct Collector {
2437 : descs: Vec<metrics::core::Desc>,
2438 : systems_created: UIntGauge,
2439 : systems_destroyed: UIntGauge,
2440 : }
2441 :
2442 : const NMETRICS: usize = 2;
2443 :
2444 : impl metrics::core::Collector for Collector {
2445 625 : fn desc(&self) -> Vec<&metrics::core::Desc> {
2446 625 : self.descs.iter().collect()
2447 625 : }
2448 :
2449 1259 : fn collect(&self) -> Vec<metrics::proto::MetricFamily> {
2450 1259 : let mut mfs = Vec::with_capacity(NMETRICS);
2451 1259 : let tokio_epoll_uring::metrics::Metrics {
2452 1259 : systems_created,
2453 1259 : systems_destroyed,
2454 1259 : } = tokio_epoll_uring::metrics::global();
2455 1259 : self.systems_created.set(systems_created);
2456 1259 : mfs.extend(self.systems_created.collect());
2457 1259 : self.systems_destroyed.set(systems_destroyed);
2458 1259 : mfs.extend(self.systems_destroyed.collect());
2459 1259 : mfs
2460 1259 : }
2461 : }
2462 :
2463 : impl Collector {
2464 : #[allow(clippy::new_without_default)]
2465 625 : pub fn new() -> Self {
2466 625 : let mut descs = Vec::new();
2467 625 :
2468 625 : let systems_created = UIntGauge::new(
2469 625 : "pageserver_tokio_epoll_uring_systems_created",
2470 625 : "counter of tokio-epoll-uring systems that were created",
2471 625 : )
2472 625 : .unwrap();
2473 625 : descs.extend(
2474 625 : metrics::core::Collector::desc(&systems_created)
2475 625 : .into_iter()
2476 625 : .cloned(),
2477 625 : );
2478 625 :
2479 625 : let systems_destroyed = UIntGauge::new(
2480 625 : "pageserver_tokio_epoll_uring_systems_destroyed",
2481 625 : "counter of tokio-epoll-uring systems that were destroyed",
2482 625 : )
2483 625 : .unwrap();
2484 625 : descs.extend(
2485 625 : metrics::core::Collector::desc(&systems_destroyed)
2486 625 : .into_iter()
2487 625 : .cloned(),
2488 625 : );
2489 625 :
2490 625 : Self {
2491 625 : descs,
2492 625 : systems_created,
2493 625 : systems_destroyed,
2494 625 : }
2495 625 : }
2496 : }
2497 : }
2498 :
2499 625 : pub fn preinitialize_metrics() {
2500 625 : // Python tests need these and on some we do alerting.
2501 625 : //
2502 625 : // FIXME(4813): make it so that we have no top level metrics as this fn will easily fall out of
2503 625 : // order:
2504 625 : // - global metrics reside in a Lazy<PageserverMetrics>
2505 625 : // - access via crate::metrics::PS_METRICS.materialized_page_cache_hit.inc()
2506 625 : // - could move the statics into TimelineMetrics::new()?
2507 625 :
2508 625 : // counters
2509 625 : [
2510 625 : &MATERIALIZED_PAGE_CACHE_HIT,
2511 625 : &MATERIALIZED_PAGE_CACHE_HIT_DIRECT,
2512 625 : &UNEXPECTED_ONDEMAND_DOWNLOADS,
2513 625 : &WALRECEIVER_STARTED_CONNECTIONS,
2514 625 : &WALRECEIVER_BROKER_UPDATES,
2515 625 : &WALRECEIVER_CANDIDATES_ADDED,
2516 625 : &WALRECEIVER_CANDIDATES_REMOVED,
2517 625 : ]
2518 625 : .into_iter()
2519 4375 : .for_each(|c| {
2520 4375 : Lazy::force(c);
2521 4375 : });
2522 625 :
2523 625 : // Deletion queue stats
2524 625 : Lazy::force(&DELETION_QUEUE);
2525 625 :
2526 625 : // Tenant stats
2527 625 : Lazy::force(&TENANT);
2528 625 :
2529 625 : // Tenant manager stats
2530 625 : Lazy::force(&TENANT_MANAGER);
2531 625 :
2532 625 : Lazy::force(&crate::tenant::storage_layer::layer::LAYER_IMPL_METRICS);
2533 625 :
2534 625 : // countervecs
2535 625 : [&BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT]
2536 625 : .into_iter()
2537 625 : .for_each(|c| {
2538 625 : Lazy::force(c);
2539 625 : });
2540 625 :
2541 625 : // gauges
2542 625 : WALRECEIVER_ACTIVE_MANAGERS.get();
2543 625 :
2544 625 : // histograms
2545 625 : [
2546 625 : &READ_NUM_FS_LAYERS,
2547 625 : &WAIT_LSN_TIME,
2548 625 : &WAL_REDO_TIME,
2549 625 : &WAL_REDO_RECORDS_HISTOGRAM,
2550 625 : &WAL_REDO_BYTES_HISTOGRAM,
2551 625 : &WAL_REDO_PROCESS_LAUNCH_DURATION_HISTOGRAM,
2552 625 : ]
2553 625 : .into_iter()
2554 3750 : .for_each(|h| {
2555 3750 : Lazy::force(h);
2556 3750 : });
2557 625 :
2558 625 : // Custom
2559 625 : Lazy::force(&RECONSTRUCT_TIME);
2560 625 : }
|
