Line data Source code
1 : use enum_map::EnumMap;
2 : use metrics::{
3 : register_counter_vec, register_gauge_vec, register_histogram, register_histogram_vec,
4 : register_int_counter, register_int_counter_pair_vec, register_int_counter_vec,
5 : register_int_gauge, register_int_gauge_vec, register_uint_gauge, register_uint_gauge_vec,
6 : Counter, CounterVec, GaugeVec, Histogram, HistogramVec, IntCounter, IntCounterPair,
7 : IntCounterPairVec, IntCounterVec, IntGauge, IntGaugeVec, UIntGauge, UIntGaugeVec,
8 : };
9 : use once_cell::sync::Lazy;
10 : use pageserver_api::shard::TenantShardId;
11 : use strum::{EnumCount, IntoEnumIterator, VariantNames};
12 : use strum_macros::{EnumVariantNames, IntoStaticStr};
13 : use tracing::warn;
14 : use utils::id::TimelineId;
15 :
16 : /// Prometheus histogram buckets (in seconds) for operations in the critical
17 : /// path. In other words, operations that directly affect that latency of user
18 : /// queries.
19 : ///
20 : /// The buckets capture the majority of latencies in the microsecond and
21 : /// millisecond range but also extend far enough up to distinguish "bad" from
22 : /// "really bad".
23 : const CRITICAL_OP_BUCKETS: &[f64] = &[
24 : 0.000_001, 0.000_010, 0.000_100, // 1 us, 10 us, 100 us
25 : 0.001_000, 0.010_000, 0.100_000, // 1 ms, 10 ms, 100 ms
26 : 1.0, 10.0, 100.0, // 1 s, 10 s, 100 s
27 : ];
28 :
29 : // Metrics collected on operations on the storage repository.
30 3040 : #[derive(Debug, EnumVariantNames, IntoStaticStr)]
31 : #[strum(serialize_all = "kebab_case")]
32 : pub(crate) enum StorageTimeOperation {
33 : #[strum(serialize = "layer flush")]
34 : LayerFlush,
35 :
36 : #[strum(serialize = "compact")]
37 : Compact,
38 :
39 : #[strum(serialize = "create images")]
40 : CreateImages,
41 :
42 : #[strum(serialize = "logical size")]
43 : LogicalSize,
44 :
45 : #[strum(serialize = "imitate logical size")]
46 : ImitateLogicalSize,
47 :
48 : #[strum(serialize = "load layer map")]
49 : LoadLayerMap,
50 :
51 : #[strum(serialize = "gc")]
52 : Gc,
53 :
54 : #[strum(serialize = "find gc cutoffs")]
55 : FindGcCutoffs,
56 :
57 : #[strum(serialize = "create tenant")]
58 : CreateTenant,
59 : }
60 :
61 138 : pub(crate) static STORAGE_TIME_SUM_PER_TIMELINE: Lazy<CounterVec> = Lazy::new(|| {
62 : register_counter_vec!(
63 : "pageserver_storage_operations_seconds_sum",
64 : "Total time spent on storage operations with operation, tenant and timeline dimensions",
65 : &["operation", "tenant_id", "shard_id", "timeline_id"],
66 : )
67 138 : .expect("failed to define a metric")
68 138 : });
69 :
70 138 : pub(crate) static STORAGE_TIME_COUNT_PER_TIMELINE: Lazy<IntCounterVec> = Lazy::new(|| {
71 : register_int_counter_vec!(
72 : "pageserver_storage_operations_seconds_count",
73 : "Count of storage operations with operation, tenant and timeline dimensions",
74 : &["operation", "tenant_id", "shard_id", "timeline_id"],
75 : )
76 138 : .expect("failed to define a metric")
77 138 : });
78 :
79 : // Buckets for background operations like compaction, GC, size calculation
80 : const STORAGE_OP_BUCKETS: &[f64] = &[0.010, 0.100, 1.0, 10.0, 100.0, 1000.0];
81 :
82 138 : pub(crate) static STORAGE_TIME_GLOBAL: Lazy<HistogramVec> = Lazy::new(|| {
83 : register_histogram_vec!(
84 : "pageserver_storage_operations_seconds_global",
85 : "Time spent on storage operations",
86 : &["operation"],
87 : STORAGE_OP_BUCKETS.into(),
88 : )
89 138 : .expect("failed to define a metric")
90 138 : });
91 :
92 134 : pub(crate) static READ_NUM_LAYERS_VISITED: Lazy<Histogram> = Lazy::new(|| {
93 : register_histogram!(
94 : "pageserver_layers_visited_per_read_global",
95 : "Number of layers visited to reconstruct one key",
96 : vec![1.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0],
97 : )
98 134 : .expect("failed to define a metric")
99 134 : });
100 :
101 24 : pub(crate) static VEC_READ_NUM_LAYERS_VISITED: Lazy<Histogram> = Lazy::new(|| {
102 : register_histogram!(
103 : "pageserver_layers_visited_per_vectored_read_global",
104 : "Average number of layers visited to reconstruct one key",
105 : vec![1.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0],
106 : )
107 24 : .expect("failed to define a metric")
108 24 : });
109 :
110 : // Metrics collected on operations on the storage repository.
111 : #[derive(
112 536 : Clone, Copy, enum_map::Enum, strum_macros::EnumString, strum_macros::Display, IntoStaticStr,
113 : )]
114 : pub(crate) enum GetKind {
115 : Singular,
116 : Vectored,
117 : }
118 :
119 : pub(crate) struct ReconstructTimeMetrics {
120 : singular: Histogram,
121 : vectored: Histogram,
122 : }
123 :
124 134 : pub(crate) static RECONSTRUCT_TIME: Lazy<ReconstructTimeMetrics> = Lazy::new(|| {
125 134 : let inner = register_histogram_vec!(
126 : "pageserver_getpage_reconstruct_seconds",
127 : "Time spent in reconstruct_value (reconstruct a page from deltas)",
128 : &["get_kind"],
129 : CRITICAL_OP_BUCKETS.into(),
130 : )
131 134 : .expect("failed to define a metric");
132 134 :
133 134 : ReconstructTimeMetrics {
134 134 : singular: inner.with_label_values(&[GetKind::Singular.into()]),
135 134 : vectored: inner.with_label_values(&[GetKind::Vectored.into()]),
136 134 : }
137 134 : });
138 :
139 : impl ReconstructTimeMetrics {
140 625691 : pub(crate) fn for_get_kind(&self, get_kind: GetKind) -> &Histogram {
141 625691 : match get_kind {
142 625621 : GetKind::Singular => &self.singular,
143 70 : GetKind::Vectored => &self.vectored,
144 : }
145 625691 : }
146 : }
147 :
148 0 : pub(crate) static MATERIALIZED_PAGE_CACHE_HIT_DIRECT: Lazy<IntCounter> = Lazy::new(|| {
149 : register_int_counter!(
150 : "pageserver_materialized_cache_hits_direct_total",
151 : "Number of cache hits from materialized page cache without redo",
152 : )
153 0 : .expect("failed to define a metric")
154 0 : });
155 :
156 : pub(crate) struct ReconstructDataTimeMetrics {
157 : singular: Histogram,
158 : vectored: Histogram,
159 : }
160 :
161 : impl ReconstructDataTimeMetrics {
162 625831 : pub(crate) fn for_get_kind(&self, get_kind: GetKind) -> &Histogram {
163 625831 : match get_kind {
164 625761 : GetKind::Singular => &self.singular,
165 70 : GetKind::Vectored => &self.vectored,
166 : }
167 625831 : }
168 : }
169 :
170 134 : pub(crate) static GET_RECONSTRUCT_DATA_TIME: Lazy<ReconstructDataTimeMetrics> = Lazy::new(|| {
171 134 : let inner = register_histogram_vec!(
172 : "pageserver_getpage_get_reconstruct_data_seconds",
173 : "Time spent in get_reconstruct_value_data",
174 : &["get_kind"],
175 : CRITICAL_OP_BUCKETS.into(),
176 : )
177 134 : .expect("failed to define a metric");
178 134 :
179 134 : ReconstructDataTimeMetrics {
180 134 : singular: inner.with_label_values(&[GetKind::Singular.into()]),
181 134 : vectored: inner.with_label_values(&[GetKind::Vectored.into()]),
182 134 : }
183 134 : });
184 :
185 0 : pub(crate) static MATERIALIZED_PAGE_CACHE_HIT: Lazy<IntCounter> = Lazy::new(|| {
186 : register_int_counter!(
187 : "pageserver_materialized_cache_hits_total",
188 : "Number of cache hits from materialized page cache",
189 : )
190 0 : .expect("failed to define a metric")
191 0 : });
192 :
193 : pub(crate) struct GetVectoredLatency {
194 : map: EnumMap<TaskKind, Option<Histogram>>,
195 : }
196 :
197 : #[allow(dead_code)]
198 : pub(crate) struct ScanLatency {
199 : map: EnumMap<TaskKind, Option<Histogram>>,
200 : }
201 :
202 : impl GetVectoredLatency {
203 : // Only these task types perform vectored gets. Filter all other tasks out to reduce total
204 : // cardinality of the metric.
205 : const TRACKED_TASK_KINDS: [TaskKind; 2] = [TaskKind::Compaction, TaskKind::PageRequestHandler];
206 :
207 928 : pub(crate) fn for_task_kind(&self, task_kind: TaskKind) -> Option<&Histogram> {
208 928 : self.map[task_kind].as_ref()
209 928 : }
210 : }
211 :
212 : impl ScanLatency {
213 : // Only these task types perform vectored gets. Filter all other tasks out to reduce total
214 : // cardinality of the metric.
215 : const TRACKED_TASK_KINDS: [TaskKind; 1] = [TaskKind::PageRequestHandler];
216 :
217 12 : pub(crate) fn for_task_kind(&self, task_kind: TaskKind) -> Option<&Histogram> {
218 12 : self.map[task_kind].as_ref()
219 12 : }
220 : }
221 :
222 : pub(crate) struct ScanLatencyOngoingRecording<'a> {
223 : parent: &'a Histogram,
224 : start: std::time::Instant,
225 : }
226 :
227 : impl<'a> ScanLatencyOngoingRecording<'a> {
228 0 : pub(crate) fn start_recording(parent: &'a Histogram) -> ScanLatencyOngoingRecording<'a> {
229 0 : let start = Instant::now();
230 0 : ScanLatencyOngoingRecording { parent, start }
231 0 : }
232 :
233 0 : pub(crate) fn observe(self, throttled: Option<Duration>) {
234 0 : let elapsed = self.start.elapsed();
235 0 : let ex_throttled = if let Some(throttled) = throttled {
236 0 : elapsed.checked_sub(throttled)
237 : } else {
238 0 : Some(elapsed)
239 : };
240 0 : if let Some(ex_throttled) = ex_throttled {
241 0 : self.parent.observe(ex_throttled.as_secs_f64());
242 0 : } else {
243 0 : use utils::rate_limit::RateLimit;
244 0 : static LOGGED: Lazy<Mutex<RateLimit>> =
245 0 : Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(10))));
246 0 : let mut rate_limit = LOGGED.lock().unwrap();
247 0 : rate_limit.call(|| {
248 0 : warn!("error deducting time spent throttled; this message is logged at a global rate limit");
249 0 : });
250 0 : }
251 0 : }
252 : }
253 :
254 128 : pub(crate) static GET_VECTORED_LATENCY: Lazy<GetVectoredLatency> = Lazy::new(|| {
255 128 : let inner = register_histogram_vec!(
256 : "pageserver_get_vectored_seconds",
257 : "Time spent in get_vectored, excluding time spent in timeline_get_throttle.",
258 : &["task_kind"],
259 : CRITICAL_OP_BUCKETS.into(),
260 : )
261 128 : .expect("failed to define a metric");
262 128 :
263 128 : GetVectoredLatency {
264 3840 : map: EnumMap::from_array(std::array::from_fn(|task_kind_idx| {
265 3840 : let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind_idx);
266 3840 :
267 3840 : if GetVectoredLatency::TRACKED_TASK_KINDS.contains(&task_kind) {
268 256 : let task_kind = task_kind.into();
269 256 : Some(inner.with_label_values(&[task_kind]))
270 : } else {
271 3584 : None
272 : }
273 3840 : })),
274 128 : }
275 128 : });
276 :
277 4 : pub(crate) static SCAN_LATENCY: Lazy<ScanLatency> = Lazy::new(|| {
278 4 : let inner = register_histogram_vec!(
279 : "pageserver_scan_seconds",
280 : "Time spent in scan, excluding time spent in timeline_get_throttle.",
281 : &["task_kind"],
282 : CRITICAL_OP_BUCKETS.into(),
283 : )
284 4 : .expect("failed to define a metric");
285 4 :
286 4 : ScanLatency {
287 120 : map: EnumMap::from_array(std::array::from_fn(|task_kind_idx| {
288 120 : let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind_idx);
289 120 :
290 120 : if ScanLatency::TRACKED_TASK_KINDS.contains(&task_kind) {
291 4 : let task_kind = task_kind.into();
292 4 : Some(inner.with_label_values(&[task_kind]))
293 : } else {
294 116 : None
295 : }
296 120 : })),
297 4 : }
298 4 : });
299 :
300 : pub(crate) struct PageCacheMetricsForTaskKind {
301 : pub read_accesses_materialized_page: IntCounter,
302 : pub read_accesses_immutable: IntCounter,
303 :
304 : pub read_hits_immutable: IntCounter,
305 : pub read_hits_materialized_page_exact: IntCounter,
306 : pub read_hits_materialized_page_older_lsn: IntCounter,
307 : }
308 :
309 : pub(crate) struct PageCacheMetrics {
310 : map: EnumMap<TaskKind, EnumMap<PageContentKind, PageCacheMetricsForTaskKind>>,
311 : }
312 :
313 136 : static PAGE_CACHE_READ_HITS: Lazy<IntCounterVec> = Lazy::new(|| {
314 : register_int_counter_vec!(
315 : "pageserver_page_cache_read_hits_total",
316 : "Number of read accesses to the page cache that hit",
317 : &["task_kind", "key_kind", "content_kind", "hit_kind"]
318 : )
319 136 : .expect("failed to define a metric")
320 136 : });
321 :
322 136 : static PAGE_CACHE_READ_ACCESSES: Lazy<IntCounterVec> = Lazy::new(|| {
323 : register_int_counter_vec!(
324 : "pageserver_page_cache_read_accesses_total",
325 : "Number of read accesses to the page cache",
326 : &["task_kind", "key_kind", "content_kind"]
327 : )
328 136 : .expect("failed to define a metric")
329 136 : });
330 :
331 136 : pub(crate) static PAGE_CACHE: Lazy<PageCacheMetrics> = Lazy::new(|| PageCacheMetrics {
332 4080 : map: EnumMap::from_array(std::array::from_fn(|task_kind| {
333 4080 : let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind);
334 4080 : let task_kind: &'static str = task_kind.into();
335 24480 : EnumMap::from_array(std::array::from_fn(|content_kind| {
336 24480 : let content_kind = <PageContentKind as enum_map::Enum>::from_usize(content_kind);
337 24480 : let content_kind: &'static str = content_kind.into();
338 24480 : PageCacheMetricsForTaskKind {
339 24480 : read_accesses_materialized_page: {
340 24480 : PAGE_CACHE_READ_ACCESSES
341 24480 : .get_metric_with_label_values(&[
342 24480 : task_kind,
343 24480 : "materialized_page",
344 24480 : content_kind,
345 24480 : ])
346 24480 : .unwrap()
347 24480 : },
348 24480 :
349 24480 : read_accesses_immutable: {
350 24480 : PAGE_CACHE_READ_ACCESSES
351 24480 : .get_metric_with_label_values(&[task_kind, "immutable", content_kind])
352 24480 : .unwrap()
353 24480 : },
354 24480 :
355 24480 : read_hits_immutable: {
356 24480 : PAGE_CACHE_READ_HITS
357 24480 : .get_metric_with_label_values(&[task_kind, "immutable", content_kind, "-"])
358 24480 : .unwrap()
359 24480 : },
360 24480 :
361 24480 : read_hits_materialized_page_exact: {
362 24480 : PAGE_CACHE_READ_HITS
363 24480 : .get_metric_with_label_values(&[
364 24480 : task_kind,
365 24480 : "materialized_page",
366 24480 : content_kind,
367 24480 : "exact",
368 24480 : ])
369 24480 : .unwrap()
370 24480 : },
371 24480 :
372 24480 : read_hits_materialized_page_older_lsn: {
373 24480 : PAGE_CACHE_READ_HITS
374 24480 : .get_metric_with_label_values(&[
375 24480 : task_kind,
376 24480 : "materialized_page",
377 24480 : content_kind,
378 24480 : "older_lsn",
379 24480 : ])
380 24480 : .unwrap()
381 24480 : },
382 24480 : }
383 24480 : }))
384 4080 : })),
385 136 : });
386 :
387 : impl PageCacheMetrics {
388 14815955 : pub(crate) fn for_ctx(&self, ctx: &RequestContext) -> &PageCacheMetricsForTaskKind {
389 14815955 : &self.map[ctx.task_kind()][ctx.page_content_kind()]
390 14815955 : }
391 : }
392 :
393 : pub(crate) struct PageCacheSizeMetrics {
394 : pub max_bytes: UIntGauge,
395 :
396 : pub current_bytes_immutable: UIntGauge,
397 : pub current_bytes_materialized_page: UIntGauge,
398 : }
399 :
400 136 : static PAGE_CACHE_SIZE_CURRENT_BYTES: Lazy<UIntGaugeVec> = Lazy::new(|| {
401 : register_uint_gauge_vec!(
402 : "pageserver_page_cache_size_current_bytes",
403 : "Current size of the page cache in bytes, by key kind",
404 : &["key_kind"]
405 : )
406 136 : .expect("failed to define a metric")
407 136 : });
408 :
409 : pub(crate) static PAGE_CACHE_SIZE: Lazy<PageCacheSizeMetrics> =
410 136 : Lazy::new(|| PageCacheSizeMetrics {
411 : max_bytes: {
412 : register_uint_gauge!(
413 : "pageserver_page_cache_size_max_bytes",
414 : "Maximum size of the page cache in bytes"
415 : )
416 136 : .expect("failed to define a metric")
417 136 : },
418 136 : current_bytes_immutable: {
419 136 : PAGE_CACHE_SIZE_CURRENT_BYTES
420 136 : .get_metric_with_label_values(&["immutable"])
421 136 : .unwrap()
422 136 : },
423 136 : current_bytes_materialized_page: {
424 136 : PAGE_CACHE_SIZE_CURRENT_BYTES
425 136 : .get_metric_with_label_values(&["materialized_page"])
426 136 : .unwrap()
427 136 : },
428 136 : });
429 :
430 : pub(crate) mod page_cache_eviction_metrics {
431 : use std::num::NonZeroUsize;
432 :
433 : use metrics::{register_int_counter_vec, IntCounter, IntCounterVec};
434 : use once_cell::sync::Lazy;
435 :
436 : #[derive(Clone, Copy)]
437 : pub(crate) enum Outcome {
438 : FoundSlotUnused { iters: NonZeroUsize },
439 : FoundSlotEvicted { iters: NonZeroUsize },
440 : ItersExceeded { iters: NonZeroUsize },
441 : }
442 :
443 70 : static ITERS_TOTAL_VEC: Lazy<IntCounterVec> = Lazy::new(|| {
444 : register_int_counter_vec!(
445 : "pageserver_page_cache_find_victim_iters_total",
446 : "Counter for the number of iterations in the find_victim loop",
447 : &["outcome"],
448 : )
449 70 : .expect("failed to define a metric")
450 70 : });
451 :
452 70 : static CALLS_VEC: Lazy<IntCounterVec> = Lazy::new(|| {
453 : register_int_counter_vec!(
454 : "pageserver_page_cache_find_victim_calls",
455 : "Incremented at the end of each find_victim() call.\
456 : Filter by outcome to get e.g., eviction rate.",
457 : &["outcome"]
458 : )
459 70 : .unwrap()
460 70 : });
461 :
462 166751 : pub(crate) fn observe(outcome: Outcome) {
463 166751 : macro_rules! dry {
464 166751 : ($label:literal, $iters:expr) => {{
465 166751 : static LABEL: &'static str = $label;
466 166751 : static ITERS_TOTAL: Lazy<IntCounter> =
467 166751 : Lazy::new(|| ITERS_TOTAL_VEC.with_label_values(&[LABEL]));
468 166751 : static CALLS: Lazy<IntCounter> =
469 166751 : Lazy::new(|| CALLS_VEC.with_label_values(&[LABEL]));
470 166751 : ITERS_TOTAL.inc_by(($iters.get()) as u64);
471 166751 : CALLS.inc();
472 166751 : }};
473 166751 : }
474 166751 : match outcome {
475 1832 : Outcome::FoundSlotUnused { iters } => dry!("found_empty", iters),
476 164919 : Outcome::FoundSlotEvicted { iters } => {
477 164919 : dry!("found_evicted", iters)
478 : }
479 0 : Outcome::ItersExceeded { iters } => {
480 0 : dry!("err_iters_exceeded", iters);
481 0 : super::page_cache_errors_inc(super::PageCacheErrorKind::EvictIterLimit);
482 0 : }
483 : }
484 166751 : }
485 : }
486 :
487 0 : static PAGE_CACHE_ERRORS: Lazy<IntCounterVec> = Lazy::new(|| {
488 : register_int_counter_vec!(
489 : "page_cache_errors_total",
490 : "Number of timeouts while acquiring a pinned slot in the page cache",
491 : &["error_kind"]
492 : )
493 0 : .expect("failed to define a metric")
494 0 : });
495 :
496 0 : #[derive(IntoStaticStr)]
497 : #[strum(serialize_all = "kebab_case")]
498 : pub(crate) enum PageCacheErrorKind {
499 : AcquirePinnedSlotTimeout,
500 : EvictIterLimit,
501 : }
502 :
503 0 : pub(crate) fn page_cache_errors_inc(error_kind: PageCacheErrorKind) {
504 0 : PAGE_CACHE_ERRORS
505 0 : .get_metric_with_label_values(&[error_kind.into()])
506 0 : .unwrap()
507 0 : .inc();
508 0 : }
509 :
510 16 : pub(crate) static WAIT_LSN_TIME: Lazy<Histogram> = Lazy::new(|| {
511 : register_histogram!(
512 : "pageserver_wait_lsn_seconds",
513 : "Time spent waiting for WAL to arrive",
514 : CRITICAL_OP_BUCKETS.into(),
515 : )
516 16 : .expect("failed to define a metric")
517 16 : });
518 :
519 138 : static LAST_RECORD_LSN: Lazy<IntGaugeVec> = Lazy::new(|| {
520 : register_int_gauge_vec!(
521 : "pageserver_last_record_lsn",
522 : "Last record LSN grouped by timeline",
523 : &["tenant_id", "shard_id", "timeline_id"]
524 : )
525 138 : .expect("failed to define a metric")
526 138 : });
527 :
528 138 : static STANDBY_HORIZON: Lazy<IntGaugeVec> = Lazy::new(|| {
529 : register_int_gauge_vec!(
530 : "pageserver_standby_horizon",
531 : "Standby apply LSN for which GC is hold off, by timeline.",
532 : &["tenant_id", "shard_id", "timeline_id"]
533 : )
534 138 : .expect("failed to define a metric")
535 138 : });
536 :
537 138 : static RESIDENT_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
538 : register_uint_gauge_vec!(
539 : "pageserver_resident_physical_size",
540 : "The size of the layer files present in the pageserver's filesystem.",
541 : &["tenant_id", "shard_id", "timeline_id"]
542 : )
543 138 : .expect("failed to define a metric")
544 138 : });
545 :
546 132 : pub(crate) static RESIDENT_PHYSICAL_SIZE_GLOBAL: Lazy<UIntGauge> = Lazy::new(|| {
547 : register_uint_gauge!(
548 : "pageserver_resident_physical_size_global",
549 : "Like `pageserver_resident_physical_size`, but without tenant/timeline dimensions."
550 : )
551 132 : .expect("failed to define a metric")
552 132 : });
553 :
554 138 : static REMOTE_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
555 : register_uint_gauge_vec!(
556 : "pageserver_remote_physical_size",
557 : "The size of the layer files present in the remote storage that are listed in the remote index_part.json.",
558 : // Corollary: If any files are missing from the index part, they won't be included here.
559 : &["tenant_id", "shard_id", "timeline_id"]
560 : )
561 138 : .expect("failed to define a metric")
562 138 : });
563 :
564 138 : static REMOTE_PHYSICAL_SIZE_GLOBAL: Lazy<UIntGauge> = Lazy::new(|| {
565 : register_uint_gauge!(
566 : "pageserver_remote_physical_size_global",
567 : "Like `pageserver_remote_physical_size`, but without tenant/timeline dimensions."
568 : )
569 138 : .expect("failed to define a metric")
570 138 : });
571 :
572 4 : pub(crate) static REMOTE_ONDEMAND_DOWNLOADED_LAYERS: Lazy<IntCounter> = Lazy::new(|| {
573 : register_int_counter!(
574 : "pageserver_remote_ondemand_downloaded_layers_total",
575 : "Total on-demand downloaded layers"
576 : )
577 4 : .unwrap()
578 4 : });
579 :
580 4 : pub(crate) static REMOTE_ONDEMAND_DOWNLOADED_BYTES: Lazy<IntCounter> = Lazy::new(|| {
581 : register_int_counter!(
582 : "pageserver_remote_ondemand_downloaded_bytes_total",
583 : "Total bytes of layers on-demand downloaded",
584 : )
585 4 : .unwrap()
586 4 : });
587 :
588 138 : static CURRENT_LOGICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
589 : register_uint_gauge_vec!(
590 : "pageserver_current_logical_size",
591 : "Current logical size grouped by timeline",
592 : &["tenant_id", "shard_id", "timeline_id"]
593 : )
594 138 : .expect("failed to define current logical size metric")
595 138 : });
596 :
597 138 : static AUX_FILE_SIZE: Lazy<IntGaugeVec> = Lazy::new(|| {
598 : register_int_gauge_vec!(
599 : "pageserver_aux_file_estimated_size",
600 : "The size of all aux files for a timeline in aux file v2 store.",
601 : &["tenant_id", "shard_id", "timeline_id"]
602 : )
603 138 : .expect("failed to define a metric")
604 138 : });
605 :
606 : pub(crate) mod initial_logical_size {
607 : use metrics::{register_int_counter, register_int_counter_vec, IntCounter, IntCounterVec};
608 : use once_cell::sync::Lazy;
609 :
610 : pub(crate) struct StartCalculation(IntCounterVec);
611 138 : pub(crate) static START_CALCULATION: Lazy<StartCalculation> = Lazy::new(|| {
612 138 : StartCalculation(
613 138 : register_int_counter_vec!(
614 : "pageserver_initial_logical_size_start_calculation",
615 : "Incremented each time we start an initial logical size calculation attempt. \
616 : The `circumstances` label provides some additional details.",
617 : &["attempt", "circumstances"]
618 138 : )
619 138 : .unwrap(),
620 138 : )
621 138 : });
622 :
623 : struct DropCalculation {
624 : first: IntCounter,
625 : retry: IntCounter,
626 : }
627 :
628 138 : static DROP_CALCULATION: Lazy<DropCalculation> = Lazy::new(|| {
629 138 : let vec = register_int_counter_vec!(
630 : "pageserver_initial_logical_size_drop_calculation",
631 : "Incremented each time we abort a started size calculation attmpt.",
632 : &["attempt"]
633 : )
634 138 : .unwrap();
635 138 : DropCalculation {
636 138 : first: vec.with_label_values(&["first"]),
637 138 : retry: vec.with_label_values(&["retry"]),
638 138 : }
639 138 : });
640 :
641 : pub(crate) struct Calculated {
642 : pub(crate) births: IntCounter,
643 : pub(crate) deaths: IntCounter,
644 : }
645 :
646 138 : pub(crate) static CALCULATED: Lazy<Calculated> = Lazy::new(|| Calculated {
647 : births: register_int_counter!(
648 : "pageserver_initial_logical_size_finish_calculation",
649 : "Incremented every time we finish calculation of initial logical size.\
650 : If everything is working well, this should happen at most once per Timeline object."
651 : )
652 138 : .unwrap(),
653 : deaths: register_int_counter!(
654 : "pageserver_initial_logical_size_drop_finished_calculation",
655 : "Incremented when we drop a finished initial logical size calculation result.\
656 : Mainly useful to turn pageserver_initial_logical_size_finish_calculation into a gauge."
657 : )
658 138 : .unwrap(),
659 138 : });
660 :
661 : pub(crate) struct OngoingCalculationGuard {
662 : inc_drop_calculation: Option<IntCounter>,
663 : }
664 :
665 150 : #[derive(strum_macros::IntoStaticStr)]
666 : pub(crate) enum StartCircumstances {
667 : EmptyInitial,
668 : SkippedConcurrencyLimiter,
669 : AfterBackgroundTasksRateLimit,
670 : }
671 :
672 : impl StartCalculation {
673 150 : pub(crate) fn first(&self, circumstances: StartCircumstances) -> OngoingCalculationGuard {
674 150 : let circumstances_label: &'static str = circumstances.into();
675 150 : self.0
676 150 : .with_label_values(&["first", circumstances_label])
677 150 : .inc();
678 150 : OngoingCalculationGuard {
679 150 : inc_drop_calculation: Some(DROP_CALCULATION.first.clone()),
680 150 : }
681 150 : }
682 0 : pub(crate) fn retry(&self, circumstances: StartCircumstances) -> OngoingCalculationGuard {
683 0 : let circumstances_label: &'static str = circumstances.into();
684 0 : self.0
685 0 : .with_label_values(&["retry", circumstances_label])
686 0 : .inc();
687 0 : OngoingCalculationGuard {
688 0 : inc_drop_calculation: Some(DROP_CALCULATION.retry.clone()),
689 0 : }
690 0 : }
691 : }
692 :
693 : impl Drop for OngoingCalculationGuard {
694 150 : fn drop(&mut self) {
695 150 : if let Some(counter) = self.inc_drop_calculation.take() {
696 0 : counter.inc();
697 150 : }
698 150 : }
699 : }
700 :
701 : impl OngoingCalculationGuard {
702 150 : pub(crate) fn calculation_result_saved(mut self) -> FinishedCalculationGuard {
703 150 : drop(self.inc_drop_calculation.take());
704 150 : CALCULATED.births.inc();
705 150 : FinishedCalculationGuard {
706 150 : inc_on_drop: CALCULATED.deaths.clone(),
707 150 : }
708 150 : }
709 : }
710 :
711 : pub(crate) struct FinishedCalculationGuard {
712 : inc_on_drop: IntCounter,
713 : }
714 :
715 : impl Drop for FinishedCalculationGuard {
716 8 : fn drop(&mut self) {
717 8 : self.inc_on_drop.inc();
718 8 : }
719 : }
720 :
721 : // context: https://github.com/neondatabase/neon/issues/5963
722 : pub(crate) static TIMELINES_WHERE_WALRECEIVER_GOT_APPROXIMATE_SIZE: Lazy<IntCounter> =
723 0 : Lazy::new(|| {
724 : register_int_counter!(
725 : "pageserver_initial_logical_size_timelines_where_walreceiver_got_approximate_size",
726 : "Counter for the following event: walreceiver calls\
727 : Timeline::get_current_logical_size() and it returns `Approximate` for the first time."
728 : )
729 0 : .unwrap()
730 0 : });
731 : }
732 :
733 0 : static DIRECTORY_ENTRIES_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
734 : register_uint_gauge_vec!(
735 : "pageserver_directory_entries_count",
736 : "Sum of the entries in pageserver-stored directory listings",
737 : &["tenant_id", "shard_id", "timeline_id"]
738 : )
739 0 : .expect("failed to define a metric")
740 0 : });
741 :
742 140 : pub(crate) static TENANT_STATE_METRIC: Lazy<UIntGaugeVec> = Lazy::new(|| {
743 : register_uint_gauge_vec!(
744 : "pageserver_tenant_states_count",
745 : "Count of tenants per state",
746 : &["state"]
747 : )
748 140 : .expect("Failed to register pageserver_tenant_states_count metric")
749 140 : });
750 :
751 : /// A set of broken tenants.
752 : ///
753 : /// These are expected to be so rare that a set is fine. Set as in a new timeseries per each broken
754 : /// tenant.
755 12 : pub(crate) static BROKEN_TENANTS_SET: Lazy<UIntGaugeVec> = Lazy::new(|| {
756 : register_uint_gauge_vec!(
757 : "pageserver_broken_tenants_count",
758 : "Set of broken tenants",
759 : &["tenant_id", "shard_id"]
760 : )
761 12 : .expect("Failed to register pageserver_tenant_states_count metric")
762 12 : });
763 :
764 6 : pub(crate) static TENANT_SYNTHETIC_SIZE_METRIC: Lazy<UIntGaugeVec> = Lazy::new(|| {
765 : register_uint_gauge_vec!(
766 : "pageserver_tenant_synthetic_cached_size_bytes",
767 : "Synthetic size of each tenant in bytes",
768 : &["tenant_id"]
769 : )
770 6 : .expect("Failed to register pageserver_tenant_synthetic_cached_size_bytes metric")
771 6 : });
772 :
773 0 : pub(crate) static EVICTION_ITERATION_DURATION: Lazy<HistogramVec> = Lazy::new(|| {
774 : register_histogram_vec!(
775 : "pageserver_eviction_iteration_duration_seconds_global",
776 : "Time spent on a single eviction iteration",
777 : &["period_secs", "threshold_secs"],
778 : STORAGE_OP_BUCKETS.into(),
779 : )
780 0 : .expect("failed to define a metric")
781 0 : });
782 :
783 138 : static EVICTIONS: Lazy<IntCounterVec> = Lazy::new(|| {
784 : register_int_counter_vec!(
785 : "pageserver_evictions",
786 : "Number of layers evicted from the pageserver",
787 : &["tenant_id", "shard_id", "timeline_id"]
788 : )
789 138 : .expect("failed to define a metric")
790 138 : });
791 :
792 138 : static EVICTIONS_WITH_LOW_RESIDENCE_DURATION: Lazy<IntCounterVec> = Lazy::new(|| {
793 : register_int_counter_vec!(
794 : "pageserver_evictions_with_low_residence_duration",
795 : "If a layer is evicted that was resident for less than `low_threshold`, it is counted to this counter. \
796 : Residence duration is determined using the `residence_duration_data_source`.",
797 : &["tenant_id", "shard_id", "timeline_id", "residence_duration_data_source", "low_threshold_secs"]
798 : )
799 138 : .expect("failed to define a metric")
800 138 : });
801 :
802 0 : pub(crate) static UNEXPECTED_ONDEMAND_DOWNLOADS: Lazy<IntCounter> = Lazy::new(|| {
803 : register_int_counter!(
804 : "pageserver_unexpected_ondemand_downloads_count",
805 : "Number of unexpected on-demand downloads. \
806 : We log more context for each increment, so, forgo any labels in this metric.",
807 : )
808 0 : .expect("failed to define a metric")
809 0 : });
810 :
811 : /// How long did we take to start up? Broken down by labels to describe
812 : /// different phases of startup.
813 0 : pub static STARTUP_DURATION: Lazy<GaugeVec> = Lazy::new(|| {
814 : register_gauge_vec!(
815 : "pageserver_startup_duration_seconds",
816 : "Time taken by phases of pageserver startup, in seconds",
817 : &["phase"]
818 : )
819 0 : .expect("Failed to register pageserver_startup_duration_seconds metric")
820 0 : });
821 :
822 0 : pub static STARTUP_IS_LOADING: Lazy<UIntGauge> = Lazy::new(|| {
823 : register_uint_gauge!(
824 : "pageserver_startup_is_loading",
825 : "1 while in initial startup load of tenants, 0 at other times"
826 : )
827 0 : .expect("Failed to register pageserver_startup_is_loading")
828 0 : });
829 :
830 132 : pub(crate) static TIMELINE_EPHEMERAL_BYTES: Lazy<UIntGauge> = Lazy::new(|| {
831 : register_uint_gauge!(
832 : "pageserver_timeline_ephemeral_bytes",
833 : "Total number of bytes in ephemeral layers, summed for all timelines. Approximate, lazily updated."
834 : )
835 132 : .expect("Failed to register metric")
836 132 : });
837 :
838 : /// Metrics related to the lifecycle of a [`crate::tenant::Tenant`] object: things
839 : /// like how long it took to load.
840 : ///
841 : /// Note that these are process-global metrics, _not_ per-tenant metrics. Per-tenant
842 : /// metrics are rather expensive, and usually fine grained stuff makes more sense
843 : /// at a timeline level than tenant level.
844 : pub(crate) struct TenantMetrics {
845 : /// How long did tenants take to go from construction to active state?
846 : pub(crate) activation: Histogram,
847 : pub(crate) preload: Histogram,
848 : pub(crate) attach: Histogram,
849 :
850 : /// How many tenants are included in the initial startup of the pagesrever?
851 : pub(crate) startup_scheduled: IntCounter,
852 : pub(crate) startup_complete: IntCounter,
853 : }
854 :
855 0 : pub(crate) static TENANT: Lazy<TenantMetrics> = Lazy::new(|| {
856 0 : TenantMetrics {
857 0 : activation: register_histogram!(
858 : "pageserver_tenant_activation_seconds",
859 : "Time taken by tenants to activate, in seconds",
860 : CRITICAL_OP_BUCKETS.into()
861 0 : )
862 0 : .expect("Failed to register metric"),
863 0 : preload: register_histogram!(
864 : "pageserver_tenant_preload_seconds",
865 : "Time taken by tenants to load remote metadata on startup/attach, in seconds",
866 : CRITICAL_OP_BUCKETS.into()
867 0 : )
868 0 : .expect("Failed to register metric"),
869 0 : attach: register_histogram!(
870 : "pageserver_tenant_attach_seconds",
871 : "Time taken by tenants to intialize, after remote metadata is already loaded",
872 : CRITICAL_OP_BUCKETS.into()
873 0 : )
874 0 : .expect("Failed to register metric"),
875 0 : startup_scheduled: register_int_counter!(
876 : "pageserver_tenant_startup_scheduled",
877 : "Number of tenants included in pageserver startup (doesn't count tenants attached later)"
878 0 : ).expect("Failed to register metric"),
879 0 : startup_complete: register_int_counter!(
880 : "pageserver_tenant_startup_complete",
881 : "Number of tenants that have completed warm-up, or activated on-demand during initial startup: \
882 : should eventually reach `pageserver_tenant_startup_scheduled_total`. Does not include broken \
883 : tenants: such cases will lead to this metric never reaching the scheduled count."
884 0 : ).expect("Failed to register metric"),
885 0 : }
886 0 : });
887 :
888 : /// Each `Timeline`'s [`EVICTIONS_WITH_LOW_RESIDENCE_DURATION`] metric.
889 : #[derive(Debug)]
890 : pub(crate) struct EvictionsWithLowResidenceDuration {
891 : data_source: &'static str,
892 : threshold: Duration,
893 : counter: Option<IntCounter>,
894 : }
895 :
896 : pub(crate) struct EvictionsWithLowResidenceDurationBuilder {
897 : data_source: &'static str,
898 : threshold: Duration,
899 : }
900 :
901 : impl EvictionsWithLowResidenceDurationBuilder {
902 380 : pub fn new(data_source: &'static str, threshold: Duration) -> Self {
903 380 : Self {
904 380 : data_source,
905 380 : threshold,
906 380 : }
907 380 : }
908 :
909 380 : fn build(
910 380 : &self,
911 380 : tenant_id: &str,
912 380 : shard_id: &str,
913 380 : timeline_id: &str,
914 380 : ) -> EvictionsWithLowResidenceDuration {
915 380 : let counter = EVICTIONS_WITH_LOW_RESIDENCE_DURATION
916 380 : .get_metric_with_label_values(&[
917 380 : tenant_id,
918 380 : shard_id,
919 380 : timeline_id,
920 380 : self.data_source,
921 380 : &EvictionsWithLowResidenceDuration::threshold_label_value(self.threshold),
922 380 : ])
923 380 : .unwrap();
924 380 : EvictionsWithLowResidenceDuration {
925 380 : data_source: self.data_source,
926 380 : threshold: self.threshold,
927 380 : counter: Some(counter),
928 380 : }
929 380 : }
930 : }
931 :
932 : impl EvictionsWithLowResidenceDuration {
933 388 : fn threshold_label_value(threshold: Duration) -> String {
934 388 : format!("{}", threshold.as_secs())
935 388 : }
936 :
937 18 : pub fn observe(&self, observed_value: Duration) {
938 18 : if observed_value < self.threshold {
939 18 : self.counter
940 18 : .as_ref()
941 18 : .expect("nobody calls this function after `remove_from_vec`")
942 18 : .inc();
943 18 : }
944 18 : }
945 :
946 8 : pub fn change_threshold(
947 8 : &mut self,
948 8 : tenant_id: &str,
949 8 : shard_id: &str,
950 8 : timeline_id: &str,
951 8 : new_threshold: Duration,
952 8 : ) {
953 8 : if new_threshold == self.threshold {
954 8 : return;
955 0 : }
956 0 : let mut with_new = EvictionsWithLowResidenceDurationBuilder::new(
957 0 : self.data_source,
958 0 : new_threshold,
959 0 : )
960 0 : .build(tenant_id, shard_id, timeline_id);
961 0 : std::mem::swap(self, &mut with_new);
962 0 : with_new.remove(tenant_id, shard_id, timeline_id);
963 8 : }
964 :
965 : // This could be a `Drop` impl, but, we need the `tenant_id` and `timeline_id`.
966 8 : fn remove(&mut self, tenant_id: &str, shard_id: &str, timeline_id: &str) {
967 8 : let Some(_counter) = self.counter.take() else {
968 0 : return;
969 : };
970 :
971 8 : let threshold = Self::threshold_label_value(self.threshold);
972 8 :
973 8 : let removed = EVICTIONS_WITH_LOW_RESIDENCE_DURATION.remove_label_values(&[
974 8 : tenant_id,
975 8 : shard_id,
976 8 : timeline_id,
977 8 : self.data_source,
978 8 : &threshold,
979 8 : ]);
980 8 :
981 8 : match removed {
982 0 : Err(e) => {
983 0 : // this has been hit in staging as
984 0 : // <https://neondatabase.sentry.io/issues/4142396994/>, but we don't know how.
985 0 : // because we can be in the drop path already, don't risk:
986 0 : // - "double-panic => illegal instruction" or
987 0 : // - future "drop panick => abort"
988 0 : //
989 0 : // so just nag: (the error has the labels)
990 0 : tracing::warn!("failed to remove EvictionsWithLowResidenceDuration, it was already removed? {e:#?}");
991 : }
992 : Ok(()) => {
993 : // to help identify cases where we double-remove the same values, let's log all
994 : // deletions?
995 8 : tracing::info!("removed EvictionsWithLowResidenceDuration with {tenant_id}, {timeline_id}, {}, {threshold}", self.data_source);
996 : }
997 : }
998 8 : }
999 : }
1000 :
1001 : // Metrics collected on disk IO operations
1002 : //
1003 : // Roughly logarithmic scale.
1004 : const STORAGE_IO_TIME_BUCKETS: &[f64] = &[
1005 : 0.000030, // 30 usec
1006 : 0.001000, // 1000 usec
1007 : 0.030, // 30 ms
1008 : 1.000, // 1000 ms
1009 : 30.000, // 30000 ms
1010 : ];
1011 :
1012 : /// VirtualFile fs operation variants.
1013 : ///
1014 : /// Operations:
1015 : /// - open ([`std::fs::OpenOptions::open`])
1016 : /// - close (dropping [`crate::virtual_file::VirtualFile`])
1017 : /// - close-by-replace (close by replacement algorithm)
1018 : /// - read (`read_at`)
1019 : /// - write (`write_at`)
1020 : /// - seek (modify internal position or file length query)
1021 : /// - fsync ([`std::fs::File::sync_all`])
1022 : /// - metadata ([`std::fs::File::metadata`])
1023 : #[derive(
1024 1404 : Debug, Clone, Copy, strum_macros::EnumCount, strum_macros::EnumIter, strum_macros::FromRepr,
1025 : )]
1026 : pub(crate) enum StorageIoOperation {
1027 : Open,
1028 : OpenAfterReplace,
1029 : Close,
1030 : CloseByReplace,
1031 : Read,
1032 : Write,
1033 : Seek,
1034 : Fsync,
1035 : Metadata,
1036 : }
1037 :
1038 : impl StorageIoOperation {
1039 1404 : pub fn as_str(&self) -> &'static str {
1040 1404 : match self {
1041 156 : StorageIoOperation::Open => "open",
1042 156 : StorageIoOperation::OpenAfterReplace => "open-after-replace",
1043 156 : StorageIoOperation::Close => "close",
1044 156 : StorageIoOperation::CloseByReplace => "close-by-replace",
1045 156 : StorageIoOperation::Read => "read",
1046 156 : StorageIoOperation::Write => "write",
1047 156 : StorageIoOperation::Seek => "seek",
1048 156 : StorageIoOperation::Fsync => "fsync",
1049 156 : StorageIoOperation::Metadata => "metadata",
1050 : }
1051 1404 : }
1052 : }
1053 :
1054 : /// Tracks time taken by fs operations near VirtualFile.
1055 : #[derive(Debug)]
1056 : pub(crate) struct StorageIoTime {
1057 : metrics: [Histogram; StorageIoOperation::COUNT],
1058 : }
1059 :
1060 : impl StorageIoTime {
1061 156 : fn new() -> Self {
1062 156 : let storage_io_histogram_vec = register_histogram_vec!(
1063 : "pageserver_io_operations_seconds",
1064 : "Time spent in IO operations",
1065 : &["operation"],
1066 : STORAGE_IO_TIME_BUCKETS.into()
1067 : )
1068 156 : .expect("failed to define a metric");
1069 1404 : let metrics = std::array::from_fn(|i| {
1070 1404 : let op = StorageIoOperation::from_repr(i).unwrap();
1071 1404 : storage_io_histogram_vec
1072 1404 : .get_metric_with_label_values(&[op.as_str()])
1073 1404 : .unwrap()
1074 1404 : });
1075 156 : Self { metrics }
1076 156 : }
1077 :
1078 1880607 : pub(crate) fn get(&self, op: StorageIoOperation) -> &Histogram {
1079 1880607 : &self.metrics[op as usize]
1080 1880607 : }
1081 : }
1082 :
1083 : pub(crate) static STORAGE_IO_TIME_METRIC: Lazy<StorageIoTime> = Lazy::new(StorageIoTime::new);
1084 :
1085 : const STORAGE_IO_SIZE_OPERATIONS: &[&str] = &["read", "write"];
1086 :
1087 : // Needed for the https://neonprod.grafana.net/d/5uK9tHL4k/picking-tenant-for-relocation?orgId=1
1088 154 : pub(crate) static STORAGE_IO_SIZE: Lazy<IntGaugeVec> = Lazy::new(|| {
1089 : register_int_gauge_vec!(
1090 : "pageserver_io_operations_bytes_total",
1091 : "Total amount of bytes read/written in IO operations",
1092 : &["operation", "tenant_id", "shard_id", "timeline_id"]
1093 : )
1094 154 : .expect("failed to define a metric")
1095 154 : });
1096 :
1097 : #[cfg(not(test))]
1098 : pub(crate) mod virtual_file_descriptor_cache {
1099 : use super::*;
1100 :
1101 0 : pub(crate) static SIZE_MAX: Lazy<UIntGauge> = Lazy::new(|| {
1102 : register_uint_gauge!(
1103 : "pageserver_virtual_file_descriptor_cache_size_max",
1104 : "Maximum number of open file descriptors in the cache."
1105 : )
1106 0 : .unwrap()
1107 0 : });
1108 :
1109 : // SIZE_CURRENT: derive it like so:
1110 : // ```
1111 : // sum (pageserver_io_operations_seconds_count{operation=~"^(open|open-after-replace)$")
1112 : // -ignoring(operation)
1113 : // sum(pageserver_io_operations_seconds_count{operation=~"^(close|close-by-replace)$"}
1114 : // ```
1115 : }
1116 :
1117 : #[cfg(not(test))]
1118 : pub(crate) mod virtual_file_io_engine {
1119 : use super::*;
1120 :
1121 0 : pub(crate) static KIND: Lazy<UIntGaugeVec> = Lazy::new(|| {
1122 : register_uint_gauge_vec!(
1123 : "pageserver_virtual_file_io_engine_kind",
1124 : "The configured io engine for VirtualFile",
1125 : &["kind"],
1126 : )
1127 0 : .unwrap()
1128 0 : });
1129 : }
1130 :
1131 : #[derive(Debug)]
1132 : struct GlobalAndPerTimelineHistogram {
1133 : global: Histogram,
1134 : per_tenant_timeline: Histogram,
1135 : }
1136 :
1137 : impl GlobalAndPerTimelineHistogram {
1138 10 : fn observe(&self, value: f64) {
1139 10 : self.global.observe(value);
1140 10 : self.per_tenant_timeline.observe(value);
1141 10 : }
1142 : }
1143 :
1144 : struct GlobalAndPerTimelineHistogramTimer<'a, 'c> {
1145 : h: &'a GlobalAndPerTimelineHistogram,
1146 : ctx: &'c RequestContext,
1147 : start: std::time::Instant,
1148 : op: SmgrQueryType,
1149 : }
1150 :
1151 : impl<'a, 'c> Drop for GlobalAndPerTimelineHistogramTimer<'a, 'c> {
1152 10 : fn drop(&mut self) {
1153 10 : let elapsed = self.start.elapsed();
1154 10 : let ex_throttled = self
1155 10 : .ctx
1156 10 : .micros_spent_throttled
1157 10 : .close_and_checked_sub_from(elapsed);
1158 10 : let ex_throttled = match ex_throttled {
1159 10 : Ok(res) => res,
1160 0 : Err(error) => {
1161 0 : use utils::rate_limit::RateLimit;
1162 0 : static LOGGED: Lazy<Mutex<enum_map::EnumMap<SmgrQueryType, RateLimit>>> =
1163 0 : Lazy::new(|| {
1164 0 : Mutex::new(enum_map::EnumMap::from_array(std::array::from_fn(|_| {
1165 0 : RateLimit::new(Duration::from_secs(10))
1166 0 : })))
1167 0 : });
1168 0 : let mut guard = LOGGED.lock().unwrap();
1169 0 : let rate_limit = &mut guard[self.op];
1170 0 : rate_limit.call(|| {
1171 0 : warn!(op=?self.op, error, "error deducting time spent throttled; this message is logged at a global rate limit");
1172 0 : });
1173 0 : elapsed
1174 : }
1175 : };
1176 10 : self.h.observe(ex_throttled.as_secs_f64());
1177 10 : }
1178 : }
1179 :
1180 : #[derive(
1181 : Debug,
1182 : Clone,
1183 : Copy,
1184 3950 : IntoStaticStr,
1185 : strum_macros::EnumCount,
1186 112 : strum_macros::EnumIter,
1187 1950 : strum_macros::FromRepr,
1188 : enum_map::Enum,
1189 : )]
1190 : #[strum(serialize_all = "snake_case")]
1191 : pub enum SmgrQueryType {
1192 : GetRelExists,
1193 : GetRelSize,
1194 : GetPageAtLsn,
1195 : GetDbSize,
1196 : GetSlruSegment,
1197 : }
1198 :
1199 : #[derive(Debug)]
1200 : pub(crate) struct SmgrQueryTimePerTimeline {
1201 : metrics: [GlobalAndPerTimelineHistogram; SmgrQueryType::COUNT],
1202 : }
1203 :
1204 140 : static SMGR_QUERY_TIME_PER_TENANT_TIMELINE: Lazy<HistogramVec> = Lazy::new(|| {
1205 : register_histogram_vec!(
1206 : "pageserver_smgr_query_seconds",
1207 : "Time spent on smgr query handling, aggegated by query type and tenant/timeline.",
1208 : &["smgr_query_type", "tenant_id", "shard_id", "timeline_id"],
1209 : CRITICAL_OP_BUCKETS.into(),
1210 : )
1211 140 : .expect("failed to define a metric")
1212 140 : });
1213 :
1214 140 : static SMGR_QUERY_TIME_GLOBAL_BUCKETS: Lazy<Vec<f64>> = Lazy::new(|| {
1215 140 : [
1216 140 : 1,
1217 140 : 10,
1218 140 : 20,
1219 140 : 40,
1220 140 : 60,
1221 140 : 80,
1222 140 : 100,
1223 140 : 200,
1224 140 : 300,
1225 140 : 400,
1226 140 : 500,
1227 140 : 600,
1228 140 : 700,
1229 140 : 800,
1230 140 : 900,
1231 140 : 1_000, // 1ms
1232 140 : 2_000,
1233 140 : 4_000,
1234 140 : 6_000,
1235 140 : 8_000,
1236 140 : 10_000, // 10ms
1237 140 : 20_000,
1238 140 : 40_000,
1239 140 : 60_000,
1240 140 : 80_000,
1241 140 : 100_000,
1242 140 : 200_000,
1243 140 : 400_000,
1244 140 : 600_000,
1245 140 : 800_000,
1246 140 : 1_000_000, // 1s
1247 140 : 2_000_000,
1248 140 : 4_000_000,
1249 140 : 6_000_000,
1250 140 : 8_000_000,
1251 140 : 10_000_000, // 10s
1252 140 : 20_000_000,
1253 140 : 50_000_000,
1254 140 : 100_000_000,
1255 140 : 200_000_000,
1256 140 : 1_000_000_000, // 1000s
1257 140 : ]
1258 140 : .into_iter()
1259 140 : .map(Duration::from_micros)
1260 5740 : .map(|d| d.as_secs_f64())
1261 140 : .collect()
1262 140 : });
1263 :
1264 140 : static SMGR_QUERY_TIME_GLOBAL: Lazy<HistogramVec> = Lazy::new(|| {
1265 : register_histogram_vec!(
1266 : "pageserver_smgr_query_seconds_global",
1267 : "Time spent on smgr query handling, aggregated by query type.",
1268 : &["smgr_query_type"],
1269 : SMGR_QUERY_TIME_GLOBAL_BUCKETS.clone(),
1270 : )
1271 140 : .expect("failed to define a metric")
1272 140 : });
1273 :
1274 : impl SmgrQueryTimePerTimeline {
1275 390 : pub(crate) fn new(tenant_shard_id: &TenantShardId, timeline_id: &TimelineId) -> Self {
1276 390 : let tenant_id = tenant_shard_id.tenant_id.to_string();
1277 390 : let shard_slug = format!("{}", tenant_shard_id.shard_slug());
1278 390 : let timeline_id = timeline_id.to_string();
1279 1950 : let metrics = std::array::from_fn(|i| {
1280 1950 : let op = SmgrQueryType::from_repr(i).unwrap();
1281 1950 : let global = SMGR_QUERY_TIME_GLOBAL
1282 1950 : .get_metric_with_label_values(&[op.into()])
1283 1950 : .unwrap();
1284 1950 : let per_tenant_timeline = SMGR_QUERY_TIME_PER_TENANT_TIMELINE
1285 1950 : .get_metric_with_label_values(&[op.into(), &tenant_id, &shard_slug, &timeline_id])
1286 1950 : .unwrap();
1287 1950 : GlobalAndPerTimelineHistogram {
1288 1950 : global,
1289 1950 : per_tenant_timeline,
1290 1950 : }
1291 1950 : });
1292 390 : Self { metrics }
1293 390 : }
1294 10 : pub(crate) fn start_timer<'c: 'a, 'a>(
1295 10 : &'a self,
1296 10 : op: SmgrQueryType,
1297 10 : ctx: &'c RequestContext,
1298 10 : ) -> impl Drop + '_ {
1299 10 : let metric = &self.metrics[op as usize];
1300 10 : let start = Instant::now();
1301 10 : match ctx.micros_spent_throttled.open() {
1302 10 : Ok(()) => (),
1303 0 : Err(error) => {
1304 0 : use utils::rate_limit::RateLimit;
1305 0 : static LOGGED: Lazy<Mutex<enum_map::EnumMap<SmgrQueryType, RateLimit>>> =
1306 0 : Lazy::new(|| {
1307 0 : Mutex::new(enum_map::EnumMap::from_array(std::array::from_fn(|_| {
1308 0 : RateLimit::new(Duration::from_secs(10))
1309 0 : })))
1310 0 : });
1311 0 : let mut guard = LOGGED.lock().unwrap();
1312 0 : let rate_limit = &mut guard[op];
1313 0 : rate_limit.call(|| {
1314 0 : warn!(?op, error, "error opening micros_spent_throttled; this message is logged at a global rate limit");
1315 0 : });
1316 0 : }
1317 : }
1318 10 : GlobalAndPerTimelineHistogramTimer {
1319 10 : h: metric,
1320 10 : ctx,
1321 10 : start,
1322 10 : op,
1323 10 : }
1324 10 : }
1325 : }
1326 :
1327 : #[cfg(test)]
1328 : mod smgr_query_time_tests {
1329 : use pageserver_api::shard::TenantShardId;
1330 : use strum::IntoEnumIterator;
1331 : use utils::id::{TenantId, TimelineId};
1332 :
1333 : use crate::{
1334 : context::{DownloadBehavior, RequestContext},
1335 : task_mgr::TaskKind,
1336 : };
1337 :
1338 : // Regression test, we used hard-coded string constants before using an enum.
1339 : #[test]
1340 2 : fn op_label_name() {
1341 2 : use super::SmgrQueryType::*;
1342 2 : let expect: [(super::SmgrQueryType, &'static str); 5] = [
1343 2 : (GetRelExists, "get_rel_exists"),
1344 2 : (GetRelSize, "get_rel_size"),
1345 2 : (GetPageAtLsn, "get_page_at_lsn"),
1346 2 : (GetDbSize, "get_db_size"),
1347 2 : (GetSlruSegment, "get_slru_segment"),
1348 2 : ];
1349 12 : for (op, expect) in expect {
1350 10 : let actual: &'static str = op.into();
1351 10 : assert_eq!(actual, expect);
1352 : }
1353 2 : }
1354 :
1355 : #[test]
1356 2 : fn basic() {
1357 2 : let ops: Vec<_> = super::SmgrQueryType::iter().collect();
1358 :
1359 12 : for op in &ops {
1360 10 : let tenant_id = TenantId::generate();
1361 10 : let timeline_id = TimelineId::generate();
1362 10 : let metrics = super::SmgrQueryTimePerTimeline::new(
1363 10 : &TenantShardId::unsharded(tenant_id),
1364 10 : &timeline_id,
1365 10 : );
1366 10 :
1367 20 : let get_counts = || {
1368 20 : let global: u64 = ops
1369 20 : .iter()
1370 100 : .map(|op| metrics.metrics[*op as usize].global.get_sample_count())
1371 20 : .sum();
1372 20 : let per_tenant_timeline: u64 = ops
1373 20 : .iter()
1374 100 : .map(|op| {
1375 100 : metrics.metrics[*op as usize]
1376 100 : .per_tenant_timeline
1377 100 : .get_sample_count()
1378 100 : })
1379 20 : .sum();
1380 20 : (global, per_tenant_timeline)
1381 20 : };
1382 :
1383 10 : let (pre_global, pre_per_tenant_timeline) = get_counts();
1384 10 : assert_eq!(pre_per_tenant_timeline, 0);
1385 :
1386 10 : let ctx = RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Download);
1387 10 : let timer = metrics.start_timer(*op, &ctx);
1388 10 : drop(timer);
1389 10 :
1390 10 : let (post_global, post_per_tenant_timeline) = get_counts();
1391 10 : assert_eq!(post_per_tenant_timeline, 1);
1392 10 : assert!(post_global > pre_global);
1393 : }
1394 2 : }
1395 : }
1396 :
1397 : // keep in sync with control plane Go code so that we can validate
1398 : // compute's basebackup_ms metric with our perspective in the context of SLI/SLO.
1399 0 : static COMPUTE_STARTUP_BUCKETS: Lazy<[f64; 28]> = Lazy::new(|| {
1400 0 : // Go code uses milliseconds. Variable is called `computeStartupBuckets`
1401 0 : [
1402 0 : 5, 10, 20, 30, 50, 70, 100, 120, 150, 200, 250, 300, 350, 400, 450, 500, 600, 800, 1000,
1403 0 : 1500, 2000, 2500, 3000, 5000, 10000, 20000, 40000, 60000,
1404 0 : ]
1405 0 : .map(|ms| (ms as f64) / 1000.0)
1406 0 : });
1407 :
1408 : pub(crate) struct BasebackupQueryTime(HistogramVec);
1409 0 : pub(crate) static BASEBACKUP_QUERY_TIME: Lazy<BasebackupQueryTime> = Lazy::new(|| {
1410 0 : BasebackupQueryTime({
1411 0 : register_histogram_vec!(
1412 : "pageserver_basebackup_query_seconds",
1413 : "Histogram of basebackup queries durations, by result type",
1414 : &["result"],
1415 : COMPUTE_STARTUP_BUCKETS.to_vec(),
1416 0 : )
1417 0 : .expect("failed to define a metric")
1418 0 : })
1419 0 : });
1420 :
1421 : pub(crate) struct BasebackupQueryTimeOngoingRecording<'a, 'c> {
1422 : parent: &'a BasebackupQueryTime,
1423 : ctx: &'c RequestContext,
1424 : start: std::time::Instant,
1425 : }
1426 :
1427 : impl BasebackupQueryTime {
1428 0 : pub(crate) fn start_recording<'c: 'a, 'a>(
1429 0 : &'a self,
1430 0 : ctx: &'c RequestContext,
1431 0 : ) -> BasebackupQueryTimeOngoingRecording<'_, '_> {
1432 0 : let start = Instant::now();
1433 0 : match ctx.micros_spent_throttled.open() {
1434 0 : Ok(()) => (),
1435 0 : Err(error) => {
1436 0 : use utils::rate_limit::RateLimit;
1437 0 : static LOGGED: Lazy<Mutex<RateLimit>> =
1438 0 : Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(10))));
1439 0 : let mut rate_limit = LOGGED.lock().unwrap();
1440 0 : rate_limit.call(|| {
1441 0 : warn!(error, "error opening micros_spent_throttled; this message is logged at a global rate limit");
1442 0 : });
1443 0 : }
1444 : }
1445 0 : BasebackupQueryTimeOngoingRecording {
1446 0 : parent: self,
1447 0 : ctx,
1448 0 : start,
1449 0 : }
1450 0 : }
1451 : }
1452 :
1453 : impl<'a, 'c> BasebackupQueryTimeOngoingRecording<'a, 'c> {
1454 0 : pub(crate) fn observe<T, E>(self, res: &Result<T, E>) {
1455 0 : let elapsed = self.start.elapsed();
1456 0 : let ex_throttled = self
1457 0 : .ctx
1458 0 : .micros_spent_throttled
1459 0 : .close_and_checked_sub_from(elapsed);
1460 0 : let ex_throttled = match ex_throttled {
1461 0 : Ok(ex_throttled) => ex_throttled,
1462 0 : Err(error) => {
1463 0 : use utils::rate_limit::RateLimit;
1464 0 : static LOGGED: Lazy<Mutex<RateLimit>> =
1465 0 : Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(10))));
1466 0 : let mut rate_limit = LOGGED.lock().unwrap();
1467 0 : rate_limit.call(|| {
1468 0 : warn!(error, "error deducting time spent throttled; this message is logged at a global rate limit");
1469 0 : });
1470 0 : elapsed
1471 : }
1472 : };
1473 0 : let label_value = if res.is_ok() { "ok" } else { "error" };
1474 0 : let metric = self
1475 0 : .parent
1476 0 : .0
1477 0 : .get_metric_with_label_values(&[label_value])
1478 0 : .unwrap();
1479 0 : metric.observe(ex_throttled.as_secs_f64());
1480 0 : }
1481 : }
1482 :
1483 0 : pub(crate) static LIVE_CONNECTIONS_COUNT: Lazy<IntGaugeVec> = Lazy::new(|| {
1484 : register_int_gauge_vec!(
1485 : "pageserver_live_connections",
1486 : "Number of live network connections",
1487 : &["pageserver_connection_kind"]
1488 : )
1489 0 : .expect("failed to define a metric")
1490 0 : });
1491 :
1492 : // remote storage metrics
1493 :
1494 134 : static REMOTE_TIMELINE_CLIENT_CALLS: Lazy<IntCounterPairVec> = Lazy::new(|| {
1495 : register_int_counter_pair_vec!(
1496 : "pageserver_remote_timeline_client_calls_started",
1497 : "Number of started calls to remote timeline client.",
1498 : "pageserver_remote_timeline_client_calls_finished",
1499 : "Number of finshed calls to remote timeline client.",
1500 : &[
1501 : "tenant_id",
1502 : "shard_id",
1503 : "timeline_id",
1504 : "file_kind",
1505 : "op_kind"
1506 : ],
1507 : )
1508 134 : .unwrap()
1509 134 : });
1510 :
1511 : static REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER: Lazy<IntCounterVec> =
1512 130 : Lazy::new(|| {
1513 : register_int_counter_vec!(
1514 : "pageserver_remote_timeline_client_bytes_started",
1515 : "Incremented by the number of bytes associated with a remote timeline client operation. \
1516 : The increment happens when the operation is scheduled.",
1517 : &["tenant_id", "shard_id", "timeline_id", "file_kind", "op_kind"],
1518 : )
1519 130 : .expect("failed to define a metric")
1520 130 : });
1521 :
1522 130 : static REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER: Lazy<IntCounterVec> = Lazy::new(|| {
1523 : register_int_counter_vec!(
1524 : "pageserver_remote_timeline_client_bytes_finished",
1525 : "Incremented by the number of bytes associated with a remote timeline client operation. \
1526 : The increment happens when the operation finishes (regardless of success/failure/shutdown).",
1527 : &["tenant_id", "shard_id", "timeline_id", "file_kind", "op_kind"],
1528 : )
1529 130 : .expect("failed to define a metric")
1530 130 : });
1531 :
1532 : pub(crate) struct TenantManagerMetrics {
1533 : tenant_slots_attached: UIntGauge,
1534 : tenant_slots_secondary: UIntGauge,
1535 : tenant_slots_inprogress: UIntGauge,
1536 : pub(crate) tenant_slot_writes: IntCounter,
1537 : pub(crate) unexpected_errors: IntCounter,
1538 : }
1539 :
1540 : impl TenantManagerMetrics {
1541 : /// Helpers for tracking slots. Note that these do not track the lifetime of TenantSlot objects
1542 : /// exactly: they track the lifetime of the slots _in the tenant map_.
1543 2 : pub(crate) fn slot_inserted(&self, slot: &TenantSlot) {
1544 2 : match slot {
1545 0 : TenantSlot::Attached(_) => {
1546 0 : self.tenant_slots_attached.inc();
1547 0 : }
1548 0 : TenantSlot::Secondary(_) => {
1549 0 : self.tenant_slots_secondary.inc();
1550 0 : }
1551 2 : TenantSlot::InProgress(_) => {
1552 2 : self.tenant_slots_inprogress.inc();
1553 2 : }
1554 : }
1555 2 : }
1556 :
1557 2 : pub(crate) fn slot_removed(&self, slot: &TenantSlot) {
1558 2 : match slot {
1559 2 : TenantSlot::Attached(_) => {
1560 2 : self.tenant_slots_attached.dec();
1561 2 : }
1562 0 : TenantSlot::Secondary(_) => {
1563 0 : self.tenant_slots_secondary.dec();
1564 0 : }
1565 0 : TenantSlot::InProgress(_) => {
1566 0 : self.tenant_slots_inprogress.dec();
1567 0 : }
1568 : }
1569 2 : }
1570 :
1571 : #[cfg(all(debug_assertions, not(test)))]
1572 0 : pub(crate) fn slots_total(&self) -> u64 {
1573 0 : self.tenant_slots_attached.get()
1574 0 : + self.tenant_slots_secondary.get()
1575 0 : + self.tenant_slots_inprogress.get()
1576 0 : }
1577 : }
1578 :
1579 2 : pub(crate) static TENANT_MANAGER: Lazy<TenantManagerMetrics> = Lazy::new(|| {
1580 2 : let tenant_slots = register_uint_gauge_vec!(
1581 : "pageserver_tenant_manager_slots",
1582 : "How many slots currently exist, including all attached, secondary and in-progress operations",
1583 : &["mode"]
1584 : )
1585 2 : .expect("failed to define a metric");
1586 2 : TenantManagerMetrics {
1587 2 : tenant_slots_attached: tenant_slots
1588 2 : .get_metric_with_label_values(&["attached"])
1589 2 : .unwrap(),
1590 2 : tenant_slots_secondary: tenant_slots
1591 2 : .get_metric_with_label_values(&["secondary"])
1592 2 : .unwrap(),
1593 2 : tenant_slots_inprogress: tenant_slots
1594 2 : .get_metric_with_label_values(&["inprogress"])
1595 2 : .unwrap(),
1596 2 : tenant_slot_writes: register_int_counter!(
1597 : "pageserver_tenant_manager_slot_writes",
1598 : "Writes to a tenant slot, including all of create/attach/detach/delete"
1599 2 : )
1600 2 : .expect("failed to define a metric"),
1601 2 : unexpected_errors: register_int_counter!(
1602 : "pageserver_tenant_manager_unexpected_errors_total",
1603 : "Number of unexpected conditions encountered: nonzero value indicates a non-fatal bug."
1604 2 : )
1605 2 : .expect("failed to define a metric"),
1606 2 : }
1607 2 : });
1608 :
1609 : pub(crate) struct DeletionQueueMetrics {
1610 : pub(crate) keys_submitted: IntCounter,
1611 : pub(crate) keys_dropped: IntCounter,
1612 : pub(crate) keys_executed: IntCounter,
1613 : pub(crate) keys_validated: IntCounter,
1614 : pub(crate) dropped_lsn_updates: IntCounter,
1615 : pub(crate) unexpected_errors: IntCounter,
1616 : pub(crate) remote_errors: IntCounterVec,
1617 : }
1618 24 : pub(crate) static DELETION_QUEUE: Lazy<DeletionQueueMetrics> = Lazy::new(|| {
1619 24 : DeletionQueueMetrics{
1620 24 :
1621 24 : keys_submitted: register_int_counter!(
1622 : "pageserver_deletion_queue_submitted_total",
1623 : "Number of objects submitted for deletion"
1624 24 : )
1625 24 : .expect("failed to define a metric"),
1626 24 :
1627 24 : keys_dropped: register_int_counter!(
1628 : "pageserver_deletion_queue_dropped_total",
1629 : "Number of object deletions dropped due to stale generation."
1630 24 : )
1631 24 : .expect("failed to define a metric"),
1632 24 :
1633 24 : keys_executed: register_int_counter!(
1634 : "pageserver_deletion_queue_executed_total",
1635 : "Number of objects deleted. Only includes objects that we actually deleted, sum with pageserver_deletion_queue_dropped_total for the total number of keys processed to completion"
1636 24 : )
1637 24 : .expect("failed to define a metric"),
1638 24 :
1639 24 : keys_validated: register_int_counter!(
1640 : "pageserver_deletion_queue_validated_total",
1641 : "Number of keys validated for deletion. Sum with pageserver_deletion_queue_dropped_total for the total number of keys that have passed through the validation stage."
1642 24 : )
1643 24 : .expect("failed to define a metric"),
1644 24 :
1645 24 : dropped_lsn_updates: register_int_counter!(
1646 : "pageserver_deletion_queue_dropped_lsn_updates_total",
1647 : "Updates to remote_consistent_lsn dropped due to stale generation number."
1648 24 : )
1649 24 : .expect("failed to define a metric"),
1650 24 : unexpected_errors: register_int_counter!(
1651 : "pageserver_deletion_queue_unexpected_errors_total",
1652 : "Number of unexpected condiions that may stall the queue: any value above zero is unexpected."
1653 24 : )
1654 24 : .expect("failed to define a metric"),
1655 24 : remote_errors: register_int_counter_vec!(
1656 : "pageserver_deletion_queue_remote_errors_total",
1657 : "Retryable remote I/O errors while executing deletions, for example 503 responses to DeleteObjects",
1658 : &["op_kind"],
1659 24 : )
1660 24 : .expect("failed to define a metric")
1661 24 : }
1662 24 : });
1663 :
1664 : pub(crate) struct SecondaryModeMetrics {
1665 : pub(crate) upload_heatmap: IntCounter,
1666 : pub(crate) upload_heatmap_errors: IntCounter,
1667 : pub(crate) upload_heatmap_duration: Histogram,
1668 : pub(crate) download_heatmap: IntCounter,
1669 : pub(crate) download_layer: IntCounter,
1670 : }
1671 0 : pub(crate) static SECONDARY_MODE: Lazy<SecondaryModeMetrics> = Lazy::new(|| {
1672 0 : SecondaryModeMetrics {
1673 0 : upload_heatmap: register_int_counter!(
1674 : "pageserver_secondary_upload_heatmap",
1675 : "Number of heatmaps written to remote storage by attached tenants"
1676 0 : )
1677 0 : .expect("failed to define a metric"),
1678 0 : upload_heatmap_errors: register_int_counter!(
1679 : "pageserver_secondary_upload_heatmap_errors",
1680 : "Failures writing heatmap to remote storage"
1681 0 : )
1682 0 : .expect("failed to define a metric"),
1683 0 : upload_heatmap_duration: register_histogram!(
1684 : "pageserver_secondary_upload_heatmap_duration",
1685 : "Time to build and upload a heatmap, including any waiting inside the S3 client"
1686 0 : )
1687 0 : .expect("failed to define a metric"),
1688 0 : download_heatmap: register_int_counter!(
1689 : "pageserver_secondary_download_heatmap",
1690 : "Number of downloads of heatmaps by secondary mode locations, including when it hasn't changed"
1691 0 : )
1692 0 : .expect("failed to define a metric"),
1693 0 : download_layer: register_int_counter!(
1694 : "pageserver_secondary_download_layer",
1695 : "Number of downloads of layers by secondary mode locations"
1696 0 : )
1697 0 : .expect("failed to define a metric"),
1698 0 : }
1699 0 : });
1700 :
1701 : #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1702 : pub enum RemoteOpKind {
1703 : Upload,
1704 : Download,
1705 : Delete,
1706 : }
1707 : impl RemoteOpKind {
1708 12693 : pub fn as_str(&self) -> &'static str {
1709 12693 : match self {
1710 11992 : Self::Upload => "upload",
1711 52 : Self::Download => "download",
1712 649 : Self::Delete => "delete",
1713 : }
1714 12693 : }
1715 : }
1716 :
1717 : #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
1718 : pub enum RemoteOpFileKind {
1719 : Layer,
1720 : Index,
1721 : }
1722 : impl RemoteOpFileKind {
1723 12693 : pub fn as_str(&self) -> &'static str {
1724 12693 : match self {
1725 8557 : Self::Layer => "layer",
1726 4136 : Self::Index => "index",
1727 : }
1728 12693 : }
1729 : }
1730 :
1731 128 : pub(crate) static REMOTE_OPERATION_TIME: Lazy<HistogramVec> = Lazy::new(|| {
1732 : register_histogram_vec!(
1733 : "pageserver_remote_operation_seconds",
1734 : "Time spent on remote storage operations. \
1735 : Grouped by tenant, timeline, operation_kind and status. \
1736 : Does not account for time spent waiting in remote timeline client's queues.",
1737 : &["file_kind", "op_kind", "status"]
1738 : )
1739 128 : .expect("failed to define a metric")
1740 128 : });
1741 :
1742 0 : pub(crate) static TENANT_TASK_EVENTS: Lazy<IntCounterVec> = Lazy::new(|| {
1743 : register_int_counter_vec!(
1744 : "pageserver_tenant_task_events",
1745 : "Number of task start/stop/fail events.",
1746 : &["event"],
1747 : )
1748 0 : .expect("Failed to register tenant_task_events metric")
1749 0 : });
1750 :
1751 20 : pub(crate) static BACKGROUND_LOOP_SEMAPHORE_WAIT_GAUGE: Lazy<IntCounterPairVec> = Lazy::new(|| {
1752 : register_int_counter_pair_vec!(
1753 : "pageserver_background_loop_semaphore_wait_start_count",
1754 : "Counter for background loop concurrency-limiting semaphore acquire calls started",
1755 : "pageserver_background_loop_semaphore_wait_finish_count",
1756 : "Counter for background loop concurrency-limiting semaphore acquire calls finished",
1757 : &["task"],
1758 : )
1759 20 : .unwrap()
1760 20 : });
1761 :
1762 0 : pub(crate) static BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT: Lazy<IntCounterVec> = Lazy::new(|| {
1763 : register_int_counter_vec!(
1764 : "pageserver_background_loop_period_overrun_count",
1765 : "Incremented whenever warn_when_period_overrun() logs a warning.",
1766 : &["task", "period"],
1767 : )
1768 0 : .expect("failed to define a metric")
1769 0 : });
1770 :
1771 : // walreceiver metrics
1772 :
1773 0 : pub(crate) static WALRECEIVER_STARTED_CONNECTIONS: Lazy<IntCounter> = Lazy::new(|| {
1774 : register_int_counter!(
1775 : "pageserver_walreceiver_started_connections_total",
1776 : "Number of started walreceiver connections"
1777 : )
1778 0 : .expect("failed to define a metric")
1779 0 : });
1780 :
1781 0 : pub(crate) static WALRECEIVER_ACTIVE_MANAGERS: Lazy<IntGauge> = Lazy::new(|| {
1782 : register_int_gauge!(
1783 : "pageserver_walreceiver_active_managers",
1784 : "Number of active walreceiver managers"
1785 : )
1786 0 : .expect("failed to define a metric")
1787 0 : });
1788 :
1789 0 : pub(crate) static WALRECEIVER_SWITCHES: Lazy<IntCounterVec> = Lazy::new(|| {
1790 : register_int_counter_vec!(
1791 : "pageserver_walreceiver_switches_total",
1792 : "Number of walreceiver manager change_connection calls",
1793 : &["reason"]
1794 : )
1795 0 : .expect("failed to define a metric")
1796 0 : });
1797 :
1798 0 : pub(crate) static WALRECEIVER_BROKER_UPDATES: Lazy<IntCounter> = Lazy::new(|| {
1799 : register_int_counter!(
1800 : "pageserver_walreceiver_broker_updates_total",
1801 : "Number of received broker updates in walreceiver"
1802 : )
1803 0 : .expect("failed to define a metric")
1804 0 : });
1805 :
1806 2 : pub(crate) static WALRECEIVER_CANDIDATES_EVENTS: Lazy<IntCounterVec> = Lazy::new(|| {
1807 : register_int_counter_vec!(
1808 : "pageserver_walreceiver_candidates_events_total",
1809 : "Number of walreceiver candidate events",
1810 : &["event"]
1811 : )
1812 2 : .expect("failed to define a metric")
1813 2 : });
1814 :
1815 : pub(crate) static WALRECEIVER_CANDIDATES_ADDED: Lazy<IntCounter> =
1816 0 : Lazy::new(|| WALRECEIVER_CANDIDATES_EVENTS.with_label_values(&["add"]));
1817 :
1818 : pub(crate) static WALRECEIVER_CANDIDATES_REMOVED: Lazy<IntCounter> =
1819 2 : Lazy::new(|| WALRECEIVER_CANDIDATES_EVENTS.with_label_values(&["remove"]));
1820 :
1821 : // Metrics collected on WAL redo operations
1822 : //
1823 : // We collect the time spent in actual WAL redo ('redo'), and time waiting
1824 : // for access to the postgres process ('wait') since there is only one for
1825 : // each tenant.
1826 :
1827 : /// Time buckets are small because we want to be able to measure the
1828 : /// smallest redo processing times. These buckets allow us to measure down
1829 : /// to 5us, which equates to 200'000 pages/sec, which equates to 1.6GB/sec.
1830 : /// This is much better than the previous 5ms aka 200 pages/sec aka 1.6MB/sec.
1831 : ///
1832 : /// Values up to 1s are recorded because metrics show that we have redo
1833 : /// durations and lock times larger than 0.250s.
1834 : macro_rules! redo_histogram_time_buckets {
1835 : () => {
1836 : vec![
1837 : 0.000_005, 0.000_010, 0.000_025, 0.000_050, 0.000_100, 0.000_250, 0.000_500, 0.001_000,
1838 : 0.002_500, 0.005_000, 0.010_000, 0.025_000, 0.050_000, 0.100_000, 0.250_000, 0.500_000,
1839 : 1.000_000,
1840 : ]
1841 : };
1842 : }
1843 :
1844 : /// While we're at it, also measure the amount of records replayed in each
1845 : /// operation. We have a global 'total replayed' counter, but that's not
1846 : /// as useful as 'what is the skew for how many records we replay in one
1847 : /// operation'.
1848 : macro_rules! redo_histogram_count_buckets {
1849 : () => {
1850 : vec![0.0, 1.0, 2.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0]
1851 : };
1852 : }
1853 :
1854 : macro_rules! redo_bytes_histogram_count_buckets {
1855 : () => {
1856 : // powers of (2^.5), from 2^4.5 to 2^15 (22 buckets)
1857 : // rounded up to the next multiple of 8 to capture any MAXALIGNed record of that size, too.
1858 : vec![
1859 : 24.0, 32.0, 48.0, 64.0, 96.0, 128.0, 184.0, 256.0, 368.0, 512.0, 728.0, 1024.0, 1456.0,
1860 : 2048.0, 2904.0, 4096.0, 5800.0, 8192.0, 11592.0, 16384.0, 23176.0, 32768.0,
1861 : ]
1862 : };
1863 : }
1864 :
1865 : pub(crate) struct WalIngestMetrics {
1866 : pub(crate) bytes_received: IntCounter,
1867 : pub(crate) records_received: IntCounter,
1868 : pub(crate) records_committed: IntCounter,
1869 : pub(crate) records_filtered: IntCounter,
1870 : }
1871 :
1872 2 : pub(crate) static WAL_INGEST: Lazy<WalIngestMetrics> = Lazy::new(|| WalIngestMetrics {
1873 : bytes_received: register_int_counter!(
1874 : "pageserver_wal_ingest_bytes_received",
1875 : "Bytes of WAL ingested from safekeepers",
1876 : )
1877 2 : .unwrap(),
1878 : records_received: register_int_counter!(
1879 : "pageserver_wal_ingest_records_received",
1880 : "Number of WAL records received from safekeepers"
1881 : )
1882 2 : .expect("failed to define a metric"),
1883 : records_committed: register_int_counter!(
1884 : "pageserver_wal_ingest_records_committed",
1885 : "Number of WAL records which resulted in writes to pageserver storage"
1886 : )
1887 2 : .expect("failed to define a metric"),
1888 : records_filtered: register_int_counter!(
1889 : "pageserver_wal_ingest_records_filtered",
1890 : "Number of WAL records filtered out due to sharding"
1891 : )
1892 2 : .expect("failed to define a metric"),
1893 2 : });
1894 :
1895 6 : pub(crate) static WAL_REDO_TIME: Lazy<Histogram> = Lazy::new(|| {
1896 : register_histogram!(
1897 : "pageserver_wal_redo_seconds",
1898 : "Time spent on WAL redo",
1899 : redo_histogram_time_buckets!()
1900 : )
1901 6 : .expect("failed to define a metric")
1902 6 : });
1903 :
1904 6 : pub(crate) static WAL_REDO_RECORDS_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
1905 : register_histogram!(
1906 : "pageserver_wal_redo_records_histogram",
1907 : "Histogram of number of records replayed per redo in the Postgres WAL redo process",
1908 : redo_histogram_count_buckets!(),
1909 : )
1910 6 : .expect("failed to define a metric")
1911 6 : });
1912 :
1913 6 : pub(crate) static WAL_REDO_BYTES_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
1914 : register_histogram!(
1915 : "pageserver_wal_redo_bytes_histogram",
1916 : "Histogram of number of records replayed per redo sent to Postgres",
1917 : redo_bytes_histogram_count_buckets!(),
1918 : )
1919 6 : .expect("failed to define a metric")
1920 6 : });
1921 :
1922 : // FIXME: isn't this already included by WAL_REDO_RECORDS_HISTOGRAM which has _count?
1923 6 : pub(crate) static WAL_REDO_RECORD_COUNTER: Lazy<IntCounter> = Lazy::new(|| {
1924 : register_int_counter!(
1925 : "pageserver_replayed_wal_records_total",
1926 : "Number of WAL records replayed in WAL redo process"
1927 : )
1928 6 : .unwrap()
1929 6 : });
1930 :
1931 : #[rustfmt::skip]
1932 6 : pub(crate) static WAL_REDO_PROCESS_LAUNCH_DURATION_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
1933 : register_histogram!(
1934 : "pageserver_wal_redo_process_launch_duration",
1935 : "Histogram of the duration of successful WalRedoProcess::launch calls",
1936 : vec![
1937 : 0.0002, 0.0004, 0.0006, 0.0008, 0.0010,
1938 : 0.0020, 0.0040, 0.0060, 0.0080, 0.0100,
1939 : 0.0200, 0.0400, 0.0600, 0.0800, 0.1000,
1940 : 0.2000, 0.4000, 0.6000, 0.8000, 1.0000,
1941 : 1.5000, 2.0000, 2.5000, 3.0000, 4.0000, 10.0000
1942 : ],
1943 : )
1944 6 : .expect("failed to define a metric")
1945 6 : });
1946 :
1947 : pub(crate) struct WalRedoProcessCounters {
1948 : pub(crate) started: IntCounter,
1949 : pub(crate) killed_by_cause: enum_map::EnumMap<WalRedoKillCause, IntCounter>,
1950 : pub(crate) active_stderr_logger_tasks_started: IntCounter,
1951 : pub(crate) active_stderr_logger_tasks_finished: IntCounter,
1952 : }
1953 :
1954 18 : #[derive(Debug, enum_map::Enum, strum_macros::IntoStaticStr)]
1955 : pub(crate) enum WalRedoKillCause {
1956 : WalRedoProcessDrop,
1957 : NoLeakChildDrop,
1958 : Startup,
1959 : }
1960 :
1961 : impl Default for WalRedoProcessCounters {
1962 6 : fn default() -> Self {
1963 6 : let started = register_int_counter!(
1964 : "pageserver_wal_redo_process_started_total",
1965 : "Number of WAL redo processes started",
1966 : )
1967 6 : .unwrap();
1968 6 :
1969 6 : let killed = register_int_counter_vec!(
1970 : "pageserver_wal_redo_process_stopped_total",
1971 : "Number of WAL redo processes stopped",
1972 : &["cause"],
1973 : )
1974 6 : .unwrap();
1975 6 :
1976 6 : let active_stderr_logger_tasks_started = register_int_counter!(
1977 : "pageserver_walredo_stderr_logger_tasks_started_total",
1978 : "Number of active walredo stderr logger tasks that have started",
1979 : )
1980 6 : .unwrap();
1981 6 :
1982 6 : let active_stderr_logger_tasks_finished = register_int_counter!(
1983 : "pageserver_walredo_stderr_logger_tasks_finished_total",
1984 : "Number of active walredo stderr logger tasks that have finished",
1985 : )
1986 6 : .unwrap();
1987 6 :
1988 6 : Self {
1989 6 : started,
1990 18 : killed_by_cause: EnumMap::from_array(std::array::from_fn(|i| {
1991 18 : let cause = <WalRedoKillCause as enum_map::Enum>::from_usize(i);
1992 18 : let cause_str: &'static str = cause.into();
1993 18 : killed.with_label_values(&[cause_str])
1994 18 : })),
1995 6 : active_stderr_logger_tasks_started,
1996 6 : active_stderr_logger_tasks_finished,
1997 6 : }
1998 6 : }
1999 : }
2000 :
2001 : pub(crate) static WAL_REDO_PROCESS_COUNTERS: Lazy<WalRedoProcessCounters> =
2002 : Lazy::new(WalRedoProcessCounters::default);
2003 :
2004 : /// Similar to `prometheus::HistogramTimer` but does not record on drop.
2005 : pub(crate) struct StorageTimeMetricsTimer {
2006 : metrics: StorageTimeMetrics,
2007 : start: Instant,
2008 : }
2009 :
2010 : impl StorageTimeMetricsTimer {
2011 4591 : fn new(metrics: StorageTimeMetrics) -> Self {
2012 4591 : Self {
2013 4591 : metrics,
2014 4591 : start: Instant::now(),
2015 4591 : }
2016 4591 : }
2017 :
2018 : /// Record the time from creation to now.
2019 3489 : pub fn stop_and_record(self) {
2020 3489 : let duration = self.start.elapsed().as_secs_f64();
2021 3489 : self.metrics.timeline_sum.inc_by(duration);
2022 3489 : self.metrics.timeline_count.inc();
2023 3489 : self.metrics.global_histogram.observe(duration);
2024 3489 : }
2025 :
2026 : /// Turns this timer into a timer, which will always record -- usually this means recording
2027 : /// regardless an early `?` path was taken in a function.
2028 754 : pub(crate) fn record_on_drop(self) -> AlwaysRecordingStorageTimeMetricsTimer {
2029 754 : AlwaysRecordingStorageTimeMetricsTimer(Some(self))
2030 754 : }
2031 : }
2032 :
2033 : pub(crate) struct AlwaysRecordingStorageTimeMetricsTimer(Option<StorageTimeMetricsTimer>);
2034 :
2035 : impl Drop for AlwaysRecordingStorageTimeMetricsTimer {
2036 754 : fn drop(&mut self) {
2037 754 : if let Some(inner) = self.0.take() {
2038 754 : inner.stop_and_record();
2039 754 : }
2040 754 : }
2041 : }
2042 :
2043 : /// Timing facilities for an globally histogrammed metric, which is supported by per tenant and
2044 : /// timeline total sum and count.
2045 : #[derive(Clone, Debug)]
2046 : pub(crate) struct StorageTimeMetrics {
2047 : /// Sum of f64 seconds, per operation, tenant_id and timeline_id
2048 : timeline_sum: Counter,
2049 : /// Number of oeprations, per operation, tenant_id and timeline_id
2050 : timeline_count: IntCounter,
2051 : /// Global histogram having only the "operation" label.
2052 : global_histogram: Histogram,
2053 : }
2054 :
2055 : impl StorageTimeMetrics {
2056 3040 : pub fn new(
2057 3040 : operation: StorageTimeOperation,
2058 3040 : tenant_id: &str,
2059 3040 : shard_id: &str,
2060 3040 : timeline_id: &str,
2061 3040 : ) -> Self {
2062 3040 : let operation: &'static str = operation.into();
2063 3040 :
2064 3040 : let timeline_sum = STORAGE_TIME_SUM_PER_TIMELINE
2065 3040 : .get_metric_with_label_values(&[operation, tenant_id, shard_id, timeline_id])
2066 3040 : .unwrap();
2067 3040 : let timeline_count = STORAGE_TIME_COUNT_PER_TIMELINE
2068 3040 : .get_metric_with_label_values(&[operation, tenant_id, shard_id, timeline_id])
2069 3040 : .unwrap();
2070 3040 : let global_histogram = STORAGE_TIME_GLOBAL
2071 3040 : .get_metric_with_label_values(&[operation])
2072 3040 : .unwrap();
2073 3040 :
2074 3040 : StorageTimeMetrics {
2075 3040 : timeline_sum,
2076 3040 : timeline_count,
2077 3040 : global_histogram,
2078 3040 : }
2079 3040 : }
2080 :
2081 : /// Starts timing a new operation.
2082 : ///
2083 : /// Note: unlike `prometheus::HistogramTimer` the returned timer does not record on drop.
2084 4591 : pub fn start_timer(&self) -> StorageTimeMetricsTimer {
2085 4591 : StorageTimeMetricsTimer::new(self.clone())
2086 4591 : }
2087 : }
2088 :
2089 : #[derive(Debug)]
2090 : pub(crate) struct TimelineMetrics {
2091 : tenant_id: String,
2092 : shard_id: String,
2093 : timeline_id: String,
2094 : pub flush_time_histo: StorageTimeMetrics,
2095 : pub compact_time_histo: StorageTimeMetrics,
2096 : pub create_images_time_histo: StorageTimeMetrics,
2097 : pub logical_size_histo: StorageTimeMetrics,
2098 : pub imitate_logical_size_histo: StorageTimeMetrics,
2099 : pub load_layer_map_histo: StorageTimeMetrics,
2100 : pub garbage_collect_histo: StorageTimeMetrics,
2101 : pub find_gc_cutoffs_histo: StorageTimeMetrics,
2102 : pub last_record_gauge: IntGauge,
2103 : pub standby_horizon_gauge: IntGauge,
2104 : pub resident_physical_size_gauge: UIntGauge,
2105 : /// copy of LayeredTimeline.current_logical_size
2106 : pub current_logical_size_gauge: UIntGauge,
2107 : pub aux_file_size_gauge: IntGauge,
2108 : pub directory_entries_count_gauge: Lazy<UIntGauge, Box<dyn Send + Fn() -> UIntGauge>>,
2109 : pub evictions: IntCounter,
2110 : pub evictions_with_low_residence_duration: std::sync::RwLock<EvictionsWithLowResidenceDuration>,
2111 : shutdown: std::sync::atomic::AtomicBool,
2112 : }
2113 :
2114 : impl TimelineMetrics {
2115 380 : pub fn new(
2116 380 : tenant_shard_id: &TenantShardId,
2117 380 : timeline_id_raw: &TimelineId,
2118 380 : evictions_with_low_residence_duration_builder: EvictionsWithLowResidenceDurationBuilder,
2119 380 : ) -> Self {
2120 380 : let tenant_id = tenant_shard_id.tenant_id.to_string();
2121 380 : let shard_id = format!("{}", tenant_shard_id.shard_slug());
2122 380 : let timeline_id = timeline_id_raw.to_string();
2123 380 : let flush_time_histo = StorageTimeMetrics::new(
2124 380 : StorageTimeOperation::LayerFlush,
2125 380 : &tenant_id,
2126 380 : &shard_id,
2127 380 : &timeline_id,
2128 380 : );
2129 380 : let compact_time_histo = StorageTimeMetrics::new(
2130 380 : StorageTimeOperation::Compact,
2131 380 : &tenant_id,
2132 380 : &shard_id,
2133 380 : &timeline_id,
2134 380 : );
2135 380 : let create_images_time_histo = StorageTimeMetrics::new(
2136 380 : StorageTimeOperation::CreateImages,
2137 380 : &tenant_id,
2138 380 : &shard_id,
2139 380 : &timeline_id,
2140 380 : );
2141 380 : let logical_size_histo = StorageTimeMetrics::new(
2142 380 : StorageTimeOperation::LogicalSize,
2143 380 : &tenant_id,
2144 380 : &shard_id,
2145 380 : &timeline_id,
2146 380 : );
2147 380 : let imitate_logical_size_histo = StorageTimeMetrics::new(
2148 380 : StorageTimeOperation::ImitateLogicalSize,
2149 380 : &tenant_id,
2150 380 : &shard_id,
2151 380 : &timeline_id,
2152 380 : );
2153 380 : let load_layer_map_histo = StorageTimeMetrics::new(
2154 380 : StorageTimeOperation::LoadLayerMap,
2155 380 : &tenant_id,
2156 380 : &shard_id,
2157 380 : &timeline_id,
2158 380 : );
2159 380 : let garbage_collect_histo = StorageTimeMetrics::new(
2160 380 : StorageTimeOperation::Gc,
2161 380 : &tenant_id,
2162 380 : &shard_id,
2163 380 : &timeline_id,
2164 380 : );
2165 380 : let find_gc_cutoffs_histo = StorageTimeMetrics::new(
2166 380 : StorageTimeOperation::FindGcCutoffs,
2167 380 : &tenant_id,
2168 380 : &shard_id,
2169 380 : &timeline_id,
2170 380 : );
2171 380 : let last_record_gauge = LAST_RECORD_LSN
2172 380 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2173 380 : .unwrap();
2174 380 : let standby_horizon_gauge = STANDBY_HORIZON
2175 380 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2176 380 : .unwrap();
2177 380 : let resident_physical_size_gauge = RESIDENT_PHYSICAL_SIZE
2178 380 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2179 380 : .unwrap();
2180 380 : // TODO: we shouldn't expose this metric
2181 380 : let current_logical_size_gauge = CURRENT_LOGICAL_SIZE
2182 380 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2183 380 : .unwrap();
2184 380 : let aux_file_size_gauge = AUX_FILE_SIZE
2185 380 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2186 380 : .unwrap();
2187 380 : // TODO use impl Trait syntax here once we have ability to use it: https://github.com/rust-lang/rust/issues/63065
2188 380 : let directory_entries_count_gauge_closure = {
2189 380 : let tenant_shard_id = *tenant_shard_id;
2190 380 : let timeline_id_raw = *timeline_id_raw;
2191 0 : move || {
2192 0 : let tenant_id = tenant_shard_id.tenant_id.to_string();
2193 0 : let shard_id = format!("{}", tenant_shard_id.shard_slug());
2194 0 : let timeline_id = timeline_id_raw.to_string();
2195 0 : let gauge: UIntGauge = DIRECTORY_ENTRIES_COUNT
2196 0 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2197 0 : .unwrap();
2198 0 : gauge
2199 0 : }
2200 : };
2201 380 : let directory_entries_count_gauge: Lazy<UIntGauge, Box<dyn Send + Fn() -> UIntGauge>> =
2202 380 : Lazy::new(Box::new(directory_entries_count_gauge_closure));
2203 380 : let evictions = EVICTIONS
2204 380 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2205 380 : .unwrap();
2206 380 : let evictions_with_low_residence_duration = evictions_with_low_residence_duration_builder
2207 380 : .build(&tenant_id, &shard_id, &timeline_id);
2208 380 :
2209 380 : TimelineMetrics {
2210 380 : tenant_id,
2211 380 : shard_id,
2212 380 : timeline_id,
2213 380 : flush_time_histo,
2214 380 : compact_time_histo,
2215 380 : create_images_time_histo,
2216 380 : logical_size_histo,
2217 380 : imitate_logical_size_histo,
2218 380 : garbage_collect_histo,
2219 380 : find_gc_cutoffs_histo,
2220 380 : load_layer_map_histo,
2221 380 : last_record_gauge,
2222 380 : standby_horizon_gauge,
2223 380 : resident_physical_size_gauge,
2224 380 : current_logical_size_gauge,
2225 380 : aux_file_size_gauge,
2226 380 : directory_entries_count_gauge,
2227 380 : evictions,
2228 380 : evictions_with_low_residence_duration: std::sync::RwLock::new(
2229 380 : evictions_with_low_residence_duration,
2230 380 : ),
2231 380 : shutdown: std::sync::atomic::AtomicBool::default(),
2232 380 : }
2233 380 : }
2234 :
2235 1460 : pub(crate) fn record_new_file_metrics(&self, sz: u64) {
2236 1460 : self.resident_physical_size_add(sz);
2237 1460 : }
2238 :
2239 441 : pub(crate) fn resident_physical_size_sub(&self, sz: u64) {
2240 441 : self.resident_physical_size_gauge.sub(sz);
2241 441 : crate::metrics::RESIDENT_PHYSICAL_SIZE_GLOBAL.sub(sz);
2242 441 : }
2243 :
2244 1490 : pub(crate) fn resident_physical_size_add(&self, sz: u64) {
2245 1490 : self.resident_physical_size_gauge.add(sz);
2246 1490 : crate::metrics::RESIDENT_PHYSICAL_SIZE_GLOBAL.add(sz);
2247 1490 : }
2248 :
2249 8 : pub(crate) fn resident_physical_size_get(&self) -> u64 {
2250 8 : self.resident_physical_size_gauge.get()
2251 8 : }
2252 :
2253 8 : pub(crate) fn shutdown(&self) {
2254 8 : let was_shutdown = self
2255 8 : .shutdown
2256 8 : .swap(true, std::sync::atomic::Ordering::Relaxed);
2257 8 :
2258 8 : if was_shutdown {
2259 : // this happens on tenant deletion because tenant first shuts down timelines, then
2260 : // invokes timeline deletion which first shuts down the timeline again.
2261 : // TODO: this can be removed once https://github.com/neondatabase/neon/issues/5080
2262 0 : return;
2263 8 : }
2264 8 :
2265 8 : let tenant_id = &self.tenant_id;
2266 8 : let timeline_id = &self.timeline_id;
2267 8 : let shard_id = &self.shard_id;
2268 8 : let _ = LAST_RECORD_LSN.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2269 8 : let _ = STANDBY_HORIZON.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2270 8 : {
2271 8 : RESIDENT_PHYSICAL_SIZE_GLOBAL.sub(self.resident_physical_size_get());
2272 8 : let _ = RESIDENT_PHYSICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2273 8 : }
2274 8 : let _ = CURRENT_LOGICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2275 8 : if let Some(metric) = Lazy::get(&DIRECTORY_ENTRIES_COUNT) {
2276 0 : let _ = metric.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2277 8 : }
2278 8 : let _ = EVICTIONS.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2279 8 : let _ = AUX_FILE_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2280 8 :
2281 8 : self.evictions_with_low_residence_duration
2282 8 : .write()
2283 8 : .unwrap()
2284 8 : .remove(tenant_id, shard_id, timeline_id);
2285 :
2286 : // The following metrics are born outside of the TimelineMetrics lifecycle but still
2287 : // removed at the end of it. The idea is to have the metrics outlive the
2288 : // entity during which they're observed, e.g., the smgr metrics shall
2289 : // outlive an individual smgr connection, but not the timeline.
2290 :
2291 80 : for op in StorageTimeOperation::VARIANTS {
2292 72 : let _ = STORAGE_TIME_SUM_PER_TIMELINE.remove_label_values(&[
2293 72 : op,
2294 72 : tenant_id,
2295 72 : shard_id,
2296 72 : timeline_id,
2297 72 : ]);
2298 72 : let _ = STORAGE_TIME_COUNT_PER_TIMELINE.remove_label_values(&[
2299 72 : op,
2300 72 : tenant_id,
2301 72 : shard_id,
2302 72 : timeline_id,
2303 72 : ]);
2304 72 : }
2305 :
2306 24 : for op in STORAGE_IO_SIZE_OPERATIONS {
2307 16 : let _ = STORAGE_IO_SIZE.remove_label_values(&[op, tenant_id, shard_id, timeline_id]);
2308 16 : }
2309 :
2310 48 : for op in SmgrQueryType::iter() {
2311 40 : let _ = SMGR_QUERY_TIME_PER_TENANT_TIMELINE.remove_label_values(&[
2312 40 : op.into(),
2313 40 : tenant_id,
2314 40 : shard_id,
2315 40 : timeline_id,
2316 40 : ]);
2317 40 : }
2318 8 : }
2319 : }
2320 :
2321 6 : pub(crate) fn remove_tenant_metrics(tenant_shard_id: &TenantShardId) {
2322 6 : // Only shard zero deals in synthetic sizes
2323 6 : if tenant_shard_id.is_shard_zero() {
2324 6 : let tid = tenant_shard_id.tenant_id.to_string();
2325 6 : let _ = TENANT_SYNTHETIC_SIZE_METRIC.remove_label_values(&[&tid]);
2326 6 : }
2327 :
2328 : // we leave the BROKEN_TENANTS_SET entry if any
2329 6 : }
2330 :
2331 : use futures::Future;
2332 : use pin_project_lite::pin_project;
2333 : use std::collections::HashMap;
2334 : use std::num::NonZeroUsize;
2335 : use std::pin::Pin;
2336 : use std::sync::atomic::AtomicU64;
2337 : use std::sync::{Arc, Mutex};
2338 : use std::task::{Context, Poll};
2339 : use std::time::{Duration, Instant};
2340 :
2341 : use crate::context::{PageContentKind, RequestContext};
2342 : use crate::task_mgr::TaskKind;
2343 : use crate::tenant::mgr::TenantSlot;
2344 :
2345 : /// Maintain a per timeline gauge in addition to the global gauge.
2346 : pub(crate) struct PerTimelineRemotePhysicalSizeGauge {
2347 : last_set: AtomicU64,
2348 : gauge: UIntGauge,
2349 : }
2350 :
2351 : impl PerTimelineRemotePhysicalSizeGauge {
2352 390 : fn new(per_timeline_gauge: UIntGauge) -> Self {
2353 390 : Self {
2354 390 : last_set: AtomicU64::new(0),
2355 390 : gauge: per_timeline_gauge,
2356 390 : }
2357 390 : }
2358 1725 : pub(crate) fn set(&self, sz: u64) {
2359 1725 : self.gauge.set(sz);
2360 1725 : let prev = self.last_set.swap(sz, std::sync::atomic::Ordering::Relaxed);
2361 1725 : if sz < prev {
2362 26 : REMOTE_PHYSICAL_SIZE_GLOBAL.sub(prev - sz);
2363 1699 : } else {
2364 1699 : REMOTE_PHYSICAL_SIZE_GLOBAL.add(sz - prev);
2365 1699 : };
2366 1725 : }
2367 2 : pub(crate) fn get(&self) -> u64 {
2368 2 : self.gauge.get()
2369 2 : }
2370 : }
2371 :
2372 : impl Drop for PerTimelineRemotePhysicalSizeGauge {
2373 20 : fn drop(&mut self) {
2374 20 : REMOTE_PHYSICAL_SIZE_GLOBAL.sub(self.last_set.load(std::sync::atomic::Ordering::Relaxed));
2375 20 : }
2376 : }
2377 :
2378 : pub(crate) struct RemoteTimelineClientMetrics {
2379 : tenant_id: String,
2380 : shard_id: String,
2381 : timeline_id: String,
2382 : pub(crate) remote_physical_size_gauge: PerTimelineRemotePhysicalSizeGauge,
2383 : calls: Mutex<HashMap<(&'static str, &'static str), IntCounterPair>>,
2384 : bytes_started_counter: Mutex<HashMap<(&'static str, &'static str), IntCounter>>,
2385 : bytes_finished_counter: Mutex<HashMap<(&'static str, &'static str), IntCounter>>,
2386 : }
2387 :
2388 : impl RemoteTimelineClientMetrics {
2389 390 : pub fn new(tenant_shard_id: &TenantShardId, timeline_id: &TimelineId) -> Self {
2390 390 : let tenant_id_str = tenant_shard_id.tenant_id.to_string();
2391 390 : let shard_id_str = format!("{}", tenant_shard_id.shard_slug());
2392 390 : let timeline_id_str = timeline_id.to_string();
2393 390 :
2394 390 : let remote_physical_size_gauge = PerTimelineRemotePhysicalSizeGauge::new(
2395 390 : REMOTE_PHYSICAL_SIZE
2396 390 : .get_metric_with_label_values(&[&tenant_id_str, &shard_id_str, &timeline_id_str])
2397 390 : .unwrap(),
2398 390 : );
2399 390 :
2400 390 : RemoteTimelineClientMetrics {
2401 390 : tenant_id: tenant_id_str,
2402 390 : shard_id: shard_id_str,
2403 390 : timeline_id: timeline_id_str,
2404 390 : calls: Mutex::new(HashMap::default()),
2405 390 : bytes_started_counter: Mutex::new(HashMap::default()),
2406 390 : bytes_finished_counter: Mutex::new(HashMap::default()),
2407 390 : remote_physical_size_gauge,
2408 390 : }
2409 390 : }
2410 :
2411 2535 : pub fn remote_operation_time(
2412 2535 : &self,
2413 2535 : file_kind: &RemoteOpFileKind,
2414 2535 : op_kind: &RemoteOpKind,
2415 2535 : status: &'static str,
2416 2535 : ) -> Histogram {
2417 2535 : let key = (file_kind.as_str(), op_kind.as_str(), status);
2418 2535 : REMOTE_OPERATION_TIME
2419 2535 : .get_metric_with_label_values(&[key.0, key.1, key.2])
2420 2535 : .unwrap()
2421 2535 : }
2422 :
2423 6054 : fn calls_counter_pair(
2424 6054 : &self,
2425 6054 : file_kind: &RemoteOpFileKind,
2426 6054 : op_kind: &RemoteOpKind,
2427 6054 : ) -> IntCounterPair {
2428 6054 : let mut guard = self.calls.lock().unwrap();
2429 6054 : let key = (file_kind.as_str(), op_kind.as_str());
2430 6054 : let metric = guard.entry(key).or_insert_with(move || {
2431 668 : REMOTE_TIMELINE_CLIENT_CALLS
2432 668 : .get_metric_with_label_values(&[
2433 668 : &self.tenant_id,
2434 668 : &self.shard_id,
2435 668 : &self.timeline_id,
2436 668 : key.0,
2437 668 : key.1,
2438 668 : ])
2439 668 : .unwrap()
2440 6054 : });
2441 6054 : metric.clone()
2442 6054 : }
2443 :
2444 1464 : fn bytes_started_counter(
2445 1464 : &self,
2446 1464 : file_kind: &RemoteOpFileKind,
2447 1464 : op_kind: &RemoteOpKind,
2448 1464 : ) -> IntCounter {
2449 1464 : let mut guard = self.bytes_started_counter.lock().unwrap();
2450 1464 : let key = (file_kind.as_str(), op_kind.as_str());
2451 1464 : let metric = guard.entry(key).or_insert_with(move || {
2452 252 : REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER
2453 252 : .get_metric_with_label_values(&[
2454 252 : &self.tenant_id,
2455 252 : &self.shard_id,
2456 252 : &self.timeline_id,
2457 252 : key.0,
2458 252 : key.1,
2459 252 : ])
2460 252 : .unwrap()
2461 1464 : });
2462 1464 : metric.clone()
2463 1464 : }
2464 :
2465 2628 : fn bytes_finished_counter(
2466 2628 : &self,
2467 2628 : file_kind: &RemoteOpFileKind,
2468 2628 : op_kind: &RemoteOpKind,
2469 2628 : ) -> IntCounter {
2470 2628 : let mut guard = self.bytes_finished_counter.lock().unwrap();
2471 2628 : let key = (file_kind.as_str(), op_kind.as_str());
2472 2628 : let metric = guard.entry(key).or_insert_with(move || {
2473 252 : REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER
2474 252 : .get_metric_with_label_values(&[
2475 252 : &self.tenant_id,
2476 252 : &self.shard_id,
2477 252 : &self.timeline_id,
2478 252 : key.0,
2479 252 : key.1,
2480 252 : ])
2481 252 : .unwrap()
2482 2628 : });
2483 2628 : metric.clone()
2484 2628 : }
2485 : }
2486 :
2487 : #[cfg(test)]
2488 : impl RemoteTimelineClientMetrics {
2489 6 : pub fn get_bytes_started_counter_value(
2490 6 : &self,
2491 6 : file_kind: &RemoteOpFileKind,
2492 6 : op_kind: &RemoteOpKind,
2493 6 : ) -> Option<u64> {
2494 6 : let guard = self.bytes_started_counter.lock().unwrap();
2495 6 : let key = (file_kind.as_str(), op_kind.as_str());
2496 6 : guard.get(&key).map(|counter| counter.get())
2497 6 : }
2498 :
2499 6 : pub fn get_bytes_finished_counter_value(
2500 6 : &self,
2501 6 : file_kind: &RemoteOpFileKind,
2502 6 : op_kind: &RemoteOpKind,
2503 6 : ) -> Option<u64> {
2504 6 : let guard = self.bytes_finished_counter.lock().unwrap();
2505 6 : let key = (file_kind.as_str(), op_kind.as_str());
2506 6 : guard.get(&key).map(|counter| counter.get())
2507 6 : }
2508 : }
2509 :
2510 : /// See [`RemoteTimelineClientMetrics::call_begin`].
2511 : #[must_use]
2512 : pub(crate) struct RemoteTimelineClientCallMetricGuard {
2513 : /// Decremented on drop.
2514 : calls_counter_pair: Option<IntCounterPair>,
2515 : /// If Some(), this references the bytes_finished metric, and we increment it by the given `u64` on drop.
2516 : bytes_finished: Option<(IntCounter, u64)>,
2517 : }
2518 :
2519 : impl RemoteTimelineClientCallMetricGuard {
2520 : /// Consume this guard object without performing the metric updates it would do on `drop()`.
2521 : /// The caller vouches to do the metric updates manually.
2522 3297 : pub fn will_decrement_manually(mut self) {
2523 3297 : let RemoteTimelineClientCallMetricGuard {
2524 3297 : calls_counter_pair,
2525 3297 : bytes_finished,
2526 3297 : } = &mut self;
2527 3297 : calls_counter_pair.take();
2528 3297 : bytes_finished.take();
2529 3297 : }
2530 : }
2531 :
2532 : impl Drop for RemoteTimelineClientCallMetricGuard {
2533 3323 : fn drop(&mut self) {
2534 3323 : let RemoteTimelineClientCallMetricGuard {
2535 3323 : calls_counter_pair,
2536 3323 : bytes_finished,
2537 3323 : } = self;
2538 3323 : if let Some(guard) = calls_counter_pair.take() {
2539 26 : guard.dec();
2540 3297 : }
2541 3323 : if let Some((bytes_finished_metric, value)) = bytes_finished {
2542 0 : bytes_finished_metric.inc_by(*value);
2543 3323 : }
2544 3323 : }
2545 : }
2546 :
2547 : /// The enum variants communicate to the [`RemoteTimelineClientMetrics`] whether to
2548 : /// track the byte size of this call in applicable metric(s).
2549 : pub(crate) enum RemoteTimelineClientMetricsCallTrackSize {
2550 : /// Do not account for this call's byte size in any metrics.
2551 : /// The `reason` field is there to make the call sites self-documenting
2552 : /// about why they don't need the metric.
2553 : DontTrackSize { reason: &'static str },
2554 : /// Track the byte size of the call in applicable metric(s).
2555 : Bytes(u64),
2556 : }
2557 :
2558 : impl RemoteTimelineClientMetrics {
2559 : /// Update the metrics that change when a call to the remote timeline client instance starts.
2560 : ///
2561 : /// Drop the returned guard object once the operation is finished to updates corresponding metrics that track completions.
2562 : /// Or, use [`RemoteTimelineClientCallMetricGuard::will_decrement_manually`] and [`call_end`](Self::call_end) if that
2563 : /// is more suitable.
2564 : /// Never do both.
2565 3323 : pub(crate) fn call_begin(
2566 3323 : &self,
2567 3323 : file_kind: &RemoteOpFileKind,
2568 3323 : op_kind: &RemoteOpKind,
2569 3323 : size: RemoteTimelineClientMetricsCallTrackSize,
2570 3323 : ) -> RemoteTimelineClientCallMetricGuard {
2571 3323 : let calls_counter_pair = self.calls_counter_pair(file_kind, op_kind);
2572 3323 : calls_counter_pair.inc();
2573 :
2574 3323 : let bytes_finished = match size {
2575 1859 : RemoteTimelineClientMetricsCallTrackSize::DontTrackSize { reason: _reason } => {
2576 1859 : // nothing to do
2577 1859 : None
2578 : }
2579 1464 : RemoteTimelineClientMetricsCallTrackSize::Bytes(size) => {
2580 1464 : self.bytes_started_counter(file_kind, op_kind).inc_by(size);
2581 1464 : let finished_counter = self.bytes_finished_counter(file_kind, op_kind);
2582 1464 : Some((finished_counter, size))
2583 : }
2584 : };
2585 3323 : RemoteTimelineClientCallMetricGuard {
2586 3323 : calls_counter_pair: Some(calls_counter_pair),
2587 3323 : bytes_finished,
2588 3323 : }
2589 3323 : }
2590 :
2591 : /// Manually udpate the metrics that track completions, instead of using the guard object.
2592 : /// Using the guard object is generally preferable.
2593 : /// See [`call_begin`](Self::call_begin) for more context.
2594 2731 : pub(crate) fn call_end(
2595 2731 : &self,
2596 2731 : file_kind: &RemoteOpFileKind,
2597 2731 : op_kind: &RemoteOpKind,
2598 2731 : size: RemoteTimelineClientMetricsCallTrackSize,
2599 2731 : ) {
2600 2731 : let calls_counter_pair = self.calls_counter_pair(file_kind, op_kind);
2601 2731 : calls_counter_pair.dec();
2602 2731 : match size {
2603 1567 : RemoteTimelineClientMetricsCallTrackSize::DontTrackSize { reason: _reason } => {}
2604 1164 : RemoteTimelineClientMetricsCallTrackSize::Bytes(size) => {
2605 1164 : self.bytes_finished_counter(file_kind, op_kind).inc_by(size);
2606 1164 : }
2607 : }
2608 2731 : }
2609 : }
2610 :
2611 : impl Drop for RemoteTimelineClientMetrics {
2612 20 : fn drop(&mut self) {
2613 20 : let RemoteTimelineClientMetrics {
2614 20 : tenant_id,
2615 20 : shard_id,
2616 20 : timeline_id,
2617 20 : remote_physical_size_gauge,
2618 20 : calls,
2619 20 : bytes_started_counter,
2620 20 : bytes_finished_counter,
2621 20 : } = self;
2622 24 : for ((a, b), _) in calls.get_mut().unwrap().drain() {
2623 24 : let mut res = [Ok(()), Ok(())];
2624 24 : REMOTE_TIMELINE_CLIENT_CALLS
2625 24 : .remove_label_values(&mut res, &[tenant_id, shard_id, timeline_id, a, b]);
2626 24 : // don't care about results
2627 24 : }
2628 20 : for ((a, b), _) in bytes_started_counter.get_mut().unwrap().drain() {
2629 6 : let _ = REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER.remove_label_values(&[
2630 6 : tenant_id,
2631 6 : shard_id,
2632 6 : timeline_id,
2633 6 : a,
2634 6 : b,
2635 6 : ]);
2636 6 : }
2637 20 : for ((a, b), _) in bytes_finished_counter.get_mut().unwrap().drain() {
2638 6 : let _ = REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER.remove_label_values(&[
2639 6 : tenant_id,
2640 6 : shard_id,
2641 6 : timeline_id,
2642 6 : a,
2643 6 : b,
2644 6 : ]);
2645 6 : }
2646 20 : {
2647 20 : let _ = remote_physical_size_gauge; // use to avoid 'unused' warning in desctructuring above
2648 20 : let _ = REMOTE_PHYSICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2649 20 : }
2650 20 : }
2651 : }
2652 :
2653 : /// Wrapper future that measures the time spent by a remote storage operation,
2654 : /// and records the time and success/failure as a prometheus metric.
2655 : pub(crate) trait MeasureRemoteOp: Sized {
2656 2762 : fn measure_remote_op(
2657 2762 : self,
2658 2762 : file_kind: RemoteOpFileKind,
2659 2762 : op: RemoteOpKind,
2660 2762 : metrics: Arc<RemoteTimelineClientMetrics>,
2661 2762 : ) -> MeasuredRemoteOp<Self> {
2662 2762 : let start = Instant::now();
2663 2762 : MeasuredRemoteOp {
2664 2762 : inner: self,
2665 2762 : file_kind,
2666 2762 : op,
2667 2762 : start,
2668 2762 : metrics,
2669 2762 : }
2670 2762 : }
2671 : }
2672 :
2673 : impl<T: Sized> MeasureRemoteOp for T {}
2674 :
2675 : pin_project! {
2676 : pub(crate) struct MeasuredRemoteOp<F>
2677 : {
2678 : #[pin]
2679 : inner: F,
2680 : file_kind: RemoteOpFileKind,
2681 : op: RemoteOpKind,
2682 : start: Instant,
2683 : metrics: Arc<RemoteTimelineClientMetrics>,
2684 : }
2685 : }
2686 :
2687 : impl<F: Future<Output = Result<O, E>>, O, E> Future for MeasuredRemoteOp<F> {
2688 : type Output = Result<O, E>;
2689 :
2690 40476 : fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
2691 40476 : let this = self.project();
2692 40476 : let poll_result = this.inner.poll(cx);
2693 40476 : if let Poll::Ready(ref res) = poll_result {
2694 2535 : let duration = this.start.elapsed();
2695 2535 : let status = if res.is_ok() { &"success" } else { &"failure" };
2696 2535 : this.metrics
2697 2535 : .remote_operation_time(this.file_kind, this.op, status)
2698 2535 : .observe(duration.as_secs_f64());
2699 37941 : }
2700 40476 : poll_result
2701 40476 : }
2702 : }
2703 :
2704 : pub mod tokio_epoll_uring {
2705 : use metrics::{register_int_counter, UIntGauge};
2706 : use once_cell::sync::Lazy;
2707 :
2708 : pub struct Collector {
2709 : descs: Vec<metrics::core::Desc>,
2710 : systems_created: UIntGauge,
2711 : systems_destroyed: UIntGauge,
2712 : }
2713 :
2714 : impl metrics::core::Collector for Collector {
2715 0 : fn desc(&self) -> Vec<&metrics::core::Desc> {
2716 0 : self.descs.iter().collect()
2717 0 : }
2718 :
2719 0 : fn collect(&self) -> Vec<metrics::proto::MetricFamily> {
2720 0 : let mut mfs = Vec::with_capacity(Self::NMETRICS);
2721 0 : let tokio_epoll_uring::metrics::Metrics {
2722 0 : systems_created,
2723 0 : systems_destroyed,
2724 0 : } = tokio_epoll_uring::metrics::global();
2725 0 : self.systems_created.set(systems_created);
2726 0 : mfs.extend(self.systems_created.collect());
2727 0 : self.systems_destroyed.set(systems_destroyed);
2728 0 : mfs.extend(self.systems_destroyed.collect());
2729 0 : mfs
2730 0 : }
2731 : }
2732 :
2733 : impl Collector {
2734 : const NMETRICS: usize = 2;
2735 :
2736 : #[allow(clippy::new_without_default)]
2737 0 : pub fn new() -> Self {
2738 0 : let mut descs = Vec::new();
2739 0 :
2740 0 : let systems_created = UIntGauge::new(
2741 0 : "pageserver_tokio_epoll_uring_systems_created",
2742 0 : "counter of tokio-epoll-uring systems that were created",
2743 0 : )
2744 0 : .unwrap();
2745 0 : descs.extend(
2746 0 : metrics::core::Collector::desc(&systems_created)
2747 0 : .into_iter()
2748 0 : .cloned(),
2749 0 : );
2750 0 :
2751 0 : let systems_destroyed = UIntGauge::new(
2752 0 : "pageserver_tokio_epoll_uring_systems_destroyed",
2753 0 : "counter of tokio-epoll-uring systems that were destroyed",
2754 0 : )
2755 0 : .unwrap();
2756 0 : descs.extend(
2757 0 : metrics::core::Collector::desc(&systems_destroyed)
2758 0 : .into_iter()
2759 0 : .cloned(),
2760 0 : );
2761 0 :
2762 0 : Self {
2763 0 : descs,
2764 0 : systems_created,
2765 0 : systems_destroyed,
2766 0 : }
2767 0 : }
2768 : }
2769 :
2770 80 : pub(crate) static THREAD_LOCAL_LAUNCH_SUCCESSES: Lazy<metrics::IntCounter> = Lazy::new(|| {
2771 : register_int_counter!(
2772 : "pageserver_tokio_epoll_uring_pageserver_thread_local_launch_success_count",
2773 : "Number of times where thread_local_system creation spanned multiple executor threads",
2774 : )
2775 80 : .unwrap()
2776 80 : });
2777 :
2778 0 : pub(crate) static THREAD_LOCAL_LAUNCH_FAILURES: Lazy<metrics::IntCounter> = Lazy::new(|| {
2779 : register_int_counter!(
2780 : "pageserver_tokio_epoll_uring_pageserver_thread_local_launch_failures_count",
2781 : "Number of times thread_local_system creation failed and was retried after back-off.",
2782 : )
2783 0 : .unwrap()
2784 0 : });
2785 : }
2786 :
2787 : pub(crate) mod tenant_throttling {
2788 : use metrics::{register_int_counter_vec, IntCounter};
2789 : use once_cell::sync::Lazy;
2790 :
2791 : use crate::tenant::{self, throttle::Metric};
2792 :
2793 : pub(crate) struct TimelineGet {
2794 : wait_time: IntCounter,
2795 : count: IntCounter,
2796 : }
2797 :
2798 140 : pub(crate) static TIMELINE_GET: Lazy<TimelineGet> = Lazy::new(|| {
2799 140 : static WAIT_USECS: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
2800 140 : register_int_counter_vec!(
2801 140 : "pageserver_tenant_throttling_wait_usecs_sum_global",
2802 140 : "Sum of microseconds that tenants spent waiting for a tenant throttle of a given kind.",
2803 140 : &["kind"]
2804 140 : )
2805 140 : .unwrap()
2806 140 : });
2807 140 :
2808 140 : static WAIT_COUNT: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
2809 140 : register_int_counter_vec!(
2810 140 : "pageserver_tenant_throttling_count_global",
2811 140 : "Count of tenant throttlings, by kind of throttle.",
2812 140 : &["kind"]
2813 140 : )
2814 140 : .unwrap()
2815 140 : });
2816 140 :
2817 140 : let kind = "timeline_get";
2818 140 : TimelineGet {
2819 140 : wait_time: WAIT_USECS.with_label_values(&[kind]),
2820 140 : count: WAIT_COUNT.with_label_values(&[kind]),
2821 140 : }
2822 140 : });
2823 :
2824 : impl Metric for &'static TimelineGet {
2825 : #[inline(always)]
2826 0 : fn observe_throttling(
2827 0 : &self,
2828 0 : tenant::throttle::Observation { wait_time }: &tenant::throttle::Observation,
2829 0 : ) {
2830 0 : let val = u64::try_from(wait_time.as_micros()).unwrap();
2831 0 : self.wait_time.inc_by(val);
2832 0 : self.count.inc();
2833 0 : }
2834 : }
2835 : }
2836 :
2837 : pub(crate) mod disk_usage_based_eviction {
2838 : use super::*;
2839 :
2840 : pub(crate) struct Metrics {
2841 : pub(crate) tenant_collection_time: Histogram,
2842 : pub(crate) tenant_layer_count: Histogram,
2843 : pub(crate) layers_collected: IntCounter,
2844 : pub(crate) layers_selected: IntCounter,
2845 : pub(crate) layers_evicted: IntCounter,
2846 : }
2847 :
2848 : impl Default for Metrics {
2849 0 : fn default() -> Self {
2850 0 : let tenant_collection_time = register_histogram!(
2851 : "pageserver_disk_usage_based_eviction_tenant_collection_seconds",
2852 : "Time spent collecting layers from a tenant -- not normalized by collected layer amount",
2853 : vec![0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0, 10.0]
2854 : )
2855 0 : .unwrap();
2856 0 :
2857 0 : let tenant_layer_count = register_histogram!(
2858 : "pageserver_disk_usage_based_eviction_tenant_collected_layers",
2859 : "Amount of layers gathered from a tenant",
2860 : vec![5.0, 50.0, 500.0, 5000.0, 50000.0]
2861 : )
2862 0 : .unwrap();
2863 0 :
2864 0 : let layers_collected = register_int_counter!(
2865 : "pageserver_disk_usage_based_eviction_collected_layers_total",
2866 : "Amount of layers collected"
2867 : )
2868 0 : .unwrap();
2869 0 :
2870 0 : let layers_selected = register_int_counter!(
2871 : "pageserver_disk_usage_based_eviction_select_layers_total",
2872 : "Amount of layers selected"
2873 : )
2874 0 : .unwrap();
2875 0 :
2876 0 : let layers_evicted = register_int_counter!(
2877 : "pageserver_disk_usage_based_eviction_evicted_layers_total",
2878 : "Amount of layers successfully evicted"
2879 : )
2880 0 : .unwrap();
2881 0 :
2882 0 : Self {
2883 0 : tenant_collection_time,
2884 0 : tenant_layer_count,
2885 0 : layers_collected,
2886 0 : layers_selected,
2887 0 : layers_evicted,
2888 0 : }
2889 0 : }
2890 : }
2891 :
2892 : pub(crate) static METRICS: Lazy<Metrics> = Lazy::new(Metrics::default);
2893 : }
2894 :
2895 132 : static TOKIO_EXECUTOR_THREAD_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
2896 : register_uint_gauge_vec!(
2897 : "pageserver_tokio_executor_thread_configured_count",
2898 : "Total number of configued tokio executor threads in the process.
2899 : The `setup` label denotes whether we're running with multiple runtimes or a single runtime.",
2900 : &["setup"],
2901 : )
2902 132 : .unwrap()
2903 132 : });
2904 :
2905 132 : pub(crate) fn set_tokio_runtime_setup(setup: &str, num_threads: NonZeroUsize) {
2906 132 : static SERIALIZE: std::sync::Mutex<()> = std::sync::Mutex::new(());
2907 132 : let _guard = SERIALIZE.lock().unwrap();
2908 132 : TOKIO_EXECUTOR_THREAD_COUNT.reset();
2909 132 : TOKIO_EXECUTOR_THREAD_COUNT
2910 132 : .get_metric_with_label_values(&[setup])
2911 132 : .unwrap()
2912 132 : .set(u64::try_from(num_threads.get()).unwrap());
2913 132 : }
2914 :
2915 0 : pub fn preinitialize_metrics() {
2916 0 : // Python tests need these and on some we do alerting.
2917 0 : //
2918 0 : // FIXME(4813): make it so that we have no top level metrics as this fn will easily fall out of
2919 0 : // order:
2920 0 : // - global metrics reside in a Lazy<PageserverMetrics>
2921 0 : // - access via crate::metrics::PS_METRICS.materialized_page_cache_hit.inc()
2922 0 : // - could move the statics into TimelineMetrics::new()?
2923 0 :
2924 0 : // counters
2925 0 : [
2926 0 : &MATERIALIZED_PAGE_CACHE_HIT,
2927 0 : &MATERIALIZED_PAGE_CACHE_HIT_DIRECT,
2928 0 : &UNEXPECTED_ONDEMAND_DOWNLOADS,
2929 0 : &WALRECEIVER_STARTED_CONNECTIONS,
2930 0 : &WALRECEIVER_BROKER_UPDATES,
2931 0 : &WALRECEIVER_CANDIDATES_ADDED,
2932 0 : &WALRECEIVER_CANDIDATES_REMOVED,
2933 0 : &tokio_epoll_uring::THREAD_LOCAL_LAUNCH_FAILURES,
2934 0 : &tokio_epoll_uring::THREAD_LOCAL_LAUNCH_SUCCESSES,
2935 0 : &REMOTE_ONDEMAND_DOWNLOADED_LAYERS,
2936 0 : &REMOTE_ONDEMAND_DOWNLOADED_BYTES,
2937 0 : ]
2938 0 : .into_iter()
2939 0 : .for_each(|c| {
2940 0 : Lazy::force(c);
2941 0 : });
2942 0 :
2943 0 : // Deletion queue stats
2944 0 : Lazy::force(&DELETION_QUEUE);
2945 0 :
2946 0 : // Tenant stats
2947 0 : Lazy::force(&TENANT);
2948 0 :
2949 0 : // Tenant manager stats
2950 0 : Lazy::force(&TENANT_MANAGER);
2951 0 :
2952 0 : Lazy::force(&crate::tenant::storage_layer::layer::LAYER_IMPL_METRICS);
2953 0 : Lazy::force(&disk_usage_based_eviction::METRICS);
2954 :
2955 0 : for state_name in pageserver_api::models::TenantState::VARIANTS {
2956 0 : // initialize the metric for all gauges, otherwise the time series might seemingly show
2957 0 : // values from last restart.
2958 0 : TENANT_STATE_METRIC.with_label_values(&[state_name]).set(0);
2959 0 : }
2960 :
2961 : // countervecs
2962 0 : [&BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT]
2963 0 : .into_iter()
2964 0 : .for_each(|c| {
2965 0 : Lazy::force(c);
2966 0 : });
2967 0 :
2968 0 : // gauges
2969 0 : WALRECEIVER_ACTIVE_MANAGERS.get();
2970 0 :
2971 0 : // histograms
2972 0 : [
2973 0 : &READ_NUM_LAYERS_VISITED,
2974 0 : &VEC_READ_NUM_LAYERS_VISITED,
2975 0 : &WAIT_LSN_TIME,
2976 0 : &WAL_REDO_TIME,
2977 0 : &WAL_REDO_RECORDS_HISTOGRAM,
2978 0 : &WAL_REDO_BYTES_HISTOGRAM,
2979 0 : &WAL_REDO_PROCESS_LAUNCH_DURATION_HISTOGRAM,
2980 0 : ]
2981 0 : .into_iter()
2982 0 : .for_each(|h| {
2983 0 : Lazy::force(h);
2984 0 : });
2985 0 :
2986 0 : // Custom
2987 0 : Lazy::force(&RECONSTRUCT_TIME);
2988 0 : Lazy::force(&tenant_throttling::TIMELINE_GET);
2989 0 : }
|
