Line data Source code
1 : use enum_map::EnumMap;
2 : use metrics::{
3 : register_counter_vec, register_gauge_vec, register_histogram, register_histogram_vec,
4 : register_int_counter, register_int_counter_pair_vec, register_int_counter_vec,
5 : register_int_gauge, register_int_gauge_vec, register_uint_gauge, register_uint_gauge_vec,
6 : Counter, CounterVec, GaugeVec, Histogram, HistogramVec, IntCounter, IntCounterPair,
7 : IntCounterPairVec, IntCounterVec, IntGauge, IntGaugeVec, UIntGauge, UIntGaugeVec,
8 : };
9 : use once_cell::sync::Lazy;
10 : use pageserver_api::shard::TenantShardId;
11 : use strum::{EnumCount, IntoEnumIterator, VariantNames};
12 : use strum_macros::{EnumVariantNames, IntoStaticStr};
13 : use tracing::warn;
14 : use utils::id::TimelineId;
15 :
16 : /// Prometheus histogram buckets (in seconds) for operations in the critical
17 : /// path. In other words, operations that directly affect that latency of user
18 : /// queries.
19 : ///
20 : /// The buckets capture the majority of latencies in the microsecond and
21 : /// millisecond range but also extend far enough up to distinguish "bad" from
22 : /// "really bad".
23 : const CRITICAL_OP_BUCKETS: &[f64] = &[
24 : 0.000_001, 0.000_010, 0.000_100, // 1 us, 10 us, 100 us
25 : 0.001_000, 0.010_000, 0.100_000, // 1 ms, 10 ms, 100 ms
26 : 1.0, 10.0, 100.0, // 1 s, 10 s, 100 s
27 : ];
28 :
29 : // Metrics collected on operations on the storage repository.
30 2672 : #[derive(Debug, EnumVariantNames, IntoStaticStr)]
31 : #[strum(serialize_all = "kebab_case")]
32 : pub(crate) enum StorageTimeOperation {
33 : #[strum(serialize = "layer flush")]
34 : LayerFlush,
35 :
36 : #[strum(serialize = "compact")]
37 : Compact,
38 :
39 : #[strum(serialize = "create images")]
40 : CreateImages,
41 :
42 : #[strum(serialize = "logical size")]
43 : LogicalSize,
44 :
45 : #[strum(serialize = "imitate logical size")]
46 : ImitateLogicalSize,
47 :
48 : #[strum(serialize = "load layer map")]
49 : LoadLayerMap,
50 :
51 : #[strum(serialize = "gc")]
52 : Gc,
53 :
54 : #[strum(serialize = "find gc cutoffs")]
55 : FindGcCutoffs,
56 :
57 : #[strum(serialize = "create tenant")]
58 : CreateTenant,
59 : }
60 :
61 106 : pub(crate) static STORAGE_TIME_SUM_PER_TIMELINE: Lazy<CounterVec> = Lazy::new(|| {
62 : register_counter_vec!(
63 : "pageserver_storage_operations_seconds_sum",
64 : "Total time spent on storage operations with operation, tenant and timeline dimensions",
65 : &["operation", "tenant_id", "shard_id", "timeline_id"],
66 : )
67 106 : .expect("failed to define a metric")
68 106 : });
69 :
70 106 : pub(crate) static STORAGE_TIME_COUNT_PER_TIMELINE: Lazy<IntCounterVec> = Lazy::new(|| {
71 : register_int_counter_vec!(
72 : "pageserver_storage_operations_seconds_count",
73 : "Count of storage operations with operation, tenant and timeline dimensions",
74 : &["operation", "tenant_id", "shard_id", "timeline_id"],
75 : )
76 106 : .expect("failed to define a metric")
77 106 : });
78 :
79 : // Buckets for background operations like compaction, GC, size calculation
80 : const STORAGE_OP_BUCKETS: &[f64] = &[0.010, 0.100, 1.0, 10.0, 100.0, 1000.0];
81 :
82 106 : pub(crate) static STORAGE_TIME_GLOBAL: Lazy<HistogramVec> = Lazy::new(|| {
83 : register_histogram_vec!(
84 : "pageserver_storage_operations_seconds_global",
85 : "Time spent on storage operations",
86 : &["operation"],
87 : STORAGE_OP_BUCKETS.into(),
88 : )
89 106 : .expect("failed to define a metric")
90 106 : });
91 :
92 104 : pub(crate) static READ_NUM_LAYERS_VISITED: Lazy<Histogram> = Lazy::new(|| {
93 : register_histogram!(
94 : "pageserver_layers_visited_per_read_global",
95 : "Number of layers visited to reconstruct one key",
96 : vec![1.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0],
97 : )
98 104 : .expect("failed to define a metric")
99 104 : });
100 :
101 8 : pub(crate) static VEC_READ_NUM_LAYERS_VISITED: Lazy<Histogram> = Lazy::new(|| {
102 : register_histogram!(
103 : "pageserver_layers_visited_per_vectored_read_global",
104 : "Average number of layers visited to reconstruct one key",
105 : vec![1.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0],
106 : )
107 8 : .expect("failed to define a metric")
108 8 : });
109 :
110 : // Metrics collected on operations on the storage repository.
111 : #[derive(
112 416 : Clone, Copy, enum_map::Enum, strum_macros::EnumString, strum_macros::Display, IntoStaticStr,
113 : )]
114 : pub(crate) enum GetKind {
115 : Singular,
116 : Vectored,
117 : }
118 :
119 : pub(crate) struct ReconstructTimeMetrics {
120 : singular: Histogram,
121 : vectored: Histogram,
122 : }
123 :
124 104 : pub(crate) static RECONSTRUCT_TIME: Lazy<ReconstructTimeMetrics> = Lazy::new(|| {
125 104 : let inner = register_histogram_vec!(
126 : "pageserver_getpage_reconstruct_seconds",
127 : "Time spent in reconstruct_value (reconstruct a page from deltas)",
128 : &["get_kind"],
129 : CRITICAL_OP_BUCKETS.into(),
130 : )
131 104 : .expect("failed to define a metric");
132 104 :
133 104 : ReconstructTimeMetrics {
134 104 : singular: inner.with_label_values(&[GetKind::Singular.into()]),
135 104 : vectored: inner.with_label_values(&[GetKind::Vectored.into()]),
136 104 : }
137 104 : });
138 :
139 : impl ReconstructTimeMetrics {
140 622849 : pub(crate) fn for_get_kind(&self, get_kind: GetKind) -> &Histogram {
141 622849 : match get_kind {
142 622817 : GetKind::Singular => &self.singular,
143 32 : GetKind::Vectored => &self.vectored,
144 : }
145 622849 : }
146 : }
147 :
148 0 : pub(crate) static MATERIALIZED_PAGE_CACHE_HIT_DIRECT: Lazy<IntCounter> = Lazy::new(|| {
149 : register_int_counter!(
150 : "pageserver_materialized_cache_hits_direct_total",
151 : "Number of cache hits from materialized page cache without redo",
152 : )
153 0 : .expect("failed to define a metric")
154 0 : });
155 :
156 : pub(crate) struct ReconstructDataTimeMetrics {
157 : singular: Histogram,
158 : vectored: Histogram,
159 : }
160 :
161 : impl ReconstructDataTimeMetrics {
162 622967 : pub(crate) fn for_get_kind(&self, get_kind: GetKind) -> &Histogram {
163 622967 : match get_kind {
164 622935 : GetKind::Singular => &self.singular,
165 32 : GetKind::Vectored => &self.vectored,
166 : }
167 622967 : }
168 : }
169 :
170 104 : pub(crate) static GET_RECONSTRUCT_DATA_TIME: Lazy<ReconstructDataTimeMetrics> = Lazy::new(|| {
171 104 : let inner = register_histogram_vec!(
172 : "pageserver_getpage_get_reconstruct_data_seconds",
173 : "Time spent in get_reconstruct_value_data",
174 : &["get_kind"],
175 : CRITICAL_OP_BUCKETS.into(),
176 : )
177 104 : .expect("failed to define a metric");
178 104 :
179 104 : ReconstructDataTimeMetrics {
180 104 : singular: inner.with_label_values(&[GetKind::Singular.into()]),
181 104 : vectored: inner.with_label_values(&[GetKind::Vectored.into()]),
182 104 : }
183 104 : });
184 :
185 0 : pub(crate) static MATERIALIZED_PAGE_CACHE_HIT: Lazy<IntCounter> = Lazy::new(|| {
186 : register_int_counter!(
187 : "pageserver_materialized_cache_hits_total",
188 : "Number of cache hits from materialized page cache",
189 : )
190 0 : .expect("failed to define a metric")
191 0 : });
192 :
193 : pub(crate) struct GetVectoredLatency {
194 : map: EnumMap<TaskKind, Option<Histogram>>,
195 : }
196 :
197 : #[allow(dead_code)]
198 : pub(crate) struct ScanLatency {
199 : map: EnumMap<TaskKind, Option<Histogram>>,
200 : }
201 :
202 : impl GetVectoredLatency {
203 : // Only these task types perform vectored gets. Filter all other tasks out to reduce total
204 : // cardinality of the metric.
205 : const TRACKED_TASK_KINDS: [TaskKind; 2] = [TaskKind::Compaction, TaskKind::PageRequestHandler];
206 :
207 616 : pub(crate) fn for_task_kind(&self, task_kind: TaskKind) -> Option<&Histogram> {
208 616 : self.map[task_kind].as_ref()
209 616 : }
210 : }
211 :
212 : impl ScanLatency {
213 : // Only these task types perform vectored gets. Filter all other tasks out to reduce total
214 : // cardinality of the metric.
215 : const TRACKED_TASK_KINDS: [TaskKind; 1] = [TaskKind::PageRequestHandler];
216 :
217 0 : pub(crate) fn for_task_kind(&self, task_kind: TaskKind) -> Option<&Histogram> {
218 0 : self.map[task_kind].as_ref()
219 0 : }
220 : }
221 :
222 : pub(crate) struct ScanLatencyOngoingRecording<'a> {
223 : parent: &'a Histogram,
224 : start: std::time::Instant,
225 : }
226 :
227 : impl<'a> ScanLatencyOngoingRecording<'a> {
228 0 : pub(crate) fn start_recording(parent: &'a Histogram) -> ScanLatencyOngoingRecording<'a> {
229 0 : let start = Instant::now();
230 0 : ScanLatencyOngoingRecording { parent, start }
231 0 : }
232 :
233 0 : pub(crate) fn observe(self, throttled: Option<Duration>) {
234 0 : let elapsed = self.start.elapsed();
235 0 : let ex_throttled = if let Some(throttled) = throttled {
236 0 : elapsed.checked_sub(throttled)
237 : } else {
238 0 : Some(elapsed)
239 : };
240 0 : if let Some(ex_throttled) = ex_throttled {
241 0 : self.parent.observe(ex_throttled.as_secs_f64());
242 0 : } else {
243 0 : use utils::rate_limit::RateLimit;
244 0 : static LOGGED: Lazy<Mutex<RateLimit>> =
245 0 : Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(10))));
246 0 : let mut rate_limit = LOGGED.lock().unwrap();
247 0 : rate_limit.call(|| {
248 0 : warn!("error deducting time spent throttled; this message is logged at a global rate limit");
249 0 : });
250 0 : }
251 0 : }
252 : }
253 :
254 98 : pub(crate) static GET_VECTORED_LATENCY: Lazy<GetVectoredLatency> = Lazy::new(|| {
255 98 : let inner = register_histogram_vec!(
256 : "pageserver_get_vectored_seconds",
257 : "Time spent in get_vectored, excluding time spent in timeline_get_throttle.",
258 : &["task_kind"],
259 : CRITICAL_OP_BUCKETS.into(),
260 : )
261 98 : .expect("failed to define a metric");
262 98 :
263 98 : GetVectoredLatency {
264 2940 : map: EnumMap::from_array(std::array::from_fn(|task_kind_idx| {
265 2940 : let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind_idx);
266 2940 :
267 2940 : if GetVectoredLatency::TRACKED_TASK_KINDS.contains(&task_kind) {
268 196 : let task_kind = task_kind.into();
269 196 : Some(inner.with_label_values(&[task_kind]))
270 : } else {
271 2744 : None
272 : }
273 2940 : })),
274 98 : }
275 98 : });
276 :
277 0 : pub(crate) static SCAN_LATENCY: Lazy<ScanLatency> = Lazy::new(|| {
278 0 : let inner = register_histogram_vec!(
279 : "pageserver_scan_seconds",
280 : "Time spent in scan, excluding time spent in timeline_get_throttle.",
281 : &["task_kind"],
282 : CRITICAL_OP_BUCKETS.into(),
283 : )
284 0 : .expect("failed to define a metric");
285 0 :
286 0 : ScanLatency {
287 0 : map: EnumMap::from_array(std::array::from_fn(|task_kind_idx| {
288 0 : let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind_idx);
289 0 :
290 0 : if ScanLatency::TRACKED_TASK_KINDS.contains(&task_kind) {
291 0 : let task_kind = task_kind.into();
292 0 : Some(inner.with_label_values(&[task_kind]))
293 : } else {
294 0 : None
295 : }
296 0 : })),
297 0 : }
298 0 : });
299 :
300 : pub(crate) struct PageCacheMetricsForTaskKind {
301 : pub read_accesses_materialized_page: IntCounter,
302 : pub read_accesses_immutable: IntCounter,
303 :
304 : pub read_hits_immutable: IntCounter,
305 : pub read_hits_materialized_page_exact: IntCounter,
306 : pub read_hits_materialized_page_older_lsn: IntCounter,
307 : }
308 :
309 : pub(crate) struct PageCacheMetrics {
310 : map: EnumMap<TaskKind, EnumMap<PageContentKind, PageCacheMetricsForTaskKind>>,
311 : }
312 :
313 106 : static PAGE_CACHE_READ_HITS: Lazy<IntCounterVec> = Lazy::new(|| {
314 : register_int_counter_vec!(
315 : "pageserver_page_cache_read_hits_total",
316 : "Number of read accesses to the page cache that hit",
317 : &["task_kind", "key_kind", "content_kind", "hit_kind"]
318 : )
319 106 : .expect("failed to define a metric")
320 106 : });
321 :
322 106 : static PAGE_CACHE_READ_ACCESSES: Lazy<IntCounterVec> = Lazy::new(|| {
323 : register_int_counter_vec!(
324 : "pageserver_page_cache_read_accesses_total",
325 : "Number of read accesses to the page cache",
326 : &["task_kind", "key_kind", "content_kind"]
327 : )
328 106 : .expect("failed to define a metric")
329 106 : });
330 :
331 106 : pub(crate) static PAGE_CACHE: Lazy<PageCacheMetrics> = Lazy::new(|| PageCacheMetrics {
332 3180 : map: EnumMap::from_array(std::array::from_fn(|task_kind| {
333 3180 : let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind);
334 3180 : let task_kind: &'static str = task_kind.into();
335 19080 : EnumMap::from_array(std::array::from_fn(|content_kind| {
336 19080 : let content_kind = <PageContentKind as enum_map::Enum>::from_usize(content_kind);
337 19080 : let content_kind: &'static str = content_kind.into();
338 19080 : PageCacheMetricsForTaskKind {
339 19080 : read_accesses_materialized_page: {
340 19080 : PAGE_CACHE_READ_ACCESSES
341 19080 : .get_metric_with_label_values(&[
342 19080 : task_kind,
343 19080 : "materialized_page",
344 19080 : content_kind,
345 19080 : ])
346 19080 : .unwrap()
347 19080 : },
348 19080 :
349 19080 : read_accesses_immutable: {
350 19080 : PAGE_CACHE_READ_ACCESSES
351 19080 : .get_metric_with_label_values(&[task_kind, "immutable", content_kind])
352 19080 : .unwrap()
353 19080 : },
354 19080 :
355 19080 : read_hits_immutable: {
356 19080 : PAGE_CACHE_READ_HITS
357 19080 : .get_metric_with_label_values(&[task_kind, "immutable", content_kind, "-"])
358 19080 : .unwrap()
359 19080 : },
360 19080 :
361 19080 : read_hits_materialized_page_exact: {
362 19080 : PAGE_CACHE_READ_HITS
363 19080 : .get_metric_with_label_values(&[
364 19080 : task_kind,
365 19080 : "materialized_page",
366 19080 : content_kind,
367 19080 : "exact",
368 19080 : ])
369 19080 : .unwrap()
370 19080 : },
371 19080 :
372 19080 : read_hits_materialized_page_older_lsn: {
373 19080 : PAGE_CACHE_READ_HITS
374 19080 : .get_metric_with_label_values(&[
375 19080 : task_kind,
376 19080 : "materialized_page",
377 19080 : content_kind,
378 19080 : "older_lsn",
379 19080 : ])
380 19080 : .unwrap()
381 19080 : },
382 19080 : }
383 19080 : }))
384 3180 : })),
385 106 : });
386 :
387 : impl PageCacheMetrics {
388 14765867 : pub(crate) fn for_ctx(&self, ctx: &RequestContext) -> &PageCacheMetricsForTaskKind {
389 14765867 : &self.map[ctx.task_kind()][ctx.page_content_kind()]
390 14765867 : }
391 : }
392 :
393 : pub(crate) struct PageCacheSizeMetrics {
394 : pub max_bytes: UIntGauge,
395 :
396 : pub current_bytes_immutable: UIntGauge,
397 : pub current_bytes_materialized_page: UIntGauge,
398 : }
399 :
400 106 : static PAGE_CACHE_SIZE_CURRENT_BYTES: Lazy<UIntGaugeVec> = Lazy::new(|| {
401 : register_uint_gauge_vec!(
402 : "pageserver_page_cache_size_current_bytes",
403 : "Current size of the page cache in bytes, by key kind",
404 : &["key_kind"]
405 : )
406 106 : .expect("failed to define a metric")
407 106 : });
408 :
409 : pub(crate) static PAGE_CACHE_SIZE: Lazy<PageCacheSizeMetrics> =
410 106 : Lazy::new(|| PageCacheSizeMetrics {
411 : max_bytes: {
412 : register_uint_gauge!(
413 : "pageserver_page_cache_size_max_bytes",
414 : "Maximum size of the page cache in bytes"
415 : )
416 106 : .expect("failed to define a metric")
417 106 : },
418 106 : current_bytes_immutable: {
419 106 : PAGE_CACHE_SIZE_CURRENT_BYTES
420 106 : .get_metric_with_label_values(&["immutable"])
421 106 : .unwrap()
422 106 : },
423 106 : current_bytes_materialized_page: {
424 106 : PAGE_CACHE_SIZE_CURRENT_BYTES
425 106 : .get_metric_with_label_values(&["materialized_page"])
426 106 : .unwrap()
427 106 : },
428 106 : });
429 :
430 : pub(crate) mod page_cache_eviction_metrics {
431 : use std::num::NonZeroUsize;
432 :
433 : use metrics::{register_int_counter_vec, IntCounter, IntCounterVec};
434 : use once_cell::sync::Lazy;
435 :
436 : #[derive(Clone, Copy)]
437 : pub(crate) enum Outcome {
438 : FoundSlotUnused { iters: NonZeroUsize },
439 : FoundSlotEvicted { iters: NonZeroUsize },
440 : ItersExceeded { iters: NonZeroUsize },
441 : }
442 :
443 44 : static ITERS_TOTAL_VEC: Lazy<IntCounterVec> = Lazy::new(|| {
444 : register_int_counter_vec!(
445 : "pageserver_page_cache_find_victim_iters_total",
446 : "Counter for the number of iterations in the find_victim loop",
447 : &["outcome"],
448 : )
449 44 : .expect("failed to define a metric")
450 44 : });
451 :
452 44 : static CALLS_VEC: Lazy<IntCounterVec> = Lazy::new(|| {
453 : register_int_counter_vec!(
454 : "pageserver_page_cache_find_victim_calls",
455 : "Incremented at the end of each find_victim() call.\
456 : Filter by outcome to get e.g., eviction rate.",
457 : &["outcome"]
458 : )
459 44 : .unwrap()
460 44 : });
461 :
462 164153 : pub(crate) fn observe(outcome: Outcome) {
463 164153 : macro_rules! dry {
464 164153 : ($label:literal, $iters:expr) => {{
465 164153 : static LABEL: &'static str = $label;
466 164153 : static ITERS_TOTAL: Lazy<IntCounter> =
467 164153 : Lazy::new(|| ITERS_TOTAL_VEC.with_label_values(&[LABEL]));
468 164153 : static CALLS: Lazy<IntCounter> =
469 164153 : Lazy::new(|| CALLS_VEC.with_label_values(&[LABEL]));
470 164153 : ITERS_TOTAL.inc_by(($iters.get()) as u64);
471 164153 : CALLS.inc();
472 164153 : }};
473 164153 : }
474 164153 : match outcome {
475 1398 : Outcome::FoundSlotUnused { iters } => dry!("found_empty", iters),
476 162755 : Outcome::FoundSlotEvicted { iters } => {
477 162755 : dry!("found_evicted", iters)
478 : }
479 0 : Outcome::ItersExceeded { iters } => {
480 0 : dry!("err_iters_exceeded", iters);
481 0 : super::page_cache_errors_inc(super::PageCacheErrorKind::EvictIterLimit);
482 0 : }
483 : }
484 164153 : }
485 : }
486 :
487 0 : static PAGE_CACHE_ERRORS: Lazy<IntCounterVec> = Lazy::new(|| {
488 : register_int_counter_vec!(
489 : "page_cache_errors_total",
490 : "Number of timeouts while acquiring a pinned slot in the page cache",
491 : &["error_kind"]
492 : )
493 0 : .expect("failed to define a metric")
494 0 : });
495 :
496 0 : #[derive(IntoStaticStr)]
497 : #[strum(serialize_all = "kebab_case")]
498 : pub(crate) enum PageCacheErrorKind {
499 : AcquirePinnedSlotTimeout,
500 : EvictIterLimit,
501 : }
502 :
503 0 : pub(crate) fn page_cache_errors_inc(error_kind: PageCacheErrorKind) {
504 0 : PAGE_CACHE_ERRORS
505 0 : .get_metric_with_label_values(&[error_kind.into()])
506 0 : .unwrap()
507 0 : .inc();
508 0 : }
509 :
510 14 : pub(crate) static WAIT_LSN_TIME: Lazy<Histogram> = Lazy::new(|| {
511 : register_histogram!(
512 : "pageserver_wait_lsn_seconds",
513 : "Time spent waiting for WAL to arrive",
514 : CRITICAL_OP_BUCKETS.into(),
515 : )
516 14 : .expect("failed to define a metric")
517 14 : });
518 :
519 106 : static LAST_RECORD_LSN: Lazy<IntGaugeVec> = Lazy::new(|| {
520 : register_int_gauge_vec!(
521 : "pageserver_last_record_lsn",
522 : "Last record LSN grouped by timeline",
523 : &["tenant_id", "shard_id", "timeline_id"]
524 : )
525 106 : .expect("failed to define a metric")
526 106 : });
527 :
528 106 : static RESIDENT_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
529 : register_uint_gauge_vec!(
530 : "pageserver_resident_physical_size",
531 : "The size of the layer files present in the pageserver's filesystem.",
532 : &["tenant_id", "shard_id", "timeline_id"]
533 : )
534 106 : .expect("failed to define a metric")
535 106 : });
536 :
537 102 : pub(crate) static RESIDENT_PHYSICAL_SIZE_GLOBAL: Lazy<UIntGauge> = Lazy::new(|| {
538 : register_uint_gauge!(
539 : "pageserver_resident_physical_size_global",
540 : "Like `pageserver_resident_physical_size`, but without tenant/timeline dimensions."
541 : )
542 102 : .expect("failed to define a metric")
543 102 : });
544 :
545 106 : static REMOTE_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
546 : register_uint_gauge_vec!(
547 : "pageserver_remote_physical_size",
548 : "The size of the layer files present in the remote storage that are listed in the remote index_part.json.",
549 : // Corollary: If any files are missing from the index part, they won't be included here.
550 : &["tenant_id", "shard_id", "timeline_id"]
551 : )
552 106 : .expect("failed to define a metric")
553 106 : });
554 :
555 106 : static REMOTE_PHYSICAL_SIZE_GLOBAL: Lazy<UIntGauge> = Lazy::new(|| {
556 : register_uint_gauge!(
557 : "pageserver_remote_physical_size_global",
558 : "Like `pageserver_remote_physical_size`, but without tenant/timeline dimensions."
559 : )
560 106 : .expect("failed to define a metric")
561 106 : });
562 :
563 4 : pub(crate) static REMOTE_ONDEMAND_DOWNLOADED_LAYERS: Lazy<IntCounter> = Lazy::new(|| {
564 : register_int_counter!(
565 : "pageserver_remote_ondemand_downloaded_layers_total",
566 : "Total on-demand downloaded layers"
567 : )
568 4 : .unwrap()
569 4 : });
570 :
571 4 : pub(crate) static REMOTE_ONDEMAND_DOWNLOADED_BYTES: Lazy<IntCounter> = Lazy::new(|| {
572 : register_int_counter!(
573 : "pageserver_remote_ondemand_downloaded_bytes_total",
574 : "Total bytes of layers on-demand downloaded",
575 : )
576 4 : .unwrap()
577 4 : });
578 :
579 106 : static CURRENT_LOGICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
580 : register_uint_gauge_vec!(
581 : "pageserver_current_logical_size",
582 : "Current logical size grouped by timeline",
583 : &["tenant_id", "shard_id", "timeline_id"]
584 : )
585 106 : .expect("failed to define current logical size metric")
586 106 : });
587 :
588 : pub(crate) mod initial_logical_size {
589 : use metrics::{register_int_counter, register_int_counter_vec, IntCounter, IntCounterVec};
590 : use once_cell::sync::Lazy;
591 :
592 : pub(crate) struct StartCalculation(IntCounterVec);
593 106 : pub(crate) static START_CALCULATION: Lazy<StartCalculation> = Lazy::new(|| {
594 106 : StartCalculation(
595 106 : register_int_counter_vec!(
596 : "pageserver_initial_logical_size_start_calculation",
597 : "Incremented each time we start an initial logical size calculation attempt. \
598 : The `circumstances` label provides some additional details.",
599 : &["attempt", "circumstances"]
600 106 : )
601 106 : .unwrap(),
602 106 : )
603 106 : });
604 :
605 : struct DropCalculation {
606 : first: IntCounter,
607 : retry: IntCounter,
608 : }
609 :
610 106 : static DROP_CALCULATION: Lazy<DropCalculation> = Lazy::new(|| {
611 106 : let vec = register_int_counter_vec!(
612 : "pageserver_initial_logical_size_drop_calculation",
613 : "Incremented each time we abort a started size calculation attmpt.",
614 : &["attempt"]
615 : )
616 106 : .unwrap();
617 106 : DropCalculation {
618 106 : first: vec.with_label_values(&["first"]),
619 106 : retry: vec.with_label_values(&["retry"]),
620 106 : }
621 106 : });
622 :
623 : pub(crate) struct Calculated {
624 : pub(crate) births: IntCounter,
625 : pub(crate) deaths: IntCounter,
626 : }
627 :
628 106 : pub(crate) static CALCULATED: Lazy<Calculated> = Lazy::new(|| Calculated {
629 : births: register_int_counter!(
630 : "pageserver_initial_logical_size_finish_calculation",
631 : "Incremented every time we finish calculation of initial logical size.\
632 : If everything is working well, this should happen at most once per Timeline object."
633 : )
634 106 : .unwrap(),
635 : deaths: register_int_counter!(
636 : "pageserver_initial_logical_size_drop_finished_calculation",
637 : "Incremented when we drop a finished initial logical size calculation result.\
638 : Mainly useful to turn pageserver_initial_logical_size_finish_calculation into a gauge."
639 : )
640 106 : .unwrap(),
641 106 : });
642 :
643 : pub(crate) struct OngoingCalculationGuard {
644 : inc_drop_calculation: Option<IntCounter>,
645 : }
646 :
647 110 : #[derive(strum_macros::IntoStaticStr)]
648 : pub(crate) enum StartCircumstances {
649 : EmptyInitial,
650 : SkippedConcurrencyLimiter,
651 : AfterBackgroundTasksRateLimit,
652 : }
653 :
654 : impl StartCalculation {
655 110 : pub(crate) fn first(&self, circumstances: StartCircumstances) -> OngoingCalculationGuard {
656 110 : let circumstances_label: &'static str = circumstances.into();
657 110 : self.0
658 110 : .with_label_values(&["first", circumstances_label])
659 110 : .inc();
660 110 : OngoingCalculationGuard {
661 110 : inc_drop_calculation: Some(DROP_CALCULATION.first.clone()),
662 110 : }
663 110 : }
664 0 : pub(crate) fn retry(&self, circumstances: StartCircumstances) -> OngoingCalculationGuard {
665 0 : let circumstances_label: &'static str = circumstances.into();
666 0 : self.0
667 0 : .with_label_values(&["retry", circumstances_label])
668 0 : .inc();
669 0 : OngoingCalculationGuard {
670 0 : inc_drop_calculation: Some(DROP_CALCULATION.retry.clone()),
671 0 : }
672 0 : }
673 : }
674 :
675 : impl Drop for OngoingCalculationGuard {
676 110 : fn drop(&mut self) {
677 110 : if let Some(counter) = self.inc_drop_calculation.take() {
678 0 : counter.inc();
679 110 : }
680 110 : }
681 : }
682 :
683 : impl OngoingCalculationGuard {
684 110 : pub(crate) fn calculation_result_saved(mut self) -> FinishedCalculationGuard {
685 110 : drop(self.inc_drop_calculation.take());
686 110 : CALCULATED.births.inc();
687 110 : FinishedCalculationGuard {
688 110 : inc_on_drop: CALCULATED.deaths.clone(),
689 110 : }
690 110 : }
691 : }
692 :
693 : pub(crate) struct FinishedCalculationGuard {
694 : inc_on_drop: IntCounter,
695 : }
696 :
697 : impl Drop for FinishedCalculationGuard {
698 6 : fn drop(&mut self) {
699 6 : self.inc_on_drop.inc();
700 6 : }
701 : }
702 :
703 : // context: https://github.com/neondatabase/neon/issues/5963
704 : pub(crate) static TIMELINES_WHERE_WALRECEIVER_GOT_APPROXIMATE_SIZE: Lazy<IntCounter> =
705 0 : Lazy::new(|| {
706 : register_int_counter!(
707 : "pageserver_initial_logical_size_timelines_where_walreceiver_got_approximate_size",
708 : "Counter for the following event: walreceiver calls\
709 : Timeline::get_current_logical_size() and it returns `Approximate` for the first time."
710 : )
711 0 : .unwrap()
712 0 : });
713 : }
714 :
715 0 : static DIRECTORY_ENTRIES_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
716 : register_uint_gauge_vec!(
717 : "pageserver_directory_entries_count",
718 : "Sum of the entries in pageserver-stored directory listings",
719 : &["tenant_id", "shard_id", "timeline_id"]
720 : )
721 0 : .expect("failed to define a metric")
722 0 : });
723 :
724 108 : pub(crate) static TENANT_STATE_METRIC: Lazy<UIntGaugeVec> = Lazy::new(|| {
725 : register_uint_gauge_vec!(
726 : "pageserver_tenant_states_count",
727 : "Count of tenants per state",
728 : &["state"]
729 : )
730 108 : .expect("Failed to register pageserver_tenant_states_count metric")
731 108 : });
732 :
733 : /// A set of broken tenants.
734 : ///
735 : /// These are expected to be so rare that a set is fine. Set as in a new timeseries per each broken
736 : /// tenant.
737 10 : pub(crate) static BROKEN_TENANTS_SET: Lazy<UIntGaugeVec> = Lazy::new(|| {
738 : register_uint_gauge_vec!(
739 : "pageserver_broken_tenants_count",
740 : "Set of broken tenants",
741 : &["tenant_id", "shard_id"]
742 : )
743 10 : .expect("Failed to register pageserver_tenant_states_count metric")
744 10 : });
745 :
746 6 : pub(crate) static TENANT_SYNTHETIC_SIZE_METRIC: Lazy<UIntGaugeVec> = Lazy::new(|| {
747 : register_uint_gauge_vec!(
748 : "pageserver_tenant_synthetic_cached_size_bytes",
749 : "Synthetic size of each tenant in bytes",
750 : &["tenant_id"]
751 : )
752 6 : .expect("Failed to register pageserver_tenant_synthetic_cached_size_bytes metric")
753 6 : });
754 :
755 0 : pub(crate) static EVICTION_ITERATION_DURATION: Lazy<HistogramVec> = Lazy::new(|| {
756 : register_histogram_vec!(
757 : "pageserver_eviction_iteration_duration_seconds_global",
758 : "Time spent on a single eviction iteration",
759 : &["period_secs", "threshold_secs"],
760 : STORAGE_OP_BUCKETS.into(),
761 : )
762 0 : .expect("failed to define a metric")
763 0 : });
764 :
765 106 : static EVICTIONS: Lazy<IntCounterVec> = Lazy::new(|| {
766 : register_int_counter_vec!(
767 : "pageserver_evictions",
768 : "Number of layers evicted from the pageserver",
769 : &["tenant_id", "shard_id", "timeline_id"]
770 : )
771 106 : .expect("failed to define a metric")
772 106 : });
773 :
774 106 : static EVICTIONS_WITH_LOW_RESIDENCE_DURATION: Lazy<IntCounterVec> = Lazy::new(|| {
775 : register_int_counter_vec!(
776 : "pageserver_evictions_with_low_residence_duration",
777 : "If a layer is evicted that was resident for less than `low_threshold`, it is counted to this counter. \
778 : Residence duration is determined using the `residence_duration_data_source`.",
779 : &["tenant_id", "shard_id", "timeline_id", "residence_duration_data_source", "low_threshold_secs"]
780 : )
781 106 : .expect("failed to define a metric")
782 106 : });
783 :
784 0 : pub(crate) static UNEXPECTED_ONDEMAND_DOWNLOADS: Lazy<IntCounter> = Lazy::new(|| {
785 : register_int_counter!(
786 : "pageserver_unexpected_ondemand_downloads_count",
787 : "Number of unexpected on-demand downloads. \
788 : We log more context for each increment, so, forgo any labels in this metric.",
789 : )
790 0 : .expect("failed to define a metric")
791 0 : });
792 :
793 : /// How long did we take to start up? Broken down by labels to describe
794 : /// different phases of startup.
795 0 : pub static STARTUP_DURATION: Lazy<GaugeVec> = Lazy::new(|| {
796 : register_gauge_vec!(
797 : "pageserver_startup_duration_seconds",
798 : "Time taken by phases of pageserver startup, in seconds",
799 : &["phase"]
800 : )
801 0 : .expect("Failed to register pageserver_startup_duration_seconds metric")
802 0 : });
803 :
804 0 : pub static STARTUP_IS_LOADING: Lazy<UIntGauge> = Lazy::new(|| {
805 : register_uint_gauge!(
806 : "pageserver_startup_is_loading",
807 : "1 while in initial startup load of tenants, 0 at other times"
808 : )
809 0 : .expect("Failed to register pageserver_startup_is_loading")
810 0 : });
811 :
812 102 : pub(crate) static TIMELINE_EPHEMERAL_BYTES: Lazy<UIntGauge> = Lazy::new(|| {
813 : register_uint_gauge!(
814 : "pageserver_timeline_ephemeral_bytes",
815 : "Total number of bytes in ephemeral layers, summed for all timelines. Approximate, lazily updated."
816 : )
817 102 : .expect("Failed to register metric")
818 102 : });
819 :
820 : /// Metrics related to the lifecycle of a [`crate::tenant::Tenant`] object: things
821 : /// like how long it took to load.
822 : ///
823 : /// Note that these are process-global metrics, _not_ per-tenant metrics. Per-tenant
824 : /// metrics are rather expensive, and usually fine grained stuff makes more sense
825 : /// at a timeline level than tenant level.
826 : pub(crate) struct TenantMetrics {
827 : /// How long did tenants take to go from construction to active state?
828 : pub(crate) activation: Histogram,
829 : pub(crate) preload: Histogram,
830 : pub(crate) attach: Histogram,
831 :
832 : /// How many tenants are included in the initial startup of the pagesrever?
833 : pub(crate) startup_scheduled: IntCounter,
834 : pub(crate) startup_complete: IntCounter,
835 : }
836 :
837 0 : pub(crate) static TENANT: Lazy<TenantMetrics> = Lazy::new(|| {
838 0 : TenantMetrics {
839 0 : activation: register_histogram!(
840 : "pageserver_tenant_activation_seconds",
841 : "Time taken by tenants to activate, in seconds",
842 : CRITICAL_OP_BUCKETS.into()
843 0 : )
844 0 : .expect("Failed to register metric"),
845 0 : preload: register_histogram!(
846 : "pageserver_tenant_preload_seconds",
847 : "Time taken by tenants to load remote metadata on startup/attach, in seconds",
848 : CRITICAL_OP_BUCKETS.into()
849 0 : )
850 0 : .expect("Failed to register metric"),
851 0 : attach: register_histogram!(
852 : "pageserver_tenant_attach_seconds",
853 : "Time taken by tenants to intialize, after remote metadata is already loaded",
854 : CRITICAL_OP_BUCKETS.into()
855 0 : )
856 0 : .expect("Failed to register metric"),
857 0 : startup_scheduled: register_int_counter!(
858 : "pageserver_tenant_startup_scheduled",
859 : "Number of tenants included in pageserver startup (doesn't count tenants attached later)"
860 0 : ).expect("Failed to register metric"),
861 0 : startup_complete: register_int_counter!(
862 : "pageserver_tenant_startup_complete",
863 : "Number of tenants that have completed warm-up, or activated on-demand during initial startup: \
864 : should eventually reach `pageserver_tenant_startup_scheduled_total`. Does not include broken \
865 : tenants: such cases will lead to this metric never reaching the scheduled count."
866 0 : ).expect("Failed to register metric"),
867 0 : }
868 0 : });
869 :
870 : /// Each `Timeline`'s [`EVICTIONS_WITH_LOW_RESIDENCE_DURATION`] metric.
871 : #[derive(Debug)]
872 : pub(crate) struct EvictionsWithLowResidenceDuration {
873 : data_source: &'static str,
874 : threshold: Duration,
875 : counter: Option<IntCounter>,
876 : }
877 :
878 : pub(crate) struct EvictionsWithLowResidenceDurationBuilder {
879 : data_source: &'static str,
880 : threshold: Duration,
881 : }
882 :
883 : impl EvictionsWithLowResidenceDurationBuilder {
884 334 : pub fn new(data_source: &'static str, threshold: Duration) -> Self {
885 334 : Self {
886 334 : data_source,
887 334 : threshold,
888 334 : }
889 334 : }
890 :
891 334 : fn build(
892 334 : &self,
893 334 : tenant_id: &str,
894 334 : shard_id: &str,
895 334 : timeline_id: &str,
896 334 : ) -> EvictionsWithLowResidenceDuration {
897 334 : let counter = EVICTIONS_WITH_LOW_RESIDENCE_DURATION
898 334 : .get_metric_with_label_values(&[
899 334 : tenant_id,
900 334 : shard_id,
901 334 : timeline_id,
902 334 : self.data_source,
903 334 : &EvictionsWithLowResidenceDuration::threshold_label_value(self.threshold),
904 334 : ])
905 334 : .unwrap();
906 334 : EvictionsWithLowResidenceDuration {
907 334 : data_source: self.data_source,
908 334 : threshold: self.threshold,
909 334 : counter: Some(counter),
910 334 : }
911 334 : }
912 : }
913 :
914 : impl EvictionsWithLowResidenceDuration {
915 342 : fn threshold_label_value(threshold: Duration) -> String {
916 342 : format!("{}", threshold.as_secs())
917 342 : }
918 :
919 18 : pub fn observe(&self, observed_value: Duration) {
920 18 : if observed_value < self.threshold {
921 18 : self.counter
922 18 : .as_ref()
923 18 : .expect("nobody calls this function after `remove_from_vec`")
924 18 : .inc();
925 18 : }
926 18 : }
927 :
928 0 : pub fn change_threshold(
929 0 : &mut self,
930 0 : tenant_id: &str,
931 0 : shard_id: &str,
932 0 : timeline_id: &str,
933 0 : new_threshold: Duration,
934 0 : ) {
935 0 : if new_threshold == self.threshold {
936 0 : return;
937 0 : }
938 0 : let mut with_new = EvictionsWithLowResidenceDurationBuilder::new(
939 0 : self.data_source,
940 0 : new_threshold,
941 0 : )
942 0 : .build(tenant_id, shard_id, timeline_id);
943 0 : std::mem::swap(self, &mut with_new);
944 0 : with_new.remove(tenant_id, shard_id, timeline_id);
945 0 : }
946 :
947 : // This could be a `Drop` impl, but, we need the `tenant_id` and `timeline_id`.
948 8 : fn remove(&mut self, tenant_id: &str, shard_id: &str, timeline_id: &str) {
949 8 : let Some(_counter) = self.counter.take() else {
950 0 : return;
951 : };
952 :
953 8 : let threshold = Self::threshold_label_value(self.threshold);
954 8 :
955 8 : let removed = EVICTIONS_WITH_LOW_RESIDENCE_DURATION.remove_label_values(&[
956 8 : tenant_id,
957 8 : shard_id,
958 8 : timeline_id,
959 8 : self.data_source,
960 8 : &threshold,
961 8 : ]);
962 8 :
963 8 : match removed {
964 0 : Err(e) => {
965 0 : // this has been hit in staging as
966 0 : // <https://neondatabase.sentry.io/issues/4142396994/>, but we don't know how.
967 0 : // because we can be in the drop path already, don't risk:
968 0 : // - "double-panic => illegal instruction" or
969 0 : // - future "drop panick => abort"
970 0 : //
971 0 : // so just nag: (the error has the labels)
972 0 : tracing::warn!("failed to remove EvictionsWithLowResidenceDuration, it was already removed? {e:#?}");
973 : }
974 : Ok(()) => {
975 : // to help identify cases where we double-remove the same values, let's log all
976 : // deletions?
977 8 : tracing::info!("removed EvictionsWithLowResidenceDuration with {tenant_id}, {timeline_id}, {}, {threshold}", self.data_source);
978 : }
979 : }
980 8 : }
981 : }
982 :
983 : // Metrics collected on disk IO operations
984 : //
985 : // Roughly logarithmic scale.
986 : const STORAGE_IO_TIME_BUCKETS: &[f64] = &[
987 : 0.000030, // 30 usec
988 : 0.001000, // 1000 usec
989 : 0.030, // 30 ms
990 : 1.000, // 1000 ms
991 : 30.000, // 30000 ms
992 : ];
993 :
994 : /// VirtualFile fs operation variants.
995 : ///
996 : /// Operations:
997 : /// - open ([`std::fs::OpenOptions::open`])
998 : /// - close (dropping [`crate::virtual_file::VirtualFile`])
999 : /// - close-by-replace (close by replacement algorithm)
1000 : /// - read (`read_at`)
1001 : /// - write (`write_at`)
1002 : /// - seek (modify internal position or file length query)
1003 : /// - fsync ([`std::fs::File::sync_all`])
1004 : /// - metadata ([`std::fs::File::metadata`])
1005 : #[derive(
1006 1134 : Debug, Clone, Copy, strum_macros::EnumCount, strum_macros::EnumIter, strum_macros::FromRepr,
1007 : )]
1008 : pub(crate) enum StorageIoOperation {
1009 : Open,
1010 : OpenAfterReplace,
1011 : Close,
1012 : CloseByReplace,
1013 : Read,
1014 : Write,
1015 : Seek,
1016 : Fsync,
1017 : Metadata,
1018 : }
1019 :
1020 : impl StorageIoOperation {
1021 1134 : pub fn as_str(&self) -> &'static str {
1022 1134 : match self {
1023 126 : StorageIoOperation::Open => "open",
1024 126 : StorageIoOperation::OpenAfterReplace => "open-after-replace",
1025 126 : StorageIoOperation::Close => "close",
1026 126 : StorageIoOperation::CloseByReplace => "close-by-replace",
1027 126 : StorageIoOperation::Read => "read",
1028 126 : StorageIoOperation::Write => "write",
1029 126 : StorageIoOperation::Seek => "seek",
1030 126 : StorageIoOperation::Fsync => "fsync",
1031 126 : StorageIoOperation::Metadata => "metadata",
1032 : }
1033 1134 : }
1034 : }
1035 :
1036 : /// Tracks time taken by fs operations near VirtualFile.
1037 : #[derive(Debug)]
1038 : pub(crate) struct StorageIoTime {
1039 : metrics: [Histogram; StorageIoOperation::COUNT],
1040 : }
1041 :
1042 : impl StorageIoTime {
1043 126 : fn new() -> Self {
1044 126 : let storage_io_histogram_vec = register_histogram_vec!(
1045 : "pageserver_io_operations_seconds",
1046 : "Time spent in IO operations",
1047 : &["operation"],
1048 : STORAGE_IO_TIME_BUCKETS.into()
1049 : )
1050 126 : .expect("failed to define a metric");
1051 1134 : let metrics = std::array::from_fn(|i| {
1052 1134 : let op = StorageIoOperation::from_repr(i).unwrap();
1053 1134 : storage_io_histogram_vec
1054 1134 : .get_metric_with_label_values(&[op.as_str()])
1055 1134 : .unwrap()
1056 1134 : });
1057 126 : Self { metrics }
1058 126 : }
1059 :
1060 797309 : pub(crate) fn get(&self, op: StorageIoOperation) -> &Histogram {
1061 797309 : &self.metrics[op as usize]
1062 797309 : }
1063 : }
1064 :
1065 : pub(crate) static STORAGE_IO_TIME_METRIC: Lazy<StorageIoTime> = Lazy::new(StorageIoTime::new);
1066 :
1067 : const STORAGE_IO_SIZE_OPERATIONS: &[&str] = &["read", "write"];
1068 :
1069 : // Needed for the https://neonprod.grafana.net/d/5uK9tHL4k/picking-tenant-for-relocation?orgId=1
1070 124 : pub(crate) static STORAGE_IO_SIZE: Lazy<IntGaugeVec> = Lazy::new(|| {
1071 : register_int_gauge_vec!(
1072 : "pageserver_io_operations_bytes_total",
1073 : "Total amount of bytes read/written in IO operations",
1074 : &["operation", "tenant_id", "shard_id", "timeline_id"]
1075 : )
1076 124 : .expect("failed to define a metric")
1077 124 : });
1078 :
1079 : #[cfg(not(test))]
1080 : pub(crate) mod virtual_file_descriptor_cache {
1081 : use super::*;
1082 :
1083 0 : pub(crate) static SIZE_MAX: Lazy<UIntGauge> = Lazy::new(|| {
1084 : register_uint_gauge!(
1085 : "pageserver_virtual_file_descriptor_cache_size_max",
1086 : "Maximum number of open file descriptors in the cache."
1087 : )
1088 0 : .unwrap()
1089 0 : });
1090 :
1091 : // SIZE_CURRENT: derive it like so:
1092 : // ```
1093 : // sum (pageserver_io_operations_seconds_count{operation=~"^(open|open-after-replace)$")
1094 : // -ignoring(operation)
1095 : // sum(pageserver_io_operations_seconds_count{operation=~"^(close|close-by-replace)$"}
1096 : // ```
1097 : }
1098 :
1099 : #[cfg(not(test))]
1100 : pub(crate) mod virtual_file_io_engine {
1101 : use super::*;
1102 :
1103 0 : pub(crate) static KIND: Lazy<UIntGaugeVec> = Lazy::new(|| {
1104 : register_uint_gauge_vec!(
1105 : "pageserver_virtual_file_io_engine_kind",
1106 : "The configured io engine for VirtualFile",
1107 : &["kind"],
1108 : )
1109 0 : .unwrap()
1110 0 : });
1111 : }
1112 :
1113 : #[derive(Debug)]
1114 : struct GlobalAndPerTimelineHistogram {
1115 : global: Histogram,
1116 : per_tenant_timeline: Histogram,
1117 : }
1118 :
1119 : impl GlobalAndPerTimelineHistogram {
1120 10 : fn observe(&self, value: f64) {
1121 10 : self.global.observe(value);
1122 10 : self.per_tenant_timeline.observe(value);
1123 10 : }
1124 : }
1125 :
1126 : struct GlobalAndPerTimelineHistogramTimer<'a, 'c> {
1127 : h: &'a GlobalAndPerTimelineHistogram,
1128 : ctx: &'c RequestContext,
1129 : start: std::time::Instant,
1130 : op: SmgrQueryType,
1131 : }
1132 :
1133 : impl<'a, 'c> Drop for GlobalAndPerTimelineHistogramTimer<'a, 'c> {
1134 10 : fn drop(&mut self) {
1135 10 : let elapsed = self.start.elapsed();
1136 10 : let ex_throttled = self
1137 10 : .ctx
1138 10 : .micros_spent_throttled
1139 10 : .close_and_checked_sub_from(elapsed);
1140 10 : let ex_throttled = match ex_throttled {
1141 10 : Ok(res) => res,
1142 0 : Err(error) => {
1143 0 : use utils::rate_limit::RateLimit;
1144 0 : static LOGGED: Lazy<Mutex<enum_map::EnumMap<SmgrQueryType, RateLimit>>> =
1145 0 : Lazy::new(|| {
1146 0 : Mutex::new(enum_map::EnumMap::from_array(std::array::from_fn(|_| {
1147 0 : RateLimit::new(Duration::from_secs(10))
1148 0 : })))
1149 0 : });
1150 0 : let mut guard = LOGGED.lock().unwrap();
1151 0 : let rate_limit = &mut guard[self.op];
1152 0 : rate_limit.call(|| {
1153 0 : warn!(op=?self.op, error, "error deducting time spent throttled; this message is logged at a global rate limit");
1154 0 : });
1155 0 : elapsed
1156 : }
1157 : };
1158 10 : self.h.observe(ex_throttled.as_secs_f64());
1159 10 : }
1160 : }
1161 :
1162 : #[derive(
1163 : Debug,
1164 : Clone,
1165 : Copy,
1166 3490 : IntoStaticStr,
1167 : strum_macros::EnumCount,
1168 112 : strum_macros::EnumIter,
1169 1720 : strum_macros::FromRepr,
1170 : enum_map::Enum,
1171 : )]
1172 : #[strum(serialize_all = "snake_case")]
1173 : pub enum SmgrQueryType {
1174 : GetRelExists,
1175 : GetRelSize,
1176 : GetPageAtLsn,
1177 : GetDbSize,
1178 : GetSlruSegment,
1179 : }
1180 :
1181 : #[derive(Debug)]
1182 : pub(crate) struct SmgrQueryTimePerTimeline {
1183 : metrics: [GlobalAndPerTimelineHistogram; SmgrQueryType::COUNT],
1184 : }
1185 :
1186 108 : static SMGR_QUERY_TIME_PER_TENANT_TIMELINE: Lazy<HistogramVec> = Lazy::new(|| {
1187 : register_histogram_vec!(
1188 : "pageserver_smgr_query_seconds",
1189 : "Time spent on smgr query handling, aggegated by query type and tenant/timeline.",
1190 : &["smgr_query_type", "tenant_id", "shard_id", "timeline_id"],
1191 : CRITICAL_OP_BUCKETS.into(),
1192 : )
1193 108 : .expect("failed to define a metric")
1194 108 : });
1195 :
1196 108 : static SMGR_QUERY_TIME_GLOBAL_BUCKETS: Lazy<Vec<f64>> = Lazy::new(|| {
1197 108 : [
1198 108 : 1,
1199 108 : 10,
1200 108 : 20,
1201 108 : 40,
1202 108 : 60,
1203 108 : 80,
1204 108 : 100,
1205 108 : 200,
1206 108 : 300,
1207 108 : 400,
1208 108 : 500,
1209 108 : 600,
1210 108 : 700,
1211 108 : 800,
1212 108 : 900,
1213 108 : 1_000, // 1ms
1214 108 : 2_000,
1215 108 : 4_000,
1216 108 : 6_000,
1217 108 : 8_000,
1218 108 : 10_000, // 10ms
1219 108 : 20_000,
1220 108 : 40_000,
1221 108 : 60_000,
1222 108 : 80_000,
1223 108 : 100_000,
1224 108 : 200_000,
1225 108 : 400_000,
1226 108 : 600_000,
1227 108 : 800_000,
1228 108 : 1_000_000, // 1s
1229 108 : 2_000_000,
1230 108 : 4_000_000,
1231 108 : 6_000_000,
1232 108 : 8_000_000,
1233 108 : 10_000_000, // 10s
1234 108 : 20_000_000,
1235 108 : 50_000_000,
1236 108 : 100_000_000,
1237 108 : 200_000_000,
1238 108 : 1_000_000_000, // 1000s
1239 108 : ]
1240 108 : .into_iter()
1241 108 : .map(Duration::from_micros)
1242 4428 : .map(|d| d.as_secs_f64())
1243 108 : .collect()
1244 108 : });
1245 :
1246 108 : static SMGR_QUERY_TIME_GLOBAL: Lazy<HistogramVec> = Lazy::new(|| {
1247 : register_histogram_vec!(
1248 : "pageserver_smgr_query_seconds_global",
1249 : "Time spent on smgr query handling, aggregated by query type.",
1250 : &["smgr_query_type"],
1251 : SMGR_QUERY_TIME_GLOBAL_BUCKETS.clone(),
1252 : )
1253 108 : .expect("failed to define a metric")
1254 108 : });
1255 :
1256 : impl SmgrQueryTimePerTimeline {
1257 344 : pub(crate) fn new(tenant_shard_id: &TenantShardId, timeline_id: &TimelineId) -> Self {
1258 344 : let tenant_id = tenant_shard_id.tenant_id.to_string();
1259 344 : let shard_slug = format!("{}", tenant_shard_id.shard_slug());
1260 344 : let timeline_id = timeline_id.to_string();
1261 1720 : let metrics = std::array::from_fn(|i| {
1262 1720 : let op = SmgrQueryType::from_repr(i).unwrap();
1263 1720 : let global = SMGR_QUERY_TIME_GLOBAL
1264 1720 : .get_metric_with_label_values(&[op.into()])
1265 1720 : .unwrap();
1266 1720 : let per_tenant_timeline = SMGR_QUERY_TIME_PER_TENANT_TIMELINE
1267 1720 : .get_metric_with_label_values(&[op.into(), &tenant_id, &shard_slug, &timeline_id])
1268 1720 : .unwrap();
1269 1720 : GlobalAndPerTimelineHistogram {
1270 1720 : global,
1271 1720 : per_tenant_timeline,
1272 1720 : }
1273 1720 : });
1274 344 : Self { metrics }
1275 344 : }
1276 10 : pub(crate) fn start_timer<'c: 'a, 'a>(
1277 10 : &'a self,
1278 10 : op: SmgrQueryType,
1279 10 : ctx: &'c RequestContext,
1280 10 : ) -> impl Drop + '_ {
1281 10 : let metric = &self.metrics[op as usize];
1282 10 : let start = Instant::now();
1283 10 : match ctx.micros_spent_throttled.open() {
1284 10 : Ok(()) => (),
1285 0 : Err(error) => {
1286 0 : use utils::rate_limit::RateLimit;
1287 0 : static LOGGED: Lazy<Mutex<enum_map::EnumMap<SmgrQueryType, RateLimit>>> =
1288 0 : Lazy::new(|| {
1289 0 : Mutex::new(enum_map::EnumMap::from_array(std::array::from_fn(|_| {
1290 0 : RateLimit::new(Duration::from_secs(10))
1291 0 : })))
1292 0 : });
1293 0 : let mut guard = LOGGED.lock().unwrap();
1294 0 : let rate_limit = &mut guard[op];
1295 0 : rate_limit.call(|| {
1296 0 : warn!(?op, error, "error opening micros_spent_throttled; this message is logged at a global rate limit");
1297 0 : });
1298 0 : }
1299 : }
1300 10 : GlobalAndPerTimelineHistogramTimer {
1301 10 : h: metric,
1302 10 : ctx,
1303 10 : start,
1304 10 : op,
1305 10 : }
1306 10 : }
1307 : }
1308 :
1309 : #[cfg(test)]
1310 : mod smgr_query_time_tests {
1311 : use pageserver_api::shard::TenantShardId;
1312 : use strum::IntoEnumIterator;
1313 : use utils::id::{TenantId, TimelineId};
1314 :
1315 : use crate::{
1316 : context::{DownloadBehavior, RequestContext},
1317 : task_mgr::TaskKind,
1318 : };
1319 :
1320 : // Regression test, we used hard-coded string constants before using an enum.
1321 : #[test]
1322 2 : fn op_label_name() {
1323 2 : use super::SmgrQueryType::*;
1324 2 : let expect: [(super::SmgrQueryType, &'static str); 5] = [
1325 2 : (GetRelExists, "get_rel_exists"),
1326 2 : (GetRelSize, "get_rel_size"),
1327 2 : (GetPageAtLsn, "get_page_at_lsn"),
1328 2 : (GetDbSize, "get_db_size"),
1329 2 : (GetSlruSegment, "get_slru_segment"),
1330 2 : ];
1331 12 : for (op, expect) in expect {
1332 10 : let actual: &'static str = op.into();
1333 10 : assert_eq!(actual, expect);
1334 : }
1335 2 : }
1336 :
1337 : #[test]
1338 2 : fn basic() {
1339 2 : let ops: Vec<_> = super::SmgrQueryType::iter().collect();
1340 :
1341 12 : for op in &ops {
1342 10 : let tenant_id = TenantId::generate();
1343 10 : let timeline_id = TimelineId::generate();
1344 10 : let metrics = super::SmgrQueryTimePerTimeline::new(
1345 10 : &TenantShardId::unsharded(tenant_id),
1346 10 : &timeline_id,
1347 10 : );
1348 10 :
1349 20 : let get_counts = || {
1350 20 : let global: u64 = ops
1351 20 : .iter()
1352 100 : .map(|op| metrics.metrics[*op as usize].global.get_sample_count())
1353 20 : .sum();
1354 20 : let per_tenant_timeline: u64 = ops
1355 20 : .iter()
1356 100 : .map(|op| {
1357 100 : metrics.metrics[*op as usize]
1358 100 : .per_tenant_timeline
1359 100 : .get_sample_count()
1360 100 : })
1361 20 : .sum();
1362 20 : (global, per_tenant_timeline)
1363 20 : };
1364 :
1365 10 : let (pre_global, pre_per_tenant_timeline) = get_counts();
1366 10 : assert_eq!(pre_per_tenant_timeline, 0);
1367 :
1368 10 : let ctx = RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Download);
1369 10 : let timer = metrics.start_timer(*op, &ctx);
1370 10 : drop(timer);
1371 10 :
1372 10 : let (post_global, post_per_tenant_timeline) = get_counts();
1373 10 : assert_eq!(post_per_tenant_timeline, 1);
1374 10 : assert!(post_global > pre_global);
1375 : }
1376 2 : }
1377 : }
1378 :
1379 : // keep in sync with control plane Go code so that we can validate
1380 : // compute's basebackup_ms metric with our perspective in the context of SLI/SLO.
1381 0 : static COMPUTE_STARTUP_BUCKETS: Lazy<[f64; 28]> = Lazy::new(|| {
1382 0 : // Go code uses milliseconds. Variable is called `computeStartupBuckets`
1383 0 : [
1384 0 : 5, 10, 20, 30, 50, 70, 100, 120, 150, 200, 250, 300, 350, 400, 450, 500, 600, 800, 1000,
1385 0 : 1500, 2000, 2500, 3000, 5000, 10000, 20000, 40000, 60000,
1386 0 : ]
1387 0 : .map(|ms| (ms as f64) / 1000.0)
1388 0 : });
1389 :
1390 : pub(crate) struct BasebackupQueryTime(HistogramVec);
1391 0 : pub(crate) static BASEBACKUP_QUERY_TIME: Lazy<BasebackupQueryTime> = Lazy::new(|| {
1392 0 : BasebackupQueryTime({
1393 0 : register_histogram_vec!(
1394 : "pageserver_basebackup_query_seconds",
1395 : "Histogram of basebackup queries durations, by result type",
1396 : &["result"],
1397 : COMPUTE_STARTUP_BUCKETS.to_vec(),
1398 0 : )
1399 0 : .expect("failed to define a metric")
1400 0 : })
1401 0 : });
1402 :
1403 : pub(crate) struct BasebackupQueryTimeOngoingRecording<'a, 'c> {
1404 : parent: &'a BasebackupQueryTime,
1405 : ctx: &'c RequestContext,
1406 : start: std::time::Instant,
1407 : }
1408 :
1409 : impl BasebackupQueryTime {
1410 0 : pub(crate) fn start_recording<'c: 'a, 'a>(
1411 0 : &'a self,
1412 0 : ctx: &'c RequestContext,
1413 0 : ) -> BasebackupQueryTimeOngoingRecording<'_, '_> {
1414 0 : let start = Instant::now();
1415 0 : match ctx.micros_spent_throttled.open() {
1416 0 : Ok(()) => (),
1417 0 : Err(error) => {
1418 0 : use utils::rate_limit::RateLimit;
1419 0 : static LOGGED: Lazy<Mutex<RateLimit>> =
1420 0 : Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(10))));
1421 0 : let mut rate_limit = LOGGED.lock().unwrap();
1422 0 : rate_limit.call(|| {
1423 0 : warn!(error, "error opening micros_spent_throttled; this message is logged at a global rate limit");
1424 0 : });
1425 0 : }
1426 : }
1427 0 : BasebackupQueryTimeOngoingRecording {
1428 0 : parent: self,
1429 0 : ctx,
1430 0 : start,
1431 0 : }
1432 0 : }
1433 : }
1434 :
1435 : impl<'a, 'c> BasebackupQueryTimeOngoingRecording<'a, 'c> {
1436 0 : pub(crate) fn observe<T, E>(self, res: &Result<T, E>) {
1437 0 : let elapsed = self.start.elapsed();
1438 0 : let ex_throttled = self
1439 0 : .ctx
1440 0 : .micros_spent_throttled
1441 0 : .close_and_checked_sub_from(elapsed);
1442 0 : let ex_throttled = match ex_throttled {
1443 0 : Ok(ex_throttled) => ex_throttled,
1444 0 : Err(error) => {
1445 0 : use utils::rate_limit::RateLimit;
1446 0 : static LOGGED: Lazy<Mutex<RateLimit>> =
1447 0 : Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(10))));
1448 0 : let mut rate_limit = LOGGED.lock().unwrap();
1449 0 : rate_limit.call(|| {
1450 0 : warn!(error, "error deducting time spent throttled; this message is logged at a global rate limit");
1451 0 : });
1452 0 : elapsed
1453 : }
1454 : };
1455 0 : let label_value = if res.is_ok() { "ok" } else { "error" };
1456 0 : let metric = self
1457 0 : .parent
1458 0 : .0
1459 0 : .get_metric_with_label_values(&[label_value])
1460 0 : .unwrap();
1461 0 : metric.observe(ex_throttled.as_secs_f64());
1462 0 : }
1463 : }
1464 :
1465 0 : pub(crate) static LIVE_CONNECTIONS_COUNT: Lazy<IntGaugeVec> = Lazy::new(|| {
1466 : register_int_gauge_vec!(
1467 : "pageserver_live_connections",
1468 : "Number of live network connections",
1469 : &["pageserver_connection_kind"]
1470 : )
1471 0 : .expect("failed to define a metric")
1472 0 : });
1473 :
1474 : // remote storage metrics
1475 :
1476 102 : static REMOTE_TIMELINE_CLIENT_CALLS: Lazy<IntCounterPairVec> = Lazy::new(|| {
1477 : register_int_counter_pair_vec!(
1478 : "pageserver_remote_timeline_client_calls_started",
1479 : "Number of started calls to remote timeline client.",
1480 : "pageserver_remote_timeline_client_calls_finished",
1481 : "Number of finshed calls to remote timeline client.",
1482 : &[
1483 : "tenant_id",
1484 : "shard_id",
1485 : "timeline_id",
1486 : "file_kind",
1487 : "op_kind"
1488 : ],
1489 : )
1490 102 : .unwrap()
1491 102 : });
1492 :
1493 : static REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER: Lazy<IntCounterVec> =
1494 100 : Lazy::new(|| {
1495 : register_int_counter_vec!(
1496 : "pageserver_remote_timeline_client_bytes_started",
1497 : "Incremented by the number of bytes associated with a remote timeline client operation. \
1498 : The increment happens when the operation is scheduled.",
1499 : &["tenant_id", "shard_id", "timeline_id", "file_kind", "op_kind"],
1500 : )
1501 100 : .expect("failed to define a metric")
1502 100 : });
1503 :
1504 100 : static REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER: Lazy<IntCounterVec> = Lazy::new(|| {
1505 : register_int_counter_vec!(
1506 : "pageserver_remote_timeline_client_bytes_finished",
1507 : "Incremented by the number of bytes associated with a remote timeline client operation. \
1508 : The increment happens when the operation finishes (regardless of success/failure/shutdown).",
1509 : &["tenant_id", "shard_id", "timeline_id", "file_kind", "op_kind"],
1510 : )
1511 100 : .expect("failed to define a metric")
1512 100 : });
1513 :
1514 : pub(crate) struct TenantManagerMetrics {
1515 : tenant_slots_attached: UIntGauge,
1516 : tenant_slots_secondary: UIntGauge,
1517 : tenant_slots_inprogress: UIntGauge,
1518 : pub(crate) tenant_slot_writes: IntCounter,
1519 : pub(crate) unexpected_errors: IntCounter,
1520 : }
1521 :
1522 : impl TenantManagerMetrics {
1523 : /// Helpers for tracking slots. Note that these do not track the lifetime of TenantSlot objects
1524 : /// exactly: they track the lifetime of the slots _in the tenant map_.
1525 2 : pub(crate) fn slot_inserted(&self, slot: &TenantSlot) {
1526 2 : match slot {
1527 0 : TenantSlot::Attached(_) => {
1528 0 : self.tenant_slots_attached.inc();
1529 0 : }
1530 0 : TenantSlot::Secondary(_) => {
1531 0 : self.tenant_slots_secondary.inc();
1532 0 : }
1533 2 : TenantSlot::InProgress(_) => {
1534 2 : self.tenant_slots_inprogress.inc();
1535 2 : }
1536 : }
1537 2 : }
1538 :
1539 2 : pub(crate) fn slot_removed(&self, slot: &TenantSlot) {
1540 2 : match slot {
1541 2 : TenantSlot::Attached(_) => {
1542 2 : self.tenant_slots_attached.dec();
1543 2 : }
1544 0 : TenantSlot::Secondary(_) => {
1545 0 : self.tenant_slots_secondary.dec();
1546 0 : }
1547 0 : TenantSlot::InProgress(_) => {
1548 0 : self.tenant_slots_inprogress.dec();
1549 0 : }
1550 : }
1551 2 : }
1552 :
1553 : #[cfg(all(debug_assertions, not(test)))]
1554 0 : pub(crate) fn slots_total(&self) -> u64 {
1555 0 : self.tenant_slots_attached.get()
1556 0 : + self.tenant_slots_secondary.get()
1557 0 : + self.tenant_slots_inprogress.get()
1558 0 : }
1559 : }
1560 :
1561 2 : pub(crate) static TENANT_MANAGER: Lazy<TenantManagerMetrics> = Lazy::new(|| {
1562 2 : let tenant_slots = register_uint_gauge_vec!(
1563 : "pageserver_tenant_manager_slots",
1564 : "How many slots currently exist, including all attached, secondary and in-progress operations",
1565 : &["mode"]
1566 : )
1567 2 : .expect("failed to define a metric");
1568 2 : TenantManagerMetrics {
1569 2 : tenant_slots_attached: tenant_slots
1570 2 : .get_metric_with_label_values(&["attached"])
1571 2 : .unwrap(),
1572 2 : tenant_slots_secondary: tenant_slots
1573 2 : .get_metric_with_label_values(&["secondary"])
1574 2 : .unwrap(),
1575 2 : tenant_slots_inprogress: tenant_slots
1576 2 : .get_metric_with_label_values(&["inprogress"])
1577 2 : .unwrap(),
1578 2 : tenant_slot_writes: register_int_counter!(
1579 : "pageserver_tenant_manager_slot_writes",
1580 : "Writes to a tenant slot, including all of create/attach/detach/delete"
1581 2 : )
1582 2 : .expect("failed to define a metric"),
1583 2 : unexpected_errors: register_int_counter!(
1584 : "pageserver_tenant_manager_unexpected_errors_total",
1585 : "Number of unexpected conditions encountered: nonzero value indicates a non-fatal bug."
1586 2 : )
1587 2 : .expect("failed to define a metric"),
1588 2 : }
1589 2 : });
1590 :
1591 : pub(crate) struct DeletionQueueMetrics {
1592 : pub(crate) keys_submitted: IntCounter,
1593 : pub(crate) keys_dropped: IntCounter,
1594 : pub(crate) keys_executed: IntCounter,
1595 : pub(crate) keys_validated: IntCounter,
1596 : pub(crate) dropped_lsn_updates: IntCounter,
1597 : pub(crate) unexpected_errors: IntCounter,
1598 : pub(crate) remote_errors: IntCounterVec,
1599 : }
1600 18 : pub(crate) static DELETION_QUEUE: Lazy<DeletionQueueMetrics> = Lazy::new(|| {
1601 18 : DeletionQueueMetrics{
1602 18 :
1603 18 : keys_submitted: register_int_counter!(
1604 : "pageserver_deletion_queue_submitted_total",
1605 : "Number of objects submitted for deletion"
1606 18 : )
1607 18 : .expect("failed to define a metric"),
1608 18 :
1609 18 : keys_dropped: register_int_counter!(
1610 : "pageserver_deletion_queue_dropped_total",
1611 : "Number of object deletions dropped due to stale generation."
1612 18 : )
1613 18 : .expect("failed to define a metric"),
1614 18 :
1615 18 : keys_executed: register_int_counter!(
1616 : "pageserver_deletion_queue_executed_total",
1617 : "Number of objects deleted. Only includes objects that we actually deleted, sum with pageserver_deletion_queue_dropped_total for the total number of keys processed to completion"
1618 18 : )
1619 18 : .expect("failed to define a metric"),
1620 18 :
1621 18 : keys_validated: register_int_counter!(
1622 : "pageserver_deletion_queue_validated_total",
1623 : "Number of keys validated for deletion. Sum with pageserver_deletion_queue_dropped_total for the total number of keys that have passed through the validation stage."
1624 18 : )
1625 18 : .expect("failed to define a metric"),
1626 18 :
1627 18 : dropped_lsn_updates: register_int_counter!(
1628 : "pageserver_deletion_queue_dropped_lsn_updates_total",
1629 : "Updates to remote_consistent_lsn dropped due to stale generation number."
1630 18 : )
1631 18 : .expect("failed to define a metric"),
1632 18 : unexpected_errors: register_int_counter!(
1633 : "pageserver_deletion_queue_unexpected_errors_total",
1634 : "Number of unexpected condiions that may stall the queue: any value above zero is unexpected."
1635 18 : )
1636 18 : .expect("failed to define a metric"),
1637 18 : remote_errors: register_int_counter_vec!(
1638 : "pageserver_deletion_queue_remote_errors_total",
1639 : "Retryable remote I/O errors while executing deletions, for example 503 responses to DeleteObjects",
1640 : &["op_kind"],
1641 18 : )
1642 18 : .expect("failed to define a metric")
1643 18 : }
1644 18 : });
1645 :
1646 : pub(crate) struct SecondaryModeMetrics {
1647 : pub(crate) upload_heatmap: IntCounter,
1648 : pub(crate) upload_heatmap_errors: IntCounter,
1649 : pub(crate) upload_heatmap_duration: Histogram,
1650 : pub(crate) download_heatmap: IntCounter,
1651 : pub(crate) download_layer: IntCounter,
1652 : }
1653 0 : pub(crate) static SECONDARY_MODE: Lazy<SecondaryModeMetrics> = Lazy::new(|| {
1654 0 : SecondaryModeMetrics {
1655 0 : upload_heatmap: register_int_counter!(
1656 : "pageserver_secondary_upload_heatmap",
1657 : "Number of heatmaps written to remote storage by attached tenants"
1658 0 : )
1659 0 : .expect("failed to define a metric"),
1660 0 : upload_heatmap_errors: register_int_counter!(
1661 : "pageserver_secondary_upload_heatmap_errors",
1662 : "Failures writing heatmap to remote storage"
1663 0 : )
1664 0 : .expect("failed to define a metric"),
1665 0 : upload_heatmap_duration: register_histogram!(
1666 : "pageserver_secondary_upload_heatmap_duration",
1667 : "Time to build and upload a heatmap, including any waiting inside the S3 client"
1668 0 : )
1669 0 : .expect("failed to define a metric"),
1670 0 : download_heatmap: register_int_counter!(
1671 : "pageserver_secondary_download_heatmap",
1672 : "Number of downloads of heatmaps by secondary mode locations, including when it hasn't changed"
1673 0 : )
1674 0 : .expect("failed to define a metric"),
1675 0 : download_layer: register_int_counter!(
1676 : "pageserver_secondary_download_layer",
1677 : "Number of downloads of layers by secondary mode locations"
1678 0 : )
1679 0 : .expect("failed to define a metric"),
1680 0 : }
1681 0 : });
1682 :
1683 : #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1684 : pub enum RemoteOpKind {
1685 : Upload,
1686 : Download,
1687 : Delete,
1688 : }
1689 : impl RemoteOpKind {
1690 10369 : pub fn as_str(&self) -> &'static str {
1691 10369 : match self {
1692 9875 : Self::Upload => "upload",
1693 52 : Self::Download => "download",
1694 442 : Self::Delete => "delete",
1695 : }
1696 10369 : }
1697 : }
1698 :
1699 : #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
1700 : pub enum RemoteOpFileKind {
1701 : Layer,
1702 : Index,
1703 : }
1704 : impl RemoteOpFileKind {
1705 10369 : pub fn as_str(&self) -> &'static str {
1706 10369 : match self {
1707 6865 : Self::Layer => "layer",
1708 3504 : Self::Index => "index",
1709 : }
1710 10369 : }
1711 : }
1712 :
1713 99 : pub(crate) static REMOTE_OPERATION_TIME: Lazy<HistogramVec> = Lazy::new(|| {
1714 : register_histogram_vec!(
1715 : "pageserver_remote_operation_seconds",
1716 : "Time spent on remote storage operations. \
1717 : Grouped by tenant, timeline, operation_kind and status. \
1718 : Does not account for time spent waiting in remote timeline client's queues.",
1719 : &["file_kind", "op_kind", "status"]
1720 : )
1721 99 : .expect("failed to define a metric")
1722 99 : });
1723 :
1724 0 : pub(crate) static TENANT_TASK_EVENTS: Lazy<IntCounterVec> = Lazy::new(|| {
1725 : register_int_counter_vec!(
1726 : "pageserver_tenant_task_events",
1727 : "Number of task start/stop/fail events.",
1728 : &["event"],
1729 : )
1730 0 : .expect("Failed to register tenant_task_events metric")
1731 0 : });
1732 :
1733 12 : pub(crate) static BACKGROUND_LOOP_SEMAPHORE_WAIT_GAUGE: Lazy<IntCounterPairVec> = Lazy::new(|| {
1734 : register_int_counter_pair_vec!(
1735 : "pageserver_background_loop_semaphore_wait_start_count",
1736 : "Counter for background loop concurrency-limiting semaphore acquire calls started",
1737 : "pageserver_background_loop_semaphore_wait_finish_count",
1738 : "Counter for background loop concurrency-limiting semaphore acquire calls finished",
1739 : &["task"],
1740 : )
1741 12 : .unwrap()
1742 12 : });
1743 :
1744 0 : pub(crate) static BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT: Lazy<IntCounterVec> = Lazy::new(|| {
1745 : register_int_counter_vec!(
1746 : "pageserver_background_loop_period_overrun_count",
1747 : "Incremented whenever warn_when_period_overrun() logs a warning.",
1748 : &["task", "period"],
1749 : )
1750 0 : .expect("failed to define a metric")
1751 0 : });
1752 :
1753 : // walreceiver metrics
1754 :
1755 0 : pub(crate) static WALRECEIVER_STARTED_CONNECTIONS: Lazy<IntCounter> = Lazy::new(|| {
1756 : register_int_counter!(
1757 : "pageserver_walreceiver_started_connections_total",
1758 : "Number of started walreceiver connections"
1759 : )
1760 0 : .expect("failed to define a metric")
1761 0 : });
1762 :
1763 0 : pub(crate) static WALRECEIVER_ACTIVE_MANAGERS: Lazy<IntGauge> = Lazy::new(|| {
1764 : register_int_gauge!(
1765 : "pageserver_walreceiver_active_managers",
1766 : "Number of active walreceiver managers"
1767 : )
1768 0 : .expect("failed to define a metric")
1769 0 : });
1770 :
1771 0 : pub(crate) static WALRECEIVER_SWITCHES: Lazy<IntCounterVec> = Lazy::new(|| {
1772 : register_int_counter_vec!(
1773 : "pageserver_walreceiver_switches_total",
1774 : "Number of walreceiver manager change_connection calls",
1775 : &["reason"]
1776 : )
1777 0 : .expect("failed to define a metric")
1778 0 : });
1779 :
1780 0 : pub(crate) static WALRECEIVER_BROKER_UPDATES: Lazy<IntCounter> = Lazy::new(|| {
1781 : register_int_counter!(
1782 : "pageserver_walreceiver_broker_updates_total",
1783 : "Number of received broker updates in walreceiver"
1784 : )
1785 0 : .expect("failed to define a metric")
1786 0 : });
1787 :
1788 2 : pub(crate) static WALRECEIVER_CANDIDATES_EVENTS: Lazy<IntCounterVec> = Lazy::new(|| {
1789 : register_int_counter_vec!(
1790 : "pageserver_walreceiver_candidates_events_total",
1791 : "Number of walreceiver candidate events",
1792 : &["event"]
1793 : )
1794 2 : .expect("failed to define a metric")
1795 2 : });
1796 :
1797 : pub(crate) static WALRECEIVER_CANDIDATES_ADDED: Lazy<IntCounter> =
1798 0 : Lazy::new(|| WALRECEIVER_CANDIDATES_EVENTS.with_label_values(&["add"]));
1799 :
1800 : pub(crate) static WALRECEIVER_CANDIDATES_REMOVED: Lazy<IntCounter> =
1801 2 : Lazy::new(|| WALRECEIVER_CANDIDATES_EVENTS.with_label_values(&["remove"]));
1802 :
1803 : // Metrics collected on WAL redo operations
1804 : //
1805 : // We collect the time spent in actual WAL redo ('redo'), and time waiting
1806 : // for access to the postgres process ('wait') since there is only one for
1807 : // each tenant.
1808 :
1809 : /// Time buckets are small because we want to be able to measure the
1810 : /// smallest redo processing times. These buckets allow us to measure down
1811 : /// to 5us, which equates to 200'000 pages/sec, which equates to 1.6GB/sec.
1812 : /// This is much better than the previous 5ms aka 200 pages/sec aka 1.6MB/sec.
1813 : ///
1814 : /// Values up to 1s are recorded because metrics show that we have redo
1815 : /// durations and lock times larger than 0.250s.
1816 : macro_rules! redo_histogram_time_buckets {
1817 : () => {
1818 : vec![
1819 : 0.000_005, 0.000_010, 0.000_025, 0.000_050, 0.000_100, 0.000_250, 0.000_500, 0.001_000,
1820 : 0.002_500, 0.005_000, 0.010_000, 0.025_000, 0.050_000, 0.100_000, 0.250_000, 0.500_000,
1821 : 1.000_000,
1822 : ]
1823 : };
1824 : }
1825 :
1826 : /// While we're at it, also measure the amount of records replayed in each
1827 : /// operation. We have a global 'total replayed' counter, but that's not
1828 : /// as useful as 'what is the skew for how many records we replay in one
1829 : /// operation'.
1830 : macro_rules! redo_histogram_count_buckets {
1831 : () => {
1832 : vec![0.0, 1.0, 2.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0]
1833 : };
1834 : }
1835 :
1836 : macro_rules! redo_bytes_histogram_count_buckets {
1837 : () => {
1838 : // powers of (2^.5), from 2^4.5 to 2^15 (22 buckets)
1839 : // rounded up to the next multiple of 8 to capture any MAXALIGNed record of that size, too.
1840 : vec![
1841 : 24.0, 32.0, 48.0, 64.0, 96.0, 128.0, 184.0, 256.0, 368.0, 512.0, 728.0, 1024.0, 1456.0,
1842 : 2048.0, 2904.0, 4096.0, 5800.0, 8192.0, 11592.0, 16384.0, 23176.0, 32768.0,
1843 : ]
1844 : };
1845 : }
1846 :
1847 : pub(crate) struct WalIngestMetrics {
1848 : pub(crate) bytes_received: IntCounter,
1849 : pub(crate) records_received: IntCounter,
1850 : pub(crate) records_committed: IntCounter,
1851 : pub(crate) records_filtered: IntCounter,
1852 : pub(crate) time_spent_on_ingest: Histogram,
1853 : }
1854 :
1855 104 : pub(crate) static WAL_INGEST: Lazy<WalIngestMetrics> = Lazy::new(|| WalIngestMetrics {
1856 : bytes_received: register_int_counter!(
1857 : "pageserver_wal_ingest_bytes_received",
1858 : "Bytes of WAL ingested from safekeepers",
1859 : )
1860 104 : .unwrap(),
1861 : records_received: register_int_counter!(
1862 : "pageserver_wal_ingest_records_received",
1863 : "Number of WAL records received from safekeepers"
1864 : )
1865 104 : .expect("failed to define a metric"),
1866 : records_committed: register_int_counter!(
1867 : "pageserver_wal_ingest_records_committed",
1868 : "Number of WAL records which resulted in writes to pageserver storage"
1869 : )
1870 104 : .expect("failed to define a metric"),
1871 : records_filtered: register_int_counter!(
1872 : "pageserver_wal_ingest_records_filtered",
1873 : "Number of WAL records filtered out due to sharding"
1874 : )
1875 104 : .expect("failed to define a metric"),
1876 : time_spent_on_ingest: register_histogram!(
1877 : "pageserver_wal_ingest_put_value_seconds",
1878 : "Actual time spent on ingesting a record",
1879 : redo_histogram_time_buckets!(),
1880 : )
1881 104 : .expect("failed to define a metric"),
1882 104 : });
1883 :
1884 6 : pub(crate) static WAL_REDO_TIME: Lazy<Histogram> = Lazy::new(|| {
1885 : register_histogram!(
1886 : "pageserver_wal_redo_seconds",
1887 : "Time spent on WAL redo",
1888 : redo_histogram_time_buckets!()
1889 : )
1890 6 : .expect("failed to define a metric")
1891 6 : });
1892 :
1893 6 : pub(crate) static WAL_REDO_RECORDS_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
1894 : register_histogram!(
1895 : "pageserver_wal_redo_records_histogram",
1896 : "Histogram of number of records replayed per redo in the Postgres WAL redo process",
1897 : redo_histogram_count_buckets!(),
1898 : )
1899 6 : .expect("failed to define a metric")
1900 6 : });
1901 :
1902 6 : pub(crate) static WAL_REDO_BYTES_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
1903 : register_histogram!(
1904 : "pageserver_wal_redo_bytes_histogram",
1905 : "Histogram of number of records replayed per redo sent to Postgres",
1906 : redo_bytes_histogram_count_buckets!(),
1907 : )
1908 6 : .expect("failed to define a metric")
1909 6 : });
1910 :
1911 : // FIXME: isn't this already included by WAL_REDO_RECORDS_HISTOGRAM which has _count?
1912 6 : pub(crate) static WAL_REDO_RECORD_COUNTER: Lazy<IntCounter> = Lazy::new(|| {
1913 : register_int_counter!(
1914 : "pageserver_replayed_wal_records_total",
1915 : "Number of WAL records replayed in WAL redo process"
1916 : )
1917 6 : .unwrap()
1918 6 : });
1919 :
1920 : #[rustfmt::skip]
1921 6 : pub(crate) static WAL_REDO_PROCESS_LAUNCH_DURATION_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
1922 : register_histogram!(
1923 : "pageserver_wal_redo_process_launch_duration",
1924 : "Histogram of the duration of successful WalRedoProcess::launch calls",
1925 : vec![
1926 : 0.0002, 0.0004, 0.0006, 0.0008, 0.0010,
1927 : 0.0020, 0.0040, 0.0060, 0.0080, 0.0100,
1928 : 0.0200, 0.0400, 0.0600, 0.0800, 0.1000,
1929 : 0.2000, 0.4000, 0.6000, 0.8000, 1.0000,
1930 : 1.5000, 2.0000, 2.5000, 3.0000, 4.0000, 10.0000
1931 : ],
1932 : )
1933 6 : .expect("failed to define a metric")
1934 6 : });
1935 :
1936 : pub(crate) struct WalRedoProcessCounters {
1937 : pub(crate) started: IntCounter,
1938 : pub(crate) killed_by_cause: enum_map::EnumMap<WalRedoKillCause, IntCounter>,
1939 : pub(crate) active_stderr_logger_tasks_started: IntCounter,
1940 : pub(crate) active_stderr_logger_tasks_finished: IntCounter,
1941 : }
1942 :
1943 18 : #[derive(Debug, enum_map::Enum, strum_macros::IntoStaticStr)]
1944 : pub(crate) enum WalRedoKillCause {
1945 : WalRedoProcessDrop,
1946 : NoLeakChildDrop,
1947 : Startup,
1948 : }
1949 :
1950 : impl Default for WalRedoProcessCounters {
1951 6 : fn default() -> Self {
1952 6 : let started = register_int_counter!(
1953 : "pageserver_wal_redo_process_started_total",
1954 : "Number of WAL redo processes started",
1955 : )
1956 6 : .unwrap();
1957 6 :
1958 6 : let killed = register_int_counter_vec!(
1959 : "pageserver_wal_redo_process_stopped_total",
1960 : "Number of WAL redo processes stopped",
1961 : &["cause"],
1962 : )
1963 6 : .unwrap();
1964 6 :
1965 6 : let active_stderr_logger_tasks_started = register_int_counter!(
1966 : "pageserver_walredo_stderr_logger_tasks_started_total",
1967 : "Number of active walredo stderr logger tasks that have started",
1968 : )
1969 6 : .unwrap();
1970 6 :
1971 6 : let active_stderr_logger_tasks_finished = register_int_counter!(
1972 : "pageserver_walredo_stderr_logger_tasks_finished_total",
1973 : "Number of active walredo stderr logger tasks that have finished",
1974 : )
1975 6 : .unwrap();
1976 6 :
1977 6 : Self {
1978 6 : started,
1979 18 : killed_by_cause: EnumMap::from_array(std::array::from_fn(|i| {
1980 18 : let cause = <WalRedoKillCause as enum_map::Enum>::from_usize(i);
1981 18 : let cause_str: &'static str = cause.into();
1982 18 : killed.with_label_values(&[cause_str])
1983 18 : })),
1984 6 : active_stderr_logger_tasks_started,
1985 6 : active_stderr_logger_tasks_finished,
1986 6 : }
1987 6 : }
1988 : }
1989 :
1990 : pub(crate) static WAL_REDO_PROCESS_COUNTERS: Lazy<WalRedoProcessCounters> =
1991 : Lazy::new(WalRedoProcessCounters::default);
1992 :
1993 : #[cfg(not(test))]
1994 : pub mod wal_redo {
1995 : use super::*;
1996 :
1997 0 : static PROCESS_KIND: Lazy<std::sync::Mutex<UIntGaugeVec>> = Lazy::new(|| {
1998 0 : std::sync::Mutex::new(
1999 0 : register_uint_gauge_vec!(
2000 : "pageserver_wal_redo_process_kind",
2001 : "The configured process kind for walredo",
2002 : &["kind"],
2003 0 : )
2004 0 : .unwrap(),
2005 0 : )
2006 0 : });
2007 :
2008 0 : pub fn set_process_kind_metric(kind: crate::walredo::ProcessKind) {
2009 0 : // use guard to avoid races around the next two steps
2010 0 : let guard = PROCESS_KIND.lock().unwrap();
2011 0 : guard.reset();
2012 0 : guard.with_label_values(&[&format!("{kind}")]).set(1);
2013 0 : }
2014 : }
2015 :
2016 : /// Similar to `prometheus::HistogramTimer` but does not record on drop.
2017 : pub(crate) struct StorageTimeMetricsTimer {
2018 : metrics: StorageTimeMetrics,
2019 : start: Instant,
2020 : }
2021 :
2022 : impl StorageTimeMetricsTimer {
2023 4416 : fn new(metrics: StorageTimeMetrics) -> Self {
2024 4416 : Self {
2025 4416 : metrics,
2026 4416 : start: Instant::now(),
2027 4416 : }
2028 4416 : }
2029 :
2030 : /// Record the time from creation to now.
2031 3482 : pub fn stop_and_record(self) {
2032 3482 : let duration = self.start.elapsed().as_secs_f64();
2033 3482 : self.metrics.timeline_sum.inc_by(duration);
2034 3482 : self.metrics.timeline_count.inc();
2035 3482 : self.metrics.global_histogram.observe(duration);
2036 3482 : }
2037 :
2038 : /// Turns this timer into a timer, which will always record -- usually this means recording
2039 : /// regardless an early `?` path was taken in a function.
2040 728 : pub(crate) fn record_on_drop(self) -> AlwaysRecordingStorageTimeMetricsTimer {
2041 728 : AlwaysRecordingStorageTimeMetricsTimer(Some(self))
2042 728 : }
2043 : }
2044 :
2045 : pub(crate) struct AlwaysRecordingStorageTimeMetricsTimer(Option<StorageTimeMetricsTimer>);
2046 :
2047 : impl Drop for AlwaysRecordingStorageTimeMetricsTimer {
2048 728 : fn drop(&mut self) {
2049 728 : if let Some(inner) = self.0.take() {
2050 728 : inner.stop_and_record();
2051 728 : }
2052 728 : }
2053 : }
2054 :
2055 : /// Timing facilities for an globally histogrammed metric, which is supported by per tenant and
2056 : /// timeline total sum and count.
2057 : #[derive(Clone, Debug)]
2058 : pub(crate) struct StorageTimeMetrics {
2059 : /// Sum of f64 seconds, per operation, tenant_id and timeline_id
2060 : timeline_sum: Counter,
2061 : /// Number of oeprations, per operation, tenant_id and timeline_id
2062 : timeline_count: IntCounter,
2063 : /// Global histogram having only the "operation" label.
2064 : global_histogram: Histogram,
2065 : }
2066 :
2067 : impl StorageTimeMetrics {
2068 2672 : pub fn new(
2069 2672 : operation: StorageTimeOperation,
2070 2672 : tenant_id: &str,
2071 2672 : shard_id: &str,
2072 2672 : timeline_id: &str,
2073 2672 : ) -> Self {
2074 2672 : let operation: &'static str = operation.into();
2075 2672 :
2076 2672 : let timeline_sum = STORAGE_TIME_SUM_PER_TIMELINE
2077 2672 : .get_metric_with_label_values(&[operation, tenant_id, shard_id, timeline_id])
2078 2672 : .unwrap();
2079 2672 : let timeline_count = STORAGE_TIME_COUNT_PER_TIMELINE
2080 2672 : .get_metric_with_label_values(&[operation, tenant_id, shard_id, timeline_id])
2081 2672 : .unwrap();
2082 2672 : let global_histogram = STORAGE_TIME_GLOBAL
2083 2672 : .get_metric_with_label_values(&[operation])
2084 2672 : .unwrap();
2085 2672 :
2086 2672 : StorageTimeMetrics {
2087 2672 : timeline_sum,
2088 2672 : timeline_count,
2089 2672 : global_histogram,
2090 2672 : }
2091 2672 : }
2092 :
2093 : /// Starts timing a new operation.
2094 : ///
2095 : /// Note: unlike `prometheus::HistogramTimer` the returned timer does not record on drop.
2096 4416 : pub fn start_timer(&self) -> StorageTimeMetricsTimer {
2097 4416 : StorageTimeMetricsTimer::new(self.clone())
2098 4416 : }
2099 : }
2100 :
2101 : #[derive(Debug)]
2102 : pub(crate) struct TimelineMetrics {
2103 : tenant_id: String,
2104 : shard_id: String,
2105 : timeline_id: String,
2106 : pub flush_time_histo: StorageTimeMetrics,
2107 : pub compact_time_histo: StorageTimeMetrics,
2108 : pub create_images_time_histo: StorageTimeMetrics,
2109 : pub logical_size_histo: StorageTimeMetrics,
2110 : pub imitate_logical_size_histo: StorageTimeMetrics,
2111 : pub load_layer_map_histo: StorageTimeMetrics,
2112 : pub garbage_collect_histo: StorageTimeMetrics,
2113 : pub find_gc_cutoffs_histo: StorageTimeMetrics,
2114 : pub last_record_gauge: IntGauge,
2115 : resident_physical_size_gauge: UIntGauge,
2116 : /// copy of LayeredTimeline.current_logical_size
2117 : pub current_logical_size_gauge: UIntGauge,
2118 : pub directory_entries_count_gauge: Lazy<UIntGauge, Box<dyn Send + Fn() -> UIntGauge>>,
2119 : pub evictions: IntCounter,
2120 : pub evictions_with_low_residence_duration: std::sync::RwLock<EvictionsWithLowResidenceDuration>,
2121 : }
2122 :
2123 : impl TimelineMetrics {
2124 334 : pub fn new(
2125 334 : tenant_shard_id: &TenantShardId,
2126 334 : timeline_id_raw: &TimelineId,
2127 334 : evictions_with_low_residence_duration_builder: EvictionsWithLowResidenceDurationBuilder,
2128 334 : ) -> Self {
2129 334 : let tenant_id = tenant_shard_id.tenant_id.to_string();
2130 334 : let shard_id = format!("{}", tenant_shard_id.shard_slug());
2131 334 : let timeline_id = timeline_id_raw.to_string();
2132 334 : let flush_time_histo = StorageTimeMetrics::new(
2133 334 : StorageTimeOperation::LayerFlush,
2134 334 : &tenant_id,
2135 334 : &shard_id,
2136 334 : &timeline_id,
2137 334 : );
2138 334 : let compact_time_histo = StorageTimeMetrics::new(
2139 334 : StorageTimeOperation::Compact,
2140 334 : &tenant_id,
2141 334 : &shard_id,
2142 334 : &timeline_id,
2143 334 : );
2144 334 : let create_images_time_histo = StorageTimeMetrics::new(
2145 334 : StorageTimeOperation::CreateImages,
2146 334 : &tenant_id,
2147 334 : &shard_id,
2148 334 : &timeline_id,
2149 334 : );
2150 334 : let logical_size_histo = StorageTimeMetrics::new(
2151 334 : StorageTimeOperation::LogicalSize,
2152 334 : &tenant_id,
2153 334 : &shard_id,
2154 334 : &timeline_id,
2155 334 : );
2156 334 : let imitate_logical_size_histo = StorageTimeMetrics::new(
2157 334 : StorageTimeOperation::ImitateLogicalSize,
2158 334 : &tenant_id,
2159 334 : &shard_id,
2160 334 : &timeline_id,
2161 334 : );
2162 334 : let load_layer_map_histo = StorageTimeMetrics::new(
2163 334 : StorageTimeOperation::LoadLayerMap,
2164 334 : &tenant_id,
2165 334 : &shard_id,
2166 334 : &timeline_id,
2167 334 : );
2168 334 : let garbage_collect_histo = StorageTimeMetrics::new(
2169 334 : StorageTimeOperation::Gc,
2170 334 : &tenant_id,
2171 334 : &shard_id,
2172 334 : &timeline_id,
2173 334 : );
2174 334 : let find_gc_cutoffs_histo = StorageTimeMetrics::new(
2175 334 : StorageTimeOperation::FindGcCutoffs,
2176 334 : &tenant_id,
2177 334 : &shard_id,
2178 334 : &timeline_id,
2179 334 : );
2180 334 : let last_record_gauge = LAST_RECORD_LSN
2181 334 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2182 334 : .unwrap();
2183 334 : let resident_physical_size_gauge = RESIDENT_PHYSICAL_SIZE
2184 334 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2185 334 : .unwrap();
2186 334 : // TODO: we shouldn't expose this metric
2187 334 : let current_logical_size_gauge = CURRENT_LOGICAL_SIZE
2188 334 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2189 334 : .unwrap();
2190 334 : // TODO use impl Trait syntax here once we have ability to use it: https://github.com/rust-lang/rust/issues/63065
2191 334 : let directory_entries_count_gauge_closure = {
2192 334 : let tenant_shard_id = *tenant_shard_id;
2193 334 : let timeline_id_raw = *timeline_id_raw;
2194 0 : move || {
2195 0 : let tenant_id = tenant_shard_id.tenant_id.to_string();
2196 0 : let shard_id = format!("{}", tenant_shard_id.shard_slug());
2197 0 : let timeline_id = timeline_id_raw.to_string();
2198 0 : let gauge: UIntGauge = DIRECTORY_ENTRIES_COUNT
2199 0 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2200 0 : .unwrap();
2201 0 : gauge
2202 0 : }
2203 : };
2204 334 : let directory_entries_count_gauge: Lazy<UIntGauge, Box<dyn Send + Fn() -> UIntGauge>> =
2205 334 : Lazy::new(Box::new(directory_entries_count_gauge_closure));
2206 334 : let evictions = EVICTIONS
2207 334 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2208 334 : .unwrap();
2209 334 : let evictions_with_low_residence_duration = evictions_with_low_residence_duration_builder
2210 334 : .build(&tenant_id, &shard_id, &timeline_id);
2211 334 :
2212 334 : TimelineMetrics {
2213 334 : tenant_id,
2214 334 : shard_id,
2215 334 : timeline_id,
2216 334 : flush_time_histo,
2217 334 : compact_time_histo,
2218 334 : create_images_time_histo,
2219 334 : logical_size_histo,
2220 334 : imitate_logical_size_histo,
2221 334 : garbage_collect_histo,
2222 334 : find_gc_cutoffs_histo,
2223 334 : load_layer_map_histo,
2224 334 : last_record_gauge,
2225 334 : resident_physical_size_gauge,
2226 334 : current_logical_size_gauge,
2227 334 : directory_entries_count_gauge,
2228 334 : evictions,
2229 334 : evictions_with_low_residence_duration: std::sync::RwLock::new(
2230 334 : evictions_with_low_residence_duration,
2231 334 : ),
2232 334 : }
2233 334 : }
2234 :
2235 1166 : pub(crate) fn record_new_file_metrics(&self, sz: u64) {
2236 1166 : self.resident_physical_size_add(sz);
2237 1166 : }
2238 :
2239 286 : pub(crate) fn resident_physical_size_sub(&self, sz: u64) {
2240 286 : self.resident_physical_size_gauge.sub(sz);
2241 286 : crate::metrics::RESIDENT_PHYSICAL_SIZE_GLOBAL.sub(sz);
2242 286 : }
2243 :
2244 1196 : pub(crate) fn resident_physical_size_add(&self, sz: u64) {
2245 1196 : self.resident_physical_size_gauge.add(sz);
2246 1196 : crate::metrics::RESIDENT_PHYSICAL_SIZE_GLOBAL.add(sz);
2247 1196 : }
2248 :
2249 8 : pub(crate) fn resident_physical_size_get(&self) -> u64 {
2250 8 : self.resident_physical_size_gauge.get()
2251 8 : }
2252 :
2253 8 : pub(crate) fn shutdown(&self) {
2254 8 : let tenant_id = &self.tenant_id;
2255 8 : let timeline_id = &self.timeline_id;
2256 8 : let shard_id = &self.shard_id;
2257 8 : let _ = LAST_RECORD_LSN.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2258 8 : {
2259 8 : RESIDENT_PHYSICAL_SIZE_GLOBAL.sub(self.resident_physical_size_get());
2260 8 : let _ = RESIDENT_PHYSICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2261 8 : }
2262 8 : let _ = CURRENT_LOGICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2263 8 : if let Some(metric) = Lazy::get(&DIRECTORY_ENTRIES_COUNT) {
2264 0 : let _ = metric.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2265 8 : }
2266 8 : let _ = EVICTIONS.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2267 8 :
2268 8 : self.evictions_with_low_residence_duration
2269 8 : .write()
2270 8 : .unwrap()
2271 8 : .remove(tenant_id, shard_id, timeline_id);
2272 :
2273 : // The following metrics are born outside of the TimelineMetrics lifecycle but still
2274 : // removed at the end of it. The idea is to have the metrics outlive the
2275 : // entity during which they're observed, e.g., the smgr metrics shall
2276 : // outlive an individual smgr connection, but not the timeline.
2277 :
2278 80 : for op in StorageTimeOperation::VARIANTS {
2279 72 : let _ = STORAGE_TIME_SUM_PER_TIMELINE.remove_label_values(&[
2280 72 : op,
2281 72 : tenant_id,
2282 72 : shard_id,
2283 72 : timeline_id,
2284 72 : ]);
2285 72 : let _ = STORAGE_TIME_COUNT_PER_TIMELINE.remove_label_values(&[
2286 72 : op,
2287 72 : tenant_id,
2288 72 : shard_id,
2289 72 : timeline_id,
2290 72 : ]);
2291 72 : }
2292 :
2293 24 : for op in STORAGE_IO_SIZE_OPERATIONS {
2294 16 : let _ = STORAGE_IO_SIZE.remove_label_values(&[op, tenant_id, shard_id, timeline_id]);
2295 16 : }
2296 :
2297 48 : for op in SmgrQueryType::iter() {
2298 40 : let _ = SMGR_QUERY_TIME_PER_TENANT_TIMELINE.remove_label_values(&[
2299 40 : op.into(),
2300 40 : tenant_id,
2301 40 : shard_id,
2302 40 : timeline_id,
2303 40 : ]);
2304 40 : }
2305 8 : }
2306 : }
2307 :
2308 6 : pub(crate) fn remove_tenant_metrics(tenant_shard_id: &TenantShardId) {
2309 6 : // Only shard zero deals in synthetic sizes
2310 6 : if tenant_shard_id.is_shard_zero() {
2311 6 : let tid = tenant_shard_id.tenant_id.to_string();
2312 6 : let _ = TENANT_SYNTHETIC_SIZE_METRIC.remove_label_values(&[&tid]);
2313 6 : }
2314 :
2315 : // we leave the BROKEN_TENANTS_SET entry if any
2316 6 : }
2317 :
2318 : use futures::Future;
2319 : use pin_project_lite::pin_project;
2320 : use std::collections::HashMap;
2321 : use std::num::NonZeroUsize;
2322 : use std::pin::Pin;
2323 : use std::sync::{Arc, Mutex};
2324 : use std::task::{Context, Poll};
2325 : use std::time::{Duration, Instant};
2326 :
2327 : use crate::context::{PageContentKind, RequestContext};
2328 : use crate::task_mgr::TaskKind;
2329 : use crate::tenant::mgr::TenantSlot;
2330 :
2331 : /// Maintain a per timeline gauge in addition to the global gauge.
2332 : struct PerTimelineRemotePhysicalSizeGauge {
2333 : last_set: u64,
2334 : gauge: UIntGauge,
2335 : }
2336 :
2337 : impl PerTimelineRemotePhysicalSizeGauge {
2338 334 : fn new(per_timeline_gauge: UIntGauge) -> Self {
2339 334 : Self {
2340 334 : last_set: per_timeline_gauge.get(),
2341 334 : gauge: per_timeline_gauge,
2342 334 : }
2343 334 : }
2344 1478 : fn set(&mut self, sz: u64) {
2345 1478 : self.gauge.set(sz);
2346 1478 : if sz < self.last_set {
2347 18 : REMOTE_PHYSICAL_SIZE_GLOBAL.sub(self.last_set - sz);
2348 1460 : } else {
2349 1460 : REMOTE_PHYSICAL_SIZE_GLOBAL.add(sz - self.last_set);
2350 1460 : };
2351 1478 : self.last_set = sz;
2352 1478 : }
2353 2 : fn get(&self) -> u64 {
2354 2 : self.gauge.get()
2355 2 : }
2356 : }
2357 :
2358 : impl Drop for PerTimelineRemotePhysicalSizeGauge {
2359 8 : fn drop(&mut self) {
2360 8 : REMOTE_PHYSICAL_SIZE_GLOBAL.sub(self.last_set);
2361 8 : }
2362 : }
2363 :
2364 : pub(crate) struct RemoteTimelineClientMetrics {
2365 : tenant_id: String,
2366 : shard_id: String,
2367 : timeline_id: String,
2368 : remote_physical_size_gauge: Mutex<Option<PerTimelineRemotePhysicalSizeGauge>>,
2369 : calls: Mutex<HashMap<(&'static str, &'static str), IntCounterPair>>,
2370 : bytes_started_counter: Mutex<HashMap<(&'static str, &'static str), IntCounter>>,
2371 : bytes_finished_counter: Mutex<HashMap<(&'static str, &'static str), IntCounter>>,
2372 : }
2373 :
2374 : impl RemoteTimelineClientMetrics {
2375 344 : pub fn new(tenant_shard_id: &TenantShardId, timeline_id: &TimelineId) -> Self {
2376 344 : RemoteTimelineClientMetrics {
2377 344 : tenant_id: tenant_shard_id.tenant_id.to_string(),
2378 344 : shard_id: format!("{}", tenant_shard_id.shard_slug()),
2379 344 : timeline_id: timeline_id.to_string(),
2380 344 : calls: Mutex::new(HashMap::default()),
2381 344 : bytes_started_counter: Mutex::new(HashMap::default()),
2382 344 : bytes_finished_counter: Mutex::new(HashMap::default()),
2383 344 : remote_physical_size_gauge: Mutex::new(None),
2384 344 : }
2385 344 : }
2386 :
2387 1478 : pub(crate) fn remote_physical_size_set(&self, sz: u64) {
2388 1478 : let mut guard = self.remote_physical_size_gauge.lock().unwrap();
2389 1478 : let gauge = guard.get_or_insert_with(|| {
2390 334 : PerTimelineRemotePhysicalSizeGauge::new(
2391 334 : REMOTE_PHYSICAL_SIZE
2392 334 : .get_metric_with_label_values(&[
2393 334 : &self.tenant_id,
2394 334 : &self.shard_id,
2395 334 : &self.timeline_id,
2396 334 : ])
2397 334 : .unwrap(),
2398 334 : )
2399 1478 : });
2400 1478 : gauge.set(sz);
2401 1478 : }
2402 :
2403 2 : pub(crate) fn remote_physical_size_get(&self) -> u64 {
2404 2 : let guard = self.remote_physical_size_gauge.lock().unwrap();
2405 2 : guard.as_ref().map(|gauge| gauge.get()).unwrap_or(0)
2406 2 : }
2407 :
2408 2129 : pub fn remote_operation_time(
2409 2129 : &self,
2410 2129 : file_kind: &RemoteOpFileKind,
2411 2129 : op_kind: &RemoteOpKind,
2412 2129 : status: &'static str,
2413 2129 : ) -> Histogram {
2414 2129 : let key = (file_kind.as_str(), op_kind.as_str(), status);
2415 2129 : REMOTE_OPERATION_TIME
2416 2129 : .get_metric_with_label_values(&[key.0, key.1, key.2])
2417 2129 : .unwrap()
2418 2129 : }
2419 :
2420 4921 : fn calls_counter_pair(
2421 4921 : &self,
2422 4921 : file_kind: &RemoteOpFileKind,
2423 4921 : op_kind: &RemoteOpKind,
2424 4921 : ) -> IntCounterPair {
2425 4921 : let mut guard = self.calls.lock().unwrap();
2426 4921 : let key = (file_kind.as_str(), op_kind.as_str());
2427 4921 : let metric = guard.entry(key).or_insert_with(move || {
2428 574 : REMOTE_TIMELINE_CLIENT_CALLS
2429 574 : .get_metric_with_label_values(&[
2430 574 : &self.tenant_id,
2431 574 : &self.shard_id,
2432 574 : &self.timeline_id,
2433 574 : key.0,
2434 574 : key.1,
2435 574 : ])
2436 574 : .unwrap()
2437 4921 : });
2438 4921 : metric.clone()
2439 4921 : }
2440 :
2441 1174 : fn bytes_started_counter(
2442 1174 : &self,
2443 1174 : file_kind: &RemoteOpFileKind,
2444 1174 : op_kind: &RemoteOpKind,
2445 1174 : ) -> IntCounter {
2446 1174 : let mut guard = self.bytes_started_counter.lock().unwrap();
2447 1174 : let key = (file_kind.as_str(), op_kind.as_str());
2448 1174 : let metric = guard.entry(key).or_insert_with(move || {
2449 214 : REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER
2450 214 : .get_metric_with_label_values(&[
2451 214 : &self.tenant_id,
2452 214 : &self.shard_id,
2453 214 : &self.timeline_id,
2454 214 : key.0,
2455 214 : key.1,
2456 214 : ])
2457 214 : .unwrap()
2458 1174 : });
2459 1174 : metric.clone()
2460 1174 : }
2461 :
2462 2133 : fn bytes_finished_counter(
2463 2133 : &self,
2464 2133 : file_kind: &RemoteOpFileKind,
2465 2133 : op_kind: &RemoteOpKind,
2466 2133 : ) -> IntCounter {
2467 2133 : let mut guard = self.bytes_finished_counter.lock().unwrap();
2468 2133 : let key = (file_kind.as_str(), op_kind.as_str());
2469 2133 : let metric = guard.entry(key).or_insert_with(move || {
2470 214 : REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER
2471 214 : .get_metric_with_label_values(&[
2472 214 : &self.tenant_id,
2473 214 : &self.shard_id,
2474 214 : &self.timeline_id,
2475 214 : key.0,
2476 214 : key.1,
2477 214 : ])
2478 214 : .unwrap()
2479 2133 : });
2480 2133 : metric.clone()
2481 2133 : }
2482 : }
2483 :
2484 : #[cfg(test)]
2485 : impl RemoteTimelineClientMetrics {
2486 6 : pub fn get_bytes_started_counter_value(
2487 6 : &self,
2488 6 : file_kind: &RemoteOpFileKind,
2489 6 : op_kind: &RemoteOpKind,
2490 6 : ) -> Option<u64> {
2491 6 : let guard = self.bytes_started_counter.lock().unwrap();
2492 6 : let key = (file_kind.as_str(), op_kind.as_str());
2493 6 : guard.get(&key).map(|counter| counter.get())
2494 6 : }
2495 :
2496 6 : pub fn get_bytes_finished_counter_value(
2497 6 : &self,
2498 6 : file_kind: &RemoteOpFileKind,
2499 6 : op_kind: &RemoteOpKind,
2500 6 : ) -> Option<u64> {
2501 6 : let guard = self.bytes_finished_counter.lock().unwrap();
2502 6 : let key = (file_kind.as_str(), op_kind.as_str());
2503 6 : guard.get(&key).map(|counter| counter.get())
2504 6 : }
2505 : }
2506 :
2507 : /// See [`RemoteTimelineClientMetrics::call_begin`].
2508 : #[must_use]
2509 : pub(crate) struct RemoteTimelineClientCallMetricGuard {
2510 : /// Decremented on drop.
2511 : calls_counter_pair: Option<IntCounterPair>,
2512 : /// If Some(), this references the bytes_finished metric, and we increment it by the given `u64` on drop.
2513 : bytes_finished: Option<(IntCounter, u64)>,
2514 : }
2515 :
2516 : impl RemoteTimelineClientCallMetricGuard {
2517 : /// Consume this guard object without performing the metric updates it would do on `drop()`.
2518 : /// The caller vouches to do the metric updates manually.
2519 2622 : pub fn will_decrement_manually(mut self) {
2520 2622 : let RemoteTimelineClientCallMetricGuard {
2521 2622 : calls_counter_pair,
2522 2622 : bytes_finished,
2523 2622 : } = &mut self;
2524 2622 : calls_counter_pair.take();
2525 2622 : bytes_finished.take();
2526 2622 : }
2527 : }
2528 :
2529 : impl Drop for RemoteTimelineClientCallMetricGuard {
2530 2648 : fn drop(&mut self) {
2531 2648 : let RemoteTimelineClientCallMetricGuard {
2532 2648 : calls_counter_pair,
2533 2648 : bytes_finished,
2534 2648 : } = self;
2535 2648 : if let Some(guard) = calls_counter_pair.take() {
2536 26 : guard.dec();
2537 2622 : }
2538 2648 : if let Some((bytes_finished_metric, value)) = bytes_finished {
2539 0 : bytes_finished_metric.inc_by(*value);
2540 2648 : }
2541 2648 : }
2542 : }
2543 :
2544 : /// The enum variants communicate to the [`RemoteTimelineClientMetrics`] whether to
2545 : /// track the byte size of this call in applicable metric(s).
2546 : pub(crate) enum RemoteTimelineClientMetricsCallTrackSize {
2547 : /// Do not account for this call's byte size in any metrics.
2548 : /// The `reason` field is there to make the call sites self-documenting
2549 : /// about why they don't need the metric.
2550 : DontTrackSize { reason: &'static str },
2551 : /// Track the byte size of the call in applicable metric(s).
2552 : Bytes(u64),
2553 : }
2554 :
2555 : impl RemoteTimelineClientMetrics {
2556 : /// Update the metrics that change when a call to the remote timeline client instance starts.
2557 : ///
2558 : /// Drop the returned guard object once the operation is finished to updates corresponding metrics that track completions.
2559 : /// Or, use [`RemoteTimelineClientCallMetricGuard::will_decrement_manually`] and [`call_end`](Self::call_end) if that
2560 : /// is more suitable.
2561 : /// Never do both.
2562 2648 : pub(crate) fn call_begin(
2563 2648 : &self,
2564 2648 : file_kind: &RemoteOpFileKind,
2565 2648 : op_kind: &RemoteOpKind,
2566 2648 : size: RemoteTimelineClientMetricsCallTrackSize,
2567 2648 : ) -> RemoteTimelineClientCallMetricGuard {
2568 2648 : let calls_counter_pair = self.calls_counter_pair(file_kind, op_kind);
2569 2648 : calls_counter_pair.inc();
2570 :
2571 2648 : let bytes_finished = match size {
2572 1474 : RemoteTimelineClientMetricsCallTrackSize::DontTrackSize { reason: _reason } => {
2573 1474 : // nothing to do
2574 1474 : None
2575 : }
2576 1174 : RemoteTimelineClientMetricsCallTrackSize::Bytes(size) => {
2577 1174 : self.bytes_started_counter(file_kind, op_kind).inc_by(size);
2578 1174 : let finished_counter = self.bytes_finished_counter(file_kind, op_kind);
2579 1174 : Some((finished_counter, size))
2580 : }
2581 : };
2582 2648 : RemoteTimelineClientCallMetricGuard {
2583 2648 : calls_counter_pair: Some(calls_counter_pair),
2584 2648 : bytes_finished,
2585 2648 : }
2586 2648 : }
2587 :
2588 : /// Manually udpate the metrics that track completions, instead of using the guard object.
2589 : /// Using the guard object is generally preferable.
2590 : /// See [`call_begin`](Self::call_begin) for more context.
2591 2273 : pub(crate) fn call_end(
2592 2273 : &self,
2593 2273 : file_kind: &RemoteOpFileKind,
2594 2273 : op_kind: &RemoteOpKind,
2595 2273 : size: RemoteTimelineClientMetricsCallTrackSize,
2596 2273 : ) {
2597 2273 : let calls_counter_pair = self.calls_counter_pair(file_kind, op_kind);
2598 2273 : calls_counter_pair.dec();
2599 2273 : match size {
2600 1314 : RemoteTimelineClientMetricsCallTrackSize::DontTrackSize { reason: _reason } => {}
2601 959 : RemoteTimelineClientMetricsCallTrackSize::Bytes(size) => {
2602 959 : self.bytes_finished_counter(file_kind, op_kind).inc_by(size);
2603 959 : }
2604 : }
2605 2273 : }
2606 : }
2607 :
2608 : impl Drop for RemoteTimelineClientMetrics {
2609 18 : fn drop(&mut self) {
2610 18 : let RemoteTimelineClientMetrics {
2611 18 : tenant_id,
2612 18 : shard_id,
2613 18 : timeline_id,
2614 18 : remote_physical_size_gauge,
2615 18 : calls,
2616 18 : bytes_started_counter,
2617 18 : bytes_finished_counter,
2618 18 : } = self;
2619 22 : for ((a, b), _) in calls.get_mut().unwrap().drain() {
2620 22 : let mut res = [Ok(()), Ok(())];
2621 22 : REMOTE_TIMELINE_CLIENT_CALLS
2622 22 : .remove_label_values(&mut res, &[tenant_id, shard_id, timeline_id, a, b]);
2623 22 : // don't care about results
2624 22 : }
2625 18 : for ((a, b), _) in bytes_started_counter.get_mut().unwrap().drain() {
2626 6 : let _ = REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER.remove_label_values(&[
2627 6 : tenant_id,
2628 6 : shard_id,
2629 6 : timeline_id,
2630 6 : a,
2631 6 : b,
2632 6 : ]);
2633 6 : }
2634 18 : for ((a, b), _) in bytes_finished_counter.get_mut().unwrap().drain() {
2635 6 : let _ = REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER.remove_label_values(&[
2636 6 : tenant_id,
2637 6 : shard_id,
2638 6 : timeline_id,
2639 6 : a,
2640 6 : b,
2641 6 : ]);
2642 6 : }
2643 18 : {
2644 18 : let _ = remote_physical_size_gauge; // use to avoid 'unused' warning in desctructuring above
2645 18 : let _ = REMOTE_PHYSICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2646 18 : }
2647 18 : }
2648 : }
2649 :
2650 : /// Wrapper future that measures the time spent by a remote storage operation,
2651 : /// and records the time and success/failure as a prometheus metric.
2652 : pub(crate) trait MeasureRemoteOp: Sized {
2653 2249 : fn measure_remote_op(
2654 2249 : self,
2655 2249 : file_kind: RemoteOpFileKind,
2656 2249 : op: RemoteOpKind,
2657 2249 : metrics: Arc<RemoteTimelineClientMetrics>,
2658 2249 : ) -> MeasuredRemoteOp<Self> {
2659 2249 : let start = Instant::now();
2660 2249 : MeasuredRemoteOp {
2661 2249 : inner: self,
2662 2249 : file_kind,
2663 2249 : op,
2664 2249 : start,
2665 2249 : metrics,
2666 2249 : }
2667 2249 : }
2668 : }
2669 :
2670 : impl<T: Sized> MeasureRemoteOp for T {}
2671 :
2672 : pin_project! {
2673 : pub(crate) struct MeasuredRemoteOp<F>
2674 : {
2675 : #[pin]
2676 : inner: F,
2677 : file_kind: RemoteOpFileKind,
2678 : op: RemoteOpKind,
2679 : start: Instant,
2680 : metrics: Arc<RemoteTimelineClientMetrics>,
2681 : }
2682 : }
2683 :
2684 : impl<F: Future<Output = Result<O, E>>, O, E> Future for MeasuredRemoteOp<F> {
2685 : type Output = Result<O, E>;
2686 :
2687 37749 : fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
2688 37749 : let this = self.project();
2689 37749 : let poll_result = this.inner.poll(cx);
2690 37749 : if let Poll::Ready(ref res) = poll_result {
2691 2129 : let duration = this.start.elapsed();
2692 2129 : let status = if res.is_ok() { &"success" } else { &"failure" };
2693 2129 : this.metrics
2694 2129 : .remote_operation_time(this.file_kind, this.op, status)
2695 2129 : .observe(duration.as_secs_f64());
2696 35620 : }
2697 37749 : poll_result
2698 37749 : }
2699 : }
2700 :
2701 : pub mod tokio_epoll_uring {
2702 : use metrics::{register_int_counter, UIntGauge};
2703 : use once_cell::sync::Lazy;
2704 :
2705 : pub struct Collector {
2706 : descs: Vec<metrics::core::Desc>,
2707 : systems_created: UIntGauge,
2708 : systems_destroyed: UIntGauge,
2709 : }
2710 :
2711 : impl metrics::core::Collector for Collector {
2712 0 : fn desc(&self) -> Vec<&metrics::core::Desc> {
2713 0 : self.descs.iter().collect()
2714 0 : }
2715 :
2716 0 : fn collect(&self) -> Vec<metrics::proto::MetricFamily> {
2717 0 : let mut mfs = Vec::with_capacity(Self::NMETRICS);
2718 0 : let tokio_epoll_uring::metrics::Metrics {
2719 0 : systems_created,
2720 0 : systems_destroyed,
2721 0 : } = tokio_epoll_uring::metrics::global();
2722 0 : self.systems_created.set(systems_created);
2723 0 : mfs.extend(self.systems_created.collect());
2724 0 : self.systems_destroyed.set(systems_destroyed);
2725 0 : mfs.extend(self.systems_destroyed.collect());
2726 0 : mfs
2727 0 : }
2728 : }
2729 :
2730 : impl Collector {
2731 : const NMETRICS: usize = 2;
2732 :
2733 : #[allow(clippy::new_without_default)]
2734 0 : pub fn new() -> Self {
2735 0 : let mut descs = Vec::new();
2736 0 :
2737 0 : let systems_created = UIntGauge::new(
2738 0 : "pageserver_tokio_epoll_uring_systems_created",
2739 0 : "counter of tokio-epoll-uring systems that were created",
2740 0 : )
2741 0 : .unwrap();
2742 0 : descs.extend(
2743 0 : metrics::core::Collector::desc(&systems_created)
2744 0 : .into_iter()
2745 0 : .cloned(),
2746 0 : );
2747 0 :
2748 0 : let systems_destroyed = UIntGauge::new(
2749 0 : "pageserver_tokio_epoll_uring_systems_destroyed",
2750 0 : "counter of tokio-epoll-uring systems that were destroyed",
2751 0 : )
2752 0 : .unwrap();
2753 0 : descs.extend(
2754 0 : metrics::core::Collector::desc(&systems_destroyed)
2755 0 : .into_iter()
2756 0 : .cloned(),
2757 0 : );
2758 0 :
2759 0 : Self {
2760 0 : descs,
2761 0 : systems_created,
2762 0 : systems_destroyed,
2763 0 : }
2764 0 : }
2765 : }
2766 :
2767 64 : pub(crate) static THREAD_LOCAL_LAUNCH_SUCCESSES: Lazy<metrics::IntCounter> = Lazy::new(|| {
2768 : register_int_counter!(
2769 : "pageserver_tokio_epoll_uring_pageserver_thread_local_launch_success_count",
2770 : "Number of times where thread_local_system creation spanned multiple executor threads",
2771 : )
2772 64 : .unwrap()
2773 64 : });
2774 :
2775 0 : pub(crate) static THREAD_LOCAL_LAUNCH_FAILURES: Lazy<metrics::IntCounter> = Lazy::new(|| {
2776 : register_int_counter!(
2777 : "pageserver_tokio_epoll_uring_pageserver_thread_local_launch_failures_count",
2778 : "Number of times thread_local_system creation failed and was retried after back-off.",
2779 : )
2780 0 : .unwrap()
2781 0 : });
2782 : }
2783 :
2784 : pub(crate) mod tenant_throttling {
2785 : use metrics::{register_int_counter_vec, IntCounter};
2786 : use once_cell::sync::Lazy;
2787 :
2788 : use crate::tenant::{self, throttle::Metric};
2789 :
2790 : pub(crate) struct TimelineGet {
2791 : wait_time: IntCounter,
2792 : count: IntCounter,
2793 : }
2794 :
2795 108 : pub(crate) static TIMELINE_GET: Lazy<TimelineGet> = Lazy::new(|| {
2796 108 : static WAIT_USECS: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
2797 108 : register_int_counter_vec!(
2798 108 : "pageserver_tenant_throttling_wait_usecs_sum_global",
2799 108 : "Sum of microseconds that tenants spent waiting for a tenant throttle of a given kind.",
2800 108 : &["kind"]
2801 108 : )
2802 108 : .unwrap()
2803 108 : });
2804 108 :
2805 108 : static WAIT_COUNT: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
2806 108 : register_int_counter_vec!(
2807 108 : "pageserver_tenant_throttling_count_global",
2808 108 : "Count of tenant throttlings, by kind of throttle.",
2809 108 : &["kind"]
2810 108 : )
2811 108 : .unwrap()
2812 108 : });
2813 108 :
2814 108 : let kind = "timeline_get";
2815 108 : TimelineGet {
2816 108 : wait_time: WAIT_USECS.with_label_values(&[kind]),
2817 108 : count: WAIT_COUNT.with_label_values(&[kind]),
2818 108 : }
2819 108 : });
2820 :
2821 : impl Metric for &'static TimelineGet {
2822 : #[inline(always)]
2823 0 : fn observe_throttling(
2824 0 : &self,
2825 0 : tenant::throttle::Observation { wait_time }: &tenant::throttle::Observation,
2826 0 : ) {
2827 0 : let val = u64::try_from(wait_time.as_micros()).unwrap();
2828 0 : self.wait_time.inc_by(val);
2829 0 : self.count.inc();
2830 0 : }
2831 : }
2832 : }
2833 :
2834 : pub(crate) mod disk_usage_based_eviction {
2835 : use super::*;
2836 :
2837 : pub(crate) struct Metrics {
2838 : pub(crate) tenant_collection_time: Histogram,
2839 : pub(crate) tenant_layer_count: Histogram,
2840 : pub(crate) layers_collected: IntCounter,
2841 : pub(crate) layers_selected: IntCounter,
2842 : pub(crate) layers_evicted: IntCounter,
2843 : }
2844 :
2845 : impl Default for Metrics {
2846 0 : fn default() -> Self {
2847 0 : let tenant_collection_time = register_histogram!(
2848 : "pageserver_disk_usage_based_eviction_tenant_collection_seconds",
2849 : "Time spent collecting layers from a tenant -- not normalized by collected layer amount",
2850 : vec![0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0, 10.0]
2851 : )
2852 0 : .unwrap();
2853 0 :
2854 0 : let tenant_layer_count = register_histogram!(
2855 : "pageserver_disk_usage_based_eviction_tenant_collected_layers",
2856 : "Amount of layers gathered from a tenant",
2857 : vec![5.0, 50.0, 500.0, 5000.0, 50000.0]
2858 : )
2859 0 : .unwrap();
2860 0 :
2861 0 : let layers_collected = register_int_counter!(
2862 : "pageserver_disk_usage_based_eviction_collected_layers_total",
2863 : "Amount of layers collected"
2864 : )
2865 0 : .unwrap();
2866 0 :
2867 0 : let layers_selected = register_int_counter!(
2868 : "pageserver_disk_usage_based_eviction_select_layers_total",
2869 : "Amount of layers selected"
2870 : )
2871 0 : .unwrap();
2872 0 :
2873 0 : let layers_evicted = register_int_counter!(
2874 : "pageserver_disk_usage_based_eviction_evicted_layers_total",
2875 : "Amount of layers successfully evicted"
2876 : )
2877 0 : .unwrap();
2878 0 :
2879 0 : Self {
2880 0 : tenant_collection_time,
2881 0 : tenant_layer_count,
2882 0 : layers_collected,
2883 0 : layers_selected,
2884 0 : layers_evicted,
2885 0 : }
2886 0 : }
2887 : }
2888 :
2889 : pub(crate) static METRICS: Lazy<Metrics> = Lazy::new(Metrics::default);
2890 : }
2891 :
2892 102 : static TOKIO_EXECUTOR_THREAD_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
2893 : register_uint_gauge_vec!(
2894 : "pageserver_tokio_executor_thread_configured_count",
2895 : "Total number of configued tokio executor threads in the process.
2896 : The `setup` label denotes whether we're running with multiple runtimes or a single runtime.",
2897 : &["setup"],
2898 : )
2899 102 : .unwrap()
2900 102 : });
2901 :
2902 102 : pub(crate) fn set_tokio_runtime_setup(setup: &str, num_threads: NonZeroUsize) {
2903 102 : static SERIALIZE: std::sync::Mutex<()> = std::sync::Mutex::new(());
2904 102 : let _guard = SERIALIZE.lock().unwrap();
2905 102 : TOKIO_EXECUTOR_THREAD_COUNT.reset();
2906 102 : TOKIO_EXECUTOR_THREAD_COUNT
2907 102 : .get_metric_with_label_values(&[setup])
2908 102 : .unwrap()
2909 102 : .set(u64::try_from(num_threads.get()).unwrap());
2910 102 : }
2911 :
2912 0 : pub fn preinitialize_metrics() {
2913 0 : // Python tests need these and on some we do alerting.
2914 0 : //
2915 0 : // FIXME(4813): make it so that we have no top level metrics as this fn will easily fall out of
2916 0 : // order:
2917 0 : // - global metrics reside in a Lazy<PageserverMetrics>
2918 0 : // - access via crate::metrics::PS_METRICS.materialized_page_cache_hit.inc()
2919 0 : // - could move the statics into TimelineMetrics::new()?
2920 0 :
2921 0 : // counters
2922 0 : [
2923 0 : &MATERIALIZED_PAGE_CACHE_HIT,
2924 0 : &MATERIALIZED_PAGE_CACHE_HIT_DIRECT,
2925 0 : &UNEXPECTED_ONDEMAND_DOWNLOADS,
2926 0 : &WALRECEIVER_STARTED_CONNECTIONS,
2927 0 : &WALRECEIVER_BROKER_UPDATES,
2928 0 : &WALRECEIVER_CANDIDATES_ADDED,
2929 0 : &WALRECEIVER_CANDIDATES_REMOVED,
2930 0 : &tokio_epoll_uring::THREAD_LOCAL_LAUNCH_FAILURES,
2931 0 : &tokio_epoll_uring::THREAD_LOCAL_LAUNCH_SUCCESSES,
2932 0 : &REMOTE_ONDEMAND_DOWNLOADED_LAYERS,
2933 0 : &REMOTE_ONDEMAND_DOWNLOADED_BYTES,
2934 0 : ]
2935 0 : .into_iter()
2936 0 : .for_each(|c| {
2937 0 : Lazy::force(c);
2938 0 : });
2939 0 :
2940 0 : // Deletion queue stats
2941 0 : Lazy::force(&DELETION_QUEUE);
2942 0 :
2943 0 : // Tenant stats
2944 0 : Lazy::force(&TENANT);
2945 0 :
2946 0 : // Tenant manager stats
2947 0 : Lazy::force(&TENANT_MANAGER);
2948 0 :
2949 0 : Lazy::force(&crate::tenant::storage_layer::layer::LAYER_IMPL_METRICS);
2950 0 : Lazy::force(&disk_usage_based_eviction::METRICS);
2951 :
2952 0 : for state_name in pageserver_api::models::TenantState::VARIANTS {
2953 0 : // initialize the metric for all gauges, otherwise the time series might seemingly show
2954 0 : // values from last restart.
2955 0 : TENANT_STATE_METRIC.with_label_values(&[state_name]).set(0);
2956 0 : }
2957 :
2958 : // countervecs
2959 0 : [&BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT]
2960 0 : .into_iter()
2961 0 : .for_each(|c| {
2962 0 : Lazy::force(c);
2963 0 : });
2964 0 :
2965 0 : // gauges
2966 0 : WALRECEIVER_ACTIVE_MANAGERS.get();
2967 0 :
2968 0 : // histograms
2969 0 : [
2970 0 : &READ_NUM_LAYERS_VISITED,
2971 0 : &VEC_READ_NUM_LAYERS_VISITED,
2972 0 : &WAIT_LSN_TIME,
2973 0 : &WAL_REDO_TIME,
2974 0 : &WAL_REDO_RECORDS_HISTOGRAM,
2975 0 : &WAL_REDO_BYTES_HISTOGRAM,
2976 0 : &WAL_REDO_PROCESS_LAUNCH_DURATION_HISTOGRAM,
2977 0 : ]
2978 0 : .into_iter()
2979 0 : .for_each(|h| {
2980 0 : Lazy::force(h);
2981 0 : });
2982 0 :
2983 0 : // Custom
2984 0 : Lazy::force(&RECONSTRUCT_TIME);
2985 0 : Lazy::force(&tenant_throttling::TIMELINE_GET);
2986 0 : }
|
