Line data Source code
1 : use std::collections::HashMap;
2 : use std::num::NonZeroUsize;
3 : use std::os::fd::RawFd;
4 : use std::pin::Pin;
5 : use std::sync::atomic::AtomicU64;
6 : use std::sync::{Arc, Mutex};
7 : use std::task::{Context, Poll};
8 : use std::time::{Duration, Instant};
9 :
10 : use enum_map::{Enum as _, EnumMap};
11 : use futures::Future;
12 : use metrics::{
13 : Counter, CounterVec, Gauge, GaugeVec, Histogram, HistogramVec, IntCounter, IntCounterPair,
14 : IntCounterPairVec, IntCounterVec, IntGauge, IntGaugeVec, UIntGauge, UIntGaugeVec,
15 : register_counter_vec, register_gauge_vec, register_histogram, register_histogram_vec,
16 : register_int_counter, register_int_counter_pair_vec, register_int_counter_vec,
17 : register_int_gauge, register_int_gauge_vec, register_uint_gauge, register_uint_gauge_vec,
18 : };
19 : use once_cell::sync::Lazy;
20 : use pageserver_api::config::{
21 : PageServicePipeliningConfig, PageServicePipeliningConfigPipelined,
22 : PageServiceProtocolPipelinedExecutionStrategy,
23 : };
24 : use pageserver_api::models::InMemoryLayerInfo;
25 : use pageserver_api::shard::TenantShardId;
26 : use pin_project_lite::pin_project;
27 : use postgres_backend::{QueryError, is_expected_io_error};
28 : use pq_proto::framed::ConnectionError;
29 : use strum::{EnumCount, IntoEnumIterator as _, VariantNames};
30 : use strum_macros::{IntoStaticStr, VariantNames};
31 : use utils::id::TimelineId;
32 :
33 : use crate::config::PageServerConf;
34 : use crate::context::{PageContentKind, RequestContext};
35 : use crate::pgdatadir_mapping::DatadirModificationStats;
36 : use crate::task_mgr::TaskKind;
37 : use crate::tenant::Timeline;
38 : use crate::tenant::layer_map::LayerMap;
39 : use crate::tenant::mgr::TenantSlot;
40 : use crate::tenant::storage_layer::{InMemoryLayer, PersistentLayerDesc};
41 : use crate::tenant::tasks::BackgroundLoopKind;
42 : use crate::tenant::throttle::ThrottleResult;
43 :
44 : /// Prometheus histogram buckets (in seconds) for operations in the critical
45 : /// path. In other words, operations that directly affect that latency of user
46 : /// queries.
47 : ///
48 : /// The buckets capture the majority of latencies in the microsecond and
49 : /// millisecond range but also extend far enough up to distinguish "bad" from
50 : /// "really bad".
51 : const CRITICAL_OP_BUCKETS: &[f64] = &[
52 : 0.000_001, 0.000_010, 0.000_100, // 1 us, 10 us, 100 us
53 : 0.001_000, 0.010_000, 0.100_000, // 1 ms, 10 ms, 100 ms
54 : 1.0, 10.0, 100.0, // 1 s, 10 s, 100 s
55 : ];
56 :
57 : // Metrics collected on operations on the storage repository.
58 : #[derive(Debug, VariantNames, IntoStaticStr)]
59 : #[strum(serialize_all = "kebab_case")]
60 : pub(crate) enum StorageTimeOperation {
61 : #[strum(serialize = "layer flush")]
62 : LayerFlush,
63 :
64 : #[strum(serialize = "layer flush delay")]
65 : LayerFlushDelay,
66 :
67 : #[strum(serialize = "compact")]
68 : Compact,
69 :
70 : #[strum(serialize = "create images")]
71 : CreateImages,
72 :
73 : #[strum(serialize = "logical size")]
74 : LogicalSize,
75 :
76 : #[strum(serialize = "imitate logical size")]
77 : ImitateLogicalSize,
78 :
79 : #[strum(serialize = "load layer map")]
80 : LoadLayerMap,
81 :
82 : #[strum(serialize = "gc")]
83 : Gc,
84 :
85 : #[strum(serialize = "find gc cutoffs")]
86 : FindGcCutoffs,
87 : }
88 :
89 414 : pub(crate) static STORAGE_TIME_SUM_PER_TIMELINE: Lazy<CounterVec> = Lazy::new(|| {
90 414 : register_counter_vec!(
91 414 : "pageserver_storage_operations_seconds_sum",
92 414 : "Total time spent on storage operations with operation, tenant and timeline dimensions",
93 414 : &["operation", "tenant_id", "shard_id", "timeline_id"],
94 414 : )
95 414 : .expect("failed to define a metric")
96 414 : });
97 :
98 414 : pub(crate) static STORAGE_TIME_COUNT_PER_TIMELINE: Lazy<IntCounterVec> = Lazy::new(|| {
99 414 : register_int_counter_vec!(
100 414 : "pageserver_storage_operations_seconds_count",
101 414 : "Count of storage operations with operation, tenant and timeline dimensions",
102 414 : &["operation", "tenant_id", "shard_id", "timeline_id"],
103 414 : )
104 414 : .expect("failed to define a metric")
105 414 : });
106 :
107 : // Buckets for background operation duration in seconds, like compaction, GC, size calculation.
108 : const STORAGE_OP_BUCKETS: &[f64] = &[0.010, 0.100, 1.0, 10.0, 100.0, 1000.0];
109 :
110 414 : pub(crate) static STORAGE_TIME_GLOBAL: Lazy<HistogramVec> = Lazy::new(|| {
111 414 : register_histogram_vec!(
112 414 : "pageserver_storage_operations_seconds_global",
113 414 : "Time spent on storage operations",
114 414 : &["operation"],
115 414 : STORAGE_OP_BUCKETS.into(),
116 414 : )
117 414 : .expect("failed to define a metric")
118 414 : });
119 :
120 : /// Measures layers visited per read (i.e. read amplification).
121 : ///
122 : /// NB: for a batch, we count all visited layers towards each read. While the cost of layer visits
123 : /// are amortized across the batch, and some layers may not intersect with a given key, each visited
124 : /// layer contributes directly to the observed latency for every read in the batch, which is what we
125 : /// care about.
126 414 : pub(crate) static LAYERS_PER_READ: Lazy<HistogramVec> = Lazy::new(|| {
127 414 : register_histogram_vec!(
128 414 : "pageserver_layers_per_read",
129 414 : "Layers visited to serve a single read (read amplification). In a batch, all visited layers count towards every read.",
130 414 : &["tenant_id", "shard_id", "timeline_id"],
131 414 : // Low resolution to reduce cardinality.
132 414 : vec![4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0],
133 414 : )
134 414 : .expect("failed to define a metric")
135 414 : });
136 :
137 406 : pub(crate) static LAYERS_PER_READ_GLOBAL: Lazy<Histogram> = Lazy::new(|| {
138 406 : register_histogram!(
139 406 : "pageserver_layers_per_read_global",
140 406 : "Layers visited to serve a single read (read amplification). In a batch, all visited layers count towards every read.",
141 406 : vec![1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0],
142 406 : )
143 406 : .expect("failed to define a metric")
144 406 : });
145 :
146 406 : pub(crate) static LAYERS_PER_READ_BATCH_GLOBAL: Lazy<Histogram> = Lazy::new(|| {
147 406 : register_histogram!(
148 406 : "pageserver_layers_per_read_batch_global",
149 406 : "Layers visited to serve a single read batch (read amplification), regardless of number of reads.",
150 406 : vec![
151 406 : 1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0
152 406 : ],
153 406 : )
154 406 : .expect("failed to define a metric")
155 406 : });
156 :
157 406 : pub(crate) static LAYERS_PER_READ_AMORTIZED_GLOBAL: Lazy<Histogram> = Lazy::new(|| {
158 406 : register_histogram!(
159 406 : "pageserver_layers_per_read_amortized_global",
160 406 : "Layers visited to serve a single read (read amplification). Amortized across a batch: \
161 406 : all visited layers are divided by number of reads.",
162 406 : vec![
163 406 : 1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0
164 406 : ],
165 406 : )
166 406 : .expect("failed to define a metric")
167 406 : });
168 :
169 406 : pub(crate) static DELTAS_PER_READ_GLOBAL: Lazy<Histogram> = Lazy::new(|| {
170 406 : // We expect this to be low because of Postgres checkpoints. Let's see if that holds.
171 406 : register_histogram!(
172 406 : "pageserver_deltas_per_read_global",
173 406 : "Number of delta pages applied to image page per read",
174 406 : vec![0.0, 1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0],
175 406 : )
176 406 : .expect("failed to define a metric")
177 406 : });
178 :
179 0 : pub(crate) static CONCURRENT_INITDBS: Lazy<UIntGauge> = Lazy::new(|| {
180 0 : register_uint_gauge!(
181 0 : "pageserver_concurrent_initdb",
182 0 : "Number of initdb processes running"
183 0 : )
184 0 : .expect("failed to define a metric")
185 0 : });
186 :
187 0 : pub(crate) static INITDB_SEMAPHORE_ACQUISITION_TIME: Lazy<Histogram> = Lazy::new(|| {
188 0 : register_histogram!(
189 0 : "pageserver_initdb_semaphore_seconds_global",
190 0 : "Time spent getting a permit from the global initdb semaphore",
191 0 : STORAGE_OP_BUCKETS.into()
192 0 : )
193 0 : .expect("failed to define metric")
194 0 : });
195 :
196 0 : pub(crate) static INITDB_RUN_TIME: Lazy<Histogram> = Lazy::new(|| {
197 0 : register_histogram!(
198 0 : "pageserver_initdb_seconds_global",
199 0 : "Time spent performing initdb",
200 0 : STORAGE_OP_BUCKETS.into()
201 0 : )
202 0 : .expect("failed to define metric")
203 0 : });
204 :
205 : pub(crate) struct GetVectoredLatency {
206 : map: EnumMap<TaskKind, Option<Histogram>>,
207 : }
208 :
209 : #[allow(dead_code)]
210 : pub(crate) struct ScanLatency {
211 : map: EnumMap<TaskKind, Option<Histogram>>,
212 : }
213 :
214 : impl GetVectoredLatency {
215 : // Only these task types perform vectored gets. Filter all other tasks out to reduce total
216 : // cardinality of the metric.
217 : const TRACKED_TASK_KINDS: [TaskKind; 2] = [TaskKind::Compaction, TaskKind::PageRequestHandler];
218 :
219 39444 : pub(crate) fn for_task_kind(&self, task_kind: TaskKind) -> Option<&Histogram> {
220 39444 : self.map[task_kind].as_ref()
221 39444 : }
222 : }
223 :
224 : impl ScanLatency {
225 : // Only these task types perform vectored gets. Filter all other tasks out to reduce total
226 : // cardinality of the metric.
227 : const TRACKED_TASK_KINDS: [TaskKind; 1] = [TaskKind::PageRequestHandler];
228 :
229 24 : pub(crate) fn for_task_kind(&self, task_kind: TaskKind) -> Option<&Histogram> {
230 24 : self.map[task_kind].as_ref()
231 24 : }
232 : }
233 :
234 : pub(crate) struct ScanLatencyOngoingRecording<'a> {
235 : parent: &'a Histogram,
236 : start: std::time::Instant,
237 : }
238 :
239 : impl<'a> ScanLatencyOngoingRecording<'a> {
240 0 : pub(crate) fn start_recording(parent: &'a Histogram) -> ScanLatencyOngoingRecording<'a> {
241 0 : let start = Instant::now();
242 0 : ScanLatencyOngoingRecording { parent, start }
243 0 : }
244 :
245 0 : pub(crate) fn observe(self) {
246 0 : let elapsed = self.start.elapsed();
247 0 : self.parent.observe(elapsed.as_secs_f64());
248 0 : }
249 : }
250 :
251 398 : pub(crate) static GET_VECTORED_LATENCY: Lazy<GetVectoredLatency> = Lazy::new(|| {
252 398 : let inner = register_histogram_vec!(
253 398 : "pageserver_get_vectored_seconds",
254 398 : "Time spent in get_vectored.",
255 398 : &["task_kind"],
256 398 : CRITICAL_OP_BUCKETS.into(),
257 398 : )
258 398 : .expect("failed to define a metric");
259 398 :
260 398 : GetVectoredLatency {
261 12338 : map: EnumMap::from_array(std::array::from_fn(|task_kind_idx| {
262 12338 : let task_kind = TaskKind::from_usize(task_kind_idx);
263 12338 :
264 12338 : if GetVectoredLatency::TRACKED_TASK_KINDS.contains(&task_kind) {
265 796 : let task_kind = task_kind.into();
266 796 : Some(inner.with_label_values(&[task_kind]))
267 : } else {
268 11542 : None
269 : }
270 12338 : })),
271 398 : }
272 398 : });
273 :
274 8 : pub(crate) static SCAN_LATENCY: Lazy<ScanLatency> = Lazy::new(|| {
275 8 : let inner = register_histogram_vec!(
276 8 : "pageserver_scan_seconds",
277 8 : "Time spent in scan.",
278 8 : &["task_kind"],
279 8 : CRITICAL_OP_BUCKETS.into(),
280 8 : )
281 8 : .expect("failed to define a metric");
282 8 :
283 8 : ScanLatency {
284 248 : map: EnumMap::from_array(std::array::from_fn(|task_kind_idx| {
285 248 : let task_kind = TaskKind::from_usize(task_kind_idx);
286 248 :
287 248 : if ScanLatency::TRACKED_TASK_KINDS.contains(&task_kind) {
288 8 : let task_kind = task_kind.into();
289 8 : Some(inner.with_label_values(&[task_kind]))
290 : } else {
291 240 : None
292 : }
293 248 : })),
294 8 : }
295 8 : });
296 :
297 : pub(crate) struct PageCacheMetricsForTaskKind {
298 : pub read_accesses_immutable: IntCounter,
299 : pub read_hits_immutable: IntCounter,
300 : }
301 :
302 : pub(crate) struct PageCacheMetrics {
303 : map: EnumMap<TaskKind, EnumMap<PageContentKind, PageCacheMetricsForTaskKind>>,
304 : }
305 :
306 189 : static PAGE_CACHE_READ_HITS: Lazy<IntCounterVec> = Lazy::new(|| {
307 189 : register_int_counter_vec!(
308 189 : "pageserver_page_cache_read_hits_total",
309 189 : "Number of read accesses to the page cache that hit",
310 189 : &["task_kind", "key_kind", "content_kind", "hit_kind"]
311 189 : )
312 189 : .expect("failed to define a metric")
313 189 : });
314 :
315 189 : static PAGE_CACHE_READ_ACCESSES: Lazy<IntCounterVec> = Lazy::new(|| {
316 189 : register_int_counter_vec!(
317 189 : "pageserver_page_cache_read_accesses_total",
318 189 : "Number of read accesses to the page cache",
319 189 : &["task_kind", "key_kind", "content_kind"]
320 189 : )
321 189 : .expect("failed to define a metric")
322 189 : });
323 :
324 189 : pub(crate) static PAGE_CACHE: Lazy<PageCacheMetrics> = Lazy::new(|| PageCacheMetrics {
325 5859 : map: EnumMap::from_array(std::array::from_fn(|task_kind| {
326 5859 : let task_kind = TaskKind::from_usize(task_kind);
327 5859 : let task_kind: &'static str = task_kind.into();
328 46872 : EnumMap::from_array(std::array::from_fn(|content_kind| {
329 46872 : let content_kind = PageContentKind::from_usize(content_kind);
330 46872 : let content_kind: &'static str = content_kind.into();
331 46872 : PageCacheMetricsForTaskKind {
332 46872 : read_accesses_immutable: {
333 46872 : PAGE_CACHE_READ_ACCESSES
334 46872 : .get_metric_with_label_values(&[task_kind, "immutable", content_kind])
335 46872 : .unwrap()
336 46872 : },
337 46872 :
338 46872 : read_hits_immutable: {
339 46872 : PAGE_CACHE_READ_HITS
340 46872 : .get_metric_with_label_values(&[task_kind, "immutable", content_kind, "-"])
341 46872 : .unwrap()
342 46872 : },
343 46872 : }
344 46872 : }))
345 5859 : })),
346 189 : });
347 :
348 : impl PageCacheMetrics {
349 1941952 : pub(crate) fn for_ctx(&self, ctx: &RequestContext) -> &PageCacheMetricsForTaskKind {
350 1941952 : &self.map[ctx.task_kind()][ctx.page_content_kind()]
351 1941952 : }
352 : }
353 :
354 : pub(crate) struct PageCacheSizeMetrics {
355 : pub max_bytes: UIntGauge,
356 :
357 : pub current_bytes_immutable: UIntGauge,
358 : }
359 :
360 189 : static PAGE_CACHE_SIZE_CURRENT_BYTES: Lazy<UIntGaugeVec> = Lazy::new(|| {
361 189 : register_uint_gauge_vec!(
362 189 : "pageserver_page_cache_size_current_bytes",
363 189 : "Current size of the page cache in bytes, by key kind",
364 189 : &["key_kind"]
365 189 : )
366 189 : .expect("failed to define a metric")
367 189 : });
368 :
369 : pub(crate) static PAGE_CACHE_SIZE: Lazy<PageCacheSizeMetrics> =
370 189 : Lazy::new(|| PageCacheSizeMetrics {
371 189 : max_bytes: {
372 189 : register_uint_gauge!(
373 189 : "pageserver_page_cache_size_max_bytes",
374 189 : "Maximum size of the page cache in bytes"
375 189 : )
376 189 : .expect("failed to define a metric")
377 189 : },
378 189 : current_bytes_immutable: {
379 189 : PAGE_CACHE_SIZE_CURRENT_BYTES
380 189 : .get_metric_with_label_values(&["immutable"])
381 189 : .unwrap()
382 189 : },
383 189 : });
384 :
385 : pub(crate) mod page_cache_eviction_metrics {
386 : use std::num::NonZeroUsize;
387 :
388 : use metrics::{IntCounter, IntCounterVec, register_int_counter_vec};
389 : use once_cell::sync::Lazy;
390 :
391 : #[derive(Clone, Copy)]
392 : pub(crate) enum Outcome {
393 : FoundSlotUnused { iters: NonZeroUsize },
394 : FoundSlotEvicted { iters: NonZeroUsize },
395 : ItersExceeded { iters: NonZeroUsize },
396 : }
397 :
398 189 : static ITERS_TOTAL_VEC: Lazy<IntCounterVec> = Lazy::new(|| {
399 189 : register_int_counter_vec!(
400 189 : "pageserver_page_cache_find_victim_iters_total",
401 189 : "Counter for the number of iterations in the find_victim loop",
402 189 : &["outcome"],
403 189 : )
404 189 : .expect("failed to define a metric")
405 189 : });
406 :
407 189 : static CALLS_VEC: Lazy<IntCounterVec> = Lazy::new(|| {
408 189 : register_int_counter_vec!(
409 189 : "pageserver_page_cache_find_victim_calls",
410 189 : "Incremented at the end of each find_victim() call.\
411 189 : Filter by outcome to get e.g., eviction rate.",
412 189 : &["outcome"]
413 189 : )
414 189 : .unwrap()
415 189 : });
416 :
417 61551 : pub(crate) fn observe(outcome: Outcome) {
418 : macro_rules! dry {
419 : ($label:literal, $iters:expr) => {{
420 : static LABEL: &'static str = $label;
421 : static ITERS_TOTAL: Lazy<IntCounter> =
422 229 : Lazy::new(|| ITERS_TOTAL_VEC.with_label_values(&[LABEL]));
423 : static CALLS: Lazy<IntCounter> =
424 229 : Lazy::new(|| CALLS_VEC.with_label_values(&[LABEL]));
425 : ITERS_TOTAL.inc_by(($iters.get()) as u64);
426 : CALLS.inc();
427 : }};
428 : }
429 61551 : match outcome {
430 3334 : Outcome::FoundSlotUnused { iters } => dry!("found_empty", iters),
431 58217 : Outcome::FoundSlotEvicted { iters } => {
432 58217 : dry!("found_evicted", iters)
433 : }
434 0 : Outcome::ItersExceeded { iters } => {
435 0 : dry!("err_iters_exceeded", iters);
436 0 : super::page_cache_errors_inc(super::PageCacheErrorKind::EvictIterLimit);
437 0 : }
438 : }
439 61551 : }
440 : }
441 :
442 0 : static PAGE_CACHE_ERRORS: Lazy<IntCounterVec> = Lazy::new(|| {
443 0 : register_int_counter_vec!(
444 0 : "page_cache_errors_total",
445 0 : "Number of timeouts while acquiring a pinned slot in the page cache",
446 0 : &["error_kind"]
447 0 : )
448 0 : .expect("failed to define a metric")
449 0 : });
450 :
451 : #[derive(IntoStaticStr)]
452 : #[strum(serialize_all = "kebab_case")]
453 : pub(crate) enum PageCacheErrorKind {
454 : AcquirePinnedSlotTimeout,
455 : EvictIterLimit,
456 : }
457 :
458 0 : pub(crate) fn page_cache_errors_inc(error_kind: PageCacheErrorKind) {
459 0 : PAGE_CACHE_ERRORS
460 0 : .get_metric_with_label_values(&[error_kind.into()])
461 0 : .unwrap()
462 0 : .inc();
463 0 : }
464 :
465 40 : pub(crate) static WAIT_LSN_TIME: Lazy<Histogram> = Lazy::new(|| {
466 40 : register_histogram!(
467 40 : "pageserver_wait_lsn_seconds",
468 40 : "Time spent waiting for WAL to arrive",
469 40 : CRITICAL_OP_BUCKETS.into(),
470 40 : )
471 40 : .expect("failed to define a metric")
472 40 : });
473 :
474 414 : static FLUSH_WAIT_UPLOAD_TIME: Lazy<GaugeVec> = Lazy::new(|| {
475 414 : register_gauge_vec!(
476 414 : "pageserver_flush_wait_upload_seconds",
477 414 : "Time spent waiting for preceding uploads during layer flush",
478 414 : &["tenant_id", "shard_id", "timeline_id"]
479 414 : )
480 414 : .expect("failed to define a metric")
481 414 : });
482 :
483 414 : static LAST_RECORD_LSN: Lazy<IntGaugeVec> = Lazy::new(|| {
484 414 : register_int_gauge_vec!(
485 414 : "pageserver_last_record_lsn",
486 414 : "Last record LSN grouped by timeline",
487 414 : &["tenant_id", "shard_id", "timeline_id"]
488 414 : )
489 414 : .expect("failed to define a metric")
490 414 : });
491 :
492 414 : static DISK_CONSISTENT_LSN: Lazy<IntGaugeVec> = Lazy::new(|| {
493 414 : register_int_gauge_vec!(
494 414 : "pageserver_disk_consistent_lsn",
495 414 : "Disk consistent LSN grouped by timeline",
496 414 : &["tenant_id", "shard_id", "timeline_id"]
497 414 : )
498 414 : .expect("failed to define a metric")
499 414 : });
500 :
501 414 : pub(crate) static PROJECTED_REMOTE_CONSISTENT_LSN: Lazy<UIntGaugeVec> = Lazy::new(|| {
502 414 : register_uint_gauge_vec!(
503 414 : "pageserver_projected_remote_consistent_lsn",
504 414 : "Projected remote consistent LSN grouped by timeline",
505 414 : &["tenant_id", "shard_id", "timeline_id"]
506 414 : )
507 414 : .expect("failed to define a metric")
508 414 : });
509 :
510 414 : static PITR_HISTORY_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
511 414 : register_uint_gauge_vec!(
512 414 : "pageserver_pitr_history_size",
513 414 : "Data written since PITR cutoff on this timeline",
514 414 : &["tenant_id", "shard_id", "timeline_id"]
515 414 : )
516 414 : .expect("failed to define a metric")
517 414 : });
518 :
519 : #[derive(
520 240 : strum_macros::EnumIter,
521 0 : strum_macros::EnumString,
522 : strum_macros::Display,
523 : strum_macros::IntoStaticStr,
524 : )]
525 : #[strum(serialize_all = "kebab_case")]
526 : pub(crate) enum LayerKind {
527 : Delta,
528 : Image,
529 : }
530 :
531 : #[derive(
532 100 : strum_macros::EnumIter,
533 0 : strum_macros::EnumString,
534 : strum_macros::Display,
535 : strum_macros::IntoStaticStr,
536 : )]
537 : #[strum(serialize_all = "kebab_case")]
538 : pub(crate) enum LayerLevel {
539 : // We don't track the currently open ephemeral layer, since there's always exactly 1 and its
540 : // size changes. See `TIMELINE_EPHEMERAL_BYTES`.
541 : Frozen,
542 : L0,
543 : L1,
544 : }
545 :
546 406 : static TIMELINE_LAYER_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
547 406 : register_uint_gauge_vec!(
548 406 : "pageserver_layer_bytes",
549 406 : "Sum of frozen, L0, and L1 layer physical sizes in bytes (excluding the open ephemeral layer)",
550 406 : &["tenant_id", "shard_id", "timeline_id", "level", "kind"]
551 406 : )
552 406 : .expect("failed to define a metric")
553 406 : });
554 :
555 406 : static TIMELINE_LAYER_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
556 406 : register_uint_gauge_vec!(
557 406 : "pageserver_layer_count",
558 406 : "Number of frozen, L0, and L1 layers (excluding the open ephemeral layer)",
559 406 : &["tenant_id", "shard_id", "timeline_id", "level", "kind"]
560 406 : )
561 406 : .expect("failed to define a metric")
562 406 : });
563 :
564 414 : static TIMELINE_ARCHIVE_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
565 414 : register_uint_gauge_vec!(
566 414 : "pageserver_archive_size",
567 414 : "Timeline's logical size if it is considered eligible for archival (outside PITR window), else zero",
568 414 : &["tenant_id", "shard_id", "timeline_id"]
569 414 : )
570 414 : .expect("failed to define a metric")
571 414 : });
572 :
573 414 : static STANDBY_HORIZON: Lazy<IntGaugeVec> = Lazy::new(|| {
574 414 : register_int_gauge_vec!(
575 414 : "pageserver_standby_horizon",
576 414 : "Standby apply LSN for which GC is hold off, by timeline.",
577 414 : &["tenant_id", "shard_id", "timeline_id"]
578 414 : )
579 414 : .expect("failed to define a metric")
580 414 : });
581 :
582 414 : static RESIDENT_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
583 414 : register_uint_gauge_vec!(
584 414 : "pageserver_resident_physical_size",
585 414 : "The size of the layer files present in the pageserver's filesystem, for attached locations.",
586 414 : &["tenant_id", "shard_id", "timeline_id"]
587 414 : )
588 414 : .expect("failed to define a metric")
589 414 : });
590 :
591 414 : static VISIBLE_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
592 414 : register_uint_gauge_vec!(
593 414 : "pageserver_visible_physical_size",
594 414 : "The size of the layer files present in the pageserver's filesystem.",
595 414 : &["tenant_id", "shard_id", "timeline_id"]
596 414 : )
597 414 : .expect("failed to define a metric")
598 414 : });
599 :
600 406 : pub(crate) static RESIDENT_PHYSICAL_SIZE_GLOBAL: Lazy<UIntGauge> = Lazy::new(|| {
601 406 : register_uint_gauge!(
602 406 : "pageserver_resident_physical_size_global",
603 406 : "Like `pageserver_resident_physical_size`, but without tenant/timeline dimensions."
604 406 : )
605 406 : .expect("failed to define a metric")
606 406 : });
607 :
608 414 : static REMOTE_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
609 414 : register_uint_gauge_vec!(
610 414 : "pageserver_remote_physical_size",
611 414 : "The size of the layer files present in the remote storage that are listed in the remote index_part.json.",
612 414 : // Corollary: If any files are missing from the index part, they won't be included here.
613 414 : &["tenant_id", "shard_id", "timeline_id"]
614 414 : )
615 414 : .expect("failed to define a metric")
616 414 : });
617 :
618 414 : static REMOTE_PHYSICAL_SIZE_GLOBAL: Lazy<UIntGauge> = Lazy::new(|| {
619 414 : register_uint_gauge!(
620 414 : "pageserver_remote_physical_size_global",
621 414 : "Like `pageserver_remote_physical_size`, but without tenant/timeline dimensions."
622 414 : )
623 414 : .expect("failed to define a metric")
624 414 : });
625 :
626 12 : pub(crate) static REMOTE_ONDEMAND_DOWNLOADED_LAYERS: Lazy<IntCounter> = Lazy::new(|| {
627 12 : register_int_counter!(
628 12 : "pageserver_remote_ondemand_downloaded_layers_total",
629 12 : "Total on-demand downloaded layers"
630 12 : )
631 12 : .unwrap()
632 12 : });
633 :
634 12 : pub(crate) static REMOTE_ONDEMAND_DOWNLOADED_BYTES: Lazy<IntCounter> = Lazy::new(|| {
635 12 : register_int_counter!(
636 12 : "pageserver_remote_ondemand_downloaded_bytes_total",
637 12 : "Total bytes of layers on-demand downloaded",
638 12 : )
639 12 : .unwrap()
640 12 : });
641 :
642 414 : static CURRENT_LOGICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
643 414 : register_uint_gauge_vec!(
644 414 : "pageserver_current_logical_size",
645 414 : "Current logical size grouped by timeline",
646 414 : &["tenant_id", "shard_id", "timeline_id"]
647 414 : )
648 414 : .expect("failed to define current logical size metric")
649 414 : });
650 :
651 414 : static AUX_FILE_SIZE: Lazy<IntGaugeVec> = Lazy::new(|| {
652 414 : register_int_gauge_vec!(
653 414 : "pageserver_aux_file_estimated_size",
654 414 : "The size of all aux files for a timeline in aux file v2 store.",
655 414 : &["tenant_id", "shard_id", "timeline_id"]
656 414 : )
657 414 : .expect("failed to define a metric")
658 414 : });
659 :
660 414 : static VALID_LSN_LEASE_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
661 414 : register_uint_gauge_vec!(
662 414 : "pageserver_valid_lsn_lease_count",
663 414 : "The number of valid leases after refreshing gc info.",
664 414 : &["tenant_id", "shard_id", "timeline_id"],
665 414 : )
666 414 : .expect("failed to define a metric")
667 414 : });
668 :
669 0 : pub(crate) static CIRCUIT_BREAKERS_BROKEN: Lazy<IntCounter> = Lazy::new(|| {
670 0 : register_int_counter!(
671 0 : "pageserver_circuit_breaker_broken",
672 0 : "How many times a circuit breaker has broken"
673 0 : )
674 0 : .expect("failed to define a metric")
675 0 : });
676 :
677 0 : pub(crate) static CIRCUIT_BREAKERS_UNBROKEN: Lazy<IntCounter> = Lazy::new(|| {
678 0 : register_int_counter!(
679 0 : "pageserver_circuit_breaker_unbroken",
680 0 : "How many times a circuit breaker has been un-broken (recovered)"
681 0 : )
682 0 : .expect("failed to define a metric")
683 0 : });
684 :
685 398 : pub(crate) static COMPRESSION_IMAGE_INPUT_BYTES: Lazy<IntCounter> = Lazy::new(|| {
686 398 : register_int_counter!(
687 398 : "pageserver_compression_image_in_bytes_total",
688 398 : "Size of data written into image layers before compression"
689 398 : )
690 398 : .expect("failed to define a metric")
691 398 : });
692 :
693 398 : pub(crate) static COMPRESSION_IMAGE_INPUT_BYTES_CONSIDERED: Lazy<IntCounter> = Lazy::new(|| {
694 398 : register_int_counter!(
695 398 : "pageserver_compression_image_in_bytes_considered",
696 398 : "Size of potentially compressible data written into image layers before compression"
697 398 : )
698 398 : .expect("failed to define a metric")
699 398 : });
700 :
701 398 : pub(crate) static COMPRESSION_IMAGE_INPUT_BYTES_CHOSEN: Lazy<IntCounter> = Lazy::new(|| {
702 398 : register_int_counter!(
703 398 : "pageserver_compression_image_in_bytes_chosen",
704 398 : "Size of data whose compressed form was written into image layers"
705 398 : )
706 398 : .expect("failed to define a metric")
707 398 : });
708 :
709 398 : pub(crate) static COMPRESSION_IMAGE_OUTPUT_BYTES: Lazy<IntCounter> = Lazy::new(|| {
710 398 : register_int_counter!(
711 398 : "pageserver_compression_image_out_bytes_total",
712 398 : "Size of compressed image layer written"
713 398 : )
714 398 : .expect("failed to define a metric")
715 398 : });
716 :
717 20 : pub(crate) static RELSIZE_CACHE_ENTRIES: Lazy<UIntGauge> = Lazy::new(|| {
718 20 : register_uint_gauge!(
719 20 : "pageserver_relsize_cache_entries",
720 20 : "Number of entries in the relation size cache",
721 20 : )
722 20 : .expect("failed to define a metric")
723 20 : });
724 :
725 20 : pub(crate) static RELSIZE_CACHE_HITS: Lazy<IntCounter> = Lazy::new(|| {
726 20 : register_int_counter!("pageserver_relsize_cache_hits", "Relation size cache hits",)
727 20 : .expect("failed to define a metric")
728 20 : });
729 :
730 20 : pub(crate) static RELSIZE_CACHE_MISSES: Lazy<IntCounter> = Lazy::new(|| {
731 20 : register_int_counter!(
732 20 : "pageserver_relsize_cache_misses",
733 20 : "Relation size cache misses",
734 20 : )
735 20 : .expect("failed to define a metric")
736 20 : });
737 :
738 8 : pub(crate) static RELSIZE_CACHE_MISSES_OLD: Lazy<IntCounter> = Lazy::new(|| {
739 8 : register_int_counter!(
740 8 : "pageserver_relsize_cache_misses_old",
741 8 : "Relation size cache misses where the lookup LSN is older than the last relation update"
742 8 : )
743 8 : .expect("failed to define a metric")
744 8 : });
745 :
746 : pub(crate) mod initial_logical_size {
747 : use metrics::{IntCounter, IntCounterVec, register_int_counter, register_int_counter_vec};
748 : use once_cell::sync::Lazy;
749 :
750 : pub(crate) struct StartCalculation(IntCounterVec);
751 414 : pub(crate) static START_CALCULATION: Lazy<StartCalculation> = Lazy::new(|| {
752 414 : StartCalculation(
753 414 : register_int_counter_vec!(
754 414 : "pageserver_initial_logical_size_start_calculation",
755 414 : "Incremented each time we start an initial logical size calculation attempt. \
756 414 : The `circumstances` label provides some additional details.",
757 414 : &["attempt", "circumstances"]
758 414 : )
759 414 : .unwrap(),
760 414 : )
761 414 : });
762 :
763 : struct DropCalculation {
764 : first: IntCounter,
765 : retry: IntCounter,
766 : }
767 :
768 414 : static DROP_CALCULATION: Lazy<DropCalculation> = Lazy::new(|| {
769 414 : let vec = register_int_counter_vec!(
770 414 : "pageserver_initial_logical_size_drop_calculation",
771 414 : "Incremented each time we abort a started size calculation attmpt.",
772 414 : &["attempt"]
773 414 : )
774 414 : .unwrap();
775 414 : DropCalculation {
776 414 : first: vec.with_label_values(&["first"]),
777 414 : retry: vec.with_label_values(&["retry"]),
778 414 : }
779 414 : });
780 :
781 : pub(crate) struct Calculated {
782 : pub(crate) births: IntCounter,
783 : pub(crate) deaths: IntCounter,
784 : }
785 :
786 414 : pub(crate) static CALCULATED: Lazy<Calculated> = Lazy::new(|| Calculated {
787 414 : births: register_int_counter!(
788 414 : "pageserver_initial_logical_size_finish_calculation",
789 414 : "Incremented every time we finish calculation of initial logical size.\
790 414 : If everything is working well, this should happen at most once per Timeline object."
791 414 : )
792 414 : .unwrap(),
793 414 : deaths: register_int_counter!(
794 414 : "pageserver_initial_logical_size_drop_finished_calculation",
795 414 : "Incremented when we drop a finished initial logical size calculation result.\
796 414 : Mainly useful to turn pageserver_initial_logical_size_finish_calculation into a gauge."
797 414 : )
798 414 : .unwrap(),
799 414 : });
800 :
801 : pub(crate) struct OngoingCalculationGuard {
802 : inc_drop_calculation: Option<IntCounter>,
803 : }
804 :
805 : #[derive(strum_macros::IntoStaticStr)]
806 : pub(crate) enum StartCircumstances {
807 : EmptyInitial,
808 : SkippedConcurrencyLimiter,
809 : AfterBackgroundTasksRateLimit,
810 : }
811 :
812 : impl StartCalculation {
813 438 : pub(crate) fn first(&self, circumstances: StartCircumstances) -> OngoingCalculationGuard {
814 438 : let circumstances_label: &'static str = circumstances.into();
815 438 : self.0
816 438 : .with_label_values(&["first", circumstances_label])
817 438 : .inc();
818 438 : OngoingCalculationGuard {
819 438 : inc_drop_calculation: Some(DROP_CALCULATION.first.clone()),
820 438 : }
821 438 : }
822 0 : pub(crate) fn retry(&self, circumstances: StartCircumstances) -> OngoingCalculationGuard {
823 0 : let circumstances_label: &'static str = circumstances.into();
824 0 : self.0
825 0 : .with_label_values(&["retry", circumstances_label])
826 0 : .inc();
827 0 : OngoingCalculationGuard {
828 0 : inc_drop_calculation: Some(DROP_CALCULATION.retry.clone()),
829 0 : }
830 0 : }
831 : }
832 :
833 : impl Drop for OngoingCalculationGuard {
834 438 : fn drop(&mut self) {
835 438 : if let Some(counter) = self.inc_drop_calculation.take() {
836 0 : counter.inc();
837 438 : }
838 438 : }
839 : }
840 :
841 : impl OngoingCalculationGuard {
842 438 : pub(crate) fn calculation_result_saved(mut self) -> FinishedCalculationGuard {
843 438 : drop(self.inc_drop_calculation.take());
844 438 : CALCULATED.births.inc();
845 438 : FinishedCalculationGuard {
846 438 : inc_on_drop: CALCULATED.deaths.clone(),
847 438 : }
848 438 : }
849 : }
850 :
851 : pub(crate) struct FinishedCalculationGuard {
852 : inc_on_drop: IntCounter,
853 : }
854 :
855 : impl Drop for FinishedCalculationGuard {
856 12 : fn drop(&mut self) {
857 12 : self.inc_on_drop.inc();
858 12 : }
859 : }
860 :
861 : // context: https://github.com/neondatabase/neon/issues/5963
862 : pub(crate) static TIMELINES_WHERE_WALRECEIVER_GOT_APPROXIMATE_SIZE: Lazy<IntCounter> =
863 0 : Lazy::new(|| {
864 0 : register_int_counter!(
865 0 : "pageserver_initial_logical_size_timelines_where_walreceiver_got_approximate_size",
866 0 : "Counter for the following event: walreceiver calls\
867 0 : Timeline::get_current_logical_size() and it returns `Approximate` for the first time."
868 0 : )
869 0 : .unwrap()
870 0 : });
871 : }
872 :
873 0 : static DIRECTORY_ENTRIES_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
874 0 : register_uint_gauge_vec!(
875 0 : "pageserver_directory_entries_count",
876 0 : "Sum of the entries in pageserver-stored directory listings",
877 0 : &["tenant_id", "shard_id", "timeline_id"]
878 0 : )
879 0 : .expect("failed to define a metric")
880 0 : });
881 :
882 418 : pub(crate) static TENANT_STATE_METRIC: Lazy<UIntGaugeVec> = Lazy::new(|| {
883 418 : register_uint_gauge_vec!(
884 418 : "pageserver_tenant_states_count",
885 418 : "Count of tenants per state",
886 418 : &["state"]
887 418 : )
888 418 : .expect("Failed to register pageserver_tenant_states_count metric")
889 418 : });
890 :
891 : /// A set of broken tenants.
892 : ///
893 : /// These are expected to be so rare that a set is fine. Set as in a new timeseries per each broken
894 : /// tenant.
895 20 : pub(crate) static BROKEN_TENANTS_SET: Lazy<UIntGaugeVec> = Lazy::new(|| {
896 20 : register_uint_gauge_vec!(
897 20 : "pageserver_broken_tenants_count",
898 20 : "Set of broken tenants",
899 20 : &["tenant_id", "shard_id"]
900 20 : )
901 20 : .expect("Failed to register pageserver_tenant_states_count metric")
902 20 : });
903 :
904 12 : pub(crate) static TENANT_SYNTHETIC_SIZE_METRIC: Lazy<UIntGaugeVec> = Lazy::new(|| {
905 12 : register_uint_gauge_vec!(
906 12 : "pageserver_tenant_synthetic_cached_size_bytes",
907 12 : "Synthetic size of each tenant in bytes",
908 12 : &["tenant_id"]
909 12 : )
910 12 : .expect("Failed to register pageserver_tenant_synthetic_cached_size_bytes metric")
911 12 : });
912 :
913 0 : pub(crate) static EVICTION_ITERATION_DURATION: Lazy<HistogramVec> = Lazy::new(|| {
914 0 : register_histogram_vec!(
915 0 : "pageserver_eviction_iteration_duration_seconds_global",
916 0 : "Time spent on a single eviction iteration",
917 0 : &["period_secs", "threshold_secs"],
918 0 : STORAGE_OP_BUCKETS.into(),
919 0 : )
920 0 : .expect("failed to define a metric")
921 0 : });
922 :
923 414 : static EVICTIONS: Lazy<IntCounterVec> = Lazy::new(|| {
924 414 : register_int_counter_vec!(
925 414 : "pageserver_evictions",
926 414 : "Number of layers evicted from the pageserver",
927 414 : &["tenant_id", "shard_id", "timeline_id"]
928 414 : )
929 414 : .expect("failed to define a metric")
930 414 : });
931 :
932 414 : static EVICTIONS_WITH_LOW_RESIDENCE_DURATION: Lazy<IntCounterVec> = Lazy::new(|| {
933 414 : register_int_counter_vec!(
934 414 : "pageserver_evictions_with_low_residence_duration",
935 414 : "If a layer is evicted that was resident for less than `low_threshold`, it is counted to this counter. \
936 414 : Residence duration is determined using the `residence_duration_data_source`.",
937 414 : &["tenant_id", "shard_id", "timeline_id", "residence_duration_data_source", "low_threshold_secs"]
938 414 : )
939 414 : .expect("failed to define a metric")
940 414 : });
941 :
942 0 : pub(crate) static UNEXPECTED_ONDEMAND_DOWNLOADS: Lazy<IntCounter> = Lazy::new(|| {
943 0 : register_int_counter!(
944 0 : "pageserver_unexpected_ondemand_downloads_count",
945 0 : "Number of unexpected on-demand downloads. \
946 0 : We log more context for each increment, so, forgo any labels in this metric.",
947 0 : )
948 0 : .expect("failed to define a metric")
949 0 : });
950 :
951 : /// How long did we take to start up? Broken down by labels to describe
952 : /// different phases of startup.
953 0 : pub static STARTUP_DURATION: Lazy<GaugeVec> = Lazy::new(|| {
954 0 : register_gauge_vec!(
955 0 : "pageserver_startup_duration_seconds",
956 0 : "Time taken by phases of pageserver startup, in seconds",
957 0 : &["phase"]
958 0 : )
959 0 : .expect("Failed to register pageserver_startup_duration_seconds metric")
960 0 : });
961 :
962 0 : pub static STARTUP_IS_LOADING: Lazy<UIntGauge> = Lazy::new(|| {
963 0 : register_uint_gauge!(
964 0 : "pageserver_startup_is_loading",
965 0 : "1 while in initial startup load of tenants, 0 at other times"
966 0 : )
967 0 : .expect("Failed to register pageserver_startup_is_loading")
968 0 : });
969 :
970 406 : pub(crate) static TIMELINE_EPHEMERAL_BYTES: Lazy<UIntGauge> = Lazy::new(|| {
971 406 : register_uint_gauge!(
972 406 : "pageserver_timeline_ephemeral_bytes",
973 406 : "Total number of bytes in ephemeral layers, summed for all timelines. Approximate, lazily updated."
974 406 : )
975 406 : .expect("Failed to register metric")
976 406 : });
977 :
978 : /// Metrics related to the lifecycle of a [`crate::tenant::Tenant`] object: things
979 : /// like how long it took to load.
980 : ///
981 : /// Note that these are process-global metrics, _not_ per-tenant metrics. Per-tenant
982 : /// metrics are rather expensive, and usually fine grained stuff makes more sense
983 : /// at a timeline level than tenant level.
984 : pub(crate) struct TenantMetrics {
985 : /// How long did tenants take to go from construction to active state?
986 : pub(crate) activation: Histogram,
987 : pub(crate) preload: Histogram,
988 : pub(crate) attach: Histogram,
989 :
990 : /// How many tenants are included in the initial startup of the pagesrever?
991 : pub(crate) startup_scheduled: IntCounter,
992 : pub(crate) startup_complete: IntCounter,
993 : }
994 :
995 0 : pub(crate) static TENANT: Lazy<TenantMetrics> = Lazy::new(|| {
996 0 : TenantMetrics {
997 0 : activation: register_histogram!(
998 0 : "pageserver_tenant_activation_seconds",
999 0 : "Time taken by tenants to activate, in seconds",
1000 0 : CRITICAL_OP_BUCKETS.into()
1001 0 : )
1002 0 : .expect("Failed to register metric"),
1003 0 : preload: register_histogram!(
1004 0 : "pageserver_tenant_preload_seconds",
1005 0 : "Time taken by tenants to load remote metadata on startup/attach, in seconds",
1006 0 : CRITICAL_OP_BUCKETS.into()
1007 0 : )
1008 0 : .expect("Failed to register metric"),
1009 0 : attach: register_histogram!(
1010 0 : "pageserver_tenant_attach_seconds",
1011 0 : "Time taken by tenants to intialize, after remote metadata is already loaded",
1012 0 : CRITICAL_OP_BUCKETS.into()
1013 0 : )
1014 0 : .expect("Failed to register metric"),
1015 0 : startup_scheduled: register_int_counter!(
1016 0 : "pageserver_tenant_startup_scheduled",
1017 0 : "Number of tenants included in pageserver startup (doesn't count tenants attached later)"
1018 0 : ).expect("Failed to register metric"),
1019 0 : startup_complete: register_int_counter!(
1020 0 : "pageserver_tenant_startup_complete",
1021 0 : "Number of tenants that have completed warm-up, or activated on-demand during initial startup: \
1022 0 : should eventually reach `pageserver_tenant_startup_scheduled_total`. Does not include broken \
1023 0 : tenants: such cases will lead to this metric never reaching the scheduled count."
1024 0 : ).expect("Failed to register metric"),
1025 0 : }
1026 0 : });
1027 :
1028 : /// Each `Timeline`'s [`EVICTIONS_WITH_LOW_RESIDENCE_DURATION`] metric.
1029 : #[derive(Debug)]
1030 : pub(crate) struct EvictionsWithLowResidenceDuration {
1031 : data_source: &'static str,
1032 : threshold: Duration,
1033 : counter: Option<IntCounter>,
1034 : }
1035 :
1036 : pub(crate) struct EvictionsWithLowResidenceDurationBuilder {
1037 : data_source: &'static str,
1038 : threshold: Duration,
1039 : }
1040 :
1041 : impl EvictionsWithLowResidenceDurationBuilder {
1042 906 : pub fn new(data_source: &'static str, threshold: Duration) -> Self {
1043 906 : Self {
1044 906 : data_source,
1045 906 : threshold,
1046 906 : }
1047 906 : }
1048 :
1049 906 : fn build(
1050 906 : &self,
1051 906 : tenant_id: &str,
1052 906 : shard_id: &str,
1053 906 : timeline_id: &str,
1054 906 : ) -> EvictionsWithLowResidenceDuration {
1055 906 : let counter = EVICTIONS_WITH_LOW_RESIDENCE_DURATION
1056 906 : .get_metric_with_label_values(&[
1057 906 : tenant_id,
1058 906 : shard_id,
1059 906 : timeline_id,
1060 906 : self.data_source,
1061 906 : &EvictionsWithLowResidenceDuration::threshold_label_value(self.threshold),
1062 906 : ])
1063 906 : .unwrap();
1064 906 : EvictionsWithLowResidenceDuration {
1065 906 : data_source: self.data_source,
1066 906 : threshold: self.threshold,
1067 906 : counter: Some(counter),
1068 906 : }
1069 906 : }
1070 : }
1071 :
1072 : impl EvictionsWithLowResidenceDuration {
1073 926 : fn threshold_label_value(threshold: Duration) -> String {
1074 926 : format!("{}", threshold.as_secs())
1075 926 : }
1076 :
1077 8 : pub fn observe(&self, observed_value: Duration) {
1078 8 : if observed_value < self.threshold {
1079 8 : self.counter
1080 8 : .as_ref()
1081 8 : .expect("nobody calls this function after `remove_from_vec`")
1082 8 : .inc();
1083 8 : }
1084 8 : }
1085 :
1086 0 : pub fn change_threshold(
1087 0 : &mut self,
1088 0 : tenant_id: &str,
1089 0 : shard_id: &str,
1090 0 : timeline_id: &str,
1091 0 : new_threshold: Duration,
1092 0 : ) {
1093 0 : if new_threshold == self.threshold {
1094 0 : return;
1095 0 : }
1096 0 : let mut with_new = EvictionsWithLowResidenceDurationBuilder::new(
1097 0 : self.data_source,
1098 0 : new_threshold,
1099 0 : )
1100 0 : .build(tenant_id, shard_id, timeline_id);
1101 0 : std::mem::swap(self, &mut with_new);
1102 0 : with_new.remove(tenant_id, shard_id, timeline_id);
1103 0 : }
1104 :
1105 : // This could be a `Drop` impl, but, we need the `tenant_id` and `timeline_id`.
1106 20 : fn remove(&mut self, tenant_id: &str, shard_id: &str, timeline_id: &str) {
1107 20 : let Some(_counter) = self.counter.take() else {
1108 0 : return;
1109 : };
1110 :
1111 20 : let threshold = Self::threshold_label_value(self.threshold);
1112 20 :
1113 20 : let removed = EVICTIONS_WITH_LOW_RESIDENCE_DURATION.remove_label_values(&[
1114 20 : tenant_id,
1115 20 : shard_id,
1116 20 : timeline_id,
1117 20 : self.data_source,
1118 20 : &threshold,
1119 20 : ]);
1120 20 :
1121 20 : match removed {
1122 0 : Err(e) => {
1123 0 : // this has been hit in staging as
1124 0 : // <https://neondatabase.sentry.io/issues/4142396994/>, but we don't know how.
1125 0 : // because we can be in the drop path already, don't risk:
1126 0 : // - "double-panic => illegal instruction" or
1127 0 : // - future "drop panick => abort"
1128 0 : //
1129 0 : // so just nag: (the error has the labels)
1130 0 : tracing::warn!(
1131 0 : "failed to remove EvictionsWithLowResidenceDuration, it was already removed? {e:#?}"
1132 : );
1133 : }
1134 : Ok(()) => {
1135 : // to help identify cases where we double-remove the same values, let's log all
1136 : // deletions?
1137 20 : tracing::info!(
1138 0 : "removed EvictionsWithLowResidenceDuration with {tenant_id}, {timeline_id}, {}, {threshold}",
1139 : self.data_source
1140 : );
1141 : }
1142 : }
1143 20 : }
1144 : }
1145 :
1146 : // Metrics collected on disk IO operations
1147 : //
1148 : // Roughly logarithmic scale.
1149 : const STORAGE_IO_TIME_BUCKETS: &[f64] = &[
1150 : 0.000030, // 30 usec
1151 : 0.001000, // 1000 usec
1152 : 0.030, // 30 ms
1153 : 1.000, // 1000 ms
1154 : 30.000, // 30000 ms
1155 : ];
1156 :
1157 : /// VirtualFile fs operation variants.
1158 : ///
1159 : /// Operations:
1160 : /// - open ([`std::fs::OpenOptions::open`])
1161 : /// - close (dropping [`crate::virtual_file::VirtualFile`])
1162 : /// - close-by-replace (close by replacement algorithm)
1163 : /// - read (`read_at`)
1164 : /// - write (`write_at`)
1165 : /// - seek (modify internal position or file length query)
1166 : /// - fsync ([`std::fs::File::sync_all`])
1167 : /// - metadata ([`std::fs::File::metadata`])
1168 : #[derive(
1169 0 : Debug, Clone, Copy, strum_macros::EnumCount, strum_macros::EnumIter, strum_macros::FromRepr,
1170 : )]
1171 : pub(crate) enum StorageIoOperation {
1172 : Open,
1173 : OpenAfterReplace,
1174 : Close,
1175 : CloseByReplace,
1176 : Read,
1177 : Write,
1178 : Seek,
1179 : Fsync,
1180 : Metadata,
1181 : }
1182 :
1183 : impl StorageIoOperation {
1184 4266 : pub fn as_str(&self) -> &'static str {
1185 4266 : match self {
1186 474 : StorageIoOperation::Open => "open",
1187 474 : StorageIoOperation::OpenAfterReplace => "open-after-replace",
1188 474 : StorageIoOperation::Close => "close",
1189 474 : StorageIoOperation::CloseByReplace => "close-by-replace",
1190 474 : StorageIoOperation::Read => "read",
1191 474 : StorageIoOperation::Write => "write",
1192 474 : StorageIoOperation::Seek => "seek",
1193 474 : StorageIoOperation::Fsync => "fsync",
1194 474 : StorageIoOperation::Metadata => "metadata",
1195 : }
1196 4266 : }
1197 : }
1198 :
1199 : /// Tracks time taken by fs operations near VirtualFile.
1200 : #[derive(Debug)]
1201 : pub(crate) struct StorageIoTime {
1202 : metrics: [Histogram; StorageIoOperation::COUNT],
1203 : }
1204 :
1205 : impl StorageIoTime {
1206 474 : fn new() -> Self {
1207 474 : let storage_io_histogram_vec = register_histogram_vec!(
1208 474 : "pageserver_io_operations_seconds",
1209 474 : "Time spent in IO operations",
1210 474 : &["operation"],
1211 474 : STORAGE_IO_TIME_BUCKETS.into()
1212 474 : )
1213 474 : .expect("failed to define a metric");
1214 4266 : let metrics = std::array::from_fn(|i| {
1215 4266 : let op = StorageIoOperation::from_repr(i).unwrap();
1216 4266 : storage_io_histogram_vec
1217 4266 : .get_metric_with_label_values(&[op.as_str()])
1218 4266 : .unwrap()
1219 4266 : });
1220 474 : Self { metrics }
1221 474 : }
1222 :
1223 4025961 : pub(crate) fn get(&self, op: StorageIoOperation) -> &Histogram {
1224 4025961 : &self.metrics[op as usize]
1225 4025961 : }
1226 : }
1227 :
1228 : pub(crate) static STORAGE_IO_TIME_METRIC: Lazy<StorageIoTime> = Lazy::new(StorageIoTime::new);
1229 :
1230 : #[derive(Clone, Copy)]
1231 : #[repr(usize)]
1232 : enum StorageIoSizeOperation {
1233 : Read,
1234 : Write,
1235 : }
1236 :
1237 : impl StorageIoSizeOperation {
1238 : const VARIANTS: &'static [&'static str] = &["read", "write"];
1239 :
1240 2912 : fn as_str(&self) -> &'static str {
1241 2912 : Self::VARIANTS[*self as usize]
1242 2912 : }
1243 : }
1244 :
1245 : // Needed for the https://neonprod.grafana.net/d/5uK9tHL4k/picking-tenant-for-relocation?orgId=1
1246 550 : static STORAGE_IO_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
1247 550 : register_uint_gauge_vec!(
1248 550 : "pageserver_io_operations_bytes_total",
1249 550 : "Total amount of bytes read/written in IO operations",
1250 550 : &["operation", "tenant_id", "shard_id", "timeline_id"]
1251 550 : )
1252 550 : .expect("failed to define a metric")
1253 550 : });
1254 :
1255 : #[derive(Clone, Debug)]
1256 : pub(crate) struct StorageIoSizeMetrics {
1257 : pub read: UIntGauge,
1258 : pub write: UIntGauge,
1259 : }
1260 :
1261 : impl StorageIoSizeMetrics {
1262 1456 : pub(crate) fn new(tenant_id: &str, shard_id: &str, timeline_id: &str) -> Self {
1263 1456 : let read = STORAGE_IO_SIZE
1264 1456 : .get_metric_with_label_values(&[
1265 1456 : StorageIoSizeOperation::Read.as_str(),
1266 1456 : tenant_id,
1267 1456 : shard_id,
1268 1456 : timeline_id,
1269 1456 : ])
1270 1456 : .unwrap();
1271 1456 : let write = STORAGE_IO_SIZE
1272 1456 : .get_metric_with_label_values(&[
1273 1456 : StorageIoSizeOperation::Write.as_str(),
1274 1456 : tenant_id,
1275 1456 : shard_id,
1276 1456 : timeline_id,
1277 1456 : ])
1278 1456 : .unwrap();
1279 1456 : Self { read, write }
1280 1456 : }
1281 : }
1282 :
1283 : #[cfg(not(test))]
1284 : pub(crate) mod virtual_file_descriptor_cache {
1285 : use super::*;
1286 :
1287 0 : pub(crate) static SIZE_MAX: Lazy<UIntGauge> = Lazy::new(|| {
1288 0 : register_uint_gauge!(
1289 0 : "pageserver_virtual_file_descriptor_cache_size_max",
1290 0 : "Maximum number of open file descriptors in the cache."
1291 0 : )
1292 0 : .unwrap()
1293 0 : });
1294 :
1295 : // SIZE_CURRENT: derive it like so:
1296 : // ```
1297 : // sum (pageserver_io_operations_seconds_count{operation=~"^(open|open-after-replace)$")
1298 : // -ignoring(operation)
1299 : // sum(pageserver_io_operations_seconds_count{operation=~"^(close|close-by-replace)$"}
1300 : // ```
1301 : }
1302 :
1303 : #[cfg(not(test))]
1304 : pub(crate) mod virtual_file_io_engine {
1305 : use super::*;
1306 :
1307 0 : pub(crate) static KIND: Lazy<UIntGaugeVec> = Lazy::new(|| {
1308 0 : register_uint_gauge_vec!(
1309 0 : "pageserver_virtual_file_io_engine_kind",
1310 0 : "The configured io engine for VirtualFile",
1311 0 : &["kind"],
1312 0 : )
1313 0 : .unwrap()
1314 0 : });
1315 : }
1316 :
1317 : pub(crate) struct SmgrOpTimer(Option<SmgrOpTimerInner>);
1318 : pub(crate) struct SmgrOpTimerInner {
1319 : global_execution_latency_histo: Histogram,
1320 : per_timeline_execution_latency_histo: Option<Histogram>,
1321 :
1322 : global_batch_wait_time: Histogram,
1323 : per_timeline_batch_wait_time: Histogram,
1324 :
1325 : global_flush_in_progress_micros: IntCounter,
1326 : per_timeline_flush_in_progress_micros: IntCounter,
1327 :
1328 : throttling: Arc<tenant_throttling::Pagestream>,
1329 :
1330 : timings: SmgrOpTimerState,
1331 : }
1332 :
1333 : /// The stages of request processing are represented by the enum variants.
1334 : /// Used as part of [`SmgrOpTimerInner::timings`].
1335 : ///
1336 : /// Request processing calls into the `SmgrOpTimer::observe_*` methods at the
1337 : /// transition points.
1338 : /// These methods bump relevant counters and then update [`SmgrOpTimerInner::timings`]
1339 : /// to the next state.
1340 : ///
1341 : /// Each request goes through every stage, in all configurations.
1342 : ///
1343 : #[derive(Debug)]
1344 : enum SmgrOpTimerState {
1345 : Received {
1346 : // In the future, we may want to track the full time the request spent
1347 : // inside pageserver process (time spent in kernel buffers can't be tracked).
1348 : // `received_at` would be used for that.
1349 : #[allow(dead_code)]
1350 : received_at: Instant,
1351 : },
1352 : Throttling {
1353 : throttle_started_at: Instant,
1354 : },
1355 : Batching {
1356 : throttle_done_at: Instant,
1357 : },
1358 : Executing {
1359 : execution_started_at: Instant,
1360 : },
1361 : Flushing,
1362 : // NB: when adding observation points, remember to update the Drop impl.
1363 : }
1364 :
1365 : // NB: when adding observation points, remember to update the Drop impl.
1366 : impl SmgrOpTimer {
1367 : /// See [`SmgrOpTimerState`] for more context.
1368 0 : pub(crate) fn observe_throttle_start(&mut self, at: Instant) {
1369 0 : let Some(inner) = self.0.as_mut() else {
1370 0 : return;
1371 : };
1372 0 : let SmgrOpTimerState::Received { received_at: _ } = &mut inner.timings else {
1373 0 : return;
1374 : };
1375 0 : inner.throttling.count_accounted_start.inc();
1376 0 : inner.timings = SmgrOpTimerState::Throttling {
1377 0 : throttle_started_at: at,
1378 0 : };
1379 0 : }
1380 :
1381 : /// See [`SmgrOpTimerState`] for more context.
1382 0 : pub(crate) fn observe_throttle_done(&mut self, throttle: ThrottleResult) {
1383 0 : let Some(inner) = self.0.as_mut() else {
1384 0 : return;
1385 : };
1386 : let SmgrOpTimerState::Throttling {
1387 0 : throttle_started_at,
1388 0 : } = &inner.timings
1389 : else {
1390 0 : return;
1391 : };
1392 0 : inner.throttling.count_accounted_finish.inc();
1393 0 : match throttle {
1394 0 : ThrottleResult::NotThrottled { end } => {
1395 0 : inner.timings = SmgrOpTimerState::Batching {
1396 0 : throttle_done_at: end,
1397 0 : };
1398 0 : }
1399 0 : ThrottleResult::Throttled { end } => {
1400 0 : // update metrics
1401 0 : inner.throttling.count_throttled.inc();
1402 0 : inner
1403 0 : .throttling
1404 0 : .wait_time
1405 0 : .inc_by((end - *throttle_started_at).as_micros().try_into().unwrap());
1406 0 : // state transition
1407 0 : inner.timings = SmgrOpTimerState::Batching {
1408 0 : throttle_done_at: end,
1409 0 : };
1410 0 : }
1411 : }
1412 0 : }
1413 :
1414 : /// See [`SmgrOpTimerState`] for more context.
1415 0 : pub(crate) fn observe_execution_start(&mut self, at: Instant) {
1416 0 : let Some(inner) = self.0.as_mut() else {
1417 0 : return;
1418 : };
1419 0 : let SmgrOpTimerState::Batching { throttle_done_at } = &inner.timings else {
1420 0 : return;
1421 : };
1422 : // update metrics
1423 0 : let batch = at - *throttle_done_at;
1424 0 : inner.global_batch_wait_time.observe(batch.as_secs_f64());
1425 0 : inner
1426 0 : .per_timeline_batch_wait_time
1427 0 : .observe(batch.as_secs_f64());
1428 0 : // state transition
1429 0 : inner.timings = SmgrOpTimerState::Executing {
1430 0 : execution_started_at: at,
1431 0 : }
1432 0 : }
1433 :
1434 : /// For all but the first caller, this is a no-op.
1435 : /// The first callers receives Some, subsequent ones None.
1436 : ///
1437 : /// See [`SmgrOpTimerState`] for more context.
1438 0 : pub(crate) fn observe_execution_end(&mut self, at: Instant) -> Option<SmgrOpFlushInProgress> {
1439 : // NB: unlike the other observe_* methods, this one take()s.
1440 : #[allow(clippy::question_mark)] // maintain similar code pattern.
1441 0 : let Some(mut inner) = self.0.take() else {
1442 0 : return None;
1443 : };
1444 : let SmgrOpTimerState::Executing {
1445 0 : execution_started_at,
1446 0 : } = &inner.timings
1447 : else {
1448 0 : return None;
1449 : };
1450 : // update metrics
1451 0 : let execution = at - *execution_started_at;
1452 0 : inner
1453 0 : .global_execution_latency_histo
1454 0 : .observe(execution.as_secs_f64());
1455 0 : if let Some(per_timeline_execution_latency_histo) =
1456 0 : &inner.per_timeline_execution_latency_histo
1457 0 : {
1458 0 : per_timeline_execution_latency_histo.observe(execution.as_secs_f64());
1459 0 : }
1460 :
1461 : // state transition
1462 0 : inner.timings = SmgrOpTimerState::Flushing;
1463 0 :
1464 0 : // return the flush in progress object which
1465 0 : // will do the remaining metrics updates
1466 0 : let SmgrOpTimerInner {
1467 0 : global_flush_in_progress_micros,
1468 0 : per_timeline_flush_in_progress_micros,
1469 0 : ..
1470 0 : } = inner;
1471 0 : Some(SmgrOpFlushInProgress {
1472 0 : global_micros: global_flush_in_progress_micros,
1473 0 : per_timeline_micros: per_timeline_flush_in_progress_micros,
1474 0 : })
1475 0 : }
1476 : }
1477 :
1478 : /// The last stage of request processing is serializing and flushing the request
1479 : /// into the TCP connection. We want to make slow flushes observable
1480 : /// _while they are occuring_, so this struct provides a wrapper method [`Self::measure`]
1481 : /// to periodically bump the metric.
1482 : ///
1483 : /// If in the future we decide that we're not interested in live updates, we can
1484 : /// add another `observe_*` method to [`SmgrOpTimer`], follow the existing pattern there,
1485 : /// and remove this struct from the code base.
1486 : pub(crate) struct SmgrOpFlushInProgress {
1487 : global_micros: IntCounter,
1488 : per_timeline_micros: IntCounter,
1489 : }
1490 :
1491 : impl Drop for SmgrOpTimer {
1492 0 : fn drop(&mut self) {
1493 0 : // In case of early drop, update any of the remaining metrics with
1494 0 : // observations so that (started,finished) counter pairs balance out
1495 0 : // and all counters on the latency path have the the same number of
1496 0 : // observations.
1497 0 : // It's technically lying and it would be better if each metric had
1498 0 : // a separate label or similar for cancelled requests.
1499 0 : // But we don't have that right now and counter pairs balancing
1500 0 : // out is useful when using the metrics in panels and whatnot.
1501 0 : let now = Instant::now();
1502 0 : self.observe_throttle_start(now);
1503 0 : self.observe_throttle_done(ThrottleResult::NotThrottled { end: now });
1504 0 : self.observe_execution_start(now);
1505 0 : let maybe_flush_timer = self.observe_execution_end(now);
1506 0 : drop(maybe_flush_timer);
1507 0 : }
1508 : }
1509 :
1510 : impl SmgrOpFlushInProgress {
1511 : /// The caller must guarantee that `socket_fd`` outlives this function.
1512 0 : pub(crate) async fn measure<Fut, O>(
1513 0 : self,
1514 0 : started_at: Instant,
1515 0 : mut fut: Fut,
1516 0 : socket_fd: RawFd,
1517 0 : ) -> O
1518 0 : where
1519 0 : Fut: std::future::Future<Output = O>,
1520 0 : {
1521 0 : let mut fut = std::pin::pin!(fut);
1522 0 :
1523 0 : let mut logged = false;
1524 0 : let mut last_counter_increment_at = started_at;
1525 0 : let mut observe_guard = scopeguard::guard(
1526 0 : |is_timeout| {
1527 0 : let now = Instant::now();
1528 0 :
1529 0 : // Increment counter
1530 0 : {
1531 0 : let elapsed_since_last_observe = now - last_counter_increment_at;
1532 0 : self.global_micros
1533 0 : .inc_by(u64::try_from(elapsed_since_last_observe.as_micros()).unwrap());
1534 0 : self.per_timeline_micros
1535 0 : .inc_by(u64::try_from(elapsed_since_last_observe.as_micros()).unwrap());
1536 0 : last_counter_increment_at = now;
1537 0 : }
1538 0 :
1539 0 : // Log something on every timeout, and on completion but only if we hit a timeout.
1540 0 : if is_timeout || logged {
1541 0 : logged = true;
1542 0 : let elapsed_total = now - started_at;
1543 0 : let msg = if is_timeout {
1544 0 : "slow flush ongoing"
1545 : } else {
1546 0 : "slow flush completed or cancelled"
1547 : };
1548 :
1549 0 : let (inq, outq) = {
1550 0 : // SAFETY: caller guarantees that `socket_fd` outlives this function.
1551 0 : #[cfg(target_os = "linux")]
1552 0 : unsafe {
1553 0 : (
1554 0 : utils::linux_socket_ioctl::inq(socket_fd).unwrap_or(-2),
1555 0 : utils::linux_socket_ioctl::outq(socket_fd).unwrap_or(-2),
1556 0 : )
1557 0 : }
1558 0 : #[cfg(not(target_os = "linux"))]
1559 0 : {
1560 0 : _ = socket_fd; // appease unused lint on macOS
1561 0 : (-1, -1)
1562 0 : }
1563 0 : };
1564 0 :
1565 0 : let elapsed_total_secs = format!("{:.6}", elapsed_total.as_secs_f64());
1566 0 : tracing::info!(elapsed_total_secs, inq, outq, msg);
1567 0 : }
1568 0 : },
1569 0 : |mut observe| {
1570 0 : observe(false);
1571 0 : },
1572 0 : );
1573 :
1574 : loop {
1575 0 : match tokio::time::timeout(Duration::from_secs(10), &mut fut).await {
1576 0 : Ok(v) => return v,
1577 0 : Err(_timeout) => {
1578 0 : (*observe_guard)(true);
1579 0 : }
1580 : }
1581 : }
1582 0 : }
1583 : }
1584 :
1585 : #[derive(
1586 : Debug,
1587 : Clone,
1588 : Copy,
1589 : IntoStaticStr,
1590 : strum_macros::EnumCount,
1591 0 : strum_macros::EnumIter,
1592 : strum_macros::FromRepr,
1593 : enum_map::Enum,
1594 : )]
1595 : #[strum(serialize_all = "snake_case")]
1596 : pub enum SmgrQueryType {
1597 : GetRelExists,
1598 : GetRelSize,
1599 : GetPageAtLsn,
1600 : GetDbSize,
1601 : GetSlruSegment,
1602 : #[cfg(feature = "testing")]
1603 : Test,
1604 : }
1605 :
1606 : pub(crate) struct SmgrQueryTimePerTimeline {
1607 : global_started: [IntCounter; SmgrQueryType::COUNT],
1608 : global_latency: [Histogram; SmgrQueryType::COUNT],
1609 : per_timeline_getpage_started: IntCounter,
1610 : per_timeline_getpage_latency: Histogram,
1611 : global_batch_size: Histogram,
1612 : per_timeline_batch_size: Histogram,
1613 : global_flush_in_progress_micros: IntCounter,
1614 : per_timeline_flush_in_progress_micros: IntCounter,
1615 : global_batch_wait_time: Histogram,
1616 : per_timeline_batch_wait_time: Histogram,
1617 : throttling: Arc<tenant_throttling::Pagestream>,
1618 : }
1619 :
1620 414 : static SMGR_QUERY_STARTED_GLOBAL: Lazy<IntCounterVec> = Lazy::new(|| {
1621 414 : register_int_counter_vec!(
1622 414 : // it's a counter, but, name is prepared to extend it to a histogram of queue depth
1623 414 : "pageserver_smgr_query_started_global_count",
1624 414 : "Number of smgr queries started, aggregated by query type.",
1625 414 : &["smgr_query_type"],
1626 414 : )
1627 414 : .expect("failed to define a metric")
1628 414 : });
1629 :
1630 414 : static SMGR_QUERY_STARTED_PER_TENANT_TIMELINE: Lazy<IntCounterVec> = Lazy::new(|| {
1631 414 : register_int_counter_vec!(
1632 414 : // it's a counter, but, name is prepared to extend it to a histogram of queue depth
1633 414 : "pageserver_smgr_query_started_count",
1634 414 : "Number of smgr queries started, aggregated by query type and tenant/timeline.",
1635 414 : &["smgr_query_type", "tenant_id", "shard_id", "timeline_id"],
1636 414 : )
1637 414 : .expect("failed to define a metric")
1638 414 : });
1639 :
1640 : // Alias so all histograms recording per-timeline smgr timings use the same buckets.
1641 : static SMGR_QUERY_TIME_PER_TENANT_TIMELINE_BUCKETS: &[f64] = CRITICAL_OP_BUCKETS;
1642 :
1643 414 : static SMGR_QUERY_TIME_PER_TENANT_TIMELINE: Lazy<HistogramVec> = Lazy::new(|| {
1644 414 : register_histogram_vec!(
1645 414 : "pageserver_smgr_query_seconds",
1646 414 : "Time spent _executing_ smgr query handling, excluding batch and throttle delays.",
1647 414 : &["smgr_query_type", "tenant_id", "shard_id", "timeline_id"],
1648 414 : SMGR_QUERY_TIME_PER_TENANT_TIMELINE_BUCKETS.into(),
1649 414 : )
1650 414 : .expect("failed to define a metric")
1651 414 : });
1652 :
1653 414 : static SMGR_QUERY_TIME_GLOBAL_BUCKETS: Lazy<Vec<f64>> = Lazy::new(|| {
1654 414 : [
1655 414 : 1,
1656 414 : 10,
1657 414 : 20,
1658 414 : 40,
1659 414 : 60,
1660 414 : 80,
1661 414 : 100,
1662 414 : 200,
1663 414 : 300,
1664 414 : 400,
1665 414 : 500,
1666 414 : 600,
1667 414 : 700,
1668 414 : 800,
1669 414 : 900,
1670 414 : 1_000, // 1ms
1671 414 : 2_000,
1672 414 : 4_000,
1673 414 : 6_000,
1674 414 : 8_000,
1675 414 : 10_000, // 10ms
1676 414 : 20_000,
1677 414 : 40_000,
1678 414 : 60_000,
1679 414 : 80_000,
1680 414 : 100_000,
1681 414 : 200_000,
1682 414 : 400_000,
1683 414 : 600_000,
1684 414 : 800_000,
1685 414 : 1_000_000, // 1s
1686 414 : 2_000_000,
1687 414 : 4_000_000,
1688 414 : 6_000_000,
1689 414 : 8_000_000,
1690 414 : 10_000_000, // 10s
1691 414 : 20_000_000,
1692 414 : 50_000_000,
1693 414 : 100_000_000,
1694 414 : 200_000_000,
1695 414 : 1_000_000_000, // 1000s
1696 414 : ]
1697 414 : .into_iter()
1698 414 : .map(Duration::from_micros)
1699 16974 : .map(|d| d.as_secs_f64())
1700 414 : .collect()
1701 414 : });
1702 :
1703 414 : static SMGR_QUERY_TIME_GLOBAL: Lazy<HistogramVec> = Lazy::new(|| {
1704 414 : register_histogram_vec!(
1705 414 : "pageserver_smgr_query_seconds_global",
1706 414 : "Like pageserver_smgr_query_seconds, but aggregated to instance level.",
1707 414 : &["smgr_query_type"],
1708 414 : SMGR_QUERY_TIME_GLOBAL_BUCKETS.clone(),
1709 414 : )
1710 414 : .expect("failed to define a metric")
1711 414 : });
1712 :
1713 414 : static PAGE_SERVICE_BATCH_SIZE_BUCKETS_GLOBAL: Lazy<Vec<f64>> = Lazy::new(|| {
1714 414 : (1..=u32::try_from(Timeline::MAX_GET_VECTORED_KEYS).unwrap())
1715 13248 : .map(|v| v.into())
1716 414 : .collect()
1717 414 : });
1718 :
1719 414 : static PAGE_SERVICE_BATCH_SIZE_GLOBAL: Lazy<Histogram> = Lazy::new(|| {
1720 414 : register_histogram!(
1721 414 : "pageserver_page_service_batch_size_global",
1722 414 : "Batch size of pageserver page service requests",
1723 414 : PAGE_SERVICE_BATCH_SIZE_BUCKETS_GLOBAL.clone(),
1724 414 : )
1725 414 : .expect("failed to define a metric")
1726 414 : });
1727 :
1728 414 : static PAGE_SERVICE_BATCH_SIZE_BUCKETS_PER_TIMELINE: Lazy<Vec<f64>> = Lazy::new(|| {
1729 414 : let mut buckets = Vec::new();
1730 2898 : for i in 0.. {
1731 2898 : let bucket = 1 << i;
1732 2898 : if bucket > u32::try_from(Timeline::MAX_GET_VECTORED_KEYS).unwrap() {
1733 414 : break;
1734 2484 : }
1735 2484 : buckets.push(bucket.into());
1736 : }
1737 414 : buckets
1738 414 : });
1739 :
1740 414 : static PAGE_SERVICE_BATCH_SIZE_PER_TENANT_TIMELINE: Lazy<HistogramVec> = Lazy::new(|| {
1741 414 : register_histogram_vec!(
1742 414 : "pageserver_page_service_batch_size",
1743 414 : "Batch size of pageserver page service requests",
1744 414 : &["tenant_id", "shard_id", "timeline_id"],
1745 414 : PAGE_SERVICE_BATCH_SIZE_BUCKETS_PER_TIMELINE.clone()
1746 414 : )
1747 414 : .expect("failed to define a metric")
1748 414 : });
1749 :
1750 0 : pub(crate) static PAGE_SERVICE_CONFIG_MAX_BATCH_SIZE: Lazy<IntGaugeVec> = Lazy::new(|| {
1751 0 : register_int_gauge_vec!(
1752 0 : "pageserver_page_service_config_max_batch_size",
1753 0 : "Configured maximum batch size for the server-side batching functionality of page_service. \
1754 0 : Labels expose more of the configuration parameters.",
1755 0 : &["mode", "execution"]
1756 0 : )
1757 0 : .expect("failed to define a metric")
1758 0 : });
1759 :
1760 0 : fn set_page_service_config_max_batch_size(conf: &PageServicePipeliningConfig) {
1761 0 : PAGE_SERVICE_CONFIG_MAX_BATCH_SIZE.reset();
1762 0 : let (label_values, value) = match conf {
1763 0 : PageServicePipeliningConfig::Serial => (["serial", "-"], 1),
1764 : PageServicePipeliningConfig::Pipelined(PageServicePipeliningConfigPipelined {
1765 0 : max_batch_size,
1766 0 : execution,
1767 0 : }) => {
1768 0 : let mode = "pipelined";
1769 0 : let execution = match execution {
1770 : PageServiceProtocolPipelinedExecutionStrategy::ConcurrentFutures => {
1771 0 : "concurrent-futures"
1772 : }
1773 0 : PageServiceProtocolPipelinedExecutionStrategy::Tasks => "tasks",
1774 : };
1775 0 : ([mode, execution], max_batch_size.get())
1776 : }
1777 : };
1778 0 : PAGE_SERVICE_CONFIG_MAX_BATCH_SIZE
1779 0 : .with_label_values(&label_values)
1780 0 : .set(value.try_into().unwrap());
1781 0 : }
1782 :
1783 414 : static PAGE_SERVICE_SMGR_FLUSH_INPROGRESS_MICROS: Lazy<IntCounterVec> = Lazy::new(|| {
1784 414 : register_int_counter_vec!(
1785 414 : "pageserver_page_service_pagestream_flush_in_progress_micros",
1786 414 : "Counter that sums up the microseconds that a pagestream response was being flushed into the TCP connection. \
1787 414 : If the flush is particularly slow, this counter will be updated periodically to make slow flushes \
1788 414 : easily discoverable in monitoring. \
1789 414 : Hence, this is NOT a completion latency historgram.",
1790 414 : &["tenant_id", "shard_id", "timeline_id"],
1791 414 : )
1792 414 : .expect("failed to define a metric")
1793 414 : });
1794 :
1795 414 : static PAGE_SERVICE_SMGR_FLUSH_INPROGRESS_MICROS_GLOBAL: Lazy<IntCounter> = Lazy::new(|| {
1796 414 : register_int_counter!(
1797 414 : "pageserver_page_service_pagestream_flush_in_progress_micros_global",
1798 414 : "Like pageserver_page_service_pagestream_flush_in_progress_seconds, but instance-wide.",
1799 414 : )
1800 414 : .expect("failed to define a metric")
1801 414 : });
1802 :
1803 414 : static PAGE_SERVICE_SMGR_BATCH_WAIT_TIME: Lazy<HistogramVec> = Lazy::new(|| {
1804 414 : register_histogram_vec!(
1805 414 : "pageserver_page_service_pagestream_batch_wait_time_seconds",
1806 414 : "Time a request spent waiting in its batch until the batch moved to throttle&execution.",
1807 414 : &["tenant_id", "shard_id", "timeline_id"],
1808 414 : SMGR_QUERY_TIME_PER_TENANT_TIMELINE_BUCKETS.into(),
1809 414 : )
1810 414 : .expect("failed to define a metric")
1811 414 : });
1812 :
1813 414 : static PAGE_SERVICE_SMGR_BATCH_WAIT_TIME_GLOBAL: Lazy<Histogram> = Lazy::new(|| {
1814 414 : register_histogram!(
1815 414 : "pageserver_page_service_pagestream_batch_wait_time_seconds_global",
1816 414 : "Like pageserver_page_service_pagestream_batch_wait_time_seconds, but aggregated to instance level.",
1817 414 : SMGR_QUERY_TIME_GLOBAL_BUCKETS.to_vec(),
1818 414 : )
1819 414 : .expect("failed to define a metric")
1820 414 : });
1821 :
1822 : impl SmgrQueryTimePerTimeline {
1823 906 : pub(crate) fn new(
1824 906 : tenant_shard_id: &TenantShardId,
1825 906 : timeline_id: &TimelineId,
1826 906 : pagestream_throttle_metrics: Arc<tenant_throttling::Pagestream>,
1827 906 : ) -> Self {
1828 906 : let tenant_id = tenant_shard_id.tenant_id.to_string();
1829 906 : let shard_slug = format!("{}", tenant_shard_id.shard_slug());
1830 906 : let timeline_id = timeline_id.to_string();
1831 5436 : let global_started = std::array::from_fn(|i| {
1832 5436 : let op = SmgrQueryType::from_repr(i).unwrap();
1833 5436 : SMGR_QUERY_STARTED_GLOBAL
1834 5436 : .get_metric_with_label_values(&[op.into()])
1835 5436 : .unwrap()
1836 5436 : });
1837 5436 : let global_latency = std::array::from_fn(|i| {
1838 5436 : let op = SmgrQueryType::from_repr(i).unwrap();
1839 5436 : SMGR_QUERY_TIME_GLOBAL
1840 5436 : .get_metric_with_label_values(&[op.into()])
1841 5436 : .unwrap()
1842 5436 : });
1843 906 :
1844 906 : let per_timeline_getpage_started = SMGR_QUERY_STARTED_PER_TENANT_TIMELINE
1845 906 : .get_metric_with_label_values(&[
1846 906 : SmgrQueryType::GetPageAtLsn.into(),
1847 906 : &tenant_id,
1848 906 : &shard_slug,
1849 906 : &timeline_id,
1850 906 : ])
1851 906 : .unwrap();
1852 906 : let per_timeline_getpage_latency = SMGR_QUERY_TIME_PER_TENANT_TIMELINE
1853 906 : .get_metric_with_label_values(&[
1854 906 : SmgrQueryType::GetPageAtLsn.into(),
1855 906 : &tenant_id,
1856 906 : &shard_slug,
1857 906 : &timeline_id,
1858 906 : ])
1859 906 : .unwrap();
1860 906 :
1861 906 : let global_batch_size = PAGE_SERVICE_BATCH_SIZE_GLOBAL.clone();
1862 906 : let per_timeline_batch_size = PAGE_SERVICE_BATCH_SIZE_PER_TENANT_TIMELINE
1863 906 : .get_metric_with_label_values(&[&tenant_id, &shard_slug, &timeline_id])
1864 906 : .unwrap();
1865 906 :
1866 906 : let global_batch_wait_time = PAGE_SERVICE_SMGR_BATCH_WAIT_TIME_GLOBAL.clone();
1867 906 : let per_timeline_batch_wait_time = PAGE_SERVICE_SMGR_BATCH_WAIT_TIME
1868 906 : .get_metric_with_label_values(&[&tenant_id, &shard_slug, &timeline_id])
1869 906 : .unwrap();
1870 906 :
1871 906 : let global_flush_in_progress_micros =
1872 906 : PAGE_SERVICE_SMGR_FLUSH_INPROGRESS_MICROS_GLOBAL.clone();
1873 906 : let per_timeline_flush_in_progress_micros = PAGE_SERVICE_SMGR_FLUSH_INPROGRESS_MICROS
1874 906 : .get_metric_with_label_values(&[&tenant_id, &shard_slug, &timeline_id])
1875 906 : .unwrap();
1876 906 :
1877 906 : Self {
1878 906 : global_started,
1879 906 : global_latency,
1880 906 : per_timeline_getpage_latency,
1881 906 : per_timeline_getpage_started,
1882 906 : global_batch_size,
1883 906 : per_timeline_batch_size,
1884 906 : global_flush_in_progress_micros,
1885 906 : per_timeline_flush_in_progress_micros,
1886 906 : global_batch_wait_time,
1887 906 : per_timeline_batch_wait_time,
1888 906 : throttling: pagestream_throttle_metrics,
1889 906 : }
1890 906 : }
1891 0 : pub(crate) fn start_smgr_op(&self, op: SmgrQueryType, received_at: Instant) -> SmgrOpTimer {
1892 0 : self.global_started[op as usize].inc();
1893 :
1894 0 : let per_timeline_latency_histo = if matches!(op, SmgrQueryType::GetPageAtLsn) {
1895 0 : self.per_timeline_getpage_started.inc();
1896 0 : Some(self.per_timeline_getpage_latency.clone())
1897 : } else {
1898 0 : None
1899 : };
1900 :
1901 0 : SmgrOpTimer(Some(SmgrOpTimerInner {
1902 0 : global_execution_latency_histo: self.global_latency[op as usize].clone(),
1903 0 : per_timeline_execution_latency_histo: per_timeline_latency_histo,
1904 0 : global_flush_in_progress_micros: self.global_flush_in_progress_micros.clone(),
1905 0 : per_timeline_flush_in_progress_micros: self
1906 0 : .per_timeline_flush_in_progress_micros
1907 0 : .clone(),
1908 0 : global_batch_wait_time: self.global_batch_wait_time.clone(),
1909 0 : per_timeline_batch_wait_time: self.per_timeline_batch_wait_time.clone(),
1910 0 : throttling: self.throttling.clone(),
1911 0 : timings: SmgrOpTimerState::Received { received_at },
1912 0 : }))
1913 0 : }
1914 :
1915 : /// TODO: do something about this? seems odd, we have a similar call on SmgrOpTimer
1916 0 : pub(crate) fn observe_getpage_batch_start(&self, batch_size: usize) {
1917 0 : self.global_batch_size.observe(batch_size as f64);
1918 0 : self.per_timeline_batch_size.observe(batch_size as f64);
1919 0 : }
1920 : }
1921 :
1922 : // keep in sync with control plane Go code so that we can validate
1923 : // compute's basebackup_ms metric with our perspective in the context of SLI/SLO.
1924 0 : static COMPUTE_STARTUP_BUCKETS: Lazy<[f64; 28]> = Lazy::new(|| {
1925 0 : // Go code uses milliseconds. Variable is called `computeStartupBuckets`
1926 0 : [
1927 0 : 5, 10, 20, 30, 50, 70, 100, 120, 150, 200, 250, 300, 350, 400, 450, 500, 600, 800, 1000,
1928 0 : 1500, 2000, 2500, 3000, 5000, 10000, 20000, 40000, 60000,
1929 0 : ]
1930 0 : .map(|ms| (ms as f64) / 1000.0)
1931 0 : });
1932 :
1933 : pub(crate) struct BasebackupQueryTime {
1934 : ok: Histogram,
1935 : error: Histogram,
1936 : client_error: Histogram,
1937 : }
1938 :
1939 0 : pub(crate) static BASEBACKUP_QUERY_TIME: Lazy<BasebackupQueryTime> = Lazy::new(|| {
1940 0 : let vec = register_histogram_vec!(
1941 0 : "pageserver_basebackup_query_seconds",
1942 0 : "Histogram of basebackup queries durations, by result type",
1943 0 : &["result"],
1944 0 : COMPUTE_STARTUP_BUCKETS.to_vec(),
1945 0 : )
1946 0 : .expect("failed to define a metric");
1947 0 : BasebackupQueryTime {
1948 0 : ok: vec.get_metric_with_label_values(&["ok"]).unwrap(),
1949 0 : error: vec.get_metric_with_label_values(&["error"]).unwrap(),
1950 0 : client_error: vec.get_metric_with_label_values(&["client_error"]).unwrap(),
1951 0 : }
1952 0 : });
1953 :
1954 : pub(crate) struct BasebackupQueryTimeOngoingRecording<'a> {
1955 : parent: &'a BasebackupQueryTime,
1956 : start: std::time::Instant,
1957 : }
1958 :
1959 : impl BasebackupQueryTime {
1960 0 : pub(crate) fn start_recording(&self) -> BasebackupQueryTimeOngoingRecording<'_> {
1961 0 : let start = Instant::now();
1962 0 : BasebackupQueryTimeOngoingRecording {
1963 0 : parent: self,
1964 0 : start,
1965 0 : }
1966 0 : }
1967 : }
1968 :
1969 : impl BasebackupQueryTimeOngoingRecording<'_> {
1970 0 : pub(crate) fn observe<T>(self, res: &Result<T, QueryError>) {
1971 0 : let elapsed = self.start.elapsed().as_secs_f64();
1972 : // If you want to change categorize of a specific error, also change it in `log_query_error`.
1973 0 : let metric = match res {
1974 0 : Ok(_) => &self.parent.ok,
1975 0 : Err(QueryError::Disconnected(ConnectionError::Io(io_error)))
1976 0 : if is_expected_io_error(io_error) =>
1977 0 : {
1978 0 : &self.parent.client_error
1979 : }
1980 0 : Err(_) => &self.parent.error,
1981 : };
1982 0 : metric.observe(elapsed);
1983 0 : }
1984 : }
1985 :
1986 0 : pub(crate) static LIVE_CONNECTIONS: Lazy<IntCounterPairVec> = Lazy::new(|| {
1987 0 : register_int_counter_pair_vec!(
1988 0 : "pageserver_live_connections_started",
1989 0 : "Number of network connections that we started handling",
1990 0 : "pageserver_live_connections_finished",
1991 0 : "Number of network connections that we finished handling",
1992 0 : &["pageserver_connection_kind"]
1993 0 : )
1994 0 : .expect("failed to define a metric")
1995 0 : });
1996 :
1997 : #[derive(Clone, Copy, enum_map::Enum, IntoStaticStr)]
1998 : pub(crate) enum ComputeCommandKind {
1999 : PageStreamV3,
2000 : PageStreamV2,
2001 : Basebackup,
2002 : Fullbackup,
2003 : LeaseLsn,
2004 : }
2005 :
2006 : pub(crate) struct ComputeCommandCounters {
2007 : map: EnumMap<ComputeCommandKind, IntCounter>,
2008 : }
2009 :
2010 0 : pub(crate) static COMPUTE_COMMANDS_COUNTERS: Lazy<ComputeCommandCounters> = Lazy::new(|| {
2011 0 : let inner = register_int_counter_vec!(
2012 0 : "pageserver_compute_commands",
2013 0 : "Number of compute -> pageserver commands processed",
2014 0 : &["command"]
2015 0 : )
2016 0 : .expect("failed to define a metric");
2017 0 :
2018 0 : ComputeCommandCounters {
2019 0 : map: EnumMap::from_array(std::array::from_fn(|i| {
2020 0 : let command = ComputeCommandKind::from_usize(i);
2021 0 : let command_str: &'static str = command.into();
2022 0 : inner.with_label_values(&[command_str])
2023 0 : })),
2024 0 : }
2025 0 : });
2026 :
2027 : impl ComputeCommandCounters {
2028 0 : pub(crate) fn for_command(&self, command: ComputeCommandKind) -> &IntCounter {
2029 0 : &self.map[command]
2030 0 : }
2031 : }
2032 :
2033 : // remote storage metrics
2034 :
2035 406 : static REMOTE_TIMELINE_CLIENT_CALLS: Lazy<IntCounterPairVec> = Lazy::new(|| {
2036 406 : register_int_counter_pair_vec!(
2037 406 : "pageserver_remote_timeline_client_calls_started",
2038 406 : "Number of started calls to remote timeline client.",
2039 406 : "pageserver_remote_timeline_client_calls_finished",
2040 406 : "Number of finshed calls to remote timeline client.",
2041 406 : &[
2042 406 : "tenant_id",
2043 406 : "shard_id",
2044 406 : "timeline_id",
2045 406 : "file_kind",
2046 406 : "op_kind"
2047 406 : ],
2048 406 : )
2049 406 : .unwrap()
2050 406 : });
2051 :
2052 : static REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER: Lazy<IntCounterVec> =
2053 402 : Lazy::new(|| {
2054 402 : register_int_counter_vec!(
2055 402 : "pageserver_remote_timeline_client_bytes_started",
2056 402 : "Incremented by the number of bytes associated with a remote timeline client operation. \
2057 402 : The increment happens when the operation is scheduled.",
2058 402 : &["tenant_id", "shard_id", "timeline_id", "file_kind", "op_kind"],
2059 402 : )
2060 402 : .expect("failed to define a metric")
2061 402 : });
2062 :
2063 402 : static REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER: Lazy<IntCounterVec> = Lazy::new(|| {
2064 402 : register_int_counter_vec!(
2065 402 : "pageserver_remote_timeline_client_bytes_finished",
2066 402 : "Incremented by the number of bytes associated with a remote timeline client operation. \
2067 402 : The increment happens when the operation finishes (regardless of success/failure/shutdown).",
2068 402 : &["tenant_id", "shard_id", "timeline_id", "file_kind", "op_kind"],
2069 402 : )
2070 402 : .expect("failed to define a metric")
2071 402 : });
2072 :
2073 : pub(crate) struct TenantManagerMetrics {
2074 : tenant_slots_attached: UIntGauge,
2075 : tenant_slots_secondary: UIntGauge,
2076 : tenant_slots_inprogress: UIntGauge,
2077 : pub(crate) tenant_slot_writes: IntCounter,
2078 : pub(crate) unexpected_errors: IntCounter,
2079 : }
2080 :
2081 : impl TenantManagerMetrics {
2082 : /// Helpers for tracking slots. Note that these do not track the lifetime of TenantSlot objects
2083 : /// exactly: they track the lifetime of the slots _in the tenant map_.
2084 4 : pub(crate) fn slot_inserted(&self, slot: &TenantSlot) {
2085 4 : match slot {
2086 0 : TenantSlot::Attached(_) => {
2087 0 : self.tenant_slots_attached.inc();
2088 0 : }
2089 0 : TenantSlot::Secondary(_) => {
2090 0 : self.tenant_slots_secondary.inc();
2091 0 : }
2092 4 : TenantSlot::InProgress(_) => {
2093 4 : self.tenant_slots_inprogress.inc();
2094 4 : }
2095 : }
2096 4 : }
2097 :
2098 4 : pub(crate) fn slot_removed(&self, slot: &TenantSlot) {
2099 4 : match slot {
2100 4 : TenantSlot::Attached(_) => {
2101 4 : self.tenant_slots_attached.dec();
2102 4 : }
2103 0 : TenantSlot::Secondary(_) => {
2104 0 : self.tenant_slots_secondary.dec();
2105 0 : }
2106 0 : TenantSlot::InProgress(_) => {
2107 0 : self.tenant_slots_inprogress.dec();
2108 0 : }
2109 : }
2110 4 : }
2111 :
2112 : #[cfg(all(debug_assertions, not(test)))]
2113 0 : pub(crate) fn slots_total(&self) -> u64 {
2114 0 : self.tenant_slots_attached.get()
2115 0 : + self.tenant_slots_secondary.get()
2116 0 : + self.tenant_slots_inprogress.get()
2117 0 : }
2118 : }
2119 :
2120 4 : pub(crate) static TENANT_MANAGER: Lazy<TenantManagerMetrics> = Lazy::new(|| {
2121 4 : let tenant_slots = register_uint_gauge_vec!(
2122 4 : "pageserver_tenant_manager_slots",
2123 4 : "How many slots currently exist, including all attached, secondary and in-progress operations",
2124 4 : &["mode"]
2125 4 : )
2126 4 : .expect("failed to define a metric");
2127 4 : TenantManagerMetrics {
2128 4 : tenant_slots_attached: tenant_slots
2129 4 : .get_metric_with_label_values(&["attached"])
2130 4 : .unwrap(),
2131 4 : tenant_slots_secondary: tenant_slots
2132 4 : .get_metric_with_label_values(&["secondary"])
2133 4 : .unwrap(),
2134 4 : tenant_slots_inprogress: tenant_slots
2135 4 : .get_metric_with_label_values(&["inprogress"])
2136 4 : .unwrap(),
2137 4 : tenant_slot_writes: register_int_counter!(
2138 4 : "pageserver_tenant_manager_slot_writes",
2139 4 : "Writes to a tenant slot, including all of create/attach/detach/delete"
2140 4 : )
2141 4 : .expect("failed to define a metric"),
2142 4 : unexpected_errors: register_int_counter!(
2143 4 : "pageserver_tenant_manager_unexpected_errors_total",
2144 4 : "Number of unexpected conditions encountered: nonzero value indicates a non-fatal bug."
2145 4 : )
2146 4 : .expect("failed to define a metric"),
2147 4 : }
2148 4 : });
2149 :
2150 : pub(crate) struct DeletionQueueMetrics {
2151 : pub(crate) keys_submitted: IntCounter,
2152 : pub(crate) keys_dropped: IntCounter,
2153 : pub(crate) keys_executed: IntCounter,
2154 : pub(crate) keys_validated: IntCounter,
2155 : pub(crate) dropped_lsn_updates: IntCounter,
2156 : pub(crate) unexpected_errors: IntCounter,
2157 : pub(crate) remote_errors: IntCounterVec,
2158 : }
2159 63 : pub(crate) static DELETION_QUEUE: Lazy<DeletionQueueMetrics> = Lazy::new(|| {
2160 63 : DeletionQueueMetrics{
2161 63 :
2162 63 : keys_submitted: register_int_counter!(
2163 63 : "pageserver_deletion_queue_submitted_total",
2164 63 : "Number of objects submitted for deletion"
2165 63 : )
2166 63 : .expect("failed to define a metric"),
2167 63 :
2168 63 : keys_dropped: register_int_counter!(
2169 63 : "pageserver_deletion_queue_dropped_total",
2170 63 : "Number of object deletions dropped due to stale generation."
2171 63 : )
2172 63 : .expect("failed to define a metric"),
2173 63 :
2174 63 : keys_executed: register_int_counter!(
2175 63 : "pageserver_deletion_queue_executed_total",
2176 63 : "Number of objects deleted. Only includes objects that we actually deleted, sum with pageserver_deletion_queue_dropped_total for the total number of keys processed to completion"
2177 63 : )
2178 63 : .expect("failed to define a metric"),
2179 63 :
2180 63 : keys_validated: register_int_counter!(
2181 63 : "pageserver_deletion_queue_validated_total",
2182 63 : "Number of keys validated for deletion. Sum with pageserver_deletion_queue_dropped_total for the total number of keys that have passed through the validation stage."
2183 63 : )
2184 63 : .expect("failed to define a metric"),
2185 63 :
2186 63 : dropped_lsn_updates: register_int_counter!(
2187 63 : "pageserver_deletion_queue_dropped_lsn_updates_total",
2188 63 : "Updates to remote_consistent_lsn dropped due to stale generation number."
2189 63 : )
2190 63 : .expect("failed to define a metric"),
2191 63 : unexpected_errors: register_int_counter!(
2192 63 : "pageserver_deletion_queue_unexpected_errors_total",
2193 63 : "Number of unexpected condiions that may stall the queue: any value above zero is unexpected."
2194 63 : )
2195 63 : .expect("failed to define a metric"),
2196 63 : remote_errors: register_int_counter_vec!(
2197 63 : "pageserver_deletion_queue_remote_errors_total",
2198 63 : "Retryable remote I/O errors while executing deletions, for example 503 responses to DeleteObjects",
2199 63 : &["op_kind"],
2200 63 : )
2201 63 : .expect("failed to define a metric")
2202 63 : }
2203 63 : });
2204 :
2205 : pub(crate) struct SecondaryModeMetrics {
2206 : pub(crate) upload_heatmap: IntCounter,
2207 : pub(crate) upload_heatmap_errors: IntCounter,
2208 : pub(crate) upload_heatmap_duration: Histogram,
2209 : pub(crate) download_heatmap: IntCounter,
2210 : pub(crate) download_layer: IntCounter,
2211 : }
2212 0 : pub(crate) static SECONDARY_MODE: Lazy<SecondaryModeMetrics> = Lazy::new(|| {
2213 0 : SecondaryModeMetrics {
2214 0 : upload_heatmap: register_int_counter!(
2215 0 : "pageserver_secondary_upload_heatmap",
2216 0 : "Number of heatmaps written to remote storage by attached tenants"
2217 0 : )
2218 0 : .expect("failed to define a metric"),
2219 0 : upload_heatmap_errors: register_int_counter!(
2220 0 : "pageserver_secondary_upload_heatmap_errors",
2221 0 : "Failures writing heatmap to remote storage"
2222 0 : )
2223 0 : .expect("failed to define a metric"),
2224 0 : upload_heatmap_duration: register_histogram!(
2225 0 : "pageserver_secondary_upload_heatmap_duration",
2226 0 : "Time to build and upload a heatmap, including any waiting inside the remote storage client"
2227 0 : )
2228 0 : .expect("failed to define a metric"),
2229 0 : download_heatmap: register_int_counter!(
2230 0 : "pageserver_secondary_download_heatmap",
2231 0 : "Number of downloads of heatmaps by secondary mode locations, including when it hasn't changed"
2232 0 : )
2233 0 : .expect("failed to define a metric"),
2234 0 : download_layer: register_int_counter!(
2235 0 : "pageserver_secondary_download_layer",
2236 0 : "Number of downloads of layers by secondary mode locations"
2237 0 : )
2238 0 : .expect("failed to define a metric"),
2239 0 : }
2240 0 : });
2241 :
2242 0 : pub(crate) static SECONDARY_RESIDENT_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
2243 0 : register_uint_gauge_vec!(
2244 0 : "pageserver_secondary_resident_physical_size",
2245 0 : "The size of the layer files present in the pageserver's filesystem, for secondary locations.",
2246 0 : &["tenant_id", "shard_id"]
2247 0 : )
2248 0 : .expect("failed to define a metric")
2249 0 : });
2250 :
2251 0 : pub(crate) static NODE_UTILIZATION_SCORE: Lazy<UIntGauge> = Lazy::new(|| {
2252 0 : register_uint_gauge!(
2253 0 : "pageserver_utilization_score",
2254 0 : "The utilization score we report to the storage controller for scheduling, where 0 is empty, 1000000 is full, and anything above is considered overloaded",
2255 0 : )
2256 0 : .expect("failed to define a metric")
2257 0 : });
2258 :
2259 0 : pub(crate) static SECONDARY_HEATMAP_TOTAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
2260 0 : register_uint_gauge_vec!(
2261 0 : "pageserver_secondary_heatmap_total_size",
2262 0 : "The total size in bytes of all layers in the most recently downloaded heatmap.",
2263 0 : &["tenant_id", "shard_id"]
2264 0 : )
2265 0 : .expect("failed to define a metric")
2266 0 : });
2267 :
2268 : #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
2269 : pub enum RemoteOpKind {
2270 : Upload,
2271 : Download,
2272 : Delete,
2273 : }
2274 : impl RemoteOpKind {
2275 28909 : pub fn as_str(&self) -> &'static str {
2276 28909 : match self {
2277 27400 : Self::Upload => "upload",
2278 136 : Self::Download => "download",
2279 1373 : Self::Delete => "delete",
2280 : }
2281 28909 : }
2282 : }
2283 :
2284 : #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
2285 : pub enum RemoteOpFileKind {
2286 : Layer,
2287 : Index,
2288 : }
2289 : impl RemoteOpFileKind {
2290 28909 : pub fn as_str(&self) -> &'static str {
2291 28909 : match self {
2292 19705 : Self::Layer => "layer",
2293 9204 : Self::Index => "index",
2294 : }
2295 28909 : }
2296 : }
2297 :
2298 402 : pub(crate) static REMOTE_OPERATION_TIME: Lazy<HistogramVec> = Lazy::new(|| {
2299 402 : register_histogram_vec!(
2300 402 : "pageserver_remote_operation_seconds",
2301 402 : "Time spent on remote storage operations. \
2302 402 : Grouped by tenant, timeline, operation_kind and status. \
2303 402 : Does not account for time spent waiting in remote timeline client's queues.",
2304 402 : &["file_kind", "op_kind", "status"]
2305 402 : )
2306 402 : .expect("failed to define a metric")
2307 402 : });
2308 :
2309 0 : pub(crate) static TENANT_TASK_EVENTS: Lazy<IntCounterVec> = Lazy::new(|| {
2310 0 : register_int_counter_vec!(
2311 0 : "pageserver_tenant_task_events",
2312 0 : "Number of task start/stop/fail events.",
2313 0 : &["event"],
2314 0 : )
2315 0 : .expect("Failed to register tenant_task_events metric")
2316 0 : });
2317 :
2318 : pub struct BackgroundLoopSemaphoreMetrics {
2319 : counters: EnumMap<BackgroundLoopKind, IntCounterPair>,
2320 : durations: EnumMap<BackgroundLoopKind, Histogram>,
2321 : waiting_tasks: EnumMap<BackgroundLoopKind, IntGauge>,
2322 : running_tasks: EnumMap<BackgroundLoopKind, IntGauge>,
2323 : }
2324 :
2325 : pub(crate) static BACKGROUND_LOOP_SEMAPHORE: Lazy<BackgroundLoopSemaphoreMetrics> =
2326 39 : Lazy::new(|| {
2327 39 : let counters = register_int_counter_pair_vec!(
2328 39 : "pageserver_background_loop_semaphore_wait_start_count",
2329 39 : "Counter for background loop concurrency-limiting semaphore acquire calls started",
2330 39 : "pageserver_background_loop_semaphore_wait_finish_count",
2331 39 : "Counter for background loop concurrency-limiting semaphore acquire calls finished",
2332 39 : &["task"],
2333 39 : )
2334 39 : .unwrap();
2335 39 :
2336 39 : let durations = register_histogram_vec!(
2337 39 : "pageserver_background_loop_semaphore_wait_seconds",
2338 39 : "Seconds spent waiting on background loop semaphore acquisition",
2339 39 : &["task"],
2340 39 : vec![0.01, 1.0, 5.0, 10.0, 30.0, 60.0, 180.0, 300.0, 600.0],
2341 39 : )
2342 39 : .unwrap();
2343 39 :
2344 39 : let waiting_tasks = register_int_gauge_vec!(
2345 39 : "pageserver_background_loop_semaphore_waiting_tasks",
2346 39 : "Number of background loop tasks waiting for semaphore",
2347 39 : &["task"],
2348 39 : )
2349 39 : .unwrap();
2350 39 :
2351 39 : let running_tasks = register_int_gauge_vec!(
2352 39 : "pageserver_background_loop_semaphore_running_tasks",
2353 39 : "Number of background loop tasks running concurrently",
2354 39 : &["task"],
2355 39 : )
2356 39 : .unwrap();
2357 39 :
2358 39 : BackgroundLoopSemaphoreMetrics {
2359 390 : counters: EnumMap::from_array(std::array::from_fn(|i| {
2360 390 : let kind = BackgroundLoopKind::from_usize(i);
2361 390 : counters.with_label_values(&[kind.into()])
2362 390 : })),
2363 390 : durations: EnumMap::from_array(std::array::from_fn(|i| {
2364 390 : let kind = BackgroundLoopKind::from_usize(i);
2365 390 : durations.with_label_values(&[kind.into()])
2366 390 : })),
2367 390 : waiting_tasks: EnumMap::from_array(std::array::from_fn(|i| {
2368 390 : let kind = BackgroundLoopKind::from_usize(i);
2369 390 : waiting_tasks.with_label_values(&[kind.into()])
2370 390 : })),
2371 390 : running_tasks: EnumMap::from_array(std::array::from_fn(|i| {
2372 390 : let kind = BackgroundLoopKind::from_usize(i);
2373 390 : running_tasks.with_label_values(&[kind.into()])
2374 390 : })),
2375 39 : }
2376 39 : });
2377 :
2378 : impl BackgroundLoopSemaphoreMetrics {
2379 : /// Starts recording semaphore metrics. Call `acquired()` on the returned recorder when the
2380 : /// semaphore is acquired, and drop it when the task completes or is cancelled.
2381 691 : pub(crate) fn record(
2382 691 : &self,
2383 691 : task: BackgroundLoopKind,
2384 691 : ) -> BackgroundLoopSemaphoreMetricsRecorder {
2385 691 : BackgroundLoopSemaphoreMetricsRecorder::start(self, task)
2386 691 : }
2387 : }
2388 :
2389 : /// Records metrics for a background task.
2390 : pub struct BackgroundLoopSemaphoreMetricsRecorder<'a> {
2391 : metrics: &'a BackgroundLoopSemaphoreMetrics,
2392 : task: BackgroundLoopKind,
2393 : start: Instant,
2394 : wait_counter_guard: Option<metrics::IntCounterPairGuard>,
2395 : }
2396 :
2397 : impl<'a> BackgroundLoopSemaphoreMetricsRecorder<'a> {
2398 : /// Starts recording semaphore metrics, by recording wait time and incrementing
2399 : /// `wait_start_count` and `waiting_tasks`.
2400 691 : fn start(metrics: &'a BackgroundLoopSemaphoreMetrics, task: BackgroundLoopKind) -> Self {
2401 691 : metrics.waiting_tasks[task].inc();
2402 691 : Self {
2403 691 : metrics,
2404 691 : task,
2405 691 : start: Instant::now(),
2406 691 : wait_counter_guard: Some(metrics.counters[task].guard()),
2407 691 : }
2408 691 : }
2409 :
2410 : /// Signals that the semaphore has been acquired, and updates relevant metrics.
2411 691 : pub fn acquired(&mut self) -> Duration {
2412 691 : let waited = self.start.elapsed();
2413 691 : self.wait_counter_guard.take().expect("already acquired");
2414 691 : self.metrics.durations[self.task].observe(waited.as_secs_f64());
2415 691 : self.metrics.waiting_tasks[self.task].dec();
2416 691 : self.metrics.running_tasks[self.task].inc();
2417 691 : waited
2418 691 : }
2419 : }
2420 :
2421 : impl Drop for BackgroundLoopSemaphoreMetricsRecorder<'_> {
2422 : /// The task either completed or was cancelled.
2423 691 : fn drop(&mut self) {
2424 691 : if self.wait_counter_guard.take().is_some() {
2425 0 : // Waiting.
2426 0 : self.metrics.durations[self.task].observe(self.start.elapsed().as_secs_f64());
2427 0 : self.metrics.waiting_tasks[self.task].dec();
2428 691 : } else {
2429 691 : // Running.
2430 691 : self.metrics.running_tasks[self.task].dec();
2431 691 : }
2432 691 : }
2433 : }
2434 :
2435 0 : pub(crate) static BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT: Lazy<IntCounterVec> = Lazy::new(|| {
2436 0 : register_int_counter_vec!(
2437 0 : "pageserver_background_loop_period_overrun_count",
2438 0 : "Incremented whenever warn_when_period_overrun() logs a warning.",
2439 0 : &["task", "period"],
2440 0 : )
2441 0 : .expect("failed to define a metric")
2442 0 : });
2443 :
2444 : // walreceiver metrics
2445 :
2446 0 : pub(crate) static WALRECEIVER_STARTED_CONNECTIONS: Lazy<IntCounter> = Lazy::new(|| {
2447 0 : register_int_counter!(
2448 0 : "pageserver_walreceiver_started_connections_total",
2449 0 : "Number of started walreceiver connections"
2450 0 : )
2451 0 : .expect("failed to define a metric")
2452 0 : });
2453 :
2454 0 : pub(crate) static WALRECEIVER_ACTIVE_MANAGERS: Lazy<IntGauge> = Lazy::new(|| {
2455 0 : register_int_gauge!(
2456 0 : "pageserver_walreceiver_active_managers",
2457 0 : "Number of active walreceiver managers"
2458 0 : )
2459 0 : .expect("failed to define a metric")
2460 0 : });
2461 :
2462 0 : pub(crate) static WALRECEIVER_SWITCHES: Lazy<IntCounterVec> = Lazy::new(|| {
2463 0 : register_int_counter_vec!(
2464 0 : "pageserver_walreceiver_switches_total",
2465 0 : "Number of walreceiver manager change_connection calls",
2466 0 : &["reason"]
2467 0 : )
2468 0 : .expect("failed to define a metric")
2469 0 : });
2470 :
2471 0 : pub(crate) static WALRECEIVER_BROKER_UPDATES: Lazy<IntCounter> = Lazy::new(|| {
2472 0 : register_int_counter!(
2473 0 : "pageserver_walreceiver_broker_updates_total",
2474 0 : "Number of received broker updates in walreceiver"
2475 0 : )
2476 0 : .expect("failed to define a metric")
2477 0 : });
2478 :
2479 4 : pub(crate) static WALRECEIVER_CANDIDATES_EVENTS: Lazy<IntCounterVec> = Lazy::new(|| {
2480 4 : register_int_counter_vec!(
2481 4 : "pageserver_walreceiver_candidates_events_total",
2482 4 : "Number of walreceiver candidate events",
2483 4 : &["event"]
2484 4 : )
2485 4 : .expect("failed to define a metric")
2486 4 : });
2487 :
2488 : pub(crate) static WALRECEIVER_CANDIDATES_ADDED: Lazy<IntCounter> =
2489 0 : Lazy::new(|| WALRECEIVER_CANDIDATES_EVENTS.with_label_values(&["add"]));
2490 :
2491 : pub(crate) static WALRECEIVER_CANDIDATES_REMOVED: Lazy<IntCounter> =
2492 4 : Lazy::new(|| WALRECEIVER_CANDIDATES_EVENTS.with_label_values(&["remove"]));
2493 :
2494 : // Metrics collected on WAL redo operations
2495 : //
2496 : // We collect the time spent in actual WAL redo ('redo'), and time waiting
2497 : // for access to the postgres process ('wait') since there is only one for
2498 : // each tenant.
2499 :
2500 : /// Time buckets are small because we want to be able to measure the
2501 : /// smallest redo processing times. These buckets allow us to measure down
2502 : /// to 5us, which equates to 200'000 pages/sec, which equates to 1.6GB/sec.
2503 : /// This is much better than the previous 5ms aka 200 pages/sec aka 1.6MB/sec.
2504 : ///
2505 : /// Values up to 1s are recorded because metrics show that we have redo
2506 : /// durations and lock times larger than 0.250s.
2507 : macro_rules! redo_histogram_time_buckets {
2508 : () => {
2509 : vec![
2510 : 0.000_005, 0.000_010, 0.000_025, 0.000_050, 0.000_100, 0.000_250, 0.000_500, 0.001_000,
2511 : 0.002_500, 0.005_000, 0.010_000, 0.025_000, 0.050_000, 0.100_000, 0.250_000, 0.500_000,
2512 : 1.000_000,
2513 : ]
2514 : };
2515 : }
2516 :
2517 : /// While we're at it, also measure the amount of records replayed in each
2518 : /// operation. We have a global 'total replayed' counter, but that's not
2519 : /// as useful as 'what is the skew for how many records we replay in one
2520 : /// operation'.
2521 : macro_rules! redo_histogram_count_buckets {
2522 : () => {
2523 : vec![0.0, 1.0, 2.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0]
2524 : };
2525 : }
2526 :
2527 : macro_rules! redo_bytes_histogram_count_buckets {
2528 : () => {
2529 : // powers of (2^.5), from 2^4.5 to 2^15 (22 buckets)
2530 : // rounded up to the next multiple of 8 to capture any MAXALIGNed record of that size, too.
2531 : vec![
2532 : 24.0, 32.0, 48.0, 64.0, 96.0, 128.0, 184.0, 256.0, 368.0, 512.0, 728.0, 1024.0, 1456.0,
2533 : 2048.0, 2904.0, 4096.0, 5800.0, 8192.0, 11592.0, 16384.0, 23176.0, 32768.0,
2534 : ]
2535 : };
2536 : }
2537 :
2538 : pub(crate) struct WalIngestMetrics {
2539 : pub(crate) bytes_received: IntCounter,
2540 : pub(crate) records_received: IntCounter,
2541 : pub(crate) records_observed: IntCounter,
2542 : pub(crate) records_committed: IntCounter,
2543 : pub(crate) records_filtered: IntCounter,
2544 : pub(crate) values_committed_metadata_images: IntCounter,
2545 : pub(crate) values_committed_metadata_deltas: IntCounter,
2546 : pub(crate) values_committed_data_images: IntCounter,
2547 : pub(crate) values_committed_data_deltas: IntCounter,
2548 : pub(crate) gap_blocks_zeroed_on_rel_extend: IntCounter,
2549 : }
2550 :
2551 : impl WalIngestMetrics {
2552 0 : pub(crate) fn inc_values_committed(&self, stats: &DatadirModificationStats) {
2553 0 : if stats.metadata_images > 0 {
2554 0 : self.values_committed_metadata_images
2555 0 : .inc_by(stats.metadata_images);
2556 0 : }
2557 0 : if stats.metadata_deltas > 0 {
2558 0 : self.values_committed_metadata_deltas
2559 0 : .inc_by(stats.metadata_deltas);
2560 0 : }
2561 0 : if stats.data_images > 0 {
2562 0 : self.values_committed_data_images.inc_by(stats.data_images);
2563 0 : }
2564 0 : if stats.data_deltas > 0 {
2565 0 : self.values_committed_data_deltas.inc_by(stats.data_deltas);
2566 0 : }
2567 0 : }
2568 : }
2569 :
2570 20 : pub(crate) static WAL_INGEST: Lazy<WalIngestMetrics> = Lazy::new(|| {
2571 20 : let values_committed = register_int_counter_vec!(
2572 20 : "pageserver_wal_ingest_values_committed",
2573 20 : "Number of values committed to pageserver storage from WAL records",
2574 20 : &["class", "kind"],
2575 20 : )
2576 20 : .expect("failed to define a metric");
2577 20 :
2578 20 : WalIngestMetrics {
2579 20 : bytes_received: register_int_counter!(
2580 20 : "pageserver_wal_ingest_bytes_received",
2581 20 : "Bytes of WAL ingested from safekeepers",
2582 20 : )
2583 20 : .unwrap(),
2584 20 : records_received: register_int_counter!(
2585 20 : "pageserver_wal_ingest_records_received",
2586 20 : "Number of WAL records received from safekeepers"
2587 20 : )
2588 20 : .expect("failed to define a metric"),
2589 20 : records_observed: register_int_counter!(
2590 20 : "pageserver_wal_ingest_records_observed",
2591 20 : "Number of WAL records observed from safekeepers. These are metadata only records for shard 0."
2592 20 : )
2593 20 : .expect("failed to define a metric"),
2594 20 : records_committed: register_int_counter!(
2595 20 : "pageserver_wal_ingest_records_committed",
2596 20 : "Number of WAL records which resulted in writes to pageserver storage"
2597 20 : )
2598 20 : .expect("failed to define a metric"),
2599 20 : records_filtered: register_int_counter!(
2600 20 : "pageserver_wal_ingest_records_filtered",
2601 20 : "Number of WAL records filtered out due to sharding"
2602 20 : )
2603 20 : .expect("failed to define a metric"),
2604 20 : values_committed_metadata_images: values_committed.with_label_values(&["metadata", "image"]),
2605 20 : values_committed_metadata_deltas: values_committed.with_label_values(&["metadata", "delta"]),
2606 20 : values_committed_data_images: values_committed.with_label_values(&["data", "image"]),
2607 20 : values_committed_data_deltas: values_committed.with_label_values(&["data", "delta"]),
2608 20 : gap_blocks_zeroed_on_rel_extend: register_int_counter!(
2609 20 : "pageserver_gap_blocks_zeroed_on_rel_extend",
2610 20 : "Total number of zero gap blocks written on relation extends"
2611 20 : )
2612 20 : .expect("failed to define a metric"),
2613 20 : }
2614 20 : });
2615 :
2616 414 : pub(crate) static PAGESERVER_TIMELINE_WAL_RECORDS_RECEIVED: Lazy<IntCounterVec> = Lazy::new(|| {
2617 414 : register_int_counter_vec!(
2618 414 : "pageserver_timeline_wal_records_received",
2619 414 : "Number of WAL records received per shard",
2620 414 : &["tenant_id", "shard_id", "timeline_id"]
2621 414 : )
2622 414 : .expect("failed to define a metric")
2623 414 : });
2624 :
2625 12 : pub(crate) static WAL_REDO_TIME: Lazy<Histogram> = Lazy::new(|| {
2626 12 : register_histogram!(
2627 12 : "pageserver_wal_redo_seconds",
2628 12 : "Time spent on WAL redo",
2629 12 : redo_histogram_time_buckets!()
2630 12 : )
2631 12 : .expect("failed to define a metric")
2632 12 : });
2633 :
2634 12 : pub(crate) static WAL_REDO_RECORDS_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
2635 12 : register_histogram!(
2636 12 : "pageserver_wal_redo_records_histogram",
2637 12 : "Histogram of number of records replayed per redo in the Postgres WAL redo process",
2638 12 : redo_histogram_count_buckets!(),
2639 12 : )
2640 12 : .expect("failed to define a metric")
2641 12 : });
2642 :
2643 12 : pub(crate) static WAL_REDO_BYTES_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
2644 12 : register_histogram!(
2645 12 : "pageserver_wal_redo_bytes_histogram",
2646 12 : "Histogram of number of records replayed per redo sent to Postgres",
2647 12 : redo_bytes_histogram_count_buckets!(),
2648 12 : )
2649 12 : .expect("failed to define a metric")
2650 12 : });
2651 :
2652 : // FIXME: isn't this already included by WAL_REDO_RECORDS_HISTOGRAM which has _count?
2653 12 : pub(crate) static WAL_REDO_RECORD_COUNTER: Lazy<IntCounter> = Lazy::new(|| {
2654 12 : register_int_counter!(
2655 12 : "pageserver_replayed_wal_records_total",
2656 12 : "Number of WAL records replayed in WAL redo process"
2657 12 : )
2658 12 : .unwrap()
2659 12 : });
2660 :
2661 : #[rustfmt::skip]
2662 16 : pub(crate) static WAL_REDO_PROCESS_LAUNCH_DURATION_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
2663 16 : register_histogram!(
2664 16 : "pageserver_wal_redo_process_launch_duration",
2665 16 : "Histogram of the duration of successful WalRedoProcess::launch calls",
2666 16 : vec![
2667 16 : 0.0002, 0.0004, 0.0006, 0.0008, 0.0010,
2668 16 : 0.0020, 0.0040, 0.0060, 0.0080, 0.0100,
2669 16 : 0.0200, 0.0400, 0.0600, 0.0800, 0.1000,
2670 16 : 0.2000, 0.4000, 0.6000, 0.8000, 1.0000,
2671 16 : 1.5000, 2.0000, 2.5000, 3.0000, 4.0000, 10.0000
2672 16 : ],
2673 16 : )
2674 16 : .expect("failed to define a metric")
2675 16 : });
2676 :
2677 : pub(crate) struct WalRedoProcessCounters {
2678 : pub(crate) started: IntCounter,
2679 : pub(crate) killed_by_cause: EnumMap<WalRedoKillCause, IntCounter>,
2680 : pub(crate) active_stderr_logger_tasks_started: IntCounter,
2681 : pub(crate) active_stderr_logger_tasks_finished: IntCounter,
2682 : }
2683 :
2684 : #[derive(Debug, enum_map::Enum, strum_macros::IntoStaticStr)]
2685 : pub(crate) enum WalRedoKillCause {
2686 : WalRedoProcessDrop,
2687 : NoLeakChildDrop,
2688 : Startup,
2689 : }
2690 :
2691 : impl Default for WalRedoProcessCounters {
2692 16 : fn default() -> Self {
2693 16 : let started = register_int_counter!(
2694 16 : "pageserver_wal_redo_process_started_total",
2695 16 : "Number of WAL redo processes started",
2696 16 : )
2697 16 : .unwrap();
2698 16 :
2699 16 : let killed = register_int_counter_vec!(
2700 16 : "pageserver_wal_redo_process_stopped_total",
2701 16 : "Number of WAL redo processes stopped",
2702 16 : &["cause"],
2703 16 : )
2704 16 : .unwrap();
2705 16 :
2706 16 : let active_stderr_logger_tasks_started = register_int_counter!(
2707 16 : "pageserver_walredo_stderr_logger_tasks_started_total",
2708 16 : "Number of active walredo stderr logger tasks that have started",
2709 16 : )
2710 16 : .unwrap();
2711 16 :
2712 16 : let active_stderr_logger_tasks_finished = register_int_counter!(
2713 16 : "pageserver_walredo_stderr_logger_tasks_finished_total",
2714 16 : "Number of active walredo stderr logger tasks that have finished",
2715 16 : )
2716 16 : .unwrap();
2717 16 :
2718 16 : Self {
2719 16 : started,
2720 48 : killed_by_cause: EnumMap::from_array(std::array::from_fn(|i| {
2721 48 : let cause = WalRedoKillCause::from_usize(i);
2722 48 : let cause_str: &'static str = cause.into();
2723 48 : killed.with_label_values(&[cause_str])
2724 48 : })),
2725 16 : active_stderr_logger_tasks_started,
2726 16 : active_stderr_logger_tasks_finished,
2727 16 : }
2728 16 : }
2729 : }
2730 :
2731 : pub(crate) static WAL_REDO_PROCESS_COUNTERS: Lazy<WalRedoProcessCounters> =
2732 : Lazy::new(WalRedoProcessCounters::default);
2733 :
2734 : /// Similar to `prometheus::HistogramTimer` but does not record on drop.
2735 : pub(crate) struct StorageTimeMetricsTimer {
2736 : metrics: StorageTimeMetrics,
2737 : start: Instant,
2738 : }
2739 :
2740 : impl StorageTimeMetricsTimer {
2741 4220 : fn new(metrics: StorageTimeMetrics) -> Self {
2742 4220 : Self {
2743 4220 : metrics,
2744 4220 : start: Instant::now(),
2745 4220 : }
2746 4220 : }
2747 :
2748 : /// Returns the elapsed duration of the timer.
2749 4220 : pub fn elapsed(&self) -> Duration {
2750 4220 : self.start.elapsed()
2751 4220 : }
2752 :
2753 : /// Record the time from creation to now and return it.
2754 4220 : pub fn stop_and_record(self) -> Duration {
2755 4220 : let duration = self.elapsed();
2756 4220 : let seconds = duration.as_secs_f64();
2757 4220 : self.metrics.timeline_sum.inc_by(seconds);
2758 4220 : self.metrics.timeline_count.inc();
2759 4220 : self.metrics.global_histogram.observe(seconds);
2760 4220 : duration
2761 4220 : }
2762 :
2763 : /// Turns this timer into a timer, which will always record -- usually this means recording
2764 : /// regardless an early `?` path was taken in a function.
2765 8 : pub(crate) fn record_on_drop(self) -> AlwaysRecordingStorageTimeMetricsTimer {
2766 8 : AlwaysRecordingStorageTimeMetricsTimer(Some(self))
2767 8 : }
2768 : }
2769 :
2770 : pub(crate) struct AlwaysRecordingStorageTimeMetricsTimer(Option<StorageTimeMetricsTimer>);
2771 :
2772 : impl Drop for AlwaysRecordingStorageTimeMetricsTimer {
2773 8 : fn drop(&mut self) {
2774 8 : if let Some(inner) = self.0.take() {
2775 8 : inner.stop_and_record();
2776 8 : }
2777 8 : }
2778 : }
2779 :
2780 : impl AlwaysRecordingStorageTimeMetricsTimer {
2781 : /// Returns the elapsed duration of the timer.
2782 0 : pub fn elapsed(&self) -> Duration {
2783 0 : self.0.as_ref().expect("not dropped yet").elapsed()
2784 0 : }
2785 : }
2786 :
2787 : /// Timing facilities for an globally histogrammed metric, which is supported by per tenant and
2788 : /// timeline total sum and count.
2789 : #[derive(Clone, Debug)]
2790 : pub(crate) struct StorageTimeMetrics {
2791 : /// Sum of f64 seconds, per operation, tenant_id and timeline_id
2792 : timeline_sum: Counter,
2793 : /// Number of oeprations, per operation, tenant_id and timeline_id
2794 : timeline_count: IntCounter,
2795 : /// Global histogram having only the "operation" label.
2796 : global_histogram: Histogram,
2797 : }
2798 :
2799 : impl StorageTimeMetrics {
2800 8154 : pub fn new(
2801 8154 : operation: StorageTimeOperation,
2802 8154 : tenant_id: &str,
2803 8154 : shard_id: &str,
2804 8154 : timeline_id: &str,
2805 8154 : ) -> Self {
2806 8154 : let operation: &'static str = operation.into();
2807 8154 :
2808 8154 : let timeline_sum = STORAGE_TIME_SUM_PER_TIMELINE
2809 8154 : .get_metric_with_label_values(&[operation, tenant_id, shard_id, timeline_id])
2810 8154 : .unwrap();
2811 8154 : let timeline_count = STORAGE_TIME_COUNT_PER_TIMELINE
2812 8154 : .get_metric_with_label_values(&[operation, tenant_id, shard_id, timeline_id])
2813 8154 : .unwrap();
2814 8154 : let global_histogram = STORAGE_TIME_GLOBAL
2815 8154 : .get_metric_with_label_values(&[operation])
2816 8154 : .unwrap();
2817 8154 :
2818 8154 : StorageTimeMetrics {
2819 8154 : timeline_sum,
2820 8154 : timeline_count,
2821 8154 : global_histogram,
2822 8154 : }
2823 8154 : }
2824 :
2825 : /// Starts timing a new operation.
2826 : ///
2827 : /// Note: unlike `prometheus::HistogramTimer` the returned timer does not record on drop.
2828 4220 : pub fn start_timer(&self) -> StorageTimeMetricsTimer {
2829 4220 : StorageTimeMetricsTimer::new(self.clone())
2830 4220 : }
2831 : }
2832 :
2833 : #[derive(Debug)]
2834 : pub(crate) struct TimelineMetrics {
2835 : tenant_id: String,
2836 : shard_id: String,
2837 : timeline_id: String,
2838 : pub flush_time_histo: StorageTimeMetrics,
2839 : pub flush_delay_histo: StorageTimeMetrics,
2840 : pub flush_wait_upload_time_gauge: Gauge,
2841 : pub compact_time_histo: StorageTimeMetrics,
2842 : pub create_images_time_histo: StorageTimeMetrics,
2843 : pub logical_size_histo: StorageTimeMetrics,
2844 : pub imitate_logical_size_histo: StorageTimeMetrics,
2845 : pub load_layer_map_histo: StorageTimeMetrics,
2846 : pub garbage_collect_histo: StorageTimeMetrics,
2847 : pub find_gc_cutoffs_histo: StorageTimeMetrics,
2848 : pub last_record_lsn_gauge: IntGauge,
2849 : pub disk_consistent_lsn_gauge: IntGauge,
2850 : pub pitr_history_size: UIntGauge,
2851 : pub archival_size: UIntGauge,
2852 : pub layers_per_read: Histogram,
2853 : pub standby_horizon_gauge: IntGauge,
2854 : pub resident_physical_size_gauge: UIntGauge,
2855 : pub visible_physical_size_gauge: UIntGauge,
2856 : /// copy of LayeredTimeline.current_logical_size
2857 : pub current_logical_size_gauge: UIntGauge,
2858 : pub aux_file_size_gauge: IntGauge,
2859 : pub directory_entries_count_gauge: Lazy<UIntGauge, Box<dyn Send + Fn() -> UIntGauge>>,
2860 : pub evictions: IntCounter,
2861 : pub evictions_with_low_residence_duration: std::sync::RwLock<EvictionsWithLowResidenceDuration>,
2862 : /// Number of valid LSN leases.
2863 : pub valid_lsn_lease_count_gauge: UIntGauge,
2864 : pub wal_records_received: IntCounter,
2865 : pub storage_io_size: StorageIoSizeMetrics,
2866 : shutdown: std::sync::atomic::AtomicBool,
2867 : }
2868 :
2869 : impl TimelineMetrics {
2870 906 : pub fn new(
2871 906 : tenant_shard_id: &TenantShardId,
2872 906 : timeline_id_raw: &TimelineId,
2873 906 : evictions_with_low_residence_duration_builder: EvictionsWithLowResidenceDurationBuilder,
2874 906 : ) -> Self {
2875 906 : let tenant_id = tenant_shard_id.tenant_id.to_string();
2876 906 : let shard_id = format!("{}", tenant_shard_id.shard_slug());
2877 906 : let timeline_id = timeline_id_raw.to_string();
2878 906 : let flush_time_histo = StorageTimeMetrics::new(
2879 906 : StorageTimeOperation::LayerFlush,
2880 906 : &tenant_id,
2881 906 : &shard_id,
2882 906 : &timeline_id,
2883 906 : );
2884 906 : let flush_delay_histo = StorageTimeMetrics::new(
2885 906 : StorageTimeOperation::LayerFlushDelay,
2886 906 : &tenant_id,
2887 906 : &shard_id,
2888 906 : &timeline_id,
2889 906 : );
2890 906 : let flush_wait_upload_time_gauge = FLUSH_WAIT_UPLOAD_TIME
2891 906 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2892 906 : .unwrap();
2893 906 : let compact_time_histo = StorageTimeMetrics::new(
2894 906 : StorageTimeOperation::Compact,
2895 906 : &tenant_id,
2896 906 : &shard_id,
2897 906 : &timeline_id,
2898 906 : );
2899 906 : let create_images_time_histo = StorageTimeMetrics::new(
2900 906 : StorageTimeOperation::CreateImages,
2901 906 : &tenant_id,
2902 906 : &shard_id,
2903 906 : &timeline_id,
2904 906 : );
2905 906 : let logical_size_histo = StorageTimeMetrics::new(
2906 906 : StorageTimeOperation::LogicalSize,
2907 906 : &tenant_id,
2908 906 : &shard_id,
2909 906 : &timeline_id,
2910 906 : );
2911 906 : let imitate_logical_size_histo = StorageTimeMetrics::new(
2912 906 : StorageTimeOperation::ImitateLogicalSize,
2913 906 : &tenant_id,
2914 906 : &shard_id,
2915 906 : &timeline_id,
2916 906 : );
2917 906 : let load_layer_map_histo = StorageTimeMetrics::new(
2918 906 : StorageTimeOperation::LoadLayerMap,
2919 906 : &tenant_id,
2920 906 : &shard_id,
2921 906 : &timeline_id,
2922 906 : );
2923 906 : let garbage_collect_histo = StorageTimeMetrics::new(
2924 906 : StorageTimeOperation::Gc,
2925 906 : &tenant_id,
2926 906 : &shard_id,
2927 906 : &timeline_id,
2928 906 : );
2929 906 : let find_gc_cutoffs_histo = StorageTimeMetrics::new(
2930 906 : StorageTimeOperation::FindGcCutoffs,
2931 906 : &tenant_id,
2932 906 : &shard_id,
2933 906 : &timeline_id,
2934 906 : );
2935 906 : let last_record_lsn_gauge = LAST_RECORD_LSN
2936 906 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2937 906 : .unwrap();
2938 906 :
2939 906 : let disk_consistent_lsn_gauge = DISK_CONSISTENT_LSN
2940 906 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2941 906 : .unwrap();
2942 906 :
2943 906 : let pitr_history_size = PITR_HISTORY_SIZE
2944 906 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2945 906 : .unwrap();
2946 906 :
2947 906 : let archival_size = TIMELINE_ARCHIVE_SIZE
2948 906 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2949 906 : .unwrap();
2950 906 :
2951 906 : let layers_per_read = LAYERS_PER_READ
2952 906 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2953 906 : .unwrap();
2954 906 :
2955 906 : let standby_horizon_gauge = STANDBY_HORIZON
2956 906 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2957 906 : .unwrap();
2958 906 : let resident_physical_size_gauge = RESIDENT_PHYSICAL_SIZE
2959 906 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2960 906 : .unwrap();
2961 906 : let visible_physical_size_gauge = VISIBLE_PHYSICAL_SIZE
2962 906 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2963 906 : .unwrap();
2964 906 : // TODO: we shouldn't expose this metric
2965 906 : let current_logical_size_gauge = CURRENT_LOGICAL_SIZE
2966 906 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2967 906 : .unwrap();
2968 906 : let aux_file_size_gauge = AUX_FILE_SIZE
2969 906 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2970 906 : .unwrap();
2971 906 : // TODO use impl Trait syntax here once we have ability to use it: https://github.com/rust-lang/rust/issues/63065
2972 906 : let directory_entries_count_gauge_closure = {
2973 906 : let tenant_shard_id = *tenant_shard_id;
2974 906 : let timeline_id_raw = *timeline_id_raw;
2975 0 : move || {
2976 0 : let tenant_id = tenant_shard_id.tenant_id.to_string();
2977 0 : let shard_id = format!("{}", tenant_shard_id.shard_slug());
2978 0 : let timeline_id = timeline_id_raw.to_string();
2979 0 : let gauge: UIntGauge = DIRECTORY_ENTRIES_COUNT
2980 0 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2981 0 : .unwrap();
2982 0 : gauge
2983 0 : }
2984 : };
2985 906 : let directory_entries_count_gauge: Lazy<UIntGauge, Box<dyn Send + Fn() -> UIntGauge>> =
2986 906 : Lazy::new(Box::new(directory_entries_count_gauge_closure));
2987 906 : let evictions = EVICTIONS
2988 906 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2989 906 : .unwrap();
2990 906 : let evictions_with_low_residence_duration = evictions_with_low_residence_duration_builder
2991 906 : .build(&tenant_id, &shard_id, &timeline_id);
2992 906 :
2993 906 : let valid_lsn_lease_count_gauge = VALID_LSN_LEASE_COUNT
2994 906 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2995 906 : .unwrap();
2996 906 :
2997 906 : let wal_records_received = PAGESERVER_TIMELINE_WAL_RECORDS_RECEIVED
2998 906 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2999 906 : .unwrap();
3000 906 :
3001 906 : let storage_io_size = StorageIoSizeMetrics::new(&tenant_id, &shard_id, &timeline_id);
3002 906 :
3003 906 : TimelineMetrics {
3004 906 : tenant_id,
3005 906 : shard_id,
3006 906 : timeline_id,
3007 906 : flush_time_histo,
3008 906 : flush_delay_histo,
3009 906 : flush_wait_upload_time_gauge,
3010 906 : compact_time_histo,
3011 906 : create_images_time_histo,
3012 906 : logical_size_histo,
3013 906 : imitate_logical_size_histo,
3014 906 : garbage_collect_histo,
3015 906 : find_gc_cutoffs_histo,
3016 906 : load_layer_map_histo,
3017 906 : last_record_lsn_gauge,
3018 906 : disk_consistent_lsn_gauge,
3019 906 : pitr_history_size,
3020 906 : archival_size,
3021 906 : layers_per_read,
3022 906 : standby_horizon_gauge,
3023 906 : resident_physical_size_gauge,
3024 906 : visible_physical_size_gauge,
3025 906 : current_logical_size_gauge,
3026 906 : aux_file_size_gauge,
3027 906 : directory_entries_count_gauge,
3028 906 : evictions,
3029 906 : evictions_with_low_residence_duration: std::sync::RwLock::new(
3030 906 : evictions_with_low_residence_duration,
3031 906 : ),
3032 906 : storage_io_size,
3033 906 : valid_lsn_lease_count_gauge,
3034 906 : wal_records_received,
3035 906 : shutdown: std::sync::atomic::AtomicBool::default(),
3036 906 : }
3037 906 : }
3038 :
3039 2901 : pub(crate) fn record_new_file_metrics(&self, sz: u64) {
3040 2901 : self.resident_physical_size_add(sz);
3041 2901 : }
3042 :
3043 914 : pub(crate) fn resident_physical_size_sub(&self, sz: u64) {
3044 914 : self.resident_physical_size_gauge.sub(sz);
3045 914 : crate::metrics::RESIDENT_PHYSICAL_SIZE_GLOBAL.sub(sz);
3046 914 : }
3047 :
3048 3173 : pub(crate) fn resident_physical_size_add(&self, sz: u64) {
3049 3173 : self.resident_physical_size_gauge.add(sz);
3050 3173 : crate::metrics::RESIDENT_PHYSICAL_SIZE_GLOBAL.add(sz);
3051 3173 : }
3052 :
3053 20 : pub(crate) fn resident_physical_size_get(&self) -> u64 {
3054 20 : self.resident_physical_size_gauge.get()
3055 20 : }
3056 :
3057 2388 : pub(crate) fn flush_wait_upload_time_gauge_add(&self, duration: f64) {
3058 2388 : self.flush_wait_upload_time_gauge.add(duration);
3059 2388 : crate::metrics::FLUSH_WAIT_UPLOAD_TIME
3060 2388 : .get_metric_with_label_values(&[&self.tenant_id, &self.shard_id, &self.timeline_id])
3061 2388 : .unwrap()
3062 2388 : .add(duration);
3063 2388 : }
3064 :
3065 : /// Generates TIMELINE_LAYER labels for a persistent layer.
3066 4816 : fn make_layer_labels(&self, layer_desc: &PersistentLayerDesc) -> [&str; 5] {
3067 4816 : let level = match LayerMap::is_l0(&layer_desc.key_range, layer_desc.is_delta()) {
3068 2711 : true => LayerLevel::L0,
3069 2105 : false => LayerLevel::L1,
3070 : };
3071 4816 : let kind = match layer_desc.is_delta() {
3072 3935 : true => LayerKind::Delta,
3073 881 : false => LayerKind::Image,
3074 : };
3075 4816 : [
3076 4816 : &self.tenant_id,
3077 4816 : &self.shard_id,
3078 4816 : &self.timeline_id,
3079 4816 : level.into(),
3080 4816 : kind.into(),
3081 4816 : ]
3082 4816 : }
3083 :
3084 : /// Generates TIMELINE_LAYER labels for a frozen ephemeral layer.
3085 4776 : fn make_frozen_layer_labels(&self, _layer: &InMemoryLayer) -> [&str; 5] {
3086 4776 : [
3087 4776 : &self.tenant_id,
3088 4776 : &self.shard_id,
3089 4776 : &self.timeline_id,
3090 4776 : LayerLevel::Frozen.into(),
3091 4776 : LayerKind::Delta.into(), // by definition
3092 4776 : ]
3093 4776 : }
3094 :
3095 : /// Removes a frozen ephemeral layer to TIMELINE_LAYER metrics.
3096 2388 : pub fn dec_frozen_layer(&self, layer: &InMemoryLayer) {
3097 2388 : assert!(matches!(layer.info(), InMemoryLayerInfo::Frozen { .. }));
3098 2388 : let labels = self.make_frozen_layer_labels(layer);
3099 2388 : let size = layer.try_len().expect("frozen layer should have no writer");
3100 2388 : TIMELINE_LAYER_COUNT
3101 2388 : .get_metric_with_label_values(&labels)
3102 2388 : .unwrap()
3103 2388 : .dec();
3104 2388 : TIMELINE_LAYER_SIZE
3105 2388 : .get_metric_with_label_values(&labels)
3106 2388 : .unwrap()
3107 2388 : .sub(size);
3108 2388 : }
3109 :
3110 : /// Adds a frozen ephemeral layer to TIMELINE_LAYER metrics.
3111 2388 : pub fn inc_frozen_layer(&self, layer: &InMemoryLayer) {
3112 2388 : assert!(matches!(layer.info(), InMemoryLayerInfo::Frozen { .. }));
3113 2388 : let labels = self.make_frozen_layer_labels(layer);
3114 2388 : let size = layer.try_len().expect("frozen layer should have no writer");
3115 2388 : TIMELINE_LAYER_COUNT
3116 2388 : .get_metric_with_label_values(&labels)
3117 2388 : .unwrap()
3118 2388 : .inc();
3119 2388 : TIMELINE_LAYER_SIZE
3120 2388 : .get_metric_with_label_values(&labels)
3121 2388 : .unwrap()
3122 2388 : .add(size);
3123 2388 : }
3124 :
3125 : /// Removes a persistent layer from TIMELINE_LAYER metrics.
3126 1211 : pub fn dec_layer(&self, layer_desc: &PersistentLayerDesc) {
3127 1211 : let labels = self.make_layer_labels(layer_desc);
3128 1211 : TIMELINE_LAYER_COUNT
3129 1211 : .get_metric_with_label_values(&labels)
3130 1211 : .unwrap()
3131 1211 : .dec();
3132 1211 : TIMELINE_LAYER_SIZE
3133 1211 : .get_metric_with_label_values(&labels)
3134 1211 : .unwrap()
3135 1211 : .sub(layer_desc.file_size);
3136 1211 : }
3137 :
3138 : /// Adds a persistent layer to TIMELINE_LAYER metrics.
3139 3605 : pub fn inc_layer(&self, layer_desc: &PersistentLayerDesc) {
3140 3605 : let labels = self.make_layer_labels(layer_desc);
3141 3605 : TIMELINE_LAYER_COUNT
3142 3605 : .get_metric_with_label_values(&labels)
3143 3605 : .unwrap()
3144 3605 : .inc();
3145 3605 : TIMELINE_LAYER_SIZE
3146 3605 : .get_metric_with_label_values(&labels)
3147 3605 : .unwrap()
3148 3605 : .add(layer_desc.file_size);
3149 3605 : }
3150 :
3151 20 : pub(crate) fn shutdown(&self) {
3152 20 : let was_shutdown = self
3153 20 : .shutdown
3154 20 : .swap(true, std::sync::atomic::Ordering::Relaxed);
3155 20 :
3156 20 : if was_shutdown {
3157 : // this happens on tenant deletion because tenant first shuts down timelines, then
3158 : // invokes timeline deletion which first shuts down the timeline again.
3159 : // TODO: this can be removed once https://github.com/neondatabase/neon/issues/5080
3160 0 : return;
3161 20 : }
3162 20 :
3163 20 : let tenant_id = &self.tenant_id;
3164 20 : let timeline_id = &self.timeline_id;
3165 20 : let shard_id = &self.shard_id;
3166 20 : let _ = LAST_RECORD_LSN.remove_label_values(&[tenant_id, shard_id, timeline_id]);
3167 20 : let _ = DISK_CONSISTENT_LSN.remove_label_values(&[tenant_id, shard_id, timeline_id]);
3168 20 : let _ = FLUSH_WAIT_UPLOAD_TIME.remove_label_values(&[tenant_id, shard_id, timeline_id]);
3169 20 : let _ = STANDBY_HORIZON.remove_label_values(&[tenant_id, shard_id, timeline_id]);
3170 20 : {
3171 20 : RESIDENT_PHYSICAL_SIZE_GLOBAL.sub(self.resident_physical_size_get());
3172 20 : let _ = RESIDENT_PHYSICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
3173 20 : }
3174 20 : let _ = VISIBLE_PHYSICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
3175 20 : let _ = CURRENT_LOGICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
3176 20 : if let Some(metric) = Lazy::get(&DIRECTORY_ENTRIES_COUNT) {
3177 0 : let _ = metric.remove_label_values(&[tenant_id, shard_id, timeline_id]);
3178 20 : }
3179 :
3180 20 : let _ = TIMELINE_ARCHIVE_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
3181 20 : let _ = PITR_HISTORY_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
3182 :
3183 80 : for ref level in LayerLevel::iter() {
3184 180 : for ref kind in LayerKind::iter() {
3185 120 : let labels: [&str; 5] =
3186 120 : [tenant_id, shard_id, timeline_id, level.into(), kind.into()];
3187 120 : let _ = TIMELINE_LAYER_SIZE.remove_label_values(&labels);
3188 120 : let _ = TIMELINE_LAYER_COUNT.remove_label_values(&labels);
3189 120 : }
3190 : }
3191 :
3192 20 : let _ = LAYERS_PER_READ.remove_label_values(&[tenant_id, shard_id, timeline_id]);
3193 20 :
3194 20 : let _ = EVICTIONS.remove_label_values(&[tenant_id, shard_id, timeline_id]);
3195 20 : let _ = AUX_FILE_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
3196 20 : let _ = VALID_LSN_LEASE_COUNT.remove_label_values(&[tenant_id, shard_id, timeline_id]);
3197 20 :
3198 20 : self.evictions_with_low_residence_duration
3199 20 : .write()
3200 20 : .unwrap()
3201 20 : .remove(tenant_id, shard_id, timeline_id);
3202 :
3203 : // The following metrics are born outside of the TimelineMetrics lifecycle but still
3204 : // removed at the end of it. The idea is to have the metrics outlive the
3205 : // entity during which they're observed, e.g., the smgr metrics shall
3206 : // outlive an individual smgr connection, but not the timeline.
3207 :
3208 200 : for op in StorageTimeOperation::VARIANTS {
3209 180 : let _ = STORAGE_TIME_SUM_PER_TIMELINE.remove_label_values(&[
3210 180 : op,
3211 180 : tenant_id,
3212 180 : shard_id,
3213 180 : timeline_id,
3214 180 : ]);
3215 180 : let _ = STORAGE_TIME_COUNT_PER_TIMELINE.remove_label_values(&[
3216 180 : op,
3217 180 : tenant_id,
3218 180 : shard_id,
3219 180 : timeline_id,
3220 180 : ]);
3221 180 : }
3222 :
3223 60 : for op in StorageIoSizeOperation::VARIANTS {
3224 40 : let _ = STORAGE_IO_SIZE.remove_label_values(&[op, tenant_id, shard_id, timeline_id]);
3225 40 : }
3226 :
3227 20 : let _ = SMGR_QUERY_STARTED_PER_TENANT_TIMELINE.remove_label_values(&[
3228 20 : SmgrQueryType::GetPageAtLsn.into(),
3229 20 : tenant_id,
3230 20 : shard_id,
3231 20 : timeline_id,
3232 20 : ]);
3233 20 : let _ = SMGR_QUERY_TIME_PER_TENANT_TIMELINE.remove_label_values(&[
3234 20 : SmgrQueryType::GetPageAtLsn.into(),
3235 20 : tenant_id,
3236 20 : shard_id,
3237 20 : timeline_id,
3238 20 : ]);
3239 20 : let _ = PAGE_SERVICE_BATCH_SIZE_PER_TENANT_TIMELINE.remove_label_values(&[
3240 20 : tenant_id,
3241 20 : shard_id,
3242 20 : timeline_id,
3243 20 : ]);
3244 20 : let _ = PAGESERVER_TIMELINE_WAL_RECORDS_RECEIVED.remove_label_values(&[
3245 20 : tenant_id,
3246 20 : shard_id,
3247 20 : timeline_id,
3248 20 : ]);
3249 20 : let _ = PAGE_SERVICE_SMGR_FLUSH_INPROGRESS_MICROS.remove_label_values(&[
3250 20 : tenant_id,
3251 20 : shard_id,
3252 20 : timeline_id,
3253 20 : ]);
3254 20 : let _ = PAGE_SERVICE_SMGR_BATCH_WAIT_TIME.remove_label_values(&[
3255 20 : tenant_id,
3256 20 : shard_id,
3257 20 : timeline_id,
3258 20 : ]);
3259 20 : }
3260 : }
3261 :
3262 12 : pub(crate) fn remove_tenant_metrics(tenant_shard_id: &TenantShardId) {
3263 12 : // Only shard zero deals in synthetic sizes
3264 12 : if tenant_shard_id.is_shard_zero() {
3265 12 : let tid = tenant_shard_id.tenant_id.to_string();
3266 12 : let _ = TENANT_SYNTHETIC_SIZE_METRIC.remove_label_values(&[&tid]);
3267 12 : }
3268 :
3269 12 : tenant_throttling::remove_tenant_metrics(tenant_shard_id);
3270 12 :
3271 12 : // we leave the BROKEN_TENANTS_SET entry if any
3272 12 : }
3273 :
3274 : /// Maintain a per timeline gauge in addition to the global gauge.
3275 : pub(crate) struct PerTimelineRemotePhysicalSizeGauge {
3276 : last_set: AtomicU64,
3277 : gauge: UIntGauge,
3278 : }
3279 :
3280 : impl PerTimelineRemotePhysicalSizeGauge {
3281 926 : fn new(per_timeline_gauge: UIntGauge) -> Self {
3282 926 : Self {
3283 926 : last_set: AtomicU64::new(0),
3284 926 : gauge: per_timeline_gauge,
3285 926 : }
3286 926 : }
3287 3908 : pub(crate) fn set(&self, sz: u64) {
3288 3908 : self.gauge.set(sz);
3289 3908 : let prev = self.last_set.swap(sz, std::sync::atomic::Ordering::Relaxed);
3290 3908 : if sz < prev {
3291 69 : REMOTE_PHYSICAL_SIZE_GLOBAL.sub(prev - sz);
3292 3839 : } else {
3293 3839 : REMOTE_PHYSICAL_SIZE_GLOBAL.add(sz - prev);
3294 3839 : };
3295 3908 : }
3296 4 : pub(crate) fn get(&self) -> u64 {
3297 4 : self.gauge.get()
3298 4 : }
3299 : }
3300 :
3301 : impl Drop for PerTimelineRemotePhysicalSizeGauge {
3302 40 : fn drop(&mut self) {
3303 40 : REMOTE_PHYSICAL_SIZE_GLOBAL.sub(self.last_set.load(std::sync::atomic::Ordering::Relaxed));
3304 40 : }
3305 : }
3306 :
3307 : pub(crate) struct RemoteTimelineClientMetrics {
3308 : tenant_id: String,
3309 : shard_id: String,
3310 : timeline_id: String,
3311 : pub(crate) remote_physical_size_gauge: PerTimelineRemotePhysicalSizeGauge,
3312 : calls: Mutex<HashMap<(&'static str, &'static str), IntCounterPair>>,
3313 : bytes_started_counter: Mutex<HashMap<(&'static str, &'static str), IntCounter>>,
3314 : bytes_finished_counter: Mutex<HashMap<(&'static str, &'static str), IntCounter>>,
3315 : pub(crate) projected_remote_consistent_lsn_gauge: UIntGauge,
3316 : }
3317 :
3318 : impl RemoteTimelineClientMetrics {
3319 926 : pub fn new(tenant_shard_id: &TenantShardId, timeline_id: &TimelineId) -> Self {
3320 926 : let tenant_id_str = tenant_shard_id.tenant_id.to_string();
3321 926 : let shard_id_str = format!("{}", tenant_shard_id.shard_slug());
3322 926 : let timeline_id_str = timeline_id.to_string();
3323 926 :
3324 926 : let remote_physical_size_gauge = PerTimelineRemotePhysicalSizeGauge::new(
3325 926 : REMOTE_PHYSICAL_SIZE
3326 926 : .get_metric_with_label_values(&[&tenant_id_str, &shard_id_str, &timeline_id_str])
3327 926 : .unwrap(),
3328 926 : );
3329 926 :
3330 926 : let projected_remote_consistent_lsn_gauge = PROJECTED_REMOTE_CONSISTENT_LSN
3331 926 : .get_metric_with_label_values(&[&tenant_id_str, &shard_id_str, &timeline_id_str])
3332 926 : .unwrap();
3333 926 :
3334 926 : RemoteTimelineClientMetrics {
3335 926 : tenant_id: tenant_id_str,
3336 926 : shard_id: shard_id_str,
3337 926 : timeline_id: timeline_id_str,
3338 926 : calls: Mutex::new(HashMap::default()),
3339 926 : bytes_started_counter: Mutex::new(HashMap::default()),
3340 926 : bytes_finished_counter: Mutex::new(HashMap::default()),
3341 926 : remote_physical_size_gauge,
3342 926 : projected_remote_consistent_lsn_gauge,
3343 926 : }
3344 926 : }
3345 :
3346 5925 : pub fn remote_operation_time(
3347 5925 : &self,
3348 5925 : file_kind: &RemoteOpFileKind,
3349 5925 : op_kind: &RemoteOpKind,
3350 5925 : status: &'static str,
3351 5925 : ) -> Histogram {
3352 5925 : let key = (file_kind.as_str(), op_kind.as_str(), status);
3353 5925 : REMOTE_OPERATION_TIME
3354 5925 : .get_metric_with_label_values(&[key.0, key.1, key.2])
3355 5925 : .unwrap()
3356 5925 : }
3357 :
3358 13647 : fn calls_counter_pair(
3359 13647 : &self,
3360 13647 : file_kind: &RemoteOpFileKind,
3361 13647 : op_kind: &RemoteOpKind,
3362 13647 : ) -> IntCounterPair {
3363 13647 : let mut guard = self.calls.lock().unwrap();
3364 13647 : let key = (file_kind.as_str(), op_kind.as_str());
3365 13647 : let metric = guard.entry(key).or_insert_with(move || {
3366 1664 : REMOTE_TIMELINE_CLIENT_CALLS
3367 1664 : .get_metric_with_label_values(&[
3368 1664 : &self.tenant_id,
3369 1664 : &self.shard_id,
3370 1664 : &self.timeline_id,
3371 1664 : key.0,
3372 1664 : key.1,
3373 1664 : ])
3374 1664 : .unwrap()
3375 13647 : });
3376 13647 : metric.clone()
3377 13647 : }
3378 :
3379 3229 : fn bytes_started_counter(
3380 3229 : &self,
3381 3229 : file_kind: &RemoteOpFileKind,
3382 3229 : op_kind: &RemoteOpKind,
3383 3229 : ) -> IntCounter {
3384 3229 : let mut guard = self.bytes_started_counter.lock().unwrap();
3385 3229 : let key = (file_kind.as_str(), op_kind.as_str());
3386 3229 : let metric = guard.entry(key).or_insert_with(move || {
3387 658 : REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER
3388 658 : .get_metric_with_label_values(&[
3389 658 : &self.tenant_id,
3390 658 : &self.shard_id,
3391 658 : &self.timeline_id,
3392 658 : key.0,
3393 658 : key.1,
3394 658 : ])
3395 658 : .unwrap()
3396 3229 : });
3397 3229 : metric.clone()
3398 3229 : }
3399 :
3400 6084 : fn bytes_finished_counter(
3401 6084 : &self,
3402 6084 : file_kind: &RemoteOpFileKind,
3403 6084 : op_kind: &RemoteOpKind,
3404 6084 : ) -> IntCounter {
3405 6084 : let mut guard = self.bytes_finished_counter.lock().unwrap();
3406 6084 : let key = (file_kind.as_str(), op_kind.as_str());
3407 6084 : let metric = guard.entry(key).or_insert_with(move || {
3408 658 : REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER
3409 658 : .get_metric_with_label_values(&[
3410 658 : &self.tenant_id,
3411 658 : &self.shard_id,
3412 658 : &self.timeline_id,
3413 658 : key.0,
3414 658 : key.1,
3415 658 : ])
3416 658 : .unwrap()
3417 6084 : });
3418 6084 : metric.clone()
3419 6084 : }
3420 : }
3421 :
3422 : #[cfg(test)]
3423 : impl RemoteTimelineClientMetrics {
3424 12 : pub fn get_bytes_started_counter_value(
3425 12 : &self,
3426 12 : file_kind: &RemoteOpFileKind,
3427 12 : op_kind: &RemoteOpKind,
3428 12 : ) -> Option<u64> {
3429 12 : let guard = self.bytes_started_counter.lock().unwrap();
3430 12 : let key = (file_kind.as_str(), op_kind.as_str());
3431 12 : guard.get(&key).map(|counter| counter.get())
3432 12 : }
3433 :
3434 12 : pub fn get_bytes_finished_counter_value(
3435 12 : &self,
3436 12 : file_kind: &RemoteOpFileKind,
3437 12 : op_kind: &RemoteOpKind,
3438 12 : ) -> Option<u64> {
3439 12 : let guard = self.bytes_finished_counter.lock().unwrap();
3440 12 : let key = (file_kind.as_str(), op_kind.as_str());
3441 12 : guard.get(&key).map(|counter| counter.get())
3442 12 : }
3443 : }
3444 :
3445 : /// See [`RemoteTimelineClientMetrics::call_begin`].
3446 : #[must_use]
3447 : pub(crate) struct RemoteTimelineClientCallMetricGuard {
3448 : /// Decremented on drop.
3449 : calls_counter_pair: Option<IntCounterPair>,
3450 : /// If Some(), this references the bytes_finished metric, and we increment it by the given `u64` on drop.
3451 : bytes_finished: Option<(IntCounter, u64)>,
3452 : }
3453 :
3454 : impl RemoteTimelineClientCallMetricGuard {
3455 : /// Consume this guard object without performing the metric updates it would do on `drop()`.
3456 : /// The caller vouches to do the metric updates manually.
3457 7196 : pub fn will_decrement_manually(mut self) {
3458 7196 : let RemoteTimelineClientCallMetricGuard {
3459 7196 : calls_counter_pair,
3460 7196 : bytes_finished,
3461 7196 : } = &mut self;
3462 7196 : calls_counter_pair.take();
3463 7196 : bytes_finished.take();
3464 7196 : }
3465 : }
3466 :
3467 : impl Drop for RemoteTimelineClientCallMetricGuard {
3468 7264 : fn drop(&mut self) {
3469 7264 : let RemoteTimelineClientCallMetricGuard {
3470 7264 : calls_counter_pair,
3471 7264 : bytes_finished,
3472 7264 : } = self;
3473 7264 : if let Some(guard) = calls_counter_pair.take() {
3474 68 : guard.dec();
3475 7196 : }
3476 7264 : if let Some((bytes_finished_metric, value)) = bytes_finished {
3477 0 : bytes_finished_metric.inc_by(*value);
3478 7264 : }
3479 7264 : }
3480 : }
3481 :
3482 : /// The enum variants communicate to the [`RemoteTimelineClientMetrics`] whether to
3483 : /// track the byte size of this call in applicable metric(s).
3484 : pub(crate) enum RemoteTimelineClientMetricsCallTrackSize {
3485 : /// Do not account for this call's byte size in any metrics.
3486 : /// The `reason` field is there to make the call sites self-documenting
3487 : /// about why they don't need the metric.
3488 : DontTrackSize { reason: &'static str },
3489 : /// Track the byte size of the call in applicable metric(s).
3490 : Bytes(u64),
3491 : }
3492 :
3493 : impl RemoteTimelineClientMetrics {
3494 : /// Update the metrics that change when a call to the remote timeline client instance starts.
3495 : ///
3496 : /// Drop the returned guard object once the operation is finished to updates corresponding metrics that track completions.
3497 : /// Or, use [`RemoteTimelineClientCallMetricGuard::will_decrement_manually`] and [`call_end`](Self::call_end) if that
3498 : /// is more suitable.
3499 : /// Never do both.
3500 7264 : pub(crate) fn call_begin(
3501 7264 : &self,
3502 7264 : file_kind: &RemoteOpFileKind,
3503 7264 : op_kind: &RemoteOpKind,
3504 7264 : size: RemoteTimelineClientMetricsCallTrackSize,
3505 7264 : ) -> RemoteTimelineClientCallMetricGuard {
3506 7264 : let calls_counter_pair = self.calls_counter_pair(file_kind, op_kind);
3507 7264 : calls_counter_pair.inc();
3508 :
3509 7264 : let bytes_finished = match size {
3510 4035 : RemoteTimelineClientMetricsCallTrackSize::DontTrackSize { reason: _reason } => {
3511 4035 : // nothing to do
3512 4035 : None
3513 : }
3514 3229 : RemoteTimelineClientMetricsCallTrackSize::Bytes(size) => {
3515 3229 : self.bytes_started_counter(file_kind, op_kind).inc_by(size);
3516 3229 : let finished_counter = self.bytes_finished_counter(file_kind, op_kind);
3517 3229 : Some((finished_counter, size))
3518 : }
3519 : };
3520 7264 : RemoteTimelineClientCallMetricGuard {
3521 7264 : calls_counter_pair: Some(calls_counter_pair),
3522 7264 : bytes_finished,
3523 7264 : }
3524 7264 : }
3525 :
3526 : /// Manually udpate the metrics that track completions, instead of using the guard object.
3527 : /// Using the guard object is generally preferable.
3528 : /// See [`call_begin`](Self::call_begin) for more context.
3529 6383 : pub(crate) fn call_end(
3530 6383 : &self,
3531 6383 : file_kind: &RemoteOpFileKind,
3532 6383 : op_kind: &RemoteOpKind,
3533 6383 : size: RemoteTimelineClientMetricsCallTrackSize,
3534 6383 : ) {
3535 6383 : let calls_counter_pair = self.calls_counter_pair(file_kind, op_kind);
3536 6383 : calls_counter_pair.dec();
3537 6383 : match size {
3538 3528 : RemoteTimelineClientMetricsCallTrackSize::DontTrackSize { reason: _reason } => {}
3539 2855 : RemoteTimelineClientMetricsCallTrackSize::Bytes(size) => {
3540 2855 : self.bytes_finished_counter(file_kind, op_kind).inc_by(size);
3541 2855 : }
3542 : }
3543 6383 : }
3544 : }
3545 :
3546 : impl Drop for RemoteTimelineClientMetrics {
3547 40 : fn drop(&mut self) {
3548 40 : let RemoteTimelineClientMetrics {
3549 40 : tenant_id,
3550 40 : shard_id,
3551 40 : timeline_id,
3552 40 : remote_physical_size_gauge,
3553 40 : calls,
3554 40 : bytes_started_counter,
3555 40 : bytes_finished_counter,
3556 40 : projected_remote_consistent_lsn_gauge,
3557 40 : } = self;
3558 48 : for ((a, b), _) in calls.get_mut().unwrap().drain() {
3559 48 : let mut res = [Ok(()), Ok(())];
3560 48 : REMOTE_TIMELINE_CLIENT_CALLS
3561 48 : .remove_label_values(&mut res, &[tenant_id, shard_id, timeline_id, a, b]);
3562 48 : // don't care about results
3563 48 : }
3564 40 : for ((a, b), _) in bytes_started_counter.get_mut().unwrap().drain() {
3565 12 : let _ = REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER.remove_label_values(&[
3566 12 : tenant_id,
3567 12 : shard_id,
3568 12 : timeline_id,
3569 12 : a,
3570 12 : b,
3571 12 : ]);
3572 12 : }
3573 40 : for ((a, b), _) in bytes_finished_counter.get_mut().unwrap().drain() {
3574 12 : let _ = REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER.remove_label_values(&[
3575 12 : tenant_id,
3576 12 : shard_id,
3577 12 : timeline_id,
3578 12 : a,
3579 12 : b,
3580 12 : ]);
3581 12 : }
3582 40 : {
3583 40 : let _ = remote_physical_size_gauge; // use to avoid 'unused' warning in desctructuring above
3584 40 : let _ = REMOTE_PHYSICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
3585 40 : }
3586 40 : {
3587 40 : let _ = projected_remote_consistent_lsn_gauge;
3588 40 : let _ = PROJECTED_REMOTE_CONSISTENT_LSN.remove_label_values(&[
3589 40 : tenant_id,
3590 40 : shard_id,
3591 40 : timeline_id,
3592 40 : ]);
3593 40 : }
3594 40 : }
3595 : }
3596 :
3597 : /// Wrapper future that measures the time spent by a remote storage operation,
3598 : /// and records the time and success/failure as a prometheus metric.
3599 : pub(crate) trait MeasureRemoteOp: Sized {
3600 6191 : fn measure_remote_op(
3601 6191 : self,
3602 6191 : file_kind: RemoteOpFileKind,
3603 6191 : op: RemoteOpKind,
3604 6191 : metrics: Arc<RemoteTimelineClientMetrics>,
3605 6191 : ) -> MeasuredRemoteOp<Self> {
3606 6191 : let start = Instant::now();
3607 6191 : MeasuredRemoteOp {
3608 6191 : inner: self,
3609 6191 : file_kind,
3610 6191 : op,
3611 6191 : start,
3612 6191 : metrics,
3613 6191 : }
3614 6191 : }
3615 : }
3616 :
3617 : impl<T: Sized> MeasureRemoteOp for T {}
3618 :
3619 : pin_project! {
3620 : pub(crate) struct MeasuredRemoteOp<F>
3621 : {
3622 : #[pin]
3623 : inner: F,
3624 : file_kind: RemoteOpFileKind,
3625 : op: RemoteOpKind,
3626 : start: Instant,
3627 : metrics: Arc<RemoteTimelineClientMetrics>,
3628 : }
3629 : }
3630 :
3631 : impl<F: Future<Output = Result<O, E>>, O, E> Future for MeasuredRemoteOp<F> {
3632 : type Output = Result<O, E>;
3633 :
3634 96481 : fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
3635 96481 : let this = self.project();
3636 96481 : let poll_result = this.inner.poll(cx);
3637 96481 : if let Poll::Ready(ref res) = poll_result {
3638 5925 : let duration = this.start.elapsed();
3639 5925 : let status = if res.is_ok() { &"success" } else { &"failure" };
3640 5925 : this.metrics
3641 5925 : .remote_operation_time(this.file_kind, this.op, status)
3642 5925 : .observe(duration.as_secs_f64());
3643 90556 : }
3644 96481 : poll_result
3645 96481 : }
3646 : }
3647 :
3648 : pub mod tokio_epoll_uring {
3649 : use std::collections::HashMap;
3650 : use std::sync::{Arc, Mutex};
3651 :
3652 : use metrics::{Histogram, LocalHistogram, UIntGauge, register_histogram, register_int_counter};
3653 : use once_cell::sync::Lazy;
3654 :
3655 : /// Shared storage for tokio-epoll-uring thread local metrics.
3656 : pub(crate) static THREAD_LOCAL_METRICS_STORAGE: Lazy<ThreadLocalMetricsStorage> =
3657 240 : Lazy::new(|| {
3658 240 : let slots_submission_queue_depth = register_histogram!(
3659 240 : "pageserver_tokio_epoll_uring_slots_submission_queue_depth",
3660 240 : "The slots waiters queue depth of each tokio_epoll_uring system",
3661 240 : vec![
3662 240 : 1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0
3663 240 : ],
3664 240 : )
3665 240 : .expect("failed to define a metric");
3666 240 : ThreadLocalMetricsStorage {
3667 240 : observers: Mutex::new(HashMap::new()),
3668 240 : slots_submission_queue_depth,
3669 240 : }
3670 240 : });
3671 :
3672 : pub struct ThreadLocalMetricsStorage {
3673 : /// List of thread local metrics observers.
3674 : observers: Mutex<HashMap<u64, Arc<ThreadLocalMetrics>>>,
3675 : /// A histogram shared between all thread local systems
3676 : /// for collecting slots submission queue depth.
3677 : slots_submission_queue_depth: Histogram,
3678 : }
3679 :
3680 : /// Each thread-local [`tokio_epoll_uring::System`] gets one of these as its
3681 : /// [`tokio_epoll_uring::metrics::PerSystemMetrics`] generic.
3682 : ///
3683 : /// The System makes observations into [`Self`] and periodically, the collector
3684 : /// comes along and flushes [`Self`] into the shared storage [`THREAD_LOCAL_METRICS_STORAGE`].
3685 : ///
3686 : /// [`LocalHistogram`] is `!Send`, so, we need to put it behind a [`Mutex`].
3687 : /// But except for the periodic flush, the lock is uncontended so there's no waiting
3688 : /// for cache coherence protocol to get an exclusive cache line.
3689 : pub struct ThreadLocalMetrics {
3690 : /// Local observer of thread local tokio-epoll-uring system's slots waiters queue depth.
3691 : slots_submission_queue_depth: Mutex<LocalHistogram>,
3692 : }
3693 :
3694 : impl ThreadLocalMetricsStorage {
3695 : /// Registers a new thread local system. Returns a thread local metrics observer.
3696 1014 : pub fn register_system(&self, id: u64) -> Arc<ThreadLocalMetrics> {
3697 1014 : let per_system_metrics = Arc::new(ThreadLocalMetrics::new(
3698 1014 : self.slots_submission_queue_depth.local(),
3699 1014 : ));
3700 1014 : let mut g = self.observers.lock().unwrap();
3701 1014 : g.insert(id, Arc::clone(&per_system_metrics));
3702 1014 : per_system_metrics
3703 1014 : }
3704 :
3705 : /// Removes metrics observer for a thread local system.
3706 : /// This should be called before dropping a thread local system.
3707 240 : pub fn remove_system(&self, id: u64) {
3708 240 : let mut g = self.observers.lock().unwrap();
3709 240 : g.remove(&id);
3710 240 : }
3711 :
3712 : /// Flush all thread local metrics to the shared storage.
3713 0 : pub fn flush_thread_local_metrics(&self) {
3714 0 : let g = self.observers.lock().unwrap();
3715 0 : g.values().for_each(|local| {
3716 0 : local.flush();
3717 0 : });
3718 0 : }
3719 : }
3720 :
3721 : impl ThreadLocalMetrics {
3722 1014 : pub fn new(slots_submission_queue_depth: LocalHistogram) -> Self {
3723 1014 : ThreadLocalMetrics {
3724 1014 : slots_submission_queue_depth: Mutex::new(slots_submission_queue_depth),
3725 1014 : }
3726 1014 : }
3727 :
3728 : /// Flushes the thread local metrics to shared aggregator.
3729 0 : pub fn flush(&self) {
3730 0 : let Self {
3731 0 : slots_submission_queue_depth,
3732 0 : } = self;
3733 0 : slots_submission_queue_depth.lock().unwrap().flush();
3734 0 : }
3735 : }
3736 :
3737 : impl tokio_epoll_uring::metrics::PerSystemMetrics for ThreadLocalMetrics {
3738 1817628 : fn observe_slots_submission_queue_depth(&self, queue_depth: u64) {
3739 1817628 : let Self {
3740 1817628 : slots_submission_queue_depth,
3741 1817628 : } = self;
3742 1817628 : slots_submission_queue_depth
3743 1817628 : .lock()
3744 1817628 : .unwrap()
3745 1817628 : .observe(queue_depth as f64);
3746 1817628 : }
3747 : }
3748 :
3749 : pub struct Collector {
3750 : descs: Vec<metrics::core::Desc>,
3751 : systems_created: UIntGauge,
3752 : systems_destroyed: UIntGauge,
3753 : thread_local_metrics_storage: &'static ThreadLocalMetricsStorage,
3754 : }
3755 :
3756 : impl metrics::core::Collector for Collector {
3757 0 : fn desc(&self) -> Vec<&metrics::core::Desc> {
3758 0 : self.descs.iter().collect()
3759 0 : }
3760 :
3761 0 : fn collect(&self) -> Vec<metrics::proto::MetricFamily> {
3762 0 : let mut mfs = Vec::with_capacity(Self::NMETRICS);
3763 0 : let tokio_epoll_uring::metrics::GlobalMetrics {
3764 0 : systems_created,
3765 0 : systems_destroyed,
3766 0 : } = tokio_epoll_uring::metrics::global();
3767 0 : self.systems_created.set(systems_created);
3768 0 : mfs.extend(self.systems_created.collect());
3769 0 : self.systems_destroyed.set(systems_destroyed);
3770 0 : mfs.extend(self.systems_destroyed.collect());
3771 0 :
3772 0 : self.thread_local_metrics_storage
3773 0 : .flush_thread_local_metrics();
3774 0 :
3775 0 : mfs.extend(
3776 0 : self.thread_local_metrics_storage
3777 0 : .slots_submission_queue_depth
3778 0 : .collect(),
3779 0 : );
3780 0 : mfs
3781 0 : }
3782 : }
3783 :
3784 : impl Collector {
3785 : const NMETRICS: usize = 3;
3786 :
3787 : #[allow(clippy::new_without_default)]
3788 0 : pub fn new() -> Self {
3789 0 : let mut descs = Vec::new();
3790 0 :
3791 0 : let systems_created = UIntGauge::new(
3792 0 : "pageserver_tokio_epoll_uring_systems_created",
3793 0 : "counter of tokio-epoll-uring systems that were created",
3794 0 : )
3795 0 : .unwrap();
3796 0 : descs.extend(
3797 0 : metrics::core::Collector::desc(&systems_created)
3798 0 : .into_iter()
3799 0 : .cloned(),
3800 0 : );
3801 0 :
3802 0 : let systems_destroyed = UIntGauge::new(
3803 0 : "pageserver_tokio_epoll_uring_systems_destroyed",
3804 0 : "counter of tokio-epoll-uring systems that were destroyed",
3805 0 : )
3806 0 : .unwrap();
3807 0 : descs.extend(
3808 0 : metrics::core::Collector::desc(&systems_destroyed)
3809 0 : .into_iter()
3810 0 : .cloned(),
3811 0 : );
3812 0 :
3813 0 : Self {
3814 0 : descs,
3815 0 : systems_created,
3816 0 : systems_destroyed,
3817 0 : thread_local_metrics_storage: &THREAD_LOCAL_METRICS_STORAGE,
3818 0 : }
3819 0 : }
3820 : }
3821 :
3822 240 : pub(crate) static THREAD_LOCAL_LAUNCH_SUCCESSES: Lazy<metrics::IntCounter> = Lazy::new(|| {
3823 240 : register_int_counter!(
3824 240 : "pageserver_tokio_epoll_uring_pageserver_thread_local_launch_success_count",
3825 240 : "Number of times where thread_local_system creation spanned multiple executor threads",
3826 240 : )
3827 240 : .unwrap()
3828 240 : });
3829 :
3830 0 : pub(crate) static THREAD_LOCAL_LAUNCH_FAILURES: Lazy<metrics::IntCounter> = Lazy::new(|| {
3831 0 : register_int_counter!(
3832 0 : "pageserver_tokio_epoll_uring_pageserver_thread_local_launch_failures_count",
3833 0 : "Number of times thread_local_system creation failed and was retried after back-off.",
3834 0 : )
3835 0 : .unwrap()
3836 0 : });
3837 : }
3838 :
3839 : pub(crate) mod tenant_throttling {
3840 : use metrics::{IntCounter, register_int_counter_vec};
3841 : use once_cell::sync::Lazy;
3842 : use utils::shard::TenantShardId;
3843 :
3844 : pub(crate) struct GlobalAndPerTenantIntCounter {
3845 : global: IntCounter,
3846 : per_tenant: IntCounter,
3847 : }
3848 :
3849 : impl GlobalAndPerTenantIntCounter {
3850 : #[inline(always)]
3851 0 : pub(crate) fn inc(&self) {
3852 0 : self.inc_by(1)
3853 0 : }
3854 : #[inline(always)]
3855 0 : pub(crate) fn inc_by(&self, n: u64) {
3856 0 : self.global.inc_by(n);
3857 0 : self.per_tenant.inc_by(n);
3858 0 : }
3859 : }
3860 :
3861 : pub(crate) struct Metrics<const KIND: usize> {
3862 : pub(super) count_accounted_start: GlobalAndPerTenantIntCounter,
3863 : pub(super) count_accounted_finish: GlobalAndPerTenantIntCounter,
3864 : pub(super) wait_time: GlobalAndPerTenantIntCounter,
3865 : pub(super) count_throttled: GlobalAndPerTenantIntCounter,
3866 : }
3867 :
3868 418 : static COUNT_ACCOUNTED_START: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
3869 418 : register_int_counter_vec!(
3870 418 : "pageserver_tenant_throttling_count_accounted_start_global",
3871 418 : "Count of tenant throttling starts, by kind of throttle.",
3872 418 : &["kind"]
3873 418 : )
3874 418 : .unwrap()
3875 418 : });
3876 418 : static COUNT_ACCOUNTED_START_PER_TENANT: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
3877 418 : register_int_counter_vec!(
3878 418 : "pageserver_tenant_throttling_count_accounted_start",
3879 418 : "Count of tenant throttling starts, by kind of throttle.",
3880 418 : &["kind", "tenant_id", "shard_id"]
3881 418 : )
3882 418 : .unwrap()
3883 418 : });
3884 418 : static COUNT_ACCOUNTED_FINISH: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
3885 418 : register_int_counter_vec!(
3886 418 : "pageserver_tenant_throttling_count_accounted_finish_global",
3887 418 : "Count of tenant throttling finishes, by kind of throttle.",
3888 418 : &["kind"]
3889 418 : )
3890 418 : .unwrap()
3891 418 : });
3892 418 : static COUNT_ACCOUNTED_FINISH_PER_TENANT: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
3893 418 : register_int_counter_vec!(
3894 418 : "pageserver_tenant_throttling_count_accounted_finish",
3895 418 : "Count of tenant throttling finishes, by kind of throttle.",
3896 418 : &["kind", "tenant_id", "shard_id"]
3897 418 : )
3898 418 : .unwrap()
3899 418 : });
3900 418 : static WAIT_USECS: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
3901 418 : register_int_counter_vec!(
3902 418 : "pageserver_tenant_throttling_wait_usecs_sum_global",
3903 418 : "Sum of microseconds that spent waiting throttle by kind of throttle.",
3904 418 : &["kind"]
3905 418 : )
3906 418 : .unwrap()
3907 418 : });
3908 418 : static WAIT_USECS_PER_TENANT: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
3909 418 : register_int_counter_vec!(
3910 418 : "pageserver_tenant_throttling_wait_usecs_sum",
3911 418 : "Sum of microseconds that spent waiting throttle by kind of throttle.",
3912 418 : &["kind", "tenant_id", "shard_id"]
3913 418 : )
3914 418 : .unwrap()
3915 418 : });
3916 :
3917 418 : static WAIT_COUNT: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
3918 418 : register_int_counter_vec!(
3919 418 : "pageserver_tenant_throttling_count_global",
3920 418 : "Count of tenant throttlings, by kind of throttle.",
3921 418 : &["kind"]
3922 418 : )
3923 418 : .unwrap()
3924 418 : });
3925 418 : static WAIT_COUNT_PER_TENANT: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
3926 418 : register_int_counter_vec!(
3927 418 : "pageserver_tenant_throttling_count",
3928 418 : "Count of tenant throttlings, by kind of throttle.",
3929 418 : &["kind", "tenant_id", "shard_id"]
3930 418 : )
3931 418 : .unwrap()
3932 418 : });
3933 :
3934 : const KINDS: &[&str] = &["pagestream"];
3935 : pub type Pagestream = Metrics<0>;
3936 :
3937 : impl<const KIND: usize> Metrics<KIND> {
3938 454 : pub(crate) fn new(tenant_shard_id: &TenantShardId) -> Self {
3939 454 : let per_tenant_label_values = &[
3940 454 : KINDS[KIND],
3941 454 : &tenant_shard_id.tenant_id.to_string(),
3942 454 : &tenant_shard_id.shard_slug().to_string(),
3943 454 : ];
3944 454 : Metrics {
3945 454 : count_accounted_start: {
3946 454 : GlobalAndPerTenantIntCounter {
3947 454 : global: COUNT_ACCOUNTED_START.with_label_values(&[KINDS[KIND]]),
3948 454 : per_tenant: COUNT_ACCOUNTED_START_PER_TENANT
3949 454 : .with_label_values(per_tenant_label_values),
3950 454 : }
3951 454 : },
3952 454 : count_accounted_finish: {
3953 454 : GlobalAndPerTenantIntCounter {
3954 454 : global: COUNT_ACCOUNTED_FINISH.with_label_values(&[KINDS[KIND]]),
3955 454 : per_tenant: COUNT_ACCOUNTED_FINISH_PER_TENANT
3956 454 : .with_label_values(per_tenant_label_values),
3957 454 : }
3958 454 : },
3959 454 : wait_time: {
3960 454 : GlobalAndPerTenantIntCounter {
3961 454 : global: WAIT_USECS.with_label_values(&[KINDS[KIND]]),
3962 454 : per_tenant: WAIT_USECS_PER_TENANT
3963 454 : .with_label_values(per_tenant_label_values),
3964 454 : }
3965 454 : },
3966 454 : count_throttled: {
3967 454 : GlobalAndPerTenantIntCounter {
3968 454 : global: WAIT_COUNT.with_label_values(&[KINDS[KIND]]),
3969 454 : per_tenant: WAIT_COUNT_PER_TENANT
3970 454 : .with_label_values(per_tenant_label_values),
3971 454 : }
3972 454 : },
3973 454 : }
3974 454 : }
3975 : }
3976 :
3977 0 : pub(crate) fn preinitialize_global_metrics() {
3978 0 : Lazy::force(&COUNT_ACCOUNTED_START);
3979 0 : Lazy::force(&COUNT_ACCOUNTED_FINISH);
3980 0 : Lazy::force(&WAIT_USECS);
3981 0 : Lazy::force(&WAIT_COUNT);
3982 0 : }
3983 :
3984 12 : pub(crate) fn remove_tenant_metrics(tenant_shard_id: &TenantShardId) {
3985 48 : for m in &[
3986 12 : &COUNT_ACCOUNTED_START_PER_TENANT,
3987 12 : &COUNT_ACCOUNTED_FINISH_PER_TENANT,
3988 12 : &WAIT_USECS_PER_TENANT,
3989 12 : &WAIT_COUNT_PER_TENANT,
3990 12 : ] {
3991 96 : for kind in KINDS {
3992 48 : let _ = m.remove_label_values(&[
3993 48 : kind,
3994 48 : &tenant_shard_id.tenant_id.to_string(),
3995 48 : &tenant_shard_id.shard_slug().to_string(),
3996 48 : ]);
3997 48 : }
3998 : }
3999 12 : }
4000 : }
4001 :
4002 : pub(crate) mod disk_usage_based_eviction {
4003 : use super::*;
4004 :
4005 : pub(crate) struct Metrics {
4006 : pub(crate) tenant_collection_time: Histogram,
4007 : pub(crate) tenant_layer_count: Histogram,
4008 : pub(crate) layers_collected: IntCounter,
4009 : pub(crate) layers_selected: IntCounter,
4010 : pub(crate) layers_evicted: IntCounter,
4011 : }
4012 :
4013 : impl Default for Metrics {
4014 0 : fn default() -> Self {
4015 0 : let tenant_collection_time = register_histogram!(
4016 0 : "pageserver_disk_usage_based_eviction_tenant_collection_seconds",
4017 0 : "Time spent collecting layers from a tenant -- not normalized by collected layer amount",
4018 0 : vec![0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0, 10.0]
4019 0 : )
4020 0 : .unwrap();
4021 0 :
4022 0 : let tenant_layer_count = register_histogram!(
4023 0 : "pageserver_disk_usage_based_eviction_tenant_collected_layers",
4024 0 : "Amount of layers gathered from a tenant",
4025 0 : vec![5.0, 50.0, 500.0, 5000.0, 50000.0]
4026 0 : )
4027 0 : .unwrap();
4028 0 :
4029 0 : let layers_collected = register_int_counter!(
4030 0 : "pageserver_disk_usage_based_eviction_collected_layers_total",
4031 0 : "Amount of layers collected"
4032 0 : )
4033 0 : .unwrap();
4034 0 :
4035 0 : let layers_selected = register_int_counter!(
4036 0 : "pageserver_disk_usage_based_eviction_select_layers_total",
4037 0 : "Amount of layers selected"
4038 0 : )
4039 0 : .unwrap();
4040 0 :
4041 0 : let layers_evicted = register_int_counter!(
4042 0 : "pageserver_disk_usage_based_eviction_evicted_layers_total",
4043 0 : "Amount of layers successfully evicted"
4044 0 : )
4045 0 : .unwrap();
4046 0 :
4047 0 : Self {
4048 0 : tenant_collection_time,
4049 0 : tenant_layer_count,
4050 0 : layers_collected,
4051 0 : layers_selected,
4052 0 : layers_evicted,
4053 0 : }
4054 0 : }
4055 : }
4056 :
4057 : pub(crate) static METRICS: Lazy<Metrics> = Lazy::new(Metrics::default);
4058 : }
4059 :
4060 406 : static TOKIO_EXECUTOR_THREAD_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
4061 406 : register_uint_gauge_vec!(
4062 406 : "pageserver_tokio_executor_thread_configured_count",
4063 406 : "Total number of configued tokio executor threads in the process.
4064 406 : The `setup` label denotes whether we're running with multiple runtimes or a single runtime.",
4065 406 : &["setup"],
4066 406 : )
4067 406 : .unwrap()
4068 406 : });
4069 :
4070 406 : pub(crate) fn set_tokio_runtime_setup(setup: &str, num_threads: NonZeroUsize) {
4071 : static SERIALIZE: std::sync::Mutex<()> = std::sync::Mutex::new(());
4072 406 : let _guard = SERIALIZE.lock().unwrap();
4073 406 : TOKIO_EXECUTOR_THREAD_COUNT.reset();
4074 406 : TOKIO_EXECUTOR_THREAD_COUNT
4075 406 : .get_metric_with_label_values(&[setup])
4076 406 : .unwrap()
4077 406 : .set(u64::try_from(num_threads.get()).unwrap());
4078 406 : }
4079 :
4080 0 : pub fn preinitialize_metrics(conf: &'static PageServerConf) {
4081 0 : set_page_service_config_max_batch_size(&conf.page_service_pipelining);
4082 0 :
4083 0 : // Python tests need these and on some we do alerting.
4084 0 : //
4085 0 : // FIXME(4813): make it so that we have no top level metrics as this fn will easily fall out of
4086 0 : // order:
4087 0 : // - global metrics reside in a Lazy<PageserverMetrics>
4088 0 : // - access via crate::metrics::PS_METRICS.some_metric.inc()
4089 0 : // - could move the statics into TimelineMetrics::new()?
4090 0 :
4091 0 : // counters
4092 0 : [
4093 0 : &UNEXPECTED_ONDEMAND_DOWNLOADS,
4094 0 : &WALRECEIVER_STARTED_CONNECTIONS,
4095 0 : &WALRECEIVER_BROKER_UPDATES,
4096 0 : &WALRECEIVER_CANDIDATES_ADDED,
4097 0 : &WALRECEIVER_CANDIDATES_REMOVED,
4098 0 : &tokio_epoll_uring::THREAD_LOCAL_LAUNCH_FAILURES,
4099 0 : &tokio_epoll_uring::THREAD_LOCAL_LAUNCH_SUCCESSES,
4100 0 : &REMOTE_ONDEMAND_DOWNLOADED_LAYERS,
4101 0 : &REMOTE_ONDEMAND_DOWNLOADED_BYTES,
4102 0 : &CIRCUIT_BREAKERS_BROKEN,
4103 0 : &CIRCUIT_BREAKERS_UNBROKEN,
4104 0 : &PAGE_SERVICE_SMGR_FLUSH_INPROGRESS_MICROS_GLOBAL,
4105 0 : ]
4106 0 : .into_iter()
4107 0 : .for_each(|c| {
4108 0 : Lazy::force(c);
4109 0 : });
4110 0 :
4111 0 : // Deletion queue stats
4112 0 : Lazy::force(&DELETION_QUEUE);
4113 0 :
4114 0 : // Tenant stats
4115 0 : Lazy::force(&TENANT);
4116 0 :
4117 0 : // Tenant manager stats
4118 0 : Lazy::force(&TENANT_MANAGER);
4119 0 :
4120 0 : Lazy::force(&crate::tenant::storage_layer::layer::LAYER_IMPL_METRICS);
4121 0 : Lazy::force(&disk_usage_based_eviction::METRICS);
4122 :
4123 0 : for state_name in pageserver_api::models::TenantState::VARIANTS {
4124 0 : // initialize the metric for all gauges, otherwise the time series might seemingly show
4125 0 : // values from last restart.
4126 0 : TENANT_STATE_METRIC.with_label_values(&[state_name]).set(0);
4127 0 : }
4128 :
4129 : // countervecs
4130 0 : [
4131 0 : &BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT,
4132 0 : &SMGR_QUERY_STARTED_GLOBAL,
4133 0 : ]
4134 0 : .into_iter()
4135 0 : .for_each(|c| {
4136 0 : Lazy::force(c);
4137 0 : });
4138 0 :
4139 0 : // gauges
4140 0 : WALRECEIVER_ACTIVE_MANAGERS.get();
4141 0 :
4142 0 : // histograms
4143 0 : [
4144 0 : &LAYERS_PER_READ_GLOBAL,
4145 0 : &LAYERS_PER_READ_BATCH_GLOBAL,
4146 0 : &LAYERS_PER_READ_AMORTIZED_GLOBAL,
4147 0 : &DELTAS_PER_READ_GLOBAL,
4148 0 : &WAIT_LSN_TIME,
4149 0 : &WAL_REDO_TIME,
4150 0 : &WAL_REDO_RECORDS_HISTOGRAM,
4151 0 : &WAL_REDO_BYTES_HISTOGRAM,
4152 0 : &WAL_REDO_PROCESS_LAUNCH_DURATION_HISTOGRAM,
4153 0 : &PAGE_SERVICE_BATCH_SIZE_GLOBAL,
4154 0 : &PAGE_SERVICE_SMGR_BATCH_WAIT_TIME_GLOBAL,
4155 0 : ]
4156 0 : .into_iter()
4157 0 : .for_each(|h| {
4158 0 : Lazy::force(h);
4159 0 : });
4160 0 :
4161 0 : // Custom
4162 0 : Lazy::force(&BASEBACKUP_QUERY_TIME);
4163 0 : Lazy::force(&COMPUTE_COMMANDS_COUNTERS);
4164 0 : Lazy::force(&tokio_epoll_uring::THREAD_LOCAL_METRICS_STORAGE);
4165 0 :
4166 0 : tenant_throttling::preinitialize_global_metrics();
4167 0 : }
|
