Line data Source code
1 : use enum_map::EnumMap;
2 : use metrics::{
3 : register_counter_vec, register_gauge_vec, register_histogram, register_histogram_vec,
4 : register_int_counter, register_int_counter_pair_vec, register_int_counter_vec,
5 : register_int_gauge, register_int_gauge_vec, register_uint_gauge, register_uint_gauge_vec,
6 : Counter, CounterVec, GaugeVec, Histogram, HistogramVec, IntCounter, IntCounterPair,
7 : IntCounterPairVec, IntCounterVec, IntGauge, IntGaugeVec, UIntGauge, UIntGaugeVec,
8 : };
9 : use once_cell::sync::Lazy;
10 : use pageserver_api::shard::TenantShardId;
11 : use strum::{EnumCount, IntoEnumIterator, VariantNames};
12 : use strum_macros::{EnumVariantNames, IntoStaticStr};
13 : use tracing::warn;
14 : use utils::id::TimelineId;
15 :
16 : /// Prometheus histogram buckets (in seconds) for operations in the critical
17 : /// path. In other words, operations that directly affect that latency of user
18 : /// queries.
19 : ///
20 : /// The buckets capture the majority of latencies in the microsecond and
21 : /// millisecond range but also extend far enough up to distinguish "bad" from
22 : /// "really bad".
23 : const CRITICAL_OP_BUCKETS: &[f64] = &[
24 : 0.000_001, 0.000_010, 0.000_100, // 1 us, 10 us, 100 us
25 : 0.001_000, 0.010_000, 0.100_000, // 1 ms, 10 ms, 100 ms
26 : 1.0, 10.0, 100.0, // 1 s, 10 s, 100 s
27 : ];
28 :
29 : // Metrics collected on operations on the storage repository.
30 2198 : #[derive(Debug, EnumVariantNames, IntoStaticStr)]
31 : #[strum(serialize_all = "kebab_case")]
32 : pub(crate) enum StorageTimeOperation {
33 : #[strum(serialize = "layer flush")]
34 : LayerFlush,
35 :
36 : #[strum(serialize = "compact")]
37 : Compact,
38 :
39 : #[strum(serialize = "create images")]
40 : CreateImages,
41 :
42 : #[strum(serialize = "logical size")]
43 : LogicalSize,
44 :
45 : #[strum(serialize = "imitate logical size")]
46 : ImitateLogicalSize,
47 :
48 : #[strum(serialize = "load layer map")]
49 : LoadLayerMap,
50 :
51 : #[strum(serialize = "gc")]
52 : Gc,
53 :
54 : #[strum(serialize = "create tenant")]
55 : CreateTenant,
56 : }
57 :
58 96 : pub(crate) static STORAGE_TIME_SUM_PER_TIMELINE: Lazy<CounterVec> = Lazy::new(|| {
59 96 : register_counter_vec!(
60 96 : "pageserver_storage_operations_seconds_sum",
61 96 : "Total time spent on storage operations with operation, tenant and timeline dimensions",
62 96 : &["operation", "tenant_id", "shard_id", "timeline_id"],
63 96 : )
64 96 : .expect("failed to define a metric")
65 96 : });
66 :
67 96 : pub(crate) static STORAGE_TIME_COUNT_PER_TIMELINE: Lazy<IntCounterVec> = Lazy::new(|| {
68 96 : register_int_counter_vec!(
69 96 : "pageserver_storage_operations_seconds_count",
70 96 : "Count of storage operations with operation, tenant and timeline dimensions",
71 96 : &["operation", "tenant_id", "shard_id", "timeline_id"],
72 96 : )
73 96 : .expect("failed to define a metric")
74 96 : });
75 :
76 : // Buckets for background operations like compaction, GC, size calculation
77 : const STORAGE_OP_BUCKETS: &[f64] = &[0.010, 0.100, 1.0, 10.0, 100.0, 1000.0];
78 :
79 96 : pub(crate) static STORAGE_TIME_GLOBAL: Lazy<HistogramVec> = Lazy::new(|| {
80 96 : register_histogram_vec!(
81 96 : "pageserver_storage_operations_seconds_global",
82 96 : "Time spent on storage operations",
83 96 : &["operation"],
84 96 : STORAGE_OP_BUCKETS.into(),
85 96 : )
86 96 : .expect("failed to define a metric")
87 96 : });
88 :
89 94 : pub(crate) static READ_NUM_FS_LAYERS: Lazy<Histogram> = Lazy::new(|| {
90 94 : register_histogram!(
91 94 : "pageserver_read_num_fs_layers",
92 94 : "Number of persistent layers accessed for processing a read request, including those in the cache",
93 94 : vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 10.0, 20.0, 50.0, 100.0],
94 94 : )
95 94 : .expect("failed to define a metric")
96 94 : });
97 :
98 : // Metrics collected on operations on the storage repository.
99 :
100 : pub(crate) struct ReconstructTimeMetrics {
101 : ok: Histogram,
102 : err: Histogram,
103 : }
104 :
105 94 : pub(crate) static RECONSTRUCT_TIME: Lazy<ReconstructTimeMetrics> = Lazy::new(|| {
106 94 : let inner = register_histogram_vec!(
107 94 : "pageserver_getpage_reconstruct_seconds",
108 94 : "Time spent in reconstruct_value (reconstruct a page from deltas)",
109 94 : &["result"],
110 94 : CRITICAL_OP_BUCKETS.into(),
111 94 : )
112 94 : .expect("failed to define a metric");
113 94 : ReconstructTimeMetrics {
114 94 : ok: inner.get_metric_with_label_values(&["ok"]).unwrap(),
115 94 : err: inner.get_metric_with_label_values(&["err"]).unwrap(),
116 94 : }
117 94 : });
118 :
119 : impl ReconstructTimeMetrics {
120 502663 : pub(crate) fn for_result<T, E>(&self, result: &Result<T, E>) -> &Histogram {
121 502663 : match result {
122 502663 : Ok(_) => &self.ok,
123 0 : Err(_) => &self.err,
124 : }
125 502663 : }
126 : }
127 :
128 0 : pub(crate) static MATERIALIZED_PAGE_CACHE_HIT_DIRECT: Lazy<IntCounter> = Lazy::new(|| {
129 0 : register_int_counter!(
130 0 : "pageserver_materialized_cache_hits_direct_total",
131 0 : "Number of cache hits from materialized page cache without redo",
132 0 : )
133 0 : .expect("failed to define a metric")
134 0 : });
135 :
136 94 : pub(crate) static GET_RECONSTRUCT_DATA_TIME: Lazy<Histogram> = Lazy::new(|| {
137 94 : register_histogram!(
138 94 : "pageserver_getpage_get_reconstruct_data_seconds",
139 94 : "Time spent in get_reconstruct_value_data",
140 94 : CRITICAL_OP_BUCKETS.into(),
141 94 : )
142 94 : .expect("failed to define a metric")
143 94 : });
144 :
145 0 : pub(crate) static MATERIALIZED_PAGE_CACHE_HIT: Lazy<IntCounter> = Lazy::new(|| {
146 0 : register_int_counter!(
147 0 : "pageserver_materialized_cache_hits_total",
148 0 : "Number of cache hits from materialized page cache",
149 0 : )
150 0 : .expect("failed to define a metric")
151 0 : });
152 :
153 : pub(crate) struct GetVectoredLatency {
154 : map: EnumMap<TaskKind, Option<Histogram>>,
155 : }
156 :
157 : impl GetVectoredLatency {
158 : // Only these task types perform vectored gets. Filter all other tasks out to reduce total
159 : // cardinality of the metric.
160 : const TRACKED_TASK_KINDS: [TaskKind; 2] = [TaskKind::Compaction, TaskKind::PageRequestHandler];
161 :
162 544 : pub(crate) fn for_task_kind(&self, task_kind: TaskKind) -> Option<&Histogram> {
163 544 : self.map[task_kind].as_ref()
164 544 : }
165 : }
166 :
167 90 : pub(crate) static GET_VECTORED_LATENCY: Lazy<GetVectoredLatency> = Lazy::new(|| {
168 90 : let inner = register_histogram_vec!(
169 90 : "pageserver_get_vectored_seconds",
170 90 : "Time spent in get_vectored, excluding time spent in timeline_get_throttle.",
171 90 : &["task_kind"],
172 90 : CRITICAL_OP_BUCKETS.into(),
173 90 : )
174 90 : .expect("failed to define a metric");
175 90 :
176 90 : GetVectoredLatency {
177 2340 : map: EnumMap::from_array(std::array::from_fn(|task_kind_idx| {
178 2340 : let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind_idx);
179 2340 :
180 2340 : if GetVectoredLatency::TRACKED_TASK_KINDS.contains(&task_kind) {
181 180 : let task_kind = task_kind.into();
182 180 : Some(inner.with_label_values(&[task_kind]))
183 : } else {
184 2160 : None
185 : }
186 2340 : })),
187 90 : }
188 90 : });
189 :
190 : pub(crate) struct PageCacheMetricsForTaskKind {
191 : pub read_accesses_materialized_page: IntCounter,
192 : pub read_accesses_immutable: IntCounter,
193 :
194 : pub read_hits_immutable: IntCounter,
195 : pub read_hits_materialized_page_exact: IntCounter,
196 : pub read_hits_materialized_page_older_lsn: IntCounter,
197 : }
198 :
199 : pub(crate) struct PageCacheMetrics {
200 : map: EnumMap<TaskKind, EnumMap<PageContentKind, PageCacheMetricsForTaskKind>>,
201 : }
202 :
203 96 : static PAGE_CACHE_READ_HITS: Lazy<IntCounterVec> = Lazy::new(|| {
204 96 : register_int_counter_vec!(
205 96 : "pageserver_page_cache_read_hits_total",
206 96 : "Number of read accesses to the page cache that hit",
207 96 : &["task_kind", "key_kind", "content_kind", "hit_kind"]
208 96 : )
209 96 : .expect("failed to define a metric")
210 96 : });
211 :
212 96 : static PAGE_CACHE_READ_ACCESSES: Lazy<IntCounterVec> = Lazy::new(|| {
213 96 : register_int_counter_vec!(
214 96 : "pageserver_page_cache_read_accesses_total",
215 96 : "Number of read accesses to the page cache",
216 96 : &["task_kind", "key_kind", "content_kind"]
217 96 : )
218 96 : .expect("failed to define a metric")
219 96 : });
220 :
221 96 : pub(crate) static PAGE_CACHE: Lazy<PageCacheMetrics> = Lazy::new(|| PageCacheMetrics {
222 2496 : map: EnumMap::from_array(std::array::from_fn(|task_kind| {
223 2496 : let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind);
224 2496 : let task_kind: &'static str = task_kind.into();
225 14976 : EnumMap::from_array(std::array::from_fn(|content_kind| {
226 14976 : let content_kind = <PageContentKind as enum_map::Enum>::from_usize(content_kind);
227 14976 : let content_kind: &'static str = content_kind.into();
228 14976 : PageCacheMetricsForTaskKind {
229 14976 : read_accesses_materialized_page: {
230 14976 : PAGE_CACHE_READ_ACCESSES
231 14976 : .get_metric_with_label_values(&[
232 14976 : task_kind,
233 14976 : "materialized_page",
234 14976 : content_kind,
235 14976 : ])
236 14976 : .unwrap()
237 14976 : },
238 14976 :
239 14976 : read_accesses_immutable: {
240 14976 : PAGE_CACHE_READ_ACCESSES
241 14976 : .get_metric_with_label_values(&[task_kind, "immutable", content_kind])
242 14976 : .unwrap()
243 14976 : },
244 14976 :
245 14976 : read_hits_immutable: {
246 14976 : PAGE_CACHE_READ_HITS
247 14976 : .get_metric_with_label_values(&[task_kind, "immutable", content_kind, "-"])
248 14976 : .unwrap()
249 14976 : },
250 14976 :
251 14976 : read_hits_materialized_page_exact: {
252 14976 : PAGE_CACHE_READ_HITS
253 14976 : .get_metric_with_label_values(&[
254 14976 : task_kind,
255 14976 : "materialized_page",
256 14976 : content_kind,
257 14976 : "exact",
258 14976 : ])
259 14976 : .unwrap()
260 14976 : },
261 14976 :
262 14976 : read_hits_materialized_page_older_lsn: {
263 14976 : PAGE_CACHE_READ_HITS
264 14976 : .get_metric_with_label_values(&[
265 14976 : task_kind,
266 14976 : "materialized_page",
267 14976 : content_kind,
268 14976 : "older_lsn",
269 14976 : ])
270 14976 : .unwrap()
271 14976 : },
272 14976 : }
273 14976 : }))
274 2496 : })),
275 96 : });
276 :
277 : impl PageCacheMetrics {
278 14339009 : pub(crate) fn for_ctx(&self, ctx: &RequestContext) -> &PageCacheMetricsForTaskKind {
279 14339009 : &self.map[ctx.task_kind()][ctx.page_content_kind()]
280 14339009 : }
281 : }
282 :
283 : pub(crate) struct PageCacheSizeMetrics {
284 : pub max_bytes: UIntGauge,
285 :
286 : pub current_bytes_immutable: UIntGauge,
287 : pub current_bytes_materialized_page: UIntGauge,
288 : }
289 :
290 96 : static PAGE_CACHE_SIZE_CURRENT_BYTES: Lazy<UIntGaugeVec> = Lazy::new(|| {
291 96 : register_uint_gauge_vec!(
292 96 : "pageserver_page_cache_size_current_bytes",
293 96 : "Current size of the page cache in bytes, by key kind",
294 96 : &["key_kind"]
295 96 : )
296 96 : .expect("failed to define a metric")
297 96 : });
298 :
299 : pub(crate) static PAGE_CACHE_SIZE: Lazy<PageCacheSizeMetrics> =
300 96 : Lazy::new(|| PageCacheSizeMetrics {
301 96 : max_bytes: {
302 96 : register_uint_gauge!(
303 96 : "pageserver_page_cache_size_max_bytes",
304 96 : "Maximum size of the page cache in bytes"
305 96 : )
306 96 : .expect("failed to define a metric")
307 96 : },
308 96 : current_bytes_immutable: {
309 96 : PAGE_CACHE_SIZE_CURRENT_BYTES
310 96 : .get_metric_with_label_values(&["immutable"])
311 96 : .unwrap()
312 96 : },
313 96 : current_bytes_materialized_page: {
314 96 : PAGE_CACHE_SIZE_CURRENT_BYTES
315 96 : .get_metric_with_label_values(&["materialized_page"])
316 96 : .unwrap()
317 96 : },
318 96 : });
319 :
320 : pub(crate) mod page_cache_eviction_metrics {
321 : use std::num::NonZeroUsize;
322 :
323 : use metrics::{register_int_counter_vec, IntCounter, IntCounterVec};
324 : use once_cell::sync::Lazy;
325 :
326 : #[derive(Clone, Copy)]
327 : pub(crate) enum Outcome {
328 : FoundSlotUnused { iters: NonZeroUsize },
329 : FoundSlotEvicted { iters: NonZeroUsize },
330 : ItersExceeded { iters: NonZeroUsize },
331 : }
332 :
333 38 : static ITERS_TOTAL_VEC: Lazy<IntCounterVec> = Lazy::new(|| {
334 38 : register_int_counter_vec!(
335 38 : "pageserver_page_cache_find_victim_iters_total",
336 38 : "Counter for the number of iterations in the find_victim loop",
337 38 : &["outcome"],
338 38 : )
339 38 : .expect("failed to define a metric")
340 38 : });
341 :
342 38 : static CALLS_VEC: Lazy<IntCounterVec> = Lazy::new(|| {
343 38 : register_int_counter_vec!(
344 38 : "pageserver_page_cache_find_victim_calls",
345 38 : "Incremented at the end of each find_victim() call.\
346 38 : Filter by outcome to get e.g., eviction rate.",
347 38 : &["outcome"]
348 38 : )
349 38 : .unwrap()
350 38 : });
351 :
352 155909 : pub(crate) fn observe(outcome: Outcome) {
353 155909 : macro_rules! dry {
354 155909 : ($label:literal, $iters:expr) => {{
355 155909 : static LABEL: &'static str = $label;
356 155909 : static ITERS_TOTAL: Lazy<IntCounter> =
357 155909 : Lazy::new(|| ITERS_TOTAL_VEC.with_label_values(&[LABEL]));
358 155909 : static CALLS: Lazy<IntCounter> =
359 155909 : Lazy::new(|| CALLS_VEC.with_label_values(&[LABEL]));
360 155909 : ITERS_TOTAL.inc_by(($iters.get()) as u64);
361 155909 : CALLS.inc();
362 155909 : }};
363 155909 : }
364 155909 : match outcome {
365 1120 : Outcome::FoundSlotUnused { iters } => dry!("found_empty", iters),
366 154789 : Outcome::FoundSlotEvicted { iters } => {
367 154789 : dry!("found_evicted", iters)
368 : }
369 0 : Outcome::ItersExceeded { iters } => {
370 0 : dry!("err_iters_exceeded", iters);
371 0 : super::page_cache_errors_inc(super::PageCacheErrorKind::EvictIterLimit);
372 0 : }
373 : }
374 155909 : }
375 : }
376 :
377 0 : static PAGE_CACHE_ERRORS: Lazy<IntCounterVec> = Lazy::new(|| {
378 0 : register_int_counter_vec!(
379 0 : "page_cache_errors_total",
380 0 : "Number of timeouts while acquiring a pinned slot in the page cache",
381 0 : &["error_kind"]
382 0 : )
383 0 : .expect("failed to define a metric")
384 0 : });
385 :
386 0 : #[derive(IntoStaticStr)]
387 : #[strum(serialize_all = "kebab_case")]
388 : pub(crate) enum PageCacheErrorKind {
389 : AcquirePinnedSlotTimeout,
390 : EvictIterLimit,
391 : }
392 :
393 0 : pub(crate) fn page_cache_errors_inc(error_kind: PageCacheErrorKind) {
394 0 : PAGE_CACHE_ERRORS
395 0 : .get_metric_with_label_values(&[error_kind.into()])
396 0 : .unwrap()
397 0 : .inc();
398 0 : }
399 :
400 12 : pub(crate) static WAIT_LSN_TIME: Lazy<Histogram> = Lazy::new(|| {
401 12 : register_histogram!(
402 12 : "pageserver_wait_lsn_seconds",
403 12 : "Time spent waiting for WAL to arrive",
404 12 : CRITICAL_OP_BUCKETS.into(),
405 12 : )
406 12 : .expect("failed to define a metric")
407 12 : });
408 :
409 96 : static LAST_RECORD_LSN: Lazy<IntGaugeVec> = Lazy::new(|| {
410 96 : register_int_gauge_vec!(
411 96 : "pageserver_last_record_lsn",
412 96 : "Last record LSN grouped by timeline",
413 96 : &["tenant_id", "shard_id", "timeline_id"]
414 96 : )
415 96 : .expect("failed to define a metric")
416 96 : });
417 :
418 96 : static RESIDENT_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
419 96 : register_uint_gauge_vec!(
420 96 : "pageserver_resident_physical_size",
421 96 : "The size of the layer files present in the pageserver's filesystem.",
422 96 : &["tenant_id", "shard_id", "timeline_id"]
423 96 : )
424 96 : .expect("failed to define a metric")
425 96 : });
426 :
427 94 : pub(crate) static RESIDENT_PHYSICAL_SIZE_GLOBAL: Lazy<UIntGauge> = Lazy::new(|| {
428 94 : register_uint_gauge!(
429 94 : "pageserver_resident_physical_size_global",
430 94 : "Like `pageserver_resident_physical_size`, but without tenant/timeline dimensions."
431 94 : )
432 94 : .expect("failed to define a metric")
433 94 : });
434 :
435 96 : static REMOTE_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
436 96 : register_uint_gauge_vec!(
437 96 : "pageserver_remote_physical_size",
438 96 : "The size of the layer files present in the remote storage that are listed in the remote index_part.json.",
439 96 : // Corollary: If any files are missing from the index part, they won't be included here.
440 96 : &["tenant_id", "shard_id", "timeline_id"]
441 96 : )
442 96 : .expect("failed to define a metric")
443 96 : });
444 :
445 96 : static REMOTE_PHYSICAL_SIZE_GLOBAL: Lazy<UIntGauge> = Lazy::new(|| {
446 96 : register_uint_gauge!(
447 96 : "pageserver_remote_physical_size_global",
448 96 : "Like `pageserver_remote_physical_size`, but without tenant/timeline dimensions."
449 96 : )
450 96 : .expect("failed to define a metric")
451 96 : });
452 :
453 4 : pub(crate) static REMOTE_ONDEMAND_DOWNLOADED_LAYERS: Lazy<IntCounter> = Lazy::new(|| {
454 4 : register_int_counter!(
455 4 : "pageserver_remote_ondemand_downloaded_layers_total",
456 4 : "Total on-demand downloaded layers"
457 4 : )
458 4 : .unwrap()
459 4 : });
460 :
461 4 : pub(crate) static REMOTE_ONDEMAND_DOWNLOADED_BYTES: Lazy<IntCounter> = Lazy::new(|| {
462 4 : register_int_counter!(
463 4 : "pageserver_remote_ondemand_downloaded_bytes_total",
464 4 : "Total bytes of layers on-demand downloaded",
465 4 : )
466 4 : .unwrap()
467 4 : });
468 :
469 96 : static CURRENT_LOGICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
470 96 : register_uint_gauge_vec!(
471 96 : "pageserver_current_logical_size",
472 96 : "Current logical size grouped by timeline",
473 96 : &["tenant_id", "shard_id", "timeline_id"]
474 96 : )
475 96 : .expect("failed to define current logical size metric")
476 96 : });
477 :
478 : pub(crate) mod initial_logical_size {
479 : use metrics::{register_int_counter, register_int_counter_vec, IntCounter, IntCounterVec};
480 : use once_cell::sync::Lazy;
481 :
482 : pub(crate) struct StartCalculation(IntCounterVec);
483 96 : pub(crate) static START_CALCULATION: Lazy<StartCalculation> = Lazy::new(|| {
484 96 : StartCalculation(
485 96 : register_int_counter_vec!(
486 96 : "pageserver_initial_logical_size_start_calculation",
487 96 : "Incremented each time we start an initial logical size calculation attempt. \
488 96 : The `circumstances` label provides some additional details.",
489 96 : &["attempt", "circumstances"]
490 96 : )
491 96 : .unwrap(),
492 96 : )
493 96 : });
494 :
495 : struct DropCalculation {
496 : first: IntCounter,
497 : retry: IntCounter,
498 : }
499 :
500 96 : static DROP_CALCULATION: Lazy<DropCalculation> = Lazy::new(|| {
501 96 : let vec = register_int_counter_vec!(
502 96 : "pageserver_initial_logical_size_drop_calculation",
503 96 : "Incremented each time we abort a started size calculation attmpt.",
504 96 : &["attempt"]
505 96 : )
506 96 : .unwrap();
507 96 : DropCalculation {
508 96 : first: vec.with_label_values(&["first"]),
509 96 : retry: vec.with_label_values(&["retry"]),
510 96 : }
511 96 : });
512 :
513 : pub(crate) struct Calculated {
514 : pub(crate) births: IntCounter,
515 : pub(crate) deaths: IntCounter,
516 : }
517 :
518 96 : pub(crate) static CALCULATED: Lazy<Calculated> = Lazy::new(|| Calculated {
519 96 : births: register_int_counter!(
520 96 : "pageserver_initial_logical_size_finish_calculation",
521 96 : "Incremented every time we finish calculation of initial logical size.\
522 96 : If everything is working well, this should happen at most once per Timeline object."
523 96 : )
524 96 : .unwrap(),
525 96 : deaths: register_int_counter!(
526 96 : "pageserver_initial_logical_size_drop_finished_calculation",
527 96 : "Incremented when we drop a finished initial logical size calculation result.\
528 96 : Mainly useful to turn pageserver_initial_logical_size_finish_calculation into a gauge."
529 96 : )
530 96 : .unwrap(),
531 96 : });
532 :
533 : pub(crate) struct OngoingCalculationGuard {
534 : inc_drop_calculation: Option<IntCounter>,
535 : }
536 :
537 96 : #[derive(strum_macros::IntoStaticStr)]
538 : pub(crate) enum StartCircumstances {
539 : EmptyInitial,
540 : SkippedConcurrencyLimiter,
541 : AfterBackgroundTasksRateLimit,
542 : }
543 :
544 : impl StartCalculation {
545 96 : pub(crate) fn first(&self, circumstances: StartCircumstances) -> OngoingCalculationGuard {
546 96 : let circumstances_label: &'static str = circumstances.into();
547 96 : self.0
548 96 : .with_label_values(&["first", circumstances_label])
549 96 : .inc();
550 96 : OngoingCalculationGuard {
551 96 : inc_drop_calculation: Some(DROP_CALCULATION.first.clone()),
552 96 : }
553 96 : }
554 0 : pub(crate) fn retry(&self, circumstances: StartCircumstances) -> OngoingCalculationGuard {
555 0 : let circumstances_label: &'static str = circumstances.into();
556 0 : self.0
557 0 : .with_label_values(&["retry", circumstances_label])
558 0 : .inc();
559 0 : OngoingCalculationGuard {
560 0 : inc_drop_calculation: Some(DROP_CALCULATION.retry.clone()),
561 0 : }
562 0 : }
563 : }
564 :
565 : impl Drop for OngoingCalculationGuard {
566 96 : fn drop(&mut self) {
567 96 : if let Some(counter) = self.inc_drop_calculation.take() {
568 0 : counter.inc();
569 96 : }
570 96 : }
571 : }
572 :
573 : impl OngoingCalculationGuard {
574 96 : pub(crate) fn calculation_result_saved(mut self) -> FinishedCalculationGuard {
575 96 : drop(self.inc_drop_calculation.take());
576 96 : CALCULATED.births.inc();
577 96 : FinishedCalculationGuard {
578 96 : inc_on_drop: CALCULATED.deaths.clone(),
579 96 : }
580 96 : }
581 : }
582 :
583 : pub(crate) struct FinishedCalculationGuard {
584 : inc_on_drop: IntCounter,
585 : }
586 :
587 : impl Drop for FinishedCalculationGuard {
588 6 : fn drop(&mut self) {
589 6 : self.inc_on_drop.inc();
590 6 : }
591 : }
592 :
593 : // context: https://github.com/neondatabase/neon/issues/5963
594 : pub(crate) static TIMELINES_WHERE_WALRECEIVER_GOT_APPROXIMATE_SIZE: Lazy<IntCounter> =
595 0 : Lazy::new(|| {
596 0 : register_int_counter!(
597 0 : "pageserver_initial_logical_size_timelines_where_walreceiver_got_approximate_size",
598 0 : "Counter for the following event: walreceiver calls\
599 0 : Timeline::get_current_logical_size() and it returns `Approximate` for the first time."
600 0 : )
601 0 : .unwrap()
602 0 : });
603 : }
604 :
605 0 : static DIRECTORY_ENTRIES_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
606 0 : register_uint_gauge_vec!(
607 0 : "pageserver_directory_entries_count",
608 0 : "Sum of the entries in pageserver-stored directory listings",
609 0 : &["tenant_id", "shard_id", "timeline_id"]
610 0 : )
611 0 : .expect("failed to define a metric")
612 0 : });
613 :
614 95 : pub(crate) static TENANT_STATE_METRIC: Lazy<UIntGaugeVec> = Lazy::new(|| {
615 95 : register_uint_gauge_vec!(
616 95 : "pageserver_tenant_states_count",
617 95 : "Count of tenants per state",
618 95 : &["state"]
619 95 : )
620 95 : .expect("Failed to register pageserver_tenant_states_count metric")
621 95 : });
622 :
623 : /// A set of broken tenants.
624 : ///
625 : /// These are expected to be so rare that a set is fine. Set as in a new timeseries per each broken
626 : /// tenant.
627 4 : pub(crate) static BROKEN_TENANTS_SET: Lazy<UIntGaugeVec> = Lazy::new(|| {
628 4 : register_uint_gauge_vec!(
629 4 : "pageserver_broken_tenants_count",
630 4 : "Set of broken tenants",
631 4 : &["tenant_id", "shard_id"]
632 4 : )
633 4 : .expect("Failed to register pageserver_tenant_states_count metric")
634 4 : });
635 :
636 6 : pub(crate) static TENANT_SYNTHETIC_SIZE_METRIC: Lazy<UIntGaugeVec> = Lazy::new(|| {
637 6 : register_uint_gauge_vec!(
638 6 : "pageserver_tenant_synthetic_cached_size_bytes",
639 6 : "Synthetic size of each tenant in bytes",
640 6 : &["tenant_id"]
641 6 : )
642 6 : .expect("Failed to register pageserver_tenant_synthetic_cached_size_bytes metric")
643 6 : });
644 :
645 0 : pub(crate) static EVICTION_ITERATION_DURATION: Lazy<HistogramVec> = Lazy::new(|| {
646 0 : register_histogram_vec!(
647 0 : "pageserver_eviction_iteration_duration_seconds_global",
648 0 : "Time spent on a single eviction iteration",
649 0 : &["period_secs", "threshold_secs"],
650 0 : STORAGE_OP_BUCKETS.into(),
651 0 : )
652 0 : .expect("failed to define a metric")
653 0 : });
654 :
655 96 : static EVICTIONS: Lazy<IntCounterVec> = Lazy::new(|| {
656 96 : register_int_counter_vec!(
657 96 : "pageserver_evictions",
658 96 : "Number of layers evicted from the pageserver",
659 96 : &["tenant_id", "shard_id", "timeline_id"]
660 96 : )
661 96 : .expect("failed to define a metric")
662 96 : });
663 :
664 96 : static EVICTIONS_WITH_LOW_RESIDENCE_DURATION: Lazy<IntCounterVec> = Lazy::new(|| {
665 96 : register_int_counter_vec!(
666 96 : "pageserver_evictions_with_low_residence_duration",
667 96 : "If a layer is evicted that was resident for less than `low_threshold`, it is counted to this counter. \
668 96 : Residence duration is determined using the `residence_duration_data_source`.",
669 96 : &["tenant_id", "shard_id", "timeline_id", "residence_duration_data_source", "low_threshold_secs"]
670 96 : )
671 96 : .expect("failed to define a metric")
672 96 : });
673 :
674 0 : pub(crate) static UNEXPECTED_ONDEMAND_DOWNLOADS: Lazy<IntCounter> = Lazy::new(|| {
675 0 : register_int_counter!(
676 0 : "pageserver_unexpected_ondemand_downloads_count",
677 0 : "Number of unexpected on-demand downloads. \
678 0 : We log more context for each increment, so, forgo any labels in this metric.",
679 0 : )
680 0 : .expect("failed to define a metric")
681 0 : });
682 :
683 : /// How long did we take to start up? Broken down by labels to describe
684 : /// different phases of startup.
685 0 : pub static STARTUP_DURATION: Lazy<GaugeVec> = Lazy::new(|| {
686 0 : register_gauge_vec!(
687 0 : "pageserver_startup_duration_seconds",
688 0 : "Time taken by phases of pageserver startup, in seconds",
689 0 : &["phase"]
690 0 : )
691 0 : .expect("Failed to register pageserver_startup_duration_seconds metric")
692 0 : });
693 :
694 0 : pub static STARTUP_IS_LOADING: Lazy<UIntGauge> = Lazy::new(|| {
695 0 : register_uint_gauge!(
696 0 : "pageserver_startup_is_loading",
697 0 : "1 while in initial startup load of tenants, 0 at other times"
698 0 : )
699 0 : .expect("Failed to register pageserver_startup_is_loading")
700 0 : });
701 :
702 94 : pub(crate) static TIMELINE_EPHEMERAL_BYTES: Lazy<UIntGauge> = Lazy::new(|| {
703 94 : register_uint_gauge!(
704 94 : "pageserver_timeline_ephemeral_bytes",
705 94 : "Total number of bytes in ephemeral layers, summed for all timelines. Approximate, lazily updated."
706 94 : )
707 94 : .expect("Failed to register metric")
708 94 : });
709 :
710 : /// Metrics related to the lifecycle of a [`crate::tenant::Tenant`] object: things
711 : /// like how long it took to load.
712 : ///
713 : /// Note that these are process-global metrics, _not_ per-tenant metrics. Per-tenant
714 : /// metrics are rather expensive, and usually fine grained stuff makes more sense
715 : /// at a timeline level than tenant level.
716 : pub(crate) struct TenantMetrics {
717 : /// How long did tenants take to go from construction to active state?
718 : pub(crate) activation: Histogram,
719 : pub(crate) preload: Histogram,
720 : pub(crate) attach: Histogram,
721 :
722 : /// How many tenants are included in the initial startup of the pagesrever?
723 : pub(crate) startup_scheduled: IntCounter,
724 : pub(crate) startup_complete: IntCounter,
725 : }
726 :
727 0 : pub(crate) static TENANT: Lazy<TenantMetrics> = Lazy::new(|| {
728 0 : TenantMetrics {
729 0 : activation: register_histogram!(
730 0 : "pageserver_tenant_activation_seconds",
731 0 : "Time taken by tenants to activate, in seconds",
732 0 : CRITICAL_OP_BUCKETS.into()
733 0 : )
734 0 : .expect("Failed to register metric"),
735 0 : preload: register_histogram!(
736 0 : "pageserver_tenant_preload_seconds",
737 0 : "Time taken by tenants to load remote metadata on startup/attach, in seconds",
738 0 : CRITICAL_OP_BUCKETS.into()
739 0 : )
740 0 : .expect("Failed to register metric"),
741 0 : attach: register_histogram!(
742 0 : "pageserver_tenant_attach_seconds",
743 0 : "Time taken by tenants to intialize, after remote metadata is already loaded",
744 0 : CRITICAL_OP_BUCKETS.into()
745 0 : )
746 0 : .expect("Failed to register metric"),
747 0 : startup_scheduled: register_int_counter!(
748 0 : "pageserver_tenant_startup_scheduled",
749 0 : "Number of tenants included in pageserver startup (doesn't count tenants attached later)"
750 0 : ).expect("Failed to register metric"),
751 0 : startup_complete: register_int_counter!(
752 0 : "pageserver_tenant_startup_complete",
753 0 : "Number of tenants that have completed warm-up, or activated on-demand during initial startup: \
754 0 : should eventually reach `pageserver_tenant_startup_scheduled_total`. Does not include broken \
755 0 : tenants: such cases will lead to this metric never reaching the scheduled count."
756 0 : ).expect("Failed to register metric"),
757 0 : }
758 0 : });
759 :
760 : /// Each `Timeline`'s [`EVICTIONS_WITH_LOW_RESIDENCE_DURATION`] metric.
761 : #[derive(Debug)]
762 : pub(crate) struct EvictionsWithLowResidenceDuration {
763 : data_source: &'static str,
764 : threshold: Duration,
765 : counter: Option<IntCounter>,
766 : }
767 :
768 : pub(crate) struct EvictionsWithLowResidenceDurationBuilder {
769 : data_source: &'static str,
770 : threshold: Duration,
771 : }
772 :
773 : impl EvictionsWithLowResidenceDurationBuilder {
774 314 : pub fn new(data_source: &'static str, threshold: Duration) -> Self {
775 314 : Self {
776 314 : data_source,
777 314 : threshold,
778 314 : }
779 314 : }
780 :
781 314 : fn build(
782 314 : &self,
783 314 : tenant_id: &str,
784 314 : shard_id: &str,
785 314 : timeline_id: &str,
786 314 : ) -> EvictionsWithLowResidenceDuration {
787 314 : let counter = EVICTIONS_WITH_LOW_RESIDENCE_DURATION
788 314 : .get_metric_with_label_values(&[
789 314 : tenant_id,
790 314 : shard_id,
791 314 : timeline_id,
792 314 : self.data_source,
793 314 : &EvictionsWithLowResidenceDuration::threshold_label_value(self.threshold),
794 314 : ])
795 314 : .unwrap();
796 314 : EvictionsWithLowResidenceDuration {
797 314 : data_source: self.data_source,
798 314 : threshold: self.threshold,
799 314 : counter: Some(counter),
800 314 : }
801 314 : }
802 : }
803 :
804 : impl EvictionsWithLowResidenceDuration {
805 322 : fn threshold_label_value(threshold: Duration) -> String {
806 322 : format!("{}", threshold.as_secs())
807 322 : }
808 :
809 18 : pub fn observe(&self, observed_value: Duration) {
810 18 : if observed_value < self.threshold {
811 18 : self.counter
812 18 : .as_ref()
813 18 : .expect("nobody calls this function after `remove_from_vec`")
814 18 : .inc();
815 18 : }
816 18 : }
817 :
818 0 : pub fn change_threshold(
819 0 : &mut self,
820 0 : tenant_id: &str,
821 0 : shard_id: &str,
822 0 : timeline_id: &str,
823 0 : new_threshold: Duration,
824 0 : ) {
825 0 : if new_threshold == self.threshold {
826 0 : return;
827 0 : }
828 0 : let mut with_new = EvictionsWithLowResidenceDurationBuilder::new(
829 0 : self.data_source,
830 0 : new_threshold,
831 0 : )
832 0 : .build(tenant_id, shard_id, timeline_id);
833 0 : std::mem::swap(self, &mut with_new);
834 0 : with_new.remove(tenant_id, shard_id, timeline_id);
835 0 : }
836 :
837 : // This could be a `Drop` impl, but, we need the `tenant_id` and `timeline_id`.
838 8 : fn remove(&mut self, tenant_id: &str, shard_id: &str, timeline_id: &str) {
839 8 : let Some(_counter) = self.counter.take() else {
840 0 : return;
841 : };
842 :
843 8 : let threshold = Self::threshold_label_value(self.threshold);
844 8 :
845 8 : let removed = EVICTIONS_WITH_LOW_RESIDENCE_DURATION.remove_label_values(&[
846 8 : tenant_id,
847 8 : shard_id,
848 8 : timeline_id,
849 8 : self.data_source,
850 8 : &threshold,
851 8 : ]);
852 8 :
853 8 : match removed {
854 0 : Err(e) => {
855 0 : // this has been hit in staging as
856 0 : // <https://neondatabase.sentry.io/issues/4142396994/>, but we don't know how.
857 0 : // because we can be in the drop path already, don't risk:
858 0 : // - "double-panic => illegal instruction" or
859 0 : // - future "drop panick => abort"
860 0 : //
861 0 : // so just nag: (the error has the labels)
862 0 : tracing::warn!("failed to remove EvictionsWithLowResidenceDuration, it was already removed? {e:#?}");
863 : }
864 : Ok(()) => {
865 : // to help identify cases where we double-remove the same values, let's log all
866 : // deletions?
867 8 : tracing::info!("removed EvictionsWithLowResidenceDuration with {tenant_id}, {timeline_id}, {}, {threshold}", self.data_source);
868 : }
869 : }
870 8 : }
871 : }
872 :
873 : // Metrics collected on disk IO operations
874 : //
875 : // Roughly logarithmic scale.
876 : const STORAGE_IO_TIME_BUCKETS: &[f64] = &[
877 : 0.000030, // 30 usec
878 : 0.001000, // 1000 usec
879 : 0.030, // 30 ms
880 : 1.000, // 1000 ms
881 : 30.000, // 30000 ms
882 : ];
883 :
884 : /// VirtualFile fs operation variants.
885 : ///
886 : /// Operations:
887 : /// - open ([`std::fs::OpenOptions::open`])
888 : /// - close (dropping [`crate::virtual_file::VirtualFile`])
889 : /// - close-by-replace (close by replacement algorithm)
890 : /// - read (`read_at`)
891 : /// - write (`write_at`)
892 : /// - seek (modify internal position or file length query)
893 : /// - fsync ([`std::fs::File::sync_all`])
894 : /// - metadata ([`std::fs::File::metadata`])
895 : #[derive(
896 1044 : Debug, Clone, Copy, strum_macros::EnumCount, strum_macros::EnumIter, strum_macros::FromRepr,
897 : )]
898 : pub(crate) enum StorageIoOperation {
899 : Open,
900 : OpenAfterReplace,
901 : Close,
902 : CloseByReplace,
903 : Read,
904 : Write,
905 : Seek,
906 : Fsync,
907 : Metadata,
908 : }
909 :
910 : impl StorageIoOperation {
911 1044 : pub fn as_str(&self) -> &'static str {
912 1044 : match self {
913 116 : StorageIoOperation::Open => "open",
914 116 : StorageIoOperation::OpenAfterReplace => "open-after-replace",
915 116 : StorageIoOperation::Close => "close",
916 116 : StorageIoOperation::CloseByReplace => "close-by-replace",
917 116 : StorageIoOperation::Read => "read",
918 116 : StorageIoOperation::Write => "write",
919 116 : StorageIoOperation::Seek => "seek",
920 116 : StorageIoOperation::Fsync => "fsync",
921 116 : StorageIoOperation::Metadata => "metadata",
922 : }
923 1044 : }
924 : }
925 :
926 : /// Tracks time taken by fs operations near VirtualFile.
927 : #[derive(Debug)]
928 : pub(crate) struct StorageIoTime {
929 : metrics: [Histogram; StorageIoOperation::COUNT],
930 : }
931 :
932 : impl StorageIoTime {
933 116 : fn new() -> Self {
934 116 : let storage_io_histogram_vec = register_histogram_vec!(
935 116 : "pageserver_io_operations_seconds",
936 116 : "Time spent in IO operations",
937 116 : &["operation"],
938 116 : STORAGE_IO_TIME_BUCKETS.into()
939 116 : )
940 116 : .expect("failed to define a metric");
941 1044 : let metrics = std::array::from_fn(|i| {
942 1044 : let op = StorageIoOperation::from_repr(i).unwrap();
943 1044 : storage_io_histogram_vec
944 1044 : .get_metric_with_label_values(&[op.as_str()])
945 1044 : .unwrap()
946 1044 : });
947 116 : Self { metrics }
948 116 : }
949 :
950 866150 : pub(crate) fn get(&self, op: StorageIoOperation) -> &Histogram {
951 866150 : &self.metrics[op as usize]
952 866150 : }
953 : }
954 :
955 : pub(crate) static STORAGE_IO_TIME_METRIC: Lazy<StorageIoTime> = Lazy::new(StorageIoTime::new);
956 :
957 : const STORAGE_IO_SIZE_OPERATIONS: &[&str] = &["read", "write"];
958 :
959 : // Needed for the https://neonprod.grafana.net/d/5uK9tHL4k/picking-tenant-for-relocation?orgId=1
960 116 : pub(crate) static STORAGE_IO_SIZE: Lazy<IntGaugeVec> = Lazy::new(|| {
961 116 : register_int_gauge_vec!(
962 116 : "pageserver_io_operations_bytes_total",
963 116 : "Total amount of bytes read/written in IO operations",
964 116 : &["operation", "tenant_id", "shard_id", "timeline_id"]
965 116 : )
966 116 : .expect("failed to define a metric")
967 116 : });
968 :
969 : #[cfg(not(test))]
970 : pub(crate) mod virtual_file_descriptor_cache {
971 : use super::*;
972 :
973 0 : pub(crate) static SIZE_MAX: Lazy<UIntGauge> = Lazy::new(|| {
974 0 : register_uint_gauge!(
975 0 : "pageserver_virtual_file_descriptor_cache_size_max",
976 0 : "Maximum number of open file descriptors in the cache."
977 0 : )
978 0 : .unwrap()
979 0 : });
980 :
981 : // SIZE_CURRENT: derive it like so:
982 : // ```
983 : // sum (pageserver_io_operations_seconds_count{operation=~"^(open|open-after-replace)$")
984 : // -ignoring(operation)
985 : // sum(pageserver_io_operations_seconds_count{operation=~"^(close|close-by-replace)$"}
986 : // ```
987 : }
988 :
989 : #[cfg(not(test))]
990 : pub(crate) mod virtual_file_io_engine {
991 : use super::*;
992 :
993 0 : pub(crate) static KIND: Lazy<UIntGaugeVec> = Lazy::new(|| {
994 0 : register_uint_gauge_vec!(
995 0 : "pageserver_virtual_file_io_engine_kind",
996 0 : "The configured io engine for VirtualFile",
997 0 : &["kind"],
998 0 : )
999 0 : .unwrap()
1000 0 : });
1001 : }
1002 :
1003 : #[derive(Debug)]
1004 : struct GlobalAndPerTimelineHistogram {
1005 : global: Histogram,
1006 : per_tenant_timeline: Histogram,
1007 : }
1008 :
1009 : impl GlobalAndPerTimelineHistogram {
1010 10 : fn observe(&self, value: f64) {
1011 10 : self.global.observe(value);
1012 10 : self.per_tenant_timeline.observe(value);
1013 10 : }
1014 : }
1015 :
1016 : struct GlobalAndPerTimelineHistogramTimer<'a, 'c> {
1017 : h: &'a GlobalAndPerTimelineHistogram,
1018 : ctx: &'c RequestContext,
1019 : start: std::time::Instant,
1020 : op: SmgrQueryType,
1021 : }
1022 :
1023 : impl<'a, 'c> Drop for GlobalAndPerTimelineHistogramTimer<'a, 'c> {
1024 10 : fn drop(&mut self) {
1025 10 : let elapsed = self.start.elapsed();
1026 10 : let ex_throttled = self
1027 10 : .ctx
1028 10 : .micros_spent_throttled
1029 10 : .close_and_checked_sub_from(elapsed);
1030 10 : let ex_throttled = match ex_throttled {
1031 10 : Ok(res) => res,
1032 0 : Err(error) => {
1033 0 : use utils::rate_limit::RateLimit;
1034 0 : static LOGGED: Lazy<Mutex<enum_map::EnumMap<SmgrQueryType, RateLimit>>> =
1035 0 : Lazy::new(|| {
1036 0 : Mutex::new(enum_map::EnumMap::from_array(std::array::from_fn(|_| {
1037 0 : RateLimit::new(Duration::from_secs(10))
1038 0 : })))
1039 0 : });
1040 0 : let mut guard = LOGGED.lock().unwrap();
1041 0 : let rate_limit = &mut guard[self.op];
1042 0 : rate_limit.call(|| {
1043 0 : warn!(op=?self.op, error, "error deducting time spent throttled; this message is logged at a global rate limit");
1044 0 : });
1045 0 : elapsed
1046 : }
1047 : };
1048 10 : self.h.observe(ex_throttled.as_secs_f64());
1049 10 : }
1050 : }
1051 :
1052 : #[derive(
1053 : Debug,
1054 : Clone,
1055 : Copy,
1056 3290 : IntoStaticStr,
1057 : strum_macros::EnumCount,
1058 112 : strum_macros::EnumIter,
1059 1620 : strum_macros::FromRepr,
1060 : enum_map::Enum,
1061 : )]
1062 : #[strum(serialize_all = "snake_case")]
1063 : pub enum SmgrQueryType {
1064 : GetRelExists,
1065 : GetRelSize,
1066 : GetPageAtLsn,
1067 : GetDbSize,
1068 : GetSlruSegment,
1069 : }
1070 :
1071 : #[derive(Debug)]
1072 : pub(crate) struct SmgrQueryTimePerTimeline {
1073 : metrics: [GlobalAndPerTimelineHistogram; SmgrQueryType::COUNT],
1074 : }
1075 :
1076 98 : static SMGR_QUERY_TIME_PER_TENANT_TIMELINE: Lazy<HistogramVec> = Lazy::new(|| {
1077 98 : register_histogram_vec!(
1078 98 : "pageserver_smgr_query_seconds",
1079 98 : "Time spent on smgr query handling, aggegated by query type and tenant/timeline.",
1080 98 : &["smgr_query_type", "tenant_id", "shard_id", "timeline_id"],
1081 98 : CRITICAL_OP_BUCKETS.into(),
1082 98 : )
1083 98 : .expect("failed to define a metric")
1084 98 : });
1085 :
1086 98 : static SMGR_QUERY_TIME_GLOBAL_BUCKETS: Lazy<Vec<f64>> = Lazy::new(|| {
1087 98 : [
1088 98 : 1,
1089 98 : 10,
1090 98 : 20,
1091 98 : 40,
1092 98 : 60,
1093 98 : 80,
1094 98 : 100,
1095 98 : 200,
1096 98 : 300,
1097 98 : 400,
1098 98 : 500,
1099 98 : 600,
1100 98 : 700,
1101 98 : 800,
1102 98 : 900,
1103 98 : 1_000, // 1ms
1104 98 : 2_000,
1105 98 : 4_000,
1106 98 : 6_000,
1107 98 : 8_000,
1108 98 : 10_000, // 10ms
1109 98 : 20_000,
1110 98 : 40_000,
1111 98 : 60_000,
1112 98 : 80_000,
1113 98 : 100_000,
1114 98 : 200_000,
1115 98 : 400_000,
1116 98 : 600_000,
1117 98 : 800_000,
1118 98 : 1_000_000, // 1s
1119 98 : 2_000_000,
1120 98 : 4_000_000,
1121 98 : 6_000_000,
1122 98 : 8_000_000,
1123 98 : 10_000_000, // 10s
1124 98 : 20_000_000,
1125 98 : 50_000_000,
1126 98 : 100_000_000,
1127 98 : 200_000_000,
1128 98 : 1_000_000_000, // 1000s
1129 98 : ]
1130 98 : .into_iter()
1131 98 : .map(Duration::from_micros)
1132 4018 : .map(|d| d.as_secs_f64())
1133 98 : .collect()
1134 98 : });
1135 :
1136 98 : static SMGR_QUERY_TIME_GLOBAL: Lazy<HistogramVec> = Lazy::new(|| {
1137 98 : register_histogram_vec!(
1138 98 : "pageserver_smgr_query_seconds_global",
1139 98 : "Time spent on smgr query handling, aggregated by query type.",
1140 98 : &["smgr_query_type"],
1141 98 : SMGR_QUERY_TIME_GLOBAL_BUCKETS.clone(),
1142 98 : )
1143 98 : .expect("failed to define a metric")
1144 98 : });
1145 :
1146 : impl SmgrQueryTimePerTimeline {
1147 324 : pub(crate) fn new(tenant_shard_id: &TenantShardId, timeline_id: &TimelineId) -> Self {
1148 324 : let tenant_id = tenant_shard_id.tenant_id.to_string();
1149 324 : let shard_slug = format!("{}", tenant_shard_id.shard_slug());
1150 324 : let timeline_id = timeline_id.to_string();
1151 1620 : let metrics = std::array::from_fn(|i| {
1152 1620 : let op = SmgrQueryType::from_repr(i).unwrap();
1153 1620 : let global = SMGR_QUERY_TIME_GLOBAL
1154 1620 : .get_metric_with_label_values(&[op.into()])
1155 1620 : .unwrap();
1156 1620 : let per_tenant_timeline = SMGR_QUERY_TIME_PER_TENANT_TIMELINE
1157 1620 : .get_metric_with_label_values(&[op.into(), &tenant_id, &shard_slug, &timeline_id])
1158 1620 : .unwrap();
1159 1620 : GlobalAndPerTimelineHistogram {
1160 1620 : global,
1161 1620 : per_tenant_timeline,
1162 1620 : }
1163 1620 : });
1164 324 : Self { metrics }
1165 324 : }
1166 10 : pub(crate) fn start_timer<'c: 'a, 'a>(
1167 10 : &'a self,
1168 10 : op: SmgrQueryType,
1169 10 : ctx: &'c RequestContext,
1170 10 : ) -> impl Drop + '_ {
1171 10 : let metric = &self.metrics[op as usize];
1172 10 : let start = Instant::now();
1173 10 : match ctx.micros_spent_throttled.open() {
1174 10 : Ok(()) => (),
1175 0 : Err(error) => {
1176 0 : use utils::rate_limit::RateLimit;
1177 0 : static LOGGED: Lazy<Mutex<enum_map::EnumMap<SmgrQueryType, RateLimit>>> =
1178 0 : Lazy::new(|| {
1179 0 : Mutex::new(enum_map::EnumMap::from_array(std::array::from_fn(|_| {
1180 0 : RateLimit::new(Duration::from_secs(10))
1181 0 : })))
1182 0 : });
1183 0 : let mut guard = LOGGED.lock().unwrap();
1184 0 : let rate_limit = &mut guard[op];
1185 0 : rate_limit.call(|| {
1186 0 : warn!(?op, error, "error opening micros_spent_throttled; this message is logged at a global rate limit");
1187 0 : });
1188 0 : }
1189 : }
1190 10 : GlobalAndPerTimelineHistogramTimer {
1191 10 : h: metric,
1192 10 : ctx,
1193 10 : start,
1194 10 : op,
1195 10 : }
1196 10 : }
1197 : }
1198 :
1199 : #[cfg(test)]
1200 : mod smgr_query_time_tests {
1201 : use pageserver_api::shard::TenantShardId;
1202 : use strum::IntoEnumIterator;
1203 : use utils::id::{TenantId, TimelineId};
1204 :
1205 : use crate::{
1206 : context::{DownloadBehavior, RequestContext},
1207 : task_mgr::TaskKind,
1208 : };
1209 :
1210 : // Regression test, we used hard-coded string constants before using an enum.
1211 : #[test]
1212 2 : fn op_label_name() {
1213 2 : use super::SmgrQueryType::*;
1214 2 : let expect: [(super::SmgrQueryType, &'static str); 5] = [
1215 2 : (GetRelExists, "get_rel_exists"),
1216 2 : (GetRelSize, "get_rel_size"),
1217 2 : (GetPageAtLsn, "get_page_at_lsn"),
1218 2 : (GetDbSize, "get_db_size"),
1219 2 : (GetSlruSegment, "get_slru_segment"),
1220 2 : ];
1221 12 : for (op, expect) in expect {
1222 10 : let actual: &'static str = op.into();
1223 10 : assert_eq!(actual, expect);
1224 : }
1225 2 : }
1226 :
1227 : #[test]
1228 2 : fn basic() {
1229 2 : let ops: Vec<_> = super::SmgrQueryType::iter().collect();
1230 :
1231 12 : for op in &ops {
1232 10 : let tenant_id = TenantId::generate();
1233 10 : let timeline_id = TimelineId::generate();
1234 10 : let metrics = super::SmgrQueryTimePerTimeline::new(
1235 10 : &TenantShardId::unsharded(tenant_id),
1236 10 : &timeline_id,
1237 10 : );
1238 10 :
1239 20 : let get_counts = || {
1240 20 : let global: u64 = ops
1241 20 : .iter()
1242 100 : .map(|op| metrics.metrics[*op as usize].global.get_sample_count())
1243 20 : .sum();
1244 20 : let per_tenant_timeline: u64 = ops
1245 20 : .iter()
1246 100 : .map(|op| {
1247 100 : metrics.metrics[*op as usize]
1248 100 : .per_tenant_timeline
1249 100 : .get_sample_count()
1250 100 : })
1251 20 : .sum();
1252 20 : (global, per_tenant_timeline)
1253 20 : };
1254 :
1255 10 : let (pre_global, pre_per_tenant_timeline) = get_counts();
1256 10 : assert_eq!(pre_per_tenant_timeline, 0);
1257 :
1258 10 : let ctx = RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Download);
1259 10 : let timer = metrics.start_timer(*op, &ctx);
1260 10 : drop(timer);
1261 10 :
1262 10 : let (post_global, post_per_tenant_timeline) = get_counts();
1263 10 : assert_eq!(post_per_tenant_timeline, 1);
1264 10 : assert!(post_global > pre_global);
1265 : }
1266 2 : }
1267 : }
1268 :
1269 : // keep in sync with control plane Go code so that we can validate
1270 : // compute's basebackup_ms metric with our perspective in the context of SLI/SLO.
1271 0 : static COMPUTE_STARTUP_BUCKETS: Lazy<[f64; 28]> = Lazy::new(|| {
1272 0 : // Go code uses milliseconds. Variable is called `computeStartupBuckets`
1273 0 : [
1274 0 : 5, 10, 20, 30, 50, 70, 100, 120, 150, 200, 250, 300, 350, 400, 450, 500, 600, 800, 1000,
1275 0 : 1500, 2000, 2500, 3000, 5000, 10000, 20000, 40000, 60000,
1276 0 : ]
1277 0 : .map(|ms| (ms as f64) / 1000.0)
1278 0 : });
1279 :
1280 : pub(crate) struct BasebackupQueryTime(HistogramVec);
1281 0 : pub(crate) static BASEBACKUP_QUERY_TIME: Lazy<BasebackupQueryTime> = Lazy::new(|| {
1282 0 : BasebackupQueryTime({
1283 0 : register_histogram_vec!(
1284 0 : "pageserver_basebackup_query_seconds",
1285 0 : "Histogram of basebackup queries durations, by result type",
1286 0 : &["result"],
1287 0 : COMPUTE_STARTUP_BUCKETS.to_vec(),
1288 0 : )
1289 0 : .expect("failed to define a metric")
1290 0 : })
1291 0 : });
1292 :
1293 : pub(crate) struct BasebackupQueryTimeOngoingRecording<'a, 'c> {
1294 : parent: &'a BasebackupQueryTime,
1295 : ctx: &'c RequestContext,
1296 : start: std::time::Instant,
1297 : }
1298 :
1299 : impl BasebackupQueryTime {
1300 0 : pub(crate) fn start_recording<'c: 'a, 'a>(
1301 0 : &'a self,
1302 0 : ctx: &'c RequestContext,
1303 0 : ) -> BasebackupQueryTimeOngoingRecording<'_, '_> {
1304 0 : let start = Instant::now();
1305 0 : match ctx.micros_spent_throttled.open() {
1306 0 : Ok(()) => (),
1307 0 : Err(error) => {
1308 0 : use utils::rate_limit::RateLimit;
1309 0 : static LOGGED: Lazy<Mutex<RateLimit>> =
1310 0 : Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(10))));
1311 0 : let mut rate_limit = LOGGED.lock().unwrap();
1312 0 : rate_limit.call(|| {
1313 0 : warn!(error, "error opening micros_spent_throttled; this message is logged at a global rate limit");
1314 0 : });
1315 0 : }
1316 : }
1317 0 : BasebackupQueryTimeOngoingRecording {
1318 0 : parent: self,
1319 0 : ctx,
1320 0 : start,
1321 0 : }
1322 0 : }
1323 : }
1324 :
1325 : impl<'a, 'c> BasebackupQueryTimeOngoingRecording<'a, 'c> {
1326 0 : pub(crate) fn observe<T, E>(self, res: &Result<T, E>) {
1327 0 : let elapsed = self.start.elapsed();
1328 0 : let ex_throttled = self
1329 0 : .ctx
1330 0 : .micros_spent_throttled
1331 0 : .close_and_checked_sub_from(elapsed);
1332 0 : let ex_throttled = match ex_throttled {
1333 0 : Ok(ex_throttled) => ex_throttled,
1334 0 : Err(error) => {
1335 0 : use utils::rate_limit::RateLimit;
1336 0 : static LOGGED: Lazy<Mutex<RateLimit>> =
1337 0 : Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(10))));
1338 0 : let mut rate_limit = LOGGED.lock().unwrap();
1339 0 : rate_limit.call(|| {
1340 0 : warn!(error, "error deducting time spent throttled; this message is logged at a global rate limit");
1341 0 : });
1342 0 : elapsed
1343 : }
1344 : };
1345 0 : let label_value = if res.is_ok() { "ok" } else { "error" };
1346 0 : let metric = self
1347 0 : .parent
1348 0 : .0
1349 0 : .get_metric_with_label_values(&[label_value])
1350 0 : .unwrap();
1351 0 : metric.observe(ex_throttled.as_secs_f64());
1352 0 : }
1353 : }
1354 :
1355 0 : pub(crate) static LIVE_CONNECTIONS_COUNT: Lazy<IntGaugeVec> = Lazy::new(|| {
1356 0 : register_int_gauge_vec!(
1357 0 : "pageserver_live_connections",
1358 0 : "Number of live network connections",
1359 0 : &["pageserver_connection_kind"]
1360 0 : )
1361 0 : .expect("failed to define a metric")
1362 0 : });
1363 :
1364 : // remote storage metrics
1365 :
1366 92 : static REMOTE_TIMELINE_CLIENT_CALLS: Lazy<IntCounterPairVec> = Lazy::new(|| {
1367 184 : register_int_counter_pair_vec!(
1368 184 : "pageserver_remote_timeline_client_calls_started",
1369 184 : "Number of started calls to remote timeline client.",
1370 184 : "pageserver_remote_timeline_client_calls_finished",
1371 184 : "Number of finshed calls to remote timeline client.",
1372 184 : &[
1373 184 : "tenant_id",
1374 184 : "shard_id",
1375 184 : "timeline_id",
1376 184 : "file_kind",
1377 184 : "op_kind"
1378 184 : ],
1379 184 : )
1380 92 : .unwrap()
1381 92 : });
1382 :
1383 : static REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER: Lazy<IntCounterVec> =
1384 92 : Lazy::new(|| {
1385 92 : register_int_counter_vec!(
1386 92 : "pageserver_remote_timeline_client_bytes_started",
1387 92 : "Incremented by the number of bytes associated with a remote timeline client operation. \
1388 92 : The increment happens when the operation is scheduled.",
1389 92 : &["tenant_id", "shard_id", "timeline_id", "file_kind", "op_kind"],
1390 92 : )
1391 92 : .expect("failed to define a metric")
1392 92 : });
1393 :
1394 92 : static REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER: Lazy<IntCounterVec> = Lazy::new(|| {
1395 92 : register_int_counter_vec!(
1396 92 : "pageserver_remote_timeline_client_bytes_finished",
1397 92 : "Incremented by the number of bytes associated with a remote timeline client operation. \
1398 92 : The increment happens when the operation finishes (regardless of success/failure/shutdown).",
1399 92 : &["tenant_id", "shard_id", "timeline_id", "file_kind", "op_kind"],
1400 92 : )
1401 92 : .expect("failed to define a metric")
1402 92 : });
1403 :
1404 : pub(crate) struct TenantManagerMetrics {
1405 : pub(crate) tenant_slots: UIntGauge,
1406 : pub(crate) tenant_slot_writes: IntCounter,
1407 : pub(crate) unexpected_errors: IntCounter,
1408 : }
1409 :
1410 2 : pub(crate) static TENANT_MANAGER: Lazy<TenantManagerMetrics> = Lazy::new(|| {
1411 2 : TenantManagerMetrics {
1412 2 : tenant_slots: register_uint_gauge!(
1413 2 : "pageserver_tenant_manager_slots",
1414 2 : "How many slots currently exist, including all attached, secondary and in-progress operations",
1415 2 : )
1416 2 : .expect("failed to define a metric"),
1417 2 : tenant_slot_writes: register_int_counter!(
1418 2 : "pageserver_tenant_manager_slot_writes",
1419 2 : "Writes to a tenant slot, including all of create/attach/detach/delete"
1420 2 : )
1421 2 : .expect("failed to define a metric"),
1422 2 : unexpected_errors: register_int_counter!(
1423 2 : "pageserver_tenant_manager_unexpected_errors_total",
1424 2 : "Number of unexpected conditions encountered: nonzero value indicates a non-fatal bug."
1425 2 : )
1426 2 : .expect("failed to define a metric"),
1427 2 : }
1428 2 : });
1429 :
1430 : pub(crate) struct DeletionQueueMetrics {
1431 : pub(crate) keys_submitted: IntCounter,
1432 : pub(crate) keys_dropped: IntCounter,
1433 : pub(crate) keys_executed: IntCounter,
1434 : pub(crate) keys_validated: IntCounter,
1435 : pub(crate) dropped_lsn_updates: IntCounter,
1436 : pub(crate) unexpected_errors: IntCounter,
1437 : pub(crate) remote_errors: IntCounterVec,
1438 : }
1439 20 : pub(crate) static DELETION_QUEUE: Lazy<DeletionQueueMetrics> = Lazy::new(|| {
1440 20 : DeletionQueueMetrics{
1441 20 :
1442 20 : keys_submitted: register_int_counter!(
1443 20 : "pageserver_deletion_queue_submitted_total",
1444 20 : "Number of objects submitted for deletion"
1445 20 : )
1446 20 : .expect("failed to define a metric"),
1447 20 :
1448 20 : keys_dropped: register_int_counter!(
1449 20 : "pageserver_deletion_queue_dropped_total",
1450 20 : "Number of object deletions dropped due to stale generation."
1451 20 : )
1452 20 : .expect("failed to define a metric"),
1453 20 :
1454 20 : keys_executed: register_int_counter!(
1455 20 : "pageserver_deletion_queue_executed_total",
1456 20 : "Number of objects deleted. Only includes objects that we actually deleted, sum with pageserver_deletion_queue_dropped_total for the total number of keys processed to completion"
1457 20 : )
1458 20 : .expect("failed to define a metric"),
1459 20 :
1460 20 : keys_validated: register_int_counter!(
1461 20 : "pageserver_deletion_queue_validated_total",
1462 20 : "Number of keys validated for deletion. Sum with pageserver_deletion_queue_dropped_total for the total number of keys that have passed through the validation stage."
1463 20 : )
1464 20 : .expect("failed to define a metric"),
1465 20 :
1466 20 : dropped_lsn_updates: register_int_counter!(
1467 20 : "pageserver_deletion_queue_dropped_lsn_updates_total",
1468 20 : "Updates to remote_consistent_lsn dropped due to stale generation number."
1469 20 : )
1470 20 : .expect("failed to define a metric"),
1471 20 : unexpected_errors: register_int_counter!(
1472 20 : "pageserver_deletion_queue_unexpected_errors_total",
1473 20 : "Number of unexpected condiions that may stall the queue: any value above zero is unexpected."
1474 20 : )
1475 20 : .expect("failed to define a metric"),
1476 20 : remote_errors: register_int_counter_vec!(
1477 20 : "pageserver_deletion_queue_remote_errors_total",
1478 20 : "Retryable remote I/O errors while executing deletions, for example 503 responses to DeleteObjects",
1479 20 : &["op_kind"],
1480 20 : )
1481 20 : .expect("failed to define a metric")
1482 20 : }
1483 20 : });
1484 :
1485 : pub(crate) struct WalIngestMetrics {
1486 : pub(crate) bytes_received: IntCounter,
1487 : pub(crate) records_received: IntCounter,
1488 : pub(crate) records_committed: IntCounter,
1489 : pub(crate) records_filtered: IntCounter,
1490 : }
1491 :
1492 2 : pub(crate) static WAL_INGEST: Lazy<WalIngestMetrics> = Lazy::new(|| WalIngestMetrics {
1493 2 : bytes_received: register_int_counter!(
1494 2 : "pageserver_wal_ingest_bytes_received",
1495 2 : "Bytes of WAL ingested from safekeepers",
1496 2 : )
1497 2 : .unwrap(),
1498 2 : records_received: register_int_counter!(
1499 2 : "pageserver_wal_ingest_records_received",
1500 2 : "Number of WAL records received from safekeepers"
1501 2 : )
1502 2 : .expect("failed to define a metric"),
1503 2 : records_committed: register_int_counter!(
1504 2 : "pageserver_wal_ingest_records_committed",
1505 2 : "Number of WAL records which resulted in writes to pageserver storage"
1506 2 : )
1507 2 : .expect("failed to define a metric"),
1508 2 : records_filtered: register_int_counter!(
1509 2 : "pageserver_wal_ingest_records_filtered",
1510 2 : "Number of WAL records filtered out due to sharding"
1511 2 : )
1512 2 : .expect("failed to define a metric"),
1513 2 : });
1514 : pub(crate) struct SecondaryModeMetrics {
1515 : pub(crate) upload_heatmap: IntCounter,
1516 : pub(crate) upload_heatmap_errors: IntCounter,
1517 : pub(crate) upload_heatmap_duration: Histogram,
1518 : pub(crate) download_heatmap: IntCounter,
1519 : pub(crate) download_layer: IntCounter,
1520 : }
1521 0 : pub(crate) static SECONDARY_MODE: Lazy<SecondaryModeMetrics> = Lazy::new(|| SecondaryModeMetrics {
1522 0 : upload_heatmap: register_int_counter!(
1523 0 : "pageserver_secondary_upload_heatmap",
1524 0 : "Number of heatmaps written to remote storage by attached tenants"
1525 0 : )
1526 0 : .expect("failed to define a metric"),
1527 0 : upload_heatmap_errors: register_int_counter!(
1528 0 : "pageserver_secondary_upload_heatmap_errors",
1529 0 : "Failures writing heatmap to remote storage"
1530 0 : )
1531 0 : .expect("failed to define a metric"),
1532 0 : upload_heatmap_duration: register_histogram!(
1533 0 : "pageserver_secondary_upload_heatmap_duration",
1534 0 : "Time to build and upload a heatmap, including any waiting inside the S3 client"
1535 0 : )
1536 0 : .expect("failed to define a metric"),
1537 0 : download_heatmap: register_int_counter!(
1538 0 : "pageserver_secondary_download_heatmap",
1539 0 : "Number of downloads of heatmaps by secondary mode locations"
1540 0 : )
1541 0 : .expect("failed to define a metric"),
1542 0 : download_layer: register_int_counter!(
1543 0 : "pageserver_secondary_download_layer",
1544 0 : "Number of downloads of layers by secondary mode locations"
1545 0 : )
1546 0 : .expect("failed to define a metric"),
1547 0 : });
1548 :
1549 : #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1550 : pub enum RemoteOpKind {
1551 : Upload,
1552 : Download,
1553 : Delete,
1554 : }
1555 : impl RemoteOpKind {
1556 8783 : pub fn as_str(&self) -> &'static str {
1557 8783 : match self {
1558 7957 : Self::Upload => "upload",
1559 52 : Self::Download => "download",
1560 774 : Self::Delete => "delete",
1561 : }
1562 8783 : }
1563 : }
1564 :
1565 : #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
1566 : pub enum RemoteOpFileKind {
1567 : Layer,
1568 : Index,
1569 : }
1570 : impl RemoteOpFileKind {
1571 8783 : pub fn as_str(&self) -> &'static str {
1572 8783 : match self {
1573 5967 : Self::Layer => "layer",
1574 2816 : Self::Index => "index",
1575 : }
1576 8783 : }
1577 : }
1578 :
1579 90 : pub(crate) static REMOTE_OPERATION_TIME: Lazy<HistogramVec> = Lazy::new(|| {
1580 90 : register_histogram_vec!(
1581 90 : "pageserver_remote_operation_seconds",
1582 90 : "Time spent on remote storage operations. \
1583 90 : Grouped by tenant, timeline, operation_kind and status. \
1584 90 : Does not account for time spent waiting in remote timeline client's queues.",
1585 90 : &["file_kind", "op_kind", "status"]
1586 90 : )
1587 90 : .expect("failed to define a metric")
1588 90 : });
1589 :
1590 0 : pub(crate) static TENANT_TASK_EVENTS: Lazy<IntCounterVec> = Lazy::new(|| {
1591 0 : register_int_counter_vec!(
1592 0 : "pageserver_tenant_task_events",
1593 0 : "Number of task start/stop/fail events.",
1594 0 : &["event"],
1595 0 : )
1596 0 : .expect("Failed to register tenant_task_events metric")
1597 0 : });
1598 :
1599 14 : pub(crate) static BACKGROUND_LOOP_SEMAPHORE_WAIT_GAUGE: Lazy<IntCounterPairVec> = Lazy::new(|| {
1600 28 : register_int_counter_pair_vec!(
1601 28 : "pageserver_background_loop_semaphore_wait_start_count",
1602 28 : "Counter for background loop concurrency-limiting semaphore acquire calls started",
1603 28 : "pageserver_background_loop_semaphore_wait_finish_count",
1604 28 : "Counter for background loop concurrency-limiting semaphore acquire calls finished",
1605 28 : &["task"],
1606 28 : )
1607 14 : .unwrap()
1608 14 : });
1609 :
1610 0 : pub(crate) static BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT: Lazy<IntCounterVec> = Lazy::new(|| {
1611 0 : register_int_counter_vec!(
1612 0 : "pageserver_background_loop_period_overrun_count",
1613 0 : "Incremented whenever warn_when_period_overrun() logs a warning.",
1614 0 : &["task", "period"],
1615 0 : )
1616 0 : .expect("failed to define a metric")
1617 0 : });
1618 :
1619 : // walreceiver metrics
1620 :
1621 0 : pub(crate) static WALRECEIVER_STARTED_CONNECTIONS: Lazy<IntCounter> = Lazy::new(|| {
1622 0 : register_int_counter!(
1623 0 : "pageserver_walreceiver_started_connections_total",
1624 0 : "Number of started walreceiver connections"
1625 0 : )
1626 0 : .expect("failed to define a metric")
1627 0 : });
1628 :
1629 0 : pub(crate) static WALRECEIVER_ACTIVE_MANAGERS: Lazy<IntGauge> = Lazy::new(|| {
1630 0 : register_int_gauge!(
1631 0 : "pageserver_walreceiver_active_managers",
1632 0 : "Number of active walreceiver managers"
1633 0 : )
1634 0 : .expect("failed to define a metric")
1635 0 : });
1636 :
1637 0 : pub(crate) static WALRECEIVER_SWITCHES: Lazy<IntCounterVec> = Lazy::new(|| {
1638 0 : register_int_counter_vec!(
1639 0 : "pageserver_walreceiver_switches_total",
1640 0 : "Number of walreceiver manager change_connection calls",
1641 0 : &["reason"]
1642 0 : )
1643 0 : .expect("failed to define a metric")
1644 0 : });
1645 :
1646 0 : pub(crate) static WALRECEIVER_BROKER_UPDATES: Lazy<IntCounter> = Lazy::new(|| {
1647 0 : register_int_counter!(
1648 0 : "pageserver_walreceiver_broker_updates_total",
1649 0 : "Number of received broker updates in walreceiver"
1650 0 : )
1651 0 : .expect("failed to define a metric")
1652 0 : });
1653 :
1654 2 : pub(crate) static WALRECEIVER_CANDIDATES_EVENTS: Lazy<IntCounterVec> = Lazy::new(|| {
1655 2 : register_int_counter_vec!(
1656 2 : "pageserver_walreceiver_candidates_events_total",
1657 2 : "Number of walreceiver candidate events",
1658 2 : &["event"]
1659 2 : )
1660 2 : .expect("failed to define a metric")
1661 2 : });
1662 :
1663 : pub(crate) static WALRECEIVER_CANDIDATES_ADDED: Lazy<IntCounter> =
1664 0 : Lazy::new(|| WALRECEIVER_CANDIDATES_EVENTS.with_label_values(&["add"]));
1665 :
1666 : pub(crate) static WALRECEIVER_CANDIDATES_REMOVED: Lazy<IntCounter> =
1667 2 : Lazy::new(|| WALRECEIVER_CANDIDATES_EVENTS.with_label_values(&["remove"]));
1668 :
1669 : // Metrics collected on WAL redo operations
1670 : //
1671 : // We collect the time spent in actual WAL redo ('redo'), and time waiting
1672 : // for access to the postgres process ('wait') since there is only one for
1673 : // each tenant.
1674 :
1675 : /// Time buckets are small because we want to be able to measure the
1676 : /// smallest redo processing times. These buckets allow us to measure down
1677 : /// to 5us, which equates to 200'000 pages/sec, which equates to 1.6GB/sec.
1678 : /// This is much better than the previous 5ms aka 200 pages/sec aka 1.6MB/sec.
1679 : ///
1680 : /// Values up to 1s are recorded because metrics show that we have redo
1681 : /// durations and lock times larger than 0.250s.
1682 : macro_rules! redo_histogram_time_buckets {
1683 : () => {
1684 : vec![
1685 : 0.000_005, 0.000_010, 0.000_025, 0.000_050, 0.000_100, 0.000_250, 0.000_500, 0.001_000,
1686 : 0.002_500, 0.005_000, 0.010_000, 0.025_000, 0.050_000, 0.100_000, 0.250_000, 0.500_000,
1687 : 1.000_000,
1688 : ]
1689 : };
1690 : }
1691 :
1692 : /// While we're at it, also measure the amount of records replayed in each
1693 : /// operation. We have a global 'total replayed' counter, but that's not
1694 : /// as useful as 'what is the skew for how many records we replay in one
1695 : /// operation'.
1696 : macro_rules! redo_histogram_count_buckets {
1697 : () => {
1698 : vec![0.0, 1.0, 2.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0]
1699 : };
1700 : }
1701 :
1702 : macro_rules! redo_bytes_histogram_count_buckets {
1703 : () => {
1704 : // powers of (2^.5), from 2^4.5 to 2^15 (22 buckets)
1705 : // rounded up to the next multiple of 8 to capture any MAXALIGNed record of that size, too.
1706 : vec![
1707 : 24.0, 32.0, 48.0, 64.0, 96.0, 128.0, 184.0, 256.0, 368.0, 512.0, 728.0, 1024.0, 1456.0,
1708 : 2048.0, 2904.0, 4096.0, 5800.0, 8192.0, 11592.0, 16384.0, 23176.0, 32768.0,
1709 : ]
1710 : };
1711 : }
1712 :
1713 6 : pub(crate) static WAL_REDO_TIME: Lazy<Histogram> = Lazy::new(|| {
1714 6 : register_histogram!(
1715 6 : "pageserver_wal_redo_seconds",
1716 6 : "Time spent on WAL redo",
1717 6 : redo_histogram_time_buckets!()
1718 6 : )
1719 6 : .expect("failed to define a metric")
1720 6 : });
1721 :
1722 6 : pub(crate) static WAL_REDO_RECORDS_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
1723 6 : register_histogram!(
1724 6 : "pageserver_wal_redo_records_histogram",
1725 6 : "Histogram of number of records replayed per redo in the Postgres WAL redo process",
1726 6 : redo_histogram_count_buckets!(),
1727 6 : )
1728 6 : .expect("failed to define a metric")
1729 6 : });
1730 :
1731 6 : pub(crate) static WAL_REDO_BYTES_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
1732 6 : register_histogram!(
1733 6 : "pageserver_wal_redo_bytes_histogram",
1734 6 : "Histogram of number of records replayed per redo sent to Postgres",
1735 6 : redo_bytes_histogram_count_buckets!(),
1736 6 : )
1737 6 : .expect("failed to define a metric")
1738 6 : });
1739 :
1740 : // FIXME: isn't this already included by WAL_REDO_RECORDS_HISTOGRAM which has _count?
1741 6 : pub(crate) static WAL_REDO_RECORD_COUNTER: Lazy<IntCounter> = Lazy::new(|| {
1742 6 : register_int_counter!(
1743 6 : "pageserver_replayed_wal_records_total",
1744 6 : "Number of WAL records replayed in WAL redo process"
1745 6 : )
1746 6 : .unwrap()
1747 6 : });
1748 :
1749 : #[rustfmt::skip]
1750 6 : pub(crate) static WAL_REDO_PROCESS_LAUNCH_DURATION_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
1751 6 : register_histogram!(
1752 6 : "pageserver_wal_redo_process_launch_duration",
1753 6 : "Histogram of the duration of successful WalRedoProcess::launch calls",
1754 6 : vec![
1755 6 : 0.0002, 0.0004, 0.0006, 0.0008, 0.0010,
1756 6 : 0.0020, 0.0040, 0.0060, 0.0080, 0.0100,
1757 6 : 0.0200, 0.0400, 0.0600, 0.0800, 0.1000,
1758 6 : 0.2000, 0.4000, 0.6000, 0.8000, 1.0000,
1759 6 : 1.5000, 2.0000, 2.5000, 3.0000, 4.0000, 10.0000
1760 6 : ],
1761 6 : )
1762 6 : .expect("failed to define a metric")
1763 6 : });
1764 :
1765 : pub(crate) struct WalRedoProcessCounters {
1766 : pub(crate) started: IntCounter,
1767 : pub(crate) killed_by_cause: enum_map::EnumMap<WalRedoKillCause, IntCounter>,
1768 : pub(crate) active_stderr_logger_tasks_started: IntCounter,
1769 : pub(crate) active_stderr_logger_tasks_finished: IntCounter,
1770 : }
1771 :
1772 18 : #[derive(Debug, enum_map::Enum, strum_macros::IntoStaticStr)]
1773 : pub(crate) enum WalRedoKillCause {
1774 : WalRedoProcessDrop,
1775 : NoLeakChildDrop,
1776 : Startup,
1777 : }
1778 :
1779 : impl Default for WalRedoProcessCounters {
1780 6 : fn default() -> Self {
1781 6 : let started = register_int_counter!(
1782 6 : "pageserver_wal_redo_process_started_total",
1783 6 : "Number of WAL redo processes started",
1784 6 : )
1785 6 : .unwrap();
1786 6 :
1787 6 : let killed = register_int_counter_vec!(
1788 6 : "pageserver_wal_redo_process_stopped_total",
1789 6 : "Number of WAL redo processes stopped",
1790 6 : &["cause"],
1791 6 : )
1792 6 : .unwrap();
1793 6 :
1794 6 : let active_stderr_logger_tasks_started = register_int_counter!(
1795 6 : "pageserver_walredo_stderr_logger_tasks_started_total",
1796 6 : "Number of active walredo stderr logger tasks that have started",
1797 6 : )
1798 6 : .unwrap();
1799 6 :
1800 6 : let active_stderr_logger_tasks_finished = register_int_counter!(
1801 6 : "pageserver_walredo_stderr_logger_tasks_finished_total",
1802 6 : "Number of active walredo stderr logger tasks that have finished",
1803 6 : )
1804 6 : .unwrap();
1805 6 :
1806 6 : Self {
1807 6 : started,
1808 18 : killed_by_cause: EnumMap::from_array(std::array::from_fn(|i| {
1809 18 : let cause = <WalRedoKillCause as enum_map::Enum>::from_usize(i);
1810 18 : let cause_str: &'static str = cause.into();
1811 18 : killed.with_label_values(&[cause_str])
1812 18 : })),
1813 6 : active_stderr_logger_tasks_started,
1814 6 : active_stderr_logger_tasks_finished,
1815 6 : }
1816 6 : }
1817 : }
1818 :
1819 : pub(crate) static WAL_REDO_PROCESS_COUNTERS: Lazy<WalRedoProcessCounters> =
1820 : Lazy::new(WalRedoProcessCounters::default);
1821 :
1822 : /// Similar to `prometheus::HistogramTimer` but does not record on drop.
1823 : pub(crate) struct StorageTimeMetricsTimer {
1824 : metrics: StorageTimeMetrics,
1825 : start: Instant,
1826 : }
1827 :
1828 : impl StorageTimeMetricsTimer {
1829 2998 : fn new(metrics: StorageTimeMetrics) -> Self {
1830 2998 : Self {
1831 2998 : metrics,
1832 2998 : start: Instant::now(),
1833 2998 : }
1834 2998 : }
1835 :
1836 : /// Record the time from creation to now.
1837 2308 : pub fn stop_and_record(self) {
1838 2308 : let duration = self.start.elapsed().as_secs_f64();
1839 2308 : self.metrics.timeline_sum.inc_by(duration);
1840 2308 : self.metrics.timeline_count.inc();
1841 2308 : self.metrics.global_histogram.observe(duration);
1842 2308 : }
1843 : }
1844 :
1845 : /// Timing facilities for an globally histogrammed metric, which is supported by per tenant and
1846 : /// timeline total sum and count.
1847 : #[derive(Clone, Debug)]
1848 : pub(crate) struct StorageTimeMetrics {
1849 : /// Sum of f64 seconds, per operation, tenant_id and timeline_id
1850 : timeline_sum: Counter,
1851 : /// Number of oeprations, per operation, tenant_id and timeline_id
1852 : timeline_count: IntCounter,
1853 : /// Global histogram having only the "operation" label.
1854 : global_histogram: Histogram,
1855 : }
1856 :
1857 : impl StorageTimeMetrics {
1858 2198 : pub fn new(
1859 2198 : operation: StorageTimeOperation,
1860 2198 : tenant_id: &str,
1861 2198 : shard_id: &str,
1862 2198 : timeline_id: &str,
1863 2198 : ) -> Self {
1864 2198 : let operation: &'static str = operation.into();
1865 2198 :
1866 2198 : let timeline_sum = STORAGE_TIME_SUM_PER_TIMELINE
1867 2198 : .get_metric_with_label_values(&[operation, tenant_id, shard_id, timeline_id])
1868 2198 : .unwrap();
1869 2198 : let timeline_count = STORAGE_TIME_COUNT_PER_TIMELINE
1870 2198 : .get_metric_with_label_values(&[operation, tenant_id, shard_id, timeline_id])
1871 2198 : .unwrap();
1872 2198 : let global_histogram = STORAGE_TIME_GLOBAL
1873 2198 : .get_metric_with_label_values(&[operation])
1874 2198 : .unwrap();
1875 2198 :
1876 2198 : StorageTimeMetrics {
1877 2198 : timeline_sum,
1878 2198 : timeline_count,
1879 2198 : global_histogram,
1880 2198 : }
1881 2198 : }
1882 :
1883 : /// Starts timing a new operation.
1884 : ///
1885 : /// Note: unlike `prometheus::HistogramTimer` the returned timer does not record on drop.
1886 2998 : pub fn start_timer(&self) -> StorageTimeMetricsTimer {
1887 2998 : StorageTimeMetricsTimer::new(self.clone())
1888 2998 : }
1889 : }
1890 :
1891 : #[derive(Debug)]
1892 : pub(crate) struct TimelineMetrics {
1893 : tenant_id: String,
1894 : shard_id: String,
1895 : timeline_id: String,
1896 : pub flush_time_histo: StorageTimeMetrics,
1897 : pub compact_time_histo: StorageTimeMetrics,
1898 : pub create_images_time_histo: StorageTimeMetrics,
1899 : pub logical_size_histo: StorageTimeMetrics,
1900 : pub imitate_logical_size_histo: StorageTimeMetrics,
1901 : pub load_layer_map_histo: StorageTimeMetrics,
1902 : pub garbage_collect_histo: StorageTimeMetrics,
1903 : pub last_record_gauge: IntGauge,
1904 : resident_physical_size_gauge: UIntGauge,
1905 : /// copy of LayeredTimeline.current_logical_size
1906 : pub current_logical_size_gauge: UIntGauge,
1907 : pub directory_entries_count_gauge: Lazy<UIntGauge, Box<dyn Send + Fn() -> UIntGauge>>,
1908 : pub evictions: IntCounter,
1909 : pub evictions_with_low_residence_duration: std::sync::RwLock<EvictionsWithLowResidenceDuration>,
1910 : }
1911 :
1912 : impl TimelineMetrics {
1913 314 : pub fn new(
1914 314 : tenant_shard_id: &TenantShardId,
1915 314 : timeline_id_raw: &TimelineId,
1916 314 : evictions_with_low_residence_duration_builder: EvictionsWithLowResidenceDurationBuilder,
1917 314 : ) -> Self {
1918 314 : let tenant_id = tenant_shard_id.tenant_id.to_string();
1919 314 : let shard_id = format!("{}", tenant_shard_id.shard_slug());
1920 314 : let timeline_id = timeline_id_raw.to_string();
1921 314 : let flush_time_histo = StorageTimeMetrics::new(
1922 314 : StorageTimeOperation::LayerFlush,
1923 314 : &tenant_id,
1924 314 : &shard_id,
1925 314 : &timeline_id,
1926 314 : );
1927 314 : let compact_time_histo = StorageTimeMetrics::new(
1928 314 : StorageTimeOperation::Compact,
1929 314 : &tenant_id,
1930 314 : &shard_id,
1931 314 : &timeline_id,
1932 314 : );
1933 314 : let create_images_time_histo = StorageTimeMetrics::new(
1934 314 : StorageTimeOperation::CreateImages,
1935 314 : &tenant_id,
1936 314 : &shard_id,
1937 314 : &timeline_id,
1938 314 : );
1939 314 : let logical_size_histo = StorageTimeMetrics::new(
1940 314 : StorageTimeOperation::LogicalSize,
1941 314 : &tenant_id,
1942 314 : &shard_id,
1943 314 : &timeline_id,
1944 314 : );
1945 314 : let imitate_logical_size_histo = StorageTimeMetrics::new(
1946 314 : StorageTimeOperation::ImitateLogicalSize,
1947 314 : &tenant_id,
1948 314 : &shard_id,
1949 314 : &timeline_id,
1950 314 : );
1951 314 : let load_layer_map_histo = StorageTimeMetrics::new(
1952 314 : StorageTimeOperation::LoadLayerMap,
1953 314 : &tenant_id,
1954 314 : &shard_id,
1955 314 : &timeline_id,
1956 314 : );
1957 314 : let garbage_collect_histo = StorageTimeMetrics::new(
1958 314 : StorageTimeOperation::Gc,
1959 314 : &tenant_id,
1960 314 : &shard_id,
1961 314 : &timeline_id,
1962 314 : );
1963 314 : let last_record_gauge = LAST_RECORD_LSN
1964 314 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
1965 314 : .unwrap();
1966 314 : let resident_physical_size_gauge = RESIDENT_PHYSICAL_SIZE
1967 314 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
1968 314 : .unwrap();
1969 314 : // TODO: we shouldn't expose this metric
1970 314 : let current_logical_size_gauge = CURRENT_LOGICAL_SIZE
1971 314 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
1972 314 : .unwrap();
1973 314 : // TODO use impl Trait syntax here once we have ability to use it: https://github.com/rust-lang/rust/issues/63065
1974 314 : let directory_entries_count_gauge_closure = {
1975 314 : let tenant_shard_id = *tenant_shard_id;
1976 314 : let timeline_id_raw = *timeline_id_raw;
1977 0 : move || {
1978 0 : let tenant_id = tenant_shard_id.tenant_id.to_string();
1979 0 : let shard_id = format!("{}", tenant_shard_id.shard_slug());
1980 0 : let timeline_id = timeline_id_raw.to_string();
1981 0 : let gauge: UIntGauge = DIRECTORY_ENTRIES_COUNT
1982 0 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
1983 0 : .unwrap();
1984 0 : gauge
1985 0 : }
1986 : };
1987 314 : let directory_entries_count_gauge: Lazy<UIntGauge, Box<dyn Send + Fn() -> UIntGauge>> =
1988 314 : Lazy::new(Box::new(directory_entries_count_gauge_closure));
1989 314 : let evictions = EVICTIONS
1990 314 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
1991 314 : .unwrap();
1992 314 : let evictions_with_low_residence_duration = evictions_with_low_residence_duration_builder
1993 314 : .build(&tenant_id, &shard_id, &timeline_id);
1994 314 :
1995 314 : TimelineMetrics {
1996 314 : tenant_id,
1997 314 : shard_id,
1998 314 : timeline_id,
1999 314 : flush_time_histo,
2000 314 : compact_time_histo,
2001 314 : create_images_time_histo,
2002 314 : logical_size_histo,
2003 314 : imitate_logical_size_histo,
2004 314 : garbage_collect_histo,
2005 314 : load_layer_map_histo,
2006 314 : last_record_gauge,
2007 314 : resident_physical_size_gauge,
2008 314 : current_logical_size_gauge,
2009 314 : directory_entries_count_gauge,
2010 314 : evictions,
2011 314 : evictions_with_low_residence_duration: std::sync::RwLock::new(
2012 314 : evictions_with_low_residence_duration,
2013 314 : ),
2014 314 : }
2015 314 : }
2016 :
2017 940 : pub(crate) fn record_new_file_metrics(&self, sz: u64) {
2018 940 : self.resident_physical_size_add(sz);
2019 940 : }
2020 :
2021 462 : pub(crate) fn resident_physical_size_sub(&self, sz: u64) {
2022 462 : self.resident_physical_size_gauge.sub(sz);
2023 462 : crate::metrics::RESIDENT_PHYSICAL_SIZE_GLOBAL.sub(sz);
2024 462 : }
2025 :
2026 970 : pub(crate) fn resident_physical_size_add(&self, sz: u64) {
2027 970 : self.resident_physical_size_gauge.add(sz);
2028 970 : crate::metrics::RESIDENT_PHYSICAL_SIZE_GLOBAL.add(sz);
2029 970 : }
2030 :
2031 8 : pub(crate) fn resident_physical_size_get(&self) -> u64 {
2032 8 : self.resident_physical_size_gauge.get()
2033 8 : }
2034 :
2035 8 : pub(crate) fn shutdown(&self) {
2036 8 : let tenant_id = &self.tenant_id;
2037 8 : let timeline_id = &self.timeline_id;
2038 8 : let shard_id = &self.shard_id;
2039 8 : let _ = LAST_RECORD_LSN.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2040 8 : {
2041 8 : RESIDENT_PHYSICAL_SIZE_GLOBAL.sub(self.resident_physical_size_get());
2042 8 : let _ = RESIDENT_PHYSICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2043 8 : }
2044 8 : let _ = CURRENT_LOGICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2045 8 : if let Some(metric) = Lazy::get(&DIRECTORY_ENTRIES_COUNT) {
2046 0 : let _ = metric.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2047 8 : }
2048 8 : let _ = EVICTIONS.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2049 8 :
2050 8 : self.evictions_with_low_residence_duration
2051 8 : .write()
2052 8 : .unwrap()
2053 8 : .remove(tenant_id, shard_id, timeline_id);
2054 :
2055 : // The following metrics are born outside of the TimelineMetrics lifecycle but still
2056 : // removed at the end of it. The idea is to have the metrics outlive the
2057 : // entity during which they're observed, e.g., the smgr metrics shall
2058 : // outlive an individual smgr connection, but not the timeline.
2059 :
2060 72 : for op in StorageTimeOperation::VARIANTS {
2061 64 : let _ = STORAGE_TIME_SUM_PER_TIMELINE.remove_label_values(&[
2062 64 : op,
2063 64 : tenant_id,
2064 64 : shard_id,
2065 64 : timeline_id,
2066 64 : ]);
2067 64 : let _ = STORAGE_TIME_COUNT_PER_TIMELINE.remove_label_values(&[
2068 64 : op,
2069 64 : tenant_id,
2070 64 : shard_id,
2071 64 : timeline_id,
2072 64 : ]);
2073 64 : }
2074 :
2075 24 : for op in STORAGE_IO_SIZE_OPERATIONS {
2076 16 : let _ = STORAGE_IO_SIZE.remove_label_values(&[op, tenant_id, shard_id, timeline_id]);
2077 16 : }
2078 :
2079 48 : for op in SmgrQueryType::iter() {
2080 40 : let _ = SMGR_QUERY_TIME_PER_TENANT_TIMELINE.remove_label_values(&[
2081 40 : op.into(),
2082 40 : tenant_id,
2083 40 : shard_id,
2084 40 : timeline_id,
2085 40 : ]);
2086 40 : }
2087 8 : }
2088 : }
2089 :
2090 6 : pub(crate) fn remove_tenant_metrics(tenant_shard_id: &TenantShardId) {
2091 6 : // Only shard zero deals in synthetic sizes
2092 6 : if tenant_shard_id.is_zero() {
2093 6 : let tid = tenant_shard_id.tenant_id.to_string();
2094 6 : let _ = TENANT_SYNTHETIC_SIZE_METRIC.remove_label_values(&[&tid]);
2095 6 : }
2096 :
2097 : // we leave the BROKEN_TENANTS_SET entry if any
2098 6 : }
2099 :
2100 : use futures::Future;
2101 : use pin_project_lite::pin_project;
2102 : use std::collections::HashMap;
2103 : use std::pin::Pin;
2104 : use std::sync::{Arc, Mutex};
2105 : use std::task::{Context, Poll};
2106 : use std::time::{Duration, Instant};
2107 :
2108 : use crate::context::{PageContentKind, RequestContext};
2109 : use crate::task_mgr::TaskKind;
2110 :
2111 : /// Maintain a per timeline gauge in addition to the global gauge.
2112 : struct PerTimelineRemotePhysicalSizeGauge {
2113 : last_set: u64,
2114 : gauge: UIntGauge,
2115 : }
2116 :
2117 : impl PerTimelineRemotePhysicalSizeGauge {
2118 314 : fn new(per_timeline_gauge: UIntGauge) -> Self {
2119 314 : Self {
2120 314 : last_set: per_timeline_gauge.get(),
2121 314 : gauge: per_timeline_gauge,
2122 314 : }
2123 314 : }
2124 1230 : fn set(&mut self, sz: u64) {
2125 1230 : self.gauge.set(sz);
2126 1230 : if sz < self.last_set {
2127 34 : REMOTE_PHYSICAL_SIZE_GLOBAL.sub(self.last_set - sz);
2128 1196 : } else {
2129 1196 : REMOTE_PHYSICAL_SIZE_GLOBAL.add(sz - self.last_set);
2130 1196 : };
2131 1230 : self.last_set = sz;
2132 1230 : }
2133 2 : fn get(&self) -> u64 {
2134 2 : self.gauge.get()
2135 2 : }
2136 : }
2137 :
2138 : impl Drop for PerTimelineRemotePhysicalSizeGauge {
2139 8 : fn drop(&mut self) {
2140 8 : REMOTE_PHYSICAL_SIZE_GLOBAL.sub(self.last_set);
2141 8 : }
2142 : }
2143 :
2144 : pub(crate) struct RemoteTimelineClientMetrics {
2145 : tenant_id: String,
2146 : shard_id: String,
2147 : timeline_id: String,
2148 : remote_physical_size_gauge: Mutex<Option<PerTimelineRemotePhysicalSizeGauge>>,
2149 : calls: Mutex<HashMap<(&'static str, &'static str), IntCounterPair>>,
2150 : bytes_started_counter: Mutex<HashMap<(&'static str, &'static str), IntCounter>>,
2151 : bytes_finished_counter: Mutex<HashMap<(&'static str, &'static str), IntCounter>>,
2152 : }
2153 :
2154 : impl RemoteTimelineClientMetrics {
2155 324 : pub fn new(tenant_shard_id: &TenantShardId, timeline_id: &TimelineId) -> Self {
2156 324 : RemoteTimelineClientMetrics {
2157 324 : tenant_id: tenant_shard_id.tenant_id.to_string(),
2158 324 : shard_id: format!("{}", tenant_shard_id.shard_slug()),
2159 324 : timeline_id: timeline_id.to_string(),
2160 324 : calls: Mutex::new(HashMap::default()),
2161 324 : bytes_started_counter: Mutex::new(HashMap::default()),
2162 324 : bytes_finished_counter: Mutex::new(HashMap::default()),
2163 324 : remote_physical_size_gauge: Mutex::new(None),
2164 324 : }
2165 324 : }
2166 :
2167 1230 : pub(crate) fn remote_physical_size_set(&self, sz: u64) {
2168 1230 : let mut guard = self.remote_physical_size_gauge.lock().unwrap();
2169 1230 : let gauge = guard.get_or_insert_with(|| {
2170 314 : PerTimelineRemotePhysicalSizeGauge::new(
2171 314 : REMOTE_PHYSICAL_SIZE
2172 314 : .get_metric_with_label_values(&[
2173 314 : &self.tenant_id,
2174 314 : &self.shard_id,
2175 314 : &self.timeline_id,
2176 314 : ])
2177 314 : .unwrap(),
2178 314 : )
2179 1230 : });
2180 1230 : gauge.set(sz);
2181 1230 : }
2182 :
2183 2 : pub(crate) fn remote_physical_size_get(&self) -> u64 {
2184 2 : let guard = self.remote_physical_size_gauge.lock().unwrap();
2185 2 : guard.as_ref().map(|gauge| gauge.get()).unwrap_or(0)
2186 2 : }
2187 :
2188 1717 : pub fn remote_operation_time(
2189 1717 : &self,
2190 1717 : file_kind: &RemoteOpFileKind,
2191 1717 : op_kind: &RemoteOpKind,
2192 1717 : status: &'static str,
2193 1717 : ) -> Histogram {
2194 1717 : let key = (file_kind.as_str(), op_kind.as_str(), status);
2195 1717 : REMOTE_OPERATION_TIME
2196 1717 : .get_metric_with_label_values(&[key.0, key.1, key.2])
2197 1717 : .unwrap()
2198 1717 : }
2199 :
2200 4383 : fn calls_counter_pair(
2201 4383 : &self,
2202 4383 : file_kind: &RemoteOpFileKind,
2203 4383 : op_kind: &RemoteOpKind,
2204 4383 : ) -> IntCounterPair {
2205 4383 : let mut guard = self.calls.lock().unwrap();
2206 4383 : let key = (file_kind.as_str(), op_kind.as_str());
2207 4383 : let metric = guard.entry(key).or_insert_with(move || {
2208 542 : REMOTE_TIMELINE_CLIENT_CALLS
2209 542 : .get_metric_with_label_values(&[
2210 542 : &self.tenant_id,
2211 542 : &self.shard_id,
2212 542 : &self.timeline_id,
2213 542 : key.0,
2214 542 : key.1,
2215 542 : ])
2216 542 : .unwrap()
2217 4383 : });
2218 4383 : metric.clone()
2219 4383 : }
2220 :
2221 948 : fn bytes_started_counter(
2222 948 : &self,
2223 948 : file_kind: &RemoteOpFileKind,
2224 948 : op_kind: &RemoteOpKind,
2225 948 : ) -> IntCounter {
2226 948 : let mut guard = self.bytes_started_counter.lock().unwrap();
2227 948 : let key = (file_kind.as_str(), op_kind.as_str());
2228 948 : let metric = guard.entry(key).or_insert_with(move || {
2229 200 : REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER
2230 200 : .get_metric_with_label_values(&[
2231 200 : &self.tenant_id,
2232 200 : &self.shard_id,
2233 200 : &self.timeline_id,
2234 200 : key.0,
2235 200 : key.1,
2236 200 : ])
2237 200 : .unwrap()
2238 948 : });
2239 948 : metric.clone()
2240 948 : }
2241 :
2242 1723 : fn bytes_finished_counter(
2243 1723 : &self,
2244 1723 : file_kind: &RemoteOpFileKind,
2245 1723 : op_kind: &RemoteOpKind,
2246 1723 : ) -> IntCounter {
2247 1723 : let mut guard = self.bytes_finished_counter.lock().unwrap();
2248 1723 : let key = (file_kind.as_str(), op_kind.as_str());
2249 1723 : let metric = guard.entry(key).or_insert_with(move || {
2250 200 : REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER
2251 200 : .get_metric_with_label_values(&[
2252 200 : &self.tenant_id,
2253 200 : &self.shard_id,
2254 200 : &self.timeline_id,
2255 200 : key.0,
2256 200 : key.1,
2257 200 : ])
2258 200 : .unwrap()
2259 1723 : });
2260 1723 : metric.clone()
2261 1723 : }
2262 : }
2263 :
2264 : #[cfg(test)]
2265 : impl RemoteTimelineClientMetrics {
2266 6 : pub fn get_bytes_started_counter_value(
2267 6 : &self,
2268 6 : file_kind: &RemoteOpFileKind,
2269 6 : op_kind: &RemoteOpKind,
2270 6 : ) -> Option<u64> {
2271 6 : let guard = self.bytes_started_counter.lock().unwrap();
2272 6 : let key = (file_kind.as_str(), op_kind.as_str());
2273 6 : guard.get(&key).map(|counter| counter.get())
2274 6 : }
2275 :
2276 6 : pub fn get_bytes_finished_counter_value(
2277 6 : &self,
2278 6 : file_kind: &RemoteOpFileKind,
2279 6 : op_kind: &RemoteOpKind,
2280 6 : ) -> Option<u64> {
2281 6 : let guard = self.bytes_finished_counter.lock().unwrap();
2282 6 : let key = (file_kind.as_str(), op_kind.as_str());
2283 6 : guard.get(&key).map(|counter| counter.get())
2284 6 : }
2285 : }
2286 :
2287 : /// See [`RemoteTimelineClientMetrics::call_begin`].
2288 : #[must_use]
2289 : pub(crate) struct RemoteTimelineClientCallMetricGuard {
2290 : /// Decremented on drop.
2291 : calls_counter_pair: Option<IntCounterPair>,
2292 : /// If Some(), this references the bytes_finished metric, and we increment it by the given `u64` on drop.
2293 : bytes_finished: Option<(IntCounter, u64)>,
2294 : }
2295 :
2296 : impl RemoteTimelineClientCallMetricGuard {
2297 : /// Consume this guard object without performing the metric updates it would do on `drop()`.
2298 : /// The caller vouches to do the metric updates manually.
2299 2340 : pub fn will_decrement_manually(mut self) {
2300 2340 : let RemoteTimelineClientCallMetricGuard {
2301 2340 : calls_counter_pair,
2302 2340 : bytes_finished,
2303 2340 : } = &mut self;
2304 2340 : calls_counter_pair.take();
2305 2340 : bytes_finished.take();
2306 2340 : }
2307 : }
2308 :
2309 : impl Drop for RemoteTimelineClientCallMetricGuard {
2310 2366 : fn drop(&mut self) {
2311 2366 : let RemoteTimelineClientCallMetricGuard {
2312 2366 : calls_counter_pair,
2313 2366 : bytes_finished,
2314 2366 : } = self;
2315 2366 : if let Some(guard) = calls_counter_pair.take() {
2316 26 : guard.dec();
2317 2340 : }
2318 2366 : if let Some((bytes_finished_metric, value)) = bytes_finished {
2319 0 : bytes_finished_metric.inc_by(*value);
2320 2366 : }
2321 2366 : }
2322 : }
2323 :
2324 : /// The enum variants communicate to the [`RemoteTimelineClientMetrics`] whether to
2325 : /// track the byte size of this call in applicable metric(s).
2326 : pub(crate) enum RemoteTimelineClientMetricsCallTrackSize {
2327 : /// Do not account for this call's byte size in any metrics.
2328 : /// The `reason` field is there to make the call sites self-documenting
2329 : /// about why they don't need the metric.
2330 : DontTrackSize { reason: &'static str },
2331 : /// Track the byte size of the call in applicable metric(s).
2332 : Bytes(u64),
2333 : }
2334 :
2335 : impl RemoteTimelineClientMetrics {
2336 : /// Update the metrics that change when a call to the remote timeline client instance starts.
2337 : ///
2338 : /// Drop the returned guard object once the operation is finished to updates corresponding metrics that track completions.
2339 : /// Or, use [`RemoteTimelineClientCallMetricGuard::will_decrement_manually`] and [`call_end`](Self::call_end) if that
2340 : /// is more suitable.
2341 : /// Never do both.
2342 2366 : pub(crate) fn call_begin(
2343 2366 : &self,
2344 2366 : file_kind: &RemoteOpFileKind,
2345 2366 : op_kind: &RemoteOpKind,
2346 2366 : size: RemoteTimelineClientMetricsCallTrackSize,
2347 2366 : ) -> RemoteTimelineClientCallMetricGuard {
2348 2366 : let calls_counter_pair = self.calls_counter_pair(file_kind, op_kind);
2349 2366 : calls_counter_pair.inc();
2350 :
2351 2366 : let bytes_finished = match size {
2352 1418 : RemoteTimelineClientMetricsCallTrackSize::DontTrackSize { reason: _reason } => {
2353 1418 : // nothing to do
2354 1418 : None
2355 : }
2356 948 : RemoteTimelineClientMetricsCallTrackSize::Bytes(size) => {
2357 948 : self.bytes_started_counter(file_kind, op_kind).inc_by(size);
2358 948 : let finished_counter = self.bytes_finished_counter(file_kind, op_kind);
2359 948 : Some((finished_counter, size))
2360 : }
2361 : };
2362 2366 : RemoteTimelineClientCallMetricGuard {
2363 2366 : calls_counter_pair: Some(calls_counter_pair),
2364 2366 : bytes_finished,
2365 2366 : }
2366 2366 : }
2367 :
2368 : /// Manually udpate the metrics that track completions, instead of using the guard object.
2369 : /// Using the guard object is generally preferable.
2370 : /// See [`call_begin`](Self::call_begin) for more context.
2371 2017 : pub(crate) fn call_end(
2372 2017 : &self,
2373 2017 : file_kind: &RemoteOpFileKind,
2374 2017 : op_kind: &RemoteOpKind,
2375 2017 : size: RemoteTimelineClientMetricsCallTrackSize,
2376 2017 : ) {
2377 2017 : let calls_counter_pair = self.calls_counter_pair(file_kind, op_kind);
2378 2017 : calls_counter_pair.dec();
2379 2017 : match size {
2380 1242 : RemoteTimelineClientMetricsCallTrackSize::DontTrackSize { reason: _reason } => {}
2381 775 : RemoteTimelineClientMetricsCallTrackSize::Bytes(size) => {
2382 775 : self.bytes_finished_counter(file_kind, op_kind).inc_by(size);
2383 775 : }
2384 : }
2385 2017 : }
2386 : }
2387 :
2388 : impl Drop for RemoteTimelineClientMetrics {
2389 18 : fn drop(&mut self) {
2390 18 : let RemoteTimelineClientMetrics {
2391 18 : tenant_id,
2392 18 : shard_id,
2393 18 : timeline_id,
2394 18 : remote_physical_size_gauge,
2395 18 : calls,
2396 18 : bytes_started_counter,
2397 18 : bytes_finished_counter,
2398 18 : } = self;
2399 22 : for ((a, b), _) in calls.get_mut().unwrap().drain() {
2400 22 : let mut res = [Ok(()), Ok(())];
2401 22 : REMOTE_TIMELINE_CLIENT_CALLS
2402 22 : .remove_label_values(&mut res, &[tenant_id, shard_id, timeline_id, a, b]);
2403 22 : // don't care about results
2404 22 : }
2405 18 : for ((a, b), _) in bytes_started_counter.get_mut().unwrap().drain() {
2406 6 : let _ = REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER.remove_label_values(&[
2407 6 : tenant_id,
2408 6 : shard_id,
2409 6 : timeline_id,
2410 6 : a,
2411 6 : b,
2412 6 : ]);
2413 6 : }
2414 18 : for ((a, b), _) in bytes_finished_counter.get_mut().unwrap().drain() {
2415 6 : let _ = REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER.remove_label_values(&[
2416 6 : tenant_id,
2417 6 : shard_id,
2418 6 : timeline_id,
2419 6 : a,
2420 6 : b,
2421 6 : ]);
2422 6 : }
2423 18 : {
2424 18 : let _ = remote_physical_size_gauge; // use to avoid 'unused' warning in desctructuring above
2425 18 : let _ = REMOTE_PHYSICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2426 18 : }
2427 18 : }
2428 : }
2429 :
2430 : /// Wrapper future that measures the time spent by a remote storage operation,
2431 : /// and records the time and success/failure as a prometheus metric.
2432 : pub(crate) trait MeasureRemoteOp: Sized {
2433 1790 : fn measure_remote_op(
2434 1790 : self,
2435 1790 : file_kind: RemoteOpFileKind,
2436 1790 : op: RemoteOpKind,
2437 1790 : metrics: Arc<RemoteTimelineClientMetrics>,
2438 1790 : ) -> MeasuredRemoteOp<Self> {
2439 1790 : let start = Instant::now();
2440 1790 : MeasuredRemoteOp {
2441 1790 : inner: self,
2442 1790 : file_kind,
2443 1790 : op,
2444 1790 : start,
2445 1790 : metrics,
2446 1790 : }
2447 1790 : }
2448 : }
2449 :
2450 : impl<T: Sized> MeasureRemoteOp for T {}
2451 :
2452 : pin_project! {
2453 : pub(crate) struct MeasuredRemoteOp<F>
2454 : {
2455 : #[pin]
2456 : inner: F,
2457 : file_kind: RemoteOpFileKind,
2458 : op: RemoteOpKind,
2459 : start: Instant,
2460 : metrics: Arc<RemoteTimelineClientMetrics>,
2461 : }
2462 : }
2463 :
2464 : impl<F: Future<Output = Result<O, E>>, O, E> Future for MeasuredRemoteOp<F> {
2465 : type Output = Result<O, E>;
2466 :
2467 32309 : fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
2468 32309 : let this = self.project();
2469 32309 : let poll_result = this.inner.poll(cx);
2470 32309 : if let Poll::Ready(ref res) = poll_result {
2471 1717 : let duration = this.start.elapsed();
2472 1717 : let status = if res.is_ok() { &"success" } else { &"failure" };
2473 1717 : this.metrics
2474 1717 : .remote_operation_time(this.file_kind, this.op, status)
2475 1717 : .observe(duration.as_secs_f64());
2476 30592 : }
2477 32309 : poll_result
2478 32309 : }
2479 : }
2480 :
2481 : pub mod tokio_epoll_uring {
2482 : use metrics::{register_int_counter, UIntGauge};
2483 : use once_cell::sync::Lazy;
2484 :
2485 : pub struct Collector {
2486 : descs: Vec<metrics::core::Desc>,
2487 : systems_created: UIntGauge,
2488 : systems_destroyed: UIntGauge,
2489 : }
2490 :
2491 : impl metrics::core::Collector for Collector {
2492 0 : fn desc(&self) -> Vec<&metrics::core::Desc> {
2493 0 : self.descs.iter().collect()
2494 0 : }
2495 :
2496 0 : fn collect(&self) -> Vec<metrics::proto::MetricFamily> {
2497 0 : let mut mfs = Vec::with_capacity(Self::NMETRICS);
2498 0 : let tokio_epoll_uring::metrics::Metrics {
2499 0 : systems_created,
2500 0 : systems_destroyed,
2501 0 : } = tokio_epoll_uring::metrics::global();
2502 0 : self.systems_created.set(systems_created);
2503 0 : mfs.extend(self.systems_created.collect());
2504 0 : self.systems_destroyed.set(systems_destroyed);
2505 0 : mfs.extend(self.systems_destroyed.collect());
2506 0 : mfs
2507 0 : }
2508 : }
2509 :
2510 : impl Collector {
2511 : const NMETRICS: usize = 2;
2512 :
2513 : #[allow(clippy::new_without_default)]
2514 0 : pub fn new() -> Self {
2515 0 : let mut descs = Vec::new();
2516 0 :
2517 0 : let systems_created = UIntGauge::new(
2518 0 : "pageserver_tokio_epoll_uring_systems_created",
2519 0 : "counter of tokio-epoll-uring systems that were created",
2520 0 : )
2521 0 : .unwrap();
2522 0 : descs.extend(
2523 0 : metrics::core::Collector::desc(&systems_created)
2524 0 : .into_iter()
2525 0 : .cloned(),
2526 0 : );
2527 0 :
2528 0 : let systems_destroyed = UIntGauge::new(
2529 0 : "pageserver_tokio_epoll_uring_systems_destroyed",
2530 0 : "counter of tokio-epoll-uring systems that were destroyed",
2531 0 : )
2532 0 : .unwrap();
2533 0 : descs.extend(
2534 0 : metrics::core::Collector::desc(&systems_destroyed)
2535 0 : .into_iter()
2536 0 : .cloned(),
2537 0 : );
2538 0 :
2539 0 : Self {
2540 0 : descs,
2541 0 : systems_created,
2542 0 : systems_destroyed,
2543 0 : }
2544 0 : }
2545 : }
2546 :
2547 59 : pub(crate) static THREAD_LOCAL_LAUNCH_SUCCESSES: Lazy<metrics::IntCounter> = Lazy::new(|| {
2548 59 : register_int_counter!(
2549 59 : "pageserver_tokio_epoll_uring_pageserver_thread_local_launch_success_count",
2550 59 : "Number of times where thread_local_system creation spanned multiple executor threads",
2551 59 : )
2552 59 : .unwrap()
2553 59 : });
2554 :
2555 0 : pub(crate) static THREAD_LOCAL_LAUNCH_FAILURES: Lazy<metrics::IntCounter> = Lazy::new(|| {
2556 0 : register_int_counter!(
2557 0 : "pageserver_tokio_epoll_uring_pageserver_thread_local_launch_failures_count",
2558 0 : "Number of times thread_local_system creation failed and was retried after back-off.",
2559 0 : )
2560 0 : .unwrap()
2561 0 : });
2562 : }
2563 :
2564 : pub(crate) mod tenant_throttling {
2565 : use metrics::{register_int_counter_vec, IntCounter};
2566 : use once_cell::sync::Lazy;
2567 :
2568 : use crate::tenant::{self, throttle::Metric};
2569 :
2570 : pub(crate) struct TimelineGet {
2571 : wait_time: IntCounter,
2572 : count: IntCounter,
2573 : }
2574 :
2575 98 : pub(crate) static TIMELINE_GET: Lazy<TimelineGet> = Lazy::new(|| {
2576 98 : static WAIT_USECS: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
2577 98 : register_int_counter_vec!(
2578 98 : "pageserver_tenant_throttling_wait_usecs_sum_global",
2579 98 : "Sum of microseconds that tenants spent waiting for a tenant throttle of a given kind.",
2580 98 : &["kind"]
2581 98 : )
2582 98 : .unwrap()
2583 98 : });
2584 98 :
2585 98 : static WAIT_COUNT: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
2586 98 : register_int_counter_vec!(
2587 98 : "pageserver_tenant_throttling_count_global",
2588 98 : "Count of tenant throttlings, by kind of throttle.",
2589 98 : &["kind"]
2590 98 : )
2591 98 : .unwrap()
2592 98 : });
2593 98 :
2594 98 : let kind = "timeline_get";
2595 98 : TimelineGet {
2596 98 : wait_time: WAIT_USECS.with_label_values(&[kind]),
2597 98 : count: WAIT_COUNT.with_label_values(&[kind]),
2598 98 : }
2599 98 : });
2600 :
2601 : impl Metric for &'static TimelineGet {
2602 : #[inline(always)]
2603 0 : fn observe_throttling(
2604 0 : &self,
2605 0 : tenant::throttle::Observation { wait_time }: &tenant::throttle::Observation,
2606 0 : ) {
2607 0 : let val = u64::try_from(wait_time.as_micros()).unwrap();
2608 0 : self.wait_time.inc_by(val);
2609 0 : self.count.inc();
2610 0 : }
2611 : }
2612 : }
2613 :
2614 : pub(crate) mod disk_usage_based_eviction {
2615 : use super::*;
2616 :
2617 : pub(crate) struct Metrics {
2618 : pub(crate) tenant_collection_time: Histogram,
2619 : pub(crate) tenant_layer_count: Histogram,
2620 : pub(crate) layers_collected: IntCounter,
2621 : pub(crate) layers_selected: IntCounter,
2622 : pub(crate) layers_evicted: IntCounter,
2623 : }
2624 :
2625 : impl Default for Metrics {
2626 0 : fn default() -> Self {
2627 0 : let tenant_collection_time = register_histogram!(
2628 0 : "pageserver_disk_usage_based_eviction_tenant_collection_seconds",
2629 0 : "Time spent collecting layers from a tenant -- not normalized by collected layer amount",
2630 0 : vec![0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0, 10.0]
2631 0 : )
2632 0 : .unwrap();
2633 0 :
2634 0 : let tenant_layer_count = register_histogram!(
2635 0 : "pageserver_disk_usage_based_eviction_tenant_collected_layers",
2636 0 : "Amount of layers gathered from a tenant",
2637 0 : vec![5.0, 50.0, 500.0, 5000.0, 50000.0]
2638 0 : )
2639 0 : .unwrap();
2640 0 :
2641 0 : let layers_collected = register_int_counter!(
2642 0 : "pageserver_disk_usage_based_eviction_collected_layers_total",
2643 0 : "Amount of layers collected"
2644 0 : )
2645 0 : .unwrap();
2646 0 :
2647 0 : let layers_selected = register_int_counter!(
2648 0 : "pageserver_disk_usage_based_eviction_select_layers_total",
2649 0 : "Amount of layers selected"
2650 0 : )
2651 0 : .unwrap();
2652 0 :
2653 0 : let layers_evicted = register_int_counter!(
2654 0 : "pageserver_disk_usage_based_eviction_evicted_layers_total",
2655 0 : "Amount of layers successfully evicted"
2656 0 : )
2657 0 : .unwrap();
2658 0 :
2659 0 : Self {
2660 0 : tenant_collection_time,
2661 0 : tenant_layer_count,
2662 0 : layers_collected,
2663 0 : layers_selected,
2664 0 : layers_evicted,
2665 0 : }
2666 0 : }
2667 : }
2668 :
2669 : pub(crate) static METRICS: Lazy<Metrics> = Lazy::new(Metrics::default);
2670 : }
2671 :
2672 0 : pub fn preinitialize_metrics() {
2673 0 : // Python tests need these and on some we do alerting.
2674 0 : //
2675 0 : // FIXME(4813): make it so that we have no top level metrics as this fn will easily fall out of
2676 0 : // order:
2677 0 : // - global metrics reside in a Lazy<PageserverMetrics>
2678 0 : // - access via crate::metrics::PS_METRICS.materialized_page_cache_hit.inc()
2679 0 : // - could move the statics into TimelineMetrics::new()?
2680 0 :
2681 0 : // counters
2682 0 : [
2683 0 : &MATERIALIZED_PAGE_CACHE_HIT,
2684 0 : &MATERIALIZED_PAGE_CACHE_HIT_DIRECT,
2685 0 : &UNEXPECTED_ONDEMAND_DOWNLOADS,
2686 0 : &WALRECEIVER_STARTED_CONNECTIONS,
2687 0 : &WALRECEIVER_BROKER_UPDATES,
2688 0 : &WALRECEIVER_CANDIDATES_ADDED,
2689 0 : &WALRECEIVER_CANDIDATES_REMOVED,
2690 0 : &tokio_epoll_uring::THREAD_LOCAL_LAUNCH_FAILURES,
2691 0 : &tokio_epoll_uring::THREAD_LOCAL_LAUNCH_SUCCESSES,
2692 0 : ]
2693 0 : .into_iter()
2694 0 : .for_each(|c| {
2695 0 : Lazy::force(c);
2696 0 : });
2697 0 :
2698 0 : // Deletion queue stats
2699 0 : Lazy::force(&DELETION_QUEUE);
2700 0 :
2701 0 : // Tenant stats
2702 0 : Lazy::force(&TENANT);
2703 0 :
2704 0 : // Tenant manager stats
2705 0 : Lazy::force(&TENANT_MANAGER);
2706 0 :
2707 0 : Lazy::force(&crate::tenant::storage_layer::layer::LAYER_IMPL_METRICS);
2708 0 : Lazy::force(&disk_usage_based_eviction::METRICS);
2709 :
2710 0 : for state_name in pageserver_api::models::TenantState::VARIANTS {
2711 0 : // initialize the metric for all gauges, otherwise the time series might seemingly show
2712 0 : // values from last restart.
2713 0 : TENANT_STATE_METRIC.with_label_values(&[state_name]).set(0);
2714 0 : }
2715 :
2716 : // countervecs
2717 0 : [&BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT]
2718 0 : .into_iter()
2719 0 : .for_each(|c| {
2720 0 : Lazy::force(c);
2721 0 : });
2722 0 :
2723 0 : // gauges
2724 0 : WALRECEIVER_ACTIVE_MANAGERS.get();
2725 0 :
2726 0 : // histograms
2727 0 : [
2728 0 : &READ_NUM_FS_LAYERS,
2729 0 : &WAIT_LSN_TIME,
2730 0 : &WAL_REDO_TIME,
2731 0 : &WAL_REDO_RECORDS_HISTOGRAM,
2732 0 : &WAL_REDO_BYTES_HISTOGRAM,
2733 0 : &WAL_REDO_PROCESS_LAUNCH_DURATION_HISTOGRAM,
2734 0 : ]
2735 0 : .into_iter()
2736 0 : .for_each(|h| {
2737 0 : Lazy::force(h);
2738 0 : });
2739 0 :
2740 0 : // Custom
2741 0 : Lazy::force(&RECONSTRUCT_TIME);
2742 0 : Lazy::force(&tenant_throttling::TIMELINE_GET);
2743 0 : }
|
