Line data Source code
1 : use enum_map::EnumMap;
2 : use metrics::{
3 : register_counter_vec, register_gauge_vec, register_histogram, register_histogram_vec,
4 : register_int_counter, register_int_counter_pair_vec, register_int_counter_vec,
5 : register_int_gauge, register_int_gauge_vec, register_uint_gauge, register_uint_gauge_vec,
6 : Counter, CounterVec, GaugeVec, Histogram, HistogramVec, IntCounter, IntCounterPair,
7 : IntCounterPairVec, IntCounterVec, IntGauge, IntGaugeVec, UIntGauge, UIntGaugeVec,
8 : };
9 : use once_cell::sync::Lazy;
10 : use pageserver_api::shard::TenantShardId;
11 : use strum::{EnumCount, IntoEnumIterator, VariantNames};
12 : use strum_macros::{EnumVariantNames, IntoStaticStr};
13 : use tracing::warn;
14 : use utils::id::TimelineId;
15 :
16 : /// Prometheus histogram buckets (in seconds) for operations in the critical
17 : /// path. In other words, operations that directly affect that latency of user
18 : /// queries.
19 : ///
20 : /// The buckets capture the majority of latencies in the microsecond and
21 : /// millisecond range but also extend far enough up to distinguish "bad" from
22 : /// "really bad".
23 : const CRITICAL_OP_BUCKETS: &[f64] = &[
24 : 0.000_001, 0.000_010, 0.000_100, // 1 us, 10 us, 100 us
25 : 0.001_000, 0.010_000, 0.100_000, // 1 ms, 10 ms, 100 ms
26 : 1.0, 10.0, 100.0, // 1 s, 10 s, 100 s
27 : ];
28 :
29 : // Metrics collected on operations on the storage repository.
30 3056 : #[derive(Debug, EnumVariantNames, IntoStaticStr)]
31 : #[strum(serialize_all = "kebab_case")]
32 : pub(crate) enum StorageTimeOperation {
33 : #[strum(serialize = "layer flush")]
34 : LayerFlush,
35 :
36 : #[strum(serialize = "compact")]
37 : Compact,
38 :
39 : #[strum(serialize = "create images")]
40 : CreateImages,
41 :
42 : #[strum(serialize = "logical size")]
43 : LogicalSize,
44 :
45 : #[strum(serialize = "imitate logical size")]
46 : ImitateLogicalSize,
47 :
48 : #[strum(serialize = "load layer map")]
49 : LoadLayerMap,
50 :
51 : #[strum(serialize = "gc")]
52 : Gc,
53 :
54 : #[strum(serialize = "find gc cutoffs")]
55 : FindGcCutoffs,
56 :
57 : #[strum(serialize = "create tenant")]
58 : CreateTenant,
59 : }
60 :
61 140 : pub(crate) static STORAGE_TIME_SUM_PER_TIMELINE: Lazy<CounterVec> = Lazy::new(|| {
62 : register_counter_vec!(
63 : "pageserver_storage_operations_seconds_sum",
64 : "Total time spent on storage operations with operation, tenant and timeline dimensions",
65 : &["operation", "tenant_id", "shard_id", "timeline_id"],
66 : )
67 140 : .expect("failed to define a metric")
68 140 : });
69 :
70 140 : pub(crate) static STORAGE_TIME_COUNT_PER_TIMELINE: Lazy<IntCounterVec> = Lazy::new(|| {
71 : register_int_counter_vec!(
72 : "pageserver_storage_operations_seconds_count",
73 : "Count of storage operations with operation, tenant and timeline dimensions",
74 : &["operation", "tenant_id", "shard_id", "timeline_id"],
75 : )
76 140 : .expect("failed to define a metric")
77 140 : });
78 :
79 : // Buckets for background operations like compaction, GC, size calculation
80 : const STORAGE_OP_BUCKETS: &[f64] = &[0.010, 0.100, 1.0, 10.0, 100.0, 1000.0];
81 :
82 140 : pub(crate) static STORAGE_TIME_GLOBAL: Lazy<HistogramVec> = Lazy::new(|| {
83 : register_histogram_vec!(
84 : "pageserver_storage_operations_seconds_global",
85 : "Time spent on storage operations",
86 : &["operation"],
87 : STORAGE_OP_BUCKETS.into(),
88 : )
89 140 : .expect("failed to define a metric")
90 140 : });
91 :
92 137 : pub(crate) static READ_NUM_LAYERS_VISITED: Lazy<Histogram> = Lazy::new(|| {
93 : register_histogram!(
94 : "pageserver_layers_visited_per_read_global",
95 : "Number of layers visited to reconstruct one key",
96 : vec![1.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0],
97 : )
98 137 : .expect("failed to define a metric")
99 137 : });
100 :
101 24 : pub(crate) static VEC_READ_NUM_LAYERS_VISITED: Lazy<Histogram> = Lazy::new(|| {
102 : register_histogram!(
103 : "pageserver_layers_visited_per_vectored_read_global",
104 : "Average number of layers visited to reconstruct one key",
105 : vec![1.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0],
106 : )
107 24 : .expect("failed to define a metric")
108 24 : });
109 :
110 : // Metrics collected on operations on the storage repository.
111 : #[derive(
112 548 : Clone, Copy, enum_map::Enum, strum_macros::EnumString, strum_macros::Display, IntoStaticStr,
113 : )]
114 : pub(crate) enum GetKind {
115 : Singular,
116 : Vectored,
117 : }
118 :
119 : pub(crate) struct ReconstructTimeMetrics {
120 : singular: Histogram,
121 : vectored: Histogram,
122 : }
123 :
124 137 : pub(crate) static RECONSTRUCT_TIME: Lazy<ReconstructTimeMetrics> = Lazy::new(|| {
125 137 : let inner = register_histogram_vec!(
126 : "pageserver_getpage_reconstruct_seconds",
127 : "Time spent in reconstruct_value (reconstruct a page from deltas)",
128 : &["get_kind"],
129 : CRITICAL_OP_BUCKETS.into(),
130 : )
131 137 : .expect("failed to define a metric");
132 137 :
133 137 : ReconstructTimeMetrics {
134 137 : singular: inner.with_label_values(&[GetKind::Singular.into()]),
135 137 : vectored: inner.with_label_values(&[GetKind::Vectored.into()]),
136 137 : }
137 137 : });
138 :
139 : impl ReconstructTimeMetrics {
140 625860 : pub(crate) fn for_get_kind(&self, get_kind: GetKind) -> &Histogram {
141 625860 : match get_kind {
142 625790 : GetKind::Singular => &self.singular,
143 70 : GetKind::Vectored => &self.vectored,
144 : }
145 625860 : }
146 : }
147 :
148 : pub(crate) struct ReconstructDataTimeMetrics {
149 : singular: Histogram,
150 : vectored: Histogram,
151 : }
152 :
153 : impl ReconstructDataTimeMetrics {
154 626000 : pub(crate) fn for_get_kind(&self, get_kind: GetKind) -> &Histogram {
155 626000 : match get_kind {
156 625930 : GetKind::Singular => &self.singular,
157 70 : GetKind::Vectored => &self.vectored,
158 : }
159 626000 : }
160 : }
161 :
162 137 : pub(crate) static GET_RECONSTRUCT_DATA_TIME: Lazy<ReconstructDataTimeMetrics> = Lazy::new(|| {
163 137 : let inner = register_histogram_vec!(
164 : "pageserver_getpage_get_reconstruct_data_seconds",
165 : "Time spent in get_reconstruct_value_data",
166 : &["get_kind"],
167 : CRITICAL_OP_BUCKETS.into(),
168 : )
169 137 : .expect("failed to define a metric");
170 137 :
171 137 : ReconstructDataTimeMetrics {
172 137 : singular: inner.with_label_values(&[GetKind::Singular.into()]),
173 137 : vectored: inner.with_label_values(&[GetKind::Vectored.into()]),
174 137 : }
175 137 : });
176 :
177 : pub(crate) struct GetVectoredLatency {
178 : map: EnumMap<TaskKind, Option<Histogram>>,
179 : }
180 :
181 : #[allow(dead_code)]
182 : pub(crate) struct ScanLatency {
183 : map: EnumMap<TaskKind, Option<Histogram>>,
184 : }
185 :
186 : impl GetVectoredLatency {
187 : // Only these task types perform vectored gets. Filter all other tasks out to reduce total
188 : // cardinality of the metric.
189 : const TRACKED_TASK_KINDS: [TaskKind; 2] = [TaskKind::Compaction, TaskKind::PageRequestHandler];
190 :
191 940 : pub(crate) fn for_task_kind(&self, task_kind: TaskKind) -> Option<&Histogram> {
192 940 : self.map[task_kind].as_ref()
193 940 : }
194 : }
195 :
196 : impl ScanLatency {
197 : // Only these task types perform vectored gets. Filter all other tasks out to reduce total
198 : // cardinality of the metric.
199 : const TRACKED_TASK_KINDS: [TaskKind; 1] = [TaskKind::PageRequestHandler];
200 :
201 12 : pub(crate) fn for_task_kind(&self, task_kind: TaskKind) -> Option<&Histogram> {
202 12 : self.map[task_kind].as_ref()
203 12 : }
204 : }
205 :
206 : pub(crate) struct ScanLatencyOngoingRecording<'a> {
207 : parent: &'a Histogram,
208 : start: std::time::Instant,
209 : }
210 :
211 : impl<'a> ScanLatencyOngoingRecording<'a> {
212 0 : pub(crate) fn start_recording(parent: &'a Histogram) -> ScanLatencyOngoingRecording<'a> {
213 0 : let start = Instant::now();
214 0 : ScanLatencyOngoingRecording { parent, start }
215 0 : }
216 :
217 0 : pub(crate) fn observe(self, throttled: Option<Duration>) {
218 0 : let elapsed = self.start.elapsed();
219 0 : let ex_throttled = if let Some(throttled) = throttled {
220 0 : elapsed.checked_sub(throttled)
221 : } else {
222 0 : Some(elapsed)
223 : };
224 0 : if let Some(ex_throttled) = ex_throttled {
225 0 : self.parent.observe(ex_throttled.as_secs_f64());
226 0 : } else {
227 0 : use utils::rate_limit::RateLimit;
228 0 : static LOGGED: Lazy<Mutex<RateLimit>> =
229 0 : Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(10))));
230 0 : let mut rate_limit = LOGGED.lock().unwrap();
231 0 : rate_limit.call(|| {
232 0 : warn!("error deducting time spent throttled; this message is logged at a global rate limit");
233 0 : });
234 0 : }
235 0 : }
236 : }
237 :
238 130 : pub(crate) static GET_VECTORED_LATENCY: Lazy<GetVectoredLatency> = Lazy::new(|| {
239 130 : let inner = register_histogram_vec!(
240 : "pageserver_get_vectored_seconds",
241 : "Time spent in get_vectored, excluding time spent in timeline_get_throttle.",
242 : &["task_kind"],
243 : CRITICAL_OP_BUCKETS.into(),
244 : )
245 130 : .expect("failed to define a metric");
246 130 :
247 130 : GetVectoredLatency {
248 3900 : map: EnumMap::from_array(std::array::from_fn(|task_kind_idx| {
249 3900 : let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind_idx);
250 3900 :
251 3900 : if GetVectoredLatency::TRACKED_TASK_KINDS.contains(&task_kind) {
252 260 : let task_kind = task_kind.into();
253 260 : Some(inner.with_label_values(&[task_kind]))
254 : } else {
255 3640 : None
256 : }
257 3900 : })),
258 130 : }
259 130 : });
260 :
261 4 : pub(crate) static SCAN_LATENCY: Lazy<ScanLatency> = Lazy::new(|| {
262 4 : let inner = register_histogram_vec!(
263 : "pageserver_scan_seconds",
264 : "Time spent in scan, excluding time spent in timeline_get_throttle.",
265 : &["task_kind"],
266 : CRITICAL_OP_BUCKETS.into(),
267 : )
268 4 : .expect("failed to define a metric");
269 4 :
270 4 : ScanLatency {
271 120 : map: EnumMap::from_array(std::array::from_fn(|task_kind_idx| {
272 120 : let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind_idx);
273 120 :
274 120 : if ScanLatency::TRACKED_TASK_KINDS.contains(&task_kind) {
275 4 : let task_kind = task_kind.into();
276 4 : Some(inner.with_label_values(&[task_kind]))
277 : } else {
278 116 : None
279 : }
280 120 : })),
281 4 : }
282 4 : });
283 :
284 : pub(crate) struct PageCacheMetricsForTaskKind {
285 : pub read_accesses_immutable: IntCounter,
286 : pub read_hits_immutable: IntCounter,
287 : }
288 :
289 : pub(crate) struct PageCacheMetrics {
290 : map: EnumMap<TaskKind, EnumMap<PageContentKind, PageCacheMetricsForTaskKind>>,
291 : }
292 :
293 72 : static PAGE_CACHE_READ_HITS: Lazy<IntCounterVec> = Lazy::new(|| {
294 : register_int_counter_vec!(
295 : "pageserver_page_cache_read_hits_total",
296 : "Number of read accesses to the page cache that hit",
297 : &["task_kind", "key_kind", "content_kind", "hit_kind"]
298 : )
299 72 : .expect("failed to define a metric")
300 72 : });
301 :
302 72 : static PAGE_CACHE_READ_ACCESSES: Lazy<IntCounterVec> = Lazy::new(|| {
303 : register_int_counter_vec!(
304 : "pageserver_page_cache_read_accesses_total",
305 : "Number of read accesses to the page cache",
306 : &["task_kind", "key_kind", "content_kind"]
307 : )
308 72 : .expect("failed to define a metric")
309 72 : });
310 :
311 72 : pub(crate) static PAGE_CACHE: Lazy<PageCacheMetrics> = Lazy::new(|| PageCacheMetrics {
312 2160 : map: EnumMap::from_array(std::array::from_fn(|task_kind| {
313 2160 : let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind);
314 2160 : let task_kind: &'static str = task_kind.into();
315 12960 : EnumMap::from_array(std::array::from_fn(|content_kind| {
316 12960 : let content_kind = <PageContentKind as enum_map::Enum>::from_usize(content_kind);
317 12960 : let content_kind: &'static str = content_kind.into();
318 12960 : PageCacheMetricsForTaskKind {
319 12960 : read_accesses_immutable: {
320 12960 : PAGE_CACHE_READ_ACCESSES
321 12960 : .get_metric_with_label_values(&[task_kind, "immutable", content_kind])
322 12960 : .unwrap()
323 12960 : },
324 12960 :
325 12960 : read_hits_immutable: {
326 12960 : PAGE_CACHE_READ_HITS
327 12960 : .get_metric_with_label_values(&[task_kind, "immutable", content_kind, "-"])
328 12960 : .unwrap()
329 12960 : },
330 12960 : }
331 12960 : }))
332 2160 : })),
333 72 : });
334 :
335 : impl PageCacheMetrics {
336 14194196 : pub(crate) fn for_ctx(&self, ctx: &RequestContext) -> &PageCacheMetricsForTaskKind {
337 14194196 : &self.map[ctx.task_kind()][ctx.page_content_kind()]
338 14194196 : }
339 : }
340 :
341 : pub(crate) struct PageCacheSizeMetrics {
342 : pub max_bytes: UIntGauge,
343 :
344 : pub current_bytes_immutable: UIntGauge,
345 : }
346 :
347 72 : static PAGE_CACHE_SIZE_CURRENT_BYTES: Lazy<UIntGaugeVec> = Lazy::new(|| {
348 : register_uint_gauge_vec!(
349 : "pageserver_page_cache_size_current_bytes",
350 : "Current size of the page cache in bytes, by key kind",
351 : &["key_kind"]
352 : )
353 72 : .expect("failed to define a metric")
354 72 : });
355 :
356 : pub(crate) static PAGE_CACHE_SIZE: Lazy<PageCacheSizeMetrics> =
357 72 : Lazy::new(|| PageCacheSizeMetrics {
358 : max_bytes: {
359 : register_uint_gauge!(
360 : "pageserver_page_cache_size_max_bytes",
361 : "Maximum size of the page cache in bytes"
362 : )
363 72 : .expect("failed to define a metric")
364 72 : },
365 72 : current_bytes_immutable: {
366 72 : PAGE_CACHE_SIZE_CURRENT_BYTES
367 72 : .get_metric_with_label_values(&["immutable"])
368 72 : .unwrap()
369 72 : },
370 72 : });
371 :
372 : pub(crate) mod page_cache_eviction_metrics {
373 : use std::num::NonZeroUsize;
374 :
375 : use metrics::{register_int_counter_vec, IntCounter, IntCounterVec};
376 : use once_cell::sync::Lazy;
377 :
378 : #[derive(Clone, Copy)]
379 : pub(crate) enum Outcome {
380 : FoundSlotUnused { iters: NonZeroUsize },
381 : FoundSlotEvicted { iters: NonZeroUsize },
382 : ItersExceeded { iters: NonZeroUsize },
383 : }
384 :
385 72 : static ITERS_TOTAL_VEC: Lazy<IntCounterVec> = Lazy::new(|| {
386 : register_int_counter_vec!(
387 : "pageserver_page_cache_find_victim_iters_total",
388 : "Counter for the number of iterations in the find_victim loop",
389 : &["outcome"],
390 : )
391 72 : .expect("failed to define a metric")
392 72 : });
393 :
394 72 : static CALLS_VEC: Lazy<IntCounterVec> = Lazy::new(|| {
395 : register_int_counter_vec!(
396 : "pageserver_page_cache_find_victim_calls",
397 : "Incremented at the end of each find_victim() call.\
398 : Filter by outcome to get e.g., eviction rate.",
399 : &["outcome"]
400 : )
401 72 : .unwrap()
402 72 : });
403 :
404 167030 : pub(crate) fn observe(outcome: Outcome) {
405 167030 : macro_rules! dry {
406 167030 : ($label:literal, $iters:expr) => {{
407 167030 : static LABEL: &'static str = $label;
408 167030 : static ITERS_TOTAL: Lazy<IntCounter> =
409 167030 : Lazy::new(|| ITERS_TOTAL_VEC.with_label_values(&[LABEL]));
410 167030 : static CALLS: Lazy<IntCounter> =
411 167030 : Lazy::new(|| CALLS_VEC.with_label_values(&[LABEL]));
412 167030 : ITERS_TOTAL.inc_by(($iters.get()) as u64);
413 167030 : CALLS.inc();
414 167030 : }};
415 167030 : }
416 167030 : match outcome {
417 1888 : Outcome::FoundSlotUnused { iters } => dry!("found_empty", iters),
418 165142 : Outcome::FoundSlotEvicted { iters } => {
419 165142 : dry!("found_evicted", iters)
420 : }
421 0 : Outcome::ItersExceeded { iters } => {
422 0 : dry!("err_iters_exceeded", iters);
423 0 : super::page_cache_errors_inc(super::PageCacheErrorKind::EvictIterLimit);
424 0 : }
425 : }
426 167030 : }
427 : }
428 :
429 0 : static PAGE_CACHE_ERRORS: Lazy<IntCounterVec> = Lazy::new(|| {
430 : register_int_counter_vec!(
431 : "page_cache_errors_total",
432 : "Number of timeouts while acquiring a pinned slot in the page cache",
433 : &["error_kind"]
434 : )
435 0 : .expect("failed to define a metric")
436 0 : });
437 :
438 0 : #[derive(IntoStaticStr)]
439 : #[strum(serialize_all = "kebab_case")]
440 : pub(crate) enum PageCacheErrorKind {
441 : AcquirePinnedSlotTimeout,
442 : EvictIterLimit,
443 : }
444 :
445 0 : pub(crate) fn page_cache_errors_inc(error_kind: PageCacheErrorKind) {
446 0 : PAGE_CACHE_ERRORS
447 0 : .get_metric_with_label_values(&[error_kind.into()])
448 0 : .unwrap()
449 0 : .inc();
450 0 : }
451 :
452 16 : pub(crate) static WAIT_LSN_TIME: Lazy<Histogram> = Lazy::new(|| {
453 : register_histogram!(
454 : "pageserver_wait_lsn_seconds",
455 : "Time spent waiting for WAL to arrive",
456 : CRITICAL_OP_BUCKETS.into(),
457 : )
458 16 : .expect("failed to define a metric")
459 16 : });
460 :
461 140 : static LAST_RECORD_LSN: Lazy<IntGaugeVec> = Lazy::new(|| {
462 : register_int_gauge_vec!(
463 : "pageserver_last_record_lsn",
464 : "Last record LSN grouped by timeline",
465 : &["tenant_id", "shard_id", "timeline_id"]
466 : )
467 140 : .expect("failed to define a metric")
468 140 : });
469 :
470 140 : static STANDBY_HORIZON: Lazy<IntGaugeVec> = Lazy::new(|| {
471 : register_int_gauge_vec!(
472 : "pageserver_standby_horizon",
473 : "Standby apply LSN for which GC is hold off, by timeline.",
474 : &["tenant_id", "shard_id", "timeline_id"]
475 : )
476 140 : .expect("failed to define a metric")
477 140 : });
478 :
479 140 : static RESIDENT_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
480 : register_uint_gauge_vec!(
481 : "pageserver_resident_physical_size",
482 : "The size of the layer files present in the pageserver's filesystem.",
483 : &["tenant_id", "shard_id", "timeline_id"]
484 : )
485 140 : .expect("failed to define a metric")
486 140 : });
487 :
488 134 : pub(crate) static RESIDENT_PHYSICAL_SIZE_GLOBAL: Lazy<UIntGauge> = Lazy::new(|| {
489 : register_uint_gauge!(
490 : "pageserver_resident_physical_size_global",
491 : "Like `pageserver_resident_physical_size`, but without tenant/timeline dimensions."
492 : )
493 134 : .expect("failed to define a metric")
494 134 : });
495 :
496 140 : static REMOTE_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
497 : register_uint_gauge_vec!(
498 : "pageserver_remote_physical_size",
499 : "The size of the layer files present in the remote storage that are listed in the remote index_part.json.",
500 : // Corollary: If any files are missing from the index part, they won't be included here.
501 : &["tenant_id", "shard_id", "timeline_id"]
502 : )
503 140 : .expect("failed to define a metric")
504 140 : });
505 :
506 140 : static REMOTE_PHYSICAL_SIZE_GLOBAL: Lazy<UIntGauge> = Lazy::new(|| {
507 : register_uint_gauge!(
508 : "pageserver_remote_physical_size_global",
509 : "Like `pageserver_remote_physical_size`, but without tenant/timeline dimensions."
510 : )
511 140 : .expect("failed to define a metric")
512 140 : });
513 :
514 4 : pub(crate) static REMOTE_ONDEMAND_DOWNLOADED_LAYERS: Lazy<IntCounter> = Lazy::new(|| {
515 : register_int_counter!(
516 : "pageserver_remote_ondemand_downloaded_layers_total",
517 : "Total on-demand downloaded layers"
518 : )
519 4 : .unwrap()
520 4 : });
521 :
522 4 : pub(crate) static REMOTE_ONDEMAND_DOWNLOADED_BYTES: Lazy<IntCounter> = Lazy::new(|| {
523 : register_int_counter!(
524 : "pageserver_remote_ondemand_downloaded_bytes_total",
525 : "Total bytes of layers on-demand downloaded",
526 : )
527 4 : .unwrap()
528 4 : });
529 :
530 140 : static CURRENT_LOGICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
531 : register_uint_gauge_vec!(
532 : "pageserver_current_logical_size",
533 : "Current logical size grouped by timeline",
534 : &["tenant_id", "shard_id", "timeline_id"]
535 : )
536 140 : .expect("failed to define current logical size metric")
537 140 : });
538 :
539 140 : static AUX_FILE_SIZE: Lazy<IntGaugeVec> = Lazy::new(|| {
540 : register_int_gauge_vec!(
541 : "pageserver_aux_file_estimated_size",
542 : "The size of all aux files for a timeline in aux file v2 store.",
543 : &["tenant_id", "shard_id", "timeline_id"]
544 : )
545 140 : .expect("failed to define a metric")
546 140 : });
547 :
548 140 : static VALID_LSN_LEASE_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
549 : register_uint_gauge_vec!(
550 : "pageserver_valid_lsn_lease_count",
551 : "The number of valid leases after refreshing gc info.",
552 : &["tenant_id", "shard_id", "timeline_id"],
553 : )
554 140 : .expect("failed to define a metric")
555 140 : });
556 :
557 : pub(crate) mod initial_logical_size {
558 : use metrics::{register_int_counter, register_int_counter_vec, IntCounter, IntCounterVec};
559 : use once_cell::sync::Lazy;
560 :
561 : pub(crate) struct StartCalculation(IntCounterVec);
562 140 : pub(crate) static START_CALCULATION: Lazy<StartCalculation> = Lazy::new(|| {
563 140 : StartCalculation(
564 140 : register_int_counter_vec!(
565 : "pageserver_initial_logical_size_start_calculation",
566 : "Incremented each time we start an initial logical size calculation attempt. \
567 : The `circumstances` label provides some additional details.",
568 : &["attempt", "circumstances"]
569 140 : )
570 140 : .unwrap(),
571 140 : )
572 140 : });
573 :
574 : struct DropCalculation {
575 : first: IntCounter,
576 : retry: IntCounter,
577 : }
578 :
579 140 : static DROP_CALCULATION: Lazy<DropCalculation> = Lazy::new(|| {
580 140 : let vec = register_int_counter_vec!(
581 : "pageserver_initial_logical_size_drop_calculation",
582 : "Incremented each time we abort a started size calculation attmpt.",
583 : &["attempt"]
584 : )
585 140 : .unwrap();
586 140 : DropCalculation {
587 140 : first: vec.with_label_values(&["first"]),
588 140 : retry: vec.with_label_values(&["retry"]),
589 140 : }
590 140 : });
591 :
592 : pub(crate) struct Calculated {
593 : pub(crate) births: IntCounter,
594 : pub(crate) deaths: IntCounter,
595 : }
596 :
597 140 : pub(crate) static CALCULATED: Lazy<Calculated> = Lazy::new(|| Calculated {
598 : births: register_int_counter!(
599 : "pageserver_initial_logical_size_finish_calculation",
600 : "Incremented every time we finish calculation of initial logical size.\
601 : If everything is working well, this should happen at most once per Timeline object."
602 : )
603 140 : .unwrap(),
604 : deaths: register_int_counter!(
605 : "pageserver_initial_logical_size_drop_finished_calculation",
606 : "Incremented when we drop a finished initial logical size calculation result.\
607 : Mainly useful to turn pageserver_initial_logical_size_finish_calculation into a gauge."
608 : )
609 140 : .unwrap(),
610 140 : });
611 :
612 : pub(crate) struct OngoingCalculationGuard {
613 : inc_drop_calculation: Option<IntCounter>,
614 : }
615 :
616 152 : #[derive(strum_macros::IntoStaticStr)]
617 : pub(crate) enum StartCircumstances {
618 : EmptyInitial,
619 : SkippedConcurrencyLimiter,
620 : AfterBackgroundTasksRateLimit,
621 : }
622 :
623 : impl StartCalculation {
624 152 : pub(crate) fn first(&self, circumstances: StartCircumstances) -> OngoingCalculationGuard {
625 152 : let circumstances_label: &'static str = circumstances.into();
626 152 : self.0
627 152 : .with_label_values(&["first", circumstances_label])
628 152 : .inc();
629 152 : OngoingCalculationGuard {
630 152 : inc_drop_calculation: Some(DROP_CALCULATION.first.clone()),
631 152 : }
632 152 : }
633 0 : pub(crate) fn retry(&self, circumstances: StartCircumstances) -> OngoingCalculationGuard {
634 0 : let circumstances_label: &'static str = circumstances.into();
635 0 : self.0
636 0 : .with_label_values(&["retry", circumstances_label])
637 0 : .inc();
638 0 : OngoingCalculationGuard {
639 0 : inc_drop_calculation: Some(DROP_CALCULATION.retry.clone()),
640 0 : }
641 0 : }
642 : }
643 :
644 : impl Drop for OngoingCalculationGuard {
645 152 : fn drop(&mut self) {
646 152 : if let Some(counter) = self.inc_drop_calculation.take() {
647 0 : counter.inc();
648 152 : }
649 152 : }
650 : }
651 :
652 : impl OngoingCalculationGuard {
653 152 : pub(crate) fn calculation_result_saved(mut self) -> FinishedCalculationGuard {
654 152 : drop(self.inc_drop_calculation.take());
655 152 : CALCULATED.births.inc();
656 152 : FinishedCalculationGuard {
657 152 : inc_on_drop: CALCULATED.deaths.clone(),
658 152 : }
659 152 : }
660 : }
661 :
662 : pub(crate) struct FinishedCalculationGuard {
663 : inc_on_drop: IntCounter,
664 : }
665 :
666 : impl Drop for FinishedCalculationGuard {
667 7 : fn drop(&mut self) {
668 7 : self.inc_on_drop.inc();
669 7 : }
670 : }
671 :
672 : // context: https://github.com/neondatabase/neon/issues/5963
673 : pub(crate) static TIMELINES_WHERE_WALRECEIVER_GOT_APPROXIMATE_SIZE: Lazy<IntCounter> =
674 0 : Lazy::new(|| {
675 : register_int_counter!(
676 : "pageserver_initial_logical_size_timelines_where_walreceiver_got_approximate_size",
677 : "Counter for the following event: walreceiver calls\
678 : Timeline::get_current_logical_size() and it returns `Approximate` for the first time."
679 : )
680 0 : .unwrap()
681 0 : });
682 : }
683 :
684 0 : static DIRECTORY_ENTRIES_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
685 : register_uint_gauge_vec!(
686 : "pageserver_directory_entries_count",
687 : "Sum of the entries in pageserver-stored directory listings",
688 : &["tenant_id", "shard_id", "timeline_id"]
689 : )
690 0 : .expect("failed to define a metric")
691 0 : });
692 :
693 142 : pub(crate) static TENANT_STATE_METRIC: Lazy<UIntGaugeVec> = Lazy::new(|| {
694 : register_uint_gauge_vec!(
695 : "pageserver_tenant_states_count",
696 : "Count of tenants per state",
697 : &["state"]
698 : )
699 142 : .expect("Failed to register pageserver_tenant_states_count metric")
700 142 : });
701 :
702 : /// A set of broken tenants.
703 : ///
704 : /// These are expected to be so rare that a set is fine. Set as in a new timeseries per each broken
705 : /// tenant.
706 12 : pub(crate) static BROKEN_TENANTS_SET: Lazy<UIntGaugeVec> = Lazy::new(|| {
707 : register_uint_gauge_vec!(
708 : "pageserver_broken_tenants_count",
709 : "Set of broken tenants",
710 : &["tenant_id", "shard_id"]
711 : )
712 12 : .expect("Failed to register pageserver_tenant_states_count metric")
713 12 : });
714 :
715 6 : pub(crate) static TENANT_SYNTHETIC_SIZE_METRIC: Lazy<UIntGaugeVec> = Lazy::new(|| {
716 : register_uint_gauge_vec!(
717 : "pageserver_tenant_synthetic_cached_size_bytes",
718 : "Synthetic size of each tenant in bytes",
719 : &["tenant_id"]
720 : )
721 6 : .expect("Failed to register pageserver_tenant_synthetic_cached_size_bytes metric")
722 6 : });
723 :
724 0 : pub(crate) static EVICTION_ITERATION_DURATION: Lazy<HistogramVec> = Lazy::new(|| {
725 : register_histogram_vec!(
726 : "pageserver_eviction_iteration_duration_seconds_global",
727 : "Time spent on a single eviction iteration",
728 : &["period_secs", "threshold_secs"],
729 : STORAGE_OP_BUCKETS.into(),
730 : )
731 0 : .expect("failed to define a metric")
732 0 : });
733 :
734 140 : static EVICTIONS: Lazy<IntCounterVec> = Lazy::new(|| {
735 : register_int_counter_vec!(
736 : "pageserver_evictions",
737 : "Number of layers evicted from the pageserver",
738 : &["tenant_id", "shard_id", "timeline_id"]
739 : )
740 140 : .expect("failed to define a metric")
741 140 : });
742 :
743 140 : static EVICTIONS_WITH_LOW_RESIDENCE_DURATION: Lazy<IntCounterVec> = Lazy::new(|| {
744 : register_int_counter_vec!(
745 : "pageserver_evictions_with_low_residence_duration",
746 : "If a layer is evicted that was resident for less than `low_threshold`, it is counted to this counter. \
747 : Residence duration is determined using the `residence_duration_data_source`.",
748 : &["tenant_id", "shard_id", "timeline_id", "residence_duration_data_source", "low_threshold_secs"]
749 : )
750 140 : .expect("failed to define a metric")
751 140 : });
752 :
753 0 : pub(crate) static UNEXPECTED_ONDEMAND_DOWNLOADS: Lazy<IntCounter> = Lazy::new(|| {
754 : register_int_counter!(
755 : "pageserver_unexpected_ondemand_downloads_count",
756 : "Number of unexpected on-demand downloads. \
757 : We log more context for each increment, so, forgo any labels in this metric.",
758 : )
759 0 : .expect("failed to define a metric")
760 0 : });
761 :
762 : /// How long did we take to start up? Broken down by labels to describe
763 : /// different phases of startup.
764 0 : pub static STARTUP_DURATION: Lazy<GaugeVec> = Lazy::new(|| {
765 : register_gauge_vec!(
766 : "pageserver_startup_duration_seconds",
767 : "Time taken by phases of pageserver startup, in seconds",
768 : &["phase"]
769 : )
770 0 : .expect("Failed to register pageserver_startup_duration_seconds metric")
771 0 : });
772 :
773 0 : pub static STARTUP_IS_LOADING: Lazy<UIntGauge> = Lazy::new(|| {
774 : register_uint_gauge!(
775 : "pageserver_startup_is_loading",
776 : "1 while in initial startup load of tenants, 0 at other times"
777 : )
778 0 : .expect("Failed to register pageserver_startup_is_loading")
779 0 : });
780 :
781 134 : pub(crate) static TIMELINE_EPHEMERAL_BYTES: Lazy<UIntGauge> = Lazy::new(|| {
782 : register_uint_gauge!(
783 : "pageserver_timeline_ephemeral_bytes",
784 : "Total number of bytes in ephemeral layers, summed for all timelines. Approximate, lazily updated."
785 : )
786 134 : .expect("Failed to register metric")
787 134 : });
788 :
789 : /// Metrics related to the lifecycle of a [`crate::tenant::Tenant`] object: things
790 : /// like how long it took to load.
791 : ///
792 : /// Note that these are process-global metrics, _not_ per-tenant metrics. Per-tenant
793 : /// metrics are rather expensive, and usually fine grained stuff makes more sense
794 : /// at a timeline level than tenant level.
795 : pub(crate) struct TenantMetrics {
796 : /// How long did tenants take to go from construction to active state?
797 : pub(crate) activation: Histogram,
798 : pub(crate) preload: Histogram,
799 : pub(crate) attach: Histogram,
800 :
801 : /// How many tenants are included in the initial startup of the pagesrever?
802 : pub(crate) startup_scheduled: IntCounter,
803 : pub(crate) startup_complete: IntCounter,
804 : }
805 :
806 0 : pub(crate) static TENANT: Lazy<TenantMetrics> = Lazy::new(|| {
807 0 : TenantMetrics {
808 0 : activation: register_histogram!(
809 : "pageserver_tenant_activation_seconds",
810 : "Time taken by tenants to activate, in seconds",
811 : CRITICAL_OP_BUCKETS.into()
812 0 : )
813 0 : .expect("Failed to register metric"),
814 0 : preload: register_histogram!(
815 : "pageserver_tenant_preload_seconds",
816 : "Time taken by tenants to load remote metadata on startup/attach, in seconds",
817 : CRITICAL_OP_BUCKETS.into()
818 0 : )
819 0 : .expect("Failed to register metric"),
820 0 : attach: register_histogram!(
821 : "pageserver_tenant_attach_seconds",
822 : "Time taken by tenants to intialize, after remote metadata is already loaded",
823 : CRITICAL_OP_BUCKETS.into()
824 0 : )
825 0 : .expect("Failed to register metric"),
826 0 : startup_scheduled: register_int_counter!(
827 : "pageserver_tenant_startup_scheduled",
828 : "Number of tenants included in pageserver startup (doesn't count tenants attached later)"
829 0 : ).expect("Failed to register metric"),
830 0 : startup_complete: register_int_counter!(
831 : "pageserver_tenant_startup_complete",
832 : "Number of tenants that have completed warm-up, or activated on-demand during initial startup: \
833 : should eventually reach `pageserver_tenant_startup_scheduled_total`. Does not include broken \
834 : tenants: such cases will lead to this metric never reaching the scheduled count."
835 0 : ).expect("Failed to register metric"),
836 0 : }
837 0 : });
838 :
839 : /// Each `Timeline`'s [`EVICTIONS_WITH_LOW_RESIDENCE_DURATION`] metric.
840 : #[derive(Debug)]
841 : pub(crate) struct EvictionsWithLowResidenceDuration {
842 : data_source: &'static str,
843 : threshold: Duration,
844 : counter: Option<IntCounter>,
845 : }
846 :
847 : pub(crate) struct EvictionsWithLowResidenceDurationBuilder {
848 : data_source: &'static str,
849 : threshold: Duration,
850 : }
851 :
852 : impl EvictionsWithLowResidenceDurationBuilder {
853 382 : pub fn new(data_source: &'static str, threshold: Duration) -> Self {
854 382 : Self {
855 382 : data_source,
856 382 : threshold,
857 382 : }
858 382 : }
859 :
860 382 : fn build(
861 382 : &self,
862 382 : tenant_id: &str,
863 382 : shard_id: &str,
864 382 : timeline_id: &str,
865 382 : ) -> EvictionsWithLowResidenceDuration {
866 382 : let counter = EVICTIONS_WITH_LOW_RESIDENCE_DURATION
867 382 : .get_metric_with_label_values(&[
868 382 : tenant_id,
869 382 : shard_id,
870 382 : timeline_id,
871 382 : self.data_source,
872 382 : &EvictionsWithLowResidenceDuration::threshold_label_value(self.threshold),
873 382 : ])
874 382 : .unwrap();
875 382 : EvictionsWithLowResidenceDuration {
876 382 : data_source: self.data_source,
877 382 : threshold: self.threshold,
878 382 : counter: Some(counter),
879 382 : }
880 382 : }
881 : }
882 :
883 : impl EvictionsWithLowResidenceDuration {
884 390 : fn threshold_label_value(threshold: Duration) -> String {
885 390 : format!("{}", threshold.as_secs())
886 390 : }
887 :
888 18 : pub fn observe(&self, observed_value: Duration) {
889 18 : if observed_value < self.threshold {
890 18 : self.counter
891 18 : .as_ref()
892 18 : .expect("nobody calls this function after `remove_from_vec`")
893 18 : .inc();
894 18 : }
895 18 : }
896 :
897 8 : pub fn change_threshold(
898 8 : &mut self,
899 8 : tenant_id: &str,
900 8 : shard_id: &str,
901 8 : timeline_id: &str,
902 8 : new_threshold: Duration,
903 8 : ) {
904 8 : if new_threshold == self.threshold {
905 8 : return;
906 0 : }
907 0 : let mut with_new = EvictionsWithLowResidenceDurationBuilder::new(
908 0 : self.data_source,
909 0 : new_threshold,
910 0 : )
911 0 : .build(tenant_id, shard_id, timeline_id);
912 0 : std::mem::swap(self, &mut with_new);
913 0 : with_new.remove(tenant_id, shard_id, timeline_id);
914 8 : }
915 :
916 : // This could be a `Drop` impl, but, we need the `tenant_id` and `timeline_id`.
917 8 : fn remove(&mut self, tenant_id: &str, shard_id: &str, timeline_id: &str) {
918 8 : let Some(_counter) = self.counter.take() else {
919 0 : return;
920 : };
921 :
922 8 : let threshold = Self::threshold_label_value(self.threshold);
923 8 :
924 8 : let removed = EVICTIONS_WITH_LOW_RESIDENCE_DURATION.remove_label_values(&[
925 8 : tenant_id,
926 8 : shard_id,
927 8 : timeline_id,
928 8 : self.data_source,
929 8 : &threshold,
930 8 : ]);
931 8 :
932 8 : match removed {
933 0 : Err(e) => {
934 0 : // this has been hit in staging as
935 0 : // <https://neondatabase.sentry.io/issues/4142396994/>, but we don't know how.
936 0 : // because we can be in the drop path already, don't risk:
937 0 : // - "double-panic => illegal instruction" or
938 0 : // - future "drop panick => abort"
939 0 : //
940 0 : // so just nag: (the error has the labels)
941 0 : tracing::warn!("failed to remove EvictionsWithLowResidenceDuration, it was already removed? {e:#?}");
942 : }
943 : Ok(()) => {
944 : // to help identify cases where we double-remove the same values, let's log all
945 : // deletions?
946 8 : tracing::info!("removed EvictionsWithLowResidenceDuration with {tenant_id}, {timeline_id}, {}, {threshold}", self.data_source);
947 : }
948 : }
949 8 : }
950 : }
951 :
952 : // Metrics collected on disk IO operations
953 : //
954 : // Roughly logarithmic scale.
955 : const STORAGE_IO_TIME_BUCKETS: &[f64] = &[
956 : 0.000030, // 30 usec
957 : 0.001000, // 1000 usec
958 : 0.030, // 30 ms
959 : 1.000, // 1000 ms
960 : 30.000, // 30000 ms
961 : ];
962 :
963 : /// VirtualFile fs operation variants.
964 : ///
965 : /// Operations:
966 : /// - open ([`std::fs::OpenOptions::open`])
967 : /// - close (dropping [`crate::virtual_file::VirtualFile`])
968 : /// - close-by-replace (close by replacement algorithm)
969 : /// - read (`read_at`)
970 : /// - write (`write_at`)
971 : /// - seek (modify internal position or file length query)
972 : /// - fsync ([`std::fs::File::sync_all`])
973 : /// - metadata ([`std::fs::File::metadata`])
974 : #[derive(
975 1431 : Debug, Clone, Copy, strum_macros::EnumCount, strum_macros::EnumIter, strum_macros::FromRepr,
976 : )]
977 : pub(crate) enum StorageIoOperation {
978 : Open,
979 : OpenAfterReplace,
980 : Close,
981 : CloseByReplace,
982 : Read,
983 : Write,
984 : Seek,
985 : Fsync,
986 : Metadata,
987 : }
988 :
989 : impl StorageIoOperation {
990 1431 : pub fn as_str(&self) -> &'static str {
991 1431 : match self {
992 159 : StorageIoOperation::Open => "open",
993 159 : StorageIoOperation::OpenAfterReplace => "open-after-replace",
994 159 : StorageIoOperation::Close => "close",
995 159 : StorageIoOperation::CloseByReplace => "close-by-replace",
996 159 : StorageIoOperation::Read => "read",
997 159 : StorageIoOperation::Write => "write",
998 159 : StorageIoOperation::Seek => "seek",
999 159 : StorageIoOperation::Fsync => "fsync",
1000 159 : StorageIoOperation::Metadata => "metadata",
1001 : }
1002 1431 : }
1003 : }
1004 :
1005 : /// Tracks time taken by fs operations near VirtualFile.
1006 : #[derive(Debug)]
1007 : pub(crate) struct StorageIoTime {
1008 : metrics: [Histogram; StorageIoOperation::COUNT],
1009 : }
1010 :
1011 : impl StorageIoTime {
1012 159 : fn new() -> Self {
1013 159 : let storage_io_histogram_vec = register_histogram_vec!(
1014 : "pageserver_io_operations_seconds",
1015 : "Time spent in IO operations",
1016 : &["operation"],
1017 : STORAGE_IO_TIME_BUCKETS.into()
1018 : )
1019 159 : .expect("failed to define a metric");
1020 1431 : let metrics = std::array::from_fn(|i| {
1021 1431 : let op = StorageIoOperation::from_repr(i).unwrap();
1022 1431 : storage_io_histogram_vec
1023 1431 : .get_metric_with_label_values(&[op.as_str()])
1024 1431 : .unwrap()
1025 1431 : });
1026 159 : Self { metrics }
1027 159 : }
1028 :
1029 1869738 : pub(crate) fn get(&self, op: StorageIoOperation) -> &Histogram {
1030 1869738 : &self.metrics[op as usize]
1031 1869738 : }
1032 : }
1033 :
1034 : pub(crate) static STORAGE_IO_TIME_METRIC: Lazy<StorageIoTime> = Lazy::new(StorageIoTime::new);
1035 :
1036 : const STORAGE_IO_SIZE_OPERATIONS: &[&str] = &["read", "write"];
1037 :
1038 : // Needed for the https://neonprod.grafana.net/d/5uK9tHL4k/picking-tenant-for-relocation?orgId=1
1039 156 : pub(crate) static STORAGE_IO_SIZE: Lazy<IntGaugeVec> = Lazy::new(|| {
1040 : register_int_gauge_vec!(
1041 : "pageserver_io_operations_bytes_total",
1042 : "Total amount of bytes read/written in IO operations",
1043 : &["operation", "tenant_id", "shard_id", "timeline_id"]
1044 : )
1045 156 : .expect("failed to define a metric")
1046 156 : });
1047 :
1048 : #[cfg(not(test))]
1049 : pub(crate) mod virtual_file_descriptor_cache {
1050 : use super::*;
1051 :
1052 0 : pub(crate) static SIZE_MAX: Lazy<UIntGauge> = Lazy::new(|| {
1053 : register_uint_gauge!(
1054 : "pageserver_virtual_file_descriptor_cache_size_max",
1055 : "Maximum number of open file descriptors in the cache."
1056 : )
1057 0 : .unwrap()
1058 0 : });
1059 :
1060 : // SIZE_CURRENT: derive it like so:
1061 : // ```
1062 : // sum (pageserver_io_operations_seconds_count{operation=~"^(open|open-after-replace)$")
1063 : // -ignoring(operation)
1064 : // sum(pageserver_io_operations_seconds_count{operation=~"^(close|close-by-replace)$"}
1065 : // ```
1066 : }
1067 :
1068 : #[cfg(not(test))]
1069 : pub(crate) mod virtual_file_io_engine {
1070 : use super::*;
1071 :
1072 0 : pub(crate) static KIND: Lazy<UIntGaugeVec> = Lazy::new(|| {
1073 : register_uint_gauge_vec!(
1074 : "pageserver_virtual_file_io_engine_kind",
1075 : "The configured io engine for VirtualFile",
1076 : &["kind"],
1077 : )
1078 0 : .unwrap()
1079 0 : });
1080 : }
1081 :
1082 : #[derive(Debug)]
1083 : struct GlobalAndPerTimelineHistogram {
1084 : global: Histogram,
1085 : per_tenant_timeline: Histogram,
1086 : }
1087 :
1088 : impl GlobalAndPerTimelineHistogram {
1089 10 : fn observe(&self, value: f64) {
1090 10 : self.global.observe(value);
1091 10 : self.per_tenant_timeline.observe(value);
1092 10 : }
1093 : }
1094 :
1095 : struct GlobalAndPerTimelineHistogramTimer<'a, 'c> {
1096 : h: &'a GlobalAndPerTimelineHistogram,
1097 : ctx: &'c RequestContext,
1098 : start: std::time::Instant,
1099 : op: SmgrQueryType,
1100 : }
1101 :
1102 : impl<'a, 'c> Drop for GlobalAndPerTimelineHistogramTimer<'a, 'c> {
1103 10 : fn drop(&mut self) {
1104 10 : let elapsed = self.start.elapsed();
1105 10 : let ex_throttled = self
1106 10 : .ctx
1107 10 : .micros_spent_throttled
1108 10 : .close_and_checked_sub_from(elapsed);
1109 10 : let ex_throttled = match ex_throttled {
1110 10 : Ok(res) => res,
1111 0 : Err(error) => {
1112 0 : use utils::rate_limit::RateLimit;
1113 0 : static LOGGED: Lazy<Mutex<enum_map::EnumMap<SmgrQueryType, RateLimit>>> =
1114 0 : Lazy::new(|| {
1115 0 : Mutex::new(enum_map::EnumMap::from_array(std::array::from_fn(|_| {
1116 0 : RateLimit::new(Duration::from_secs(10))
1117 0 : })))
1118 0 : });
1119 0 : let mut guard = LOGGED.lock().unwrap();
1120 0 : let rate_limit = &mut guard[self.op];
1121 0 : rate_limit.call(|| {
1122 0 : warn!(op=?self.op, error, "error deducting time spent throttled; this message is logged at a global rate limit");
1123 0 : });
1124 0 : elapsed
1125 : }
1126 : };
1127 10 : self.h.observe(ex_throttled.as_secs_f64());
1128 10 : }
1129 : }
1130 :
1131 : #[derive(
1132 : Debug,
1133 : Clone,
1134 : Copy,
1135 3970 : IntoStaticStr,
1136 : strum_macros::EnumCount,
1137 112 : strum_macros::EnumIter,
1138 1960 : strum_macros::FromRepr,
1139 : enum_map::Enum,
1140 : )]
1141 : #[strum(serialize_all = "snake_case")]
1142 : pub enum SmgrQueryType {
1143 : GetRelExists,
1144 : GetRelSize,
1145 : GetPageAtLsn,
1146 : GetDbSize,
1147 : GetSlruSegment,
1148 : }
1149 :
1150 : #[derive(Debug)]
1151 : pub(crate) struct SmgrQueryTimePerTimeline {
1152 : metrics: [GlobalAndPerTimelineHistogram; SmgrQueryType::COUNT],
1153 : }
1154 :
1155 142 : static SMGR_QUERY_TIME_PER_TENANT_TIMELINE: Lazy<HistogramVec> = Lazy::new(|| {
1156 : register_histogram_vec!(
1157 : "pageserver_smgr_query_seconds",
1158 : "Time spent on smgr query handling, aggegated by query type and tenant/timeline.",
1159 : &["smgr_query_type", "tenant_id", "shard_id", "timeline_id"],
1160 : CRITICAL_OP_BUCKETS.into(),
1161 : )
1162 142 : .expect("failed to define a metric")
1163 142 : });
1164 :
1165 142 : static SMGR_QUERY_TIME_GLOBAL_BUCKETS: Lazy<Vec<f64>> = Lazy::new(|| {
1166 142 : [
1167 142 : 1,
1168 142 : 10,
1169 142 : 20,
1170 142 : 40,
1171 142 : 60,
1172 142 : 80,
1173 142 : 100,
1174 142 : 200,
1175 142 : 300,
1176 142 : 400,
1177 142 : 500,
1178 142 : 600,
1179 142 : 700,
1180 142 : 800,
1181 142 : 900,
1182 142 : 1_000, // 1ms
1183 142 : 2_000,
1184 142 : 4_000,
1185 142 : 6_000,
1186 142 : 8_000,
1187 142 : 10_000, // 10ms
1188 142 : 20_000,
1189 142 : 40_000,
1190 142 : 60_000,
1191 142 : 80_000,
1192 142 : 100_000,
1193 142 : 200_000,
1194 142 : 400_000,
1195 142 : 600_000,
1196 142 : 800_000,
1197 142 : 1_000_000, // 1s
1198 142 : 2_000_000,
1199 142 : 4_000_000,
1200 142 : 6_000_000,
1201 142 : 8_000_000,
1202 142 : 10_000_000, // 10s
1203 142 : 20_000_000,
1204 142 : 50_000_000,
1205 142 : 100_000_000,
1206 142 : 200_000_000,
1207 142 : 1_000_000_000, // 1000s
1208 142 : ]
1209 142 : .into_iter()
1210 142 : .map(Duration::from_micros)
1211 5822 : .map(|d| d.as_secs_f64())
1212 142 : .collect()
1213 142 : });
1214 :
1215 142 : static SMGR_QUERY_TIME_GLOBAL: Lazy<HistogramVec> = Lazy::new(|| {
1216 : register_histogram_vec!(
1217 : "pageserver_smgr_query_seconds_global",
1218 : "Time spent on smgr query handling, aggregated by query type.",
1219 : &["smgr_query_type"],
1220 : SMGR_QUERY_TIME_GLOBAL_BUCKETS.clone(),
1221 : )
1222 142 : .expect("failed to define a metric")
1223 142 : });
1224 :
1225 : impl SmgrQueryTimePerTimeline {
1226 392 : pub(crate) fn new(tenant_shard_id: &TenantShardId, timeline_id: &TimelineId) -> Self {
1227 392 : let tenant_id = tenant_shard_id.tenant_id.to_string();
1228 392 : let shard_slug = format!("{}", tenant_shard_id.shard_slug());
1229 392 : let timeline_id = timeline_id.to_string();
1230 1960 : let metrics = std::array::from_fn(|i| {
1231 1960 : let op = SmgrQueryType::from_repr(i).unwrap();
1232 1960 : let global = SMGR_QUERY_TIME_GLOBAL
1233 1960 : .get_metric_with_label_values(&[op.into()])
1234 1960 : .unwrap();
1235 1960 : let per_tenant_timeline = SMGR_QUERY_TIME_PER_TENANT_TIMELINE
1236 1960 : .get_metric_with_label_values(&[op.into(), &tenant_id, &shard_slug, &timeline_id])
1237 1960 : .unwrap();
1238 1960 : GlobalAndPerTimelineHistogram {
1239 1960 : global,
1240 1960 : per_tenant_timeline,
1241 1960 : }
1242 1960 : });
1243 392 : Self { metrics }
1244 392 : }
1245 10 : pub(crate) fn start_timer<'c: 'a, 'a>(
1246 10 : &'a self,
1247 10 : op: SmgrQueryType,
1248 10 : ctx: &'c RequestContext,
1249 10 : ) -> impl Drop + '_ {
1250 10 : let metric = &self.metrics[op as usize];
1251 10 : let start = Instant::now();
1252 10 : match ctx.micros_spent_throttled.open() {
1253 10 : Ok(()) => (),
1254 0 : Err(error) => {
1255 0 : use utils::rate_limit::RateLimit;
1256 0 : static LOGGED: Lazy<Mutex<enum_map::EnumMap<SmgrQueryType, RateLimit>>> =
1257 0 : Lazy::new(|| {
1258 0 : Mutex::new(enum_map::EnumMap::from_array(std::array::from_fn(|_| {
1259 0 : RateLimit::new(Duration::from_secs(10))
1260 0 : })))
1261 0 : });
1262 0 : let mut guard = LOGGED.lock().unwrap();
1263 0 : let rate_limit = &mut guard[op];
1264 0 : rate_limit.call(|| {
1265 0 : warn!(?op, error, "error opening micros_spent_throttled; this message is logged at a global rate limit");
1266 0 : });
1267 0 : }
1268 : }
1269 10 : GlobalAndPerTimelineHistogramTimer {
1270 10 : h: metric,
1271 10 : ctx,
1272 10 : start,
1273 10 : op,
1274 10 : }
1275 10 : }
1276 : }
1277 :
1278 : #[cfg(test)]
1279 : mod smgr_query_time_tests {
1280 : use pageserver_api::shard::TenantShardId;
1281 : use strum::IntoEnumIterator;
1282 : use utils::id::{TenantId, TimelineId};
1283 :
1284 : use crate::{
1285 : context::{DownloadBehavior, RequestContext},
1286 : task_mgr::TaskKind,
1287 : };
1288 :
1289 : // Regression test, we used hard-coded string constants before using an enum.
1290 : #[test]
1291 2 : fn op_label_name() {
1292 2 : use super::SmgrQueryType::*;
1293 2 : let expect: [(super::SmgrQueryType, &'static str); 5] = [
1294 2 : (GetRelExists, "get_rel_exists"),
1295 2 : (GetRelSize, "get_rel_size"),
1296 2 : (GetPageAtLsn, "get_page_at_lsn"),
1297 2 : (GetDbSize, "get_db_size"),
1298 2 : (GetSlruSegment, "get_slru_segment"),
1299 2 : ];
1300 12 : for (op, expect) in expect {
1301 10 : let actual: &'static str = op.into();
1302 10 : assert_eq!(actual, expect);
1303 : }
1304 2 : }
1305 :
1306 : #[test]
1307 2 : fn basic() {
1308 2 : let ops: Vec<_> = super::SmgrQueryType::iter().collect();
1309 :
1310 12 : for op in &ops {
1311 10 : let tenant_id = TenantId::generate();
1312 10 : let timeline_id = TimelineId::generate();
1313 10 : let metrics = super::SmgrQueryTimePerTimeline::new(
1314 10 : &TenantShardId::unsharded(tenant_id),
1315 10 : &timeline_id,
1316 10 : );
1317 10 :
1318 20 : let get_counts = || {
1319 20 : let global: u64 = ops
1320 20 : .iter()
1321 100 : .map(|op| metrics.metrics[*op as usize].global.get_sample_count())
1322 20 : .sum();
1323 20 : let per_tenant_timeline: u64 = ops
1324 20 : .iter()
1325 100 : .map(|op| {
1326 100 : metrics.metrics[*op as usize]
1327 100 : .per_tenant_timeline
1328 100 : .get_sample_count()
1329 100 : })
1330 20 : .sum();
1331 20 : (global, per_tenant_timeline)
1332 20 : };
1333 :
1334 10 : let (pre_global, pre_per_tenant_timeline) = get_counts();
1335 10 : assert_eq!(pre_per_tenant_timeline, 0);
1336 :
1337 10 : let ctx = RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Download);
1338 10 : let timer = metrics.start_timer(*op, &ctx);
1339 10 : drop(timer);
1340 10 :
1341 10 : let (post_global, post_per_tenant_timeline) = get_counts();
1342 10 : assert_eq!(post_per_tenant_timeline, 1);
1343 10 : assert!(post_global > pre_global);
1344 : }
1345 2 : }
1346 : }
1347 :
1348 : // keep in sync with control plane Go code so that we can validate
1349 : // compute's basebackup_ms metric with our perspective in the context of SLI/SLO.
1350 0 : static COMPUTE_STARTUP_BUCKETS: Lazy<[f64; 28]> = Lazy::new(|| {
1351 0 : // Go code uses milliseconds. Variable is called `computeStartupBuckets`
1352 0 : [
1353 0 : 5, 10, 20, 30, 50, 70, 100, 120, 150, 200, 250, 300, 350, 400, 450, 500, 600, 800, 1000,
1354 0 : 1500, 2000, 2500, 3000, 5000, 10000, 20000, 40000, 60000,
1355 0 : ]
1356 0 : .map(|ms| (ms as f64) / 1000.0)
1357 0 : });
1358 :
1359 : pub(crate) struct BasebackupQueryTime {
1360 : ok: Histogram,
1361 : error: Histogram,
1362 : }
1363 :
1364 0 : pub(crate) static BASEBACKUP_QUERY_TIME: Lazy<BasebackupQueryTime> = Lazy::new(|| {
1365 0 : let vec = register_histogram_vec!(
1366 : "pageserver_basebackup_query_seconds",
1367 : "Histogram of basebackup queries durations, by result type",
1368 : &["result"],
1369 : COMPUTE_STARTUP_BUCKETS.to_vec(),
1370 : )
1371 0 : .expect("failed to define a metric");
1372 0 : BasebackupQueryTime {
1373 0 : ok: vec.get_metric_with_label_values(&["ok"]).unwrap(),
1374 0 : error: vec.get_metric_with_label_values(&["error"]).unwrap(),
1375 0 : }
1376 0 : });
1377 :
1378 : pub(crate) struct BasebackupQueryTimeOngoingRecording<'a, 'c> {
1379 : parent: &'a BasebackupQueryTime,
1380 : ctx: &'c RequestContext,
1381 : start: std::time::Instant,
1382 : }
1383 :
1384 : impl BasebackupQueryTime {
1385 0 : pub(crate) fn start_recording<'c: 'a, 'a>(
1386 0 : &'a self,
1387 0 : ctx: &'c RequestContext,
1388 0 : ) -> BasebackupQueryTimeOngoingRecording<'_, '_> {
1389 0 : let start = Instant::now();
1390 0 : match ctx.micros_spent_throttled.open() {
1391 0 : Ok(()) => (),
1392 0 : Err(error) => {
1393 0 : use utils::rate_limit::RateLimit;
1394 0 : static LOGGED: Lazy<Mutex<RateLimit>> =
1395 0 : Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(10))));
1396 0 : let mut rate_limit = LOGGED.lock().unwrap();
1397 0 : rate_limit.call(|| {
1398 0 : warn!(error, "error opening micros_spent_throttled; this message is logged at a global rate limit");
1399 0 : });
1400 0 : }
1401 : }
1402 0 : BasebackupQueryTimeOngoingRecording {
1403 0 : parent: self,
1404 0 : ctx,
1405 0 : start,
1406 0 : }
1407 0 : }
1408 : }
1409 :
1410 : impl<'a, 'c> BasebackupQueryTimeOngoingRecording<'a, 'c> {
1411 0 : pub(crate) fn observe<T, E>(self, res: &Result<T, E>) {
1412 0 : let elapsed = self.start.elapsed();
1413 0 : let ex_throttled = self
1414 0 : .ctx
1415 0 : .micros_spent_throttled
1416 0 : .close_and_checked_sub_from(elapsed);
1417 0 : let ex_throttled = match ex_throttled {
1418 0 : Ok(ex_throttled) => ex_throttled,
1419 0 : Err(error) => {
1420 0 : use utils::rate_limit::RateLimit;
1421 0 : static LOGGED: Lazy<Mutex<RateLimit>> =
1422 0 : Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(10))));
1423 0 : let mut rate_limit = LOGGED.lock().unwrap();
1424 0 : rate_limit.call(|| {
1425 0 : warn!(error, "error deducting time spent throttled; this message is logged at a global rate limit");
1426 0 : });
1427 0 : elapsed
1428 : }
1429 : };
1430 0 : let metric = if res.is_ok() {
1431 0 : &self.parent.ok
1432 : } else {
1433 0 : &self.parent.error
1434 : };
1435 0 : metric.observe(ex_throttled.as_secs_f64());
1436 0 : }
1437 : }
1438 :
1439 0 : pub(crate) static LIVE_CONNECTIONS_COUNT: Lazy<IntGaugeVec> = Lazy::new(|| {
1440 : register_int_gauge_vec!(
1441 : "pageserver_live_connections",
1442 : "Number of live network connections",
1443 : &["pageserver_connection_kind"]
1444 : )
1445 0 : .expect("failed to define a metric")
1446 0 : });
1447 :
1448 : // remote storage metrics
1449 :
1450 136 : static REMOTE_TIMELINE_CLIENT_CALLS: Lazy<IntCounterPairVec> = Lazy::new(|| {
1451 : register_int_counter_pair_vec!(
1452 : "pageserver_remote_timeline_client_calls_started",
1453 : "Number of started calls to remote timeline client.",
1454 : "pageserver_remote_timeline_client_calls_finished",
1455 : "Number of finshed calls to remote timeline client.",
1456 : &[
1457 : "tenant_id",
1458 : "shard_id",
1459 : "timeline_id",
1460 : "file_kind",
1461 : "op_kind"
1462 : ],
1463 : )
1464 136 : .unwrap()
1465 136 : });
1466 :
1467 : static REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER: Lazy<IntCounterVec> =
1468 132 : Lazy::new(|| {
1469 : register_int_counter_vec!(
1470 : "pageserver_remote_timeline_client_bytes_started",
1471 : "Incremented by the number of bytes associated with a remote timeline client operation. \
1472 : The increment happens when the operation is scheduled.",
1473 : &["tenant_id", "shard_id", "timeline_id", "file_kind", "op_kind"],
1474 : )
1475 132 : .expect("failed to define a metric")
1476 132 : });
1477 :
1478 132 : static REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER: Lazy<IntCounterVec> = Lazy::new(|| {
1479 : register_int_counter_vec!(
1480 : "pageserver_remote_timeline_client_bytes_finished",
1481 : "Incremented by the number of bytes associated with a remote timeline client operation. \
1482 : The increment happens when the operation finishes (regardless of success/failure/shutdown).",
1483 : &["tenant_id", "shard_id", "timeline_id", "file_kind", "op_kind"],
1484 : )
1485 132 : .expect("failed to define a metric")
1486 132 : });
1487 :
1488 : pub(crate) struct TenantManagerMetrics {
1489 : tenant_slots_attached: UIntGauge,
1490 : tenant_slots_secondary: UIntGauge,
1491 : tenant_slots_inprogress: UIntGauge,
1492 : pub(crate) tenant_slot_writes: IntCounter,
1493 : pub(crate) unexpected_errors: IntCounter,
1494 : }
1495 :
1496 : impl TenantManagerMetrics {
1497 : /// Helpers for tracking slots. Note that these do not track the lifetime of TenantSlot objects
1498 : /// exactly: they track the lifetime of the slots _in the tenant map_.
1499 2 : pub(crate) fn slot_inserted(&self, slot: &TenantSlot) {
1500 2 : match slot {
1501 0 : TenantSlot::Attached(_) => {
1502 0 : self.tenant_slots_attached.inc();
1503 0 : }
1504 0 : TenantSlot::Secondary(_) => {
1505 0 : self.tenant_slots_secondary.inc();
1506 0 : }
1507 2 : TenantSlot::InProgress(_) => {
1508 2 : self.tenant_slots_inprogress.inc();
1509 2 : }
1510 : }
1511 2 : }
1512 :
1513 2 : pub(crate) fn slot_removed(&self, slot: &TenantSlot) {
1514 2 : match slot {
1515 2 : TenantSlot::Attached(_) => {
1516 2 : self.tenant_slots_attached.dec();
1517 2 : }
1518 0 : TenantSlot::Secondary(_) => {
1519 0 : self.tenant_slots_secondary.dec();
1520 0 : }
1521 0 : TenantSlot::InProgress(_) => {
1522 0 : self.tenant_slots_inprogress.dec();
1523 0 : }
1524 : }
1525 2 : }
1526 :
1527 : #[cfg(all(debug_assertions, not(test)))]
1528 0 : pub(crate) fn slots_total(&self) -> u64 {
1529 0 : self.tenant_slots_attached.get()
1530 0 : + self.tenant_slots_secondary.get()
1531 0 : + self.tenant_slots_inprogress.get()
1532 0 : }
1533 : }
1534 :
1535 2 : pub(crate) static TENANT_MANAGER: Lazy<TenantManagerMetrics> = Lazy::new(|| {
1536 2 : let tenant_slots = register_uint_gauge_vec!(
1537 : "pageserver_tenant_manager_slots",
1538 : "How many slots currently exist, including all attached, secondary and in-progress operations",
1539 : &["mode"]
1540 : )
1541 2 : .expect("failed to define a metric");
1542 2 : TenantManagerMetrics {
1543 2 : tenant_slots_attached: tenant_slots
1544 2 : .get_metric_with_label_values(&["attached"])
1545 2 : .unwrap(),
1546 2 : tenant_slots_secondary: tenant_slots
1547 2 : .get_metric_with_label_values(&["secondary"])
1548 2 : .unwrap(),
1549 2 : tenant_slots_inprogress: tenant_slots
1550 2 : .get_metric_with_label_values(&["inprogress"])
1551 2 : .unwrap(),
1552 2 : tenant_slot_writes: register_int_counter!(
1553 : "pageserver_tenant_manager_slot_writes",
1554 : "Writes to a tenant slot, including all of create/attach/detach/delete"
1555 2 : )
1556 2 : .expect("failed to define a metric"),
1557 2 : unexpected_errors: register_int_counter!(
1558 : "pageserver_tenant_manager_unexpected_errors_total",
1559 : "Number of unexpected conditions encountered: nonzero value indicates a non-fatal bug."
1560 2 : )
1561 2 : .expect("failed to define a metric"),
1562 2 : }
1563 2 : });
1564 :
1565 : pub(crate) struct DeletionQueueMetrics {
1566 : pub(crate) keys_submitted: IntCounter,
1567 : pub(crate) keys_dropped: IntCounter,
1568 : pub(crate) keys_executed: IntCounter,
1569 : pub(crate) keys_validated: IntCounter,
1570 : pub(crate) dropped_lsn_updates: IntCounter,
1571 : pub(crate) unexpected_errors: IntCounter,
1572 : pub(crate) remote_errors: IntCounterVec,
1573 : }
1574 24 : pub(crate) static DELETION_QUEUE: Lazy<DeletionQueueMetrics> = Lazy::new(|| {
1575 24 : DeletionQueueMetrics{
1576 24 :
1577 24 : keys_submitted: register_int_counter!(
1578 : "pageserver_deletion_queue_submitted_total",
1579 : "Number of objects submitted for deletion"
1580 24 : )
1581 24 : .expect("failed to define a metric"),
1582 24 :
1583 24 : keys_dropped: register_int_counter!(
1584 : "pageserver_deletion_queue_dropped_total",
1585 : "Number of object deletions dropped due to stale generation."
1586 24 : )
1587 24 : .expect("failed to define a metric"),
1588 24 :
1589 24 : keys_executed: register_int_counter!(
1590 : "pageserver_deletion_queue_executed_total",
1591 : "Number of objects deleted. Only includes objects that we actually deleted, sum with pageserver_deletion_queue_dropped_total for the total number of keys processed to completion"
1592 24 : )
1593 24 : .expect("failed to define a metric"),
1594 24 :
1595 24 : keys_validated: register_int_counter!(
1596 : "pageserver_deletion_queue_validated_total",
1597 : "Number of keys validated for deletion. Sum with pageserver_deletion_queue_dropped_total for the total number of keys that have passed through the validation stage."
1598 24 : )
1599 24 : .expect("failed to define a metric"),
1600 24 :
1601 24 : dropped_lsn_updates: register_int_counter!(
1602 : "pageserver_deletion_queue_dropped_lsn_updates_total",
1603 : "Updates to remote_consistent_lsn dropped due to stale generation number."
1604 24 : )
1605 24 : .expect("failed to define a metric"),
1606 24 : unexpected_errors: register_int_counter!(
1607 : "pageserver_deletion_queue_unexpected_errors_total",
1608 : "Number of unexpected condiions that may stall the queue: any value above zero is unexpected."
1609 24 : )
1610 24 : .expect("failed to define a metric"),
1611 24 : remote_errors: register_int_counter_vec!(
1612 : "pageserver_deletion_queue_remote_errors_total",
1613 : "Retryable remote I/O errors while executing deletions, for example 503 responses to DeleteObjects",
1614 : &["op_kind"],
1615 24 : )
1616 24 : .expect("failed to define a metric")
1617 24 : }
1618 24 : });
1619 :
1620 : pub(crate) struct SecondaryModeMetrics {
1621 : pub(crate) upload_heatmap: IntCounter,
1622 : pub(crate) upload_heatmap_errors: IntCounter,
1623 : pub(crate) upload_heatmap_duration: Histogram,
1624 : pub(crate) download_heatmap: IntCounter,
1625 : pub(crate) download_layer: IntCounter,
1626 : }
1627 0 : pub(crate) static SECONDARY_MODE: Lazy<SecondaryModeMetrics> = Lazy::new(|| {
1628 0 : SecondaryModeMetrics {
1629 0 : upload_heatmap: register_int_counter!(
1630 : "pageserver_secondary_upload_heatmap",
1631 : "Number of heatmaps written to remote storage by attached tenants"
1632 0 : )
1633 0 : .expect("failed to define a metric"),
1634 0 : upload_heatmap_errors: register_int_counter!(
1635 : "pageserver_secondary_upload_heatmap_errors",
1636 : "Failures writing heatmap to remote storage"
1637 0 : )
1638 0 : .expect("failed to define a metric"),
1639 0 : upload_heatmap_duration: register_histogram!(
1640 : "pageserver_secondary_upload_heatmap_duration",
1641 : "Time to build and upload a heatmap, including any waiting inside the S3 client"
1642 0 : )
1643 0 : .expect("failed to define a metric"),
1644 0 : download_heatmap: register_int_counter!(
1645 : "pageserver_secondary_download_heatmap",
1646 : "Number of downloads of heatmaps by secondary mode locations, including when it hasn't changed"
1647 0 : )
1648 0 : .expect("failed to define a metric"),
1649 0 : download_layer: register_int_counter!(
1650 : "pageserver_secondary_download_layer",
1651 : "Number of downloads of layers by secondary mode locations"
1652 0 : )
1653 0 : .expect("failed to define a metric"),
1654 0 : }
1655 0 : });
1656 :
1657 : #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1658 : pub enum RemoteOpKind {
1659 : Upload,
1660 : Download,
1661 : Delete,
1662 : }
1663 : impl RemoteOpKind {
1664 12877 : pub fn as_str(&self) -> &'static str {
1665 12877 : match self {
1666 12166 : Self::Upload => "upload",
1667 52 : Self::Download => "download",
1668 659 : Self::Delete => "delete",
1669 : }
1670 12877 : }
1671 : }
1672 :
1673 : #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
1674 : pub enum RemoteOpFileKind {
1675 : Layer,
1676 : Index,
1677 : }
1678 : impl RemoteOpFileKind {
1679 12877 : pub fn as_str(&self) -> &'static str {
1680 12877 : match self {
1681 8723 : Self::Layer => "layer",
1682 4154 : Self::Index => "index",
1683 : }
1684 12877 : }
1685 : }
1686 :
1687 131 : pub(crate) static REMOTE_OPERATION_TIME: Lazy<HistogramVec> = Lazy::new(|| {
1688 : register_histogram_vec!(
1689 : "pageserver_remote_operation_seconds",
1690 : "Time spent on remote storage operations. \
1691 : Grouped by tenant, timeline, operation_kind and status. \
1692 : Does not account for time spent waiting in remote timeline client's queues.",
1693 : &["file_kind", "op_kind", "status"]
1694 : )
1695 131 : .expect("failed to define a metric")
1696 131 : });
1697 :
1698 0 : pub(crate) static TENANT_TASK_EVENTS: Lazy<IntCounterVec> = Lazy::new(|| {
1699 : register_int_counter_vec!(
1700 : "pageserver_tenant_task_events",
1701 : "Number of task start/stop/fail events.",
1702 : &["event"],
1703 : )
1704 0 : .expect("Failed to register tenant_task_events metric")
1705 0 : });
1706 :
1707 20 : pub(crate) static BACKGROUND_LOOP_SEMAPHORE_WAIT_GAUGE: Lazy<IntCounterPairVec> = Lazy::new(|| {
1708 : register_int_counter_pair_vec!(
1709 : "pageserver_background_loop_semaphore_wait_start_count",
1710 : "Counter for background loop concurrency-limiting semaphore acquire calls started",
1711 : "pageserver_background_loop_semaphore_wait_finish_count",
1712 : "Counter for background loop concurrency-limiting semaphore acquire calls finished",
1713 : &["task"],
1714 : )
1715 20 : .unwrap()
1716 20 : });
1717 :
1718 0 : pub(crate) static BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT: Lazy<IntCounterVec> = Lazy::new(|| {
1719 : register_int_counter_vec!(
1720 : "pageserver_background_loop_period_overrun_count",
1721 : "Incremented whenever warn_when_period_overrun() logs a warning.",
1722 : &["task", "period"],
1723 : )
1724 0 : .expect("failed to define a metric")
1725 0 : });
1726 :
1727 : // walreceiver metrics
1728 :
1729 0 : pub(crate) static WALRECEIVER_STARTED_CONNECTIONS: Lazy<IntCounter> = Lazy::new(|| {
1730 : register_int_counter!(
1731 : "pageserver_walreceiver_started_connections_total",
1732 : "Number of started walreceiver connections"
1733 : )
1734 0 : .expect("failed to define a metric")
1735 0 : });
1736 :
1737 0 : pub(crate) static WALRECEIVER_ACTIVE_MANAGERS: Lazy<IntGauge> = Lazy::new(|| {
1738 : register_int_gauge!(
1739 : "pageserver_walreceiver_active_managers",
1740 : "Number of active walreceiver managers"
1741 : )
1742 0 : .expect("failed to define a metric")
1743 0 : });
1744 :
1745 0 : pub(crate) static WALRECEIVER_SWITCHES: Lazy<IntCounterVec> = Lazy::new(|| {
1746 : register_int_counter_vec!(
1747 : "pageserver_walreceiver_switches_total",
1748 : "Number of walreceiver manager change_connection calls",
1749 : &["reason"]
1750 : )
1751 0 : .expect("failed to define a metric")
1752 0 : });
1753 :
1754 0 : pub(crate) static WALRECEIVER_BROKER_UPDATES: Lazy<IntCounter> = Lazy::new(|| {
1755 : register_int_counter!(
1756 : "pageserver_walreceiver_broker_updates_total",
1757 : "Number of received broker updates in walreceiver"
1758 : )
1759 0 : .expect("failed to define a metric")
1760 0 : });
1761 :
1762 2 : pub(crate) static WALRECEIVER_CANDIDATES_EVENTS: Lazy<IntCounterVec> = Lazy::new(|| {
1763 : register_int_counter_vec!(
1764 : "pageserver_walreceiver_candidates_events_total",
1765 : "Number of walreceiver candidate events",
1766 : &["event"]
1767 : )
1768 2 : .expect("failed to define a metric")
1769 2 : });
1770 :
1771 : pub(crate) static WALRECEIVER_CANDIDATES_ADDED: Lazy<IntCounter> =
1772 0 : Lazy::new(|| WALRECEIVER_CANDIDATES_EVENTS.with_label_values(&["add"]));
1773 :
1774 : pub(crate) static WALRECEIVER_CANDIDATES_REMOVED: Lazy<IntCounter> =
1775 2 : Lazy::new(|| WALRECEIVER_CANDIDATES_EVENTS.with_label_values(&["remove"]));
1776 :
1777 : // Metrics collected on WAL redo operations
1778 : //
1779 : // We collect the time spent in actual WAL redo ('redo'), and time waiting
1780 : // for access to the postgres process ('wait') since there is only one for
1781 : // each tenant.
1782 :
1783 : /// Time buckets are small because we want to be able to measure the
1784 : /// smallest redo processing times. These buckets allow us to measure down
1785 : /// to 5us, which equates to 200'000 pages/sec, which equates to 1.6GB/sec.
1786 : /// This is much better than the previous 5ms aka 200 pages/sec aka 1.6MB/sec.
1787 : ///
1788 : /// Values up to 1s are recorded because metrics show that we have redo
1789 : /// durations and lock times larger than 0.250s.
1790 : macro_rules! redo_histogram_time_buckets {
1791 : () => {
1792 : vec![
1793 : 0.000_005, 0.000_010, 0.000_025, 0.000_050, 0.000_100, 0.000_250, 0.000_500, 0.001_000,
1794 : 0.002_500, 0.005_000, 0.010_000, 0.025_000, 0.050_000, 0.100_000, 0.250_000, 0.500_000,
1795 : 1.000_000,
1796 : ]
1797 : };
1798 : }
1799 :
1800 : /// While we're at it, also measure the amount of records replayed in each
1801 : /// operation. We have a global 'total replayed' counter, but that's not
1802 : /// as useful as 'what is the skew for how many records we replay in one
1803 : /// operation'.
1804 : macro_rules! redo_histogram_count_buckets {
1805 : () => {
1806 : vec![0.0, 1.0, 2.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0]
1807 : };
1808 : }
1809 :
1810 : macro_rules! redo_bytes_histogram_count_buckets {
1811 : () => {
1812 : // powers of (2^.5), from 2^4.5 to 2^15 (22 buckets)
1813 : // rounded up to the next multiple of 8 to capture any MAXALIGNed record of that size, too.
1814 : vec![
1815 : 24.0, 32.0, 48.0, 64.0, 96.0, 128.0, 184.0, 256.0, 368.0, 512.0, 728.0, 1024.0, 1456.0,
1816 : 2048.0, 2904.0, 4096.0, 5800.0, 8192.0, 11592.0, 16384.0, 23176.0, 32768.0,
1817 : ]
1818 : };
1819 : }
1820 :
1821 : pub(crate) struct WalIngestMetrics {
1822 : pub(crate) bytes_received: IntCounter,
1823 : pub(crate) records_received: IntCounter,
1824 : pub(crate) records_committed: IntCounter,
1825 : pub(crate) records_filtered: IntCounter,
1826 : }
1827 :
1828 2 : pub(crate) static WAL_INGEST: Lazy<WalIngestMetrics> = Lazy::new(|| WalIngestMetrics {
1829 : bytes_received: register_int_counter!(
1830 : "pageserver_wal_ingest_bytes_received",
1831 : "Bytes of WAL ingested from safekeepers",
1832 : )
1833 2 : .unwrap(),
1834 : records_received: register_int_counter!(
1835 : "pageserver_wal_ingest_records_received",
1836 : "Number of WAL records received from safekeepers"
1837 : )
1838 2 : .expect("failed to define a metric"),
1839 : records_committed: register_int_counter!(
1840 : "pageserver_wal_ingest_records_committed",
1841 : "Number of WAL records which resulted in writes to pageserver storage"
1842 : )
1843 2 : .expect("failed to define a metric"),
1844 : records_filtered: register_int_counter!(
1845 : "pageserver_wal_ingest_records_filtered",
1846 : "Number of WAL records filtered out due to sharding"
1847 : )
1848 2 : .expect("failed to define a metric"),
1849 2 : });
1850 :
1851 6 : pub(crate) static WAL_REDO_TIME: Lazy<Histogram> = Lazy::new(|| {
1852 : register_histogram!(
1853 : "pageserver_wal_redo_seconds",
1854 : "Time spent on WAL redo",
1855 : redo_histogram_time_buckets!()
1856 : )
1857 6 : .expect("failed to define a metric")
1858 6 : });
1859 :
1860 6 : pub(crate) static WAL_REDO_RECORDS_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
1861 : register_histogram!(
1862 : "pageserver_wal_redo_records_histogram",
1863 : "Histogram of number of records replayed per redo in the Postgres WAL redo process",
1864 : redo_histogram_count_buckets!(),
1865 : )
1866 6 : .expect("failed to define a metric")
1867 6 : });
1868 :
1869 6 : pub(crate) static WAL_REDO_BYTES_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
1870 : register_histogram!(
1871 : "pageserver_wal_redo_bytes_histogram",
1872 : "Histogram of number of records replayed per redo sent to Postgres",
1873 : redo_bytes_histogram_count_buckets!(),
1874 : )
1875 6 : .expect("failed to define a metric")
1876 6 : });
1877 :
1878 : // FIXME: isn't this already included by WAL_REDO_RECORDS_HISTOGRAM which has _count?
1879 6 : pub(crate) static WAL_REDO_RECORD_COUNTER: Lazy<IntCounter> = Lazy::new(|| {
1880 : register_int_counter!(
1881 : "pageserver_replayed_wal_records_total",
1882 : "Number of WAL records replayed in WAL redo process"
1883 : )
1884 6 : .unwrap()
1885 6 : });
1886 :
1887 : #[rustfmt::skip]
1888 6 : pub(crate) static WAL_REDO_PROCESS_LAUNCH_DURATION_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
1889 : register_histogram!(
1890 : "pageserver_wal_redo_process_launch_duration",
1891 : "Histogram of the duration of successful WalRedoProcess::launch calls",
1892 : vec![
1893 : 0.0002, 0.0004, 0.0006, 0.0008, 0.0010,
1894 : 0.0020, 0.0040, 0.0060, 0.0080, 0.0100,
1895 : 0.0200, 0.0400, 0.0600, 0.0800, 0.1000,
1896 : 0.2000, 0.4000, 0.6000, 0.8000, 1.0000,
1897 : 1.5000, 2.0000, 2.5000, 3.0000, 4.0000, 10.0000
1898 : ],
1899 : )
1900 6 : .expect("failed to define a metric")
1901 6 : });
1902 :
1903 : pub(crate) struct WalRedoProcessCounters {
1904 : pub(crate) started: IntCounter,
1905 : pub(crate) killed_by_cause: enum_map::EnumMap<WalRedoKillCause, IntCounter>,
1906 : pub(crate) active_stderr_logger_tasks_started: IntCounter,
1907 : pub(crate) active_stderr_logger_tasks_finished: IntCounter,
1908 : }
1909 :
1910 18 : #[derive(Debug, enum_map::Enum, strum_macros::IntoStaticStr)]
1911 : pub(crate) enum WalRedoKillCause {
1912 : WalRedoProcessDrop,
1913 : NoLeakChildDrop,
1914 : Startup,
1915 : }
1916 :
1917 : impl Default for WalRedoProcessCounters {
1918 6 : fn default() -> Self {
1919 6 : let started = register_int_counter!(
1920 : "pageserver_wal_redo_process_started_total",
1921 : "Number of WAL redo processes started",
1922 : )
1923 6 : .unwrap();
1924 6 :
1925 6 : let killed = register_int_counter_vec!(
1926 : "pageserver_wal_redo_process_stopped_total",
1927 : "Number of WAL redo processes stopped",
1928 : &["cause"],
1929 : )
1930 6 : .unwrap();
1931 6 :
1932 6 : let active_stderr_logger_tasks_started = register_int_counter!(
1933 : "pageserver_walredo_stderr_logger_tasks_started_total",
1934 : "Number of active walredo stderr logger tasks that have started",
1935 : )
1936 6 : .unwrap();
1937 6 :
1938 6 : let active_stderr_logger_tasks_finished = register_int_counter!(
1939 : "pageserver_walredo_stderr_logger_tasks_finished_total",
1940 : "Number of active walredo stderr logger tasks that have finished",
1941 : )
1942 6 : .unwrap();
1943 6 :
1944 6 : Self {
1945 6 : started,
1946 18 : killed_by_cause: EnumMap::from_array(std::array::from_fn(|i| {
1947 18 : let cause = <WalRedoKillCause as enum_map::Enum>::from_usize(i);
1948 18 : let cause_str: &'static str = cause.into();
1949 18 : killed.with_label_values(&[cause_str])
1950 18 : })),
1951 6 : active_stderr_logger_tasks_started,
1952 6 : active_stderr_logger_tasks_finished,
1953 6 : }
1954 6 : }
1955 : }
1956 :
1957 : pub(crate) static WAL_REDO_PROCESS_COUNTERS: Lazy<WalRedoProcessCounters> =
1958 : Lazy::new(WalRedoProcessCounters::default);
1959 :
1960 : /// Similar to `prometheus::HistogramTimer` but does not record on drop.
1961 : pub(crate) struct StorageTimeMetricsTimer {
1962 : metrics: StorageTimeMetrics,
1963 : start: Instant,
1964 : }
1965 :
1966 : impl StorageTimeMetricsTimer {
1967 4596 : fn new(metrics: StorageTimeMetrics) -> Self {
1968 4596 : Self {
1969 4596 : metrics,
1970 4596 : start: Instant::now(),
1971 4596 : }
1972 4596 : }
1973 :
1974 : /// Record the time from creation to now.
1975 3493 : pub fn stop_and_record(self) {
1976 3493 : let duration = self.start.elapsed().as_secs_f64();
1977 3493 : self.metrics.timeline_sum.inc_by(duration);
1978 3493 : self.metrics.timeline_count.inc();
1979 3493 : self.metrics.global_histogram.observe(duration);
1980 3493 : }
1981 :
1982 : /// Turns this timer into a timer, which will always record -- usually this means recording
1983 : /// regardless an early `?` path was taken in a function.
1984 754 : pub(crate) fn record_on_drop(self) -> AlwaysRecordingStorageTimeMetricsTimer {
1985 754 : AlwaysRecordingStorageTimeMetricsTimer(Some(self))
1986 754 : }
1987 : }
1988 :
1989 : pub(crate) struct AlwaysRecordingStorageTimeMetricsTimer(Option<StorageTimeMetricsTimer>);
1990 :
1991 : impl Drop for AlwaysRecordingStorageTimeMetricsTimer {
1992 754 : fn drop(&mut self) {
1993 754 : if let Some(inner) = self.0.take() {
1994 754 : inner.stop_and_record();
1995 754 : }
1996 754 : }
1997 : }
1998 :
1999 : /// Timing facilities for an globally histogrammed metric, which is supported by per tenant and
2000 : /// timeline total sum and count.
2001 : #[derive(Clone, Debug)]
2002 : pub(crate) struct StorageTimeMetrics {
2003 : /// Sum of f64 seconds, per operation, tenant_id and timeline_id
2004 : timeline_sum: Counter,
2005 : /// Number of oeprations, per operation, tenant_id and timeline_id
2006 : timeline_count: IntCounter,
2007 : /// Global histogram having only the "operation" label.
2008 : global_histogram: Histogram,
2009 : }
2010 :
2011 : impl StorageTimeMetrics {
2012 3056 : pub fn new(
2013 3056 : operation: StorageTimeOperation,
2014 3056 : tenant_id: &str,
2015 3056 : shard_id: &str,
2016 3056 : timeline_id: &str,
2017 3056 : ) -> Self {
2018 3056 : let operation: &'static str = operation.into();
2019 3056 :
2020 3056 : let timeline_sum = STORAGE_TIME_SUM_PER_TIMELINE
2021 3056 : .get_metric_with_label_values(&[operation, tenant_id, shard_id, timeline_id])
2022 3056 : .unwrap();
2023 3056 : let timeline_count = STORAGE_TIME_COUNT_PER_TIMELINE
2024 3056 : .get_metric_with_label_values(&[operation, tenant_id, shard_id, timeline_id])
2025 3056 : .unwrap();
2026 3056 : let global_histogram = STORAGE_TIME_GLOBAL
2027 3056 : .get_metric_with_label_values(&[operation])
2028 3056 : .unwrap();
2029 3056 :
2030 3056 : StorageTimeMetrics {
2031 3056 : timeline_sum,
2032 3056 : timeline_count,
2033 3056 : global_histogram,
2034 3056 : }
2035 3056 : }
2036 :
2037 : /// Starts timing a new operation.
2038 : ///
2039 : /// Note: unlike `prometheus::HistogramTimer` the returned timer does not record on drop.
2040 4596 : pub fn start_timer(&self) -> StorageTimeMetricsTimer {
2041 4596 : StorageTimeMetricsTimer::new(self.clone())
2042 4596 : }
2043 : }
2044 :
2045 : #[derive(Debug)]
2046 : pub(crate) struct TimelineMetrics {
2047 : tenant_id: String,
2048 : shard_id: String,
2049 : timeline_id: String,
2050 : pub flush_time_histo: StorageTimeMetrics,
2051 : pub compact_time_histo: StorageTimeMetrics,
2052 : pub create_images_time_histo: StorageTimeMetrics,
2053 : pub logical_size_histo: StorageTimeMetrics,
2054 : pub imitate_logical_size_histo: StorageTimeMetrics,
2055 : pub load_layer_map_histo: StorageTimeMetrics,
2056 : pub garbage_collect_histo: StorageTimeMetrics,
2057 : pub find_gc_cutoffs_histo: StorageTimeMetrics,
2058 : pub last_record_gauge: IntGauge,
2059 : pub standby_horizon_gauge: IntGauge,
2060 : pub resident_physical_size_gauge: UIntGauge,
2061 : /// copy of LayeredTimeline.current_logical_size
2062 : pub current_logical_size_gauge: UIntGauge,
2063 : pub aux_file_size_gauge: IntGauge,
2064 : pub directory_entries_count_gauge: Lazy<UIntGauge, Box<dyn Send + Fn() -> UIntGauge>>,
2065 : pub evictions: IntCounter,
2066 : pub evictions_with_low_residence_duration: std::sync::RwLock<EvictionsWithLowResidenceDuration>,
2067 : /// Number of valid LSN leases.
2068 : pub valid_lsn_lease_count_gauge: UIntGauge,
2069 : shutdown: std::sync::atomic::AtomicBool,
2070 : }
2071 :
2072 : impl TimelineMetrics {
2073 382 : pub fn new(
2074 382 : tenant_shard_id: &TenantShardId,
2075 382 : timeline_id_raw: &TimelineId,
2076 382 : evictions_with_low_residence_duration_builder: EvictionsWithLowResidenceDurationBuilder,
2077 382 : ) -> Self {
2078 382 : let tenant_id = tenant_shard_id.tenant_id.to_string();
2079 382 : let shard_id = format!("{}", tenant_shard_id.shard_slug());
2080 382 : let timeline_id = timeline_id_raw.to_string();
2081 382 : let flush_time_histo = StorageTimeMetrics::new(
2082 382 : StorageTimeOperation::LayerFlush,
2083 382 : &tenant_id,
2084 382 : &shard_id,
2085 382 : &timeline_id,
2086 382 : );
2087 382 : let compact_time_histo = StorageTimeMetrics::new(
2088 382 : StorageTimeOperation::Compact,
2089 382 : &tenant_id,
2090 382 : &shard_id,
2091 382 : &timeline_id,
2092 382 : );
2093 382 : let create_images_time_histo = StorageTimeMetrics::new(
2094 382 : StorageTimeOperation::CreateImages,
2095 382 : &tenant_id,
2096 382 : &shard_id,
2097 382 : &timeline_id,
2098 382 : );
2099 382 : let logical_size_histo = StorageTimeMetrics::new(
2100 382 : StorageTimeOperation::LogicalSize,
2101 382 : &tenant_id,
2102 382 : &shard_id,
2103 382 : &timeline_id,
2104 382 : );
2105 382 : let imitate_logical_size_histo = StorageTimeMetrics::new(
2106 382 : StorageTimeOperation::ImitateLogicalSize,
2107 382 : &tenant_id,
2108 382 : &shard_id,
2109 382 : &timeline_id,
2110 382 : );
2111 382 : let load_layer_map_histo = StorageTimeMetrics::new(
2112 382 : StorageTimeOperation::LoadLayerMap,
2113 382 : &tenant_id,
2114 382 : &shard_id,
2115 382 : &timeline_id,
2116 382 : );
2117 382 : let garbage_collect_histo = StorageTimeMetrics::new(
2118 382 : StorageTimeOperation::Gc,
2119 382 : &tenant_id,
2120 382 : &shard_id,
2121 382 : &timeline_id,
2122 382 : );
2123 382 : let find_gc_cutoffs_histo = StorageTimeMetrics::new(
2124 382 : StorageTimeOperation::FindGcCutoffs,
2125 382 : &tenant_id,
2126 382 : &shard_id,
2127 382 : &timeline_id,
2128 382 : );
2129 382 : let last_record_gauge = LAST_RECORD_LSN
2130 382 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2131 382 : .unwrap();
2132 382 : let standby_horizon_gauge = STANDBY_HORIZON
2133 382 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2134 382 : .unwrap();
2135 382 : let resident_physical_size_gauge = RESIDENT_PHYSICAL_SIZE
2136 382 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2137 382 : .unwrap();
2138 382 : // TODO: we shouldn't expose this metric
2139 382 : let current_logical_size_gauge = CURRENT_LOGICAL_SIZE
2140 382 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2141 382 : .unwrap();
2142 382 : let aux_file_size_gauge = AUX_FILE_SIZE
2143 382 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2144 382 : .unwrap();
2145 382 : // TODO use impl Trait syntax here once we have ability to use it: https://github.com/rust-lang/rust/issues/63065
2146 382 : let directory_entries_count_gauge_closure = {
2147 382 : let tenant_shard_id = *tenant_shard_id;
2148 382 : let timeline_id_raw = *timeline_id_raw;
2149 0 : move || {
2150 0 : let tenant_id = tenant_shard_id.tenant_id.to_string();
2151 0 : let shard_id = format!("{}", tenant_shard_id.shard_slug());
2152 0 : let timeline_id = timeline_id_raw.to_string();
2153 0 : let gauge: UIntGauge = DIRECTORY_ENTRIES_COUNT
2154 0 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2155 0 : .unwrap();
2156 0 : gauge
2157 0 : }
2158 : };
2159 382 : let directory_entries_count_gauge: Lazy<UIntGauge, Box<dyn Send + Fn() -> UIntGauge>> =
2160 382 : Lazy::new(Box::new(directory_entries_count_gauge_closure));
2161 382 : let evictions = EVICTIONS
2162 382 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2163 382 : .unwrap();
2164 382 : let evictions_with_low_residence_duration = evictions_with_low_residence_duration_builder
2165 382 : .build(&tenant_id, &shard_id, &timeline_id);
2166 382 :
2167 382 : let valid_lsn_lease_count_gauge = VALID_LSN_LEASE_COUNT
2168 382 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2169 382 : .unwrap();
2170 382 :
2171 382 : TimelineMetrics {
2172 382 : tenant_id,
2173 382 : shard_id,
2174 382 : timeline_id,
2175 382 : flush_time_histo,
2176 382 : compact_time_histo,
2177 382 : create_images_time_histo,
2178 382 : logical_size_histo,
2179 382 : imitate_logical_size_histo,
2180 382 : garbage_collect_histo,
2181 382 : find_gc_cutoffs_histo,
2182 382 : load_layer_map_histo,
2183 382 : last_record_gauge,
2184 382 : standby_horizon_gauge,
2185 382 : resident_physical_size_gauge,
2186 382 : current_logical_size_gauge,
2187 382 : aux_file_size_gauge,
2188 382 : directory_entries_count_gauge,
2189 382 : evictions,
2190 382 : evictions_with_low_residence_duration: std::sync::RwLock::new(
2191 382 : evictions_with_low_residence_duration,
2192 382 : ),
2193 382 : valid_lsn_lease_count_gauge,
2194 382 : shutdown: std::sync::atomic::AtomicBool::default(),
2195 382 : }
2196 382 : }
2197 :
2198 1466 : pub(crate) fn record_new_file_metrics(&self, sz: u64) {
2199 1466 : self.resident_physical_size_add(sz);
2200 1466 : }
2201 :
2202 449 : pub(crate) fn resident_physical_size_sub(&self, sz: u64) {
2203 449 : self.resident_physical_size_gauge.sub(sz);
2204 449 : crate::metrics::RESIDENT_PHYSICAL_SIZE_GLOBAL.sub(sz);
2205 449 : }
2206 :
2207 1496 : pub(crate) fn resident_physical_size_add(&self, sz: u64) {
2208 1496 : self.resident_physical_size_gauge.add(sz);
2209 1496 : crate::metrics::RESIDENT_PHYSICAL_SIZE_GLOBAL.add(sz);
2210 1496 : }
2211 :
2212 8 : pub(crate) fn resident_physical_size_get(&self) -> u64 {
2213 8 : self.resident_physical_size_gauge.get()
2214 8 : }
2215 :
2216 8 : pub(crate) fn shutdown(&self) {
2217 8 : let was_shutdown = self
2218 8 : .shutdown
2219 8 : .swap(true, std::sync::atomic::Ordering::Relaxed);
2220 8 :
2221 8 : if was_shutdown {
2222 : // this happens on tenant deletion because tenant first shuts down timelines, then
2223 : // invokes timeline deletion which first shuts down the timeline again.
2224 : // TODO: this can be removed once https://github.com/neondatabase/neon/issues/5080
2225 0 : return;
2226 8 : }
2227 8 :
2228 8 : let tenant_id = &self.tenant_id;
2229 8 : let timeline_id = &self.timeline_id;
2230 8 : let shard_id = &self.shard_id;
2231 8 : let _ = LAST_RECORD_LSN.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2232 8 : let _ = STANDBY_HORIZON.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2233 8 : {
2234 8 : RESIDENT_PHYSICAL_SIZE_GLOBAL.sub(self.resident_physical_size_get());
2235 8 : let _ = RESIDENT_PHYSICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2236 8 : }
2237 8 : let _ = CURRENT_LOGICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2238 8 : if let Some(metric) = Lazy::get(&DIRECTORY_ENTRIES_COUNT) {
2239 0 : let _ = metric.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2240 8 : }
2241 8 : let _ = EVICTIONS.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2242 8 : let _ = AUX_FILE_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2243 8 : let _ = VALID_LSN_LEASE_COUNT.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2244 8 :
2245 8 : self.evictions_with_low_residence_duration
2246 8 : .write()
2247 8 : .unwrap()
2248 8 : .remove(tenant_id, shard_id, timeline_id);
2249 :
2250 : // The following metrics are born outside of the TimelineMetrics lifecycle but still
2251 : // removed at the end of it. The idea is to have the metrics outlive the
2252 : // entity during which they're observed, e.g., the smgr metrics shall
2253 : // outlive an individual smgr connection, but not the timeline.
2254 :
2255 80 : for op in StorageTimeOperation::VARIANTS {
2256 72 : let _ = STORAGE_TIME_SUM_PER_TIMELINE.remove_label_values(&[
2257 72 : op,
2258 72 : tenant_id,
2259 72 : shard_id,
2260 72 : timeline_id,
2261 72 : ]);
2262 72 : let _ = STORAGE_TIME_COUNT_PER_TIMELINE.remove_label_values(&[
2263 72 : op,
2264 72 : tenant_id,
2265 72 : shard_id,
2266 72 : timeline_id,
2267 72 : ]);
2268 72 : }
2269 :
2270 24 : for op in STORAGE_IO_SIZE_OPERATIONS {
2271 16 : let _ = STORAGE_IO_SIZE.remove_label_values(&[op, tenant_id, shard_id, timeline_id]);
2272 16 : }
2273 :
2274 48 : for op in SmgrQueryType::iter() {
2275 40 : let _ = SMGR_QUERY_TIME_PER_TENANT_TIMELINE.remove_label_values(&[
2276 40 : op.into(),
2277 40 : tenant_id,
2278 40 : shard_id,
2279 40 : timeline_id,
2280 40 : ]);
2281 40 : }
2282 8 : }
2283 : }
2284 :
2285 6 : pub(crate) fn remove_tenant_metrics(tenant_shard_id: &TenantShardId) {
2286 6 : // Only shard zero deals in synthetic sizes
2287 6 : if tenant_shard_id.is_shard_zero() {
2288 6 : let tid = tenant_shard_id.tenant_id.to_string();
2289 6 : let _ = TENANT_SYNTHETIC_SIZE_METRIC.remove_label_values(&[&tid]);
2290 6 : }
2291 :
2292 : // we leave the BROKEN_TENANTS_SET entry if any
2293 6 : }
2294 :
2295 : use futures::Future;
2296 : use pin_project_lite::pin_project;
2297 : use std::collections::HashMap;
2298 : use std::num::NonZeroUsize;
2299 : use std::pin::Pin;
2300 : use std::sync::atomic::AtomicU64;
2301 : use std::sync::{Arc, Mutex};
2302 : use std::task::{Context, Poll};
2303 : use std::time::{Duration, Instant};
2304 :
2305 : use crate::context::{PageContentKind, RequestContext};
2306 : use crate::task_mgr::TaskKind;
2307 : use crate::tenant::mgr::TenantSlot;
2308 :
2309 : /// Maintain a per timeline gauge in addition to the global gauge.
2310 : pub(crate) struct PerTimelineRemotePhysicalSizeGauge {
2311 : last_set: AtomicU64,
2312 : gauge: UIntGauge,
2313 : }
2314 :
2315 : impl PerTimelineRemotePhysicalSizeGauge {
2316 392 : fn new(per_timeline_gauge: UIntGauge) -> Self {
2317 392 : Self {
2318 392 : last_set: AtomicU64::new(0),
2319 392 : gauge: per_timeline_gauge,
2320 392 : }
2321 392 : }
2322 1735 : pub(crate) fn set(&self, sz: u64) {
2323 1735 : self.gauge.set(sz);
2324 1735 : let prev = self.last_set.swap(sz, std::sync::atomic::Ordering::Relaxed);
2325 1735 : if sz < prev {
2326 26 : REMOTE_PHYSICAL_SIZE_GLOBAL.sub(prev - sz);
2327 1709 : } else {
2328 1709 : REMOTE_PHYSICAL_SIZE_GLOBAL.add(sz - prev);
2329 1709 : };
2330 1735 : }
2331 2 : pub(crate) fn get(&self) -> u64 {
2332 2 : self.gauge.get()
2333 2 : }
2334 : }
2335 :
2336 : impl Drop for PerTimelineRemotePhysicalSizeGauge {
2337 19 : fn drop(&mut self) {
2338 19 : REMOTE_PHYSICAL_SIZE_GLOBAL.sub(self.last_set.load(std::sync::atomic::Ordering::Relaxed));
2339 19 : }
2340 : }
2341 :
2342 : pub(crate) struct RemoteTimelineClientMetrics {
2343 : tenant_id: String,
2344 : shard_id: String,
2345 : timeline_id: String,
2346 : pub(crate) remote_physical_size_gauge: PerTimelineRemotePhysicalSizeGauge,
2347 : calls: Mutex<HashMap<(&'static str, &'static str), IntCounterPair>>,
2348 : bytes_started_counter: Mutex<HashMap<(&'static str, &'static str), IntCounter>>,
2349 : bytes_finished_counter: Mutex<HashMap<(&'static str, &'static str), IntCounter>>,
2350 : }
2351 :
2352 : impl RemoteTimelineClientMetrics {
2353 392 : pub fn new(tenant_shard_id: &TenantShardId, timeline_id: &TimelineId) -> Self {
2354 392 : let tenant_id_str = tenant_shard_id.tenant_id.to_string();
2355 392 : let shard_id_str = format!("{}", tenant_shard_id.shard_slug());
2356 392 : let timeline_id_str = timeline_id.to_string();
2357 392 :
2358 392 : let remote_physical_size_gauge = PerTimelineRemotePhysicalSizeGauge::new(
2359 392 : REMOTE_PHYSICAL_SIZE
2360 392 : .get_metric_with_label_values(&[&tenant_id_str, &shard_id_str, &timeline_id_str])
2361 392 : .unwrap(),
2362 392 : );
2363 392 :
2364 392 : RemoteTimelineClientMetrics {
2365 392 : tenant_id: tenant_id_str,
2366 392 : shard_id: shard_id_str,
2367 392 : timeline_id: timeline_id_str,
2368 392 : calls: Mutex::new(HashMap::default()),
2369 392 : bytes_started_counter: Mutex::new(HashMap::default()),
2370 392 : bytes_finished_counter: Mutex::new(HashMap::default()),
2371 392 : remote_physical_size_gauge,
2372 392 : }
2373 392 : }
2374 :
2375 2593 : pub fn remote_operation_time(
2376 2593 : &self,
2377 2593 : file_kind: &RemoteOpFileKind,
2378 2593 : op_kind: &RemoteOpKind,
2379 2593 : status: &'static str,
2380 2593 : ) -> Histogram {
2381 2593 : let key = (file_kind.as_str(), op_kind.as_str(), status);
2382 2593 : REMOTE_OPERATION_TIME
2383 2593 : .get_metric_with_label_values(&[key.0, key.1, key.2])
2384 2593 : .unwrap()
2385 2593 : }
2386 :
2387 6126 : fn calls_counter_pair(
2388 6126 : &self,
2389 6126 : file_kind: &RemoteOpFileKind,
2390 6126 : op_kind: &RemoteOpKind,
2391 6126 : ) -> IntCounterPair {
2392 6126 : let mut guard = self.calls.lock().unwrap();
2393 6126 : let key = (file_kind.as_str(), op_kind.as_str());
2394 6126 : let metric = guard.entry(key).or_insert_with(move || {
2395 674 : REMOTE_TIMELINE_CLIENT_CALLS
2396 674 : .get_metric_with_label_values(&[
2397 674 : &self.tenant_id,
2398 674 : &self.shard_id,
2399 674 : &self.timeline_id,
2400 674 : key.0,
2401 674 : key.1,
2402 674 : ])
2403 674 : .unwrap()
2404 6126 : });
2405 6126 : metric.clone()
2406 6126 : }
2407 :
2408 1466 : fn bytes_started_counter(
2409 1466 : &self,
2410 1466 : file_kind: &RemoteOpFileKind,
2411 1466 : op_kind: &RemoteOpKind,
2412 1466 : ) -> IntCounter {
2413 1466 : let mut guard = self.bytes_started_counter.lock().unwrap();
2414 1466 : let key = (file_kind.as_str(), op_kind.as_str());
2415 1466 : let metric = guard.entry(key).or_insert_with(move || {
2416 254 : REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER
2417 254 : .get_metric_with_label_values(&[
2418 254 : &self.tenant_id,
2419 254 : &self.shard_id,
2420 254 : &self.timeline_id,
2421 254 : key.0,
2422 254 : key.1,
2423 254 : ])
2424 254 : .unwrap()
2425 1466 : });
2426 1466 : metric.clone()
2427 1466 : }
2428 :
2429 2680 : fn bytes_finished_counter(
2430 2680 : &self,
2431 2680 : file_kind: &RemoteOpFileKind,
2432 2680 : op_kind: &RemoteOpKind,
2433 2680 : ) -> IntCounter {
2434 2680 : let mut guard = self.bytes_finished_counter.lock().unwrap();
2435 2680 : let key = (file_kind.as_str(), op_kind.as_str());
2436 2680 : let metric = guard.entry(key).or_insert_with(move || {
2437 254 : REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER
2438 254 : .get_metric_with_label_values(&[
2439 254 : &self.tenant_id,
2440 254 : &self.shard_id,
2441 254 : &self.timeline_id,
2442 254 : key.0,
2443 254 : key.1,
2444 254 : ])
2445 254 : .unwrap()
2446 2680 : });
2447 2680 : metric.clone()
2448 2680 : }
2449 : }
2450 :
2451 : #[cfg(test)]
2452 : impl RemoteTimelineClientMetrics {
2453 6 : pub fn get_bytes_started_counter_value(
2454 6 : &self,
2455 6 : file_kind: &RemoteOpFileKind,
2456 6 : op_kind: &RemoteOpKind,
2457 6 : ) -> Option<u64> {
2458 6 : let guard = self.bytes_started_counter.lock().unwrap();
2459 6 : let key = (file_kind.as_str(), op_kind.as_str());
2460 6 : guard.get(&key).map(|counter| counter.get())
2461 6 : }
2462 :
2463 6 : pub fn get_bytes_finished_counter_value(
2464 6 : &self,
2465 6 : file_kind: &RemoteOpFileKind,
2466 6 : op_kind: &RemoteOpKind,
2467 6 : ) -> Option<u64> {
2468 6 : let guard = self.bytes_finished_counter.lock().unwrap();
2469 6 : let key = (file_kind.as_str(), op_kind.as_str());
2470 6 : guard.get(&key).map(|counter| counter.get())
2471 6 : }
2472 : }
2473 :
2474 : /// See [`RemoteTimelineClientMetrics::call_begin`].
2475 : #[must_use]
2476 : pub(crate) struct RemoteTimelineClientCallMetricGuard {
2477 : /// Decremented on drop.
2478 : calls_counter_pair: Option<IntCounterPair>,
2479 : /// If Some(), this references the bytes_finished metric, and we increment it by the given `u64` on drop.
2480 : bytes_finished: Option<(IntCounter, u64)>,
2481 : }
2482 :
2483 : impl RemoteTimelineClientCallMetricGuard {
2484 : /// Consume this guard object without performing the metric updates it would do on `drop()`.
2485 : /// The caller vouches to do the metric updates manually.
2486 3309 : pub fn will_decrement_manually(mut self) {
2487 3309 : let RemoteTimelineClientCallMetricGuard {
2488 3309 : calls_counter_pair,
2489 3309 : bytes_finished,
2490 3309 : } = &mut self;
2491 3309 : calls_counter_pair.take();
2492 3309 : bytes_finished.take();
2493 3309 : }
2494 : }
2495 :
2496 : impl Drop for RemoteTimelineClientCallMetricGuard {
2497 3335 : fn drop(&mut self) {
2498 3335 : let RemoteTimelineClientCallMetricGuard {
2499 3335 : calls_counter_pair,
2500 3335 : bytes_finished,
2501 3335 : } = self;
2502 3335 : if let Some(guard) = calls_counter_pair.take() {
2503 26 : guard.dec();
2504 3309 : }
2505 3335 : if let Some((bytes_finished_metric, value)) = bytes_finished {
2506 0 : bytes_finished_metric.inc_by(*value);
2507 3335 : }
2508 3335 : }
2509 : }
2510 :
2511 : /// The enum variants communicate to the [`RemoteTimelineClientMetrics`] whether to
2512 : /// track the byte size of this call in applicable metric(s).
2513 : pub(crate) enum RemoteTimelineClientMetricsCallTrackSize {
2514 : /// Do not account for this call's byte size in any metrics.
2515 : /// The `reason` field is there to make the call sites self-documenting
2516 : /// about why they don't need the metric.
2517 : DontTrackSize { reason: &'static str },
2518 : /// Track the byte size of the call in applicable metric(s).
2519 : Bytes(u64),
2520 : }
2521 :
2522 : impl RemoteTimelineClientMetrics {
2523 : /// Update the metrics that change when a call to the remote timeline client instance starts.
2524 : ///
2525 : /// Drop the returned guard object once the operation is finished to updates corresponding metrics that track completions.
2526 : /// Or, use [`RemoteTimelineClientCallMetricGuard::will_decrement_manually`] and [`call_end`](Self::call_end) if that
2527 : /// is more suitable.
2528 : /// Never do both.
2529 3335 : pub(crate) fn call_begin(
2530 3335 : &self,
2531 3335 : file_kind: &RemoteOpFileKind,
2532 3335 : op_kind: &RemoteOpKind,
2533 3335 : size: RemoteTimelineClientMetricsCallTrackSize,
2534 3335 : ) -> RemoteTimelineClientCallMetricGuard {
2535 3335 : let calls_counter_pair = self.calls_counter_pair(file_kind, op_kind);
2536 3335 : calls_counter_pair.inc();
2537 :
2538 3335 : let bytes_finished = match size {
2539 1869 : RemoteTimelineClientMetricsCallTrackSize::DontTrackSize { reason: _reason } => {
2540 1869 : // nothing to do
2541 1869 : None
2542 : }
2543 1466 : RemoteTimelineClientMetricsCallTrackSize::Bytes(size) => {
2544 1466 : self.bytes_started_counter(file_kind, op_kind).inc_by(size);
2545 1466 : let finished_counter = self.bytes_finished_counter(file_kind, op_kind);
2546 1466 : Some((finished_counter, size))
2547 : }
2548 : };
2549 3335 : RemoteTimelineClientCallMetricGuard {
2550 3335 : calls_counter_pair: Some(calls_counter_pair),
2551 3335 : bytes_finished,
2552 3335 : }
2553 3335 : }
2554 :
2555 : /// Manually udpate the metrics that track completions, instead of using the guard object.
2556 : /// Using the guard object is generally preferable.
2557 : /// See [`call_begin`](Self::call_begin) for more context.
2558 2791 : pub(crate) fn call_end(
2559 2791 : &self,
2560 2791 : file_kind: &RemoteOpFileKind,
2561 2791 : op_kind: &RemoteOpKind,
2562 2791 : size: RemoteTimelineClientMetricsCallTrackSize,
2563 2791 : ) {
2564 2791 : let calls_counter_pair = self.calls_counter_pair(file_kind, op_kind);
2565 2791 : calls_counter_pair.dec();
2566 2791 : match size {
2567 1577 : RemoteTimelineClientMetricsCallTrackSize::DontTrackSize { reason: _reason } => {}
2568 1214 : RemoteTimelineClientMetricsCallTrackSize::Bytes(size) => {
2569 1214 : self.bytes_finished_counter(file_kind, op_kind).inc_by(size);
2570 1214 : }
2571 : }
2572 2791 : }
2573 : }
2574 :
2575 : impl Drop for RemoteTimelineClientMetrics {
2576 19 : fn drop(&mut self) {
2577 19 : let RemoteTimelineClientMetrics {
2578 19 : tenant_id,
2579 19 : shard_id,
2580 19 : timeline_id,
2581 19 : remote_physical_size_gauge,
2582 19 : calls,
2583 19 : bytes_started_counter,
2584 19 : bytes_finished_counter,
2585 19 : } = self;
2586 24 : for ((a, b), _) in calls.get_mut().unwrap().drain() {
2587 24 : let mut res = [Ok(()), Ok(())];
2588 24 : REMOTE_TIMELINE_CLIENT_CALLS
2589 24 : .remove_label_values(&mut res, &[tenant_id, shard_id, timeline_id, a, b]);
2590 24 : // don't care about results
2591 24 : }
2592 19 : for ((a, b), _) in bytes_started_counter.get_mut().unwrap().drain() {
2593 6 : let _ = REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER.remove_label_values(&[
2594 6 : tenant_id,
2595 6 : shard_id,
2596 6 : timeline_id,
2597 6 : a,
2598 6 : b,
2599 6 : ]);
2600 6 : }
2601 19 : for ((a, b), _) in bytes_finished_counter.get_mut().unwrap().drain() {
2602 6 : let _ = REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER.remove_label_values(&[
2603 6 : tenant_id,
2604 6 : shard_id,
2605 6 : timeline_id,
2606 6 : a,
2607 6 : b,
2608 6 : ]);
2609 6 : }
2610 19 : {
2611 19 : let _ = remote_physical_size_gauge; // use to avoid 'unused' warning in desctructuring above
2612 19 : let _ = REMOTE_PHYSICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2613 19 : }
2614 19 : }
2615 : }
2616 :
2617 : /// Wrapper future that measures the time spent by a remote storage operation,
2618 : /// and records the time and success/failure as a prometheus metric.
2619 : pub(crate) trait MeasureRemoteOp: Sized {
2620 2711 : fn measure_remote_op(
2621 2711 : self,
2622 2711 : file_kind: RemoteOpFileKind,
2623 2711 : op: RemoteOpKind,
2624 2711 : metrics: Arc<RemoteTimelineClientMetrics>,
2625 2711 : ) -> MeasuredRemoteOp<Self> {
2626 2711 : let start = Instant::now();
2627 2711 : MeasuredRemoteOp {
2628 2711 : inner: self,
2629 2711 : file_kind,
2630 2711 : op,
2631 2711 : start,
2632 2711 : metrics,
2633 2711 : }
2634 2711 : }
2635 : }
2636 :
2637 : impl<T: Sized> MeasureRemoteOp for T {}
2638 :
2639 : pin_project! {
2640 : pub(crate) struct MeasuredRemoteOp<F>
2641 : {
2642 : #[pin]
2643 : inner: F,
2644 : file_kind: RemoteOpFileKind,
2645 : op: RemoteOpKind,
2646 : start: Instant,
2647 : metrics: Arc<RemoteTimelineClientMetrics>,
2648 : }
2649 : }
2650 :
2651 : impl<F: Future<Output = Result<O, E>>, O, E> Future for MeasuredRemoteOp<F> {
2652 : type Output = Result<O, E>;
2653 :
2654 41736 : fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
2655 41736 : let this = self.project();
2656 41736 : let poll_result = this.inner.poll(cx);
2657 41736 : if let Poll::Ready(ref res) = poll_result {
2658 2593 : let duration = this.start.elapsed();
2659 2593 : let status = if res.is_ok() { &"success" } else { &"failure" };
2660 2593 : this.metrics
2661 2593 : .remote_operation_time(this.file_kind, this.op, status)
2662 2593 : .observe(duration.as_secs_f64());
2663 39143 : }
2664 41736 : poll_result
2665 41736 : }
2666 : }
2667 :
2668 : pub mod tokio_epoll_uring {
2669 : use metrics::{register_int_counter, UIntGauge};
2670 : use once_cell::sync::Lazy;
2671 :
2672 : pub struct Collector {
2673 : descs: Vec<metrics::core::Desc>,
2674 : systems_created: UIntGauge,
2675 : systems_destroyed: UIntGauge,
2676 : }
2677 :
2678 : impl metrics::core::Collector for Collector {
2679 0 : fn desc(&self) -> Vec<&metrics::core::Desc> {
2680 0 : self.descs.iter().collect()
2681 0 : }
2682 :
2683 0 : fn collect(&self) -> Vec<metrics::proto::MetricFamily> {
2684 0 : let mut mfs = Vec::with_capacity(Self::NMETRICS);
2685 0 : let tokio_epoll_uring::metrics::Metrics {
2686 0 : systems_created,
2687 0 : systems_destroyed,
2688 0 : } = tokio_epoll_uring::metrics::global();
2689 0 : self.systems_created.set(systems_created);
2690 0 : mfs.extend(self.systems_created.collect());
2691 0 : self.systems_destroyed.set(systems_destroyed);
2692 0 : mfs.extend(self.systems_destroyed.collect());
2693 0 : mfs
2694 0 : }
2695 : }
2696 :
2697 : impl Collector {
2698 : const NMETRICS: usize = 2;
2699 :
2700 : #[allow(clippy::new_without_default)]
2701 0 : pub fn new() -> Self {
2702 0 : let mut descs = Vec::new();
2703 0 :
2704 0 : let systems_created = UIntGauge::new(
2705 0 : "pageserver_tokio_epoll_uring_systems_created",
2706 0 : "counter of tokio-epoll-uring systems that were created",
2707 0 : )
2708 0 : .unwrap();
2709 0 : descs.extend(
2710 0 : metrics::core::Collector::desc(&systems_created)
2711 0 : .into_iter()
2712 0 : .cloned(),
2713 0 : );
2714 0 :
2715 0 : let systems_destroyed = UIntGauge::new(
2716 0 : "pageserver_tokio_epoll_uring_systems_destroyed",
2717 0 : "counter of tokio-epoll-uring systems that were destroyed",
2718 0 : )
2719 0 : .unwrap();
2720 0 : descs.extend(
2721 0 : metrics::core::Collector::desc(&systems_destroyed)
2722 0 : .into_iter()
2723 0 : .cloned(),
2724 0 : );
2725 0 :
2726 0 : Self {
2727 0 : descs,
2728 0 : systems_created,
2729 0 : systems_destroyed,
2730 0 : }
2731 0 : }
2732 : }
2733 :
2734 81 : pub(crate) static THREAD_LOCAL_LAUNCH_SUCCESSES: Lazy<metrics::IntCounter> = Lazy::new(|| {
2735 : register_int_counter!(
2736 : "pageserver_tokio_epoll_uring_pageserver_thread_local_launch_success_count",
2737 : "Number of times where thread_local_system creation spanned multiple executor threads",
2738 : )
2739 81 : .unwrap()
2740 81 : });
2741 :
2742 0 : pub(crate) static THREAD_LOCAL_LAUNCH_FAILURES: Lazy<metrics::IntCounter> = Lazy::new(|| {
2743 : register_int_counter!(
2744 : "pageserver_tokio_epoll_uring_pageserver_thread_local_launch_failures_count",
2745 : "Number of times thread_local_system creation failed and was retried after back-off.",
2746 : )
2747 0 : .unwrap()
2748 0 : });
2749 : }
2750 :
2751 : pub(crate) mod tenant_throttling {
2752 : use metrics::{register_int_counter_vec, IntCounter};
2753 : use once_cell::sync::Lazy;
2754 :
2755 : use crate::tenant::{self, throttle::Metric};
2756 :
2757 : pub(crate) struct TimelineGet {
2758 : wait_time: IntCounter,
2759 : count: IntCounter,
2760 : }
2761 :
2762 142 : pub(crate) static TIMELINE_GET: Lazy<TimelineGet> = Lazy::new(|| {
2763 142 : static WAIT_USECS: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
2764 142 : register_int_counter_vec!(
2765 142 : "pageserver_tenant_throttling_wait_usecs_sum_global",
2766 142 : "Sum of microseconds that tenants spent waiting for a tenant throttle of a given kind.",
2767 142 : &["kind"]
2768 142 : )
2769 142 : .unwrap()
2770 142 : });
2771 142 :
2772 142 : static WAIT_COUNT: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
2773 142 : register_int_counter_vec!(
2774 142 : "pageserver_tenant_throttling_count_global",
2775 142 : "Count of tenant throttlings, by kind of throttle.",
2776 142 : &["kind"]
2777 142 : )
2778 142 : .unwrap()
2779 142 : });
2780 142 :
2781 142 : let kind = "timeline_get";
2782 142 : TimelineGet {
2783 142 : wait_time: WAIT_USECS.with_label_values(&[kind]),
2784 142 : count: WAIT_COUNT.with_label_values(&[kind]),
2785 142 : }
2786 142 : });
2787 :
2788 : impl Metric for &'static TimelineGet {
2789 : #[inline(always)]
2790 0 : fn observe_throttling(
2791 0 : &self,
2792 0 : tenant::throttle::Observation { wait_time }: &tenant::throttle::Observation,
2793 0 : ) {
2794 0 : let val = u64::try_from(wait_time.as_micros()).unwrap();
2795 0 : self.wait_time.inc_by(val);
2796 0 : self.count.inc();
2797 0 : }
2798 : }
2799 : }
2800 :
2801 : pub(crate) mod disk_usage_based_eviction {
2802 : use super::*;
2803 :
2804 : pub(crate) struct Metrics {
2805 : pub(crate) tenant_collection_time: Histogram,
2806 : pub(crate) tenant_layer_count: Histogram,
2807 : pub(crate) layers_collected: IntCounter,
2808 : pub(crate) layers_selected: IntCounter,
2809 : pub(crate) layers_evicted: IntCounter,
2810 : }
2811 :
2812 : impl Default for Metrics {
2813 0 : fn default() -> Self {
2814 0 : let tenant_collection_time = register_histogram!(
2815 : "pageserver_disk_usage_based_eviction_tenant_collection_seconds",
2816 : "Time spent collecting layers from a tenant -- not normalized by collected layer amount",
2817 : vec![0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0, 10.0]
2818 : )
2819 0 : .unwrap();
2820 0 :
2821 0 : let tenant_layer_count = register_histogram!(
2822 : "pageserver_disk_usage_based_eviction_tenant_collected_layers",
2823 : "Amount of layers gathered from a tenant",
2824 : vec![5.0, 50.0, 500.0, 5000.0, 50000.0]
2825 : )
2826 0 : .unwrap();
2827 0 :
2828 0 : let layers_collected = register_int_counter!(
2829 : "pageserver_disk_usage_based_eviction_collected_layers_total",
2830 : "Amount of layers collected"
2831 : )
2832 0 : .unwrap();
2833 0 :
2834 0 : let layers_selected = register_int_counter!(
2835 : "pageserver_disk_usage_based_eviction_select_layers_total",
2836 : "Amount of layers selected"
2837 : )
2838 0 : .unwrap();
2839 0 :
2840 0 : let layers_evicted = register_int_counter!(
2841 : "pageserver_disk_usage_based_eviction_evicted_layers_total",
2842 : "Amount of layers successfully evicted"
2843 : )
2844 0 : .unwrap();
2845 0 :
2846 0 : Self {
2847 0 : tenant_collection_time,
2848 0 : tenant_layer_count,
2849 0 : layers_collected,
2850 0 : layers_selected,
2851 0 : layers_evicted,
2852 0 : }
2853 0 : }
2854 : }
2855 :
2856 : pub(crate) static METRICS: Lazy<Metrics> = Lazy::new(Metrics::default);
2857 : }
2858 :
2859 135 : static TOKIO_EXECUTOR_THREAD_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
2860 : register_uint_gauge_vec!(
2861 : "pageserver_tokio_executor_thread_configured_count",
2862 : "Total number of configued tokio executor threads in the process.
2863 : The `setup` label denotes whether we're running with multiple runtimes or a single runtime.",
2864 : &["setup"],
2865 : )
2866 135 : .unwrap()
2867 135 : });
2868 :
2869 135 : pub(crate) fn set_tokio_runtime_setup(setup: &str, num_threads: NonZeroUsize) {
2870 135 : static SERIALIZE: std::sync::Mutex<()> = std::sync::Mutex::new(());
2871 135 : let _guard = SERIALIZE.lock().unwrap();
2872 135 : TOKIO_EXECUTOR_THREAD_COUNT.reset();
2873 135 : TOKIO_EXECUTOR_THREAD_COUNT
2874 135 : .get_metric_with_label_values(&[setup])
2875 135 : .unwrap()
2876 135 : .set(u64::try_from(num_threads.get()).unwrap());
2877 135 : }
2878 :
2879 0 : pub fn preinitialize_metrics() {
2880 0 : // Python tests need these and on some we do alerting.
2881 0 : //
2882 0 : // FIXME(4813): make it so that we have no top level metrics as this fn will easily fall out of
2883 0 : // order:
2884 0 : // - global metrics reside in a Lazy<PageserverMetrics>
2885 0 : // - access via crate::metrics::PS_METRICS.some_metric.inc()
2886 0 : // - could move the statics into TimelineMetrics::new()?
2887 0 :
2888 0 : // counters
2889 0 : [
2890 0 : &UNEXPECTED_ONDEMAND_DOWNLOADS,
2891 0 : &WALRECEIVER_STARTED_CONNECTIONS,
2892 0 : &WALRECEIVER_BROKER_UPDATES,
2893 0 : &WALRECEIVER_CANDIDATES_ADDED,
2894 0 : &WALRECEIVER_CANDIDATES_REMOVED,
2895 0 : &tokio_epoll_uring::THREAD_LOCAL_LAUNCH_FAILURES,
2896 0 : &tokio_epoll_uring::THREAD_LOCAL_LAUNCH_SUCCESSES,
2897 0 : &REMOTE_ONDEMAND_DOWNLOADED_LAYERS,
2898 0 : &REMOTE_ONDEMAND_DOWNLOADED_BYTES,
2899 0 : ]
2900 0 : .into_iter()
2901 0 : .for_each(|c| {
2902 0 : Lazy::force(c);
2903 0 : });
2904 0 :
2905 0 : // Deletion queue stats
2906 0 : Lazy::force(&DELETION_QUEUE);
2907 0 :
2908 0 : // Tenant stats
2909 0 : Lazy::force(&TENANT);
2910 0 :
2911 0 : // Tenant manager stats
2912 0 : Lazy::force(&TENANT_MANAGER);
2913 0 :
2914 0 : Lazy::force(&crate::tenant::storage_layer::layer::LAYER_IMPL_METRICS);
2915 0 : Lazy::force(&disk_usage_based_eviction::METRICS);
2916 :
2917 0 : for state_name in pageserver_api::models::TenantState::VARIANTS {
2918 0 : // initialize the metric for all gauges, otherwise the time series might seemingly show
2919 0 : // values from last restart.
2920 0 : TENANT_STATE_METRIC.with_label_values(&[state_name]).set(0);
2921 0 : }
2922 :
2923 : // countervecs
2924 0 : [&BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT]
2925 0 : .into_iter()
2926 0 : .for_each(|c| {
2927 0 : Lazy::force(c);
2928 0 : });
2929 0 :
2930 0 : // gauges
2931 0 : WALRECEIVER_ACTIVE_MANAGERS.get();
2932 0 :
2933 0 : // histograms
2934 0 : [
2935 0 : &READ_NUM_LAYERS_VISITED,
2936 0 : &VEC_READ_NUM_LAYERS_VISITED,
2937 0 : &WAIT_LSN_TIME,
2938 0 : &WAL_REDO_TIME,
2939 0 : &WAL_REDO_RECORDS_HISTOGRAM,
2940 0 : &WAL_REDO_BYTES_HISTOGRAM,
2941 0 : &WAL_REDO_PROCESS_LAUNCH_DURATION_HISTOGRAM,
2942 0 : ]
2943 0 : .into_iter()
2944 0 : .for_each(|h| {
2945 0 : Lazy::force(h);
2946 0 : });
2947 0 :
2948 0 : // Custom
2949 0 : Lazy::force(&RECONSTRUCT_TIME);
2950 0 : Lazy::force(&tenant_throttling::TIMELINE_GET);
2951 0 : Lazy::force(&BASEBACKUP_QUERY_TIME);
2952 0 : }
|
