Line data Source code
1 : use enum_map::EnumMap;
2 : use metrics::{
3 : register_counter_vec, register_gauge_vec, register_histogram, register_histogram_vec,
4 : register_int_counter, register_int_counter_pair_vec, register_int_counter_vec,
5 : register_int_gauge, register_int_gauge_vec, register_uint_gauge, register_uint_gauge_vec,
6 : Counter, CounterVec, GaugeVec, Histogram, HistogramVec, IntCounter, IntCounterPair,
7 : IntCounterPairVec, IntCounterVec, IntGauge, IntGaugeVec, UIntGauge, UIntGaugeVec,
8 : };
9 : use once_cell::sync::Lazy;
10 : use pageserver_api::shard::TenantShardId;
11 : use strum::{EnumCount, VariantNames};
12 : use strum_macros::{EnumVariantNames, IntoStaticStr};
13 : use tracing::warn;
14 : use utils::id::TimelineId;
15 :
16 : /// Prometheus histogram buckets (in seconds) for operations in the critical
17 : /// path. In other words, operations that directly affect that latency of user
18 : /// queries.
19 : ///
20 : /// The buckets capture the majority of latencies in the microsecond and
21 : /// millisecond range but also extend far enough up to distinguish "bad" from
22 : /// "really bad".
23 : const CRITICAL_OP_BUCKETS: &[f64] = &[
24 : 0.000_001, 0.000_010, 0.000_100, // 1 us, 10 us, 100 us
25 : 0.001_000, 0.010_000, 0.100_000, // 1 ms, 10 ms, 100 ms
26 : 1.0, 10.0, 100.0, // 1 s, 10 s, 100 s
27 : ];
28 :
29 : // Metrics collected on operations on the storage repository.
30 3184 : #[derive(Debug, EnumVariantNames, IntoStaticStr)]
31 : #[strum(serialize_all = "kebab_case")]
32 : pub(crate) enum StorageTimeOperation {
33 : #[strum(serialize = "layer flush")]
34 : LayerFlush,
35 :
36 : #[strum(serialize = "compact")]
37 : Compact,
38 :
39 : #[strum(serialize = "create images")]
40 : CreateImages,
41 :
42 : #[strum(serialize = "logical size")]
43 : LogicalSize,
44 :
45 : #[strum(serialize = "imitate logical size")]
46 : ImitateLogicalSize,
47 :
48 : #[strum(serialize = "load layer map")]
49 : LoadLayerMap,
50 :
51 : #[strum(serialize = "gc")]
52 : Gc,
53 :
54 : #[strum(serialize = "find gc cutoffs")]
55 : FindGcCutoffs,
56 : }
57 :
58 154 : pub(crate) static STORAGE_TIME_SUM_PER_TIMELINE: Lazy<CounterVec> = Lazy::new(|| {
59 : register_counter_vec!(
60 : "pageserver_storage_operations_seconds_sum",
61 : "Total time spent on storage operations with operation, tenant and timeline dimensions",
62 : &["operation", "tenant_id", "shard_id", "timeline_id"],
63 : )
64 154 : .expect("failed to define a metric")
65 154 : });
66 :
67 154 : pub(crate) static STORAGE_TIME_COUNT_PER_TIMELINE: Lazy<IntCounterVec> = Lazy::new(|| {
68 : register_int_counter_vec!(
69 : "pageserver_storage_operations_seconds_count",
70 : "Count of storage operations with operation, tenant and timeline dimensions",
71 : &["operation", "tenant_id", "shard_id", "timeline_id"],
72 : )
73 154 : .expect("failed to define a metric")
74 154 : });
75 :
76 : // Buckets for background operations like compaction, GC, size calculation
77 : const STORAGE_OP_BUCKETS: &[f64] = &[0.010, 0.100, 1.0, 10.0, 100.0, 1000.0];
78 :
79 154 : pub(crate) static STORAGE_TIME_GLOBAL: Lazy<HistogramVec> = Lazy::new(|| {
80 : register_histogram_vec!(
81 : "pageserver_storage_operations_seconds_global",
82 : "Time spent on storage operations",
83 : &["operation"],
84 : STORAGE_OP_BUCKETS.into(),
85 : )
86 154 : .expect("failed to define a metric")
87 154 : });
88 :
89 4 : pub(crate) static READ_NUM_LAYERS_VISITED: Lazy<Histogram> = Lazy::new(|| {
90 : register_histogram!(
91 : "pageserver_layers_visited_per_read_global",
92 : "Number of layers visited to reconstruct one key",
93 : vec![1.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0],
94 : )
95 4 : .expect("failed to define a metric")
96 4 : });
97 :
98 150 : pub(crate) static VEC_READ_NUM_LAYERS_VISITED: Lazy<Histogram> = Lazy::new(|| {
99 : register_histogram!(
100 : "pageserver_layers_visited_per_vectored_read_global",
101 : "Average number of layers visited to reconstruct one key",
102 : vec![1.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0],
103 : )
104 150 : .expect("failed to define a metric")
105 150 : });
106 :
107 : // Metrics collected on operations on the storage repository.
108 : #[derive(
109 608 : Clone, Copy, enum_map::Enum, strum_macros::EnumString, strum_macros::Display, IntoStaticStr,
110 : )]
111 : pub(crate) enum GetKind {
112 : Singular,
113 : Vectored,
114 : }
115 :
116 : pub(crate) struct ReconstructTimeMetrics {
117 : singular: Histogram,
118 : vectored: Histogram,
119 : }
120 :
121 152 : pub(crate) static RECONSTRUCT_TIME: Lazy<ReconstructTimeMetrics> = Lazy::new(|| {
122 152 : let inner = register_histogram_vec!(
123 : "pageserver_getpage_reconstruct_seconds",
124 : "Time spent in reconstruct_value (reconstruct a page from deltas)",
125 : &["get_kind"],
126 : CRITICAL_OP_BUCKETS.into(),
127 : )
128 152 : .expect("failed to define a metric");
129 152 :
130 152 : ReconstructTimeMetrics {
131 152 : singular: inner.with_label_values(&[GetKind::Singular.into()]),
132 152 : vectored: inner.with_label_values(&[GetKind::Vectored.into()]),
133 152 : }
134 152 : });
135 :
136 : impl ReconstructTimeMetrics {
137 626352 : pub(crate) fn for_get_kind(&self, get_kind: GetKind) -> &Histogram {
138 626352 : match get_kind {
139 626112 : GetKind::Singular => &self.singular,
140 240 : GetKind::Vectored => &self.vectored,
141 : }
142 626352 : }
143 : }
144 :
145 : pub(crate) struct ReconstructDataTimeMetrics {
146 : singular: Histogram,
147 : vectored: Histogram,
148 : }
149 :
150 : impl ReconstructDataTimeMetrics {
151 626370 : pub(crate) fn for_get_kind(&self, get_kind: GetKind) -> &Histogram {
152 626370 : match get_kind {
153 626130 : GetKind::Singular => &self.singular,
154 240 : GetKind::Vectored => &self.vectored,
155 : }
156 626370 : }
157 : }
158 :
159 152 : pub(crate) static GET_RECONSTRUCT_DATA_TIME: Lazy<ReconstructDataTimeMetrics> = Lazy::new(|| {
160 152 : let inner = register_histogram_vec!(
161 : "pageserver_getpage_get_reconstruct_data_seconds",
162 : "Time spent in get_reconstruct_value_data",
163 : &["get_kind"],
164 : CRITICAL_OP_BUCKETS.into(),
165 : )
166 152 : .expect("failed to define a metric");
167 152 :
168 152 : ReconstructDataTimeMetrics {
169 152 : singular: inner.with_label_values(&[GetKind::Singular.into()]),
170 152 : vectored: inner.with_label_values(&[GetKind::Vectored.into()]),
171 152 : }
172 152 : });
173 :
174 : pub(crate) struct GetVectoredLatency {
175 : map: EnumMap<TaskKind, Option<Histogram>>,
176 : }
177 :
178 : #[allow(dead_code)]
179 : pub(crate) struct ScanLatency {
180 : map: EnumMap<TaskKind, Option<Histogram>>,
181 : }
182 :
183 : impl GetVectoredLatency {
184 : // Only these task types perform vectored gets. Filter all other tasks out to reduce total
185 : // cardinality of the metric.
186 : const TRACKED_TASK_KINDS: [TaskKind; 2] = [TaskKind::Compaction, TaskKind::PageRequestHandler];
187 :
188 1036 : pub(crate) fn for_task_kind(&self, task_kind: TaskKind) -> Option<&Histogram> {
189 1036 : self.map[task_kind].as_ref()
190 1036 : }
191 : }
192 :
193 : impl ScanLatency {
194 : // Only these task types perform vectored gets. Filter all other tasks out to reduce total
195 : // cardinality of the metric.
196 : const TRACKED_TASK_KINDS: [TaskKind; 1] = [TaskKind::PageRequestHandler];
197 :
198 12 : pub(crate) fn for_task_kind(&self, task_kind: TaskKind) -> Option<&Histogram> {
199 12 : self.map[task_kind].as_ref()
200 12 : }
201 : }
202 :
203 : pub(crate) struct ScanLatencyOngoingRecording<'a> {
204 : parent: &'a Histogram,
205 : start: std::time::Instant,
206 : }
207 :
208 : impl<'a> ScanLatencyOngoingRecording<'a> {
209 0 : pub(crate) fn start_recording(parent: &'a Histogram) -> ScanLatencyOngoingRecording<'a> {
210 0 : let start = Instant::now();
211 0 : ScanLatencyOngoingRecording { parent, start }
212 0 : }
213 :
214 0 : pub(crate) fn observe(self, throttled: Option<Duration>) {
215 0 : let elapsed = self.start.elapsed();
216 0 : let ex_throttled = if let Some(throttled) = throttled {
217 0 : elapsed.checked_sub(throttled)
218 : } else {
219 0 : Some(elapsed)
220 : };
221 0 : if let Some(ex_throttled) = ex_throttled {
222 0 : self.parent.observe(ex_throttled.as_secs_f64());
223 0 : } else {
224 0 : use utils::rate_limit::RateLimit;
225 0 : static LOGGED: Lazy<Mutex<RateLimit>> =
226 0 : Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(10))));
227 0 : let mut rate_limit = LOGGED.lock().unwrap();
228 0 : rate_limit.call(|| {
229 0 : warn!("error deducting time spent throttled; this message is logged at a global rate limit");
230 0 : });
231 0 : }
232 0 : }
233 : }
234 :
235 146 : pub(crate) static GET_VECTORED_LATENCY: Lazy<GetVectoredLatency> = Lazy::new(|| {
236 146 : let inner = register_histogram_vec!(
237 : "pageserver_get_vectored_seconds",
238 : "Time spent in get_vectored, excluding time spent in timeline_get_throttle.",
239 : &["task_kind"],
240 : CRITICAL_OP_BUCKETS.into(),
241 : )
242 146 : .expect("failed to define a metric");
243 146 :
244 146 : GetVectoredLatency {
245 4380 : map: EnumMap::from_array(std::array::from_fn(|task_kind_idx| {
246 4380 : let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind_idx);
247 4380 :
248 4380 : if GetVectoredLatency::TRACKED_TASK_KINDS.contains(&task_kind) {
249 292 : let task_kind = task_kind.into();
250 292 : Some(inner.with_label_values(&[task_kind]))
251 : } else {
252 4088 : None
253 : }
254 4380 : })),
255 146 : }
256 146 : });
257 :
258 4 : pub(crate) static SCAN_LATENCY: Lazy<ScanLatency> = Lazy::new(|| {
259 4 : let inner = register_histogram_vec!(
260 : "pageserver_scan_seconds",
261 : "Time spent in scan, excluding time spent in timeline_get_throttle.",
262 : &["task_kind"],
263 : CRITICAL_OP_BUCKETS.into(),
264 : )
265 4 : .expect("failed to define a metric");
266 4 :
267 4 : ScanLatency {
268 120 : map: EnumMap::from_array(std::array::from_fn(|task_kind_idx| {
269 120 : let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind_idx);
270 120 :
271 120 : if ScanLatency::TRACKED_TASK_KINDS.contains(&task_kind) {
272 4 : let task_kind = task_kind.into();
273 4 : Some(inner.with_label_values(&[task_kind]))
274 : } else {
275 116 : None
276 : }
277 120 : })),
278 4 : }
279 4 : });
280 :
281 : pub(crate) struct PageCacheMetricsForTaskKind {
282 : pub read_accesses_immutable: IntCounter,
283 : pub read_hits_immutable: IntCounter,
284 : }
285 :
286 : pub(crate) struct PageCacheMetrics {
287 : map: EnumMap<TaskKind, EnumMap<PageContentKind, PageCacheMetricsForTaskKind>>,
288 : }
289 :
290 86 : static PAGE_CACHE_READ_HITS: Lazy<IntCounterVec> = Lazy::new(|| {
291 : register_int_counter_vec!(
292 : "pageserver_page_cache_read_hits_total",
293 : "Number of read accesses to the page cache that hit",
294 : &["task_kind", "key_kind", "content_kind", "hit_kind"]
295 : )
296 86 : .expect("failed to define a metric")
297 86 : });
298 :
299 86 : static PAGE_CACHE_READ_ACCESSES: Lazy<IntCounterVec> = Lazy::new(|| {
300 : register_int_counter_vec!(
301 : "pageserver_page_cache_read_accesses_total",
302 : "Number of read accesses to the page cache",
303 : &["task_kind", "key_kind", "content_kind"]
304 : )
305 86 : .expect("failed to define a metric")
306 86 : });
307 :
308 86 : pub(crate) static PAGE_CACHE: Lazy<PageCacheMetrics> = Lazy::new(|| PageCacheMetrics {
309 2580 : map: EnumMap::from_array(std::array::from_fn(|task_kind| {
310 2580 : let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind);
311 2580 : let task_kind: &'static str = task_kind.into();
312 15480 : EnumMap::from_array(std::array::from_fn(|content_kind| {
313 15480 : let content_kind = <PageContentKind as enum_map::Enum>::from_usize(content_kind);
314 15480 : let content_kind: &'static str = content_kind.into();
315 15480 : PageCacheMetricsForTaskKind {
316 15480 : read_accesses_immutable: {
317 15480 : PAGE_CACHE_READ_ACCESSES
318 15480 : .get_metric_with_label_values(&[task_kind, "immutable", content_kind])
319 15480 : .unwrap()
320 15480 : },
321 15480 :
322 15480 : read_hits_immutable: {
323 15480 : PAGE_CACHE_READ_HITS
324 15480 : .get_metric_with_label_values(&[task_kind, "immutable", content_kind, "-"])
325 15480 : .unwrap()
326 15480 : },
327 15480 : }
328 15480 : }))
329 2580 : })),
330 86 : });
331 :
332 : impl PageCacheMetrics {
333 5669576 : pub(crate) fn for_ctx(&self, ctx: &RequestContext) -> &PageCacheMetricsForTaskKind {
334 5669576 : &self.map[ctx.task_kind()][ctx.page_content_kind()]
335 5669576 : }
336 : }
337 :
338 : pub(crate) struct PageCacheSizeMetrics {
339 : pub max_bytes: UIntGauge,
340 :
341 : pub current_bytes_immutable: UIntGauge,
342 : }
343 :
344 86 : static PAGE_CACHE_SIZE_CURRENT_BYTES: Lazy<UIntGaugeVec> = Lazy::new(|| {
345 : register_uint_gauge_vec!(
346 : "pageserver_page_cache_size_current_bytes",
347 : "Current size of the page cache in bytes, by key kind",
348 : &["key_kind"]
349 : )
350 86 : .expect("failed to define a metric")
351 86 : });
352 :
353 : pub(crate) static PAGE_CACHE_SIZE: Lazy<PageCacheSizeMetrics> =
354 86 : Lazy::new(|| PageCacheSizeMetrics {
355 : max_bytes: {
356 : register_uint_gauge!(
357 : "pageserver_page_cache_size_max_bytes",
358 : "Maximum size of the page cache in bytes"
359 : )
360 86 : .expect("failed to define a metric")
361 86 : },
362 86 : current_bytes_immutable: {
363 86 : PAGE_CACHE_SIZE_CURRENT_BYTES
364 86 : .get_metric_with_label_values(&["immutable"])
365 86 : .unwrap()
366 86 : },
367 86 : });
368 :
369 : pub(crate) mod page_cache_eviction_metrics {
370 : use std::num::NonZeroUsize;
371 :
372 : use metrics::{register_int_counter_vec, IntCounter, IntCounterVec};
373 : use once_cell::sync::Lazy;
374 :
375 : #[derive(Clone, Copy)]
376 : pub(crate) enum Outcome {
377 : FoundSlotUnused { iters: NonZeroUsize },
378 : FoundSlotEvicted { iters: NonZeroUsize },
379 : ItersExceeded { iters: NonZeroUsize },
380 : }
381 :
382 86 : static ITERS_TOTAL_VEC: Lazy<IntCounterVec> = Lazy::new(|| {
383 : register_int_counter_vec!(
384 : "pageserver_page_cache_find_victim_iters_total",
385 : "Counter for the number of iterations in the find_victim loop",
386 : &["outcome"],
387 : )
388 86 : .expect("failed to define a metric")
389 86 : });
390 :
391 86 : static CALLS_VEC: Lazy<IntCounterVec> = Lazy::new(|| {
392 : register_int_counter_vec!(
393 : "pageserver_page_cache_find_victim_calls",
394 : "Incremented at the end of each find_victim() call.\
395 : Filter by outcome to get e.g., eviction rate.",
396 : &["outcome"]
397 : )
398 86 : .unwrap()
399 86 : });
400 :
401 61926 : pub(crate) fn observe(outcome: Outcome) {
402 61926 : macro_rules! dry {
403 61926 : ($label:literal, $iters:expr) => {{
404 61926 : static LABEL: &'static str = $label;
405 61926 : static ITERS_TOTAL: Lazy<IntCounter> =
406 61926 : Lazy::new(|| ITERS_TOTAL_VEC.with_label_values(&[LABEL]));
407 61926 : static CALLS: Lazy<IntCounter> =
408 61926 : Lazy::new(|| CALLS_VEC.with_label_values(&[LABEL]));
409 61926 : ITERS_TOTAL.inc_by(($iters.get()) as u64);
410 61926 : CALLS.inc();
411 61926 : }};
412 61926 : }
413 61926 : match outcome {
414 1906 : Outcome::FoundSlotUnused { iters } => dry!("found_empty", iters),
415 60020 : Outcome::FoundSlotEvicted { iters } => {
416 60020 : dry!("found_evicted", iters)
417 : }
418 0 : Outcome::ItersExceeded { iters } => {
419 0 : dry!("err_iters_exceeded", iters);
420 0 : super::page_cache_errors_inc(super::PageCacheErrorKind::EvictIterLimit);
421 0 : }
422 : }
423 61926 : }
424 : }
425 :
426 0 : static PAGE_CACHE_ERRORS: Lazy<IntCounterVec> = Lazy::new(|| {
427 : register_int_counter_vec!(
428 : "page_cache_errors_total",
429 : "Number of timeouts while acquiring a pinned slot in the page cache",
430 : &["error_kind"]
431 : )
432 0 : .expect("failed to define a metric")
433 0 : });
434 :
435 0 : #[derive(IntoStaticStr)]
436 : #[strum(serialize_all = "kebab_case")]
437 : pub(crate) enum PageCacheErrorKind {
438 : AcquirePinnedSlotTimeout,
439 : EvictIterLimit,
440 : }
441 :
442 0 : pub(crate) fn page_cache_errors_inc(error_kind: PageCacheErrorKind) {
443 0 : PAGE_CACHE_ERRORS
444 0 : .get_metric_with_label_values(&[error_kind.into()])
445 0 : .unwrap()
446 0 : .inc();
447 0 : }
448 :
449 18 : pub(crate) static WAIT_LSN_TIME: Lazy<Histogram> = Lazy::new(|| {
450 : register_histogram!(
451 : "pageserver_wait_lsn_seconds",
452 : "Time spent waiting for WAL to arrive",
453 : CRITICAL_OP_BUCKETS.into(),
454 : )
455 18 : .expect("failed to define a metric")
456 18 : });
457 :
458 154 : static LAST_RECORD_LSN: Lazy<IntGaugeVec> = Lazy::new(|| {
459 : register_int_gauge_vec!(
460 : "pageserver_last_record_lsn",
461 : "Last record LSN grouped by timeline",
462 : &["tenant_id", "shard_id", "timeline_id"]
463 : )
464 154 : .expect("failed to define a metric")
465 154 : });
466 :
467 154 : static PITR_HISTORY_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
468 : register_uint_gauge_vec!(
469 : "pageserver_pitr_history_size",
470 : "Data written since PITR cutoff on this timeline",
471 : &["tenant_id", "shard_id", "timeline_id"]
472 : )
473 154 : .expect("failed to define a metric")
474 154 : });
475 :
476 1624 : #[derive(strum_macros::EnumString, strum_macros::Display, strum_macros::IntoStaticStr)]
477 : #[strum(serialize_all = "kebab_case")]
478 : pub(crate) enum MetricLayerKind {
479 : Delta,
480 : Image,
481 : }
482 :
483 154 : static TIMELINE_LAYER_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
484 : register_uint_gauge_vec!(
485 : "pageserver_layer_bytes",
486 : "Sum of layer physical sizes in bytes",
487 : &["tenant_id", "shard_id", "timeline_id", "kind"]
488 : )
489 154 : .expect("failed to define a metric")
490 154 : });
491 :
492 154 : static TIMELINE_LAYER_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
493 : register_uint_gauge_vec!(
494 : "pageserver_layer_count",
495 : "Number of layers that exist",
496 : &["tenant_id", "shard_id", "timeline_id", "kind"]
497 : )
498 154 : .expect("failed to define a metric")
499 154 : });
500 :
501 154 : static TIMELINE_ARCHIVE_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
502 : register_uint_gauge_vec!(
503 : "pageserver_archive_size",
504 : "Timeline's logical size if it is considered eligible for archival (outside PITR window), else zero",
505 : &["tenant_id", "shard_id", "timeline_id"]
506 : )
507 154 : .expect("failed to define a metric")
508 154 : });
509 :
510 154 : static STANDBY_HORIZON: Lazy<IntGaugeVec> = Lazy::new(|| {
511 : register_int_gauge_vec!(
512 : "pageserver_standby_horizon",
513 : "Standby apply LSN for which GC is hold off, by timeline.",
514 : &["tenant_id", "shard_id", "timeline_id"]
515 : )
516 154 : .expect("failed to define a metric")
517 154 : });
518 :
519 154 : static RESIDENT_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
520 : register_uint_gauge_vec!(
521 : "pageserver_resident_physical_size",
522 : "The size of the layer files present in the pageserver's filesystem, for attached locations.",
523 : &["tenant_id", "shard_id", "timeline_id"]
524 : )
525 154 : .expect("failed to define a metric")
526 154 : });
527 :
528 154 : static VISIBLE_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
529 : register_uint_gauge_vec!(
530 : "pageserver_visible_physical_size",
531 : "The size of the layer files present in the pageserver's filesystem.",
532 : &["tenant_id", "shard_id", "timeline_id"]
533 : )
534 154 : .expect("failed to define a metric")
535 154 : });
536 :
537 150 : pub(crate) static RESIDENT_PHYSICAL_SIZE_GLOBAL: Lazy<UIntGauge> = Lazy::new(|| {
538 : register_uint_gauge!(
539 : "pageserver_resident_physical_size_global",
540 : "Like `pageserver_resident_physical_size`, but without tenant/timeline dimensions."
541 : )
542 150 : .expect("failed to define a metric")
543 150 : });
544 :
545 154 : static REMOTE_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
546 : register_uint_gauge_vec!(
547 : "pageserver_remote_physical_size",
548 : "The size of the layer files present in the remote storage that are listed in the remote index_part.json.",
549 : // Corollary: If any files are missing from the index part, they won't be included here.
550 : &["tenant_id", "shard_id", "timeline_id"]
551 : )
552 154 : .expect("failed to define a metric")
553 154 : });
554 :
555 154 : static REMOTE_PHYSICAL_SIZE_GLOBAL: Lazy<UIntGauge> = Lazy::new(|| {
556 : register_uint_gauge!(
557 : "pageserver_remote_physical_size_global",
558 : "Like `pageserver_remote_physical_size`, but without tenant/timeline dimensions."
559 : )
560 154 : .expect("failed to define a metric")
561 154 : });
562 :
563 4 : pub(crate) static REMOTE_ONDEMAND_DOWNLOADED_LAYERS: Lazy<IntCounter> = Lazy::new(|| {
564 : register_int_counter!(
565 : "pageserver_remote_ondemand_downloaded_layers_total",
566 : "Total on-demand downloaded layers"
567 : )
568 4 : .unwrap()
569 4 : });
570 :
571 4 : pub(crate) static REMOTE_ONDEMAND_DOWNLOADED_BYTES: Lazy<IntCounter> = Lazy::new(|| {
572 : register_int_counter!(
573 : "pageserver_remote_ondemand_downloaded_bytes_total",
574 : "Total bytes of layers on-demand downloaded",
575 : )
576 4 : .unwrap()
577 4 : });
578 :
579 154 : static CURRENT_LOGICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
580 : register_uint_gauge_vec!(
581 : "pageserver_current_logical_size",
582 : "Current logical size grouped by timeline",
583 : &["tenant_id", "shard_id", "timeline_id"]
584 : )
585 154 : .expect("failed to define current logical size metric")
586 154 : });
587 :
588 154 : static AUX_FILE_SIZE: Lazy<IntGaugeVec> = Lazy::new(|| {
589 : register_int_gauge_vec!(
590 : "pageserver_aux_file_estimated_size",
591 : "The size of all aux files for a timeline in aux file v2 store.",
592 : &["tenant_id", "shard_id", "timeline_id"]
593 : )
594 154 : .expect("failed to define a metric")
595 154 : });
596 :
597 154 : static VALID_LSN_LEASE_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
598 : register_uint_gauge_vec!(
599 : "pageserver_valid_lsn_lease_count",
600 : "The number of valid leases after refreshing gc info.",
601 : &["tenant_id", "shard_id", "timeline_id"],
602 : )
603 154 : .expect("failed to define a metric")
604 154 : });
605 :
606 0 : pub(crate) static CIRCUIT_BREAKERS_BROKEN: Lazy<IntCounter> = Lazy::new(|| {
607 : register_int_counter!(
608 : "pageserver_circuit_breaker_broken",
609 : "How many times a circuit breaker has broken"
610 : )
611 0 : .expect("failed to define a metric")
612 0 : });
613 :
614 0 : pub(crate) static CIRCUIT_BREAKERS_UNBROKEN: Lazy<IntCounter> = Lazy::new(|| {
615 : register_int_counter!(
616 : "pageserver_circuit_breaker_unbroken",
617 : "How many times a circuit breaker has been un-broken (recovered)"
618 : )
619 0 : .expect("failed to define a metric")
620 0 : });
621 :
622 146 : pub(crate) static COMPRESSION_IMAGE_INPUT_BYTES: Lazy<IntCounter> = Lazy::new(|| {
623 : register_int_counter!(
624 : "pageserver_compression_image_in_bytes_total",
625 : "Size of data written into image layers before compression"
626 : )
627 146 : .expect("failed to define a metric")
628 146 : });
629 :
630 146 : pub(crate) static COMPRESSION_IMAGE_INPUT_BYTES_CONSIDERED: Lazy<IntCounter> = Lazy::new(|| {
631 : register_int_counter!(
632 : "pageserver_compression_image_in_bytes_considered",
633 : "Size of potentially compressible data written into image layers before compression"
634 : )
635 146 : .expect("failed to define a metric")
636 146 : });
637 :
638 146 : pub(crate) static COMPRESSION_IMAGE_INPUT_BYTES_CHOSEN: Lazy<IntCounter> = Lazy::new(|| {
639 : register_int_counter!(
640 : "pageserver_compression_image_in_bytes_chosen",
641 : "Size of data whose compressed form was written into image layers"
642 : )
643 146 : .expect("failed to define a metric")
644 146 : });
645 :
646 146 : pub(crate) static COMPRESSION_IMAGE_OUTPUT_BYTES: Lazy<IntCounter> = Lazy::new(|| {
647 : register_int_counter!(
648 : "pageserver_compression_image_out_bytes_total",
649 : "Size of compressed image layer written"
650 : )
651 146 : .expect("failed to define a metric")
652 146 : });
653 :
654 : pub(crate) mod initial_logical_size {
655 : use metrics::{register_int_counter, register_int_counter_vec, IntCounter, IntCounterVec};
656 : use once_cell::sync::Lazy;
657 :
658 : pub(crate) struct StartCalculation(IntCounterVec);
659 154 : pub(crate) static START_CALCULATION: Lazy<StartCalculation> = Lazy::new(|| {
660 154 : StartCalculation(
661 154 : register_int_counter_vec!(
662 : "pageserver_initial_logical_size_start_calculation",
663 : "Incremented each time we start an initial logical size calculation attempt. \
664 : The `circumstances` label provides some additional details.",
665 : &["attempt", "circumstances"]
666 154 : )
667 154 : .unwrap(),
668 154 : )
669 154 : });
670 :
671 : struct DropCalculation {
672 : first: IntCounter,
673 : retry: IntCounter,
674 : }
675 :
676 154 : static DROP_CALCULATION: Lazy<DropCalculation> = Lazy::new(|| {
677 154 : let vec = register_int_counter_vec!(
678 : "pageserver_initial_logical_size_drop_calculation",
679 : "Incremented each time we abort a started size calculation attmpt.",
680 : &["attempt"]
681 : )
682 154 : .unwrap();
683 154 : DropCalculation {
684 154 : first: vec.with_label_values(&["first"]),
685 154 : retry: vec.with_label_values(&["retry"]),
686 154 : }
687 154 : });
688 :
689 : pub(crate) struct Calculated {
690 : pub(crate) births: IntCounter,
691 : pub(crate) deaths: IntCounter,
692 : }
693 :
694 154 : pub(crate) static CALCULATED: Lazy<Calculated> = Lazy::new(|| Calculated {
695 : births: register_int_counter!(
696 : "pageserver_initial_logical_size_finish_calculation",
697 : "Incremented every time we finish calculation of initial logical size.\
698 : If everything is working well, this should happen at most once per Timeline object."
699 : )
700 154 : .unwrap(),
701 : deaths: register_int_counter!(
702 : "pageserver_initial_logical_size_drop_finished_calculation",
703 : "Incremented when we drop a finished initial logical size calculation result.\
704 : Mainly useful to turn pageserver_initial_logical_size_finish_calculation into a gauge."
705 : )
706 154 : .unwrap(),
707 154 : });
708 :
709 : pub(crate) struct OngoingCalculationGuard {
710 : inc_drop_calculation: Option<IntCounter>,
711 : }
712 :
713 166 : #[derive(strum_macros::IntoStaticStr)]
714 : pub(crate) enum StartCircumstances {
715 : EmptyInitial,
716 : SkippedConcurrencyLimiter,
717 : AfterBackgroundTasksRateLimit,
718 : }
719 :
720 : impl StartCalculation {
721 166 : pub(crate) fn first(&self, circumstances: StartCircumstances) -> OngoingCalculationGuard {
722 166 : let circumstances_label: &'static str = circumstances.into();
723 166 : self.0
724 166 : .with_label_values(&["first", circumstances_label])
725 166 : .inc();
726 166 : OngoingCalculationGuard {
727 166 : inc_drop_calculation: Some(DROP_CALCULATION.first.clone()),
728 166 : }
729 166 : }
730 0 : pub(crate) fn retry(&self, circumstances: StartCircumstances) -> OngoingCalculationGuard {
731 0 : let circumstances_label: &'static str = circumstances.into();
732 0 : self.0
733 0 : .with_label_values(&["retry", circumstances_label])
734 0 : .inc();
735 0 : OngoingCalculationGuard {
736 0 : inc_drop_calculation: Some(DROP_CALCULATION.retry.clone()),
737 0 : }
738 0 : }
739 : }
740 :
741 : impl Drop for OngoingCalculationGuard {
742 166 : fn drop(&mut self) {
743 166 : if let Some(counter) = self.inc_drop_calculation.take() {
744 0 : counter.inc();
745 166 : }
746 166 : }
747 : }
748 :
749 : impl OngoingCalculationGuard {
750 166 : pub(crate) fn calculation_result_saved(mut self) -> FinishedCalculationGuard {
751 166 : drop(self.inc_drop_calculation.take());
752 166 : CALCULATED.births.inc();
753 166 : FinishedCalculationGuard {
754 166 : inc_on_drop: CALCULATED.deaths.clone(),
755 166 : }
756 166 : }
757 : }
758 :
759 : pub(crate) struct FinishedCalculationGuard {
760 : inc_on_drop: IntCounter,
761 : }
762 :
763 : impl Drop for FinishedCalculationGuard {
764 6 : fn drop(&mut self) {
765 6 : self.inc_on_drop.inc();
766 6 : }
767 : }
768 :
769 : // context: https://github.com/neondatabase/neon/issues/5963
770 : pub(crate) static TIMELINES_WHERE_WALRECEIVER_GOT_APPROXIMATE_SIZE: Lazy<IntCounter> =
771 0 : Lazy::new(|| {
772 : register_int_counter!(
773 : "pageserver_initial_logical_size_timelines_where_walreceiver_got_approximate_size",
774 : "Counter for the following event: walreceiver calls\
775 : Timeline::get_current_logical_size() and it returns `Approximate` for the first time."
776 : )
777 0 : .unwrap()
778 0 : });
779 : }
780 :
781 0 : static DIRECTORY_ENTRIES_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
782 : register_uint_gauge_vec!(
783 : "pageserver_directory_entries_count",
784 : "Sum of the entries in pageserver-stored directory listings",
785 : &["tenant_id", "shard_id", "timeline_id"]
786 : )
787 0 : .expect("failed to define a metric")
788 0 : });
789 :
790 156 : pub(crate) static TENANT_STATE_METRIC: Lazy<UIntGaugeVec> = Lazy::new(|| {
791 : register_uint_gauge_vec!(
792 : "pageserver_tenant_states_count",
793 : "Count of tenants per state",
794 : &["state"]
795 : )
796 156 : .expect("Failed to register pageserver_tenant_states_count metric")
797 156 : });
798 :
799 : /// A set of broken tenants.
800 : ///
801 : /// These are expected to be so rare that a set is fine. Set as in a new timeseries per each broken
802 : /// tenant.
803 12 : pub(crate) static BROKEN_TENANTS_SET: Lazy<UIntGaugeVec> = Lazy::new(|| {
804 : register_uint_gauge_vec!(
805 : "pageserver_broken_tenants_count",
806 : "Set of broken tenants",
807 : &["tenant_id", "shard_id"]
808 : )
809 12 : .expect("Failed to register pageserver_tenant_states_count metric")
810 12 : });
811 :
812 6 : pub(crate) static TENANT_SYNTHETIC_SIZE_METRIC: Lazy<UIntGaugeVec> = Lazy::new(|| {
813 : register_uint_gauge_vec!(
814 : "pageserver_tenant_synthetic_cached_size_bytes",
815 : "Synthetic size of each tenant in bytes",
816 : &["tenant_id"]
817 : )
818 6 : .expect("Failed to register pageserver_tenant_synthetic_cached_size_bytes metric")
819 6 : });
820 :
821 0 : pub(crate) static EVICTION_ITERATION_DURATION: Lazy<HistogramVec> = Lazy::new(|| {
822 : register_histogram_vec!(
823 : "pageserver_eviction_iteration_duration_seconds_global",
824 : "Time spent on a single eviction iteration",
825 : &["period_secs", "threshold_secs"],
826 : STORAGE_OP_BUCKETS.into(),
827 : )
828 0 : .expect("failed to define a metric")
829 0 : });
830 :
831 154 : static EVICTIONS: Lazy<IntCounterVec> = Lazy::new(|| {
832 : register_int_counter_vec!(
833 : "pageserver_evictions",
834 : "Number of layers evicted from the pageserver",
835 : &["tenant_id", "shard_id", "timeline_id"]
836 : )
837 154 : .expect("failed to define a metric")
838 154 : });
839 :
840 154 : static EVICTIONS_WITH_LOW_RESIDENCE_DURATION: Lazy<IntCounterVec> = Lazy::new(|| {
841 : register_int_counter_vec!(
842 : "pageserver_evictions_with_low_residence_duration",
843 : "If a layer is evicted that was resident for less than `low_threshold`, it is counted to this counter. \
844 : Residence duration is determined using the `residence_duration_data_source`.",
845 : &["tenant_id", "shard_id", "timeline_id", "residence_duration_data_source", "low_threshold_secs"]
846 : )
847 154 : .expect("failed to define a metric")
848 154 : });
849 :
850 0 : pub(crate) static UNEXPECTED_ONDEMAND_DOWNLOADS: Lazy<IntCounter> = Lazy::new(|| {
851 : register_int_counter!(
852 : "pageserver_unexpected_ondemand_downloads_count",
853 : "Number of unexpected on-demand downloads. \
854 : We log more context for each increment, so, forgo any labels in this metric.",
855 : )
856 0 : .expect("failed to define a metric")
857 0 : });
858 :
859 : /// How long did we take to start up? Broken down by labels to describe
860 : /// different phases of startup.
861 0 : pub static STARTUP_DURATION: Lazy<GaugeVec> = Lazy::new(|| {
862 : register_gauge_vec!(
863 : "pageserver_startup_duration_seconds",
864 : "Time taken by phases of pageserver startup, in seconds",
865 : &["phase"]
866 : )
867 0 : .expect("Failed to register pageserver_startup_duration_seconds metric")
868 0 : });
869 :
870 0 : pub static STARTUP_IS_LOADING: Lazy<UIntGauge> = Lazy::new(|| {
871 : register_uint_gauge!(
872 : "pageserver_startup_is_loading",
873 : "1 while in initial startup load of tenants, 0 at other times"
874 : )
875 0 : .expect("Failed to register pageserver_startup_is_loading")
876 0 : });
877 :
878 150 : pub(crate) static TIMELINE_EPHEMERAL_BYTES: Lazy<UIntGauge> = Lazy::new(|| {
879 : register_uint_gauge!(
880 : "pageserver_timeline_ephemeral_bytes",
881 : "Total number of bytes in ephemeral layers, summed for all timelines. Approximate, lazily updated."
882 : )
883 150 : .expect("Failed to register metric")
884 150 : });
885 :
886 : /// Metrics related to the lifecycle of a [`crate::tenant::Tenant`] object: things
887 : /// like how long it took to load.
888 : ///
889 : /// Note that these are process-global metrics, _not_ per-tenant metrics. Per-tenant
890 : /// metrics are rather expensive, and usually fine grained stuff makes more sense
891 : /// at a timeline level than tenant level.
892 : pub(crate) struct TenantMetrics {
893 : /// How long did tenants take to go from construction to active state?
894 : pub(crate) activation: Histogram,
895 : pub(crate) preload: Histogram,
896 : pub(crate) attach: Histogram,
897 :
898 : /// How many tenants are included in the initial startup of the pagesrever?
899 : pub(crate) startup_scheduled: IntCounter,
900 : pub(crate) startup_complete: IntCounter,
901 : }
902 :
903 0 : pub(crate) static TENANT: Lazy<TenantMetrics> = Lazy::new(|| {
904 0 : TenantMetrics {
905 0 : activation: register_histogram!(
906 : "pageserver_tenant_activation_seconds",
907 : "Time taken by tenants to activate, in seconds",
908 : CRITICAL_OP_BUCKETS.into()
909 0 : )
910 0 : .expect("Failed to register metric"),
911 0 : preload: register_histogram!(
912 : "pageserver_tenant_preload_seconds",
913 : "Time taken by tenants to load remote metadata on startup/attach, in seconds",
914 : CRITICAL_OP_BUCKETS.into()
915 0 : )
916 0 : .expect("Failed to register metric"),
917 0 : attach: register_histogram!(
918 : "pageserver_tenant_attach_seconds",
919 : "Time taken by tenants to intialize, after remote metadata is already loaded",
920 : CRITICAL_OP_BUCKETS.into()
921 0 : )
922 0 : .expect("Failed to register metric"),
923 0 : startup_scheduled: register_int_counter!(
924 : "pageserver_tenant_startup_scheduled",
925 : "Number of tenants included in pageserver startup (doesn't count tenants attached later)"
926 0 : ).expect("Failed to register metric"),
927 0 : startup_complete: register_int_counter!(
928 : "pageserver_tenant_startup_complete",
929 : "Number of tenants that have completed warm-up, or activated on-demand during initial startup: \
930 : should eventually reach `pageserver_tenant_startup_scheduled_total`. Does not include broken \
931 : tenants: such cases will lead to this metric never reaching the scheduled count."
932 0 : ).expect("Failed to register metric"),
933 0 : }
934 0 : });
935 :
936 : /// Each `Timeline`'s [`EVICTIONS_WITH_LOW_RESIDENCE_DURATION`] metric.
937 : #[derive(Debug)]
938 : pub(crate) struct EvictionsWithLowResidenceDuration {
939 : data_source: &'static str,
940 : threshold: Duration,
941 : counter: Option<IntCounter>,
942 : }
943 :
944 : pub(crate) struct EvictionsWithLowResidenceDurationBuilder {
945 : data_source: &'static str,
946 : threshold: Duration,
947 : }
948 :
949 : impl EvictionsWithLowResidenceDurationBuilder {
950 398 : pub fn new(data_source: &'static str, threshold: Duration) -> Self {
951 398 : Self {
952 398 : data_source,
953 398 : threshold,
954 398 : }
955 398 : }
956 :
957 398 : fn build(
958 398 : &self,
959 398 : tenant_id: &str,
960 398 : shard_id: &str,
961 398 : timeline_id: &str,
962 398 : ) -> EvictionsWithLowResidenceDuration {
963 398 : let counter = EVICTIONS_WITH_LOW_RESIDENCE_DURATION
964 398 : .get_metric_with_label_values(&[
965 398 : tenant_id,
966 398 : shard_id,
967 398 : timeline_id,
968 398 : self.data_source,
969 398 : &EvictionsWithLowResidenceDuration::threshold_label_value(self.threshold),
970 398 : ])
971 398 : .unwrap();
972 398 : EvictionsWithLowResidenceDuration {
973 398 : data_source: self.data_source,
974 398 : threshold: self.threshold,
975 398 : counter: Some(counter),
976 398 : }
977 398 : }
978 : }
979 :
980 : impl EvictionsWithLowResidenceDuration {
981 406 : fn threshold_label_value(threshold: Duration) -> String {
982 406 : format!("{}", threshold.as_secs())
983 406 : }
984 :
985 4 : pub fn observe(&self, observed_value: Duration) {
986 4 : if observed_value < self.threshold {
987 4 : self.counter
988 4 : .as_ref()
989 4 : .expect("nobody calls this function after `remove_from_vec`")
990 4 : .inc();
991 4 : }
992 4 : }
993 :
994 8 : pub fn change_threshold(
995 8 : &mut self,
996 8 : tenant_id: &str,
997 8 : shard_id: &str,
998 8 : timeline_id: &str,
999 8 : new_threshold: Duration,
1000 8 : ) {
1001 8 : if new_threshold == self.threshold {
1002 8 : return;
1003 0 : }
1004 0 : let mut with_new = EvictionsWithLowResidenceDurationBuilder::new(
1005 0 : self.data_source,
1006 0 : new_threshold,
1007 0 : )
1008 0 : .build(tenant_id, shard_id, timeline_id);
1009 0 : std::mem::swap(self, &mut with_new);
1010 0 : with_new.remove(tenant_id, shard_id, timeline_id);
1011 8 : }
1012 :
1013 : // This could be a `Drop` impl, but, we need the `tenant_id` and `timeline_id`.
1014 8 : fn remove(&mut self, tenant_id: &str, shard_id: &str, timeline_id: &str) {
1015 8 : let Some(_counter) = self.counter.take() else {
1016 0 : return;
1017 : };
1018 :
1019 8 : let threshold = Self::threshold_label_value(self.threshold);
1020 8 :
1021 8 : let removed = EVICTIONS_WITH_LOW_RESIDENCE_DURATION.remove_label_values(&[
1022 8 : tenant_id,
1023 8 : shard_id,
1024 8 : timeline_id,
1025 8 : self.data_source,
1026 8 : &threshold,
1027 8 : ]);
1028 8 :
1029 8 : match removed {
1030 0 : Err(e) => {
1031 0 : // this has been hit in staging as
1032 0 : // <https://neondatabase.sentry.io/issues/4142396994/>, but we don't know how.
1033 0 : // because we can be in the drop path already, don't risk:
1034 0 : // - "double-panic => illegal instruction" or
1035 0 : // - future "drop panick => abort"
1036 0 : //
1037 0 : // so just nag: (the error has the labels)
1038 0 : tracing::warn!("failed to remove EvictionsWithLowResidenceDuration, it was already removed? {e:#?}");
1039 : }
1040 : Ok(()) => {
1041 : // to help identify cases where we double-remove the same values, let's log all
1042 : // deletions?
1043 8 : tracing::info!("removed EvictionsWithLowResidenceDuration with {tenant_id}, {timeline_id}, {}, {threshold}", self.data_source);
1044 : }
1045 : }
1046 8 : }
1047 : }
1048 :
1049 : // Metrics collected on disk IO operations
1050 : //
1051 : // Roughly logarithmic scale.
1052 : const STORAGE_IO_TIME_BUCKETS: &[f64] = &[
1053 : 0.000030, // 30 usec
1054 : 0.001000, // 1000 usec
1055 : 0.030, // 30 ms
1056 : 1.000, // 1000 ms
1057 : 30.000, // 30000 ms
1058 : ];
1059 :
1060 : /// VirtualFile fs operation variants.
1061 : ///
1062 : /// Operations:
1063 : /// - open ([`std::fs::OpenOptions::open`])
1064 : /// - close (dropping [`crate::virtual_file::VirtualFile`])
1065 : /// - close-by-replace (close by replacement algorithm)
1066 : /// - read (`read_at`)
1067 : /// - write (`write_at`)
1068 : /// - seek (modify internal position or file length query)
1069 : /// - fsync ([`std::fs::File::sync_all`])
1070 : /// - metadata ([`std::fs::File::metadata`])
1071 : #[derive(
1072 1602 : Debug, Clone, Copy, strum_macros::EnumCount, strum_macros::EnumIter, strum_macros::FromRepr,
1073 : )]
1074 : pub(crate) enum StorageIoOperation {
1075 : Open,
1076 : OpenAfterReplace,
1077 : Close,
1078 : CloseByReplace,
1079 : Read,
1080 : Write,
1081 : Seek,
1082 : Fsync,
1083 : Metadata,
1084 : }
1085 :
1086 : impl StorageIoOperation {
1087 1602 : pub fn as_str(&self) -> &'static str {
1088 1602 : match self {
1089 178 : StorageIoOperation::Open => "open",
1090 178 : StorageIoOperation::OpenAfterReplace => "open-after-replace",
1091 178 : StorageIoOperation::Close => "close",
1092 178 : StorageIoOperation::CloseByReplace => "close-by-replace",
1093 178 : StorageIoOperation::Read => "read",
1094 178 : StorageIoOperation::Write => "write",
1095 178 : StorageIoOperation::Seek => "seek",
1096 178 : StorageIoOperation::Fsync => "fsync",
1097 178 : StorageIoOperation::Metadata => "metadata",
1098 : }
1099 1602 : }
1100 : }
1101 :
1102 : /// Tracks time taken by fs operations near VirtualFile.
1103 : #[derive(Debug)]
1104 : pub(crate) struct StorageIoTime {
1105 : metrics: [Histogram; StorageIoOperation::COUNT],
1106 : }
1107 :
1108 : impl StorageIoTime {
1109 178 : fn new() -> Self {
1110 178 : let storage_io_histogram_vec = register_histogram_vec!(
1111 : "pageserver_io_operations_seconds",
1112 : "Time spent in IO operations",
1113 : &["operation"],
1114 : STORAGE_IO_TIME_BUCKETS.into()
1115 : )
1116 178 : .expect("failed to define a metric");
1117 1602 : let metrics = std::array::from_fn(|i| {
1118 1602 : let op = StorageIoOperation::from_repr(i).unwrap();
1119 1602 : storage_io_histogram_vec
1120 1602 : .get_metric_with_label_values(&[op.as_str()])
1121 1602 : .unwrap()
1122 1602 : });
1123 178 : Self { metrics }
1124 178 : }
1125 :
1126 2011723 : pub(crate) fn get(&self, op: StorageIoOperation) -> &Histogram {
1127 2011723 : &self.metrics[op as usize]
1128 2011723 : }
1129 : }
1130 :
1131 : pub(crate) static STORAGE_IO_TIME_METRIC: Lazy<StorageIoTime> = Lazy::new(StorageIoTime::new);
1132 :
1133 : const STORAGE_IO_SIZE_OPERATIONS: &[&str] = &["read", "write"];
1134 :
1135 : // Needed for the https://neonprod.grafana.net/d/5uK9tHL4k/picking-tenant-for-relocation?orgId=1
1136 176 : pub(crate) static STORAGE_IO_SIZE: Lazy<IntGaugeVec> = Lazy::new(|| {
1137 : register_int_gauge_vec!(
1138 : "pageserver_io_operations_bytes_total",
1139 : "Total amount of bytes read/written in IO operations",
1140 : &["operation", "tenant_id", "shard_id", "timeline_id"]
1141 : )
1142 176 : .expect("failed to define a metric")
1143 176 : });
1144 :
1145 : #[cfg(not(test))]
1146 : pub(crate) mod virtual_file_descriptor_cache {
1147 : use super::*;
1148 :
1149 0 : pub(crate) static SIZE_MAX: Lazy<UIntGauge> = Lazy::new(|| {
1150 : register_uint_gauge!(
1151 : "pageserver_virtual_file_descriptor_cache_size_max",
1152 : "Maximum number of open file descriptors in the cache."
1153 : )
1154 0 : .unwrap()
1155 0 : });
1156 :
1157 : // SIZE_CURRENT: derive it like so:
1158 : // ```
1159 : // sum (pageserver_io_operations_seconds_count{operation=~"^(open|open-after-replace)$")
1160 : // -ignoring(operation)
1161 : // sum(pageserver_io_operations_seconds_count{operation=~"^(close|close-by-replace)$"}
1162 : // ```
1163 : }
1164 :
1165 : #[cfg(not(test))]
1166 : pub(crate) mod virtual_file_io_engine {
1167 : use super::*;
1168 :
1169 0 : pub(crate) static KIND: Lazy<UIntGaugeVec> = Lazy::new(|| {
1170 : register_uint_gauge_vec!(
1171 : "pageserver_virtual_file_io_engine_kind",
1172 : "The configured io engine for VirtualFile",
1173 : &["kind"],
1174 : )
1175 0 : .unwrap()
1176 0 : });
1177 : }
1178 :
1179 : struct GlobalAndPerTimelineHistogramTimer<'a, 'c> {
1180 : global_metric: &'a Histogram,
1181 :
1182 : // Optional because not all op types are tracked per-timeline
1183 : timeline_metric: Option<&'a Histogram>,
1184 :
1185 : ctx: &'c RequestContext,
1186 : start: std::time::Instant,
1187 : op: SmgrQueryType,
1188 : }
1189 :
1190 : impl<'a, 'c> Drop for GlobalAndPerTimelineHistogramTimer<'a, 'c> {
1191 10 : fn drop(&mut self) {
1192 10 : let elapsed = self.start.elapsed();
1193 10 : let ex_throttled = self
1194 10 : .ctx
1195 10 : .micros_spent_throttled
1196 10 : .close_and_checked_sub_from(elapsed);
1197 10 : let ex_throttled = match ex_throttled {
1198 10 : Ok(res) => res,
1199 0 : Err(error) => {
1200 0 : use utils::rate_limit::RateLimit;
1201 0 : static LOGGED: Lazy<Mutex<enum_map::EnumMap<SmgrQueryType, RateLimit>>> =
1202 0 : Lazy::new(|| {
1203 0 : Mutex::new(enum_map::EnumMap::from_array(std::array::from_fn(|_| {
1204 0 : RateLimit::new(Duration::from_secs(10))
1205 0 : })))
1206 0 : });
1207 0 : let mut guard = LOGGED.lock().unwrap();
1208 0 : let rate_limit = &mut guard[self.op];
1209 0 : rate_limit.call(|| {
1210 0 : warn!(op=?self.op, error, "error deducting time spent throttled; this message is logged at a global rate limit");
1211 0 : });
1212 0 : elapsed
1213 : }
1214 : };
1215 10 : self.global_metric.observe(ex_throttled.as_secs_f64());
1216 10 : if let Some(timeline_metric) = self.timeline_metric {
1217 2 : timeline_metric.observe(ex_throttled.as_secs_f64());
1218 8 : }
1219 10 : }
1220 : }
1221 :
1222 : #[derive(
1223 : Debug,
1224 : Clone,
1225 : Copy,
1226 2466 : IntoStaticStr,
1227 : strum_macros::EnumCount,
1228 24 : strum_macros::EnumIter,
1229 2040 : strum_macros::FromRepr,
1230 : enum_map::Enum,
1231 : )]
1232 : #[strum(serialize_all = "snake_case")]
1233 : pub enum SmgrQueryType {
1234 : GetRelExists,
1235 : GetRelSize,
1236 : GetPageAtLsn,
1237 : GetDbSize,
1238 : GetSlruSegment,
1239 : }
1240 :
1241 : #[derive(Debug)]
1242 : pub(crate) struct SmgrQueryTimePerTimeline {
1243 : global_metrics: [Histogram; SmgrQueryType::COUNT],
1244 : per_timeline_getpage: Histogram,
1245 : }
1246 :
1247 156 : static SMGR_QUERY_TIME_PER_TENANT_TIMELINE: Lazy<HistogramVec> = Lazy::new(|| {
1248 : register_histogram_vec!(
1249 : "pageserver_smgr_query_seconds",
1250 : "Time spent on smgr query handling, aggegated by query type and tenant/timeline.",
1251 : &["smgr_query_type", "tenant_id", "shard_id", "timeline_id"],
1252 : CRITICAL_OP_BUCKETS.into(),
1253 : )
1254 156 : .expect("failed to define a metric")
1255 156 : });
1256 :
1257 156 : static SMGR_QUERY_TIME_GLOBAL_BUCKETS: Lazy<Vec<f64>> = Lazy::new(|| {
1258 156 : [
1259 156 : 1,
1260 156 : 10,
1261 156 : 20,
1262 156 : 40,
1263 156 : 60,
1264 156 : 80,
1265 156 : 100,
1266 156 : 200,
1267 156 : 300,
1268 156 : 400,
1269 156 : 500,
1270 156 : 600,
1271 156 : 700,
1272 156 : 800,
1273 156 : 900,
1274 156 : 1_000, // 1ms
1275 156 : 2_000,
1276 156 : 4_000,
1277 156 : 6_000,
1278 156 : 8_000,
1279 156 : 10_000, // 10ms
1280 156 : 20_000,
1281 156 : 40_000,
1282 156 : 60_000,
1283 156 : 80_000,
1284 156 : 100_000,
1285 156 : 200_000,
1286 156 : 400_000,
1287 156 : 600_000,
1288 156 : 800_000,
1289 156 : 1_000_000, // 1s
1290 156 : 2_000_000,
1291 156 : 4_000_000,
1292 156 : 6_000_000,
1293 156 : 8_000_000,
1294 156 : 10_000_000, // 10s
1295 156 : 20_000_000,
1296 156 : 50_000_000,
1297 156 : 100_000_000,
1298 156 : 200_000_000,
1299 156 : 1_000_000_000, // 1000s
1300 156 : ]
1301 156 : .into_iter()
1302 156 : .map(Duration::from_micros)
1303 6396 : .map(|d| d.as_secs_f64())
1304 156 : .collect()
1305 156 : });
1306 :
1307 156 : static SMGR_QUERY_TIME_GLOBAL: Lazy<HistogramVec> = Lazy::new(|| {
1308 : register_histogram_vec!(
1309 : "pageserver_smgr_query_seconds_global",
1310 : "Time spent on smgr query handling, aggregated by query type.",
1311 : &["smgr_query_type"],
1312 : SMGR_QUERY_TIME_GLOBAL_BUCKETS.clone(),
1313 : )
1314 156 : .expect("failed to define a metric")
1315 156 : });
1316 :
1317 : impl SmgrQueryTimePerTimeline {
1318 408 : pub(crate) fn new(tenant_shard_id: &TenantShardId, timeline_id: &TimelineId) -> Self {
1319 408 : let tenant_id = tenant_shard_id.tenant_id.to_string();
1320 408 : let shard_slug = format!("{}", tenant_shard_id.shard_slug());
1321 408 : let timeline_id = timeline_id.to_string();
1322 2040 : let global_metrics = std::array::from_fn(|i| {
1323 2040 : let op = SmgrQueryType::from_repr(i).unwrap();
1324 2040 : SMGR_QUERY_TIME_GLOBAL
1325 2040 : .get_metric_with_label_values(&[op.into()])
1326 2040 : .unwrap()
1327 2040 : });
1328 408 :
1329 408 : let per_timeline_getpage = SMGR_QUERY_TIME_PER_TENANT_TIMELINE
1330 408 : .get_metric_with_label_values(&[
1331 408 : SmgrQueryType::GetPageAtLsn.into(),
1332 408 : &tenant_id,
1333 408 : &shard_slug,
1334 408 : &timeline_id,
1335 408 : ])
1336 408 : .unwrap();
1337 408 : Self {
1338 408 : global_metrics,
1339 408 : per_timeline_getpage,
1340 408 : }
1341 408 : }
1342 10 : pub(crate) fn start_timer<'c: 'a, 'a>(
1343 10 : &'a self,
1344 10 : op: SmgrQueryType,
1345 10 : ctx: &'c RequestContext,
1346 10 : ) -> Option<impl Drop + '_> {
1347 10 : let global_metric = &self.global_metrics[op as usize];
1348 10 : let start = Instant::now();
1349 10 : match ctx.micros_spent_throttled.open() {
1350 10 : Ok(()) => (),
1351 0 : Err(error) => {
1352 0 : use utils::rate_limit::RateLimit;
1353 0 : static LOGGED: Lazy<Mutex<enum_map::EnumMap<SmgrQueryType, RateLimit>>> =
1354 0 : Lazy::new(|| {
1355 0 : Mutex::new(enum_map::EnumMap::from_array(std::array::from_fn(|_| {
1356 0 : RateLimit::new(Duration::from_secs(10))
1357 0 : })))
1358 0 : });
1359 0 : let mut guard = LOGGED.lock().unwrap();
1360 0 : let rate_limit = &mut guard[op];
1361 0 : rate_limit.call(|| {
1362 0 : warn!(?op, error, "error opening micros_spent_throttled; this message is logged at a global rate limit");
1363 0 : });
1364 0 : }
1365 : }
1366 :
1367 10 : let timeline_metric = if matches!(op, SmgrQueryType::GetPageAtLsn) {
1368 2 : Some(&self.per_timeline_getpage)
1369 : } else {
1370 8 : None
1371 : };
1372 :
1373 10 : Some(GlobalAndPerTimelineHistogramTimer {
1374 10 : global_metric,
1375 10 : timeline_metric,
1376 10 : ctx,
1377 10 : start,
1378 10 : op,
1379 10 : })
1380 10 : }
1381 : }
1382 :
1383 : #[cfg(test)]
1384 : mod smgr_query_time_tests {
1385 : use pageserver_api::shard::TenantShardId;
1386 : use strum::IntoEnumIterator;
1387 : use utils::id::{TenantId, TimelineId};
1388 :
1389 : use crate::{
1390 : context::{DownloadBehavior, RequestContext},
1391 : task_mgr::TaskKind,
1392 : };
1393 :
1394 : // Regression test, we used hard-coded string constants before using an enum.
1395 : #[test]
1396 2 : fn op_label_name() {
1397 2 : use super::SmgrQueryType::*;
1398 2 : let expect: [(super::SmgrQueryType, &'static str); 5] = [
1399 2 : (GetRelExists, "get_rel_exists"),
1400 2 : (GetRelSize, "get_rel_size"),
1401 2 : (GetPageAtLsn, "get_page_at_lsn"),
1402 2 : (GetDbSize, "get_db_size"),
1403 2 : (GetSlruSegment, "get_slru_segment"),
1404 2 : ];
1405 12 : for (op, expect) in expect {
1406 10 : let actual: &'static str = op.into();
1407 10 : assert_eq!(actual, expect);
1408 : }
1409 2 : }
1410 :
1411 : #[test]
1412 2 : fn basic() {
1413 2 : let ops: Vec<_> = super::SmgrQueryType::iter().collect();
1414 :
1415 12 : for op in &ops {
1416 10 : let tenant_id = TenantId::generate();
1417 10 : let timeline_id = TimelineId::generate();
1418 10 : let metrics = super::SmgrQueryTimePerTimeline::new(
1419 10 : &TenantShardId::unsharded(tenant_id),
1420 10 : &timeline_id,
1421 10 : );
1422 10 :
1423 20 : let get_counts = || {
1424 20 : let global: u64 = ops
1425 20 : .iter()
1426 100 : .map(|op| metrics.global_metrics[*op as usize].get_sample_count())
1427 20 : .sum();
1428 20 : (global, metrics.per_timeline_getpage.get_sample_count())
1429 20 : };
1430 :
1431 10 : let (pre_global, pre_per_tenant_timeline) = get_counts();
1432 10 : assert_eq!(pre_per_tenant_timeline, 0);
1433 :
1434 10 : let ctx = RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Download);
1435 10 : let timer = metrics.start_timer(*op, &ctx);
1436 10 : drop(timer);
1437 10 :
1438 10 : let (post_global, post_per_tenant_timeline) = get_counts();
1439 10 : if matches!(op, super::SmgrQueryType::GetPageAtLsn) {
1440 : // getpage ops are tracked per-timeline, others aren't
1441 2 : assert_eq!(post_per_tenant_timeline, 1);
1442 : } else {
1443 8 : assert_eq!(post_per_tenant_timeline, 0);
1444 : }
1445 10 : assert!(post_global > pre_global);
1446 : }
1447 2 : }
1448 : }
1449 :
1450 : // keep in sync with control plane Go code so that we can validate
1451 : // compute's basebackup_ms metric with our perspective in the context of SLI/SLO.
1452 0 : static COMPUTE_STARTUP_BUCKETS: Lazy<[f64; 28]> = Lazy::new(|| {
1453 0 : // Go code uses milliseconds. Variable is called `computeStartupBuckets`
1454 0 : [
1455 0 : 5, 10, 20, 30, 50, 70, 100, 120, 150, 200, 250, 300, 350, 400, 450, 500, 600, 800, 1000,
1456 0 : 1500, 2000, 2500, 3000, 5000, 10000, 20000, 40000, 60000,
1457 0 : ]
1458 0 : .map(|ms| (ms as f64) / 1000.0)
1459 0 : });
1460 :
1461 : pub(crate) struct BasebackupQueryTime {
1462 : ok: Histogram,
1463 : error: Histogram,
1464 : }
1465 :
1466 0 : pub(crate) static BASEBACKUP_QUERY_TIME: Lazy<BasebackupQueryTime> = Lazy::new(|| {
1467 0 : let vec = register_histogram_vec!(
1468 : "pageserver_basebackup_query_seconds",
1469 : "Histogram of basebackup queries durations, by result type",
1470 : &["result"],
1471 : COMPUTE_STARTUP_BUCKETS.to_vec(),
1472 : )
1473 0 : .expect("failed to define a metric");
1474 0 : BasebackupQueryTime {
1475 0 : ok: vec.get_metric_with_label_values(&["ok"]).unwrap(),
1476 0 : error: vec.get_metric_with_label_values(&["error"]).unwrap(),
1477 0 : }
1478 0 : });
1479 :
1480 : pub(crate) struct BasebackupQueryTimeOngoingRecording<'a, 'c> {
1481 : parent: &'a BasebackupQueryTime,
1482 : ctx: &'c RequestContext,
1483 : start: std::time::Instant,
1484 : }
1485 :
1486 : impl BasebackupQueryTime {
1487 0 : pub(crate) fn start_recording<'c: 'a, 'a>(
1488 0 : &'a self,
1489 0 : ctx: &'c RequestContext,
1490 0 : ) -> BasebackupQueryTimeOngoingRecording<'_, '_> {
1491 0 : let start = Instant::now();
1492 0 : match ctx.micros_spent_throttled.open() {
1493 0 : Ok(()) => (),
1494 0 : Err(error) => {
1495 0 : use utils::rate_limit::RateLimit;
1496 0 : static LOGGED: Lazy<Mutex<RateLimit>> =
1497 0 : Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(10))));
1498 0 : let mut rate_limit = LOGGED.lock().unwrap();
1499 0 : rate_limit.call(|| {
1500 0 : warn!(error, "error opening micros_spent_throttled; this message is logged at a global rate limit");
1501 0 : });
1502 0 : }
1503 : }
1504 0 : BasebackupQueryTimeOngoingRecording {
1505 0 : parent: self,
1506 0 : ctx,
1507 0 : start,
1508 0 : }
1509 0 : }
1510 : }
1511 :
1512 : impl<'a, 'c> BasebackupQueryTimeOngoingRecording<'a, 'c> {
1513 0 : pub(crate) fn observe<T, E>(self, res: &Result<T, E>) {
1514 0 : let elapsed = self.start.elapsed();
1515 0 : let ex_throttled = self
1516 0 : .ctx
1517 0 : .micros_spent_throttled
1518 0 : .close_and_checked_sub_from(elapsed);
1519 0 : let ex_throttled = match ex_throttled {
1520 0 : Ok(ex_throttled) => ex_throttled,
1521 0 : Err(error) => {
1522 0 : use utils::rate_limit::RateLimit;
1523 0 : static LOGGED: Lazy<Mutex<RateLimit>> =
1524 0 : Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(10))));
1525 0 : let mut rate_limit = LOGGED.lock().unwrap();
1526 0 : rate_limit.call(|| {
1527 0 : warn!(error, "error deducting time spent throttled; this message is logged at a global rate limit");
1528 0 : });
1529 0 : elapsed
1530 : }
1531 : };
1532 0 : let metric = if res.is_ok() {
1533 0 : &self.parent.ok
1534 : } else {
1535 0 : &self.parent.error
1536 : };
1537 0 : metric.observe(ex_throttled.as_secs_f64());
1538 0 : }
1539 : }
1540 :
1541 0 : pub(crate) static LIVE_CONNECTIONS: Lazy<IntCounterPairVec> = Lazy::new(|| {
1542 : register_int_counter_pair_vec!(
1543 : "pageserver_live_connections_started",
1544 : "Number of network connections that we started handling",
1545 : "pageserver_live_connections_finished",
1546 : "Number of network connections that we finished handling",
1547 : &["pageserver_connection_kind"]
1548 : )
1549 0 : .expect("failed to define a metric")
1550 0 : });
1551 :
1552 0 : #[derive(Clone, Copy, enum_map::Enum, IntoStaticStr)]
1553 : pub(crate) enum ComputeCommandKind {
1554 : PageStreamV2,
1555 : PageStream,
1556 : Basebackup,
1557 : Fullbackup,
1558 : LeaseLsn,
1559 : }
1560 :
1561 : pub(crate) struct ComputeCommandCounters {
1562 : map: EnumMap<ComputeCommandKind, IntCounter>,
1563 : }
1564 :
1565 0 : pub(crate) static COMPUTE_COMMANDS_COUNTERS: Lazy<ComputeCommandCounters> = Lazy::new(|| {
1566 0 : let inner = register_int_counter_vec!(
1567 : "pageserver_compute_commands",
1568 : "Number of compute -> pageserver commands processed",
1569 : &["command"]
1570 : )
1571 0 : .expect("failed to define a metric");
1572 0 :
1573 0 : ComputeCommandCounters {
1574 0 : map: EnumMap::from_array(std::array::from_fn(|i| {
1575 0 : let command = <ComputeCommandKind as enum_map::Enum>::from_usize(i);
1576 0 : let command_str: &'static str = command.into();
1577 0 : inner.with_label_values(&[command_str])
1578 0 : })),
1579 0 : }
1580 0 : });
1581 :
1582 : impl ComputeCommandCounters {
1583 0 : pub(crate) fn for_command(&self, command: ComputeCommandKind) -> &IntCounter {
1584 0 : &self.map[command]
1585 0 : }
1586 : }
1587 :
1588 : // remote storage metrics
1589 :
1590 152 : static REMOTE_TIMELINE_CLIENT_CALLS: Lazy<IntCounterPairVec> = Lazy::new(|| {
1591 : register_int_counter_pair_vec!(
1592 : "pageserver_remote_timeline_client_calls_started",
1593 : "Number of started calls to remote timeline client.",
1594 : "pageserver_remote_timeline_client_calls_finished",
1595 : "Number of finshed calls to remote timeline client.",
1596 : &[
1597 : "tenant_id",
1598 : "shard_id",
1599 : "timeline_id",
1600 : "file_kind",
1601 : "op_kind"
1602 : ],
1603 : )
1604 152 : .unwrap()
1605 152 : });
1606 :
1607 : static REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER: Lazy<IntCounterVec> =
1608 148 : Lazy::new(|| {
1609 : register_int_counter_vec!(
1610 : "pageserver_remote_timeline_client_bytes_started",
1611 : "Incremented by the number of bytes associated with a remote timeline client operation. \
1612 : The increment happens when the operation is scheduled.",
1613 : &["tenant_id", "shard_id", "timeline_id", "file_kind", "op_kind"],
1614 : )
1615 148 : .expect("failed to define a metric")
1616 148 : });
1617 :
1618 148 : static REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER: Lazy<IntCounterVec> = Lazy::new(|| {
1619 : register_int_counter_vec!(
1620 : "pageserver_remote_timeline_client_bytes_finished",
1621 : "Incremented by the number of bytes associated with a remote timeline client operation. \
1622 : The increment happens when the operation finishes (regardless of success/failure/shutdown).",
1623 : &["tenant_id", "shard_id", "timeline_id", "file_kind", "op_kind"],
1624 : )
1625 148 : .expect("failed to define a metric")
1626 148 : });
1627 :
1628 : pub(crate) struct TenantManagerMetrics {
1629 : tenant_slots_attached: UIntGauge,
1630 : tenant_slots_secondary: UIntGauge,
1631 : tenant_slots_inprogress: UIntGauge,
1632 : pub(crate) tenant_slot_writes: IntCounter,
1633 : pub(crate) unexpected_errors: IntCounter,
1634 : }
1635 :
1636 : impl TenantManagerMetrics {
1637 : /// Helpers for tracking slots. Note that these do not track the lifetime of TenantSlot objects
1638 : /// exactly: they track the lifetime of the slots _in the tenant map_.
1639 2 : pub(crate) fn slot_inserted(&self, slot: &TenantSlot) {
1640 2 : match slot {
1641 0 : TenantSlot::Attached(_) => {
1642 0 : self.tenant_slots_attached.inc();
1643 0 : }
1644 0 : TenantSlot::Secondary(_) => {
1645 0 : self.tenant_slots_secondary.inc();
1646 0 : }
1647 2 : TenantSlot::InProgress(_) => {
1648 2 : self.tenant_slots_inprogress.inc();
1649 2 : }
1650 : }
1651 2 : }
1652 :
1653 2 : pub(crate) fn slot_removed(&self, slot: &TenantSlot) {
1654 2 : match slot {
1655 2 : TenantSlot::Attached(_) => {
1656 2 : self.tenant_slots_attached.dec();
1657 2 : }
1658 0 : TenantSlot::Secondary(_) => {
1659 0 : self.tenant_slots_secondary.dec();
1660 0 : }
1661 0 : TenantSlot::InProgress(_) => {
1662 0 : self.tenant_slots_inprogress.dec();
1663 0 : }
1664 : }
1665 2 : }
1666 :
1667 : #[cfg(all(debug_assertions, not(test)))]
1668 0 : pub(crate) fn slots_total(&self) -> u64 {
1669 0 : self.tenant_slots_attached.get()
1670 0 : + self.tenant_slots_secondary.get()
1671 0 : + self.tenant_slots_inprogress.get()
1672 0 : }
1673 : }
1674 :
1675 2 : pub(crate) static TENANT_MANAGER: Lazy<TenantManagerMetrics> = Lazy::new(|| {
1676 2 : let tenant_slots = register_uint_gauge_vec!(
1677 : "pageserver_tenant_manager_slots",
1678 : "How many slots currently exist, including all attached, secondary and in-progress operations",
1679 : &["mode"]
1680 : )
1681 2 : .expect("failed to define a metric");
1682 2 : TenantManagerMetrics {
1683 2 : tenant_slots_attached: tenant_slots
1684 2 : .get_metric_with_label_values(&["attached"])
1685 2 : .unwrap(),
1686 2 : tenant_slots_secondary: tenant_slots
1687 2 : .get_metric_with_label_values(&["secondary"])
1688 2 : .unwrap(),
1689 2 : tenant_slots_inprogress: tenant_slots
1690 2 : .get_metric_with_label_values(&["inprogress"])
1691 2 : .unwrap(),
1692 2 : tenant_slot_writes: register_int_counter!(
1693 : "pageserver_tenant_manager_slot_writes",
1694 : "Writes to a tenant slot, including all of create/attach/detach/delete"
1695 2 : )
1696 2 : .expect("failed to define a metric"),
1697 2 : unexpected_errors: register_int_counter!(
1698 : "pageserver_tenant_manager_unexpected_errors_total",
1699 : "Number of unexpected conditions encountered: nonzero value indicates a non-fatal bug."
1700 2 : )
1701 2 : .expect("failed to define a metric"),
1702 2 : }
1703 2 : });
1704 :
1705 : pub(crate) struct DeletionQueueMetrics {
1706 : pub(crate) keys_submitted: IntCounter,
1707 : pub(crate) keys_dropped: IntCounter,
1708 : pub(crate) keys_executed: IntCounter,
1709 : pub(crate) keys_validated: IntCounter,
1710 : pub(crate) dropped_lsn_updates: IntCounter,
1711 : pub(crate) unexpected_errors: IntCounter,
1712 : pub(crate) remote_errors: IntCounterVec,
1713 : }
1714 27 : pub(crate) static DELETION_QUEUE: Lazy<DeletionQueueMetrics> = Lazy::new(|| {
1715 27 : DeletionQueueMetrics{
1716 27 :
1717 27 : keys_submitted: register_int_counter!(
1718 : "pageserver_deletion_queue_submitted_total",
1719 : "Number of objects submitted for deletion"
1720 27 : )
1721 27 : .expect("failed to define a metric"),
1722 27 :
1723 27 : keys_dropped: register_int_counter!(
1724 : "pageserver_deletion_queue_dropped_total",
1725 : "Number of object deletions dropped due to stale generation."
1726 27 : )
1727 27 : .expect("failed to define a metric"),
1728 27 :
1729 27 : keys_executed: register_int_counter!(
1730 : "pageserver_deletion_queue_executed_total",
1731 : "Number of objects deleted. Only includes objects that we actually deleted, sum with pageserver_deletion_queue_dropped_total for the total number of keys processed to completion"
1732 27 : )
1733 27 : .expect("failed to define a metric"),
1734 27 :
1735 27 : keys_validated: register_int_counter!(
1736 : "pageserver_deletion_queue_validated_total",
1737 : "Number of keys validated for deletion. Sum with pageserver_deletion_queue_dropped_total for the total number of keys that have passed through the validation stage."
1738 27 : )
1739 27 : .expect("failed to define a metric"),
1740 27 :
1741 27 : dropped_lsn_updates: register_int_counter!(
1742 : "pageserver_deletion_queue_dropped_lsn_updates_total",
1743 : "Updates to remote_consistent_lsn dropped due to stale generation number."
1744 27 : )
1745 27 : .expect("failed to define a metric"),
1746 27 : unexpected_errors: register_int_counter!(
1747 : "pageserver_deletion_queue_unexpected_errors_total",
1748 : "Number of unexpected condiions that may stall the queue: any value above zero is unexpected."
1749 27 : )
1750 27 : .expect("failed to define a metric"),
1751 27 : remote_errors: register_int_counter_vec!(
1752 : "pageserver_deletion_queue_remote_errors_total",
1753 : "Retryable remote I/O errors while executing deletions, for example 503 responses to DeleteObjects",
1754 : &["op_kind"],
1755 27 : )
1756 27 : .expect("failed to define a metric")
1757 27 : }
1758 27 : });
1759 :
1760 : pub(crate) struct SecondaryModeMetrics {
1761 : pub(crate) upload_heatmap: IntCounter,
1762 : pub(crate) upload_heatmap_errors: IntCounter,
1763 : pub(crate) upload_heatmap_duration: Histogram,
1764 : pub(crate) download_heatmap: IntCounter,
1765 : pub(crate) download_layer: IntCounter,
1766 : }
1767 0 : pub(crate) static SECONDARY_MODE: Lazy<SecondaryModeMetrics> = Lazy::new(|| {
1768 0 : SecondaryModeMetrics {
1769 0 : upload_heatmap: register_int_counter!(
1770 : "pageserver_secondary_upload_heatmap",
1771 : "Number of heatmaps written to remote storage by attached tenants"
1772 0 : )
1773 0 : .expect("failed to define a metric"),
1774 0 : upload_heatmap_errors: register_int_counter!(
1775 : "pageserver_secondary_upload_heatmap_errors",
1776 : "Failures writing heatmap to remote storage"
1777 0 : )
1778 0 : .expect("failed to define a metric"),
1779 0 : upload_heatmap_duration: register_histogram!(
1780 : "pageserver_secondary_upload_heatmap_duration",
1781 : "Time to build and upload a heatmap, including any waiting inside the S3 client"
1782 0 : )
1783 0 : .expect("failed to define a metric"),
1784 0 : download_heatmap: register_int_counter!(
1785 : "pageserver_secondary_download_heatmap",
1786 : "Number of downloads of heatmaps by secondary mode locations, including when it hasn't changed"
1787 0 : )
1788 0 : .expect("failed to define a metric"),
1789 0 : download_layer: register_int_counter!(
1790 : "pageserver_secondary_download_layer",
1791 : "Number of downloads of layers by secondary mode locations"
1792 0 : )
1793 0 : .expect("failed to define a metric"),
1794 0 : }
1795 0 : });
1796 :
1797 0 : pub(crate) static SECONDARY_RESIDENT_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
1798 : register_uint_gauge_vec!(
1799 : "pageserver_secondary_resident_physical_size",
1800 : "The size of the layer files present in the pageserver's filesystem, for secondary locations.",
1801 : &["tenant_id", "shard_id"]
1802 : )
1803 0 : .expect("failed to define a metric")
1804 0 : });
1805 :
1806 : #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1807 : pub enum RemoteOpKind {
1808 : Upload,
1809 : Download,
1810 : Delete,
1811 : }
1812 : impl RemoteOpKind {
1813 13167 : pub fn as_str(&self) -> &'static str {
1814 13167 : match self {
1815 12423 : Self::Upload => "upload",
1816 52 : Self::Download => "download",
1817 692 : Self::Delete => "delete",
1818 : }
1819 13167 : }
1820 : }
1821 :
1822 : #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
1823 : pub enum RemoteOpFileKind {
1824 : Layer,
1825 : Index,
1826 : }
1827 : impl RemoteOpFileKind {
1828 13167 : pub fn as_str(&self) -> &'static str {
1829 13167 : match self {
1830 8903 : Self::Layer => "layer",
1831 4264 : Self::Index => "index",
1832 : }
1833 13167 : }
1834 : }
1835 :
1836 148 : pub(crate) static REMOTE_OPERATION_TIME: Lazy<HistogramVec> = Lazy::new(|| {
1837 : register_histogram_vec!(
1838 : "pageserver_remote_operation_seconds",
1839 : "Time spent on remote storage operations. \
1840 : Grouped by tenant, timeline, operation_kind and status. \
1841 : Does not account for time spent waiting in remote timeline client's queues.",
1842 : &["file_kind", "op_kind", "status"]
1843 : )
1844 148 : .expect("failed to define a metric")
1845 148 : });
1846 :
1847 0 : pub(crate) static TENANT_TASK_EVENTS: Lazy<IntCounterVec> = Lazy::new(|| {
1848 : register_int_counter_vec!(
1849 : "pageserver_tenant_task_events",
1850 : "Number of task start/stop/fail events.",
1851 : &["event"],
1852 : )
1853 0 : .expect("Failed to register tenant_task_events metric")
1854 0 : });
1855 :
1856 20 : pub(crate) static BACKGROUND_LOOP_SEMAPHORE_WAIT_GAUGE: Lazy<IntCounterPairVec> = Lazy::new(|| {
1857 : register_int_counter_pair_vec!(
1858 : "pageserver_background_loop_semaphore_wait_start_count",
1859 : "Counter for background loop concurrency-limiting semaphore acquire calls started",
1860 : "pageserver_background_loop_semaphore_wait_finish_count",
1861 : "Counter for background loop concurrency-limiting semaphore acquire calls finished",
1862 : &["task"],
1863 : )
1864 20 : .unwrap()
1865 20 : });
1866 :
1867 0 : pub(crate) static BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT: Lazy<IntCounterVec> = Lazy::new(|| {
1868 : register_int_counter_vec!(
1869 : "pageserver_background_loop_period_overrun_count",
1870 : "Incremented whenever warn_when_period_overrun() logs a warning.",
1871 : &["task", "period"],
1872 : )
1873 0 : .expect("failed to define a metric")
1874 0 : });
1875 :
1876 : // walreceiver metrics
1877 :
1878 0 : pub(crate) static WALRECEIVER_STARTED_CONNECTIONS: Lazy<IntCounter> = Lazy::new(|| {
1879 : register_int_counter!(
1880 : "pageserver_walreceiver_started_connections_total",
1881 : "Number of started walreceiver connections"
1882 : )
1883 0 : .expect("failed to define a metric")
1884 0 : });
1885 :
1886 0 : pub(crate) static WALRECEIVER_ACTIVE_MANAGERS: Lazy<IntGauge> = Lazy::new(|| {
1887 : register_int_gauge!(
1888 : "pageserver_walreceiver_active_managers",
1889 : "Number of active walreceiver managers"
1890 : )
1891 0 : .expect("failed to define a metric")
1892 0 : });
1893 :
1894 0 : pub(crate) static WALRECEIVER_SWITCHES: Lazy<IntCounterVec> = Lazy::new(|| {
1895 : register_int_counter_vec!(
1896 : "pageserver_walreceiver_switches_total",
1897 : "Number of walreceiver manager change_connection calls",
1898 : &["reason"]
1899 : )
1900 0 : .expect("failed to define a metric")
1901 0 : });
1902 :
1903 0 : pub(crate) static WALRECEIVER_BROKER_UPDATES: Lazy<IntCounter> = Lazy::new(|| {
1904 : register_int_counter!(
1905 : "pageserver_walreceiver_broker_updates_total",
1906 : "Number of received broker updates in walreceiver"
1907 : )
1908 0 : .expect("failed to define a metric")
1909 0 : });
1910 :
1911 2 : pub(crate) static WALRECEIVER_CANDIDATES_EVENTS: Lazy<IntCounterVec> = Lazy::new(|| {
1912 : register_int_counter_vec!(
1913 : "pageserver_walreceiver_candidates_events_total",
1914 : "Number of walreceiver candidate events",
1915 : &["event"]
1916 : )
1917 2 : .expect("failed to define a metric")
1918 2 : });
1919 :
1920 : pub(crate) static WALRECEIVER_CANDIDATES_ADDED: Lazy<IntCounter> =
1921 0 : Lazy::new(|| WALRECEIVER_CANDIDATES_EVENTS.with_label_values(&["add"]));
1922 :
1923 : pub(crate) static WALRECEIVER_CANDIDATES_REMOVED: Lazy<IntCounter> =
1924 2 : Lazy::new(|| WALRECEIVER_CANDIDATES_EVENTS.with_label_values(&["remove"]));
1925 :
1926 : // Metrics collected on WAL redo operations
1927 : //
1928 : // We collect the time spent in actual WAL redo ('redo'), and time waiting
1929 : // for access to the postgres process ('wait') since there is only one for
1930 : // each tenant.
1931 :
1932 : /// Time buckets are small because we want to be able to measure the
1933 : /// smallest redo processing times. These buckets allow us to measure down
1934 : /// to 5us, which equates to 200'000 pages/sec, which equates to 1.6GB/sec.
1935 : /// This is much better than the previous 5ms aka 200 pages/sec aka 1.6MB/sec.
1936 : ///
1937 : /// Values up to 1s are recorded because metrics show that we have redo
1938 : /// durations and lock times larger than 0.250s.
1939 : macro_rules! redo_histogram_time_buckets {
1940 : () => {
1941 : vec![
1942 : 0.000_005, 0.000_010, 0.000_025, 0.000_050, 0.000_100, 0.000_250, 0.000_500, 0.001_000,
1943 : 0.002_500, 0.005_000, 0.010_000, 0.025_000, 0.050_000, 0.100_000, 0.250_000, 0.500_000,
1944 : 1.000_000,
1945 : ]
1946 : };
1947 : }
1948 :
1949 : /// While we're at it, also measure the amount of records replayed in each
1950 : /// operation. We have a global 'total replayed' counter, but that's not
1951 : /// as useful as 'what is the skew for how many records we replay in one
1952 : /// operation'.
1953 : macro_rules! redo_histogram_count_buckets {
1954 : () => {
1955 : vec![0.0, 1.0, 2.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0]
1956 : };
1957 : }
1958 :
1959 : macro_rules! redo_bytes_histogram_count_buckets {
1960 : () => {
1961 : // powers of (2^.5), from 2^4.5 to 2^15 (22 buckets)
1962 : // rounded up to the next multiple of 8 to capture any MAXALIGNed record of that size, too.
1963 : vec![
1964 : 24.0, 32.0, 48.0, 64.0, 96.0, 128.0, 184.0, 256.0, 368.0, 512.0, 728.0, 1024.0, 1456.0,
1965 : 2048.0, 2904.0, 4096.0, 5800.0, 8192.0, 11592.0, 16384.0, 23176.0, 32768.0,
1966 : ]
1967 : };
1968 : }
1969 :
1970 : pub(crate) struct WalIngestMetrics {
1971 : pub(crate) bytes_received: IntCounter,
1972 : pub(crate) records_received: IntCounter,
1973 : pub(crate) records_committed: IntCounter,
1974 : pub(crate) records_filtered: IntCounter,
1975 : }
1976 :
1977 2 : pub(crate) static WAL_INGEST: Lazy<WalIngestMetrics> = Lazy::new(|| WalIngestMetrics {
1978 : bytes_received: register_int_counter!(
1979 : "pageserver_wal_ingest_bytes_received",
1980 : "Bytes of WAL ingested from safekeepers",
1981 : )
1982 2 : .unwrap(),
1983 : records_received: register_int_counter!(
1984 : "pageserver_wal_ingest_records_received",
1985 : "Number of WAL records received from safekeepers"
1986 : )
1987 2 : .expect("failed to define a metric"),
1988 : records_committed: register_int_counter!(
1989 : "pageserver_wal_ingest_records_committed",
1990 : "Number of WAL records which resulted in writes to pageserver storage"
1991 : )
1992 2 : .expect("failed to define a metric"),
1993 : records_filtered: register_int_counter!(
1994 : "pageserver_wal_ingest_records_filtered",
1995 : "Number of WAL records filtered out due to sharding"
1996 : )
1997 2 : .expect("failed to define a metric"),
1998 2 : });
1999 :
2000 6 : pub(crate) static WAL_REDO_TIME: Lazy<Histogram> = Lazy::new(|| {
2001 : register_histogram!(
2002 : "pageserver_wal_redo_seconds",
2003 : "Time spent on WAL redo",
2004 : redo_histogram_time_buckets!()
2005 : )
2006 6 : .expect("failed to define a metric")
2007 6 : });
2008 :
2009 6 : pub(crate) static WAL_REDO_RECORDS_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
2010 : register_histogram!(
2011 : "pageserver_wal_redo_records_histogram",
2012 : "Histogram of number of records replayed per redo in the Postgres WAL redo process",
2013 : redo_histogram_count_buckets!(),
2014 : )
2015 6 : .expect("failed to define a metric")
2016 6 : });
2017 :
2018 6 : pub(crate) static WAL_REDO_BYTES_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
2019 : register_histogram!(
2020 : "pageserver_wal_redo_bytes_histogram",
2021 : "Histogram of number of records replayed per redo sent to Postgres",
2022 : redo_bytes_histogram_count_buckets!(),
2023 : )
2024 6 : .expect("failed to define a metric")
2025 6 : });
2026 :
2027 : // FIXME: isn't this already included by WAL_REDO_RECORDS_HISTOGRAM which has _count?
2028 6 : pub(crate) static WAL_REDO_RECORD_COUNTER: Lazy<IntCounter> = Lazy::new(|| {
2029 : register_int_counter!(
2030 : "pageserver_replayed_wal_records_total",
2031 : "Number of WAL records replayed in WAL redo process"
2032 : )
2033 6 : .unwrap()
2034 6 : });
2035 :
2036 : #[rustfmt::skip]
2037 6 : pub(crate) static WAL_REDO_PROCESS_LAUNCH_DURATION_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
2038 : register_histogram!(
2039 : "pageserver_wal_redo_process_launch_duration",
2040 : "Histogram of the duration of successful WalRedoProcess::launch calls",
2041 : vec![
2042 : 0.0002, 0.0004, 0.0006, 0.0008, 0.0010,
2043 : 0.0020, 0.0040, 0.0060, 0.0080, 0.0100,
2044 : 0.0200, 0.0400, 0.0600, 0.0800, 0.1000,
2045 : 0.2000, 0.4000, 0.6000, 0.8000, 1.0000,
2046 : 1.5000, 2.0000, 2.5000, 3.0000, 4.0000, 10.0000
2047 : ],
2048 : )
2049 6 : .expect("failed to define a metric")
2050 6 : });
2051 :
2052 : pub(crate) struct WalRedoProcessCounters {
2053 : pub(crate) started: IntCounter,
2054 : pub(crate) killed_by_cause: enum_map::EnumMap<WalRedoKillCause, IntCounter>,
2055 : pub(crate) active_stderr_logger_tasks_started: IntCounter,
2056 : pub(crate) active_stderr_logger_tasks_finished: IntCounter,
2057 : }
2058 :
2059 18 : #[derive(Debug, enum_map::Enum, strum_macros::IntoStaticStr)]
2060 : pub(crate) enum WalRedoKillCause {
2061 : WalRedoProcessDrop,
2062 : NoLeakChildDrop,
2063 : Startup,
2064 : }
2065 :
2066 : impl Default for WalRedoProcessCounters {
2067 6 : fn default() -> Self {
2068 6 : let started = register_int_counter!(
2069 : "pageserver_wal_redo_process_started_total",
2070 : "Number of WAL redo processes started",
2071 : )
2072 6 : .unwrap();
2073 6 :
2074 6 : let killed = register_int_counter_vec!(
2075 : "pageserver_wal_redo_process_stopped_total",
2076 : "Number of WAL redo processes stopped",
2077 : &["cause"],
2078 : )
2079 6 : .unwrap();
2080 6 :
2081 6 : let active_stderr_logger_tasks_started = register_int_counter!(
2082 : "pageserver_walredo_stderr_logger_tasks_started_total",
2083 : "Number of active walredo stderr logger tasks that have started",
2084 : )
2085 6 : .unwrap();
2086 6 :
2087 6 : let active_stderr_logger_tasks_finished = register_int_counter!(
2088 : "pageserver_walredo_stderr_logger_tasks_finished_total",
2089 : "Number of active walredo stderr logger tasks that have finished",
2090 : )
2091 6 : .unwrap();
2092 6 :
2093 6 : Self {
2094 6 : started,
2095 18 : killed_by_cause: EnumMap::from_array(std::array::from_fn(|i| {
2096 18 : let cause = <WalRedoKillCause as enum_map::Enum>::from_usize(i);
2097 18 : let cause_str: &'static str = cause.into();
2098 18 : killed.with_label_values(&[cause_str])
2099 18 : })),
2100 6 : active_stderr_logger_tasks_started,
2101 6 : active_stderr_logger_tasks_finished,
2102 6 : }
2103 6 : }
2104 : }
2105 :
2106 : pub(crate) static WAL_REDO_PROCESS_COUNTERS: Lazy<WalRedoProcessCounters> =
2107 : Lazy::new(WalRedoProcessCounters::default);
2108 :
2109 : /// Similar to `prometheus::HistogramTimer` but does not record on drop.
2110 : pub(crate) struct StorageTimeMetricsTimer {
2111 : metrics: StorageTimeMetrics,
2112 : start: Instant,
2113 : }
2114 :
2115 : impl StorageTimeMetricsTimer {
2116 4639 : fn new(metrics: StorageTimeMetrics) -> Self {
2117 4639 : Self {
2118 4639 : metrics,
2119 4639 : start: Instant::now(),
2120 4639 : }
2121 4639 : }
2122 :
2123 : /// Record the time from creation to now.
2124 3525 : pub fn stop_and_record(self) {
2125 3525 : let duration = self.start.elapsed().as_secs_f64();
2126 3525 : self.metrics.timeline_sum.inc_by(duration);
2127 3525 : self.metrics.timeline_count.inc();
2128 3525 : self.metrics.global_histogram.observe(duration);
2129 3525 : }
2130 :
2131 : /// Turns this timer into a timer, which will always record -- usually this means recording
2132 : /// regardless an early `?` path was taken in a function.
2133 754 : pub(crate) fn record_on_drop(self) -> AlwaysRecordingStorageTimeMetricsTimer {
2134 754 : AlwaysRecordingStorageTimeMetricsTimer(Some(self))
2135 754 : }
2136 : }
2137 :
2138 : pub(crate) struct AlwaysRecordingStorageTimeMetricsTimer(Option<StorageTimeMetricsTimer>);
2139 :
2140 : impl Drop for AlwaysRecordingStorageTimeMetricsTimer {
2141 754 : fn drop(&mut self) {
2142 754 : if let Some(inner) = self.0.take() {
2143 754 : inner.stop_and_record();
2144 754 : }
2145 754 : }
2146 : }
2147 :
2148 : /// Timing facilities for an globally histogrammed metric, which is supported by per tenant and
2149 : /// timeline total sum and count.
2150 : #[derive(Clone, Debug)]
2151 : pub(crate) struct StorageTimeMetrics {
2152 : /// Sum of f64 seconds, per operation, tenant_id and timeline_id
2153 : timeline_sum: Counter,
2154 : /// Number of oeprations, per operation, tenant_id and timeline_id
2155 : timeline_count: IntCounter,
2156 : /// Global histogram having only the "operation" label.
2157 : global_histogram: Histogram,
2158 : }
2159 :
2160 : impl StorageTimeMetrics {
2161 3184 : pub fn new(
2162 3184 : operation: StorageTimeOperation,
2163 3184 : tenant_id: &str,
2164 3184 : shard_id: &str,
2165 3184 : timeline_id: &str,
2166 3184 : ) -> Self {
2167 3184 : let operation: &'static str = operation.into();
2168 3184 :
2169 3184 : let timeline_sum = STORAGE_TIME_SUM_PER_TIMELINE
2170 3184 : .get_metric_with_label_values(&[operation, tenant_id, shard_id, timeline_id])
2171 3184 : .unwrap();
2172 3184 : let timeline_count = STORAGE_TIME_COUNT_PER_TIMELINE
2173 3184 : .get_metric_with_label_values(&[operation, tenant_id, shard_id, timeline_id])
2174 3184 : .unwrap();
2175 3184 : let global_histogram = STORAGE_TIME_GLOBAL
2176 3184 : .get_metric_with_label_values(&[operation])
2177 3184 : .unwrap();
2178 3184 :
2179 3184 : StorageTimeMetrics {
2180 3184 : timeline_sum,
2181 3184 : timeline_count,
2182 3184 : global_histogram,
2183 3184 : }
2184 3184 : }
2185 :
2186 : /// Starts timing a new operation.
2187 : ///
2188 : /// Note: unlike `prometheus::HistogramTimer` the returned timer does not record on drop.
2189 4639 : pub fn start_timer(&self) -> StorageTimeMetricsTimer {
2190 4639 : StorageTimeMetricsTimer::new(self.clone())
2191 4639 : }
2192 : }
2193 :
2194 : #[derive(Debug)]
2195 : pub(crate) struct TimelineMetrics {
2196 : tenant_id: String,
2197 : shard_id: String,
2198 : timeline_id: String,
2199 : pub flush_time_histo: StorageTimeMetrics,
2200 : pub compact_time_histo: StorageTimeMetrics,
2201 : pub create_images_time_histo: StorageTimeMetrics,
2202 : pub logical_size_histo: StorageTimeMetrics,
2203 : pub imitate_logical_size_histo: StorageTimeMetrics,
2204 : pub load_layer_map_histo: StorageTimeMetrics,
2205 : pub garbage_collect_histo: StorageTimeMetrics,
2206 : pub find_gc_cutoffs_histo: StorageTimeMetrics,
2207 : pub last_record_gauge: IntGauge,
2208 : pub pitr_history_size: UIntGauge,
2209 : pub archival_size: UIntGauge,
2210 : pub(crate) layer_size_image: UIntGauge,
2211 : pub(crate) layer_count_image: UIntGauge,
2212 : pub(crate) layer_size_delta: UIntGauge,
2213 : pub(crate) layer_count_delta: UIntGauge,
2214 : pub standby_horizon_gauge: IntGauge,
2215 : pub resident_physical_size_gauge: UIntGauge,
2216 : pub visible_physical_size_gauge: UIntGauge,
2217 : /// copy of LayeredTimeline.current_logical_size
2218 : pub current_logical_size_gauge: UIntGauge,
2219 : pub aux_file_size_gauge: IntGauge,
2220 : pub directory_entries_count_gauge: Lazy<UIntGauge, Box<dyn Send + Fn() -> UIntGauge>>,
2221 : pub evictions: IntCounter,
2222 : pub evictions_with_low_residence_duration: std::sync::RwLock<EvictionsWithLowResidenceDuration>,
2223 : /// Number of valid LSN leases.
2224 : pub valid_lsn_lease_count_gauge: UIntGauge,
2225 : shutdown: std::sync::atomic::AtomicBool,
2226 : }
2227 :
2228 : impl TimelineMetrics {
2229 398 : pub fn new(
2230 398 : tenant_shard_id: &TenantShardId,
2231 398 : timeline_id_raw: &TimelineId,
2232 398 : evictions_with_low_residence_duration_builder: EvictionsWithLowResidenceDurationBuilder,
2233 398 : ) -> Self {
2234 398 : let tenant_id = tenant_shard_id.tenant_id.to_string();
2235 398 : let shard_id = format!("{}", tenant_shard_id.shard_slug());
2236 398 : let timeline_id = timeline_id_raw.to_string();
2237 398 : let flush_time_histo = StorageTimeMetrics::new(
2238 398 : StorageTimeOperation::LayerFlush,
2239 398 : &tenant_id,
2240 398 : &shard_id,
2241 398 : &timeline_id,
2242 398 : );
2243 398 : let compact_time_histo = StorageTimeMetrics::new(
2244 398 : StorageTimeOperation::Compact,
2245 398 : &tenant_id,
2246 398 : &shard_id,
2247 398 : &timeline_id,
2248 398 : );
2249 398 : let create_images_time_histo = StorageTimeMetrics::new(
2250 398 : StorageTimeOperation::CreateImages,
2251 398 : &tenant_id,
2252 398 : &shard_id,
2253 398 : &timeline_id,
2254 398 : );
2255 398 : let logical_size_histo = StorageTimeMetrics::new(
2256 398 : StorageTimeOperation::LogicalSize,
2257 398 : &tenant_id,
2258 398 : &shard_id,
2259 398 : &timeline_id,
2260 398 : );
2261 398 : let imitate_logical_size_histo = StorageTimeMetrics::new(
2262 398 : StorageTimeOperation::ImitateLogicalSize,
2263 398 : &tenant_id,
2264 398 : &shard_id,
2265 398 : &timeline_id,
2266 398 : );
2267 398 : let load_layer_map_histo = StorageTimeMetrics::new(
2268 398 : StorageTimeOperation::LoadLayerMap,
2269 398 : &tenant_id,
2270 398 : &shard_id,
2271 398 : &timeline_id,
2272 398 : );
2273 398 : let garbage_collect_histo = StorageTimeMetrics::new(
2274 398 : StorageTimeOperation::Gc,
2275 398 : &tenant_id,
2276 398 : &shard_id,
2277 398 : &timeline_id,
2278 398 : );
2279 398 : let find_gc_cutoffs_histo = StorageTimeMetrics::new(
2280 398 : StorageTimeOperation::FindGcCutoffs,
2281 398 : &tenant_id,
2282 398 : &shard_id,
2283 398 : &timeline_id,
2284 398 : );
2285 398 : let last_record_gauge = LAST_RECORD_LSN
2286 398 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2287 398 : .unwrap();
2288 398 :
2289 398 : let pitr_history_size = PITR_HISTORY_SIZE
2290 398 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2291 398 : .unwrap();
2292 398 :
2293 398 : let archival_size = TIMELINE_ARCHIVE_SIZE
2294 398 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2295 398 : .unwrap();
2296 398 :
2297 398 : let layer_size_image = TIMELINE_LAYER_SIZE
2298 398 : .get_metric_with_label_values(&[
2299 398 : &tenant_id,
2300 398 : &shard_id,
2301 398 : &timeline_id,
2302 398 : MetricLayerKind::Image.into(),
2303 398 : ])
2304 398 : .unwrap();
2305 398 :
2306 398 : let layer_count_image = TIMELINE_LAYER_COUNT
2307 398 : .get_metric_with_label_values(&[
2308 398 : &tenant_id,
2309 398 : &shard_id,
2310 398 : &timeline_id,
2311 398 : MetricLayerKind::Image.into(),
2312 398 : ])
2313 398 : .unwrap();
2314 398 :
2315 398 : let layer_size_delta = TIMELINE_LAYER_SIZE
2316 398 : .get_metric_with_label_values(&[
2317 398 : &tenant_id,
2318 398 : &shard_id,
2319 398 : &timeline_id,
2320 398 : MetricLayerKind::Delta.into(),
2321 398 : ])
2322 398 : .unwrap();
2323 398 :
2324 398 : let layer_count_delta = TIMELINE_LAYER_COUNT
2325 398 : .get_metric_with_label_values(&[
2326 398 : &tenant_id,
2327 398 : &shard_id,
2328 398 : &timeline_id,
2329 398 : MetricLayerKind::Delta.into(),
2330 398 : ])
2331 398 : .unwrap();
2332 398 :
2333 398 : let standby_horizon_gauge = STANDBY_HORIZON
2334 398 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2335 398 : .unwrap();
2336 398 : let resident_physical_size_gauge = RESIDENT_PHYSICAL_SIZE
2337 398 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2338 398 : .unwrap();
2339 398 : let visible_physical_size_gauge = VISIBLE_PHYSICAL_SIZE
2340 398 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2341 398 : .unwrap();
2342 398 : // TODO: we shouldn't expose this metric
2343 398 : let current_logical_size_gauge = CURRENT_LOGICAL_SIZE
2344 398 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2345 398 : .unwrap();
2346 398 : let aux_file_size_gauge = AUX_FILE_SIZE
2347 398 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2348 398 : .unwrap();
2349 398 : // TODO use impl Trait syntax here once we have ability to use it: https://github.com/rust-lang/rust/issues/63065
2350 398 : let directory_entries_count_gauge_closure = {
2351 398 : let tenant_shard_id = *tenant_shard_id;
2352 398 : let timeline_id_raw = *timeline_id_raw;
2353 0 : move || {
2354 0 : let tenant_id = tenant_shard_id.tenant_id.to_string();
2355 0 : let shard_id = format!("{}", tenant_shard_id.shard_slug());
2356 0 : let timeline_id = timeline_id_raw.to_string();
2357 0 : let gauge: UIntGauge = DIRECTORY_ENTRIES_COUNT
2358 0 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2359 0 : .unwrap();
2360 0 : gauge
2361 0 : }
2362 : };
2363 398 : let directory_entries_count_gauge: Lazy<UIntGauge, Box<dyn Send + Fn() -> UIntGauge>> =
2364 398 : Lazy::new(Box::new(directory_entries_count_gauge_closure));
2365 398 : let evictions = EVICTIONS
2366 398 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2367 398 : .unwrap();
2368 398 : let evictions_with_low_residence_duration = evictions_with_low_residence_duration_builder
2369 398 : .build(&tenant_id, &shard_id, &timeline_id);
2370 398 :
2371 398 : let valid_lsn_lease_count_gauge = VALID_LSN_LEASE_COUNT
2372 398 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2373 398 : .unwrap();
2374 398 :
2375 398 : TimelineMetrics {
2376 398 : tenant_id,
2377 398 : shard_id,
2378 398 : timeline_id,
2379 398 : flush_time_histo,
2380 398 : compact_time_histo,
2381 398 : create_images_time_histo,
2382 398 : logical_size_histo,
2383 398 : imitate_logical_size_histo,
2384 398 : garbage_collect_histo,
2385 398 : find_gc_cutoffs_histo,
2386 398 : load_layer_map_histo,
2387 398 : last_record_gauge,
2388 398 : pitr_history_size,
2389 398 : archival_size,
2390 398 : layer_size_image,
2391 398 : layer_count_image,
2392 398 : layer_size_delta,
2393 398 : layer_count_delta,
2394 398 : standby_horizon_gauge,
2395 398 : resident_physical_size_gauge,
2396 398 : visible_physical_size_gauge,
2397 398 : current_logical_size_gauge,
2398 398 : aux_file_size_gauge,
2399 398 : directory_entries_count_gauge,
2400 398 : evictions,
2401 398 : evictions_with_low_residence_duration: std::sync::RwLock::new(
2402 398 : evictions_with_low_residence_duration,
2403 398 : ),
2404 398 : valid_lsn_lease_count_gauge,
2405 398 : shutdown: std::sync::atomic::AtomicBool::default(),
2406 398 : }
2407 398 : }
2408 :
2409 1508 : pub(crate) fn record_new_file_metrics(&self, sz: u64) {
2410 1508 : self.resident_physical_size_add(sz);
2411 1508 : }
2412 :
2413 470 : pub(crate) fn resident_physical_size_sub(&self, sz: u64) {
2414 470 : self.resident_physical_size_gauge.sub(sz);
2415 470 : crate::metrics::RESIDENT_PHYSICAL_SIZE_GLOBAL.sub(sz);
2416 470 : }
2417 :
2418 1538 : pub(crate) fn resident_physical_size_add(&self, sz: u64) {
2419 1538 : self.resident_physical_size_gauge.add(sz);
2420 1538 : crate::metrics::RESIDENT_PHYSICAL_SIZE_GLOBAL.add(sz);
2421 1538 : }
2422 :
2423 8 : pub(crate) fn resident_physical_size_get(&self) -> u64 {
2424 8 : self.resident_physical_size_gauge.get()
2425 8 : }
2426 :
2427 8 : pub(crate) fn shutdown(&self) {
2428 8 : let was_shutdown = self
2429 8 : .shutdown
2430 8 : .swap(true, std::sync::atomic::Ordering::Relaxed);
2431 8 :
2432 8 : if was_shutdown {
2433 : // this happens on tenant deletion because tenant first shuts down timelines, then
2434 : // invokes timeline deletion which first shuts down the timeline again.
2435 : // TODO: this can be removed once https://github.com/neondatabase/neon/issues/5080
2436 0 : return;
2437 8 : }
2438 8 :
2439 8 : let tenant_id = &self.tenant_id;
2440 8 : let timeline_id = &self.timeline_id;
2441 8 : let shard_id = &self.shard_id;
2442 8 : let _ = LAST_RECORD_LSN.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2443 8 : let _ = STANDBY_HORIZON.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2444 8 : {
2445 8 : RESIDENT_PHYSICAL_SIZE_GLOBAL.sub(self.resident_physical_size_get());
2446 8 : let _ = RESIDENT_PHYSICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2447 8 : }
2448 8 : let _ = VISIBLE_PHYSICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2449 8 : let _ = CURRENT_LOGICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2450 8 : if let Some(metric) = Lazy::get(&DIRECTORY_ENTRIES_COUNT) {
2451 0 : let _ = metric.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2452 8 : }
2453 :
2454 8 : let _ = TIMELINE_ARCHIVE_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2455 8 : let _ = PITR_HISTORY_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2456 8 :
2457 8 : let _ = TIMELINE_LAYER_SIZE.remove_label_values(&[
2458 8 : tenant_id,
2459 8 : shard_id,
2460 8 : timeline_id,
2461 8 : MetricLayerKind::Image.into(),
2462 8 : ]);
2463 8 : let _ = TIMELINE_LAYER_COUNT.remove_label_values(&[
2464 8 : tenant_id,
2465 8 : shard_id,
2466 8 : timeline_id,
2467 8 : MetricLayerKind::Image.into(),
2468 8 : ]);
2469 8 : let _ = TIMELINE_LAYER_SIZE.remove_label_values(&[
2470 8 : tenant_id,
2471 8 : shard_id,
2472 8 : timeline_id,
2473 8 : MetricLayerKind::Delta.into(),
2474 8 : ]);
2475 8 : let _ = TIMELINE_LAYER_COUNT.remove_label_values(&[
2476 8 : tenant_id,
2477 8 : shard_id,
2478 8 : timeline_id,
2479 8 : MetricLayerKind::Delta.into(),
2480 8 : ]);
2481 8 :
2482 8 : let _ = EVICTIONS.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2483 8 : let _ = AUX_FILE_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2484 8 : let _ = VALID_LSN_LEASE_COUNT.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2485 8 :
2486 8 : self.evictions_with_low_residence_duration
2487 8 : .write()
2488 8 : .unwrap()
2489 8 : .remove(tenant_id, shard_id, timeline_id);
2490 :
2491 : // The following metrics are born outside of the TimelineMetrics lifecycle but still
2492 : // removed at the end of it. The idea is to have the metrics outlive the
2493 : // entity during which they're observed, e.g., the smgr metrics shall
2494 : // outlive an individual smgr connection, but not the timeline.
2495 :
2496 72 : for op in StorageTimeOperation::VARIANTS {
2497 64 : let _ = STORAGE_TIME_SUM_PER_TIMELINE.remove_label_values(&[
2498 64 : op,
2499 64 : tenant_id,
2500 64 : shard_id,
2501 64 : timeline_id,
2502 64 : ]);
2503 64 : let _ = STORAGE_TIME_COUNT_PER_TIMELINE.remove_label_values(&[
2504 64 : op,
2505 64 : tenant_id,
2506 64 : shard_id,
2507 64 : timeline_id,
2508 64 : ]);
2509 64 : }
2510 :
2511 24 : for op in STORAGE_IO_SIZE_OPERATIONS {
2512 16 : let _ = STORAGE_IO_SIZE.remove_label_values(&[op, tenant_id, shard_id, timeline_id]);
2513 16 : }
2514 :
2515 8 : let _ = SMGR_QUERY_TIME_PER_TENANT_TIMELINE.remove_label_values(&[
2516 8 : SmgrQueryType::GetPageAtLsn.into(),
2517 8 : tenant_id,
2518 8 : shard_id,
2519 8 : timeline_id,
2520 8 : ]);
2521 8 : }
2522 : }
2523 :
2524 6 : pub(crate) fn remove_tenant_metrics(tenant_shard_id: &TenantShardId) {
2525 6 : // Only shard zero deals in synthetic sizes
2526 6 : if tenant_shard_id.is_shard_zero() {
2527 6 : let tid = tenant_shard_id.tenant_id.to_string();
2528 6 : let _ = TENANT_SYNTHETIC_SIZE_METRIC.remove_label_values(&[&tid]);
2529 6 : }
2530 :
2531 : // we leave the BROKEN_TENANTS_SET entry if any
2532 6 : }
2533 :
2534 : use futures::Future;
2535 : use pin_project_lite::pin_project;
2536 : use std::collections::HashMap;
2537 : use std::num::NonZeroUsize;
2538 : use std::pin::Pin;
2539 : use std::sync::atomic::AtomicU64;
2540 : use std::sync::{Arc, Mutex};
2541 : use std::task::{Context, Poll};
2542 : use std::time::{Duration, Instant};
2543 :
2544 : use crate::context::{PageContentKind, RequestContext};
2545 : use crate::task_mgr::TaskKind;
2546 : use crate::tenant::mgr::TenantSlot;
2547 :
2548 : /// Maintain a per timeline gauge in addition to the global gauge.
2549 : pub(crate) struct PerTimelineRemotePhysicalSizeGauge {
2550 : last_set: AtomicU64,
2551 : gauge: UIntGauge,
2552 : }
2553 :
2554 : impl PerTimelineRemotePhysicalSizeGauge {
2555 408 : fn new(per_timeline_gauge: UIntGauge) -> Self {
2556 408 : Self {
2557 408 : last_set: AtomicU64::new(0),
2558 408 : gauge: per_timeline_gauge,
2559 408 : }
2560 408 : }
2561 1790 : pub(crate) fn set(&self, sz: u64) {
2562 1790 : self.gauge.set(sz);
2563 1790 : let prev = self.last_set.swap(sz, std::sync::atomic::Ordering::Relaxed);
2564 1790 : if sz < prev {
2565 26 : REMOTE_PHYSICAL_SIZE_GLOBAL.sub(prev - sz);
2566 1764 : } else {
2567 1764 : REMOTE_PHYSICAL_SIZE_GLOBAL.add(sz - prev);
2568 1764 : };
2569 1790 : }
2570 2 : pub(crate) fn get(&self) -> u64 {
2571 2 : self.gauge.get()
2572 2 : }
2573 : }
2574 :
2575 : impl Drop for PerTimelineRemotePhysicalSizeGauge {
2576 18 : fn drop(&mut self) {
2577 18 : REMOTE_PHYSICAL_SIZE_GLOBAL.sub(self.last_set.load(std::sync::atomic::Ordering::Relaxed));
2578 18 : }
2579 : }
2580 :
2581 : pub(crate) struct RemoteTimelineClientMetrics {
2582 : tenant_id: String,
2583 : shard_id: String,
2584 : timeline_id: String,
2585 : pub(crate) remote_physical_size_gauge: PerTimelineRemotePhysicalSizeGauge,
2586 : calls: Mutex<HashMap<(&'static str, &'static str), IntCounterPair>>,
2587 : bytes_started_counter: Mutex<HashMap<(&'static str, &'static str), IntCounter>>,
2588 : bytes_finished_counter: Mutex<HashMap<(&'static str, &'static str), IntCounter>>,
2589 : }
2590 :
2591 : impl RemoteTimelineClientMetrics {
2592 408 : pub fn new(tenant_shard_id: &TenantShardId, timeline_id: &TimelineId) -> Self {
2593 408 : let tenant_id_str = tenant_shard_id.tenant_id.to_string();
2594 408 : let shard_id_str = format!("{}", tenant_shard_id.shard_slug());
2595 408 : let timeline_id_str = timeline_id.to_string();
2596 408 :
2597 408 : let remote_physical_size_gauge = PerTimelineRemotePhysicalSizeGauge::new(
2598 408 : REMOTE_PHYSICAL_SIZE
2599 408 : .get_metric_with_label_values(&[&tenant_id_str, &shard_id_str, &timeline_id_str])
2600 408 : .unwrap(),
2601 408 : );
2602 408 :
2603 408 : RemoteTimelineClientMetrics {
2604 408 : tenant_id: tenant_id_str,
2605 408 : shard_id: shard_id_str,
2606 408 : timeline_id: timeline_id_str,
2607 408 : calls: Mutex::new(HashMap::default()),
2608 408 : bytes_started_counter: Mutex::new(HashMap::default()),
2609 408 : bytes_finished_counter: Mutex::new(HashMap::default()),
2610 408 : remote_physical_size_gauge,
2611 408 : }
2612 408 : }
2613 :
2614 2631 : pub fn remote_operation_time(
2615 2631 : &self,
2616 2631 : file_kind: &RemoteOpFileKind,
2617 2631 : op_kind: &RemoteOpKind,
2618 2631 : status: &'static str,
2619 2631 : ) -> Histogram {
2620 2631 : let key = (file_kind.as_str(), op_kind.as_str(), status);
2621 2631 : REMOTE_OPERATION_TIME
2622 2631 : .get_metric_with_label_values(&[key.0, key.1, key.2])
2623 2631 : .unwrap()
2624 2631 : }
2625 :
2626 6279 : fn calls_counter_pair(
2627 6279 : &self,
2628 6279 : file_kind: &RemoteOpFileKind,
2629 6279 : op_kind: &RemoteOpKind,
2630 6279 : ) -> IntCounterPair {
2631 6279 : let mut guard = self.calls.lock().unwrap();
2632 6279 : let key = (file_kind.as_str(), op_kind.as_str());
2633 6279 : let metric = guard.entry(key).or_insert_with(move || {
2634 714 : REMOTE_TIMELINE_CLIENT_CALLS
2635 714 : .get_metric_with_label_values(&[
2636 714 : &self.tenant_id,
2637 714 : &self.shard_id,
2638 714 : &self.timeline_id,
2639 714 : key.0,
2640 714 : key.1,
2641 714 : ])
2642 714 : .unwrap()
2643 6279 : });
2644 6279 : metric.clone()
2645 6279 : }
2646 :
2647 1516 : fn bytes_started_counter(
2648 1516 : &self,
2649 1516 : file_kind: &RemoteOpFileKind,
2650 1516 : op_kind: &RemoteOpKind,
2651 1516 : ) -> IntCounter {
2652 1516 : let mut guard = self.bytes_started_counter.lock().unwrap();
2653 1516 : let key = (file_kind.as_str(), op_kind.as_str());
2654 1516 : let metric = guard.entry(key).or_insert_with(move || {
2655 272 : REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER
2656 272 : .get_metric_with_label_values(&[
2657 272 : &self.tenant_id,
2658 272 : &self.shard_id,
2659 272 : &self.timeline_id,
2660 272 : key.0,
2661 272 : key.1,
2662 272 : ])
2663 272 : .unwrap()
2664 1516 : });
2665 1516 : metric.clone()
2666 1516 : }
2667 :
2668 2729 : fn bytes_finished_counter(
2669 2729 : &self,
2670 2729 : file_kind: &RemoteOpFileKind,
2671 2729 : op_kind: &RemoteOpKind,
2672 2729 : ) -> IntCounter {
2673 2729 : let mut guard = self.bytes_finished_counter.lock().unwrap();
2674 2729 : let key = (file_kind.as_str(), op_kind.as_str());
2675 2729 : let metric = guard.entry(key).or_insert_with(move || {
2676 272 : REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER
2677 272 : .get_metric_with_label_values(&[
2678 272 : &self.tenant_id,
2679 272 : &self.shard_id,
2680 272 : &self.timeline_id,
2681 272 : key.0,
2682 272 : key.1,
2683 272 : ])
2684 272 : .unwrap()
2685 2729 : });
2686 2729 : metric.clone()
2687 2729 : }
2688 : }
2689 :
2690 : #[cfg(test)]
2691 : impl RemoteTimelineClientMetrics {
2692 6 : pub fn get_bytes_started_counter_value(
2693 6 : &self,
2694 6 : file_kind: &RemoteOpFileKind,
2695 6 : op_kind: &RemoteOpKind,
2696 6 : ) -> Option<u64> {
2697 6 : let guard = self.bytes_started_counter.lock().unwrap();
2698 6 : let key = (file_kind.as_str(), op_kind.as_str());
2699 6 : guard.get(&key).map(|counter| counter.get())
2700 6 : }
2701 :
2702 6 : pub fn get_bytes_finished_counter_value(
2703 6 : &self,
2704 6 : file_kind: &RemoteOpFileKind,
2705 6 : op_kind: &RemoteOpKind,
2706 6 : ) -> Option<u64> {
2707 6 : let guard = self.bytes_finished_counter.lock().unwrap();
2708 6 : let key = (file_kind.as_str(), op_kind.as_str());
2709 6 : guard.get(&key).map(|counter| counter.get())
2710 6 : }
2711 : }
2712 :
2713 : /// See [`RemoteTimelineClientMetrics::call_begin`].
2714 : #[must_use]
2715 : pub(crate) struct RemoteTimelineClientCallMetricGuard {
2716 : /// Decremented on drop.
2717 : calls_counter_pair: Option<IntCounterPair>,
2718 : /// If Some(), this references the bytes_finished metric, and we increment it by the given `u64` on drop.
2719 : bytes_finished: Option<(IntCounter, u64)>,
2720 : }
2721 :
2722 : impl RemoteTimelineClientCallMetricGuard {
2723 : /// Consume this guard object without performing the metric updates it would do on `drop()`.
2724 : /// The caller vouches to do the metric updates manually.
2725 3412 : pub fn will_decrement_manually(mut self) {
2726 3412 : let RemoteTimelineClientCallMetricGuard {
2727 3412 : calls_counter_pair,
2728 3412 : bytes_finished,
2729 3412 : } = &mut self;
2730 3412 : calls_counter_pair.take();
2731 3412 : bytes_finished.take();
2732 3412 : }
2733 : }
2734 :
2735 : impl Drop for RemoteTimelineClientCallMetricGuard {
2736 3438 : fn drop(&mut self) {
2737 3438 : let RemoteTimelineClientCallMetricGuard {
2738 3438 : calls_counter_pair,
2739 3438 : bytes_finished,
2740 3438 : } = self;
2741 3438 : if let Some(guard) = calls_counter_pair.take() {
2742 26 : guard.dec();
2743 3412 : }
2744 3438 : if let Some((bytes_finished_metric, value)) = bytes_finished {
2745 0 : bytes_finished_metric.inc_by(*value);
2746 3438 : }
2747 3438 : }
2748 : }
2749 :
2750 : /// The enum variants communicate to the [`RemoteTimelineClientMetrics`] whether to
2751 : /// track the byte size of this call in applicable metric(s).
2752 : pub(crate) enum RemoteTimelineClientMetricsCallTrackSize {
2753 : /// Do not account for this call's byte size in any metrics.
2754 : /// The `reason` field is there to make the call sites self-documenting
2755 : /// about why they don't need the metric.
2756 : DontTrackSize { reason: &'static str },
2757 : /// Track the byte size of the call in applicable metric(s).
2758 : Bytes(u64),
2759 : }
2760 :
2761 : impl RemoteTimelineClientMetrics {
2762 : /// Update the metrics that change when a call to the remote timeline client instance starts.
2763 : ///
2764 : /// Drop the returned guard object once the operation is finished to updates corresponding metrics that track completions.
2765 : /// Or, use [`RemoteTimelineClientCallMetricGuard::will_decrement_manually`] and [`call_end`](Self::call_end) if that
2766 : /// is more suitable.
2767 : /// Never do both.
2768 3438 : pub(crate) fn call_begin(
2769 3438 : &self,
2770 3438 : file_kind: &RemoteOpFileKind,
2771 3438 : op_kind: &RemoteOpKind,
2772 3438 : size: RemoteTimelineClientMetricsCallTrackSize,
2773 3438 : ) -> RemoteTimelineClientCallMetricGuard {
2774 3438 : let calls_counter_pair = self.calls_counter_pair(file_kind, op_kind);
2775 3438 : calls_counter_pair.inc();
2776 :
2777 3438 : let bytes_finished = match size {
2778 1922 : RemoteTimelineClientMetricsCallTrackSize::DontTrackSize { reason: _reason } => {
2779 1922 : // nothing to do
2780 1922 : None
2781 : }
2782 1516 : RemoteTimelineClientMetricsCallTrackSize::Bytes(size) => {
2783 1516 : self.bytes_started_counter(file_kind, op_kind).inc_by(size);
2784 1516 : let finished_counter = self.bytes_finished_counter(file_kind, op_kind);
2785 1516 : Some((finished_counter, size))
2786 : }
2787 : };
2788 3438 : RemoteTimelineClientCallMetricGuard {
2789 3438 : calls_counter_pair: Some(calls_counter_pair),
2790 3438 : bytes_finished,
2791 3438 : }
2792 3438 : }
2793 :
2794 : /// Manually udpate the metrics that track completions, instead of using the guard object.
2795 : /// Using the guard object is generally preferable.
2796 : /// See [`call_begin`](Self::call_begin) for more context.
2797 2841 : pub(crate) fn call_end(
2798 2841 : &self,
2799 2841 : file_kind: &RemoteOpFileKind,
2800 2841 : op_kind: &RemoteOpKind,
2801 2841 : size: RemoteTimelineClientMetricsCallTrackSize,
2802 2841 : ) {
2803 2841 : let calls_counter_pair = self.calls_counter_pair(file_kind, op_kind);
2804 2841 : calls_counter_pair.dec();
2805 2841 : match size {
2806 1628 : RemoteTimelineClientMetricsCallTrackSize::DontTrackSize { reason: _reason } => {}
2807 1213 : RemoteTimelineClientMetricsCallTrackSize::Bytes(size) => {
2808 1213 : self.bytes_finished_counter(file_kind, op_kind).inc_by(size);
2809 1213 : }
2810 : }
2811 2841 : }
2812 : }
2813 :
2814 : impl Drop for RemoteTimelineClientMetrics {
2815 18 : fn drop(&mut self) {
2816 18 : let RemoteTimelineClientMetrics {
2817 18 : tenant_id,
2818 18 : shard_id,
2819 18 : timeline_id,
2820 18 : remote_physical_size_gauge,
2821 18 : calls,
2822 18 : bytes_started_counter,
2823 18 : bytes_finished_counter,
2824 18 : } = self;
2825 24 : for ((a, b), _) in calls.get_mut().unwrap().drain() {
2826 24 : let mut res = [Ok(()), Ok(())];
2827 24 : REMOTE_TIMELINE_CLIENT_CALLS
2828 24 : .remove_label_values(&mut res, &[tenant_id, shard_id, timeline_id, a, b]);
2829 24 : // don't care about results
2830 24 : }
2831 18 : for ((a, b), _) in bytes_started_counter.get_mut().unwrap().drain() {
2832 6 : let _ = REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER.remove_label_values(&[
2833 6 : tenant_id,
2834 6 : shard_id,
2835 6 : timeline_id,
2836 6 : a,
2837 6 : b,
2838 6 : ]);
2839 6 : }
2840 18 : for ((a, b), _) in bytes_finished_counter.get_mut().unwrap().drain() {
2841 6 : let _ = REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER.remove_label_values(&[
2842 6 : tenant_id,
2843 6 : shard_id,
2844 6 : timeline_id,
2845 6 : a,
2846 6 : b,
2847 6 : ]);
2848 6 : }
2849 18 : {
2850 18 : let _ = remote_physical_size_gauge; // use to avoid 'unused' warning in desctructuring above
2851 18 : let _ = REMOTE_PHYSICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2852 18 : }
2853 18 : }
2854 : }
2855 :
2856 : /// Wrapper future that measures the time spent by a remote storage operation,
2857 : /// and records the time and success/failure as a prometheus metric.
2858 : pub(crate) trait MeasureRemoteOp: Sized {
2859 2791 : fn measure_remote_op(
2860 2791 : self,
2861 2791 : file_kind: RemoteOpFileKind,
2862 2791 : op: RemoteOpKind,
2863 2791 : metrics: Arc<RemoteTimelineClientMetrics>,
2864 2791 : ) -> MeasuredRemoteOp<Self> {
2865 2791 : let start = Instant::now();
2866 2791 : MeasuredRemoteOp {
2867 2791 : inner: self,
2868 2791 : file_kind,
2869 2791 : op,
2870 2791 : start,
2871 2791 : metrics,
2872 2791 : }
2873 2791 : }
2874 : }
2875 :
2876 : impl<T: Sized> MeasureRemoteOp for T {}
2877 :
2878 : pin_project! {
2879 : pub(crate) struct MeasuredRemoteOp<F>
2880 : {
2881 : #[pin]
2882 : inner: F,
2883 : file_kind: RemoteOpFileKind,
2884 : op: RemoteOpKind,
2885 : start: Instant,
2886 : metrics: Arc<RemoteTimelineClientMetrics>,
2887 : }
2888 : }
2889 :
2890 : impl<F: Future<Output = Result<O, E>>, O, E> Future for MeasuredRemoteOp<F> {
2891 : type Output = Result<O, E>;
2892 :
2893 44254 : fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
2894 44254 : let this = self.project();
2895 44254 : let poll_result = this.inner.poll(cx);
2896 44254 : if let Poll::Ready(ref res) = poll_result {
2897 2631 : let duration = this.start.elapsed();
2898 2631 : let status = if res.is_ok() { &"success" } else { &"failure" };
2899 2631 : this.metrics
2900 2631 : .remote_operation_time(this.file_kind, this.op, status)
2901 2631 : .observe(duration.as_secs_f64());
2902 41623 : }
2903 44254 : poll_result
2904 44254 : }
2905 : }
2906 :
2907 : pub mod tokio_epoll_uring {
2908 : use metrics::{register_int_counter, UIntGauge};
2909 : use once_cell::sync::Lazy;
2910 :
2911 : pub struct Collector {
2912 : descs: Vec<metrics::core::Desc>,
2913 : systems_created: UIntGauge,
2914 : systems_destroyed: UIntGauge,
2915 : }
2916 :
2917 : impl metrics::core::Collector for Collector {
2918 0 : fn desc(&self) -> Vec<&metrics::core::Desc> {
2919 0 : self.descs.iter().collect()
2920 0 : }
2921 :
2922 0 : fn collect(&self) -> Vec<metrics::proto::MetricFamily> {
2923 0 : let mut mfs = Vec::with_capacity(Self::NMETRICS);
2924 0 : let tokio_epoll_uring::metrics::Metrics {
2925 0 : systems_created,
2926 0 : systems_destroyed,
2927 0 : } = tokio_epoll_uring::metrics::global();
2928 0 : self.systems_created.set(systems_created);
2929 0 : mfs.extend(self.systems_created.collect());
2930 0 : self.systems_destroyed.set(systems_destroyed);
2931 0 : mfs.extend(self.systems_destroyed.collect());
2932 0 : mfs
2933 0 : }
2934 : }
2935 :
2936 : impl Collector {
2937 : const NMETRICS: usize = 2;
2938 :
2939 : #[allow(clippy::new_without_default)]
2940 0 : pub fn new() -> Self {
2941 0 : let mut descs = Vec::new();
2942 0 :
2943 0 : let systems_created = UIntGauge::new(
2944 0 : "pageserver_tokio_epoll_uring_systems_created",
2945 0 : "counter of tokio-epoll-uring systems that were created",
2946 0 : )
2947 0 : .unwrap();
2948 0 : descs.extend(
2949 0 : metrics::core::Collector::desc(&systems_created)
2950 0 : .into_iter()
2951 0 : .cloned(),
2952 0 : );
2953 0 :
2954 0 : let systems_destroyed = UIntGauge::new(
2955 0 : "pageserver_tokio_epoll_uring_systems_destroyed",
2956 0 : "counter of tokio-epoll-uring systems that were destroyed",
2957 0 : )
2958 0 : .unwrap();
2959 0 : descs.extend(
2960 0 : metrics::core::Collector::desc(&systems_destroyed)
2961 0 : .into_iter()
2962 0 : .cloned(),
2963 0 : );
2964 0 :
2965 0 : Self {
2966 0 : descs,
2967 0 : systems_created,
2968 0 : systems_destroyed,
2969 0 : }
2970 0 : }
2971 : }
2972 :
2973 90 : pub(crate) static THREAD_LOCAL_LAUNCH_SUCCESSES: Lazy<metrics::IntCounter> = Lazy::new(|| {
2974 : register_int_counter!(
2975 : "pageserver_tokio_epoll_uring_pageserver_thread_local_launch_success_count",
2976 : "Number of times where thread_local_system creation spanned multiple executor threads",
2977 : )
2978 90 : .unwrap()
2979 90 : });
2980 :
2981 0 : pub(crate) static THREAD_LOCAL_LAUNCH_FAILURES: Lazy<metrics::IntCounter> = Lazy::new(|| {
2982 : register_int_counter!(
2983 : "pageserver_tokio_epoll_uring_pageserver_thread_local_launch_failures_count",
2984 : "Number of times thread_local_system creation failed and was retried after back-off.",
2985 : )
2986 0 : .unwrap()
2987 0 : });
2988 : }
2989 :
2990 : pub(crate) mod tenant_throttling {
2991 : use metrics::{register_int_counter_vec, IntCounter};
2992 : use once_cell::sync::Lazy;
2993 :
2994 : use crate::tenant::{self, throttle::Metric};
2995 :
2996 : pub(crate) struct TimelineGet {
2997 : wait_time: IntCounter,
2998 : count: IntCounter,
2999 : }
3000 :
3001 156 : pub(crate) static TIMELINE_GET: Lazy<TimelineGet> = Lazy::new(|| {
3002 156 : static WAIT_USECS: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
3003 156 : register_int_counter_vec!(
3004 156 : "pageserver_tenant_throttling_wait_usecs_sum_global",
3005 156 : "Sum of microseconds that tenants spent waiting for a tenant throttle of a given kind.",
3006 156 : &["kind"]
3007 156 : )
3008 156 : .unwrap()
3009 156 : });
3010 156 :
3011 156 : static WAIT_COUNT: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
3012 156 : register_int_counter_vec!(
3013 156 : "pageserver_tenant_throttling_count_global",
3014 156 : "Count of tenant throttlings, by kind of throttle.",
3015 156 : &["kind"]
3016 156 : )
3017 156 : .unwrap()
3018 156 : });
3019 156 :
3020 156 : let kind = "timeline_get";
3021 156 : TimelineGet {
3022 156 : wait_time: WAIT_USECS.with_label_values(&[kind]),
3023 156 : count: WAIT_COUNT.with_label_values(&[kind]),
3024 156 : }
3025 156 : });
3026 :
3027 : impl Metric for &'static TimelineGet {
3028 : #[inline(always)]
3029 0 : fn observe_throttling(
3030 0 : &self,
3031 0 : tenant::throttle::Observation { wait_time }: &tenant::throttle::Observation,
3032 0 : ) {
3033 0 : let val = u64::try_from(wait_time.as_micros()).unwrap();
3034 0 : self.wait_time.inc_by(val);
3035 0 : self.count.inc();
3036 0 : }
3037 : }
3038 : }
3039 :
3040 : pub(crate) mod disk_usage_based_eviction {
3041 : use super::*;
3042 :
3043 : pub(crate) struct Metrics {
3044 : pub(crate) tenant_collection_time: Histogram,
3045 : pub(crate) tenant_layer_count: Histogram,
3046 : pub(crate) layers_collected: IntCounter,
3047 : pub(crate) layers_selected: IntCounter,
3048 : pub(crate) layers_evicted: IntCounter,
3049 : }
3050 :
3051 : impl Default for Metrics {
3052 0 : fn default() -> Self {
3053 0 : let tenant_collection_time = register_histogram!(
3054 : "pageserver_disk_usage_based_eviction_tenant_collection_seconds",
3055 : "Time spent collecting layers from a tenant -- not normalized by collected layer amount",
3056 : vec![0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0, 10.0]
3057 : )
3058 0 : .unwrap();
3059 0 :
3060 0 : let tenant_layer_count = register_histogram!(
3061 : "pageserver_disk_usage_based_eviction_tenant_collected_layers",
3062 : "Amount of layers gathered from a tenant",
3063 : vec![5.0, 50.0, 500.0, 5000.0, 50000.0]
3064 : )
3065 0 : .unwrap();
3066 0 :
3067 0 : let layers_collected = register_int_counter!(
3068 : "pageserver_disk_usage_based_eviction_collected_layers_total",
3069 : "Amount of layers collected"
3070 : )
3071 0 : .unwrap();
3072 0 :
3073 0 : let layers_selected = register_int_counter!(
3074 : "pageserver_disk_usage_based_eviction_select_layers_total",
3075 : "Amount of layers selected"
3076 : )
3077 0 : .unwrap();
3078 0 :
3079 0 : let layers_evicted = register_int_counter!(
3080 : "pageserver_disk_usage_based_eviction_evicted_layers_total",
3081 : "Amount of layers successfully evicted"
3082 : )
3083 0 : .unwrap();
3084 0 :
3085 0 : Self {
3086 0 : tenant_collection_time,
3087 0 : tenant_layer_count,
3088 0 : layers_collected,
3089 0 : layers_selected,
3090 0 : layers_evicted,
3091 0 : }
3092 0 : }
3093 : }
3094 :
3095 : pub(crate) static METRICS: Lazy<Metrics> = Lazy::new(Metrics::default);
3096 : }
3097 :
3098 150 : static TOKIO_EXECUTOR_THREAD_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
3099 : register_uint_gauge_vec!(
3100 : "pageserver_tokio_executor_thread_configured_count",
3101 : "Total number of configued tokio executor threads in the process.
3102 : The `setup` label denotes whether we're running with multiple runtimes or a single runtime.",
3103 : &["setup"],
3104 : )
3105 150 : .unwrap()
3106 150 : });
3107 :
3108 150 : pub(crate) fn set_tokio_runtime_setup(setup: &str, num_threads: NonZeroUsize) {
3109 150 : static SERIALIZE: std::sync::Mutex<()> = std::sync::Mutex::new(());
3110 150 : let _guard = SERIALIZE.lock().unwrap();
3111 150 : TOKIO_EXECUTOR_THREAD_COUNT.reset();
3112 150 : TOKIO_EXECUTOR_THREAD_COUNT
3113 150 : .get_metric_with_label_values(&[setup])
3114 150 : .unwrap()
3115 150 : .set(u64::try_from(num_threads.get()).unwrap());
3116 150 : }
3117 :
3118 0 : pub fn preinitialize_metrics() {
3119 0 : // Python tests need these and on some we do alerting.
3120 0 : //
3121 0 : // FIXME(4813): make it so that we have no top level metrics as this fn will easily fall out of
3122 0 : // order:
3123 0 : // - global metrics reside in a Lazy<PageserverMetrics>
3124 0 : // - access via crate::metrics::PS_METRICS.some_metric.inc()
3125 0 : // - could move the statics into TimelineMetrics::new()?
3126 0 :
3127 0 : // counters
3128 0 : [
3129 0 : &UNEXPECTED_ONDEMAND_DOWNLOADS,
3130 0 : &WALRECEIVER_STARTED_CONNECTIONS,
3131 0 : &WALRECEIVER_BROKER_UPDATES,
3132 0 : &WALRECEIVER_CANDIDATES_ADDED,
3133 0 : &WALRECEIVER_CANDIDATES_REMOVED,
3134 0 : &tokio_epoll_uring::THREAD_LOCAL_LAUNCH_FAILURES,
3135 0 : &tokio_epoll_uring::THREAD_LOCAL_LAUNCH_SUCCESSES,
3136 0 : &REMOTE_ONDEMAND_DOWNLOADED_LAYERS,
3137 0 : &REMOTE_ONDEMAND_DOWNLOADED_BYTES,
3138 0 : &CIRCUIT_BREAKERS_BROKEN,
3139 0 : &CIRCUIT_BREAKERS_UNBROKEN,
3140 0 : ]
3141 0 : .into_iter()
3142 0 : .for_each(|c| {
3143 0 : Lazy::force(c);
3144 0 : });
3145 0 :
3146 0 : // Deletion queue stats
3147 0 : Lazy::force(&DELETION_QUEUE);
3148 0 :
3149 0 : // Tenant stats
3150 0 : Lazy::force(&TENANT);
3151 0 :
3152 0 : // Tenant manager stats
3153 0 : Lazy::force(&TENANT_MANAGER);
3154 0 :
3155 0 : Lazy::force(&crate::tenant::storage_layer::layer::LAYER_IMPL_METRICS);
3156 0 : Lazy::force(&disk_usage_based_eviction::METRICS);
3157 :
3158 0 : for state_name in pageserver_api::models::TenantState::VARIANTS {
3159 0 : // initialize the metric for all gauges, otherwise the time series might seemingly show
3160 0 : // values from last restart.
3161 0 : TENANT_STATE_METRIC.with_label_values(&[state_name]).set(0);
3162 0 : }
3163 :
3164 : // countervecs
3165 0 : [&BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT]
3166 0 : .into_iter()
3167 0 : .for_each(|c| {
3168 0 : Lazy::force(c);
3169 0 : });
3170 0 :
3171 0 : // gauges
3172 0 : WALRECEIVER_ACTIVE_MANAGERS.get();
3173 0 :
3174 0 : // histograms
3175 0 : [
3176 0 : &READ_NUM_LAYERS_VISITED,
3177 0 : &VEC_READ_NUM_LAYERS_VISITED,
3178 0 : &WAIT_LSN_TIME,
3179 0 : &WAL_REDO_TIME,
3180 0 : &WAL_REDO_RECORDS_HISTOGRAM,
3181 0 : &WAL_REDO_BYTES_HISTOGRAM,
3182 0 : &WAL_REDO_PROCESS_LAUNCH_DURATION_HISTOGRAM,
3183 0 : ]
3184 0 : .into_iter()
3185 0 : .for_each(|h| {
3186 0 : Lazy::force(h);
3187 0 : });
3188 0 :
3189 0 : // Custom
3190 0 : Lazy::force(&RECONSTRUCT_TIME);
3191 0 : Lazy::force(&tenant_throttling::TIMELINE_GET);
3192 0 : Lazy::force(&BASEBACKUP_QUERY_TIME);
3193 0 : Lazy::force(&COMPUTE_COMMANDS_COUNTERS);
3194 0 : }
|
