Line data Source code
1 : use enum_map::EnumMap;
2 : use metrics::{
3 : register_counter_vec, register_gauge_vec, register_histogram, register_histogram_vec,
4 : register_int_counter, register_int_counter_pair_vec, register_int_counter_vec,
5 : register_int_gauge, register_int_gauge_vec, register_uint_gauge, register_uint_gauge_vec,
6 : Counter, CounterVec, GaugeVec, Histogram, HistogramVec, IntCounter, IntCounterPair,
7 : IntCounterPairVec, IntCounterVec, IntGauge, IntGaugeVec, UIntGauge, UIntGaugeVec,
8 : };
9 : use once_cell::sync::Lazy;
10 : use pageserver_api::shard::TenantShardId;
11 : use strum::{EnumCount, IntoEnumIterator, VariantNames};
12 : use strum_macros::{EnumVariantNames, IntoStaticStr};
13 : use tracing::warn;
14 : use utils::id::TimelineId;
15 :
16 : /// Prometheus histogram buckets (in seconds) for operations in the critical
17 : /// path. In other words, operations that directly affect that latency of user
18 : /// queries.
19 : ///
20 : /// The buckets capture the majority of latencies in the microsecond and
21 : /// millisecond range but also extend far enough up to distinguish "bad" from
22 : /// "really bad".
23 : const CRITICAL_OP_BUCKETS: &[f64] = &[
24 : 0.000_001, 0.000_010, 0.000_100, // 1 us, 10 us, 100 us
25 : 0.001_000, 0.010_000, 0.100_000, // 1 ms, 10 ms, 100 ms
26 : 1.0, 10.0, 100.0, // 1 s, 10 s, 100 s
27 : ];
28 :
29 : // Metrics collected on operations on the storage repository.
30 3072 : #[derive(Debug, EnumVariantNames, IntoStaticStr)]
31 : #[strum(serialize_all = "kebab_case")]
32 : pub(crate) enum StorageTimeOperation {
33 : #[strum(serialize = "layer flush")]
34 : LayerFlush,
35 :
36 : #[strum(serialize = "compact")]
37 : Compact,
38 :
39 : #[strum(serialize = "create images")]
40 : CreateImages,
41 :
42 : #[strum(serialize = "logical size")]
43 : LogicalSize,
44 :
45 : #[strum(serialize = "imitate logical size")]
46 : ImitateLogicalSize,
47 :
48 : #[strum(serialize = "load layer map")]
49 : LoadLayerMap,
50 :
51 : #[strum(serialize = "gc")]
52 : Gc,
53 :
54 : #[strum(serialize = "find gc cutoffs")]
55 : FindGcCutoffs,
56 : }
57 :
58 142 : pub(crate) static STORAGE_TIME_SUM_PER_TIMELINE: Lazy<CounterVec> = Lazy::new(|| {
59 : register_counter_vec!(
60 : "pageserver_storage_operations_seconds_sum",
61 : "Total time spent on storage operations with operation, tenant and timeline dimensions",
62 : &["operation", "tenant_id", "shard_id", "timeline_id"],
63 : )
64 142 : .expect("failed to define a metric")
65 142 : });
66 :
67 142 : pub(crate) static STORAGE_TIME_COUNT_PER_TIMELINE: Lazy<IntCounterVec> = Lazy::new(|| {
68 : register_int_counter_vec!(
69 : "pageserver_storage_operations_seconds_count",
70 : "Count of storage operations with operation, tenant and timeline dimensions",
71 : &["operation", "tenant_id", "shard_id", "timeline_id"],
72 : )
73 142 : .expect("failed to define a metric")
74 142 : });
75 :
76 : // Buckets for background operations like compaction, GC, size calculation
77 : const STORAGE_OP_BUCKETS: &[f64] = &[0.010, 0.100, 1.0, 10.0, 100.0, 1000.0];
78 :
79 142 : pub(crate) static STORAGE_TIME_GLOBAL: Lazy<HistogramVec> = Lazy::new(|| {
80 : register_histogram_vec!(
81 : "pageserver_storage_operations_seconds_global",
82 : "Time spent on storage operations",
83 : &["operation"],
84 : STORAGE_OP_BUCKETS.into(),
85 : )
86 142 : .expect("failed to define a metric")
87 142 : });
88 :
89 140 : pub(crate) static READ_NUM_LAYERS_VISITED: Lazy<Histogram> = Lazy::new(|| {
90 : register_histogram!(
91 : "pageserver_layers_visited_per_read_global",
92 : "Number of layers visited to reconstruct one key",
93 : vec![1.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0],
94 : )
95 140 : .expect("failed to define a metric")
96 140 : });
97 :
98 24 : pub(crate) static VEC_READ_NUM_LAYERS_VISITED: Lazy<Histogram> = Lazy::new(|| {
99 : register_histogram!(
100 : "pageserver_layers_visited_per_vectored_read_global",
101 : "Average number of layers visited to reconstruct one key",
102 : vec![1.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0],
103 : )
104 24 : .expect("failed to define a metric")
105 24 : });
106 :
107 : // Metrics collected on operations on the storage repository.
108 : #[derive(
109 560 : Clone, Copy, enum_map::Enum, strum_macros::EnumString, strum_macros::Display, IntoStaticStr,
110 : )]
111 : pub(crate) enum GetKind {
112 : Singular,
113 : Vectored,
114 : }
115 :
116 : pub(crate) struct ReconstructTimeMetrics {
117 : singular: Histogram,
118 : vectored: Histogram,
119 : }
120 :
121 140 : pub(crate) static RECONSTRUCT_TIME: Lazy<ReconstructTimeMetrics> = Lazy::new(|| {
122 140 : let inner = register_histogram_vec!(
123 : "pageserver_getpage_reconstruct_seconds",
124 : "Time spent in reconstruct_value (reconstruct a page from deltas)",
125 : &["get_kind"],
126 : CRITICAL_OP_BUCKETS.into(),
127 : )
128 140 : .expect("failed to define a metric");
129 140 :
130 140 : ReconstructTimeMetrics {
131 140 : singular: inner.with_label_values(&[GetKind::Singular.into()]),
132 140 : vectored: inner.with_label_values(&[GetKind::Vectored.into()]),
133 140 : }
134 140 : });
135 :
136 : impl ReconstructTimeMetrics {
137 625906 : pub(crate) fn for_get_kind(&self, get_kind: GetKind) -> &Histogram {
138 625906 : match get_kind {
139 625836 : GetKind::Singular => &self.singular,
140 70 : GetKind::Vectored => &self.vectored,
141 : }
142 625906 : }
143 : }
144 :
145 : pub(crate) struct ReconstructDataTimeMetrics {
146 : singular: Histogram,
147 : vectored: Histogram,
148 : }
149 :
150 : impl ReconstructDataTimeMetrics {
151 626046 : pub(crate) fn for_get_kind(&self, get_kind: GetKind) -> &Histogram {
152 626046 : match get_kind {
153 625976 : GetKind::Singular => &self.singular,
154 70 : GetKind::Vectored => &self.vectored,
155 : }
156 626046 : }
157 : }
158 :
159 140 : pub(crate) static GET_RECONSTRUCT_DATA_TIME: Lazy<ReconstructDataTimeMetrics> = Lazy::new(|| {
160 140 : let inner = register_histogram_vec!(
161 : "pageserver_getpage_get_reconstruct_data_seconds",
162 : "Time spent in get_reconstruct_value_data",
163 : &["get_kind"],
164 : CRITICAL_OP_BUCKETS.into(),
165 : )
166 140 : .expect("failed to define a metric");
167 140 :
168 140 : ReconstructDataTimeMetrics {
169 140 : singular: inner.with_label_values(&[GetKind::Singular.into()]),
170 140 : vectored: inner.with_label_values(&[GetKind::Vectored.into()]),
171 140 : }
172 140 : });
173 :
174 : pub(crate) struct GetVectoredLatency {
175 : map: EnumMap<TaskKind, Option<Histogram>>,
176 : }
177 :
178 : #[allow(dead_code)]
179 : pub(crate) struct ScanLatency {
180 : map: EnumMap<TaskKind, Option<Histogram>>,
181 : }
182 :
183 : impl GetVectoredLatency {
184 : // Only these task types perform vectored gets. Filter all other tasks out to reduce total
185 : // cardinality of the metric.
186 : const TRACKED_TASK_KINDS: [TaskKind; 2] = [TaskKind::Compaction, TaskKind::PageRequestHandler];
187 :
188 964 : pub(crate) fn for_task_kind(&self, task_kind: TaskKind) -> Option<&Histogram> {
189 964 : self.map[task_kind].as_ref()
190 964 : }
191 : }
192 :
193 : impl ScanLatency {
194 : // Only these task types perform vectored gets. Filter all other tasks out to reduce total
195 : // cardinality of the metric.
196 : const TRACKED_TASK_KINDS: [TaskKind; 1] = [TaskKind::PageRequestHandler];
197 :
198 12 : pub(crate) fn for_task_kind(&self, task_kind: TaskKind) -> Option<&Histogram> {
199 12 : self.map[task_kind].as_ref()
200 12 : }
201 : }
202 :
203 : pub(crate) struct ScanLatencyOngoingRecording<'a> {
204 : parent: &'a Histogram,
205 : start: std::time::Instant,
206 : }
207 :
208 : impl<'a> ScanLatencyOngoingRecording<'a> {
209 0 : pub(crate) fn start_recording(parent: &'a Histogram) -> ScanLatencyOngoingRecording<'a> {
210 0 : let start = Instant::now();
211 0 : ScanLatencyOngoingRecording { parent, start }
212 0 : }
213 :
214 0 : pub(crate) fn observe(self, throttled: Option<Duration>) {
215 0 : let elapsed = self.start.elapsed();
216 0 : let ex_throttled = if let Some(throttled) = throttled {
217 0 : elapsed.checked_sub(throttled)
218 : } else {
219 0 : Some(elapsed)
220 : };
221 0 : if let Some(ex_throttled) = ex_throttled {
222 0 : self.parent.observe(ex_throttled.as_secs_f64());
223 0 : } else {
224 0 : use utils::rate_limit::RateLimit;
225 0 : static LOGGED: Lazy<Mutex<RateLimit>> =
226 0 : Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(10))));
227 0 : let mut rate_limit = LOGGED.lock().unwrap();
228 0 : rate_limit.call(|| {
229 0 : warn!("error deducting time spent throttled; this message is logged at a global rate limit");
230 0 : });
231 0 : }
232 0 : }
233 : }
234 :
235 134 : pub(crate) static GET_VECTORED_LATENCY: Lazy<GetVectoredLatency> = Lazy::new(|| {
236 134 : let inner = register_histogram_vec!(
237 : "pageserver_get_vectored_seconds",
238 : "Time spent in get_vectored, excluding time spent in timeline_get_throttle.",
239 : &["task_kind"],
240 : CRITICAL_OP_BUCKETS.into(),
241 : )
242 134 : .expect("failed to define a metric");
243 134 :
244 134 : GetVectoredLatency {
245 4020 : map: EnumMap::from_array(std::array::from_fn(|task_kind_idx| {
246 4020 : let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind_idx);
247 4020 :
248 4020 : if GetVectoredLatency::TRACKED_TASK_KINDS.contains(&task_kind) {
249 268 : let task_kind = task_kind.into();
250 268 : Some(inner.with_label_values(&[task_kind]))
251 : } else {
252 3752 : None
253 : }
254 4020 : })),
255 134 : }
256 134 : });
257 :
258 4 : pub(crate) static SCAN_LATENCY: Lazy<ScanLatency> = Lazy::new(|| {
259 4 : let inner = register_histogram_vec!(
260 : "pageserver_scan_seconds",
261 : "Time spent in scan, excluding time spent in timeline_get_throttle.",
262 : &["task_kind"],
263 : CRITICAL_OP_BUCKETS.into(),
264 : )
265 4 : .expect("failed to define a metric");
266 4 :
267 4 : ScanLatency {
268 120 : map: EnumMap::from_array(std::array::from_fn(|task_kind_idx| {
269 120 : let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind_idx);
270 120 :
271 120 : if ScanLatency::TRACKED_TASK_KINDS.contains(&task_kind) {
272 4 : let task_kind = task_kind.into();
273 4 : Some(inner.with_label_values(&[task_kind]))
274 : } else {
275 116 : None
276 : }
277 120 : })),
278 4 : }
279 4 : });
280 :
281 : pub(crate) struct PageCacheMetricsForTaskKind {
282 : pub read_accesses_immutable: IntCounter,
283 : pub read_hits_immutable: IntCounter,
284 : }
285 :
286 : pub(crate) struct PageCacheMetrics {
287 : map: EnumMap<TaskKind, EnumMap<PageContentKind, PageCacheMetricsForTaskKind>>,
288 : }
289 :
290 76 : static PAGE_CACHE_READ_HITS: Lazy<IntCounterVec> = Lazy::new(|| {
291 : register_int_counter_vec!(
292 : "pageserver_page_cache_read_hits_total",
293 : "Number of read accesses to the page cache that hit",
294 : &["task_kind", "key_kind", "content_kind", "hit_kind"]
295 : )
296 76 : .expect("failed to define a metric")
297 76 : });
298 :
299 76 : static PAGE_CACHE_READ_ACCESSES: Lazy<IntCounterVec> = Lazy::new(|| {
300 : register_int_counter_vec!(
301 : "pageserver_page_cache_read_accesses_total",
302 : "Number of read accesses to the page cache",
303 : &["task_kind", "key_kind", "content_kind"]
304 : )
305 76 : .expect("failed to define a metric")
306 76 : });
307 :
308 76 : pub(crate) static PAGE_CACHE: Lazy<PageCacheMetrics> = Lazy::new(|| PageCacheMetrics {
309 2280 : map: EnumMap::from_array(std::array::from_fn(|task_kind| {
310 2280 : let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind);
311 2280 : let task_kind: &'static str = task_kind.into();
312 13680 : EnumMap::from_array(std::array::from_fn(|content_kind| {
313 13680 : let content_kind = <PageContentKind as enum_map::Enum>::from_usize(content_kind);
314 13680 : let content_kind: &'static str = content_kind.into();
315 13680 : PageCacheMetricsForTaskKind {
316 13680 : read_accesses_immutable: {
317 13680 : PAGE_CACHE_READ_ACCESSES
318 13680 : .get_metric_with_label_values(&[task_kind, "immutable", content_kind])
319 13680 : .unwrap()
320 13680 : },
321 13680 :
322 13680 : read_hits_immutable: {
323 13680 : PAGE_CACHE_READ_HITS
324 13680 : .get_metric_with_label_values(&[task_kind, "immutable", content_kind, "-"])
325 13680 : .unwrap()
326 13680 : },
327 13680 : }
328 13680 : }))
329 2280 : })),
330 76 : });
331 :
332 : impl PageCacheMetrics {
333 14191614 : pub(crate) fn for_ctx(&self, ctx: &RequestContext) -> &PageCacheMetricsForTaskKind {
334 14191614 : &self.map[ctx.task_kind()][ctx.page_content_kind()]
335 14191614 : }
336 : }
337 :
338 : pub(crate) struct PageCacheSizeMetrics {
339 : pub max_bytes: UIntGauge,
340 :
341 : pub current_bytes_immutable: UIntGauge,
342 : }
343 :
344 76 : static PAGE_CACHE_SIZE_CURRENT_BYTES: Lazy<UIntGaugeVec> = Lazy::new(|| {
345 : register_uint_gauge_vec!(
346 : "pageserver_page_cache_size_current_bytes",
347 : "Current size of the page cache in bytes, by key kind",
348 : &["key_kind"]
349 : )
350 76 : .expect("failed to define a metric")
351 76 : });
352 :
353 : pub(crate) static PAGE_CACHE_SIZE: Lazy<PageCacheSizeMetrics> =
354 76 : Lazy::new(|| PageCacheSizeMetrics {
355 : max_bytes: {
356 : register_uint_gauge!(
357 : "pageserver_page_cache_size_max_bytes",
358 : "Maximum size of the page cache in bytes"
359 : )
360 76 : .expect("failed to define a metric")
361 76 : },
362 76 : current_bytes_immutable: {
363 76 : PAGE_CACHE_SIZE_CURRENT_BYTES
364 76 : .get_metric_with_label_values(&["immutable"])
365 76 : .unwrap()
366 76 : },
367 76 : });
368 :
369 : pub(crate) mod page_cache_eviction_metrics {
370 : use std::num::NonZeroUsize;
371 :
372 : use metrics::{register_int_counter_vec, IntCounter, IntCounterVec};
373 : use once_cell::sync::Lazy;
374 :
375 : #[derive(Clone, Copy)]
376 : pub(crate) enum Outcome {
377 : FoundSlotUnused { iters: NonZeroUsize },
378 : FoundSlotEvicted { iters: NonZeroUsize },
379 : ItersExceeded { iters: NonZeroUsize },
380 : }
381 :
382 76 : static ITERS_TOTAL_VEC: Lazy<IntCounterVec> = Lazy::new(|| {
383 : register_int_counter_vec!(
384 : "pageserver_page_cache_find_victim_iters_total",
385 : "Counter for the number of iterations in the find_victim loop",
386 : &["outcome"],
387 : )
388 76 : .expect("failed to define a metric")
389 76 : });
390 :
391 76 : static CALLS_VEC: Lazy<IntCounterVec> = Lazy::new(|| {
392 : register_int_counter_vec!(
393 : "pageserver_page_cache_find_victim_calls",
394 : "Incremented at the end of each find_victim() call.\
395 : Filter by outcome to get e.g., eviction rate.",
396 : &["outcome"]
397 : )
398 76 : .unwrap()
399 76 : });
400 :
401 166859 : pub(crate) fn observe(outcome: Outcome) {
402 166859 : macro_rules! dry {
403 166859 : ($label:literal, $iters:expr) => {{
404 166859 : static LABEL: &'static str = $label;
405 166859 : static ITERS_TOTAL: Lazy<IntCounter> =
406 166859 : Lazy::new(|| ITERS_TOTAL_VEC.with_label_values(&[LABEL]));
407 166859 : static CALLS: Lazy<IntCounter> =
408 166859 : Lazy::new(|| CALLS_VEC.with_label_values(&[LABEL]));
409 166859 : ITERS_TOTAL.inc_by(($iters.get()) as u64);
410 166859 : CALLS.inc();
411 166859 : }};
412 166859 : }
413 166859 : match outcome {
414 1900 : Outcome::FoundSlotUnused { iters } => dry!("found_empty", iters),
415 164959 : Outcome::FoundSlotEvicted { iters } => {
416 164959 : dry!("found_evicted", iters)
417 : }
418 0 : Outcome::ItersExceeded { iters } => {
419 0 : dry!("err_iters_exceeded", iters);
420 0 : super::page_cache_errors_inc(super::PageCacheErrorKind::EvictIterLimit);
421 0 : }
422 : }
423 166859 : }
424 : }
425 :
426 0 : static PAGE_CACHE_ERRORS: Lazy<IntCounterVec> = Lazy::new(|| {
427 : register_int_counter_vec!(
428 : "page_cache_errors_total",
429 : "Number of timeouts while acquiring a pinned slot in the page cache",
430 : &["error_kind"]
431 : )
432 0 : .expect("failed to define a metric")
433 0 : });
434 :
435 0 : #[derive(IntoStaticStr)]
436 : #[strum(serialize_all = "kebab_case")]
437 : pub(crate) enum PageCacheErrorKind {
438 : AcquirePinnedSlotTimeout,
439 : EvictIterLimit,
440 : }
441 :
442 0 : pub(crate) fn page_cache_errors_inc(error_kind: PageCacheErrorKind) {
443 0 : PAGE_CACHE_ERRORS
444 0 : .get_metric_with_label_values(&[error_kind.into()])
445 0 : .unwrap()
446 0 : .inc();
447 0 : }
448 :
449 16 : pub(crate) static WAIT_LSN_TIME: Lazy<Histogram> = Lazy::new(|| {
450 : register_histogram!(
451 : "pageserver_wait_lsn_seconds",
452 : "Time spent waiting for WAL to arrive",
453 : CRITICAL_OP_BUCKETS.into(),
454 : )
455 16 : .expect("failed to define a metric")
456 16 : });
457 :
458 142 : static LAST_RECORD_LSN: Lazy<IntGaugeVec> = Lazy::new(|| {
459 : register_int_gauge_vec!(
460 : "pageserver_last_record_lsn",
461 : "Last record LSN grouped by timeline",
462 : &["tenant_id", "shard_id", "timeline_id"]
463 : )
464 142 : .expect("failed to define a metric")
465 142 : });
466 :
467 142 : static STANDBY_HORIZON: Lazy<IntGaugeVec> = Lazy::new(|| {
468 : register_int_gauge_vec!(
469 : "pageserver_standby_horizon",
470 : "Standby apply LSN for which GC is hold off, by timeline.",
471 : &["tenant_id", "shard_id", "timeline_id"]
472 : )
473 142 : .expect("failed to define a metric")
474 142 : });
475 :
476 142 : static RESIDENT_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
477 : register_uint_gauge_vec!(
478 : "pageserver_resident_physical_size",
479 : "The size of the layer files present in the pageserver's filesystem, for attached locations.",
480 : &["tenant_id", "shard_id", "timeline_id"]
481 : )
482 142 : .expect("failed to define a metric")
483 142 : });
484 :
485 138 : pub(crate) static RESIDENT_PHYSICAL_SIZE_GLOBAL: Lazy<UIntGauge> = Lazy::new(|| {
486 : register_uint_gauge!(
487 : "pageserver_resident_physical_size_global",
488 : "Like `pageserver_resident_physical_size`, but without tenant/timeline dimensions."
489 : )
490 138 : .expect("failed to define a metric")
491 138 : });
492 :
493 142 : static REMOTE_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
494 : register_uint_gauge_vec!(
495 : "pageserver_remote_physical_size",
496 : "The size of the layer files present in the remote storage that are listed in the remote index_part.json.",
497 : // Corollary: If any files are missing from the index part, they won't be included here.
498 : &["tenant_id", "shard_id", "timeline_id"]
499 : )
500 142 : .expect("failed to define a metric")
501 142 : });
502 :
503 142 : static REMOTE_PHYSICAL_SIZE_GLOBAL: Lazy<UIntGauge> = Lazy::new(|| {
504 : register_uint_gauge!(
505 : "pageserver_remote_physical_size_global",
506 : "Like `pageserver_remote_physical_size`, but without tenant/timeline dimensions."
507 : )
508 142 : .expect("failed to define a metric")
509 142 : });
510 :
511 4 : pub(crate) static REMOTE_ONDEMAND_DOWNLOADED_LAYERS: Lazy<IntCounter> = Lazy::new(|| {
512 : register_int_counter!(
513 : "pageserver_remote_ondemand_downloaded_layers_total",
514 : "Total on-demand downloaded layers"
515 : )
516 4 : .unwrap()
517 4 : });
518 :
519 4 : pub(crate) static REMOTE_ONDEMAND_DOWNLOADED_BYTES: Lazy<IntCounter> = Lazy::new(|| {
520 : register_int_counter!(
521 : "pageserver_remote_ondemand_downloaded_bytes_total",
522 : "Total bytes of layers on-demand downloaded",
523 : )
524 4 : .unwrap()
525 4 : });
526 :
527 142 : static CURRENT_LOGICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
528 : register_uint_gauge_vec!(
529 : "pageserver_current_logical_size",
530 : "Current logical size grouped by timeline",
531 : &["tenant_id", "shard_id", "timeline_id"]
532 : )
533 142 : .expect("failed to define current logical size metric")
534 142 : });
535 :
536 142 : static AUX_FILE_SIZE: Lazy<IntGaugeVec> = Lazy::new(|| {
537 : register_int_gauge_vec!(
538 : "pageserver_aux_file_estimated_size",
539 : "The size of all aux files for a timeline in aux file v2 store.",
540 : &["tenant_id", "shard_id", "timeline_id"]
541 : )
542 142 : .expect("failed to define a metric")
543 142 : });
544 :
545 142 : static VALID_LSN_LEASE_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
546 : register_uint_gauge_vec!(
547 : "pageserver_valid_lsn_lease_count",
548 : "The number of valid leases after refreshing gc info.",
549 : &["tenant_id", "shard_id", "timeline_id"],
550 : )
551 142 : .expect("failed to define a metric")
552 142 : });
553 :
554 : pub(crate) mod initial_logical_size {
555 : use metrics::{register_int_counter, register_int_counter_vec, IntCounter, IntCounterVec};
556 : use once_cell::sync::Lazy;
557 :
558 : pub(crate) struct StartCalculation(IntCounterVec);
559 142 : pub(crate) static START_CALCULATION: Lazy<StartCalculation> = Lazy::new(|| {
560 142 : StartCalculation(
561 142 : register_int_counter_vec!(
562 : "pageserver_initial_logical_size_start_calculation",
563 : "Incremented each time we start an initial logical size calculation attempt. \
564 : The `circumstances` label provides some additional details.",
565 : &["attempt", "circumstances"]
566 142 : )
567 142 : .unwrap(),
568 142 : )
569 142 : });
570 :
571 : struct DropCalculation {
572 : first: IntCounter,
573 : retry: IntCounter,
574 : }
575 :
576 142 : static DROP_CALCULATION: Lazy<DropCalculation> = Lazy::new(|| {
577 142 : let vec = register_int_counter_vec!(
578 : "pageserver_initial_logical_size_drop_calculation",
579 : "Incremented each time we abort a started size calculation attmpt.",
580 : &["attempt"]
581 : )
582 142 : .unwrap();
583 142 : DropCalculation {
584 142 : first: vec.with_label_values(&["first"]),
585 142 : retry: vec.with_label_values(&["retry"]),
586 142 : }
587 142 : });
588 :
589 : pub(crate) struct Calculated {
590 : pub(crate) births: IntCounter,
591 : pub(crate) deaths: IntCounter,
592 : }
593 :
594 142 : pub(crate) static CALCULATED: Lazy<Calculated> = Lazy::new(|| Calculated {
595 : births: register_int_counter!(
596 : "pageserver_initial_logical_size_finish_calculation",
597 : "Incremented every time we finish calculation of initial logical size.\
598 : If everything is working well, this should happen at most once per Timeline object."
599 : )
600 142 : .unwrap(),
601 : deaths: register_int_counter!(
602 : "pageserver_initial_logical_size_drop_finished_calculation",
603 : "Incremented when we drop a finished initial logical size calculation result.\
604 : Mainly useful to turn pageserver_initial_logical_size_finish_calculation into a gauge."
605 : )
606 142 : .unwrap(),
607 142 : });
608 :
609 : pub(crate) struct OngoingCalculationGuard {
610 : inc_drop_calculation: Option<IntCounter>,
611 : }
612 :
613 154 : #[derive(strum_macros::IntoStaticStr)]
614 : pub(crate) enum StartCircumstances {
615 : EmptyInitial,
616 : SkippedConcurrencyLimiter,
617 : AfterBackgroundTasksRateLimit,
618 : }
619 :
620 : impl StartCalculation {
621 154 : pub(crate) fn first(&self, circumstances: StartCircumstances) -> OngoingCalculationGuard {
622 154 : let circumstances_label: &'static str = circumstances.into();
623 154 : self.0
624 154 : .with_label_values(&["first", circumstances_label])
625 154 : .inc();
626 154 : OngoingCalculationGuard {
627 154 : inc_drop_calculation: Some(DROP_CALCULATION.first.clone()),
628 154 : }
629 154 : }
630 0 : pub(crate) fn retry(&self, circumstances: StartCircumstances) -> OngoingCalculationGuard {
631 0 : let circumstances_label: &'static str = circumstances.into();
632 0 : self.0
633 0 : .with_label_values(&["retry", circumstances_label])
634 0 : .inc();
635 0 : OngoingCalculationGuard {
636 0 : inc_drop_calculation: Some(DROP_CALCULATION.retry.clone()),
637 0 : }
638 0 : }
639 : }
640 :
641 : impl Drop for OngoingCalculationGuard {
642 154 : fn drop(&mut self) {
643 154 : if let Some(counter) = self.inc_drop_calculation.take() {
644 0 : counter.inc();
645 154 : }
646 154 : }
647 : }
648 :
649 : impl OngoingCalculationGuard {
650 154 : pub(crate) fn calculation_result_saved(mut self) -> FinishedCalculationGuard {
651 154 : drop(self.inc_drop_calculation.take());
652 154 : CALCULATED.births.inc();
653 154 : FinishedCalculationGuard {
654 154 : inc_on_drop: CALCULATED.deaths.clone(),
655 154 : }
656 154 : }
657 : }
658 :
659 : pub(crate) struct FinishedCalculationGuard {
660 : inc_on_drop: IntCounter,
661 : }
662 :
663 : impl Drop for FinishedCalculationGuard {
664 6 : fn drop(&mut self) {
665 6 : self.inc_on_drop.inc();
666 6 : }
667 : }
668 :
669 : // context: https://github.com/neondatabase/neon/issues/5963
670 : pub(crate) static TIMELINES_WHERE_WALRECEIVER_GOT_APPROXIMATE_SIZE: Lazy<IntCounter> =
671 0 : Lazy::new(|| {
672 : register_int_counter!(
673 : "pageserver_initial_logical_size_timelines_where_walreceiver_got_approximate_size",
674 : "Counter for the following event: walreceiver calls\
675 : Timeline::get_current_logical_size() and it returns `Approximate` for the first time."
676 : )
677 0 : .unwrap()
678 0 : });
679 : }
680 :
681 0 : static DIRECTORY_ENTRIES_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
682 : register_uint_gauge_vec!(
683 : "pageserver_directory_entries_count",
684 : "Sum of the entries in pageserver-stored directory listings",
685 : &["tenant_id", "shard_id", "timeline_id"]
686 : )
687 0 : .expect("failed to define a metric")
688 0 : });
689 :
690 144 : pub(crate) static TENANT_STATE_METRIC: Lazy<UIntGaugeVec> = Lazy::new(|| {
691 : register_uint_gauge_vec!(
692 : "pageserver_tenant_states_count",
693 : "Count of tenants per state",
694 : &["state"]
695 : )
696 144 : .expect("Failed to register pageserver_tenant_states_count metric")
697 144 : });
698 :
699 : /// A set of broken tenants.
700 : ///
701 : /// These are expected to be so rare that a set is fine. Set as in a new timeseries per each broken
702 : /// tenant.
703 12 : pub(crate) static BROKEN_TENANTS_SET: Lazy<UIntGaugeVec> = Lazy::new(|| {
704 : register_uint_gauge_vec!(
705 : "pageserver_broken_tenants_count",
706 : "Set of broken tenants",
707 : &["tenant_id", "shard_id"]
708 : )
709 12 : .expect("Failed to register pageserver_tenant_states_count metric")
710 12 : });
711 :
712 6 : pub(crate) static TENANT_SYNTHETIC_SIZE_METRIC: Lazy<UIntGaugeVec> = Lazy::new(|| {
713 : register_uint_gauge_vec!(
714 : "pageserver_tenant_synthetic_cached_size_bytes",
715 : "Synthetic size of each tenant in bytes",
716 : &["tenant_id"]
717 : )
718 6 : .expect("Failed to register pageserver_tenant_synthetic_cached_size_bytes metric")
719 6 : });
720 :
721 0 : pub(crate) static EVICTION_ITERATION_DURATION: Lazy<HistogramVec> = Lazy::new(|| {
722 : register_histogram_vec!(
723 : "pageserver_eviction_iteration_duration_seconds_global",
724 : "Time spent on a single eviction iteration",
725 : &["period_secs", "threshold_secs"],
726 : STORAGE_OP_BUCKETS.into(),
727 : )
728 0 : .expect("failed to define a metric")
729 0 : });
730 :
731 142 : static EVICTIONS: Lazy<IntCounterVec> = Lazy::new(|| {
732 : register_int_counter_vec!(
733 : "pageserver_evictions",
734 : "Number of layers evicted from the pageserver",
735 : &["tenant_id", "shard_id", "timeline_id"]
736 : )
737 142 : .expect("failed to define a metric")
738 142 : });
739 :
740 142 : static EVICTIONS_WITH_LOW_RESIDENCE_DURATION: Lazy<IntCounterVec> = Lazy::new(|| {
741 : register_int_counter_vec!(
742 : "pageserver_evictions_with_low_residence_duration",
743 : "If a layer is evicted that was resident for less than `low_threshold`, it is counted to this counter. \
744 : Residence duration is determined using the `residence_duration_data_source`.",
745 : &["tenant_id", "shard_id", "timeline_id", "residence_duration_data_source", "low_threshold_secs"]
746 : )
747 142 : .expect("failed to define a metric")
748 142 : });
749 :
750 0 : pub(crate) static UNEXPECTED_ONDEMAND_DOWNLOADS: Lazy<IntCounter> = Lazy::new(|| {
751 : register_int_counter!(
752 : "pageserver_unexpected_ondemand_downloads_count",
753 : "Number of unexpected on-demand downloads. \
754 : We log more context for each increment, so, forgo any labels in this metric.",
755 : )
756 0 : .expect("failed to define a metric")
757 0 : });
758 :
759 : /// How long did we take to start up? Broken down by labels to describe
760 : /// different phases of startup.
761 0 : pub static STARTUP_DURATION: Lazy<GaugeVec> = Lazy::new(|| {
762 : register_gauge_vec!(
763 : "pageserver_startup_duration_seconds",
764 : "Time taken by phases of pageserver startup, in seconds",
765 : &["phase"]
766 : )
767 0 : .expect("Failed to register pageserver_startup_duration_seconds metric")
768 0 : });
769 :
770 0 : pub static STARTUP_IS_LOADING: Lazy<UIntGauge> = Lazy::new(|| {
771 : register_uint_gauge!(
772 : "pageserver_startup_is_loading",
773 : "1 while in initial startup load of tenants, 0 at other times"
774 : )
775 0 : .expect("Failed to register pageserver_startup_is_loading")
776 0 : });
777 :
778 138 : pub(crate) static TIMELINE_EPHEMERAL_BYTES: Lazy<UIntGauge> = Lazy::new(|| {
779 : register_uint_gauge!(
780 : "pageserver_timeline_ephemeral_bytes",
781 : "Total number of bytes in ephemeral layers, summed for all timelines. Approximate, lazily updated."
782 : )
783 138 : .expect("Failed to register metric")
784 138 : });
785 :
786 : /// Metrics related to the lifecycle of a [`crate::tenant::Tenant`] object: things
787 : /// like how long it took to load.
788 : ///
789 : /// Note that these are process-global metrics, _not_ per-tenant metrics. Per-tenant
790 : /// metrics are rather expensive, and usually fine grained stuff makes more sense
791 : /// at a timeline level than tenant level.
792 : pub(crate) struct TenantMetrics {
793 : /// How long did tenants take to go from construction to active state?
794 : pub(crate) activation: Histogram,
795 : pub(crate) preload: Histogram,
796 : pub(crate) attach: Histogram,
797 :
798 : /// How many tenants are included in the initial startup of the pagesrever?
799 : pub(crate) startup_scheduled: IntCounter,
800 : pub(crate) startup_complete: IntCounter,
801 : }
802 :
803 0 : pub(crate) static TENANT: Lazy<TenantMetrics> = Lazy::new(|| {
804 0 : TenantMetrics {
805 0 : activation: register_histogram!(
806 : "pageserver_tenant_activation_seconds",
807 : "Time taken by tenants to activate, in seconds",
808 : CRITICAL_OP_BUCKETS.into()
809 0 : )
810 0 : .expect("Failed to register metric"),
811 0 : preload: register_histogram!(
812 : "pageserver_tenant_preload_seconds",
813 : "Time taken by tenants to load remote metadata on startup/attach, in seconds",
814 : CRITICAL_OP_BUCKETS.into()
815 0 : )
816 0 : .expect("Failed to register metric"),
817 0 : attach: register_histogram!(
818 : "pageserver_tenant_attach_seconds",
819 : "Time taken by tenants to intialize, after remote metadata is already loaded",
820 : CRITICAL_OP_BUCKETS.into()
821 0 : )
822 0 : .expect("Failed to register metric"),
823 0 : startup_scheduled: register_int_counter!(
824 : "pageserver_tenant_startup_scheduled",
825 : "Number of tenants included in pageserver startup (doesn't count tenants attached later)"
826 0 : ).expect("Failed to register metric"),
827 0 : startup_complete: register_int_counter!(
828 : "pageserver_tenant_startup_complete",
829 : "Number of tenants that have completed warm-up, or activated on-demand during initial startup: \
830 : should eventually reach `pageserver_tenant_startup_scheduled_total`. Does not include broken \
831 : tenants: such cases will lead to this metric never reaching the scheduled count."
832 0 : ).expect("Failed to register metric"),
833 0 : }
834 0 : });
835 :
836 : /// Each `Timeline`'s [`EVICTIONS_WITH_LOW_RESIDENCE_DURATION`] metric.
837 : #[derive(Debug)]
838 : pub(crate) struct EvictionsWithLowResidenceDuration {
839 : data_source: &'static str,
840 : threshold: Duration,
841 : counter: Option<IntCounter>,
842 : }
843 :
844 : pub(crate) struct EvictionsWithLowResidenceDurationBuilder {
845 : data_source: &'static str,
846 : threshold: Duration,
847 : }
848 :
849 : impl EvictionsWithLowResidenceDurationBuilder {
850 384 : pub fn new(data_source: &'static str, threshold: Duration) -> Self {
851 384 : Self {
852 384 : data_source,
853 384 : threshold,
854 384 : }
855 384 : }
856 :
857 384 : fn build(
858 384 : &self,
859 384 : tenant_id: &str,
860 384 : shard_id: &str,
861 384 : timeline_id: &str,
862 384 : ) -> EvictionsWithLowResidenceDuration {
863 384 : let counter = EVICTIONS_WITH_LOW_RESIDENCE_DURATION
864 384 : .get_metric_with_label_values(&[
865 384 : tenant_id,
866 384 : shard_id,
867 384 : timeline_id,
868 384 : self.data_source,
869 384 : &EvictionsWithLowResidenceDuration::threshold_label_value(self.threshold),
870 384 : ])
871 384 : .unwrap();
872 384 : EvictionsWithLowResidenceDuration {
873 384 : data_source: self.data_source,
874 384 : threshold: self.threshold,
875 384 : counter: Some(counter),
876 384 : }
877 384 : }
878 : }
879 :
880 : impl EvictionsWithLowResidenceDuration {
881 392 : fn threshold_label_value(threshold: Duration) -> String {
882 392 : format!("{}", threshold.as_secs())
883 392 : }
884 :
885 18 : pub fn observe(&self, observed_value: Duration) {
886 18 : if observed_value < self.threshold {
887 18 : self.counter
888 18 : .as_ref()
889 18 : .expect("nobody calls this function after `remove_from_vec`")
890 18 : .inc();
891 18 : }
892 18 : }
893 :
894 8 : pub fn change_threshold(
895 8 : &mut self,
896 8 : tenant_id: &str,
897 8 : shard_id: &str,
898 8 : timeline_id: &str,
899 8 : new_threshold: Duration,
900 8 : ) {
901 8 : if new_threshold == self.threshold {
902 8 : return;
903 0 : }
904 0 : let mut with_new = EvictionsWithLowResidenceDurationBuilder::new(
905 0 : self.data_source,
906 0 : new_threshold,
907 0 : )
908 0 : .build(tenant_id, shard_id, timeline_id);
909 0 : std::mem::swap(self, &mut with_new);
910 0 : with_new.remove(tenant_id, shard_id, timeline_id);
911 8 : }
912 :
913 : // This could be a `Drop` impl, but, we need the `tenant_id` and `timeline_id`.
914 8 : fn remove(&mut self, tenant_id: &str, shard_id: &str, timeline_id: &str) {
915 8 : let Some(_counter) = self.counter.take() else {
916 0 : return;
917 : };
918 :
919 8 : let threshold = Self::threshold_label_value(self.threshold);
920 8 :
921 8 : let removed = EVICTIONS_WITH_LOW_RESIDENCE_DURATION.remove_label_values(&[
922 8 : tenant_id,
923 8 : shard_id,
924 8 : timeline_id,
925 8 : self.data_source,
926 8 : &threshold,
927 8 : ]);
928 8 :
929 8 : match removed {
930 0 : Err(e) => {
931 0 : // this has been hit in staging as
932 0 : // <https://neondatabase.sentry.io/issues/4142396994/>, but we don't know how.
933 0 : // because we can be in the drop path already, don't risk:
934 0 : // - "double-panic => illegal instruction" or
935 0 : // - future "drop panick => abort"
936 0 : //
937 0 : // so just nag: (the error has the labels)
938 0 : tracing::warn!("failed to remove EvictionsWithLowResidenceDuration, it was already removed? {e:#?}");
939 : }
940 : Ok(()) => {
941 : // to help identify cases where we double-remove the same values, let's log all
942 : // deletions?
943 8 : tracing::info!("removed EvictionsWithLowResidenceDuration with {tenant_id}, {timeline_id}, {}, {threshold}", self.data_source);
944 : }
945 : }
946 8 : }
947 : }
948 :
949 : // Metrics collected on disk IO operations
950 : //
951 : // Roughly logarithmic scale.
952 : const STORAGE_IO_TIME_BUCKETS: &[f64] = &[
953 : 0.000030, // 30 usec
954 : 0.001000, // 1000 usec
955 : 0.030, // 30 ms
956 : 1.000, // 1000 ms
957 : 30.000, // 30000 ms
958 : ];
959 :
960 : /// VirtualFile fs operation variants.
961 : ///
962 : /// Operations:
963 : /// - open ([`std::fs::OpenOptions::open`])
964 : /// - close (dropping [`crate::virtual_file::VirtualFile`])
965 : /// - close-by-replace (close by replacement algorithm)
966 : /// - read (`read_at`)
967 : /// - write (`write_at`)
968 : /// - seek (modify internal position or file length query)
969 : /// - fsync ([`std::fs::File::sync_all`])
970 : /// - metadata ([`std::fs::File::metadata`])
971 : #[derive(
972 1458 : Debug, Clone, Copy, strum_macros::EnumCount, strum_macros::EnumIter, strum_macros::FromRepr,
973 : )]
974 : pub(crate) enum StorageIoOperation {
975 : Open,
976 : OpenAfterReplace,
977 : Close,
978 : CloseByReplace,
979 : Read,
980 : Write,
981 : Seek,
982 : Fsync,
983 : Metadata,
984 : }
985 :
986 : impl StorageIoOperation {
987 1458 : pub fn as_str(&self) -> &'static str {
988 1458 : match self {
989 162 : StorageIoOperation::Open => "open",
990 162 : StorageIoOperation::OpenAfterReplace => "open-after-replace",
991 162 : StorageIoOperation::Close => "close",
992 162 : StorageIoOperation::CloseByReplace => "close-by-replace",
993 162 : StorageIoOperation::Read => "read",
994 162 : StorageIoOperation::Write => "write",
995 162 : StorageIoOperation::Seek => "seek",
996 162 : StorageIoOperation::Fsync => "fsync",
997 162 : StorageIoOperation::Metadata => "metadata",
998 : }
999 1458 : }
1000 : }
1001 :
1002 : /// Tracks time taken by fs operations near VirtualFile.
1003 : #[derive(Debug)]
1004 : pub(crate) struct StorageIoTime {
1005 : metrics: [Histogram; StorageIoOperation::COUNT],
1006 : }
1007 :
1008 : impl StorageIoTime {
1009 162 : fn new() -> Self {
1010 162 : let storage_io_histogram_vec = register_histogram_vec!(
1011 : "pageserver_io_operations_seconds",
1012 : "Time spent in IO operations",
1013 : &["operation"],
1014 : STORAGE_IO_TIME_BUCKETS.into()
1015 : )
1016 162 : .expect("failed to define a metric");
1017 1458 : let metrics = std::array::from_fn(|i| {
1018 1458 : let op = StorageIoOperation::from_repr(i).unwrap();
1019 1458 : storage_io_histogram_vec
1020 1458 : .get_metric_with_label_values(&[op.as_str()])
1021 1458 : .unwrap()
1022 1458 : });
1023 162 : Self { metrics }
1024 162 : }
1025 :
1026 1922801 : pub(crate) fn get(&self, op: StorageIoOperation) -> &Histogram {
1027 1922801 : &self.metrics[op as usize]
1028 1922801 : }
1029 : }
1030 :
1031 : pub(crate) static STORAGE_IO_TIME_METRIC: Lazy<StorageIoTime> = Lazy::new(StorageIoTime::new);
1032 :
1033 : const STORAGE_IO_SIZE_OPERATIONS: &[&str] = &["read", "write"];
1034 :
1035 : // Needed for the https://neonprod.grafana.net/d/5uK9tHL4k/picking-tenant-for-relocation?orgId=1
1036 160 : pub(crate) static STORAGE_IO_SIZE: Lazy<IntGaugeVec> = Lazy::new(|| {
1037 : register_int_gauge_vec!(
1038 : "pageserver_io_operations_bytes_total",
1039 : "Total amount of bytes read/written in IO operations",
1040 : &["operation", "tenant_id", "shard_id", "timeline_id"]
1041 : )
1042 160 : .expect("failed to define a metric")
1043 160 : });
1044 :
1045 : #[cfg(not(test))]
1046 : pub(crate) mod virtual_file_descriptor_cache {
1047 : use super::*;
1048 :
1049 0 : pub(crate) static SIZE_MAX: Lazy<UIntGauge> = Lazy::new(|| {
1050 : register_uint_gauge!(
1051 : "pageserver_virtual_file_descriptor_cache_size_max",
1052 : "Maximum number of open file descriptors in the cache."
1053 : )
1054 0 : .unwrap()
1055 0 : });
1056 :
1057 : // SIZE_CURRENT: derive it like so:
1058 : // ```
1059 : // sum (pageserver_io_operations_seconds_count{operation=~"^(open|open-after-replace)$")
1060 : // -ignoring(operation)
1061 : // sum(pageserver_io_operations_seconds_count{operation=~"^(close|close-by-replace)$"}
1062 : // ```
1063 : }
1064 :
1065 : #[cfg(not(test))]
1066 : pub(crate) mod virtual_file_io_engine {
1067 : use super::*;
1068 :
1069 0 : pub(crate) static KIND: Lazy<UIntGaugeVec> = Lazy::new(|| {
1070 : register_uint_gauge_vec!(
1071 : "pageserver_virtual_file_io_engine_kind",
1072 : "The configured io engine for VirtualFile",
1073 : &["kind"],
1074 : )
1075 0 : .unwrap()
1076 0 : });
1077 : }
1078 :
1079 : #[derive(Debug)]
1080 : struct GlobalAndPerTimelineHistogram {
1081 : global: Histogram,
1082 : per_tenant_timeline: Histogram,
1083 : }
1084 :
1085 : impl GlobalAndPerTimelineHistogram {
1086 10 : fn observe(&self, value: f64) {
1087 10 : self.global.observe(value);
1088 10 : self.per_tenant_timeline.observe(value);
1089 10 : }
1090 : }
1091 :
1092 : struct GlobalAndPerTimelineHistogramTimer<'a, 'c> {
1093 : h: &'a GlobalAndPerTimelineHistogram,
1094 : ctx: &'c RequestContext,
1095 : start: std::time::Instant,
1096 : op: SmgrQueryType,
1097 : }
1098 :
1099 : impl<'a, 'c> Drop for GlobalAndPerTimelineHistogramTimer<'a, 'c> {
1100 10 : fn drop(&mut self) {
1101 10 : let elapsed = self.start.elapsed();
1102 10 : let ex_throttled = self
1103 10 : .ctx
1104 10 : .micros_spent_throttled
1105 10 : .close_and_checked_sub_from(elapsed);
1106 10 : let ex_throttled = match ex_throttled {
1107 10 : Ok(res) => res,
1108 0 : Err(error) => {
1109 0 : use utils::rate_limit::RateLimit;
1110 0 : static LOGGED: Lazy<Mutex<enum_map::EnumMap<SmgrQueryType, RateLimit>>> =
1111 0 : Lazy::new(|| {
1112 0 : Mutex::new(enum_map::EnumMap::from_array(std::array::from_fn(|_| {
1113 0 : RateLimit::new(Duration::from_secs(10))
1114 0 : })))
1115 0 : });
1116 0 : let mut guard = LOGGED.lock().unwrap();
1117 0 : let rate_limit = &mut guard[self.op];
1118 0 : rate_limit.call(|| {
1119 0 : warn!(op=?self.op, error, "error deducting time spent throttled; this message is logged at a global rate limit");
1120 0 : });
1121 0 : elapsed
1122 : }
1123 : };
1124 10 : self.h.observe(ex_throttled.as_secs_f64());
1125 10 : }
1126 : }
1127 :
1128 : #[derive(
1129 : Debug,
1130 : Clone,
1131 : Copy,
1132 3990 : IntoStaticStr,
1133 : strum_macros::EnumCount,
1134 112 : strum_macros::EnumIter,
1135 1970 : strum_macros::FromRepr,
1136 : enum_map::Enum,
1137 : )]
1138 : #[strum(serialize_all = "snake_case")]
1139 : pub enum SmgrQueryType {
1140 : GetRelExists,
1141 : GetRelSize,
1142 : GetPageAtLsn,
1143 : GetDbSize,
1144 : GetSlruSegment,
1145 : }
1146 :
1147 : #[derive(Debug)]
1148 : pub(crate) struct SmgrQueryTimePerTimeline {
1149 : metrics: [GlobalAndPerTimelineHistogram; SmgrQueryType::COUNT],
1150 : }
1151 :
1152 144 : static SMGR_QUERY_TIME_PER_TENANT_TIMELINE: Lazy<HistogramVec> = Lazy::new(|| {
1153 : register_histogram_vec!(
1154 : "pageserver_smgr_query_seconds",
1155 : "Time spent on smgr query handling, aggegated by query type and tenant/timeline.",
1156 : &["smgr_query_type", "tenant_id", "shard_id", "timeline_id"],
1157 : CRITICAL_OP_BUCKETS.into(),
1158 : )
1159 144 : .expect("failed to define a metric")
1160 144 : });
1161 :
1162 144 : static SMGR_QUERY_TIME_GLOBAL_BUCKETS: Lazy<Vec<f64>> = Lazy::new(|| {
1163 144 : [
1164 144 : 1,
1165 144 : 10,
1166 144 : 20,
1167 144 : 40,
1168 144 : 60,
1169 144 : 80,
1170 144 : 100,
1171 144 : 200,
1172 144 : 300,
1173 144 : 400,
1174 144 : 500,
1175 144 : 600,
1176 144 : 700,
1177 144 : 800,
1178 144 : 900,
1179 144 : 1_000, // 1ms
1180 144 : 2_000,
1181 144 : 4_000,
1182 144 : 6_000,
1183 144 : 8_000,
1184 144 : 10_000, // 10ms
1185 144 : 20_000,
1186 144 : 40_000,
1187 144 : 60_000,
1188 144 : 80_000,
1189 144 : 100_000,
1190 144 : 200_000,
1191 144 : 400_000,
1192 144 : 600_000,
1193 144 : 800_000,
1194 144 : 1_000_000, // 1s
1195 144 : 2_000_000,
1196 144 : 4_000_000,
1197 144 : 6_000_000,
1198 144 : 8_000_000,
1199 144 : 10_000_000, // 10s
1200 144 : 20_000_000,
1201 144 : 50_000_000,
1202 144 : 100_000_000,
1203 144 : 200_000_000,
1204 144 : 1_000_000_000, // 1000s
1205 144 : ]
1206 144 : .into_iter()
1207 144 : .map(Duration::from_micros)
1208 5904 : .map(|d| d.as_secs_f64())
1209 144 : .collect()
1210 144 : });
1211 :
1212 144 : static SMGR_QUERY_TIME_GLOBAL: Lazy<HistogramVec> = Lazy::new(|| {
1213 : register_histogram_vec!(
1214 : "pageserver_smgr_query_seconds_global",
1215 : "Time spent on smgr query handling, aggregated by query type.",
1216 : &["smgr_query_type"],
1217 : SMGR_QUERY_TIME_GLOBAL_BUCKETS.clone(),
1218 : )
1219 144 : .expect("failed to define a metric")
1220 144 : });
1221 :
1222 : impl SmgrQueryTimePerTimeline {
1223 394 : pub(crate) fn new(tenant_shard_id: &TenantShardId, timeline_id: &TimelineId) -> Self {
1224 394 : let tenant_id = tenant_shard_id.tenant_id.to_string();
1225 394 : let shard_slug = format!("{}", tenant_shard_id.shard_slug());
1226 394 : let timeline_id = timeline_id.to_string();
1227 1970 : let metrics = std::array::from_fn(|i| {
1228 1970 : let op = SmgrQueryType::from_repr(i).unwrap();
1229 1970 : let global = SMGR_QUERY_TIME_GLOBAL
1230 1970 : .get_metric_with_label_values(&[op.into()])
1231 1970 : .unwrap();
1232 1970 : let per_tenant_timeline = SMGR_QUERY_TIME_PER_TENANT_TIMELINE
1233 1970 : .get_metric_with_label_values(&[op.into(), &tenant_id, &shard_slug, &timeline_id])
1234 1970 : .unwrap();
1235 1970 : GlobalAndPerTimelineHistogram {
1236 1970 : global,
1237 1970 : per_tenant_timeline,
1238 1970 : }
1239 1970 : });
1240 394 : Self { metrics }
1241 394 : }
1242 10 : pub(crate) fn start_timer<'c: 'a, 'a>(
1243 10 : &'a self,
1244 10 : op: SmgrQueryType,
1245 10 : ctx: &'c RequestContext,
1246 10 : ) -> impl Drop + '_ {
1247 10 : let metric = &self.metrics[op as usize];
1248 10 : let start = Instant::now();
1249 10 : match ctx.micros_spent_throttled.open() {
1250 10 : Ok(()) => (),
1251 0 : Err(error) => {
1252 0 : use utils::rate_limit::RateLimit;
1253 0 : static LOGGED: Lazy<Mutex<enum_map::EnumMap<SmgrQueryType, RateLimit>>> =
1254 0 : Lazy::new(|| {
1255 0 : Mutex::new(enum_map::EnumMap::from_array(std::array::from_fn(|_| {
1256 0 : RateLimit::new(Duration::from_secs(10))
1257 0 : })))
1258 0 : });
1259 0 : let mut guard = LOGGED.lock().unwrap();
1260 0 : let rate_limit = &mut guard[op];
1261 0 : rate_limit.call(|| {
1262 0 : warn!(?op, error, "error opening micros_spent_throttled; this message is logged at a global rate limit");
1263 0 : });
1264 0 : }
1265 : }
1266 10 : GlobalAndPerTimelineHistogramTimer {
1267 10 : h: metric,
1268 10 : ctx,
1269 10 : start,
1270 10 : op,
1271 10 : }
1272 10 : }
1273 : }
1274 :
1275 : #[cfg(test)]
1276 : mod smgr_query_time_tests {
1277 : use pageserver_api::shard::TenantShardId;
1278 : use strum::IntoEnumIterator;
1279 : use utils::id::{TenantId, TimelineId};
1280 :
1281 : use crate::{
1282 : context::{DownloadBehavior, RequestContext},
1283 : task_mgr::TaskKind,
1284 : };
1285 :
1286 : // Regression test, we used hard-coded string constants before using an enum.
1287 : #[test]
1288 2 : fn op_label_name() {
1289 2 : use super::SmgrQueryType::*;
1290 2 : let expect: [(super::SmgrQueryType, &'static str); 5] = [
1291 2 : (GetRelExists, "get_rel_exists"),
1292 2 : (GetRelSize, "get_rel_size"),
1293 2 : (GetPageAtLsn, "get_page_at_lsn"),
1294 2 : (GetDbSize, "get_db_size"),
1295 2 : (GetSlruSegment, "get_slru_segment"),
1296 2 : ];
1297 12 : for (op, expect) in expect {
1298 10 : let actual: &'static str = op.into();
1299 10 : assert_eq!(actual, expect);
1300 : }
1301 2 : }
1302 :
1303 : #[test]
1304 2 : fn basic() {
1305 2 : let ops: Vec<_> = super::SmgrQueryType::iter().collect();
1306 :
1307 12 : for op in &ops {
1308 10 : let tenant_id = TenantId::generate();
1309 10 : let timeline_id = TimelineId::generate();
1310 10 : let metrics = super::SmgrQueryTimePerTimeline::new(
1311 10 : &TenantShardId::unsharded(tenant_id),
1312 10 : &timeline_id,
1313 10 : );
1314 10 :
1315 20 : let get_counts = || {
1316 20 : let global: u64 = ops
1317 20 : .iter()
1318 100 : .map(|op| metrics.metrics[*op as usize].global.get_sample_count())
1319 20 : .sum();
1320 20 : let per_tenant_timeline: u64 = ops
1321 20 : .iter()
1322 100 : .map(|op| {
1323 100 : metrics.metrics[*op as usize]
1324 100 : .per_tenant_timeline
1325 100 : .get_sample_count()
1326 100 : })
1327 20 : .sum();
1328 20 : (global, per_tenant_timeline)
1329 20 : };
1330 :
1331 10 : let (pre_global, pre_per_tenant_timeline) = get_counts();
1332 10 : assert_eq!(pre_per_tenant_timeline, 0);
1333 :
1334 10 : let ctx = RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Download);
1335 10 : let timer = metrics.start_timer(*op, &ctx);
1336 10 : drop(timer);
1337 10 :
1338 10 : let (post_global, post_per_tenant_timeline) = get_counts();
1339 10 : assert_eq!(post_per_tenant_timeline, 1);
1340 10 : assert!(post_global > pre_global);
1341 : }
1342 2 : }
1343 : }
1344 :
1345 : // keep in sync with control plane Go code so that we can validate
1346 : // compute's basebackup_ms metric with our perspective in the context of SLI/SLO.
1347 0 : static COMPUTE_STARTUP_BUCKETS: Lazy<[f64; 28]> = Lazy::new(|| {
1348 0 : // Go code uses milliseconds. Variable is called `computeStartupBuckets`
1349 0 : [
1350 0 : 5, 10, 20, 30, 50, 70, 100, 120, 150, 200, 250, 300, 350, 400, 450, 500, 600, 800, 1000,
1351 0 : 1500, 2000, 2500, 3000, 5000, 10000, 20000, 40000, 60000,
1352 0 : ]
1353 0 : .map(|ms| (ms as f64) / 1000.0)
1354 0 : });
1355 :
1356 : pub(crate) struct BasebackupQueryTime {
1357 : ok: Histogram,
1358 : error: Histogram,
1359 : }
1360 :
1361 0 : pub(crate) static BASEBACKUP_QUERY_TIME: Lazy<BasebackupQueryTime> = Lazy::new(|| {
1362 0 : let vec = register_histogram_vec!(
1363 : "pageserver_basebackup_query_seconds",
1364 : "Histogram of basebackup queries durations, by result type",
1365 : &["result"],
1366 : COMPUTE_STARTUP_BUCKETS.to_vec(),
1367 : )
1368 0 : .expect("failed to define a metric");
1369 0 : BasebackupQueryTime {
1370 0 : ok: vec.get_metric_with_label_values(&["ok"]).unwrap(),
1371 0 : error: vec.get_metric_with_label_values(&["error"]).unwrap(),
1372 0 : }
1373 0 : });
1374 :
1375 : pub(crate) struct BasebackupQueryTimeOngoingRecording<'a, 'c> {
1376 : parent: &'a BasebackupQueryTime,
1377 : ctx: &'c RequestContext,
1378 : start: std::time::Instant,
1379 : }
1380 :
1381 : impl BasebackupQueryTime {
1382 0 : pub(crate) fn start_recording<'c: 'a, 'a>(
1383 0 : &'a self,
1384 0 : ctx: &'c RequestContext,
1385 0 : ) -> BasebackupQueryTimeOngoingRecording<'_, '_> {
1386 0 : let start = Instant::now();
1387 0 : match ctx.micros_spent_throttled.open() {
1388 0 : Ok(()) => (),
1389 0 : Err(error) => {
1390 0 : use utils::rate_limit::RateLimit;
1391 0 : static LOGGED: Lazy<Mutex<RateLimit>> =
1392 0 : Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(10))));
1393 0 : let mut rate_limit = LOGGED.lock().unwrap();
1394 0 : rate_limit.call(|| {
1395 0 : warn!(error, "error opening micros_spent_throttled; this message is logged at a global rate limit");
1396 0 : });
1397 0 : }
1398 : }
1399 0 : BasebackupQueryTimeOngoingRecording {
1400 0 : parent: self,
1401 0 : ctx,
1402 0 : start,
1403 0 : }
1404 0 : }
1405 : }
1406 :
1407 : impl<'a, 'c> BasebackupQueryTimeOngoingRecording<'a, 'c> {
1408 0 : pub(crate) fn observe<T, E>(self, res: &Result<T, E>) {
1409 0 : let elapsed = self.start.elapsed();
1410 0 : let ex_throttled = self
1411 0 : .ctx
1412 0 : .micros_spent_throttled
1413 0 : .close_and_checked_sub_from(elapsed);
1414 0 : let ex_throttled = match ex_throttled {
1415 0 : Ok(ex_throttled) => ex_throttled,
1416 0 : Err(error) => {
1417 0 : use utils::rate_limit::RateLimit;
1418 0 : static LOGGED: Lazy<Mutex<RateLimit>> =
1419 0 : Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(10))));
1420 0 : let mut rate_limit = LOGGED.lock().unwrap();
1421 0 : rate_limit.call(|| {
1422 0 : warn!(error, "error deducting time spent throttled; this message is logged at a global rate limit");
1423 0 : });
1424 0 : elapsed
1425 : }
1426 : };
1427 0 : let metric = if res.is_ok() {
1428 0 : &self.parent.ok
1429 : } else {
1430 0 : &self.parent.error
1431 : };
1432 0 : metric.observe(ex_throttled.as_secs_f64());
1433 0 : }
1434 : }
1435 :
1436 0 : pub(crate) static LIVE_CONNECTIONS_COUNT: Lazy<IntGaugeVec> = Lazy::new(|| {
1437 : register_int_gauge_vec!(
1438 : "pageserver_live_connections",
1439 : "Number of live network connections",
1440 : &["pageserver_connection_kind"]
1441 : )
1442 0 : .expect("failed to define a metric")
1443 0 : });
1444 :
1445 0 : #[derive(Clone, Copy, enum_map::Enum, IntoStaticStr)]
1446 : pub(crate) enum ComputeCommandKind {
1447 : PageStreamV2,
1448 : PageStream,
1449 : Basebackup,
1450 : GetLastRecordRlsn,
1451 : Fullbackup,
1452 : ImportBasebackup,
1453 : ImportWal,
1454 : LeaseLsn,
1455 : Show,
1456 : }
1457 :
1458 : pub(crate) struct ComputeCommandCounters {
1459 : map: EnumMap<ComputeCommandKind, IntCounter>,
1460 : }
1461 :
1462 0 : pub(crate) static COMPUTE_COMMANDS_COUNTERS: Lazy<ComputeCommandCounters> = Lazy::new(|| {
1463 0 : let inner = register_int_counter_vec!(
1464 : "pageserver_compute_commands",
1465 : "Number of compute -> pageserver commands processed",
1466 : &["command"]
1467 : )
1468 0 : .expect("failed to define a metric");
1469 0 :
1470 0 : ComputeCommandCounters {
1471 0 : map: EnumMap::from_array(std::array::from_fn(|i| {
1472 0 : let command = <ComputeCommandKind as enum_map::Enum>::from_usize(i);
1473 0 : let command_str: &'static str = command.into();
1474 0 : inner.with_label_values(&[command_str])
1475 0 : })),
1476 0 : }
1477 0 : });
1478 :
1479 : impl ComputeCommandCounters {
1480 0 : pub(crate) fn for_command(&self, command: ComputeCommandKind) -> &IntCounter {
1481 0 : &self.map[command]
1482 0 : }
1483 : }
1484 :
1485 : // remote storage metrics
1486 :
1487 140 : static REMOTE_TIMELINE_CLIENT_CALLS: Lazy<IntCounterPairVec> = Lazy::new(|| {
1488 : register_int_counter_pair_vec!(
1489 : "pageserver_remote_timeline_client_calls_started",
1490 : "Number of started calls to remote timeline client.",
1491 : "pageserver_remote_timeline_client_calls_finished",
1492 : "Number of finshed calls to remote timeline client.",
1493 : &[
1494 : "tenant_id",
1495 : "shard_id",
1496 : "timeline_id",
1497 : "file_kind",
1498 : "op_kind"
1499 : ],
1500 : )
1501 140 : .unwrap()
1502 140 : });
1503 :
1504 : static REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER: Lazy<IntCounterVec> =
1505 136 : Lazy::new(|| {
1506 : register_int_counter_vec!(
1507 : "pageserver_remote_timeline_client_bytes_started",
1508 : "Incremented by the number of bytes associated with a remote timeline client operation. \
1509 : The increment happens when the operation is scheduled.",
1510 : &["tenant_id", "shard_id", "timeline_id", "file_kind", "op_kind"],
1511 : )
1512 136 : .expect("failed to define a metric")
1513 136 : });
1514 :
1515 136 : static REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER: Lazy<IntCounterVec> = Lazy::new(|| {
1516 : register_int_counter_vec!(
1517 : "pageserver_remote_timeline_client_bytes_finished",
1518 : "Incremented by the number of bytes associated with a remote timeline client operation. \
1519 : The increment happens when the operation finishes (regardless of success/failure/shutdown).",
1520 : &["tenant_id", "shard_id", "timeline_id", "file_kind", "op_kind"],
1521 : )
1522 136 : .expect("failed to define a metric")
1523 136 : });
1524 :
1525 : pub(crate) struct TenantManagerMetrics {
1526 : tenant_slots_attached: UIntGauge,
1527 : tenant_slots_secondary: UIntGauge,
1528 : tenant_slots_inprogress: UIntGauge,
1529 : pub(crate) tenant_slot_writes: IntCounter,
1530 : pub(crate) unexpected_errors: IntCounter,
1531 : }
1532 :
1533 : impl TenantManagerMetrics {
1534 : /// Helpers for tracking slots. Note that these do not track the lifetime of TenantSlot objects
1535 : /// exactly: they track the lifetime of the slots _in the tenant map_.
1536 2 : pub(crate) fn slot_inserted(&self, slot: &TenantSlot) {
1537 2 : match slot {
1538 0 : TenantSlot::Attached(_) => {
1539 0 : self.tenant_slots_attached.inc();
1540 0 : }
1541 0 : TenantSlot::Secondary(_) => {
1542 0 : self.tenant_slots_secondary.inc();
1543 0 : }
1544 2 : TenantSlot::InProgress(_) => {
1545 2 : self.tenant_slots_inprogress.inc();
1546 2 : }
1547 : }
1548 2 : }
1549 :
1550 2 : pub(crate) fn slot_removed(&self, slot: &TenantSlot) {
1551 2 : match slot {
1552 2 : TenantSlot::Attached(_) => {
1553 2 : self.tenant_slots_attached.dec();
1554 2 : }
1555 0 : TenantSlot::Secondary(_) => {
1556 0 : self.tenant_slots_secondary.dec();
1557 0 : }
1558 0 : TenantSlot::InProgress(_) => {
1559 0 : self.tenant_slots_inprogress.dec();
1560 0 : }
1561 : }
1562 2 : }
1563 :
1564 : #[cfg(all(debug_assertions, not(test)))]
1565 0 : pub(crate) fn slots_total(&self) -> u64 {
1566 0 : self.tenant_slots_attached.get()
1567 0 : + self.tenant_slots_secondary.get()
1568 0 : + self.tenant_slots_inprogress.get()
1569 0 : }
1570 : }
1571 :
1572 2 : pub(crate) static TENANT_MANAGER: Lazy<TenantManagerMetrics> = Lazy::new(|| {
1573 2 : let tenant_slots = register_uint_gauge_vec!(
1574 : "pageserver_tenant_manager_slots",
1575 : "How many slots currently exist, including all attached, secondary and in-progress operations",
1576 : &["mode"]
1577 : )
1578 2 : .expect("failed to define a metric");
1579 2 : TenantManagerMetrics {
1580 2 : tenant_slots_attached: tenant_slots
1581 2 : .get_metric_with_label_values(&["attached"])
1582 2 : .unwrap(),
1583 2 : tenant_slots_secondary: tenant_slots
1584 2 : .get_metric_with_label_values(&["secondary"])
1585 2 : .unwrap(),
1586 2 : tenant_slots_inprogress: tenant_slots
1587 2 : .get_metric_with_label_values(&["inprogress"])
1588 2 : .unwrap(),
1589 2 : tenant_slot_writes: register_int_counter!(
1590 : "pageserver_tenant_manager_slot_writes",
1591 : "Writes to a tenant slot, including all of create/attach/detach/delete"
1592 2 : )
1593 2 : .expect("failed to define a metric"),
1594 2 : unexpected_errors: register_int_counter!(
1595 : "pageserver_tenant_manager_unexpected_errors_total",
1596 : "Number of unexpected conditions encountered: nonzero value indicates a non-fatal bug."
1597 2 : )
1598 2 : .expect("failed to define a metric"),
1599 2 : }
1600 2 : });
1601 :
1602 : pub(crate) struct DeletionQueueMetrics {
1603 : pub(crate) keys_submitted: IntCounter,
1604 : pub(crate) keys_dropped: IntCounter,
1605 : pub(crate) keys_executed: IntCounter,
1606 : pub(crate) keys_validated: IntCounter,
1607 : pub(crate) dropped_lsn_updates: IntCounter,
1608 : pub(crate) unexpected_errors: IntCounter,
1609 : pub(crate) remote_errors: IntCounterVec,
1610 : }
1611 22 : pub(crate) static DELETION_QUEUE: Lazy<DeletionQueueMetrics> = Lazy::new(|| {
1612 22 : DeletionQueueMetrics{
1613 22 :
1614 22 : keys_submitted: register_int_counter!(
1615 : "pageserver_deletion_queue_submitted_total",
1616 : "Number of objects submitted for deletion"
1617 22 : )
1618 22 : .expect("failed to define a metric"),
1619 22 :
1620 22 : keys_dropped: register_int_counter!(
1621 : "pageserver_deletion_queue_dropped_total",
1622 : "Number of object deletions dropped due to stale generation."
1623 22 : )
1624 22 : .expect("failed to define a metric"),
1625 22 :
1626 22 : keys_executed: register_int_counter!(
1627 : "pageserver_deletion_queue_executed_total",
1628 : "Number of objects deleted. Only includes objects that we actually deleted, sum with pageserver_deletion_queue_dropped_total for the total number of keys processed to completion"
1629 22 : )
1630 22 : .expect("failed to define a metric"),
1631 22 :
1632 22 : keys_validated: register_int_counter!(
1633 : "pageserver_deletion_queue_validated_total",
1634 : "Number of keys validated for deletion. Sum with pageserver_deletion_queue_dropped_total for the total number of keys that have passed through the validation stage."
1635 22 : )
1636 22 : .expect("failed to define a metric"),
1637 22 :
1638 22 : dropped_lsn_updates: register_int_counter!(
1639 : "pageserver_deletion_queue_dropped_lsn_updates_total",
1640 : "Updates to remote_consistent_lsn dropped due to stale generation number."
1641 22 : )
1642 22 : .expect("failed to define a metric"),
1643 22 : unexpected_errors: register_int_counter!(
1644 : "pageserver_deletion_queue_unexpected_errors_total",
1645 : "Number of unexpected condiions that may stall the queue: any value above zero is unexpected."
1646 22 : )
1647 22 : .expect("failed to define a metric"),
1648 22 : remote_errors: register_int_counter_vec!(
1649 : "pageserver_deletion_queue_remote_errors_total",
1650 : "Retryable remote I/O errors while executing deletions, for example 503 responses to DeleteObjects",
1651 : &["op_kind"],
1652 22 : )
1653 22 : .expect("failed to define a metric")
1654 22 : }
1655 22 : });
1656 :
1657 : pub(crate) struct SecondaryModeMetrics {
1658 : pub(crate) upload_heatmap: IntCounter,
1659 : pub(crate) upload_heatmap_errors: IntCounter,
1660 : pub(crate) upload_heatmap_duration: Histogram,
1661 : pub(crate) download_heatmap: IntCounter,
1662 : pub(crate) download_layer: IntCounter,
1663 : }
1664 0 : pub(crate) static SECONDARY_MODE: Lazy<SecondaryModeMetrics> = Lazy::new(|| {
1665 0 : SecondaryModeMetrics {
1666 0 : upload_heatmap: register_int_counter!(
1667 : "pageserver_secondary_upload_heatmap",
1668 : "Number of heatmaps written to remote storage by attached tenants"
1669 0 : )
1670 0 : .expect("failed to define a metric"),
1671 0 : upload_heatmap_errors: register_int_counter!(
1672 : "pageserver_secondary_upload_heatmap_errors",
1673 : "Failures writing heatmap to remote storage"
1674 0 : )
1675 0 : .expect("failed to define a metric"),
1676 0 : upload_heatmap_duration: register_histogram!(
1677 : "pageserver_secondary_upload_heatmap_duration",
1678 : "Time to build and upload a heatmap, including any waiting inside the S3 client"
1679 0 : )
1680 0 : .expect("failed to define a metric"),
1681 0 : download_heatmap: register_int_counter!(
1682 : "pageserver_secondary_download_heatmap",
1683 : "Number of downloads of heatmaps by secondary mode locations, including when it hasn't changed"
1684 0 : )
1685 0 : .expect("failed to define a metric"),
1686 0 : download_layer: register_int_counter!(
1687 : "pageserver_secondary_download_layer",
1688 : "Number of downloads of layers by secondary mode locations"
1689 0 : )
1690 0 : .expect("failed to define a metric"),
1691 0 : }
1692 0 : });
1693 :
1694 0 : pub(crate) static SECONDARY_RESIDENT_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
1695 : register_uint_gauge_vec!(
1696 : "pageserver_secondary_resident_physical_size",
1697 : "The size of the layer files present in the pageserver's filesystem, for secondary locations.",
1698 : &["tenant_id", "shard_id"]
1699 : )
1700 0 : .expect("failed to define a metric")
1701 0 : });
1702 :
1703 : #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1704 : pub enum RemoteOpKind {
1705 : Upload,
1706 : Download,
1707 : Delete,
1708 : }
1709 : impl RemoteOpKind {
1710 12739 : pub fn as_str(&self) -> &'static str {
1711 12739 : match self {
1712 12037 : Self::Upload => "upload",
1713 52 : Self::Download => "download",
1714 650 : Self::Delete => "delete",
1715 : }
1716 12739 : }
1717 : }
1718 :
1719 : #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
1720 : pub enum RemoteOpFileKind {
1721 : Layer,
1722 : Index,
1723 : }
1724 : impl RemoteOpFileKind {
1725 12739 : pub fn as_str(&self) -> &'static str {
1726 12739 : match self {
1727 8585 : Self::Layer => "layer",
1728 4154 : Self::Index => "index",
1729 : }
1730 12739 : }
1731 : }
1732 :
1733 134 : pub(crate) static REMOTE_OPERATION_TIME: Lazy<HistogramVec> = Lazy::new(|| {
1734 : register_histogram_vec!(
1735 : "pageserver_remote_operation_seconds",
1736 : "Time spent on remote storage operations. \
1737 : Grouped by tenant, timeline, operation_kind and status. \
1738 : Does not account for time spent waiting in remote timeline client's queues.",
1739 : &["file_kind", "op_kind", "status"]
1740 : )
1741 134 : .expect("failed to define a metric")
1742 134 : });
1743 :
1744 0 : pub(crate) static TENANT_TASK_EVENTS: Lazy<IntCounterVec> = Lazy::new(|| {
1745 : register_int_counter_vec!(
1746 : "pageserver_tenant_task_events",
1747 : "Number of task start/stop/fail events.",
1748 : &["event"],
1749 : )
1750 0 : .expect("Failed to register tenant_task_events metric")
1751 0 : });
1752 :
1753 20 : pub(crate) static BACKGROUND_LOOP_SEMAPHORE_WAIT_GAUGE: Lazy<IntCounterPairVec> = Lazy::new(|| {
1754 : register_int_counter_pair_vec!(
1755 : "pageserver_background_loop_semaphore_wait_start_count",
1756 : "Counter for background loop concurrency-limiting semaphore acquire calls started",
1757 : "pageserver_background_loop_semaphore_wait_finish_count",
1758 : "Counter for background loop concurrency-limiting semaphore acquire calls finished",
1759 : &["task"],
1760 : )
1761 20 : .unwrap()
1762 20 : });
1763 :
1764 0 : pub(crate) static BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT: Lazy<IntCounterVec> = Lazy::new(|| {
1765 : register_int_counter_vec!(
1766 : "pageserver_background_loop_period_overrun_count",
1767 : "Incremented whenever warn_when_period_overrun() logs a warning.",
1768 : &["task", "period"],
1769 : )
1770 0 : .expect("failed to define a metric")
1771 0 : });
1772 :
1773 : // walreceiver metrics
1774 :
1775 0 : pub(crate) static WALRECEIVER_STARTED_CONNECTIONS: Lazy<IntCounter> = Lazy::new(|| {
1776 : register_int_counter!(
1777 : "pageserver_walreceiver_started_connections_total",
1778 : "Number of started walreceiver connections"
1779 : )
1780 0 : .expect("failed to define a metric")
1781 0 : });
1782 :
1783 0 : pub(crate) static WALRECEIVER_ACTIVE_MANAGERS: Lazy<IntGauge> = Lazy::new(|| {
1784 : register_int_gauge!(
1785 : "pageserver_walreceiver_active_managers",
1786 : "Number of active walreceiver managers"
1787 : )
1788 0 : .expect("failed to define a metric")
1789 0 : });
1790 :
1791 0 : pub(crate) static WALRECEIVER_SWITCHES: Lazy<IntCounterVec> = Lazy::new(|| {
1792 : register_int_counter_vec!(
1793 : "pageserver_walreceiver_switches_total",
1794 : "Number of walreceiver manager change_connection calls",
1795 : &["reason"]
1796 : )
1797 0 : .expect("failed to define a metric")
1798 0 : });
1799 :
1800 0 : pub(crate) static WALRECEIVER_BROKER_UPDATES: Lazy<IntCounter> = Lazy::new(|| {
1801 : register_int_counter!(
1802 : "pageserver_walreceiver_broker_updates_total",
1803 : "Number of received broker updates in walreceiver"
1804 : )
1805 0 : .expect("failed to define a metric")
1806 0 : });
1807 :
1808 2 : pub(crate) static WALRECEIVER_CANDIDATES_EVENTS: Lazy<IntCounterVec> = Lazy::new(|| {
1809 : register_int_counter_vec!(
1810 : "pageserver_walreceiver_candidates_events_total",
1811 : "Number of walreceiver candidate events",
1812 : &["event"]
1813 : )
1814 2 : .expect("failed to define a metric")
1815 2 : });
1816 :
1817 : pub(crate) static WALRECEIVER_CANDIDATES_ADDED: Lazy<IntCounter> =
1818 0 : Lazy::new(|| WALRECEIVER_CANDIDATES_EVENTS.with_label_values(&["add"]));
1819 :
1820 : pub(crate) static WALRECEIVER_CANDIDATES_REMOVED: Lazy<IntCounter> =
1821 2 : Lazy::new(|| WALRECEIVER_CANDIDATES_EVENTS.with_label_values(&["remove"]));
1822 :
1823 : // Metrics collected on WAL redo operations
1824 : //
1825 : // We collect the time spent in actual WAL redo ('redo'), and time waiting
1826 : // for access to the postgres process ('wait') since there is only one for
1827 : // each tenant.
1828 :
1829 : /// Time buckets are small because we want to be able to measure the
1830 : /// smallest redo processing times. These buckets allow us to measure down
1831 : /// to 5us, which equates to 200'000 pages/sec, which equates to 1.6GB/sec.
1832 : /// This is much better than the previous 5ms aka 200 pages/sec aka 1.6MB/sec.
1833 : ///
1834 : /// Values up to 1s are recorded because metrics show that we have redo
1835 : /// durations and lock times larger than 0.250s.
1836 : macro_rules! redo_histogram_time_buckets {
1837 : () => {
1838 : vec![
1839 : 0.000_005, 0.000_010, 0.000_025, 0.000_050, 0.000_100, 0.000_250, 0.000_500, 0.001_000,
1840 : 0.002_500, 0.005_000, 0.010_000, 0.025_000, 0.050_000, 0.100_000, 0.250_000, 0.500_000,
1841 : 1.000_000,
1842 : ]
1843 : };
1844 : }
1845 :
1846 : /// While we're at it, also measure the amount of records replayed in each
1847 : /// operation. We have a global 'total replayed' counter, but that's not
1848 : /// as useful as 'what is the skew for how many records we replay in one
1849 : /// operation'.
1850 : macro_rules! redo_histogram_count_buckets {
1851 : () => {
1852 : vec![0.0, 1.0, 2.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0]
1853 : };
1854 : }
1855 :
1856 : macro_rules! redo_bytes_histogram_count_buckets {
1857 : () => {
1858 : // powers of (2^.5), from 2^4.5 to 2^15 (22 buckets)
1859 : // rounded up to the next multiple of 8 to capture any MAXALIGNed record of that size, too.
1860 : vec![
1861 : 24.0, 32.0, 48.0, 64.0, 96.0, 128.0, 184.0, 256.0, 368.0, 512.0, 728.0, 1024.0, 1456.0,
1862 : 2048.0, 2904.0, 4096.0, 5800.0, 8192.0, 11592.0, 16384.0, 23176.0, 32768.0,
1863 : ]
1864 : };
1865 : }
1866 :
1867 : pub(crate) struct WalIngestMetrics {
1868 : pub(crate) bytes_received: IntCounter,
1869 : pub(crate) records_received: IntCounter,
1870 : pub(crate) records_committed: IntCounter,
1871 : pub(crate) records_filtered: IntCounter,
1872 : }
1873 :
1874 2 : pub(crate) static WAL_INGEST: Lazy<WalIngestMetrics> = Lazy::new(|| WalIngestMetrics {
1875 : bytes_received: register_int_counter!(
1876 : "pageserver_wal_ingest_bytes_received",
1877 : "Bytes of WAL ingested from safekeepers",
1878 : )
1879 2 : .unwrap(),
1880 : records_received: register_int_counter!(
1881 : "pageserver_wal_ingest_records_received",
1882 : "Number of WAL records received from safekeepers"
1883 : )
1884 2 : .expect("failed to define a metric"),
1885 : records_committed: register_int_counter!(
1886 : "pageserver_wal_ingest_records_committed",
1887 : "Number of WAL records which resulted in writes to pageserver storage"
1888 : )
1889 2 : .expect("failed to define a metric"),
1890 : records_filtered: register_int_counter!(
1891 : "pageserver_wal_ingest_records_filtered",
1892 : "Number of WAL records filtered out due to sharding"
1893 : )
1894 2 : .expect("failed to define a metric"),
1895 2 : });
1896 :
1897 6 : pub(crate) static WAL_REDO_TIME: Lazy<Histogram> = Lazy::new(|| {
1898 : register_histogram!(
1899 : "pageserver_wal_redo_seconds",
1900 : "Time spent on WAL redo",
1901 : redo_histogram_time_buckets!()
1902 : )
1903 6 : .expect("failed to define a metric")
1904 6 : });
1905 :
1906 6 : pub(crate) static WAL_REDO_RECORDS_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
1907 : register_histogram!(
1908 : "pageserver_wal_redo_records_histogram",
1909 : "Histogram of number of records replayed per redo in the Postgres WAL redo process",
1910 : redo_histogram_count_buckets!(),
1911 : )
1912 6 : .expect("failed to define a metric")
1913 6 : });
1914 :
1915 6 : pub(crate) static WAL_REDO_BYTES_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
1916 : register_histogram!(
1917 : "pageserver_wal_redo_bytes_histogram",
1918 : "Histogram of number of records replayed per redo sent to Postgres",
1919 : redo_bytes_histogram_count_buckets!(),
1920 : )
1921 6 : .expect("failed to define a metric")
1922 6 : });
1923 :
1924 : // FIXME: isn't this already included by WAL_REDO_RECORDS_HISTOGRAM which has _count?
1925 6 : pub(crate) static WAL_REDO_RECORD_COUNTER: Lazy<IntCounter> = Lazy::new(|| {
1926 : register_int_counter!(
1927 : "pageserver_replayed_wal_records_total",
1928 : "Number of WAL records replayed in WAL redo process"
1929 : )
1930 6 : .unwrap()
1931 6 : });
1932 :
1933 : #[rustfmt::skip]
1934 6 : pub(crate) static WAL_REDO_PROCESS_LAUNCH_DURATION_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
1935 : register_histogram!(
1936 : "pageserver_wal_redo_process_launch_duration",
1937 : "Histogram of the duration of successful WalRedoProcess::launch calls",
1938 : vec![
1939 : 0.0002, 0.0004, 0.0006, 0.0008, 0.0010,
1940 : 0.0020, 0.0040, 0.0060, 0.0080, 0.0100,
1941 : 0.0200, 0.0400, 0.0600, 0.0800, 0.1000,
1942 : 0.2000, 0.4000, 0.6000, 0.8000, 1.0000,
1943 : 1.5000, 2.0000, 2.5000, 3.0000, 4.0000, 10.0000
1944 : ],
1945 : )
1946 6 : .expect("failed to define a metric")
1947 6 : });
1948 :
1949 : pub(crate) struct WalRedoProcessCounters {
1950 : pub(crate) started: IntCounter,
1951 : pub(crate) killed_by_cause: enum_map::EnumMap<WalRedoKillCause, IntCounter>,
1952 : pub(crate) active_stderr_logger_tasks_started: IntCounter,
1953 : pub(crate) active_stderr_logger_tasks_finished: IntCounter,
1954 : }
1955 :
1956 18 : #[derive(Debug, enum_map::Enum, strum_macros::IntoStaticStr)]
1957 : pub(crate) enum WalRedoKillCause {
1958 : WalRedoProcessDrop,
1959 : NoLeakChildDrop,
1960 : Startup,
1961 : }
1962 :
1963 : impl Default for WalRedoProcessCounters {
1964 6 : fn default() -> Self {
1965 6 : let started = register_int_counter!(
1966 : "pageserver_wal_redo_process_started_total",
1967 : "Number of WAL redo processes started",
1968 : )
1969 6 : .unwrap();
1970 6 :
1971 6 : let killed = register_int_counter_vec!(
1972 : "pageserver_wal_redo_process_stopped_total",
1973 : "Number of WAL redo processes stopped",
1974 : &["cause"],
1975 : )
1976 6 : .unwrap();
1977 6 :
1978 6 : let active_stderr_logger_tasks_started = register_int_counter!(
1979 : "pageserver_walredo_stderr_logger_tasks_started_total",
1980 : "Number of active walredo stderr logger tasks that have started",
1981 : )
1982 6 : .unwrap();
1983 6 :
1984 6 : let active_stderr_logger_tasks_finished = register_int_counter!(
1985 : "pageserver_walredo_stderr_logger_tasks_finished_total",
1986 : "Number of active walredo stderr logger tasks that have finished",
1987 : )
1988 6 : .unwrap();
1989 6 :
1990 6 : Self {
1991 6 : started,
1992 18 : killed_by_cause: EnumMap::from_array(std::array::from_fn(|i| {
1993 18 : let cause = <WalRedoKillCause as enum_map::Enum>::from_usize(i);
1994 18 : let cause_str: &'static str = cause.into();
1995 18 : killed.with_label_values(&[cause_str])
1996 18 : })),
1997 6 : active_stderr_logger_tasks_started,
1998 6 : active_stderr_logger_tasks_finished,
1999 6 : }
2000 6 : }
2001 : }
2002 :
2003 : pub(crate) static WAL_REDO_PROCESS_COUNTERS: Lazy<WalRedoProcessCounters> =
2004 : Lazy::new(WalRedoProcessCounters::default);
2005 :
2006 : /// Similar to `prometheus::HistogramTimer` but does not record on drop.
2007 : pub(crate) struct StorageTimeMetricsTimer {
2008 : metrics: StorageTimeMetrics,
2009 : start: Instant,
2010 : }
2011 :
2012 : impl StorageTimeMetricsTimer {
2013 4605 : fn new(metrics: StorageTimeMetrics) -> Self {
2014 4605 : Self {
2015 4605 : metrics,
2016 4605 : start: Instant::now(),
2017 4605 : }
2018 4605 : }
2019 :
2020 : /// Record the time from creation to now.
2021 3501 : pub fn stop_and_record(self) {
2022 3501 : let duration = self.start.elapsed().as_secs_f64();
2023 3501 : self.metrics.timeline_sum.inc_by(duration);
2024 3501 : self.metrics.timeline_count.inc();
2025 3501 : self.metrics.global_histogram.observe(duration);
2026 3501 : }
2027 :
2028 : /// Turns this timer into a timer, which will always record -- usually this means recording
2029 : /// regardless an early `?` path was taken in a function.
2030 754 : pub(crate) fn record_on_drop(self) -> AlwaysRecordingStorageTimeMetricsTimer {
2031 754 : AlwaysRecordingStorageTimeMetricsTimer(Some(self))
2032 754 : }
2033 : }
2034 :
2035 : pub(crate) struct AlwaysRecordingStorageTimeMetricsTimer(Option<StorageTimeMetricsTimer>);
2036 :
2037 : impl Drop for AlwaysRecordingStorageTimeMetricsTimer {
2038 754 : fn drop(&mut self) {
2039 754 : if let Some(inner) = self.0.take() {
2040 754 : inner.stop_and_record();
2041 754 : }
2042 754 : }
2043 : }
2044 :
2045 : /// Timing facilities for an globally histogrammed metric, which is supported by per tenant and
2046 : /// timeline total sum and count.
2047 : #[derive(Clone, Debug)]
2048 : pub(crate) struct StorageTimeMetrics {
2049 : /// Sum of f64 seconds, per operation, tenant_id and timeline_id
2050 : timeline_sum: Counter,
2051 : /// Number of oeprations, per operation, tenant_id and timeline_id
2052 : timeline_count: IntCounter,
2053 : /// Global histogram having only the "operation" label.
2054 : global_histogram: Histogram,
2055 : }
2056 :
2057 : impl StorageTimeMetrics {
2058 3072 : pub fn new(
2059 3072 : operation: StorageTimeOperation,
2060 3072 : tenant_id: &str,
2061 3072 : shard_id: &str,
2062 3072 : timeline_id: &str,
2063 3072 : ) -> Self {
2064 3072 : let operation: &'static str = operation.into();
2065 3072 :
2066 3072 : let timeline_sum = STORAGE_TIME_SUM_PER_TIMELINE
2067 3072 : .get_metric_with_label_values(&[operation, tenant_id, shard_id, timeline_id])
2068 3072 : .unwrap();
2069 3072 : let timeline_count = STORAGE_TIME_COUNT_PER_TIMELINE
2070 3072 : .get_metric_with_label_values(&[operation, tenant_id, shard_id, timeline_id])
2071 3072 : .unwrap();
2072 3072 : let global_histogram = STORAGE_TIME_GLOBAL
2073 3072 : .get_metric_with_label_values(&[operation])
2074 3072 : .unwrap();
2075 3072 :
2076 3072 : StorageTimeMetrics {
2077 3072 : timeline_sum,
2078 3072 : timeline_count,
2079 3072 : global_histogram,
2080 3072 : }
2081 3072 : }
2082 :
2083 : /// Starts timing a new operation.
2084 : ///
2085 : /// Note: unlike `prometheus::HistogramTimer` the returned timer does not record on drop.
2086 4605 : pub fn start_timer(&self) -> StorageTimeMetricsTimer {
2087 4605 : StorageTimeMetricsTimer::new(self.clone())
2088 4605 : }
2089 : }
2090 :
2091 : #[derive(Debug)]
2092 : pub(crate) struct TimelineMetrics {
2093 : tenant_id: String,
2094 : shard_id: String,
2095 : timeline_id: String,
2096 : pub flush_time_histo: StorageTimeMetrics,
2097 : pub compact_time_histo: StorageTimeMetrics,
2098 : pub create_images_time_histo: StorageTimeMetrics,
2099 : pub logical_size_histo: StorageTimeMetrics,
2100 : pub imitate_logical_size_histo: StorageTimeMetrics,
2101 : pub load_layer_map_histo: StorageTimeMetrics,
2102 : pub garbage_collect_histo: StorageTimeMetrics,
2103 : pub find_gc_cutoffs_histo: StorageTimeMetrics,
2104 : pub last_record_gauge: IntGauge,
2105 : pub standby_horizon_gauge: IntGauge,
2106 : pub resident_physical_size_gauge: UIntGauge,
2107 : /// copy of LayeredTimeline.current_logical_size
2108 : pub current_logical_size_gauge: UIntGauge,
2109 : pub aux_file_size_gauge: IntGauge,
2110 : pub directory_entries_count_gauge: Lazy<UIntGauge, Box<dyn Send + Fn() -> UIntGauge>>,
2111 : pub evictions: IntCounter,
2112 : pub evictions_with_low_residence_duration: std::sync::RwLock<EvictionsWithLowResidenceDuration>,
2113 : /// Number of valid LSN leases.
2114 : pub valid_lsn_lease_count_gauge: UIntGauge,
2115 : shutdown: std::sync::atomic::AtomicBool,
2116 : }
2117 :
2118 : impl TimelineMetrics {
2119 384 : pub fn new(
2120 384 : tenant_shard_id: &TenantShardId,
2121 384 : timeline_id_raw: &TimelineId,
2122 384 : evictions_with_low_residence_duration_builder: EvictionsWithLowResidenceDurationBuilder,
2123 384 : ) -> Self {
2124 384 : let tenant_id = tenant_shard_id.tenant_id.to_string();
2125 384 : let shard_id = format!("{}", tenant_shard_id.shard_slug());
2126 384 : let timeline_id = timeline_id_raw.to_string();
2127 384 : let flush_time_histo = StorageTimeMetrics::new(
2128 384 : StorageTimeOperation::LayerFlush,
2129 384 : &tenant_id,
2130 384 : &shard_id,
2131 384 : &timeline_id,
2132 384 : );
2133 384 : let compact_time_histo = StorageTimeMetrics::new(
2134 384 : StorageTimeOperation::Compact,
2135 384 : &tenant_id,
2136 384 : &shard_id,
2137 384 : &timeline_id,
2138 384 : );
2139 384 : let create_images_time_histo = StorageTimeMetrics::new(
2140 384 : StorageTimeOperation::CreateImages,
2141 384 : &tenant_id,
2142 384 : &shard_id,
2143 384 : &timeline_id,
2144 384 : );
2145 384 : let logical_size_histo = StorageTimeMetrics::new(
2146 384 : StorageTimeOperation::LogicalSize,
2147 384 : &tenant_id,
2148 384 : &shard_id,
2149 384 : &timeline_id,
2150 384 : );
2151 384 : let imitate_logical_size_histo = StorageTimeMetrics::new(
2152 384 : StorageTimeOperation::ImitateLogicalSize,
2153 384 : &tenant_id,
2154 384 : &shard_id,
2155 384 : &timeline_id,
2156 384 : );
2157 384 : let load_layer_map_histo = StorageTimeMetrics::new(
2158 384 : StorageTimeOperation::LoadLayerMap,
2159 384 : &tenant_id,
2160 384 : &shard_id,
2161 384 : &timeline_id,
2162 384 : );
2163 384 : let garbage_collect_histo = StorageTimeMetrics::new(
2164 384 : StorageTimeOperation::Gc,
2165 384 : &tenant_id,
2166 384 : &shard_id,
2167 384 : &timeline_id,
2168 384 : );
2169 384 : let find_gc_cutoffs_histo = StorageTimeMetrics::new(
2170 384 : StorageTimeOperation::FindGcCutoffs,
2171 384 : &tenant_id,
2172 384 : &shard_id,
2173 384 : &timeline_id,
2174 384 : );
2175 384 : let last_record_gauge = LAST_RECORD_LSN
2176 384 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2177 384 : .unwrap();
2178 384 : let standby_horizon_gauge = STANDBY_HORIZON
2179 384 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2180 384 : .unwrap();
2181 384 : let resident_physical_size_gauge = RESIDENT_PHYSICAL_SIZE
2182 384 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2183 384 : .unwrap();
2184 384 : // TODO: we shouldn't expose this metric
2185 384 : let current_logical_size_gauge = CURRENT_LOGICAL_SIZE
2186 384 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2187 384 : .unwrap();
2188 384 : let aux_file_size_gauge = AUX_FILE_SIZE
2189 384 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2190 384 : .unwrap();
2191 384 : // TODO use impl Trait syntax here once we have ability to use it: https://github.com/rust-lang/rust/issues/63065
2192 384 : let directory_entries_count_gauge_closure = {
2193 384 : let tenant_shard_id = *tenant_shard_id;
2194 384 : let timeline_id_raw = *timeline_id_raw;
2195 0 : move || {
2196 0 : let tenant_id = tenant_shard_id.tenant_id.to_string();
2197 0 : let shard_id = format!("{}", tenant_shard_id.shard_slug());
2198 0 : let timeline_id = timeline_id_raw.to_string();
2199 0 : let gauge: UIntGauge = DIRECTORY_ENTRIES_COUNT
2200 0 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2201 0 : .unwrap();
2202 0 : gauge
2203 0 : }
2204 : };
2205 384 : let directory_entries_count_gauge: Lazy<UIntGauge, Box<dyn Send + Fn() -> UIntGauge>> =
2206 384 : Lazy::new(Box::new(directory_entries_count_gauge_closure));
2207 384 : let evictions = EVICTIONS
2208 384 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2209 384 : .unwrap();
2210 384 : let evictions_with_low_residence_duration = evictions_with_low_residence_duration_builder
2211 384 : .build(&tenant_id, &shard_id, &timeline_id);
2212 384 :
2213 384 : let valid_lsn_lease_count_gauge = VALID_LSN_LEASE_COUNT
2214 384 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2215 384 : .unwrap();
2216 384 :
2217 384 : TimelineMetrics {
2218 384 : tenant_id,
2219 384 : shard_id,
2220 384 : timeline_id,
2221 384 : flush_time_histo,
2222 384 : compact_time_histo,
2223 384 : create_images_time_histo,
2224 384 : logical_size_histo,
2225 384 : imitate_logical_size_histo,
2226 384 : garbage_collect_histo,
2227 384 : find_gc_cutoffs_histo,
2228 384 : load_layer_map_histo,
2229 384 : last_record_gauge,
2230 384 : standby_horizon_gauge,
2231 384 : resident_physical_size_gauge,
2232 384 : current_logical_size_gauge,
2233 384 : aux_file_size_gauge,
2234 384 : directory_entries_count_gauge,
2235 384 : evictions,
2236 384 : evictions_with_low_residence_duration: std::sync::RwLock::new(
2237 384 : evictions_with_low_residence_duration,
2238 384 : ),
2239 384 : valid_lsn_lease_count_gauge,
2240 384 : shutdown: std::sync::atomic::AtomicBool::default(),
2241 384 : }
2242 384 : }
2243 :
2244 1470 : pub(crate) fn record_new_file_metrics(&self, sz: u64) {
2245 1470 : self.resident_physical_size_add(sz);
2246 1470 : }
2247 :
2248 448 : pub(crate) fn resident_physical_size_sub(&self, sz: u64) {
2249 448 : self.resident_physical_size_gauge.sub(sz);
2250 448 : crate::metrics::RESIDENT_PHYSICAL_SIZE_GLOBAL.sub(sz);
2251 448 : }
2252 :
2253 1500 : pub(crate) fn resident_physical_size_add(&self, sz: u64) {
2254 1500 : self.resident_physical_size_gauge.add(sz);
2255 1500 : crate::metrics::RESIDENT_PHYSICAL_SIZE_GLOBAL.add(sz);
2256 1500 : }
2257 :
2258 8 : pub(crate) fn resident_physical_size_get(&self) -> u64 {
2259 8 : self.resident_physical_size_gauge.get()
2260 8 : }
2261 :
2262 8 : pub(crate) fn shutdown(&self) {
2263 8 : let was_shutdown = self
2264 8 : .shutdown
2265 8 : .swap(true, std::sync::atomic::Ordering::Relaxed);
2266 8 :
2267 8 : if was_shutdown {
2268 : // this happens on tenant deletion because tenant first shuts down timelines, then
2269 : // invokes timeline deletion which first shuts down the timeline again.
2270 : // TODO: this can be removed once https://github.com/neondatabase/neon/issues/5080
2271 0 : return;
2272 8 : }
2273 8 :
2274 8 : let tenant_id = &self.tenant_id;
2275 8 : let timeline_id = &self.timeline_id;
2276 8 : let shard_id = &self.shard_id;
2277 8 : let _ = LAST_RECORD_LSN.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2278 8 : let _ = STANDBY_HORIZON.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2279 8 : {
2280 8 : RESIDENT_PHYSICAL_SIZE_GLOBAL.sub(self.resident_physical_size_get());
2281 8 : let _ = RESIDENT_PHYSICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2282 8 : }
2283 8 : let _ = CURRENT_LOGICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2284 8 : if let Some(metric) = Lazy::get(&DIRECTORY_ENTRIES_COUNT) {
2285 0 : let _ = metric.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2286 8 : }
2287 8 : let _ = EVICTIONS.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2288 8 : let _ = AUX_FILE_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2289 8 : let _ = VALID_LSN_LEASE_COUNT.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2290 8 :
2291 8 : self.evictions_with_low_residence_duration
2292 8 : .write()
2293 8 : .unwrap()
2294 8 : .remove(tenant_id, shard_id, timeline_id);
2295 :
2296 : // The following metrics are born outside of the TimelineMetrics lifecycle but still
2297 : // removed at the end of it. The idea is to have the metrics outlive the
2298 : // entity during which they're observed, e.g., the smgr metrics shall
2299 : // outlive an individual smgr connection, but not the timeline.
2300 :
2301 72 : for op in StorageTimeOperation::VARIANTS {
2302 64 : let _ = STORAGE_TIME_SUM_PER_TIMELINE.remove_label_values(&[
2303 64 : op,
2304 64 : tenant_id,
2305 64 : shard_id,
2306 64 : timeline_id,
2307 64 : ]);
2308 64 : let _ = STORAGE_TIME_COUNT_PER_TIMELINE.remove_label_values(&[
2309 64 : op,
2310 64 : tenant_id,
2311 64 : shard_id,
2312 64 : timeline_id,
2313 64 : ]);
2314 64 : }
2315 :
2316 24 : for op in STORAGE_IO_SIZE_OPERATIONS {
2317 16 : let _ = STORAGE_IO_SIZE.remove_label_values(&[op, tenant_id, shard_id, timeline_id]);
2318 16 : }
2319 :
2320 48 : for op in SmgrQueryType::iter() {
2321 40 : let _ = SMGR_QUERY_TIME_PER_TENANT_TIMELINE.remove_label_values(&[
2322 40 : op.into(),
2323 40 : tenant_id,
2324 40 : shard_id,
2325 40 : timeline_id,
2326 40 : ]);
2327 40 : }
2328 8 : }
2329 : }
2330 :
2331 6 : pub(crate) fn remove_tenant_metrics(tenant_shard_id: &TenantShardId) {
2332 6 : // Only shard zero deals in synthetic sizes
2333 6 : if tenant_shard_id.is_shard_zero() {
2334 6 : let tid = tenant_shard_id.tenant_id.to_string();
2335 6 : let _ = TENANT_SYNTHETIC_SIZE_METRIC.remove_label_values(&[&tid]);
2336 6 : }
2337 :
2338 : // we leave the BROKEN_TENANTS_SET entry if any
2339 6 : }
2340 :
2341 : use futures::Future;
2342 : use pin_project_lite::pin_project;
2343 : use std::collections::HashMap;
2344 : use std::num::NonZeroUsize;
2345 : use std::pin::Pin;
2346 : use std::sync::atomic::AtomicU64;
2347 : use std::sync::{Arc, Mutex};
2348 : use std::task::{Context, Poll};
2349 : use std::time::{Duration, Instant};
2350 :
2351 : use crate::context::{PageContentKind, RequestContext};
2352 : use crate::task_mgr::TaskKind;
2353 : use crate::tenant::mgr::TenantSlot;
2354 :
2355 : /// Maintain a per timeline gauge in addition to the global gauge.
2356 : pub(crate) struct PerTimelineRemotePhysicalSizeGauge {
2357 : last_set: AtomicU64,
2358 : gauge: UIntGauge,
2359 : }
2360 :
2361 : impl PerTimelineRemotePhysicalSizeGauge {
2362 394 : fn new(per_timeline_gauge: UIntGauge) -> Self {
2363 394 : Self {
2364 394 : last_set: AtomicU64::new(0),
2365 394 : gauge: per_timeline_gauge,
2366 394 : }
2367 394 : }
2368 1733 : pub(crate) fn set(&self, sz: u64) {
2369 1733 : self.gauge.set(sz);
2370 1733 : let prev = self.last_set.swap(sz, std::sync::atomic::Ordering::Relaxed);
2371 1733 : if sz < prev {
2372 26 : REMOTE_PHYSICAL_SIZE_GLOBAL.sub(prev - sz);
2373 1707 : } else {
2374 1707 : REMOTE_PHYSICAL_SIZE_GLOBAL.add(sz - prev);
2375 1707 : };
2376 1733 : }
2377 2 : pub(crate) fn get(&self) -> u64 {
2378 2 : self.gauge.get()
2379 2 : }
2380 : }
2381 :
2382 : impl Drop for PerTimelineRemotePhysicalSizeGauge {
2383 18 : fn drop(&mut self) {
2384 18 : REMOTE_PHYSICAL_SIZE_GLOBAL.sub(self.last_set.load(std::sync::atomic::Ordering::Relaxed));
2385 18 : }
2386 : }
2387 :
2388 : pub(crate) struct RemoteTimelineClientMetrics {
2389 : tenant_id: String,
2390 : shard_id: String,
2391 : timeline_id: String,
2392 : pub(crate) remote_physical_size_gauge: PerTimelineRemotePhysicalSizeGauge,
2393 : calls: Mutex<HashMap<(&'static str, &'static str), IntCounterPair>>,
2394 : bytes_started_counter: Mutex<HashMap<(&'static str, &'static str), IntCounter>>,
2395 : bytes_finished_counter: Mutex<HashMap<(&'static str, &'static str), IntCounter>>,
2396 : }
2397 :
2398 : impl RemoteTimelineClientMetrics {
2399 394 : pub fn new(tenant_shard_id: &TenantShardId, timeline_id: &TimelineId) -> Self {
2400 394 : let tenant_id_str = tenant_shard_id.tenant_id.to_string();
2401 394 : let shard_id_str = format!("{}", tenant_shard_id.shard_slug());
2402 394 : let timeline_id_str = timeline_id.to_string();
2403 394 :
2404 394 : let remote_physical_size_gauge = PerTimelineRemotePhysicalSizeGauge::new(
2405 394 : REMOTE_PHYSICAL_SIZE
2406 394 : .get_metric_with_label_values(&[&tenant_id_str, &shard_id_str, &timeline_id_str])
2407 394 : .unwrap(),
2408 394 : );
2409 394 :
2410 394 : RemoteTimelineClientMetrics {
2411 394 : tenant_id: tenant_id_str,
2412 394 : shard_id: shard_id_str,
2413 394 : timeline_id: timeline_id_str,
2414 394 : calls: Mutex::new(HashMap::default()),
2415 394 : bytes_started_counter: Mutex::new(HashMap::default()),
2416 394 : bytes_finished_counter: Mutex::new(HashMap::default()),
2417 394 : remote_physical_size_gauge,
2418 394 : }
2419 394 : }
2420 :
2421 2534 : pub fn remote_operation_time(
2422 2534 : &self,
2423 2534 : file_kind: &RemoteOpFileKind,
2424 2534 : op_kind: &RemoteOpKind,
2425 2534 : status: &'static str,
2426 2534 : ) -> Histogram {
2427 2534 : let key = (file_kind.as_str(), op_kind.as_str(), status);
2428 2534 : REMOTE_OPERATION_TIME
2429 2534 : .get_metric_with_label_values(&[key.0, key.1, key.2])
2430 2534 : .unwrap()
2431 2534 : }
2432 :
2433 6078 : fn calls_counter_pair(
2434 6078 : &self,
2435 6078 : file_kind: &RemoteOpFileKind,
2436 6078 : op_kind: &RemoteOpKind,
2437 6078 : ) -> IntCounterPair {
2438 6078 : let mut guard = self.calls.lock().unwrap();
2439 6078 : let key = (file_kind.as_str(), op_kind.as_str());
2440 6078 : let metric = guard.entry(key).or_insert_with(move || {
2441 681 : REMOTE_TIMELINE_CLIENT_CALLS
2442 681 : .get_metric_with_label_values(&[
2443 681 : &self.tenant_id,
2444 681 : &self.shard_id,
2445 681 : &self.timeline_id,
2446 681 : key.0,
2447 681 : key.1,
2448 681 : ])
2449 681 : .unwrap()
2450 6078 : });
2451 6078 : metric.clone()
2452 6078 : }
2453 :
2454 1478 : fn bytes_started_counter(
2455 1478 : &self,
2456 1478 : file_kind: &RemoteOpFileKind,
2457 1478 : op_kind: &RemoteOpKind,
2458 1478 : ) -> IntCounter {
2459 1478 : let mut guard = self.bytes_started_counter.lock().unwrap();
2460 1478 : let key = (file_kind.as_str(), op_kind.as_str());
2461 1478 : let metric = guard.entry(key).or_insert_with(move || {
2462 258 : REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER
2463 258 : .get_metric_with_label_values(&[
2464 258 : &self.tenant_id,
2465 258 : &self.shard_id,
2466 258 : &self.timeline_id,
2467 258 : key.0,
2468 258 : key.1,
2469 258 : ])
2470 258 : .unwrap()
2471 1478 : });
2472 1478 : metric.clone()
2473 1478 : }
2474 :
2475 2637 : fn bytes_finished_counter(
2476 2637 : &self,
2477 2637 : file_kind: &RemoteOpFileKind,
2478 2637 : op_kind: &RemoteOpKind,
2479 2637 : ) -> IntCounter {
2480 2637 : let mut guard = self.bytes_finished_counter.lock().unwrap();
2481 2637 : let key = (file_kind.as_str(), op_kind.as_str());
2482 2637 : let metric = guard.entry(key).or_insert_with(move || {
2483 258 : REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER
2484 258 : .get_metric_with_label_values(&[
2485 258 : &self.tenant_id,
2486 258 : &self.shard_id,
2487 258 : &self.timeline_id,
2488 258 : key.0,
2489 258 : key.1,
2490 258 : ])
2491 258 : .unwrap()
2492 2637 : });
2493 2637 : metric.clone()
2494 2637 : }
2495 : }
2496 :
2497 : #[cfg(test)]
2498 : impl RemoteTimelineClientMetrics {
2499 6 : pub fn get_bytes_started_counter_value(
2500 6 : &self,
2501 6 : file_kind: &RemoteOpFileKind,
2502 6 : op_kind: &RemoteOpKind,
2503 6 : ) -> Option<u64> {
2504 6 : let guard = self.bytes_started_counter.lock().unwrap();
2505 6 : let key = (file_kind.as_str(), op_kind.as_str());
2506 6 : guard.get(&key).map(|counter| counter.get())
2507 6 : }
2508 :
2509 6 : pub fn get_bytes_finished_counter_value(
2510 6 : &self,
2511 6 : file_kind: &RemoteOpFileKind,
2512 6 : op_kind: &RemoteOpKind,
2513 6 : ) -> Option<u64> {
2514 6 : let guard = self.bytes_finished_counter.lock().unwrap();
2515 6 : let key = (file_kind.as_str(), op_kind.as_str());
2516 6 : guard.get(&key).map(|counter| counter.get())
2517 6 : }
2518 : }
2519 :
2520 : /// See [`RemoteTimelineClientMetrics::call_begin`].
2521 : #[must_use]
2522 : pub(crate) struct RemoteTimelineClientCallMetricGuard {
2523 : /// Decremented on drop.
2524 : calls_counter_pair: Option<IntCounterPair>,
2525 : /// If Some(), this references the bytes_finished metric, and we increment it by the given `u64` on drop.
2526 : bytes_finished: Option<(IntCounter, u64)>,
2527 : }
2528 :
2529 : impl RemoteTimelineClientCallMetricGuard {
2530 : /// Consume this guard object without performing the metric updates it would do on `drop()`.
2531 : /// The caller vouches to do the metric updates manually.
2532 3328 : pub fn will_decrement_manually(mut self) {
2533 3328 : let RemoteTimelineClientCallMetricGuard {
2534 3328 : calls_counter_pair,
2535 3328 : bytes_finished,
2536 3328 : } = &mut self;
2537 3328 : calls_counter_pair.take();
2538 3328 : bytes_finished.take();
2539 3328 : }
2540 : }
2541 :
2542 : impl Drop for RemoteTimelineClientCallMetricGuard {
2543 3354 : fn drop(&mut self) {
2544 3354 : let RemoteTimelineClientCallMetricGuard {
2545 3354 : calls_counter_pair,
2546 3354 : bytes_finished,
2547 3354 : } = self;
2548 3354 : if let Some(guard) = calls_counter_pair.take() {
2549 26 : guard.dec();
2550 3328 : }
2551 3354 : if let Some((bytes_finished_metric, value)) = bytes_finished {
2552 0 : bytes_finished_metric.inc_by(*value);
2553 3354 : }
2554 3354 : }
2555 : }
2556 :
2557 : /// The enum variants communicate to the [`RemoteTimelineClientMetrics`] whether to
2558 : /// track the byte size of this call in applicable metric(s).
2559 : pub(crate) enum RemoteTimelineClientMetricsCallTrackSize {
2560 : /// Do not account for this call's byte size in any metrics.
2561 : /// The `reason` field is there to make the call sites self-documenting
2562 : /// about why they don't need the metric.
2563 : DontTrackSize { reason: &'static str },
2564 : /// Track the byte size of the call in applicable metric(s).
2565 : Bytes(u64),
2566 : }
2567 :
2568 : impl RemoteTimelineClientMetrics {
2569 : /// Update the metrics that change when a call to the remote timeline client instance starts.
2570 : ///
2571 : /// Drop the returned guard object once the operation is finished to updates corresponding metrics that track completions.
2572 : /// Or, use [`RemoteTimelineClientCallMetricGuard::will_decrement_manually`] and [`call_end`](Self::call_end) if that
2573 : /// is more suitable.
2574 : /// Never do both.
2575 3354 : pub(crate) fn call_begin(
2576 3354 : &self,
2577 3354 : file_kind: &RemoteOpFileKind,
2578 3354 : op_kind: &RemoteOpKind,
2579 3354 : size: RemoteTimelineClientMetricsCallTrackSize,
2580 3354 : ) -> RemoteTimelineClientCallMetricGuard {
2581 3354 : let calls_counter_pair = self.calls_counter_pair(file_kind, op_kind);
2582 3354 : calls_counter_pair.inc();
2583 :
2584 3354 : let bytes_finished = match size {
2585 1876 : RemoteTimelineClientMetricsCallTrackSize::DontTrackSize { reason: _reason } => {
2586 1876 : // nothing to do
2587 1876 : None
2588 : }
2589 1478 : RemoteTimelineClientMetricsCallTrackSize::Bytes(size) => {
2590 1478 : self.bytes_started_counter(file_kind, op_kind).inc_by(size);
2591 1478 : let finished_counter = self.bytes_finished_counter(file_kind, op_kind);
2592 1478 : Some((finished_counter, size))
2593 : }
2594 : };
2595 3354 : RemoteTimelineClientCallMetricGuard {
2596 3354 : calls_counter_pair: Some(calls_counter_pair),
2597 3354 : bytes_finished,
2598 3354 : }
2599 3354 : }
2600 :
2601 : /// Manually udpate the metrics that track completions, instead of using the guard object.
2602 : /// Using the guard object is generally preferable.
2603 : /// See [`call_begin`](Self::call_begin) for more context.
2604 2724 : pub(crate) fn call_end(
2605 2724 : &self,
2606 2724 : file_kind: &RemoteOpFileKind,
2607 2724 : op_kind: &RemoteOpKind,
2608 2724 : size: RemoteTimelineClientMetricsCallTrackSize,
2609 2724 : ) {
2610 2724 : let calls_counter_pair = self.calls_counter_pair(file_kind, op_kind);
2611 2724 : calls_counter_pair.dec();
2612 2724 : match size {
2613 1565 : RemoteTimelineClientMetricsCallTrackSize::DontTrackSize { reason: _reason } => {}
2614 1159 : RemoteTimelineClientMetricsCallTrackSize::Bytes(size) => {
2615 1159 : self.bytes_finished_counter(file_kind, op_kind).inc_by(size);
2616 1159 : }
2617 : }
2618 2724 : }
2619 : }
2620 :
2621 : impl Drop for RemoteTimelineClientMetrics {
2622 18 : fn drop(&mut self) {
2623 18 : let RemoteTimelineClientMetrics {
2624 18 : tenant_id,
2625 18 : shard_id,
2626 18 : timeline_id,
2627 18 : remote_physical_size_gauge,
2628 18 : calls,
2629 18 : bytes_started_counter,
2630 18 : bytes_finished_counter,
2631 18 : } = self;
2632 24 : for ((a, b), _) in calls.get_mut().unwrap().drain() {
2633 24 : let mut res = [Ok(()), Ok(())];
2634 24 : REMOTE_TIMELINE_CLIENT_CALLS
2635 24 : .remove_label_values(&mut res, &[tenant_id, shard_id, timeline_id, a, b]);
2636 24 : // don't care about results
2637 24 : }
2638 18 : for ((a, b), _) in bytes_started_counter.get_mut().unwrap().drain() {
2639 6 : let _ = REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER.remove_label_values(&[
2640 6 : tenant_id,
2641 6 : shard_id,
2642 6 : timeline_id,
2643 6 : a,
2644 6 : b,
2645 6 : ]);
2646 6 : }
2647 18 : for ((a, b), _) in bytes_finished_counter.get_mut().unwrap().drain() {
2648 6 : let _ = REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER.remove_label_values(&[
2649 6 : tenant_id,
2650 6 : shard_id,
2651 6 : timeline_id,
2652 6 : a,
2653 6 : b,
2654 6 : ]);
2655 6 : }
2656 18 : {
2657 18 : let _ = remote_physical_size_gauge; // use to avoid 'unused' warning in desctructuring above
2658 18 : let _ = REMOTE_PHYSICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2659 18 : }
2660 18 : }
2661 : }
2662 :
2663 : /// Wrapper future that measures the time spent by a remote storage operation,
2664 : /// and records the time and success/failure as a prometheus metric.
2665 : pub(crate) trait MeasureRemoteOp: Sized {
2666 2714 : fn measure_remote_op(
2667 2714 : self,
2668 2714 : file_kind: RemoteOpFileKind,
2669 2714 : op: RemoteOpKind,
2670 2714 : metrics: Arc<RemoteTimelineClientMetrics>,
2671 2714 : ) -> MeasuredRemoteOp<Self> {
2672 2714 : let start = Instant::now();
2673 2714 : MeasuredRemoteOp {
2674 2714 : inner: self,
2675 2714 : file_kind,
2676 2714 : op,
2677 2714 : start,
2678 2714 : metrics,
2679 2714 : }
2680 2714 : }
2681 : }
2682 :
2683 : impl<T: Sized> MeasureRemoteOp for T {}
2684 :
2685 : pin_project! {
2686 : pub(crate) struct MeasuredRemoteOp<F>
2687 : {
2688 : #[pin]
2689 : inner: F,
2690 : file_kind: RemoteOpFileKind,
2691 : op: RemoteOpKind,
2692 : start: Instant,
2693 : metrics: Arc<RemoteTimelineClientMetrics>,
2694 : }
2695 : }
2696 :
2697 : impl<F: Future<Output = Result<O, E>>, O, E> Future for MeasuredRemoteOp<F> {
2698 : type Output = Result<O, E>;
2699 :
2700 39728 : fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
2701 39728 : let this = self.project();
2702 39728 : let poll_result = this.inner.poll(cx);
2703 39728 : if let Poll::Ready(ref res) = poll_result {
2704 2534 : let duration = this.start.elapsed();
2705 2534 : let status = if res.is_ok() { &"success" } else { &"failure" };
2706 2534 : this.metrics
2707 2534 : .remote_operation_time(this.file_kind, this.op, status)
2708 2534 : .observe(duration.as_secs_f64());
2709 37194 : }
2710 39728 : poll_result
2711 39728 : }
2712 : }
2713 :
2714 : pub mod tokio_epoll_uring {
2715 : use metrics::{register_int_counter, UIntGauge};
2716 : use once_cell::sync::Lazy;
2717 :
2718 : pub struct Collector {
2719 : descs: Vec<metrics::core::Desc>,
2720 : systems_created: UIntGauge,
2721 : systems_destroyed: UIntGauge,
2722 : }
2723 :
2724 : impl metrics::core::Collector for Collector {
2725 0 : fn desc(&self) -> Vec<&metrics::core::Desc> {
2726 0 : self.descs.iter().collect()
2727 0 : }
2728 :
2729 0 : fn collect(&self) -> Vec<metrics::proto::MetricFamily> {
2730 0 : let mut mfs = Vec::with_capacity(Self::NMETRICS);
2731 0 : let tokio_epoll_uring::metrics::Metrics {
2732 0 : systems_created,
2733 0 : systems_destroyed,
2734 0 : } = tokio_epoll_uring::metrics::global();
2735 0 : self.systems_created.set(systems_created);
2736 0 : mfs.extend(self.systems_created.collect());
2737 0 : self.systems_destroyed.set(systems_destroyed);
2738 0 : mfs.extend(self.systems_destroyed.collect());
2739 0 : mfs
2740 0 : }
2741 : }
2742 :
2743 : impl Collector {
2744 : const NMETRICS: usize = 2;
2745 :
2746 : #[allow(clippy::new_without_default)]
2747 0 : pub fn new() -> Self {
2748 0 : let mut descs = Vec::new();
2749 0 :
2750 0 : let systems_created = UIntGauge::new(
2751 0 : "pageserver_tokio_epoll_uring_systems_created",
2752 0 : "counter of tokio-epoll-uring systems that were created",
2753 0 : )
2754 0 : .unwrap();
2755 0 : descs.extend(
2756 0 : metrics::core::Collector::desc(&systems_created)
2757 0 : .into_iter()
2758 0 : .cloned(),
2759 0 : );
2760 0 :
2761 0 : let systems_destroyed = UIntGauge::new(
2762 0 : "pageserver_tokio_epoll_uring_systems_destroyed",
2763 0 : "counter of tokio-epoll-uring systems that were destroyed",
2764 0 : )
2765 0 : .unwrap();
2766 0 : descs.extend(
2767 0 : metrics::core::Collector::desc(&systems_destroyed)
2768 0 : .into_iter()
2769 0 : .cloned(),
2770 0 : );
2771 0 :
2772 0 : Self {
2773 0 : descs,
2774 0 : systems_created,
2775 0 : systems_destroyed,
2776 0 : }
2777 0 : }
2778 : }
2779 :
2780 82 : pub(crate) static THREAD_LOCAL_LAUNCH_SUCCESSES: Lazy<metrics::IntCounter> = Lazy::new(|| {
2781 : register_int_counter!(
2782 : "pageserver_tokio_epoll_uring_pageserver_thread_local_launch_success_count",
2783 : "Number of times where thread_local_system creation spanned multiple executor threads",
2784 : )
2785 82 : .unwrap()
2786 82 : });
2787 :
2788 0 : pub(crate) static THREAD_LOCAL_LAUNCH_FAILURES: Lazy<metrics::IntCounter> = Lazy::new(|| {
2789 : register_int_counter!(
2790 : "pageserver_tokio_epoll_uring_pageserver_thread_local_launch_failures_count",
2791 : "Number of times thread_local_system creation failed and was retried after back-off.",
2792 : )
2793 0 : .unwrap()
2794 0 : });
2795 : }
2796 :
2797 : pub(crate) mod tenant_throttling {
2798 : use metrics::{register_int_counter_vec, IntCounter};
2799 : use once_cell::sync::Lazy;
2800 :
2801 : use crate::tenant::{self, throttle::Metric};
2802 :
2803 : pub(crate) struct TimelineGet {
2804 : wait_time: IntCounter,
2805 : count: IntCounter,
2806 : }
2807 :
2808 144 : pub(crate) static TIMELINE_GET: Lazy<TimelineGet> = Lazy::new(|| {
2809 144 : static WAIT_USECS: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
2810 144 : register_int_counter_vec!(
2811 144 : "pageserver_tenant_throttling_wait_usecs_sum_global",
2812 144 : "Sum of microseconds that tenants spent waiting for a tenant throttle of a given kind.",
2813 144 : &["kind"]
2814 144 : )
2815 144 : .unwrap()
2816 144 : });
2817 144 :
2818 144 : static WAIT_COUNT: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
2819 144 : register_int_counter_vec!(
2820 144 : "pageserver_tenant_throttling_count_global",
2821 144 : "Count of tenant throttlings, by kind of throttle.",
2822 144 : &["kind"]
2823 144 : )
2824 144 : .unwrap()
2825 144 : });
2826 144 :
2827 144 : let kind = "timeline_get";
2828 144 : TimelineGet {
2829 144 : wait_time: WAIT_USECS.with_label_values(&[kind]),
2830 144 : count: WAIT_COUNT.with_label_values(&[kind]),
2831 144 : }
2832 144 : });
2833 :
2834 : impl Metric for &'static TimelineGet {
2835 : #[inline(always)]
2836 0 : fn observe_throttling(
2837 0 : &self,
2838 0 : tenant::throttle::Observation { wait_time }: &tenant::throttle::Observation,
2839 0 : ) {
2840 0 : let val = u64::try_from(wait_time.as_micros()).unwrap();
2841 0 : self.wait_time.inc_by(val);
2842 0 : self.count.inc();
2843 0 : }
2844 : }
2845 : }
2846 :
2847 : pub(crate) mod disk_usage_based_eviction {
2848 : use super::*;
2849 :
2850 : pub(crate) struct Metrics {
2851 : pub(crate) tenant_collection_time: Histogram,
2852 : pub(crate) tenant_layer_count: Histogram,
2853 : pub(crate) layers_collected: IntCounter,
2854 : pub(crate) layers_selected: IntCounter,
2855 : pub(crate) layers_evicted: IntCounter,
2856 : }
2857 :
2858 : impl Default for Metrics {
2859 0 : fn default() -> Self {
2860 0 : let tenant_collection_time = register_histogram!(
2861 : "pageserver_disk_usage_based_eviction_tenant_collection_seconds",
2862 : "Time spent collecting layers from a tenant -- not normalized by collected layer amount",
2863 : vec![0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0, 10.0]
2864 : )
2865 0 : .unwrap();
2866 0 :
2867 0 : let tenant_layer_count = register_histogram!(
2868 : "pageserver_disk_usage_based_eviction_tenant_collected_layers",
2869 : "Amount of layers gathered from a tenant",
2870 : vec![5.0, 50.0, 500.0, 5000.0, 50000.0]
2871 : )
2872 0 : .unwrap();
2873 0 :
2874 0 : let layers_collected = register_int_counter!(
2875 : "pageserver_disk_usage_based_eviction_collected_layers_total",
2876 : "Amount of layers collected"
2877 : )
2878 0 : .unwrap();
2879 0 :
2880 0 : let layers_selected = register_int_counter!(
2881 : "pageserver_disk_usage_based_eviction_select_layers_total",
2882 : "Amount of layers selected"
2883 : )
2884 0 : .unwrap();
2885 0 :
2886 0 : let layers_evicted = register_int_counter!(
2887 : "pageserver_disk_usage_based_eviction_evicted_layers_total",
2888 : "Amount of layers successfully evicted"
2889 : )
2890 0 : .unwrap();
2891 0 :
2892 0 : Self {
2893 0 : tenant_collection_time,
2894 0 : tenant_layer_count,
2895 0 : layers_collected,
2896 0 : layers_selected,
2897 0 : layers_evicted,
2898 0 : }
2899 0 : }
2900 : }
2901 :
2902 : pub(crate) static METRICS: Lazy<Metrics> = Lazy::new(Metrics::default);
2903 : }
2904 :
2905 138 : static TOKIO_EXECUTOR_THREAD_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
2906 : register_uint_gauge_vec!(
2907 : "pageserver_tokio_executor_thread_configured_count",
2908 : "Total number of configued tokio executor threads in the process.
2909 : The `setup` label denotes whether we're running with multiple runtimes or a single runtime.",
2910 : &["setup"],
2911 : )
2912 138 : .unwrap()
2913 138 : });
2914 :
2915 138 : pub(crate) fn set_tokio_runtime_setup(setup: &str, num_threads: NonZeroUsize) {
2916 138 : static SERIALIZE: std::sync::Mutex<()> = std::sync::Mutex::new(());
2917 138 : let _guard = SERIALIZE.lock().unwrap();
2918 138 : TOKIO_EXECUTOR_THREAD_COUNT.reset();
2919 138 : TOKIO_EXECUTOR_THREAD_COUNT
2920 138 : .get_metric_with_label_values(&[setup])
2921 138 : .unwrap()
2922 138 : .set(u64::try_from(num_threads.get()).unwrap());
2923 138 : }
2924 :
2925 0 : pub fn preinitialize_metrics() {
2926 0 : // Python tests need these and on some we do alerting.
2927 0 : //
2928 0 : // FIXME(4813): make it so that we have no top level metrics as this fn will easily fall out of
2929 0 : // order:
2930 0 : // - global metrics reside in a Lazy<PageserverMetrics>
2931 0 : // - access via crate::metrics::PS_METRICS.some_metric.inc()
2932 0 : // - could move the statics into TimelineMetrics::new()?
2933 0 :
2934 0 : // counters
2935 0 : [
2936 0 : &UNEXPECTED_ONDEMAND_DOWNLOADS,
2937 0 : &WALRECEIVER_STARTED_CONNECTIONS,
2938 0 : &WALRECEIVER_BROKER_UPDATES,
2939 0 : &WALRECEIVER_CANDIDATES_ADDED,
2940 0 : &WALRECEIVER_CANDIDATES_REMOVED,
2941 0 : &tokio_epoll_uring::THREAD_LOCAL_LAUNCH_FAILURES,
2942 0 : &tokio_epoll_uring::THREAD_LOCAL_LAUNCH_SUCCESSES,
2943 0 : &REMOTE_ONDEMAND_DOWNLOADED_LAYERS,
2944 0 : &REMOTE_ONDEMAND_DOWNLOADED_BYTES,
2945 0 : ]
2946 0 : .into_iter()
2947 0 : .for_each(|c| {
2948 0 : Lazy::force(c);
2949 0 : });
2950 0 :
2951 0 : // Deletion queue stats
2952 0 : Lazy::force(&DELETION_QUEUE);
2953 0 :
2954 0 : // Tenant stats
2955 0 : Lazy::force(&TENANT);
2956 0 :
2957 0 : // Tenant manager stats
2958 0 : Lazy::force(&TENANT_MANAGER);
2959 0 :
2960 0 : Lazy::force(&crate::tenant::storage_layer::layer::LAYER_IMPL_METRICS);
2961 0 : Lazy::force(&disk_usage_based_eviction::METRICS);
2962 :
2963 0 : for state_name in pageserver_api::models::TenantState::VARIANTS {
2964 0 : // initialize the metric for all gauges, otherwise the time series might seemingly show
2965 0 : // values from last restart.
2966 0 : TENANT_STATE_METRIC.with_label_values(&[state_name]).set(0);
2967 0 : }
2968 :
2969 : // countervecs
2970 0 : [&BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT]
2971 0 : .into_iter()
2972 0 : .for_each(|c| {
2973 0 : Lazy::force(c);
2974 0 : });
2975 0 :
2976 0 : // gauges
2977 0 : WALRECEIVER_ACTIVE_MANAGERS.get();
2978 0 :
2979 0 : // histograms
2980 0 : [
2981 0 : &READ_NUM_LAYERS_VISITED,
2982 0 : &VEC_READ_NUM_LAYERS_VISITED,
2983 0 : &WAIT_LSN_TIME,
2984 0 : &WAL_REDO_TIME,
2985 0 : &WAL_REDO_RECORDS_HISTOGRAM,
2986 0 : &WAL_REDO_BYTES_HISTOGRAM,
2987 0 : &WAL_REDO_PROCESS_LAUNCH_DURATION_HISTOGRAM,
2988 0 : ]
2989 0 : .into_iter()
2990 0 : .for_each(|h| {
2991 0 : Lazy::force(h);
2992 0 : });
2993 0 :
2994 0 : // Custom
2995 0 : Lazy::force(&RECONSTRUCT_TIME);
2996 0 : Lazy::force(&tenant_throttling::TIMELINE_GET);
2997 0 : Lazy::force(&BASEBACKUP_QUERY_TIME);
2998 0 : Lazy::force(&COMPUTE_COMMANDS_COUNTERS);
2999 0 : }
|
