Line data Source code
1 : use enum_map::EnumMap;
2 : use metrics::{
3 : register_counter_vec, register_gauge_vec, register_histogram, register_histogram_vec,
4 : register_int_counter, register_int_counter_pair_vec, register_int_counter_vec,
5 : register_int_gauge, register_int_gauge_vec, register_uint_gauge, register_uint_gauge_vec,
6 : Counter, CounterVec, GaugeVec, Histogram, HistogramVec, IntCounter, IntCounterPair,
7 : IntCounterPairVec, IntCounterVec, IntGauge, IntGaugeVec, UIntGauge, UIntGaugeVec,
8 : };
9 : use once_cell::sync::Lazy;
10 : use pageserver_api::shard::TenantShardId;
11 : use strum::{EnumCount, IntoEnumIterator, VariantNames};
12 : use strum_macros::{EnumVariantNames, IntoStaticStr};
13 : use tracing::warn;
14 : use utils::id::TimelineId;
15 :
16 : /// Prometheus histogram buckets (in seconds) for operations in the critical
17 : /// path. In other words, operations that directly affect that latency of user
18 : /// queries.
19 : ///
20 : /// The buckets capture the majority of latencies in the microsecond and
21 : /// millisecond range but also extend far enough up to distinguish "bad" from
22 : /// "really bad".
23 : const CRITICAL_OP_BUCKETS: &[f64] = &[
24 : 0.000_001, 0.000_010, 0.000_100, // 1 us, 10 us, 100 us
25 : 0.001_000, 0.010_000, 0.100_000, // 1 ms, 10 ms, 100 ms
26 : 1.0, 10.0, 100.0, // 1 s, 10 s, 100 s
27 : ];
28 :
29 : // Metrics collected on operations on the storage repository.
30 3064 : #[derive(Debug, EnumVariantNames, IntoStaticStr)]
31 : #[strum(serialize_all = "kebab_case")]
32 : pub(crate) enum StorageTimeOperation {
33 : #[strum(serialize = "layer flush")]
34 : LayerFlush,
35 :
36 : #[strum(serialize = "compact")]
37 : Compact,
38 :
39 : #[strum(serialize = "create images")]
40 : CreateImages,
41 :
42 : #[strum(serialize = "logical size")]
43 : LogicalSize,
44 :
45 : #[strum(serialize = "imitate logical size")]
46 : ImitateLogicalSize,
47 :
48 : #[strum(serialize = "load layer map")]
49 : LoadLayerMap,
50 :
51 : #[strum(serialize = "gc")]
52 : Gc,
53 :
54 : #[strum(serialize = "find gc cutoffs")]
55 : FindGcCutoffs,
56 : }
57 :
58 141 : pub(crate) static STORAGE_TIME_SUM_PER_TIMELINE: Lazy<CounterVec> = Lazy::new(|| {
59 : register_counter_vec!(
60 : "pageserver_storage_operations_seconds_sum",
61 : "Total time spent on storage operations with operation, tenant and timeline dimensions",
62 : &["operation", "tenant_id", "shard_id", "timeline_id"],
63 : )
64 141 : .expect("failed to define a metric")
65 141 : });
66 :
67 141 : pub(crate) static STORAGE_TIME_COUNT_PER_TIMELINE: Lazy<IntCounterVec> = Lazy::new(|| {
68 : register_int_counter_vec!(
69 : "pageserver_storage_operations_seconds_count",
70 : "Count of storage operations with operation, tenant and timeline dimensions",
71 : &["operation", "tenant_id", "shard_id", "timeline_id"],
72 : )
73 141 : .expect("failed to define a metric")
74 141 : });
75 :
76 : // Buckets for background operations like compaction, GC, size calculation
77 : const STORAGE_OP_BUCKETS: &[f64] = &[0.010, 0.100, 1.0, 10.0, 100.0, 1000.0];
78 :
79 141 : pub(crate) static STORAGE_TIME_GLOBAL: Lazy<HistogramVec> = Lazy::new(|| {
80 : register_histogram_vec!(
81 : "pageserver_storage_operations_seconds_global",
82 : "Time spent on storage operations",
83 : &["operation"],
84 : STORAGE_OP_BUCKETS.into(),
85 : )
86 141 : .expect("failed to define a metric")
87 141 : });
88 :
89 138 : pub(crate) static READ_NUM_LAYERS_VISITED: Lazy<Histogram> = Lazy::new(|| {
90 : register_histogram!(
91 : "pageserver_layers_visited_per_read_global",
92 : "Number of layers visited to reconstruct one key",
93 : vec![1.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0],
94 : )
95 138 : .expect("failed to define a metric")
96 138 : });
97 :
98 24 : pub(crate) static VEC_READ_NUM_LAYERS_VISITED: Lazy<Histogram> = Lazy::new(|| {
99 : register_histogram!(
100 : "pageserver_layers_visited_per_vectored_read_global",
101 : "Average number of layers visited to reconstruct one key",
102 : vec![1.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0],
103 : )
104 24 : .expect("failed to define a metric")
105 24 : });
106 :
107 : // Metrics collected on operations on the storage repository.
108 : #[derive(
109 552 : Clone, Copy, enum_map::Enum, strum_macros::EnumString, strum_macros::Display, IntoStaticStr,
110 : )]
111 : pub(crate) enum GetKind {
112 : Singular,
113 : Vectored,
114 : }
115 :
116 : pub(crate) struct ReconstructTimeMetrics {
117 : singular: Histogram,
118 : vectored: Histogram,
119 : }
120 :
121 138 : pub(crate) static RECONSTRUCT_TIME: Lazy<ReconstructTimeMetrics> = Lazy::new(|| {
122 138 : let inner = register_histogram_vec!(
123 : "pageserver_getpage_reconstruct_seconds",
124 : "Time spent in reconstruct_value (reconstruct a page from deltas)",
125 : &["get_kind"],
126 : CRITICAL_OP_BUCKETS.into(),
127 : )
128 138 : .expect("failed to define a metric");
129 138 :
130 138 : ReconstructTimeMetrics {
131 138 : singular: inner.with_label_values(&[GetKind::Singular.into()]),
132 138 : vectored: inner.with_label_values(&[GetKind::Vectored.into()]),
133 138 : }
134 138 : });
135 :
136 : impl ReconstructTimeMetrics {
137 625861 : pub(crate) fn for_get_kind(&self, get_kind: GetKind) -> &Histogram {
138 625861 : match get_kind {
139 625791 : GetKind::Singular => &self.singular,
140 70 : GetKind::Vectored => &self.vectored,
141 : }
142 625861 : }
143 : }
144 :
145 : pub(crate) struct ReconstructDataTimeMetrics {
146 : singular: Histogram,
147 : vectored: Histogram,
148 : }
149 :
150 : impl ReconstructDataTimeMetrics {
151 626001 : pub(crate) fn for_get_kind(&self, get_kind: GetKind) -> &Histogram {
152 626001 : match get_kind {
153 625931 : GetKind::Singular => &self.singular,
154 70 : GetKind::Vectored => &self.vectored,
155 : }
156 626001 : }
157 : }
158 :
159 138 : pub(crate) static GET_RECONSTRUCT_DATA_TIME: Lazy<ReconstructDataTimeMetrics> = Lazy::new(|| {
160 138 : let inner = register_histogram_vec!(
161 : "pageserver_getpage_get_reconstruct_data_seconds",
162 : "Time spent in get_reconstruct_value_data",
163 : &["get_kind"],
164 : CRITICAL_OP_BUCKETS.into(),
165 : )
166 138 : .expect("failed to define a metric");
167 138 :
168 138 : ReconstructDataTimeMetrics {
169 138 : singular: inner.with_label_values(&[GetKind::Singular.into()]),
170 138 : vectored: inner.with_label_values(&[GetKind::Vectored.into()]),
171 138 : }
172 138 : });
173 :
174 : pub(crate) struct GetVectoredLatency {
175 : map: EnumMap<TaskKind, Option<Histogram>>,
176 : }
177 :
178 : #[allow(dead_code)]
179 : pub(crate) struct ScanLatency {
180 : map: EnumMap<TaskKind, Option<Histogram>>,
181 : }
182 :
183 : impl GetVectoredLatency {
184 : // Only these task types perform vectored gets. Filter all other tasks out to reduce total
185 : // cardinality of the metric.
186 : const TRACKED_TASK_KINDS: [TaskKind; 2] = [TaskKind::Compaction, TaskKind::PageRequestHandler];
187 :
188 952 : pub(crate) fn for_task_kind(&self, task_kind: TaskKind) -> Option<&Histogram> {
189 952 : self.map[task_kind].as_ref()
190 952 : }
191 : }
192 :
193 : impl ScanLatency {
194 : // Only these task types perform vectored gets. Filter all other tasks out to reduce total
195 : // cardinality of the metric.
196 : const TRACKED_TASK_KINDS: [TaskKind; 1] = [TaskKind::PageRequestHandler];
197 :
198 12 : pub(crate) fn for_task_kind(&self, task_kind: TaskKind) -> Option<&Histogram> {
199 12 : self.map[task_kind].as_ref()
200 12 : }
201 : }
202 :
203 : pub(crate) struct ScanLatencyOngoingRecording<'a> {
204 : parent: &'a Histogram,
205 : start: std::time::Instant,
206 : }
207 :
208 : impl<'a> ScanLatencyOngoingRecording<'a> {
209 0 : pub(crate) fn start_recording(parent: &'a Histogram) -> ScanLatencyOngoingRecording<'a> {
210 0 : let start = Instant::now();
211 0 : ScanLatencyOngoingRecording { parent, start }
212 0 : }
213 :
214 0 : pub(crate) fn observe(self, throttled: Option<Duration>) {
215 0 : let elapsed = self.start.elapsed();
216 0 : let ex_throttled = if let Some(throttled) = throttled {
217 0 : elapsed.checked_sub(throttled)
218 : } else {
219 0 : Some(elapsed)
220 : };
221 0 : if let Some(ex_throttled) = ex_throttled {
222 0 : self.parent.observe(ex_throttled.as_secs_f64());
223 0 : } else {
224 0 : use utils::rate_limit::RateLimit;
225 0 : static LOGGED: Lazy<Mutex<RateLimit>> =
226 0 : Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(10))));
227 0 : let mut rate_limit = LOGGED.lock().unwrap();
228 0 : rate_limit.call(|| {
229 0 : warn!("error deducting time spent throttled; this message is logged at a global rate limit");
230 0 : });
231 0 : }
232 0 : }
233 : }
234 :
235 132 : pub(crate) static GET_VECTORED_LATENCY: Lazy<GetVectoredLatency> = Lazy::new(|| {
236 132 : let inner = register_histogram_vec!(
237 : "pageserver_get_vectored_seconds",
238 : "Time spent in get_vectored, excluding time spent in timeline_get_throttle.",
239 : &["task_kind"],
240 : CRITICAL_OP_BUCKETS.into(),
241 : )
242 132 : .expect("failed to define a metric");
243 132 :
244 132 : GetVectoredLatency {
245 3960 : map: EnumMap::from_array(std::array::from_fn(|task_kind_idx| {
246 3960 : let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind_idx);
247 3960 :
248 3960 : if GetVectoredLatency::TRACKED_TASK_KINDS.contains(&task_kind) {
249 264 : let task_kind = task_kind.into();
250 264 : Some(inner.with_label_values(&[task_kind]))
251 : } else {
252 3696 : None
253 : }
254 3960 : })),
255 132 : }
256 132 : });
257 :
258 4 : pub(crate) static SCAN_LATENCY: Lazy<ScanLatency> = Lazy::new(|| {
259 4 : let inner = register_histogram_vec!(
260 : "pageserver_scan_seconds",
261 : "Time spent in scan, excluding time spent in timeline_get_throttle.",
262 : &["task_kind"],
263 : CRITICAL_OP_BUCKETS.into(),
264 : )
265 4 : .expect("failed to define a metric");
266 4 :
267 4 : ScanLatency {
268 120 : map: EnumMap::from_array(std::array::from_fn(|task_kind_idx| {
269 120 : let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind_idx);
270 120 :
271 120 : if ScanLatency::TRACKED_TASK_KINDS.contains(&task_kind) {
272 4 : let task_kind = task_kind.into();
273 4 : Some(inner.with_label_values(&[task_kind]))
274 : } else {
275 116 : None
276 : }
277 120 : })),
278 4 : }
279 4 : });
280 :
281 : pub(crate) struct PageCacheMetricsForTaskKind {
282 : pub read_accesses_immutable: IntCounter,
283 : pub read_hits_immutable: IntCounter,
284 : }
285 :
286 : pub(crate) struct PageCacheMetrics {
287 : map: EnumMap<TaskKind, EnumMap<PageContentKind, PageCacheMetricsForTaskKind>>,
288 : }
289 :
290 74 : static PAGE_CACHE_READ_HITS: Lazy<IntCounterVec> = Lazy::new(|| {
291 : register_int_counter_vec!(
292 : "pageserver_page_cache_read_hits_total",
293 : "Number of read accesses to the page cache that hit",
294 : &["task_kind", "key_kind", "content_kind", "hit_kind"]
295 : )
296 74 : .expect("failed to define a metric")
297 74 : });
298 :
299 74 : static PAGE_CACHE_READ_ACCESSES: Lazy<IntCounterVec> = Lazy::new(|| {
300 : register_int_counter_vec!(
301 : "pageserver_page_cache_read_accesses_total",
302 : "Number of read accesses to the page cache",
303 : &["task_kind", "key_kind", "content_kind"]
304 : )
305 74 : .expect("failed to define a metric")
306 74 : });
307 :
308 74 : pub(crate) static PAGE_CACHE: Lazy<PageCacheMetrics> = Lazy::new(|| PageCacheMetrics {
309 2220 : map: EnumMap::from_array(std::array::from_fn(|task_kind| {
310 2220 : let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind);
311 2220 : let task_kind: &'static str = task_kind.into();
312 13320 : EnumMap::from_array(std::array::from_fn(|content_kind| {
313 13320 : let content_kind = <PageContentKind as enum_map::Enum>::from_usize(content_kind);
314 13320 : let content_kind: &'static str = content_kind.into();
315 13320 : PageCacheMetricsForTaskKind {
316 13320 : read_accesses_immutable: {
317 13320 : PAGE_CACHE_READ_ACCESSES
318 13320 : .get_metric_with_label_values(&[task_kind, "immutable", content_kind])
319 13320 : .unwrap()
320 13320 : },
321 13320 :
322 13320 : read_hits_immutable: {
323 13320 : PAGE_CACHE_READ_HITS
324 13320 : .get_metric_with_label_values(&[task_kind, "immutable", content_kind, "-"])
325 13320 : .unwrap()
326 13320 : },
327 13320 : }
328 13320 : }))
329 2220 : })),
330 74 : });
331 :
332 : impl PageCacheMetrics {
333 14189396 : pub(crate) fn for_ctx(&self, ctx: &RequestContext) -> &PageCacheMetricsForTaskKind {
334 14189396 : &self.map[ctx.task_kind()][ctx.page_content_kind()]
335 14189396 : }
336 : }
337 :
338 : pub(crate) struct PageCacheSizeMetrics {
339 : pub max_bytes: UIntGauge,
340 :
341 : pub current_bytes_immutable: UIntGauge,
342 : }
343 :
344 74 : static PAGE_CACHE_SIZE_CURRENT_BYTES: Lazy<UIntGaugeVec> = Lazy::new(|| {
345 : register_uint_gauge_vec!(
346 : "pageserver_page_cache_size_current_bytes",
347 : "Current size of the page cache in bytes, by key kind",
348 : &["key_kind"]
349 : )
350 74 : .expect("failed to define a metric")
351 74 : });
352 :
353 : pub(crate) static PAGE_CACHE_SIZE: Lazy<PageCacheSizeMetrics> =
354 74 : Lazy::new(|| PageCacheSizeMetrics {
355 : max_bytes: {
356 : register_uint_gauge!(
357 : "pageserver_page_cache_size_max_bytes",
358 : "Maximum size of the page cache in bytes"
359 : )
360 74 : .expect("failed to define a metric")
361 74 : },
362 74 : current_bytes_immutable: {
363 74 : PAGE_CACHE_SIZE_CURRENT_BYTES
364 74 : .get_metric_with_label_values(&["immutable"])
365 74 : .unwrap()
366 74 : },
367 74 : });
368 :
369 : pub(crate) mod page_cache_eviction_metrics {
370 : use std::num::NonZeroUsize;
371 :
372 : use metrics::{register_int_counter_vec, IntCounter, IntCounterVec};
373 : use once_cell::sync::Lazy;
374 :
375 : #[derive(Clone, Copy)]
376 : pub(crate) enum Outcome {
377 : FoundSlotUnused { iters: NonZeroUsize },
378 : FoundSlotEvicted { iters: NonZeroUsize },
379 : ItersExceeded { iters: NonZeroUsize },
380 : }
381 :
382 74 : static ITERS_TOTAL_VEC: Lazy<IntCounterVec> = Lazy::new(|| {
383 : register_int_counter_vec!(
384 : "pageserver_page_cache_find_victim_iters_total",
385 : "Counter for the number of iterations in the find_victim loop",
386 : &["outcome"],
387 : )
388 74 : .expect("failed to define a metric")
389 74 : });
390 :
391 74 : static CALLS_VEC: Lazy<IntCounterVec> = Lazy::new(|| {
392 : register_int_counter_vec!(
393 : "pageserver_page_cache_find_victim_calls",
394 : "Incremented at the end of each find_victim() call.\
395 : Filter by outcome to get e.g., eviction rate.",
396 : &["outcome"]
397 : )
398 74 : .unwrap()
399 74 : });
400 :
401 166722 : pub(crate) fn observe(outcome: Outcome) {
402 166722 : macro_rules! dry {
403 166722 : ($label:literal, $iters:expr) => {{
404 166722 : static LABEL: &'static str = $label;
405 166722 : static ITERS_TOTAL: Lazy<IntCounter> =
406 166722 : Lazy::new(|| ITERS_TOTAL_VEC.with_label_values(&[LABEL]));
407 166722 : static CALLS: Lazy<IntCounter> =
408 166722 : Lazy::new(|| CALLS_VEC.with_label_values(&[LABEL]));
409 166722 : ITERS_TOTAL.inc_by(($iters.get()) as u64);
410 166722 : CALLS.inc();
411 166722 : }};
412 166722 : }
413 166722 : match outcome {
414 1892 : Outcome::FoundSlotUnused { iters } => dry!("found_empty", iters),
415 164830 : Outcome::FoundSlotEvicted { iters } => {
416 164830 : dry!("found_evicted", iters)
417 : }
418 0 : Outcome::ItersExceeded { iters } => {
419 0 : dry!("err_iters_exceeded", iters);
420 0 : super::page_cache_errors_inc(super::PageCacheErrorKind::EvictIterLimit);
421 0 : }
422 : }
423 166722 : }
424 : }
425 :
426 0 : static PAGE_CACHE_ERRORS: Lazy<IntCounterVec> = Lazy::new(|| {
427 : register_int_counter_vec!(
428 : "page_cache_errors_total",
429 : "Number of timeouts while acquiring a pinned slot in the page cache",
430 : &["error_kind"]
431 : )
432 0 : .expect("failed to define a metric")
433 0 : });
434 :
435 0 : #[derive(IntoStaticStr)]
436 : #[strum(serialize_all = "kebab_case")]
437 : pub(crate) enum PageCacheErrorKind {
438 : AcquirePinnedSlotTimeout,
439 : EvictIterLimit,
440 : }
441 :
442 0 : pub(crate) fn page_cache_errors_inc(error_kind: PageCacheErrorKind) {
443 0 : PAGE_CACHE_ERRORS
444 0 : .get_metric_with_label_values(&[error_kind.into()])
445 0 : .unwrap()
446 0 : .inc();
447 0 : }
448 :
449 16 : pub(crate) static WAIT_LSN_TIME: Lazy<Histogram> = Lazy::new(|| {
450 : register_histogram!(
451 : "pageserver_wait_lsn_seconds",
452 : "Time spent waiting for WAL to arrive",
453 : CRITICAL_OP_BUCKETS.into(),
454 : )
455 16 : .expect("failed to define a metric")
456 16 : });
457 :
458 141 : static LAST_RECORD_LSN: Lazy<IntGaugeVec> = Lazy::new(|| {
459 : register_int_gauge_vec!(
460 : "pageserver_last_record_lsn",
461 : "Last record LSN grouped by timeline",
462 : &["tenant_id", "shard_id", "timeline_id"]
463 : )
464 141 : .expect("failed to define a metric")
465 141 : });
466 :
467 141 : static STANDBY_HORIZON: Lazy<IntGaugeVec> = Lazy::new(|| {
468 : register_int_gauge_vec!(
469 : "pageserver_standby_horizon",
470 : "Standby apply LSN for which GC is hold off, by timeline.",
471 : &["tenant_id", "shard_id", "timeline_id"]
472 : )
473 141 : .expect("failed to define a metric")
474 141 : });
475 :
476 141 : static RESIDENT_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
477 : register_uint_gauge_vec!(
478 : "pageserver_resident_physical_size",
479 : "The size of the layer files present in the pageserver's filesystem.",
480 : &["tenant_id", "shard_id", "timeline_id"]
481 : )
482 141 : .expect("failed to define a metric")
483 141 : });
484 :
485 136 : pub(crate) static RESIDENT_PHYSICAL_SIZE_GLOBAL: Lazy<UIntGauge> = Lazy::new(|| {
486 : register_uint_gauge!(
487 : "pageserver_resident_physical_size_global",
488 : "Like `pageserver_resident_physical_size`, but without tenant/timeline dimensions."
489 : )
490 136 : .expect("failed to define a metric")
491 136 : });
492 :
493 141 : static REMOTE_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
494 : register_uint_gauge_vec!(
495 : "pageserver_remote_physical_size",
496 : "The size of the layer files present in the remote storage that are listed in the remote index_part.json.",
497 : // Corollary: If any files are missing from the index part, they won't be included here.
498 : &["tenant_id", "shard_id", "timeline_id"]
499 : )
500 141 : .expect("failed to define a metric")
501 141 : });
502 :
503 141 : static REMOTE_PHYSICAL_SIZE_GLOBAL: Lazy<UIntGauge> = Lazy::new(|| {
504 : register_uint_gauge!(
505 : "pageserver_remote_physical_size_global",
506 : "Like `pageserver_remote_physical_size`, but without tenant/timeline dimensions."
507 : )
508 141 : .expect("failed to define a metric")
509 141 : });
510 :
511 4 : pub(crate) static REMOTE_ONDEMAND_DOWNLOADED_LAYERS: Lazy<IntCounter> = Lazy::new(|| {
512 : register_int_counter!(
513 : "pageserver_remote_ondemand_downloaded_layers_total",
514 : "Total on-demand downloaded layers"
515 : )
516 4 : .unwrap()
517 4 : });
518 :
519 4 : pub(crate) static REMOTE_ONDEMAND_DOWNLOADED_BYTES: Lazy<IntCounter> = Lazy::new(|| {
520 : register_int_counter!(
521 : "pageserver_remote_ondemand_downloaded_bytes_total",
522 : "Total bytes of layers on-demand downloaded",
523 : )
524 4 : .unwrap()
525 4 : });
526 :
527 141 : static CURRENT_LOGICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
528 : register_uint_gauge_vec!(
529 : "pageserver_current_logical_size",
530 : "Current logical size grouped by timeline",
531 : &["tenant_id", "shard_id", "timeline_id"]
532 : )
533 141 : .expect("failed to define current logical size metric")
534 141 : });
535 :
536 141 : static AUX_FILE_SIZE: Lazy<IntGaugeVec> = Lazy::new(|| {
537 : register_int_gauge_vec!(
538 : "pageserver_aux_file_estimated_size",
539 : "The size of all aux files for a timeline in aux file v2 store.",
540 : &["tenant_id", "shard_id", "timeline_id"]
541 : )
542 141 : .expect("failed to define a metric")
543 141 : });
544 :
545 141 : static VALID_LSN_LEASE_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
546 : register_uint_gauge_vec!(
547 : "pageserver_valid_lsn_lease_count",
548 : "The number of valid leases after refreshing gc info.",
549 : &["tenant_id", "shard_id", "timeline_id"],
550 : )
551 141 : .expect("failed to define a metric")
552 141 : });
553 :
554 : pub(crate) mod initial_logical_size {
555 : use metrics::{register_int_counter, register_int_counter_vec, IntCounter, IntCounterVec};
556 : use once_cell::sync::Lazy;
557 :
558 : pub(crate) struct StartCalculation(IntCounterVec);
559 141 : pub(crate) static START_CALCULATION: Lazy<StartCalculation> = Lazy::new(|| {
560 141 : StartCalculation(
561 141 : register_int_counter_vec!(
562 : "pageserver_initial_logical_size_start_calculation",
563 : "Incremented each time we start an initial logical size calculation attempt. \
564 : The `circumstances` label provides some additional details.",
565 : &["attempt", "circumstances"]
566 141 : )
567 141 : .unwrap(),
568 141 : )
569 141 : });
570 :
571 : struct DropCalculation {
572 : first: IntCounter,
573 : retry: IntCounter,
574 : }
575 :
576 141 : static DROP_CALCULATION: Lazy<DropCalculation> = Lazy::new(|| {
577 141 : let vec = register_int_counter_vec!(
578 : "pageserver_initial_logical_size_drop_calculation",
579 : "Incremented each time we abort a started size calculation attmpt.",
580 : &["attempt"]
581 : )
582 141 : .unwrap();
583 141 : DropCalculation {
584 141 : first: vec.with_label_values(&["first"]),
585 141 : retry: vec.with_label_values(&["retry"]),
586 141 : }
587 141 : });
588 :
589 : pub(crate) struct Calculated {
590 : pub(crate) births: IntCounter,
591 : pub(crate) deaths: IntCounter,
592 : }
593 :
594 141 : pub(crate) static CALCULATED: Lazy<Calculated> = Lazy::new(|| Calculated {
595 : births: register_int_counter!(
596 : "pageserver_initial_logical_size_finish_calculation",
597 : "Incremented every time we finish calculation of initial logical size.\
598 : If everything is working well, this should happen at most once per Timeline object."
599 : )
600 141 : .unwrap(),
601 : deaths: register_int_counter!(
602 : "pageserver_initial_logical_size_drop_finished_calculation",
603 : "Incremented when we drop a finished initial logical size calculation result.\
604 : Mainly useful to turn pageserver_initial_logical_size_finish_calculation into a gauge."
605 : )
606 141 : .unwrap(),
607 141 : });
608 :
609 : pub(crate) struct OngoingCalculationGuard {
610 : inc_drop_calculation: Option<IntCounter>,
611 : }
612 :
613 153 : #[derive(strum_macros::IntoStaticStr)]
614 : pub(crate) enum StartCircumstances {
615 : EmptyInitial,
616 : SkippedConcurrencyLimiter,
617 : AfterBackgroundTasksRateLimit,
618 : }
619 :
620 : impl StartCalculation {
621 153 : pub(crate) fn first(&self, circumstances: StartCircumstances) -> OngoingCalculationGuard {
622 153 : let circumstances_label: &'static str = circumstances.into();
623 153 : self.0
624 153 : .with_label_values(&["first", circumstances_label])
625 153 : .inc();
626 153 : OngoingCalculationGuard {
627 153 : inc_drop_calculation: Some(DROP_CALCULATION.first.clone()),
628 153 : }
629 153 : }
630 0 : pub(crate) fn retry(&self, circumstances: StartCircumstances) -> OngoingCalculationGuard {
631 0 : let circumstances_label: &'static str = circumstances.into();
632 0 : self.0
633 0 : .with_label_values(&["retry", circumstances_label])
634 0 : .inc();
635 0 : OngoingCalculationGuard {
636 0 : inc_drop_calculation: Some(DROP_CALCULATION.retry.clone()),
637 0 : }
638 0 : }
639 : }
640 :
641 : impl Drop for OngoingCalculationGuard {
642 153 : fn drop(&mut self) {
643 153 : if let Some(counter) = self.inc_drop_calculation.take() {
644 0 : counter.inc();
645 153 : }
646 153 : }
647 : }
648 :
649 : impl OngoingCalculationGuard {
650 153 : pub(crate) fn calculation_result_saved(mut self) -> FinishedCalculationGuard {
651 153 : drop(self.inc_drop_calculation.take());
652 153 : CALCULATED.births.inc();
653 153 : FinishedCalculationGuard {
654 153 : inc_on_drop: CALCULATED.deaths.clone(),
655 153 : }
656 153 : }
657 : }
658 :
659 : pub(crate) struct FinishedCalculationGuard {
660 : inc_on_drop: IntCounter,
661 : }
662 :
663 : impl Drop for FinishedCalculationGuard {
664 7 : fn drop(&mut self) {
665 7 : self.inc_on_drop.inc();
666 7 : }
667 : }
668 :
669 : // context: https://github.com/neondatabase/neon/issues/5963
670 : pub(crate) static TIMELINES_WHERE_WALRECEIVER_GOT_APPROXIMATE_SIZE: Lazy<IntCounter> =
671 0 : Lazy::new(|| {
672 : register_int_counter!(
673 : "pageserver_initial_logical_size_timelines_where_walreceiver_got_approximate_size",
674 : "Counter for the following event: walreceiver calls\
675 : Timeline::get_current_logical_size() and it returns `Approximate` for the first time."
676 : )
677 0 : .unwrap()
678 0 : });
679 : }
680 :
681 0 : static DIRECTORY_ENTRIES_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
682 : register_uint_gauge_vec!(
683 : "pageserver_directory_entries_count",
684 : "Sum of the entries in pageserver-stored directory listings",
685 : &["tenant_id", "shard_id", "timeline_id"]
686 : )
687 0 : .expect("failed to define a metric")
688 0 : });
689 :
690 143 : pub(crate) static TENANT_STATE_METRIC: Lazy<UIntGaugeVec> = Lazy::new(|| {
691 : register_uint_gauge_vec!(
692 : "pageserver_tenant_states_count",
693 : "Count of tenants per state",
694 : &["state"]
695 : )
696 143 : .expect("Failed to register pageserver_tenant_states_count metric")
697 143 : });
698 :
699 : /// A set of broken tenants.
700 : ///
701 : /// These are expected to be so rare that a set is fine. Set as in a new timeseries per each broken
702 : /// tenant.
703 12 : pub(crate) static BROKEN_TENANTS_SET: Lazy<UIntGaugeVec> = Lazy::new(|| {
704 : register_uint_gauge_vec!(
705 : "pageserver_broken_tenants_count",
706 : "Set of broken tenants",
707 : &["tenant_id", "shard_id"]
708 : )
709 12 : .expect("Failed to register pageserver_tenant_states_count metric")
710 12 : });
711 :
712 6 : pub(crate) static TENANT_SYNTHETIC_SIZE_METRIC: Lazy<UIntGaugeVec> = Lazy::new(|| {
713 : register_uint_gauge_vec!(
714 : "pageserver_tenant_synthetic_cached_size_bytes",
715 : "Synthetic size of each tenant in bytes",
716 : &["tenant_id"]
717 : )
718 6 : .expect("Failed to register pageserver_tenant_synthetic_cached_size_bytes metric")
719 6 : });
720 :
721 0 : pub(crate) static EVICTION_ITERATION_DURATION: Lazy<HistogramVec> = Lazy::new(|| {
722 : register_histogram_vec!(
723 : "pageserver_eviction_iteration_duration_seconds_global",
724 : "Time spent on a single eviction iteration",
725 : &["period_secs", "threshold_secs"],
726 : STORAGE_OP_BUCKETS.into(),
727 : )
728 0 : .expect("failed to define a metric")
729 0 : });
730 :
731 141 : static EVICTIONS: Lazy<IntCounterVec> = Lazy::new(|| {
732 : register_int_counter_vec!(
733 : "pageserver_evictions",
734 : "Number of layers evicted from the pageserver",
735 : &["tenant_id", "shard_id", "timeline_id"]
736 : )
737 141 : .expect("failed to define a metric")
738 141 : });
739 :
740 141 : static EVICTIONS_WITH_LOW_RESIDENCE_DURATION: Lazy<IntCounterVec> = Lazy::new(|| {
741 : register_int_counter_vec!(
742 : "pageserver_evictions_with_low_residence_duration",
743 : "If a layer is evicted that was resident for less than `low_threshold`, it is counted to this counter. \
744 : Residence duration is determined using the `residence_duration_data_source`.",
745 : &["tenant_id", "shard_id", "timeline_id", "residence_duration_data_source", "low_threshold_secs"]
746 : )
747 141 : .expect("failed to define a metric")
748 141 : });
749 :
750 0 : pub(crate) static UNEXPECTED_ONDEMAND_DOWNLOADS: Lazy<IntCounter> = Lazy::new(|| {
751 : register_int_counter!(
752 : "pageserver_unexpected_ondemand_downloads_count",
753 : "Number of unexpected on-demand downloads. \
754 : We log more context for each increment, so, forgo any labels in this metric.",
755 : )
756 0 : .expect("failed to define a metric")
757 0 : });
758 :
759 : /// How long did we take to start up? Broken down by labels to describe
760 : /// different phases of startup.
761 0 : pub static STARTUP_DURATION: Lazy<GaugeVec> = Lazy::new(|| {
762 : register_gauge_vec!(
763 : "pageserver_startup_duration_seconds",
764 : "Time taken by phases of pageserver startup, in seconds",
765 : &["phase"]
766 : )
767 0 : .expect("Failed to register pageserver_startup_duration_seconds metric")
768 0 : });
769 :
770 0 : pub static STARTUP_IS_LOADING: Lazy<UIntGauge> = Lazy::new(|| {
771 : register_uint_gauge!(
772 : "pageserver_startup_is_loading",
773 : "1 while in initial startup load of tenants, 0 at other times"
774 : )
775 0 : .expect("Failed to register pageserver_startup_is_loading")
776 0 : });
777 :
778 136 : pub(crate) static TIMELINE_EPHEMERAL_BYTES: Lazy<UIntGauge> = Lazy::new(|| {
779 : register_uint_gauge!(
780 : "pageserver_timeline_ephemeral_bytes",
781 : "Total number of bytes in ephemeral layers, summed for all timelines. Approximate, lazily updated."
782 : )
783 136 : .expect("Failed to register metric")
784 136 : });
785 :
786 : /// Metrics related to the lifecycle of a [`crate::tenant::Tenant`] object: things
787 : /// like how long it took to load.
788 : ///
789 : /// Note that these are process-global metrics, _not_ per-tenant metrics. Per-tenant
790 : /// metrics are rather expensive, and usually fine grained stuff makes more sense
791 : /// at a timeline level than tenant level.
792 : pub(crate) struct TenantMetrics {
793 : /// How long did tenants take to go from construction to active state?
794 : pub(crate) activation: Histogram,
795 : pub(crate) preload: Histogram,
796 : pub(crate) attach: Histogram,
797 :
798 : /// How many tenants are included in the initial startup of the pagesrever?
799 : pub(crate) startup_scheduled: IntCounter,
800 : pub(crate) startup_complete: IntCounter,
801 : }
802 :
803 0 : pub(crate) static TENANT: Lazy<TenantMetrics> = Lazy::new(|| {
804 0 : TenantMetrics {
805 0 : activation: register_histogram!(
806 : "pageserver_tenant_activation_seconds",
807 : "Time taken by tenants to activate, in seconds",
808 : CRITICAL_OP_BUCKETS.into()
809 0 : )
810 0 : .expect("Failed to register metric"),
811 0 : preload: register_histogram!(
812 : "pageserver_tenant_preload_seconds",
813 : "Time taken by tenants to load remote metadata on startup/attach, in seconds",
814 : CRITICAL_OP_BUCKETS.into()
815 0 : )
816 0 : .expect("Failed to register metric"),
817 0 : attach: register_histogram!(
818 : "pageserver_tenant_attach_seconds",
819 : "Time taken by tenants to intialize, after remote metadata is already loaded",
820 : CRITICAL_OP_BUCKETS.into()
821 0 : )
822 0 : .expect("Failed to register metric"),
823 0 : startup_scheduled: register_int_counter!(
824 : "pageserver_tenant_startup_scheduled",
825 : "Number of tenants included in pageserver startup (doesn't count tenants attached later)"
826 0 : ).expect("Failed to register metric"),
827 0 : startup_complete: register_int_counter!(
828 : "pageserver_tenant_startup_complete",
829 : "Number of tenants that have completed warm-up, or activated on-demand during initial startup: \
830 : should eventually reach `pageserver_tenant_startup_scheduled_total`. Does not include broken \
831 : tenants: such cases will lead to this metric never reaching the scheduled count."
832 0 : ).expect("Failed to register metric"),
833 0 : }
834 0 : });
835 :
836 : /// Each `Timeline`'s [`EVICTIONS_WITH_LOW_RESIDENCE_DURATION`] metric.
837 : #[derive(Debug)]
838 : pub(crate) struct EvictionsWithLowResidenceDuration {
839 : data_source: &'static str,
840 : threshold: Duration,
841 : counter: Option<IntCounter>,
842 : }
843 :
844 : pub(crate) struct EvictionsWithLowResidenceDurationBuilder {
845 : data_source: &'static str,
846 : threshold: Duration,
847 : }
848 :
849 : impl EvictionsWithLowResidenceDurationBuilder {
850 383 : pub fn new(data_source: &'static str, threshold: Duration) -> Self {
851 383 : Self {
852 383 : data_source,
853 383 : threshold,
854 383 : }
855 383 : }
856 :
857 383 : fn build(
858 383 : &self,
859 383 : tenant_id: &str,
860 383 : shard_id: &str,
861 383 : timeline_id: &str,
862 383 : ) -> EvictionsWithLowResidenceDuration {
863 383 : let counter = EVICTIONS_WITH_LOW_RESIDENCE_DURATION
864 383 : .get_metric_with_label_values(&[
865 383 : tenant_id,
866 383 : shard_id,
867 383 : timeline_id,
868 383 : self.data_source,
869 383 : &EvictionsWithLowResidenceDuration::threshold_label_value(self.threshold),
870 383 : ])
871 383 : .unwrap();
872 383 : EvictionsWithLowResidenceDuration {
873 383 : data_source: self.data_source,
874 383 : threshold: self.threshold,
875 383 : counter: Some(counter),
876 383 : }
877 383 : }
878 : }
879 :
880 : impl EvictionsWithLowResidenceDuration {
881 391 : fn threshold_label_value(threshold: Duration) -> String {
882 391 : format!("{}", threshold.as_secs())
883 391 : }
884 :
885 18 : pub fn observe(&self, observed_value: Duration) {
886 18 : if observed_value < self.threshold {
887 18 : self.counter
888 18 : .as_ref()
889 18 : .expect("nobody calls this function after `remove_from_vec`")
890 18 : .inc();
891 18 : }
892 18 : }
893 :
894 8 : pub fn change_threshold(
895 8 : &mut self,
896 8 : tenant_id: &str,
897 8 : shard_id: &str,
898 8 : timeline_id: &str,
899 8 : new_threshold: Duration,
900 8 : ) {
901 8 : if new_threshold == self.threshold {
902 8 : return;
903 0 : }
904 0 : let mut with_new = EvictionsWithLowResidenceDurationBuilder::new(
905 0 : self.data_source,
906 0 : new_threshold,
907 0 : )
908 0 : .build(tenant_id, shard_id, timeline_id);
909 0 : std::mem::swap(self, &mut with_new);
910 0 : with_new.remove(tenant_id, shard_id, timeline_id);
911 8 : }
912 :
913 : // This could be a `Drop` impl, but, we need the `tenant_id` and `timeline_id`.
914 8 : fn remove(&mut self, tenant_id: &str, shard_id: &str, timeline_id: &str) {
915 8 : let Some(_counter) = self.counter.take() else {
916 0 : return;
917 : };
918 :
919 8 : let threshold = Self::threshold_label_value(self.threshold);
920 8 :
921 8 : let removed = EVICTIONS_WITH_LOW_RESIDENCE_DURATION.remove_label_values(&[
922 8 : tenant_id,
923 8 : shard_id,
924 8 : timeline_id,
925 8 : self.data_source,
926 8 : &threshold,
927 8 : ]);
928 8 :
929 8 : match removed {
930 0 : Err(e) => {
931 0 : // this has been hit in staging as
932 0 : // <https://neondatabase.sentry.io/issues/4142396994/>, but we don't know how.
933 0 : // because we can be in the drop path already, don't risk:
934 0 : // - "double-panic => illegal instruction" or
935 0 : // - future "drop panick => abort"
936 0 : //
937 0 : // so just nag: (the error has the labels)
938 0 : tracing::warn!("failed to remove EvictionsWithLowResidenceDuration, it was already removed? {e:#?}");
939 : }
940 : Ok(()) => {
941 : // to help identify cases where we double-remove the same values, let's log all
942 : // deletions?
943 8 : tracing::info!("removed EvictionsWithLowResidenceDuration with {tenant_id}, {timeline_id}, {}, {threshold}", self.data_source);
944 : }
945 : }
946 8 : }
947 : }
948 :
949 : // Metrics collected on disk IO operations
950 : //
951 : // Roughly logarithmic scale.
952 : const STORAGE_IO_TIME_BUCKETS: &[f64] = &[
953 : 0.000030, // 30 usec
954 : 0.001000, // 1000 usec
955 : 0.030, // 30 ms
956 : 1.000, // 1000 ms
957 : 30.000, // 30000 ms
958 : ];
959 :
960 : /// VirtualFile fs operation variants.
961 : ///
962 : /// Operations:
963 : /// - open ([`std::fs::OpenOptions::open`])
964 : /// - close (dropping [`crate::virtual_file::VirtualFile`])
965 : /// - close-by-replace (close by replacement algorithm)
966 : /// - read (`read_at`)
967 : /// - write (`write_at`)
968 : /// - seek (modify internal position or file length query)
969 : /// - fsync ([`std::fs::File::sync_all`])
970 : /// - metadata ([`std::fs::File::metadata`])
971 : #[derive(
972 1440 : Debug, Clone, Copy, strum_macros::EnumCount, strum_macros::EnumIter, strum_macros::FromRepr,
973 : )]
974 : pub(crate) enum StorageIoOperation {
975 : Open,
976 : OpenAfterReplace,
977 : Close,
978 : CloseByReplace,
979 : Read,
980 : Write,
981 : Seek,
982 : Fsync,
983 : Metadata,
984 : }
985 :
986 : impl StorageIoOperation {
987 1440 : pub fn as_str(&self) -> &'static str {
988 1440 : match self {
989 160 : StorageIoOperation::Open => "open",
990 160 : StorageIoOperation::OpenAfterReplace => "open-after-replace",
991 160 : StorageIoOperation::Close => "close",
992 160 : StorageIoOperation::CloseByReplace => "close-by-replace",
993 160 : StorageIoOperation::Read => "read",
994 160 : StorageIoOperation::Write => "write",
995 160 : StorageIoOperation::Seek => "seek",
996 160 : StorageIoOperation::Fsync => "fsync",
997 160 : StorageIoOperation::Metadata => "metadata",
998 : }
999 1440 : }
1000 : }
1001 :
1002 : /// Tracks time taken by fs operations near VirtualFile.
1003 : #[derive(Debug)]
1004 : pub(crate) struct StorageIoTime {
1005 : metrics: [Histogram; StorageIoOperation::COUNT],
1006 : }
1007 :
1008 : impl StorageIoTime {
1009 160 : fn new() -> Self {
1010 160 : let storage_io_histogram_vec = register_histogram_vec!(
1011 : "pageserver_io_operations_seconds",
1012 : "Time spent in IO operations",
1013 : &["operation"],
1014 : STORAGE_IO_TIME_BUCKETS.into()
1015 : )
1016 160 : .expect("failed to define a metric");
1017 1440 : let metrics = std::array::from_fn(|i| {
1018 1440 : let op = StorageIoOperation::from_repr(i).unwrap();
1019 1440 : storage_io_histogram_vec
1020 1440 : .get_metric_with_label_values(&[op.as_str()])
1021 1440 : .unwrap()
1022 1440 : });
1023 160 : Self { metrics }
1024 160 : }
1025 :
1026 1892232 : pub(crate) fn get(&self, op: StorageIoOperation) -> &Histogram {
1027 1892232 : &self.metrics[op as usize]
1028 1892232 : }
1029 : }
1030 :
1031 : pub(crate) static STORAGE_IO_TIME_METRIC: Lazy<StorageIoTime> = Lazy::new(StorageIoTime::new);
1032 :
1033 : const STORAGE_IO_SIZE_OPERATIONS: &[&str] = &["read", "write"];
1034 :
1035 : // Needed for the https://neonprod.grafana.net/d/5uK9tHL4k/picking-tenant-for-relocation?orgId=1
1036 158 : pub(crate) static STORAGE_IO_SIZE: Lazy<IntGaugeVec> = Lazy::new(|| {
1037 : register_int_gauge_vec!(
1038 : "pageserver_io_operations_bytes_total",
1039 : "Total amount of bytes read/written in IO operations",
1040 : &["operation", "tenant_id", "shard_id", "timeline_id"]
1041 : )
1042 158 : .expect("failed to define a metric")
1043 158 : });
1044 :
1045 : #[cfg(not(test))]
1046 : pub(crate) mod virtual_file_descriptor_cache {
1047 : use super::*;
1048 :
1049 0 : pub(crate) static SIZE_MAX: Lazy<UIntGauge> = Lazy::new(|| {
1050 : register_uint_gauge!(
1051 : "pageserver_virtual_file_descriptor_cache_size_max",
1052 : "Maximum number of open file descriptors in the cache."
1053 : )
1054 0 : .unwrap()
1055 0 : });
1056 :
1057 : // SIZE_CURRENT: derive it like so:
1058 : // ```
1059 : // sum (pageserver_io_operations_seconds_count{operation=~"^(open|open-after-replace)$")
1060 : // -ignoring(operation)
1061 : // sum(pageserver_io_operations_seconds_count{operation=~"^(close|close-by-replace)$"}
1062 : // ```
1063 : }
1064 :
1065 : #[cfg(not(test))]
1066 : pub(crate) mod virtual_file_io_engine {
1067 : use super::*;
1068 :
1069 0 : pub(crate) static KIND: Lazy<UIntGaugeVec> = Lazy::new(|| {
1070 : register_uint_gauge_vec!(
1071 : "pageserver_virtual_file_io_engine_kind",
1072 : "The configured io engine for VirtualFile",
1073 : &["kind"],
1074 : )
1075 0 : .unwrap()
1076 0 : });
1077 : }
1078 :
1079 : #[derive(Debug)]
1080 : struct GlobalAndPerTimelineHistogram {
1081 : global: Histogram,
1082 : per_tenant_timeline: Histogram,
1083 : }
1084 :
1085 : impl GlobalAndPerTimelineHistogram {
1086 10 : fn observe(&self, value: f64) {
1087 10 : self.global.observe(value);
1088 10 : self.per_tenant_timeline.observe(value);
1089 10 : }
1090 : }
1091 :
1092 : struct GlobalAndPerTimelineHistogramTimer<'a, 'c> {
1093 : h: &'a GlobalAndPerTimelineHistogram,
1094 : ctx: &'c RequestContext,
1095 : start: std::time::Instant,
1096 : op: SmgrQueryType,
1097 : }
1098 :
1099 : impl<'a, 'c> Drop for GlobalAndPerTimelineHistogramTimer<'a, 'c> {
1100 10 : fn drop(&mut self) {
1101 10 : let elapsed = self.start.elapsed();
1102 10 : let ex_throttled = self
1103 10 : .ctx
1104 10 : .micros_spent_throttled
1105 10 : .close_and_checked_sub_from(elapsed);
1106 10 : let ex_throttled = match ex_throttled {
1107 10 : Ok(res) => res,
1108 0 : Err(error) => {
1109 0 : use utils::rate_limit::RateLimit;
1110 0 : static LOGGED: Lazy<Mutex<enum_map::EnumMap<SmgrQueryType, RateLimit>>> =
1111 0 : Lazy::new(|| {
1112 0 : Mutex::new(enum_map::EnumMap::from_array(std::array::from_fn(|_| {
1113 0 : RateLimit::new(Duration::from_secs(10))
1114 0 : })))
1115 0 : });
1116 0 : let mut guard = LOGGED.lock().unwrap();
1117 0 : let rate_limit = &mut guard[self.op];
1118 0 : rate_limit.call(|| {
1119 0 : warn!(op=?self.op, error, "error deducting time spent throttled; this message is logged at a global rate limit");
1120 0 : });
1121 0 : elapsed
1122 : }
1123 : };
1124 10 : self.h.observe(ex_throttled.as_secs_f64());
1125 10 : }
1126 : }
1127 :
1128 : #[derive(
1129 : Debug,
1130 : Clone,
1131 : Copy,
1132 3980 : IntoStaticStr,
1133 : strum_macros::EnumCount,
1134 112 : strum_macros::EnumIter,
1135 1965 : strum_macros::FromRepr,
1136 : enum_map::Enum,
1137 : )]
1138 : #[strum(serialize_all = "snake_case")]
1139 : pub enum SmgrQueryType {
1140 : GetRelExists,
1141 : GetRelSize,
1142 : GetPageAtLsn,
1143 : GetDbSize,
1144 : GetSlruSegment,
1145 : }
1146 :
1147 : #[derive(Debug)]
1148 : pub(crate) struct SmgrQueryTimePerTimeline {
1149 : metrics: [GlobalAndPerTimelineHistogram; SmgrQueryType::COUNT],
1150 : }
1151 :
1152 143 : static SMGR_QUERY_TIME_PER_TENANT_TIMELINE: Lazy<HistogramVec> = Lazy::new(|| {
1153 : register_histogram_vec!(
1154 : "pageserver_smgr_query_seconds",
1155 : "Time spent on smgr query handling, aggegated by query type and tenant/timeline.",
1156 : &["smgr_query_type", "tenant_id", "shard_id", "timeline_id"],
1157 : CRITICAL_OP_BUCKETS.into(),
1158 : )
1159 143 : .expect("failed to define a metric")
1160 143 : });
1161 :
1162 143 : static SMGR_QUERY_TIME_GLOBAL_BUCKETS: Lazy<Vec<f64>> = Lazy::new(|| {
1163 143 : [
1164 143 : 1,
1165 143 : 10,
1166 143 : 20,
1167 143 : 40,
1168 143 : 60,
1169 143 : 80,
1170 143 : 100,
1171 143 : 200,
1172 143 : 300,
1173 143 : 400,
1174 143 : 500,
1175 143 : 600,
1176 143 : 700,
1177 143 : 800,
1178 143 : 900,
1179 143 : 1_000, // 1ms
1180 143 : 2_000,
1181 143 : 4_000,
1182 143 : 6_000,
1183 143 : 8_000,
1184 143 : 10_000, // 10ms
1185 143 : 20_000,
1186 143 : 40_000,
1187 143 : 60_000,
1188 143 : 80_000,
1189 143 : 100_000,
1190 143 : 200_000,
1191 143 : 400_000,
1192 143 : 600_000,
1193 143 : 800_000,
1194 143 : 1_000_000, // 1s
1195 143 : 2_000_000,
1196 143 : 4_000_000,
1197 143 : 6_000_000,
1198 143 : 8_000_000,
1199 143 : 10_000_000, // 10s
1200 143 : 20_000_000,
1201 143 : 50_000_000,
1202 143 : 100_000_000,
1203 143 : 200_000_000,
1204 143 : 1_000_000_000, // 1000s
1205 143 : ]
1206 143 : .into_iter()
1207 143 : .map(Duration::from_micros)
1208 5863 : .map(|d| d.as_secs_f64())
1209 143 : .collect()
1210 143 : });
1211 :
1212 143 : static SMGR_QUERY_TIME_GLOBAL: Lazy<HistogramVec> = Lazy::new(|| {
1213 : register_histogram_vec!(
1214 : "pageserver_smgr_query_seconds_global",
1215 : "Time spent on smgr query handling, aggregated by query type.",
1216 : &["smgr_query_type"],
1217 : SMGR_QUERY_TIME_GLOBAL_BUCKETS.clone(),
1218 : )
1219 143 : .expect("failed to define a metric")
1220 143 : });
1221 :
1222 : impl SmgrQueryTimePerTimeline {
1223 393 : pub(crate) fn new(tenant_shard_id: &TenantShardId, timeline_id: &TimelineId) -> Self {
1224 393 : let tenant_id = tenant_shard_id.tenant_id.to_string();
1225 393 : let shard_slug = format!("{}", tenant_shard_id.shard_slug());
1226 393 : let timeline_id = timeline_id.to_string();
1227 1965 : let metrics = std::array::from_fn(|i| {
1228 1965 : let op = SmgrQueryType::from_repr(i).unwrap();
1229 1965 : let global = SMGR_QUERY_TIME_GLOBAL
1230 1965 : .get_metric_with_label_values(&[op.into()])
1231 1965 : .unwrap();
1232 1965 : let per_tenant_timeline = SMGR_QUERY_TIME_PER_TENANT_TIMELINE
1233 1965 : .get_metric_with_label_values(&[op.into(), &tenant_id, &shard_slug, &timeline_id])
1234 1965 : .unwrap();
1235 1965 : GlobalAndPerTimelineHistogram {
1236 1965 : global,
1237 1965 : per_tenant_timeline,
1238 1965 : }
1239 1965 : });
1240 393 : Self { metrics }
1241 393 : }
1242 10 : pub(crate) fn start_timer<'c: 'a, 'a>(
1243 10 : &'a self,
1244 10 : op: SmgrQueryType,
1245 10 : ctx: &'c RequestContext,
1246 10 : ) -> impl Drop + '_ {
1247 10 : let metric = &self.metrics[op as usize];
1248 10 : let start = Instant::now();
1249 10 : match ctx.micros_spent_throttled.open() {
1250 10 : Ok(()) => (),
1251 0 : Err(error) => {
1252 0 : use utils::rate_limit::RateLimit;
1253 0 : static LOGGED: Lazy<Mutex<enum_map::EnumMap<SmgrQueryType, RateLimit>>> =
1254 0 : Lazy::new(|| {
1255 0 : Mutex::new(enum_map::EnumMap::from_array(std::array::from_fn(|_| {
1256 0 : RateLimit::new(Duration::from_secs(10))
1257 0 : })))
1258 0 : });
1259 0 : let mut guard = LOGGED.lock().unwrap();
1260 0 : let rate_limit = &mut guard[op];
1261 0 : rate_limit.call(|| {
1262 0 : warn!(?op, error, "error opening micros_spent_throttled; this message is logged at a global rate limit");
1263 0 : });
1264 0 : }
1265 : }
1266 10 : GlobalAndPerTimelineHistogramTimer {
1267 10 : h: metric,
1268 10 : ctx,
1269 10 : start,
1270 10 : op,
1271 10 : }
1272 10 : }
1273 : }
1274 :
1275 : #[cfg(test)]
1276 : mod smgr_query_time_tests {
1277 : use pageserver_api::shard::TenantShardId;
1278 : use strum::IntoEnumIterator;
1279 : use utils::id::{TenantId, TimelineId};
1280 :
1281 : use crate::{
1282 : context::{DownloadBehavior, RequestContext},
1283 : task_mgr::TaskKind,
1284 : };
1285 :
1286 : // Regression test, we used hard-coded string constants before using an enum.
1287 : #[test]
1288 2 : fn op_label_name() {
1289 2 : use super::SmgrQueryType::*;
1290 2 : let expect: [(super::SmgrQueryType, &'static str); 5] = [
1291 2 : (GetRelExists, "get_rel_exists"),
1292 2 : (GetRelSize, "get_rel_size"),
1293 2 : (GetPageAtLsn, "get_page_at_lsn"),
1294 2 : (GetDbSize, "get_db_size"),
1295 2 : (GetSlruSegment, "get_slru_segment"),
1296 2 : ];
1297 12 : for (op, expect) in expect {
1298 10 : let actual: &'static str = op.into();
1299 10 : assert_eq!(actual, expect);
1300 : }
1301 2 : }
1302 :
1303 : #[test]
1304 2 : fn basic() {
1305 2 : let ops: Vec<_> = super::SmgrQueryType::iter().collect();
1306 :
1307 12 : for op in &ops {
1308 10 : let tenant_id = TenantId::generate();
1309 10 : let timeline_id = TimelineId::generate();
1310 10 : let metrics = super::SmgrQueryTimePerTimeline::new(
1311 10 : &TenantShardId::unsharded(tenant_id),
1312 10 : &timeline_id,
1313 10 : );
1314 10 :
1315 20 : let get_counts = || {
1316 20 : let global: u64 = ops
1317 20 : .iter()
1318 100 : .map(|op| metrics.metrics[*op as usize].global.get_sample_count())
1319 20 : .sum();
1320 20 : let per_tenant_timeline: u64 = ops
1321 20 : .iter()
1322 100 : .map(|op| {
1323 100 : metrics.metrics[*op as usize]
1324 100 : .per_tenant_timeline
1325 100 : .get_sample_count()
1326 100 : })
1327 20 : .sum();
1328 20 : (global, per_tenant_timeline)
1329 20 : };
1330 :
1331 10 : let (pre_global, pre_per_tenant_timeline) = get_counts();
1332 10 : assert_eq!(pre_per_tenant_timeline, 0);
1333 :
1334 10 : let ctx = RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Download);
1335 10 : let timer = metrics.start_timer(*op, &ctx);
1336 10 : drop(timer);
1337 10 :
1338 10 : let (post_global, post_per_tenant_timeline) = get_counts();
1339 10 : assert_eq!(post_per_tenant_timeline, 1);
1340 10 : assert!(post_global > pre_global);
1341 : }
1342 2 : }
1343 : }
1344 :
1345 : // keep in sync with control plane Go code so that we can validate
1346 : // compute's basebackup_ms metric with our perspective in the context of SLI/SLO.
1347 0 : static COMPUTE_STARTUP_BUCKETS: Lazy<[f64; 28]> = Lazy::new(|| {
1348 0 : // Go code uses milliseconds. Variable is called `computeStartupBuckets`
1349 0 : [
1350 0 : 5, 10, 20, 30, 50, 70, 100, 120, 150, 200, 250, 300, 350, 400, 450, 500, 600, 800, 1000,
1351 0 : 1500, 2000, 2500, 3000, 5000, 10000, 20000, 40000, 60000,
1352 0 : ]
1353 0 : .map(|ms| (ms as f64) / 1000.0)
1354 0 : });
1355 :
1356 : pub(crate) struct BasebackupQueryTime {
1357 : ok: Histogram,
1358 : error: Histogram,
1359 : }
1360 :
1361 0 : pub(crate) static BASEBACKUP_QUERY_TIME: Lazy<BasebackupQueryTime> = Lazy::new(|| {
1362 0 : let vec = register_histogram_vec!(
1363 : "pageserver_basebackup_query_seconds",
1364 : "Histogram of basebackup queries durations, by result type",
1365 : &["result"],
1366 : COMPUTE_STARTUP_BUCKETS.to_vec(),
1367 : )
1368 0 : .expect("failed to define a metric");
1369 0 : BasebackupQueryTime {
1370 0 : ok: vec.get_metric_with_label_values(&["ok"]).unwrap(),
1371 0 : error: vec.get_metric_with_label_values(&["error"]).unwrap(),
1372 0 : }
1373 0 : });
1374 :
1375 : pub(crate) struct BasebackupQueryTimeOngoingRecording<'a, 'c> {
1376 : parent: &'a BasebackupQueryTime,
1377 : ctx: &'c RequestContext,
1378 : start: std::time::Instant,
1379 : }
1380 :
1381 : impl BasebackupQueryTime {
1382 0 : pub(crate) fn start_recording<'c: 'a, 'a>(
1383 0 : &'a self,
1384 0 : ctx: &'c RequestContext,
1385 0 : ) -> BasebackupQueryTimeOngoingRecording<'_, '_> {
1386 0 : let start = Instant::now();
1387 0 : match ctx.micros_spent_throttled.open() {
1388 0 : Ok(()) => (),
1389 0 : Err(error) => {
1390 0 : use utils::rate_limit::RateLimit;
1391 0 : static LOGGED: Lazy<Mutex<RateLimit>> =
1392 0 : Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(10))));
1393 0 : let mut rate_limit = LOGGED.lock().unwrap();
1394 0 : rate_limit.call(|| {
1395 0 : warn!(error, "error opening micros_spent_throttled; this message is logged at a global rate limit");
1396 0 : });
1397 0 : }
1398 : }
1399 0 : BasebackupQueryTimeOngoingRecording {
1400 0 : parent: self,
1401 0 : ctx,
1402 0 : start,
1403 0 : }
1404 0 : }
1405 : }
1406 :
1407 : impl<'a, 'c> BasebackupQueryTimeOngoingRecording<'a, 'c> {
1408 0 : pub(crate) fn observe<T, E>(self, res: &Result<T, E>) {
1409 0 : let elapsed = self.start.elapsed();
1410 0 : let ex_throttled = self
1411 0 : .ctx
1412 0 : .micros_spent_throttled
1413 0 : .close_and_checked_sub_from(elapsed);
1414 0 : let ex_throttled = match ex_throttled {
1415 0 : Ok(ex_throttled) => ex_throttled,
1416 0 : Err(error) => {
1417 0 : use utils::rate_limit::RateLimit;
1418 0 : static LOGGED: Lazy<Mutex<RateLimit>> =
1419 0 : Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(10))));
1420 0 : let mut rate_limit = LOGGED.lock().unwrap();
1421 0 : rate_limit.call(|| {
1422 0 : warn!(error, "error deducting time spent throttled; this message is logged at a global rate limit");
1423 0 : });
1424 0 : elapsed
1425 : }
1426 : };
1427 0 : let metric = if res.is_ok() {
1428 0 : &self.parent.ok
1429 : } else {
1430 0 : &self.parent.error
1431 : };
1432 0 : metric.observe(ex_throttled.as_secs_f64());
1433 0 : }
1434 : }
1435 :
1436 0 : pub(crate) static LIVE_CONNECTIONS_COUNT: Lazy<IntGaugeVec> = Lazy::new(|| {
1437 : register_int_gauge_vec!(
1438 : "pageserver_live_connections",
1439 : "Number of live network connections",
1440 : &["pageserver_connection_kind"]
1441 : )
1442 0 : .expect("failed to define a metric")
1443 0 : });
1444 :
1445 0 : #[derive(Clone, Copy, enum_map::Enum, IntoStaticStr)]
1446 : pub(crate) enum ComputeCommandKind {
1447 : PageStreamV2,
1448 : PageStream,
1449 : Basebackup,
1450 : GetLastRecordRlsn,
1451 : Fullbackup,
1452 : ImportBasebackup,
1453 : ImportWal,
1454 : LeaseLsn,
1455 : Show,
1456 : }
1457 :
1458 : pub(crate) struct ComputeCommandCounters {
1459 : map: EnumMap<ComputeCommandKind, IntCounter>,
1460 : }
1461 :
1462 0 : pub(crate) static COMPUTE_COMMANDS_COUNTERS: Lazy<ComputeCommandCounters> = Lazy::new(|| {
1463 0 : let inner = register_int_counter_vec!(
1464 : "pageserver_compute_commands",
1465 : "Number of compute -> pageserver commands processed",
1466 : &["command"]
1467 : )
1468 0 : .expect("failed to define a metric");
1469 0 :
1470 0 : ComputeCommandCounters {
1471 0 : map: EnumMap::from_array(std::array::from_fn(|i| {
1472 0 : let command = <ComputeCommandKind as enum_map::Enum>::from_usize(i);
1473 0 : let command_str: &'static str = command.into();
1474 0 : inner.with_label_values(&[command_str])
1475 0 : })),
1476 0 : }
1477 0 : });
1478 :
1479 : impl ComputeCommandCounters {
1480 0 : pub(crate) fn for_command(&self, command: ComputeCommandKind) -> &IntCounter {
1481 0 : &self.map[command]
1482 0 : }
1483 : }
1484 :
1485 : // remote storage metrics
1486 :
1487 138 : static REMOTE_TIMELINE_CLIENT_CALLS: Lazy<IntCounterPairVec> = Lazy::new(|| {
1488 : register_int_counter_pair_vec!(
1489 : "pageserver_remote_timeline_client_calls_started",
1490 : "Number of started calls to remote timeline client.",
1491 : "pageserver_remote_timeline_client_calls_finished",
1492 : "Number of finshed calls to remote timeline client.",
1493 : &[
1494 : "tenant_id",
1495 : "shard_id",
1496 : "timeline_id",
1497 : "file_kind",
1498 : "op_kind"
1499 : ],
1500 : )
1501 138 : .unwrap()
1502 138 : });
1503 :
1504 : static REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER: Lazy<IntCounterVec> =
1505 134 : Lazy::new(|| {
1506 : register_int_counter_vec!(
1507 : "pageserver_remote_timeline_client_bytes_started",
1508 : "Incremented by the number of bytes associated with a remote timeline client operation. \
1509 : The increment happens when the operation is scheduled.",
1510 : &["tenant_id", "shard_id", "timeline_id", "file_kind", "op_kind"],
1511 : )
1512 134 : .expect("failed to define a metric")
1513 134 : });
1514 :
1515 134 : static REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER: Lazy<IntCounterVec> = Lazy::new(|| {
1516 : register_int_counter_vec!(
1517 : "pageserver_remote_timeline_client_bytes_finished",
1518 : "Incremented by the number of bytes associated with a remote timeline client operation. \
1519 : The increment happens when the operation finishes (regardless of success/failure/shutdown).",
1520 : &["tenant_id", "shard_id", "timeline_id", "file_kind", "op_kind"],
1521 : )
1522 134 : .expect("failed to define a metric")
1523 134 : });
1524 :
1525 : pub(crate) struct TenantManagerMetrics {
1526 : tenant_slots_attached: UIntGauge,
1527 : tenant_slots_secondary: UIntGauge,
1528 : tenant_slots_inprogress: UIntGauge,
1529 : pub(crate) tenant_slot_writes: IntCounter,
1530 : pub(crate) unexpected_errors: IntCounter,
1531 : }
1532 :
1533 : impl TenantManagerMetrics {
1534 : /// Helpers for tracking slots. Note that these do not track the lifetime of TenantSlot objects
1535 : /// exactly: they track the lifetime of the slots _in the tenant map_.
1536 2 : pub(crate) fn slot_inserted(&self, slot: &TenantSlot) {
1537 2 : match slot {
1538 0 : TenantSlot::Attached(_) => {
1539 0 : self.tenant_slots_attached.inc();
1540 0 : }
1541 0 : TenantSlot::Secondary(_) => {
1542 0 : self.tenant_slots_secondary.inc();
1543 0 : }
1544 2 : TenantSlot::InProgress(_) => {
1545 2 : self.tenant_slots_inprogress.inc();
1546 2 : }
1547 : }
1548 2 : }
1549 :
1550 2 : pub(crate) fn slot_removed(&self, slot: &TenantSlot) {
1551 2 : match slot {
1552 2 : TenantSlot::Attached(_) => {
1553 2 : self.tenant_slots_attached.dec();
1554 2 : }
1555 0 : TenantSlot::Secondary(_) => {
1556 0 : self.tenant_slots_secondary.dec();
1557 0 : }
1558 0 : TenantSlot::InProgress(_) => {
1559 0 : self.tenant_slots_inprogress.dec();
1560 0 : }
1561 : }
1562 2 : }
1563 :
1564 : #[cfg(all(debug_assertions, not(test)))]
1565 0 : pub(crate) fn slots_total(&self) -> u64 {
1566 0 : self.tenant_slots_attached.get()
1567 0 : + self.tenant_slots_secondary.get()
1568 0 : + self.tenant_slots_inprogress.get()
1569 0 : }
1570 : }
1571 :
1572 2 : pub(crate) static TENANT_MANAGER: Lazy<TenantManagerMetrics> = Lazy::new(|| {
1573 2 : let tenant_slots = register_uint_gauge_vec!(
1574 : "pageserver_tenant_manager_slots",
1575 : "How many slots currently exist, including all attached, secondary and in-progress operations",
1576 : &["mode"]
1577 : )
1578 2 : .expect("failed to define a metric");
1579 2 : TenantManagerMetrics {
1580 2 : tenant_slots_attached: tenant_slots
1581 2 : .get_metric_with_label_values(&["attached"])
1582 2 : .unwrap(),
1583 2 : tenant_slots_secondary: tenant_slots
1584 2 : .get_metric_with_label_values(&["secondary"])
1585 2 : .unwrap(),
1586 2 : tenant_slots_inprogress: tenant_slots
1587 2 : .get_metric_with_label_values(&["inprogress"])
1588 2 : .unwrap(),
1589 2 : tenant_slot_writes: register_int_counter!(
1590 : "pageserver_tenant_manager_slot_writes",
1591 : "Writes to a tenant slot, including all of create/attach/detach/delete"
1592 2 : )
1593 2 : .expect("failed to define a metric"),
1594 2 : unexpected_errors: register_int_counter!(
1595 : "pageserver_tenant_manager_unexpected_errors_total",
1596 : "Number of unexpected conditions encountered: nonzero value indicates a non-fatal bug."
1597 2 : )
1598 2 : .expect("failed to define a metric"),
1599 2 : }
1600 2 : });
1601 :
1602 : pub(crate) struct DeletionQueueMetrics {
1603 : pub(crate) keys_submitted: IntCounter,
1604 : pub(crate) keys_dropped: IntCounter,
1605 : pub(crate) keys_executed: IntCounter,
1606 : pub(crate) keys_validated: IntCounter,
1607 : pub(crate) dropped_lsn_updates: IntCounter,
1608 : pub(crate) unexpected_errors: IntCounter,
1609 : pub(crate) remote_errors: IntCounterVec,
1610 : }
1611 25 : pub(crate) static DELETION_QUEUE: Lazy<DeletionQueueMetrics> = Lazy::new(|| {
1612 25 : DeletionQueueMetrics{
1613 25 :
1614 25 : keys_submitted: register_int_counter!(
1615 : "pageserver_deletion_queue_submitted_total",
1616 : "Number of objects submitted for deletion"
1617 25 : )
1618 25 : .expect("failed to define a metric"),
1619 25 :
1620 25 : keys_dropped: register_int_counter!(
1621 : "pageserver_deletion_queue_dropped_total",
1622 : "Number of object deletions dropped due to stale generation."
1623 25 : )
1624 25 : .expect("failed to define a metric"),
1625 25 :
1626 25 : keys_executed: register_int_counter!(
1627 : "pageserver_deletion_queue_executed_total",
1628 : "Number of objects deleted. Only includes objects that we actually deleted, sum with pageserver_deletion_queue_dropped_total for the total number of keys processed to completion"
1629 25 : )
1630 25 : .expect("failed to define a metric"),
1631 25 :
1632 25 : keys_validated: register_int_counter!(
1633 : "pageserver_deletion_queue_validated_total",
1634 : "Number of keys validated for deletion. Sum with pageserver_deletion_queue_dropped_total for the total number of keys that have passed through the validation stage."
1635 25 : )
1636 25 : .expect("failed to define a metric"),
1637 25 :
1638 25 : dropped_lsn_updates: register_int_counter!(
1639 : "pageserver_deletion_queue_dropped_lsn_updates_total",
1640 : "Updates to remote_consistent_lsn dropped due to stale generation number."
1641 25 : )
1642 25 : .expect("failed to define a metric"),
1643 25 : unexpected_errors: register_int_counter!(
1644 : "pageserver_deletion_queue_unexpected_errors_total",
1645 : "Number of unexpected condiions that may stall the queue: any value above zero is unexpected."
1646 25 : )
1647 25 : .expect("failed to define a metric"),
1648 25 : remote_errors: register_int_counter_vec!(
1649 : "pageserver_deletion_queue_remote_errors_total",
1650 : "Retryable remote I/O errors while executing deletions, for example 503 responses to DeleteObjects",
1651 : &["op_kind"],
1652 25 : )
1653 25 : .expect("failed to define a metric")
1654 25 : }
1655 25 : });
1656 :
1657 : pub(crate) struct SecondaryModeMetrics {
1658 : pub(crate) upload_heatmap: IntCounter,
1659 : pub(crate) upload_heatmap_errors: IntCounter,
1660 : pub(crate) upload_heatmap_duration: Histogram,
1661 : pub(crate) download_heatmap: IntCounter,
1662 : pub(crate) download_layer: IntCounter,
1663 : }
1664 0 : pub(crate) static SECONDARY_MODE: Lazy<SecondaryModeMetrics> = Lazy::new(|| {
1665 0 : SecondaryModeMetrics {
1666 0 : upload_heatmap: register_int_counter!(
1667 : "pageserver_secondary_upload_heatmap",
1668 : "Number of heatmaps written to remote storage by attached tenants"
1669 0 : )
1670 0 : .expect("failed to define a metric"),
1671 0 : upload_heatmap_errors: register_int_counter!(
1672 : "pageserver_secondary_upload_heatmap_errors",
1673 : "Failures writing heatmap to remote storage"
1674 0 : )
1675 0 : .expect("failed to define a metric"),
1676 0 : upload_heatmap_duration: register_histogram!(
1677 : "pageserver_secondary_upload_heatmap_duration",
1678 : "Time to build and upload a heatmap, including any waiting inside the S3 client"
1679 0 : )
1680 0 : .expect("failed to define a metric"),
1681 0 : download_heatmap: register_int_counter!(
1682 : "pageserver_secondary_download_heatmap",
1683 : "Number of downloads of heatmaps by secondary mode locations, including when it hasn't changed"
1684 0 : )
1685 0 : .expect("failed to define a metric"),
1686 0 : download_layer: register_int_counter!(
1687 : "pageserver_secondary_download_layer",
1688 : "Number of downloads of layers by secondary mode locations"
1689 0 : )
1690 0 : .expect("failed to define a metric"),
1691 0 : }
1692 0 : });
1693 :
1694 : #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1695 : pub enum RemoteOpKind {
1696 : Upload,
1697 : Download,
1698 : Delete,
1699 : }
1700 : impl RemoteOpKind {
1701 13287 : pub fn as_str(&self) -> &'static str {
1702 13287 : match self {
1703 12514 : Self::Upload => "upload",
1704 52 : Self::Download => "download",
1705 721 : Self::Delete => "delete",
1706 : }
1707 13287 : }
1708 : }
1709 :
1710 : #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
1711 : pub enum RemoteOpFileKind {
1712 : Layer,
1713 : Index,
1714 : }
1715 : impl RemoteOpFileKind {
1716 13287 : pub fn as_str(&self) -> &'static str {
1717 13287 : match self {
1718 9115 : Self::Layer => "layer",
1719 4172 : Self::Index => "index",
1720 : }
1721 13287 : }
1722 : }
1723 :
1724 133 : pub(crate) static REMOTE_OPERATION_TIME: Lazy<HistogramVec> = Lazy::new(|| {
1725 : register_histogram_vec!(
1726 : "pageserver_remote_operation_seconds",
1727 : "Time spent on remote storage operations. \
1728 : Grouped by tenant, timeline, operation_kind and status. \
1729 : Does not account for time spent waiting in remote timeline client's queues.",
1730 : &["file_kind", "op_kind", "status"]
1731 : )
1732 133 : .expect("failed to define a metric")
1733 133 : });
1734 :
1735 0 : pub(crate) static TENANT_TASK_EVENTS: Lazy<IntCounterVec> = Lazy::new(|| {
1736 : register_int_counter_vec!(
1737 : "pageserver_tenant_task_events",
1738 : "Number of task start/stop/fail events.",
1739 : &["event"],
1740 : )
1741 0 : .expect("Failed to register tenant_task_events metric")
1742 0 : });
1743 :
1744 20 : pub(crate) static BACKGROUND_LOOP_SEMAPHORE_WAIT_GAUGE: Lazy<IntCounterPairVec> = Lazy::new(|| {
1745 : register_int_counter_pair_vec!(
1746 : "pageserver_background_loop_semaphore_wait_start_count",
1747 : "Counter for background loop concurrency-limiting semaphore acquire calls started",
1748 : "pageserver_background_loop_semaphore_wait_finish_count",
1749 : "Counter for background loop concurrency-limiting semaphore acquire calls finished",
1750 : &["task"],
1751 : )
1752 20 : .unwrap()
1753 20 : });
1754 :
1755 0 : pub(crate) static BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT: Lazy<IntCounterVec> = Lazy::new(|| {
1756 : register_int_counter_vec!(
1757 : "pageserver_background_loop_period_overrun_count",
1758 : "Incremented whenever warn_when_period_overrun() logs a warning.",
1759 : &["task", "period"],
1760 : )
1761 0 : .expect("failed to define a metric")
1762 0 : });
1763 :
1764 : // walreceiver metrics
1765 :
1766 0 : pub(crate) static WALRECEIVER_STARTED_CONNECTIONS: Lazy<IntCounter> = Lazy::new(|| {
1767 : register_int_counter!(
1768 : "pageserver_walreceiver_started_connections_total",
1769 : "Number of started walreceiver connections"
1770 : )
1771 0 : .expect("failed to define a metric")
1772 0 : });
1773 :
1774 0 : pub(crate) static WALRECEIVER_ACTIVE_MANAGERS: Lazy<IntGauge> = Lazy::new(|| {
1775 : register_int_gauge!(
1776 : "pageserver_walreceiver_active_managers",
1777 : "Number of active walreceiver managers"
1778 : )
1779 0 : .expect("failed to define a metric")
1780 0 : });
1781 :
1782 0 : pub(crate) static WALRECEIVER_SWITCHES: Lazy<IntCounterVec> = Lazy::new(|| {
1783 : register_int_counter_vec!(
1784 : "pageserver_walreceiver_switches_total",
1785 : "Number of walreceiver manager change_connection calls",
1786 : &["reason"]
1787 : )
1788 0 : .expect("failed to define a metric")
1789 0 : });
1790 :
1791 0 : pub(crate) static WALRECEIVER_BROKER_UPDATES: Lazy<IntCounter> = Lazy::new(|| {
1792 : register_int_counter!(
1793 : "pageserver_walreceiver_broker_updates_total",
1794 : "Number of received broker updates in walreceiver"
1795 : )
1796 0 : .expect("failed to define a metric")
1797 0 : });
1798 :
1799 2 : pub(crate) static WALRECEIVER_CANDIDATES_EVENTS: Lazy<IntCounterVec> = Lazy::new(|| {
1800 : register_int_counter_vec!(
1801 : "pageserver_walreceiver_candidates_events_total",
1802 : "Number of walreceiver candidate events",
1803 : &["event"]
1804 : )
1805 2 : .expect("failed to define a metric")
1806 2 : });
1807 :
1808 : pub(crate) static WALRECEIVER_CANDIDATES_ADDED: Lazy<IntCounter> =
1809 0 : Lazy::new(|| WALRECEIVER_CANDIDATES_EVENTS.with_label_values(&["add"]));
1810 :
1811 : pub(crate) static WALRECEIVER_CANDIDATES_REMOVED: Lazy<IntCounter> =
1812 2 : Lazy::new(|| WALRECEIVER_CANDIDATES_EVENTS.with_label_values(&["remove"]));
1813 :
1814 : // Metrics collected on WAL redo operations
1815 : //
1816 : // We collect the time spent in actual WAL redo ('redo'), and time waiting
1817 : // for access to the postgres process ('wait') since there is only one for
1818 : // each tenant.
1819 :
1820 : /// Time buckets are small because we want to be able to measure the
1821 : /// smallest redo processing times. These buckets allow us to measure down
1822 : /// to 5us, which equates to 200'000 pages/sec, which equates to 1.6GB/sec.
1823 : /// This is much better than the previous 5ms aka 200 pages/sec aka 1.6MB/sec.
1824 : ///
1825 : /// Values up to 1s are recorded because metrics show that we have redo
1826 : /// durations and lock times larger than 0.250s.
1827 : macro_rules! redo_histogram_time_buckets {
1828 : () => {
1829 : vec![
1830 : 0.000_005, 0.000_010, 0.000_025, 0.000_050, 0.000_100, 0.000_250, 0.000_500, 0.001_000,
1831 : 0.002_500, 0.005_000, 0.010_000, 0.025_000, 0.050_000, 0.100_000, 0.250_000, 0.500_000,
1832 : 1.000_000,
1833 : ]
1834 : };
1835 : }
1836 :
1837 : /// While we're at it, also measure the amount of records replayed in each
1838 : /// operation. We have a global 'total replayed' counter, but that's not
1839 : /// as useful as 'what is the skew for how many records we replay in one
1840 : /// operation'.
1841 : macro_rules! redo_histogram_count_buckets {
1842 : () => {
1843 : vec![0.0, 1.0, 2.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0]
1844 : };
1845 : }
1846 :
1847 : macro_rules! redo_bytes_histogram_count_buckets {
1848 : () => {
1849 : // powers of (2^.5), from 2^4.5 to 2^15 (22 buckets)
1850 : // rounded up to the next multiple of 8 to capture any MAXALIGNed record of that size, too.
1851 : vec![
1852 : 24.0, 32.0, 48.0, 64.0, 96.0, 128.0, 184.0, 256.0, 368.0, 512.0, 728.0, 1024.0, 1456.0,
1853 : 2048.0, 2904.0, 4096.0, 5800.0, 8192.0, 11592.0, 16384.0, 23176.0, 32768.0,
1854 : ]
1855 : };
1856 : }
1857 :
1858 : pub(crate) struct WalIngestMetrics {
1859 : pub(crate) bytes_received: IntCounter,
1860 : pub(crate) records_received: IntCounter,
1861 : pub(crate) records_committed: IntCounter,
1862 : pub(crate) records_filtered: IntCounter,
1863 : }
1864 :
1865 2 : pub(crate) static WAL_INGEST: Lazy<WalIngestMetrics> = Lazy::new(|| WalIngestMetrics {
1866 : bytes_received: register_int_counter!(
1867 : "pageserver_wal_ingest_bytes_received",
1868 : "Bytes of WAL ingested from safekeepers",
1869 : )
1870 2 : .unwrap(),
1871 : records_received: register_int_counter!(
1872 : "pageserver_wal_ingest_records_received",
1873 : "Number of WAL records received from safekeepers"
1874 : )
1875 2 : .expect("failed to define a metric"),
1876 : records_committed: register_int_counter!(
1877 : "pageserver_wal_ingest_records_committed",
1878 : "Number of WAL records which resulted in writes to pageserver storage"
1879 : )
1880 2 : .expect("failed to define a metric"),
1881 : records_filtered: register_int_counter!(
1882 : "pageserver_wal_ingest_records_filtered",
1883 : "Number of WAL records filtered out due to sharding"
1884 : )
1885 2 : .expect("failed to define a metric"),
1886 2 : });
1887 :
1888 6 : pub(crate) static WAL_REDO_TIME: Lazy<Histogram> = Lazy::new(|| {
1889 : register_histogram!(
1890 : "pageserver_wal_redo_seconds",
1891 : "Time spent on WAL redo",
1892 : redo_histogram_time_buckets!()
1893 : )
1894 6 : .expect("failed to define a metric")
1895 6 : });
1896 :
1897 6 : pub(crate) static WAL_REDO_RECORDS_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
1898 : register_histogram!(
1899 : "pageserver_wal_redo_records_histogram",
1900 : "Histogram of number of records replayed per redo in the Postgres WAL redo process",
1901 : redo_histogram_count_buckets!(),
1902 : )
1903 6 : .expect("failed to define a metric")
1904 6 : });
1905 :
1906 6 : pub(crate) static WAL_REDO_BYTES_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
1907 : register_histogram!(
1908 : "pageserver_wal_redo_bytes_histogram",
1909 : "Histogram of number of records replayed per redo sent to Postgres",
1910 : redo_bytes_histogram_count_buckets!(),
1911 : )
1912 6 : .expect("failed to define a metric")
1913 6 : });
1914 :
1915 : // FIXME: isn't this already included by WAL_REDO_RECORDS_HISTOGRAM which has _count?
1916 6 : pub(crate) static WAL_REDO_RECORD_COUNTER: Lazy<IntCounter> = Lazy::new(|| {
1917 : register_int_counter!(
1918 : "pageserver_replayed_wal_records_total",
1919 : "Number of WAL records replayed in WAL redo process"
1920 : )
1921 6 : .unwrap()
1922 6 : });
1923 :
1924 : #[rustfmt::skip]
1925 6 : pub(crate) static WAL_REDO_PROCESS_LAUNCH_DURATION_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
1926 : register_histogram!(
1927 : "pageserver_wal_redo_process_launch_duration",
1928 : "Histogram of the duration of successful WalRedoProcess::launch calls",
1929 : vec![
1930 : 0.0002, 0.0004, 0.0006, 0.0008, 0.0010,
1931 : 0.0020, 0.0040, 0.0060, 0.0080, 0.0100,
1932 : 0.0200, 0.0400, 0.0600, 0.0800, 0.1000,
1933 : 0.2000, 0.4000, 0.6000, 0.8000, 1.0000,
1934 : 1.5000, 2.0000, 2.5000, 3.0000, 4.0000, 10.0000
1935 : ],
1936 : )
1937 6 : .expect("failed to define a metric")
1938 6 : });
1939 :
1940 : pub(crate) struct WalRedoProcessCounters {
1941 : pub(crate) started: IntCounter,
1942 : pub(crate) killed_by_cause: enum_map::EnumMap<WalRedoKillCause, IntCounter>,
1943 : pub(crate) active_stderr_logger_tasks_started: IntCounter,
1944 : pub(crate) active_stderr_logger_tasks_finished: IntCounter,
1945 : }
1946 :
1947 18 : #[derive(Debug, enum_map::Enum, strum_macros::IntoStaticStr)]
1948 : pub(crate) enum WalRedoKillCause {
1949 : WalRedoProcessDrop,
1950 : NoLeakChildDrop,
1951 : Startup,
1952 : }
1953 :
1954 : impl Default for WalRedoProcessCounters {
1955 6 : fn default() -> Self {
1956 6 : let started = register_int_counter!(
1957 : "pageserver_wal_redo_process_started_total",
1958 : "Number of WAL redo processes started",
1959 : )
1960 6 : .unwrap();
1961 6 :
1962 6 : let killed = register_int_counter_vec!(
1963 : "pageserver_wal_redo_process_stopped_total",
1964 : "Number of WAL redo processes stopped",
1965 : &["cause"],
1966 : )
1967 6 : .unwrap();
1968 6 :
1969 6 : let active_stderr_logger_tasks_started = register_int_counter!(
1970 : "pageserver_walredo_stderr_logger_tasks_started_total",
1971 : "Number of active walredo stderr logger tasks that have started",
1972 : )
1973 6 : .unwrap();
1974 6 :
1975 6 : let active_stderr_logger_tasks_finished = register_int_counter!(
1976 : "pageserver_walredo_stderr_logger_tasks_finished_total",
1977 : "Number of active walredo stderr logger tasks that have finished",
1978 : )
1979 6 : .unwrap();
1980 6 :
1981 6 : Self {
1982 6 : started,
1983 18 : killed_by_cause: EnumMap::from_array(std::array::from_fn(|i| {
1984 18 : let cause = <WalRedoKillCause as enum_map::Enum>::from_usize(i);
1985 18 : let cause_str: &'static str = cause.into();
1986 18 : killed.with_label_values(&[cause_str])
1987 18 : })),
1988 6 : active_stderr_logger_tasks_started,
1989 6 : active_stderr_logger_tasks_finished,
1990 6 : }
1991 6 : }
1992 : }
1993 :
1994 : pub(crate) static WAL_REDO_PROCESS_COUNTERS: Lazy<WalRedoProcessCounters> =
1995 : Lazy::new(WalRedoProcessCounters::default);
1996 :
1997 : /// Similar to `prometheus::HistogramTimer` but does not record on drop.
1998 : pub(crate) struct StorageTimeMetricsTimer {
1999 : metrics: StorageTimeMetrics,
2000 : start: Instant,
2001 : }
2002 :
2003 : impl StorageTimeMetricsTimer {
2004 4602 : fn new(metrics: StorageTimeMetrics) -> Self {
2005 4602 : Self {
2006 4602 : metrics,
2007 4602 : start: Instant::now(),
2008 4602 : }
2009 4602 : }
2010 :
2011 : /// Record the time from creation to now.
2012 3496 : pub fn stop_and_record(self) {
2013 3496 : let duration = self.start.elapsed().as_secs_f64();
2014 3496 : self.metrics.timeline_sum.inc_by(duration);
2015 3496 : self.metrics.timeline_count.inc();
2016 3496 : self.metrics.global_histogram.observe(duration);
2017 3496 : }
2018 :
2019 : /// Turns this timer into a timer, which will always record -- usually this means recording
2020 : /// regardless an early `?` path was taken in a function.
2021 754 : pub(crate) fn record_on_drop(self) -> AlwaysRecordingStorageTimeMetricsTimer {
2022 754 : AlwaysRecordingStorageTimeMetricsTimer(Some(self))
2023 754 : }
2024 : }
2025 :
2026 : pub(crate) struct AlwaysRecordingStorageTimeMetricsTimer(Option<StorageTimeMetricsTimer>);
2027 :
2028 : impl Drop for AlwaysRecordingStorageTimeMetricsTimer {
2029 754 : fn drop(&mut self) {
2030 754 : if let Some(inner) = self.0.take() {
2031 754 : inner.stop_and_record();
2032 754 : }
2033 754 : }
2034 : }
2035 :
2036 : /// Timing facilities for an globally histogrammed metric, which is supported by per tenant and
2037 : /// timeline total sum and count.
2038 : #[derive(Clone, Debug)]
2039 : pub(crate) struct StorageTimeMetrics {
2040 : /// Sum of f64 seconds, per operation, tenant_id and timeline_id
2041 : timeline_sum: Counter,
2042 : /// Number of oeprations, per operation, tenant_id and timeline_id
2043 : timeline_count: IntCounter,
2044 : /// Global histogram having only the "operation" label.
2045 : global_histogram: Histogram,
2046 : }
2047 :
2048 : impl StorageTimeMetrics {
2049 3064 : pub fn new(
2050 3064 : operation: StorageTimeOperation,
2051 3064 : tenant_id: &str,
2052 3064 : shard_id: &str,
2053 3064 : timeline_id: &str,
2054 3064 : ) -> Self {
2055 3064 : let operation: &'static str = operation.into();
2056 3064 :
2057 3064 : let timeline_sum = STORAGE_TIME_SUM_PER_TIMELINE
2058 3064 : .get_metric_with_label_values(&[operation, tenant_id, shard_id, timeline_id])
2059 3064 : .unwrap();
2060 3064 : let timeline_count = STORAGE_TIME_COUNT_PER_TIMELINE
2061 3064 : .get_metric_with_label_values(&[operation, tenant_id, shard_id, timeline_id])
2062 3064 : .unwrap();
2063 3064 : let global_histogram = STORAGE_TIME_GLOBAL
2064 3064 : .get_metric_with_label_values(&[operation])
2065 3064 : .unwrap();
2066 3064 :
2067 3064 : StorageTimeMetrics {
2068 3064 : timeline_sum,
2069 3064 : timeline_count,
2070 3064 : global_histogram,
2071 3064 : }
2072 3064 : }
2073 :
2074 : /// Starts timing a new operation.
2075 : ///
2076 : /// Note: unlike `prometheus::HistogramTimer` the returned timer does not record on drop.
2077 4602 : pub fn start_timer(&self) -> StorageTimeMetricsTimer {
2078 4602 : StorageTimeMetricsTimer::new(self.clone())
2079 4602 : }
2080 : }
2081 :
2082 : #[derive(Debug)]
2083 : pub(crate) struct TimelineMetrics {
2084 : tenant_id: String,
2085 : shard_id: String,
2086 : timeline_id: String,
2087 : pub flush_time_histo: StorageTimeMetrics,
2088 : pub compact_time_histo: StorageTimeMetrics,
2089 : pub create_images_time_histo: StorageTimeMetrics,
2090 : pub logical_size_histo: StorageTimeMetrics,
2091 : pub imitate_logical_size_histo: StorageTimeMetrics,
2092 : pub load_layer_map_histo: StorageTimeMetrics,
2093 : pub garbage_collect_histo: StorageTimeMetrics,
2094 : pub find_gc_cutoffs_histo: StorageTimeMetrics,
2095 : pub last_record_gauge: IntGauge,
2096 : pub standby_horizon_gauge: IntGauge,
2097 : pub resident_physical_size_gauge: UIntGauge,
2098 : /// copy of LayeredTimeline.current_logical_size
2099 : pub current_logical_size_gauge: UIntGauge,
2100 : pub aux_file_size_gauge: IntGauge,
2101 : pub directory_entries_count_gauge: Lazy<UIntGauge, Box<dyn Send + Fn() -> UIntGauge>>,
2102 : pub evictions: IntCounter,
2103 : pub evictions_with_low_residence_duration: std::sync::RwLock<EvictionsWithLowResidenceDuration>,
2104 : /// Number of valid LSN leases.
2105 : pub valid_lsn_lease_count_gauge: UIntGauge,
2106 : shutdown: std::sync::atomic::AtomicBool,
2107 : }
2108 :
2109 : impl TimelineMetrics {
2110 383 : pub fn new(
2111 383 : tenant_shard_id: &TenantShardId,
2112 383 : timeline_id_raw: &TimelineId,
2113 383 : evictions_with_low_residence_duration_builder: EvictionsWithLowResidenceDurationBuilder,
2114 383 : ) -> Self {
2115 383 : let tenant_id = tenant_shard_id.tenant_id.to_string();
2116 383 : let shard_id = format!("{}", tenant_shard_id.shard_slug());
2117 383 : let timeline_id = timeline_id_raw.to_string();
2118 383 : let flush_time_histo = StorageTimeMetrics::new(
2119 383 : StorageTimeOperation::LayerFlush,
2120 383 : &tenant_id,
2121 383 : &shard_id,
2122 383 : &timeline_id,
2123 383 : );
2124 383 : let compact_time_histo = StorageTimeMetrics::new(
2125 383 : StorageTimeOperation::Compact,
2126 383 : &tenant_id,
2127 383 : &shard_id,
2128 383 : &timeline_id,
2129 383 : );
2130 383 : let create_images_time_histo = StorageTimeMetrics::new(
2131 383 : StorageTimeOperation::CreateImages,
2132 383 : &tenant_id,
2133 383 : &shard_id,
2134 383 : &timeline_id,
2135 383 : );
2136 383 : let logical_size_histo = StorageTimeMetrics::new(
2137 383 : StorageTimeOperation::LogicalSize,
2138 383 : &tenant_id,
2139 383 : &shard_id,
2140 383 : &timeline_id,
2141 383 : );
2142 383 : let imitate_logical_size_histo = StorageTimeMetrics::new(
2143 383 : StorageTimeOperation::ImitateLogicalSize,
2144 383 : &tenant_id,
2145 383 : &shard_id,
2146 383 : &timeline_id,
2147 383 : );
2148 383 : let load_layer_map_histo = StorageTimeMetrics::new(
2149 383 : StorageTimeOperation::LoadLayerMap,
2150 383 : &tenant_id,
2151 383 : &shard_id,
2152 383 : &timeline_id,
2153 383 : );
2154 383 : let garbage_collect_histo = StorageTimeMetrics::new(
2155 383 : StorageTimeOperation::Gc,
2156 383 : &tenant_id,
2157 383 : &shard_id,
2158 383 : &timeline_id,
2159 383 : );
2160 383 : let find_gc_cutoffs_histo = StorageTimeMetrics::new(
2161 383 : StorageTimeOperation::FindGcCutoffs,
2162 383 : &tenant_id,
2163 383 : &shard_id,
2164 383 : &timeline_id,
2165 383 : );
2166 383 : let last_record_gauge = LAST_RECORD_LSN
2167 383 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2168 383 : .unwrap();
2169 383 : let standby_horizon_gauge = STANDBY_HORIZON
2170 383 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2171 383 : .unwrap();
2172 383 : let resident_physical_size_gauge = RESIDENT_PHYSICAL_SIZE
2173 383 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2174 383 : .unwrap();
2175 383 : // TODO: we shouldn't expose this metric
2176 383 : let current_logical_size_gauge = CURRENT_LOGICAL_SIZE
2177 383 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2178 383 : .unwrap();
2179 383 : let aux_file_size_gauge = AUX_FILE_SIZE
2180 383 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2181 383 : .unwrap();
2182 383 : // TODO use impl Trait syntax here once we have ability to use it: https://github.com/rust-lang/rust/issues/63065
2183 383 : let directory_entries_count_gauge_closure = {
2184 383 : let tenant_shard_id = *tenant_shard_id;
2185 383 : let timeline_id_raw = *timeline_id_raw;
2186 0 : move || {
2187 0 : let tenant_id = tenant_shard_id.tenant_id.to_string();
2188 0 : let shard_id = format!("{}", tenant_shard_id.shard_slug());
2189 0 : let timeline_id = timeline_id_raw.to_string();
2190 0 : let gauge: UIntGauge = DIRECTORY_ENTRIES_COUNT
2191 0 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2192 0 : .unwrap();
2193 0 : gauge
2194 0 : }
2195 : };
2196 383 : let directory_entries_count_gauge: Lazy<UIntGauge, Box<dyn Send + Fn() -> UIntGauge>> =
2197 383 : Lazy::new(Box::new(directory_entries_count_gauge_closure));
2198 383 : let evictions = EVICTIONS
2199 383 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2200 383 : .unwrap();
2201 383 : let evictions_with_low_residence_duration = evictions_with_low_residence_duration_builder
2202 383 : .build(&tenant_id, &shard_id, &timeline_id);
2203 383 :
2204 383 : let valid_lsn_lease_count_gauge = VALID_LSN_LEASE_COUNT
2205 383 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2206 383 : .unwrap();
2207 383 :
2208 383 : TimelineMetrics {
2209 383 : tenant_id,
2210 383 : shard_id,
2211 383 : timeline_id,
2212 383 : flush_time_histo,
2213 383 : compact_time_histo,
2214 383 : create_images_time_histo,
2215 383 : logical_size_histo,
2216 383 : imitate_logical_size_histo,
2217 383 : garbage_collect_histo,
2218 383 : find_gc_cutoffs_histo,
2219 383 : load_layer_map_histo,
2220 383 : last_record_gauge,
2221 383 : standby_horizon_gauge,
2222 383 : resident_physical_size_gauge,
2223 383 : current_logical_size_gauge,
2224 383 : aux_file_size_gauge,
2225 383 : directory_entries_count_gauge,
2226 383 : evictions,
2227 383 : evictions_with_low_residence_duration: std::sync::RwLock::new(
2228 383 : evictions_with_low_residence_duration,
2229 383 : ),
2230 383 : valid_lsn_lease_count_gauge,
2231 383 : shutdown: std::sync::atomic::AtomicBool::default(),
2232 383 : }
2233 383 : }
2234 :
2235 1468 : pub(crate) fn record_new_file_metrics(&self, sz: u64) {
2236 1468 : self.resident_physical_size_add(sz);
2237 1468 : }
2238 :
2239 450 : pub(crate) fn resident_physical_size_sub(&self, sz: u64) {
2240 450 : self.resident_physical_size_gauge.sub(sz);
2241 450 : crate::metrics::RESIDENT_PHYSICAL_SIZE_GLOBAL.sub(sz);
2242 450 : }
2243 :
2244 1498 : pub(crate) fn resident_physical_size_add(&self, sz: u64) {
2245 1498 : self.resident_physical_size_gauge.add(sz);
2246 1498 : crate::metrics::RESIDENT_PHYSICAL_SIZE_GLOBAL.add(sz);
2247 1498 : }
2248 :
2249 8 : pub(crate) fn resident_physical_size_get(&self) -> u64 {
2250 8 : self.resident_physical_size_gauge.get()
2251 8 : }
2252 :
2253 8 : pub(crate) fn shutdown(&self) {
2254 8 : let was_shutdown = self
2255 8 : .shutdown
2256 8 : .swap(true, std::sync::atomic::Ordering::Relaxed);
2257 8 :
2258 8 : if was_shutdown {
2259 : // this happens on tenant deletion because tenant first shuts down timelines, then
2260 : // invokes timeline deletion which first shuts down the timeline again.
2261 : // TODO: this can be removed once https://github.com/neondatabase/neon/issues/5080
2262 0 : return;
2263 8 : }
2264 8 :
2265 8 : let tenant_id = &self.tenant_id;
2266 8 : let timeline_id = &self.timeline_id;
2267 8 : let shard_id = &self.shard_id;
2268 8 : let _ = LAST_RECORD_LSN.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2269 8 : let _ = STANDBY_HORIZON.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2270 8 : {
2271 8 : RESIDENT_PHYSICAL_SIZE_GLOBAL.sub(self.resident_physical_size_get());
2272 8 : let _ = RESIDENT_PHYSICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2273 8 : }
2274 8 : let _ = CURRENT_LOGICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2275 8 : if let Some(metric) = Lazy::get(&DIRECTORY_ENTRIES_COUNT) {
2276 0 : let _ = metric.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2277 8 : }
2278 8 : let _ = EVICTIONS.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2279 8 : let _ = AUX_FILE_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2280 8 : let _ = VALID_LSN_LEASE_COUNT.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2281 8 :
2282 8 : self.evictions_with_low_residence_duration
2283 8 : .write()
2284 8 : .unwrap()
2285 8 : .remove(tenant_id, shard_id, timeline_id);
2286 :
2287 : // The following metrics are born outside of the TimelineMetrics lifecycle but still
2288 : // removed at the end of it. The idea is to have the metrics outlive the
2289 : // entity during which they're observed, e.g., the smgr metrics shall
2290 : // outlive an individual smgr connection, but not the timeline.
2291 :
2292 72 : for op in StorageTimeOperation::VARIANTS {
2293 64 : let _ = STORAGE_TIME_SUM_PER_TIMELINE.remove_label_values(&[
2294 64 : op,
2295 64 : tenant_id,
2296 64 : shard_id,
2297 64 : timeline_id,
2298 64 : ]);
2299 64 : let _ = STORAGE_TIME_COUNT_PER_TIMELINE.remove_label_values(&[
2300 64 : op,
2301 64 : tenant_id,
2302 64 : shard_id,
2303 64 : timeline_id,
2304 64 : ]);
2305 64 : }
2306 :
2307 24 : for op in STORAGE_IO_SIZE_OPERATIONS {
2308 16 : let _ = STORAGE_IO_SIZE.remove_label_values(&[op, tenant_id, shard_id, timeline_id]);
2309 16 : }
2310 :
2311 48 : for op in SmgrQueryType::iter() {
2312 40 : let _ = SMGR_QUERY_TIME_PER_TENANT_TIMELINE.remove_label_values(&[
2313 40 : op.into(),
2314 40 : tenant_id,
2315 40 : shard_id,
2316 40 : timeline_id,
2317 40 : ]);
2318 40 : }
2319 8 : }
2320 : }
2321 :
2322 6 : pub(crate) fn remove_tenant_metrics(tenant_shard_id: &TenantShardId) {
2323 6 : // Only shard zero deals in synthetic sizes
2324 6 : if tenant_shard_id.is_shard_zero() {
2325 6 : let tid = tenant_shard_id.tenant_id.to_string();
2326 6 : let _ = TENANT_SYNTHETIC_SIZE_METRIC.remove_label_values(&[&tid]);
2327 6 : }
2328 :
2329 : // we leave the BROKEN_TENANTS_SET entry if any
2330 6 : }
2331 :
2332 : use futures::Future;
2333 : use pin_project_lite::pin_project;
2334 : use std::collections::HashMap;
2335 : use std::num::NonZeroUsize;
2336 : use std::pin::Pin;
2337 : use std::sync::atomic::AtomicU64;
2338 : use std::sync::{Arc, Mutex};
2339 : use std::task::{Context, Poll};
2340 : use std::time::{Duration, Instant};
2341 :
2342 : use crate::context::{PageContentKind, RequestContext};
2343 : use crate::task_mgr::TaskKind;
2344 : use crate::tenant::mgr::TenantSlot;
2345 :
2346 : /// Maintain a per timeline gauge in addition to the global gauge.
2347 : pub(crate) struct PerTimelineRemotePhysicalSizeGauge {
2348 : last_set: AtomicU64,
2349 : gauge: UIntGauge,
2350 : }
2351 :
2352 : impl PerTimelineRemotePhysicalSizeGauge {
2353 393 : fn new(per_timeline_gauge: UIntGauge) -> Self {
2354 393 : Self {
2355 393 : last_set: AtomicU64::new(0),
2356 393 : gauge: per_timeline_gauge,
2357 393 : }
2358 393 : }
2359 1742 : pub(crate) fn set(&self, sz: u64) {
2360 1742 : self.gauge.set(sz);
2361 1742 : let prev = self.last_set.swap(sz, std::sync::atomic::Ordering::Relaxed);
2362 1742 : if sz < prev {
2363 26 : REMOTE_PHYSICAL_SIZE_GLOBAL.sub(prev - sz);
2364 1716 : } else {
2365 1716 : REMOTE_PHYSICAL_SIZE_GLOBAL.add(sz - prev);
2366 1716 : };
2367 1742 : }
2368 2 : pub(crate) fn get(&self) -> u64 {
2369 2 : self.gauge.get()
2370 2 : }
2371 : }
2372 :
2373 : impl Drop for PerTimelineRemotePhysicalSizeGauge {
2374 19 : fn drop(&mut self) {
2375 19 : REMOTE_PHYSICAL_SIZE_GLOBAL.sub(self.last_set.load(std::sync::atomic::Ordering::Relaxed));
2376 19 : }
2377 : }
2378 :
2379 : pub(crate) struct RemoteTimelineClientMetrics {
2380 : tenant_id: String,
2381 : shard_id: String,
2382 : timeline_id: String,
2383 : pub(crate) remote_physical_size_gauge: PerTimelineRemotePhysicalSizeGauge,
2384 : calls: Mutex<HashMap<(&'static str, &'static str), IntCounterPair>>,
2385 : bytes_started_counter: Mutex<HashMap<(&'static str, &'static str), IntCounter>>,
2386 : bytes_finished_counter: Mutex<HashMap<(&'static str, &'static str), IntCounter>>,
2387 : }
2388 :
2389 : impl RemoteTimelineClientMetrics {
2390 393 : pub fn new(tenant_shard_id: &TenantShardId, timeline_id: &TimelineId) -> Self {
2391 393 : let tenant_id_str = tenant_shard_id.tenant_id.to_string();
2392 393 : let shard_id_str = format!("{}", tenant_shard_id.shard_slug());
2393 393 : let timeline_id_str = timeline_id.to_string();
2394 393 :
2395 393 : let remote_physical_size_gauge = PerTimelineRemotePhysicalSizeGauge::new(
2396 393 : REMOTE_PHYSICAL_SIZE
2397 393 : .get_metric_with_label_values(&[&tenant_id_str, &shard_id_str, &timeline_id_str])
2398 393 : .unwrap(),
2399 393 : );
2400 393 :
2401 393 : RemoteTimelineClientMetrics {
2402 393 : tenant_id: tenant_id_str,
2403 393 : shard_id: shard_id_str,
2404 393 : timeline_id: timeline_id_str,
2405 393 : calls: Mutex::new(HashMap::default()),
2406 393 : bytes_started_counter: Mutex::new(HashMap::default()),
2407 393 : bytes_finished_counter: Mutex::new(HashMap::default()),
2408 393 : remote_physical_size_gauge,
2409 393 : }
2410 393 : }
2411 :
2412 2699 : pub fn remote_operation_time(
2413 2699 : &self,
2414 2699 : file_kind: &RemoteOpFileKind,
2415 2699 : op_kind: &RemoteOpKind,
2416 2699 : status: &'static str,
2417 2699 : ) -> Histogram {
2418 2699 : let key = (file_kind.as_str(), op_kind.as_str(), status);
2419 2699 : REMOTE_OPERATION_TIME
2420 2699 : .get_metric_with_label_values(&[key.0, key.1, key.2])
2421 2699 : .unwrap()
2422 2699 : }
2423 :
2424 6310 : fn calls_counter_pair(
2425 6310 : &self,
2426 6310 : file_kind: &RemoteOpFileKind,
2427 6310 : op_kind: &RemoteOpKind,
2428 6310 : ) -> IntCounterPair {
2429 6310 : let mut guard = self.calls.lock().unwrap();
2430 6310 : let key = (file_kind.as_str(), op_kind.as_str());
2431 6310 : let metric = guard.entry(key).or_insert_with(move || {
2432 678 : REMOTE_TIMELINE_CLIENT_CALLS
2433 678 : .get_metric_with_label_values(&[
2434 678 : &self.tenant_id,
2435 678 : &self.shard_id,
2436 678 : &self.timeline_id,
2437 678 : key.0,
2438 678 : key.1,
2439 678 : ])
2440 678 : .unwrap()
2441 6310 : });
2442 6310 : metric.clone()
2443 6310 : }
2444 :
2445 1476 : fn bytes_started_counter(
2446 1476 : &self,
2447 1476 : file_kind: &RemoteOpFileKind,
2448 1476 : op_kind: &RemoteOpKind,
2449 1476 : ) -> IntCounter {
2450 1476 : let mut guard = self.bytes_started_counter.lock().unwrap();
2451 1476 : let key = (file_kind.as_str(), op_kind.as_str());
2452 1476 : let metric = guard.entry(key).or_insert_with(move || {
2453 256 : REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER
2454 256 : .get_metric_with_label_values(&[
2455 256 : &self.tenant_id,
2456 256 : &self.shard_id,
2457 256 : &self.timeline_id,
2458 256 : key.0,
2459 256 : key.1,
2460 256 : ])
2461 256 : .unwrap()
2462 1476 : });
2463 1476 : metric.clone()
2464 1476 : }
2465 :
2466 2790 : fn bytes_finished_counter(
2467 2790 : &self,
2468 2790 : file_kind: &RemoteOpFileKind,
2469 2790 : op_kind: &RemoteOpKind,
2470 2790 : ) -> IntCounter {
2471 2790 : let mut guard = self.bytes_finished_counter.lock().unwrap();
2472 2790 : let key = (file_kind.as_str(), op_kind.as_str());
2473 2790 : let metric = guard.entry(key).or_insert_with(move || {
2474 256 : REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER
2475 256 : .get_metric_with_label_values(&[
2476 256 : &self.tenant_id,
2477 256 : &self.shard_id,
2478 256 : &self.timeline_id,
2479 256 : key.0,
2480 256 : key.1,
2481 256 : ])
2482 256 : .unwrap()
2483 2790 : });
2484 2790 : metric.clone()
2485 2790 : }
2486 : }
2487 :
2488 : #[cfg(test)]
2489 : impl RemoteTimelineClientMetrics {
2490 6 : pub fn get_bytes_started_counter_value(
2491 6 : &self,
2492 6 : file_kind: &RemoteOpFileKind,
2493 6 : op_kind: &RemoteOpKind,
2494 6 : ) -> Option<u64> {
2495 6 : let guard = self.bytes_started_counter.lock().unwrap();
2496 6 : let key = (file_kind.as_str(), op_kind.as_str());
2497 6 : guard.get(&key).map(|counter| counter.get())
2498 6 : }
2499 :
2500 6 : pub fn get_bytes_finished_counter_value(
2501 6 : &self,
2502 6 : file_kind: &RemoteOpFileKind,
2503 6 : op_kind: &RemoteOpKind,
2504 6 : ) -> Option<u64> {
2505 6 : let guard = self.bytes_finished_counter.lock().unwrap();
2506 6 : let key = (file_kind.as_str(), op_kind.as_str());
2507 6 : guard.get(&key).map(|counter| counter.get())
2508 6 : }
2509 : }
2510 :
2511 : /// See [`RemoteTimelineClientMetrics::call_begin`].
2512 : #[must_use]
2513 : pub(crate) struct RemoteTimelineClientCallMetricGuard {
2514 : /// Decremented on drop.
2515 : calls_counter_pair: Option<IntCounterPair>,
2516 : /// If Some(), this references the bytes_finished metric, and we increment it by the given `u64` on drop.
2517 : bytes_finished: Option<(IntCounter, u64)>,
2518 : }
2519 :
2520 : impl RemoteTimelineClientCallMetricGuard {
2521 : /// Consume this guard object without performing the metric updates it would do on `drop()`.
2522 : /// The caller vouches to do the metric updates manually.
2523 3326 : pub fn will_decrement_manually(mut self) {
2524 3326 : let RemoteTimelineClientCallMetricGuard {
2525 3326 : calls_counter_pair,
2526 3326 : bytes_finished,
2527 3326 : } = &mut self;
2528 3326 : calls_counter_pair.take();
2529 3326 : bytes_finished.take();
2530 3326 : }
2531 : }
2532 :
2533 : impl Drop for RemoteTimelineClientCallMetricGuard {
2534 3352 : fn drop(&mut self) {
2535 3352 : let RemoteTimelineClientCallMetricGuard {
2536 3352 : calls_counter_pair,
2537 3352 : bytes_finished,
2538 3352 : } = self;
2539 3352 : if let Some(guard) = calls_counter_pair.take() {
2540 26 : guard.dec();
2541 3326 : }
2542 3352 : if let Some((bytes_finished_metric, value)) = bytes_finished {
2543 0 : bytes_finished_metric.inc_by(*value);
2544 3352 : }
2545 3352 : }
2546 : }
2547 :
2548 : /// The enum variants communicate to the [`RemoteTimelineClientMetrics`] whether to
2549 : /// track the byte size of this call in applicable metric(s).
2550 : pub(crate) enum RemoteTimelineClientMetricsCallTrackSize {
2551 : /// Do not account for this call's byte size in any metrics.
2552 : /// The `reason` field is there to make the call sites self-documenting
2553 : /// about why they don't need the metric.
2554 : DontTrackSize { reason: &'static str },
2555 : /// Track the byte size of the call in applicable metric(s).
2556 : Bytes(u64),
2557 : }
2558 :
2559 : impl RemoteTimelineClientMetrics {
2560 : /// Update the metrics that change when a call to the remote timeline client instance starts.
2561 : ///
2562 : /// Drop the returned guard object once the operation is finished to updates corresponding metrics that track completions.
2563 : /// Or, use [`RemoteTimelineClientCallMetricGuard::will_decrement_manually`] and [`call_end`](Self::call_end) if that
2564 : /// is more suitable.
2565 : /// Never do both.
2566 3352 : pub(crate) fn call_begin(
2567 3352 : &self,
2568 3352 : file_kind: &RemoteOpFileKind,
2569 3352 : op_kind: &RemoteOpKind,
2570 3352 : size: RemoteTimelineClientMetricsCallTrackSize,
2571 3352 : ) -> RemoteTimelineClientCallMetricGuard {
2572 3352 : let calls_counter_pair = self.calls_counter_pair(file_kind, op_kind);
2573 3352 : calls_counter_pair.inc();
2574 :
2575 3352 : let bytes_finished = match size {
2576 1876 : RemoteTimelineClientMetricsCallTrackSize::DontTrackSize { reason: _reason } => {
2577 1876 : // nothing to do
2578 1876 : None
2579 : }
2580 1476 : RemoteTimelineClientMetricsCallTrackSize::Bytes(size) => {
2581 1476 : self.bytes_started_counter(file_kind, op_kind).inc_by(size);
2582 1476 : let finished_counter = self.bytes_finished_counter(file_kind, op_kind);
2583 1476 : Some((finished_counter, size))
2584 : }
2585 : };
2586 3352 : RemoteTimelineClientCallMetricGuard {
2587 3352 : calls_counter_pair: Some(calls_counter_pair),
2588 3352 : bytes_finished,
2589 3352 : }
2590 3352 : }
2591 :
2592 : /// Manually udpate the metrics that track completions, instead of using the guard object.
2593 : /// Using the guard object is generally preferable.
2594 : /// See [`call_begin`](Self::call_begin) for more context.
2595 2958 : pub(crate) fn call_end(
2596 2958 : &self,
2597 2958 : file_kind: &RemoteOpFileKind,
2598 2958 : op_kind: &RemoteOpKind,
2599 2958 : size: RemoteTimelineClientMetricsCallTrackSize,
2600 2958 : ) {
2601 2958 : let calls_counter_pair = self.calls_counter_pair(file_kind, op_kind);
2602 2958 : calls_counter_pair.dec();
2603 2958 : match size {
2604 1644 : RemoteTimelineClientMetricsCallTrackSize::DontTrackSize { reason: _reason } => {}
2605 1314 : RemoteTimelineClientMetricsCallTrackSize::Bytes(size) => {
2606 1314 : self.bytes_finished_counter(file_kind, op_kind).inc_by(size);
2607 1314 : }
2608 : }
2609 2958 : }
2610 : }
2611 :
2612 : impl Drop for RemoteTimelineClientMetrics {
2613 19 : fn drop(&mut self) {
2614 19 : let RemoteTimelineClientMetrics {
2615 19 : tenant_id,
2616 19 : shard_id,
2617 19 : timeline_id,
2618 19 : remote_physical_size_gauge,
2619 19 : calls,
2620 19 : bytes_started_counter,
2621 19 : bytes_finished_counter,
2622 19 : } = self;
2623 24 : for ((a, b), _) in calls.get_mut().unwrap().drain() {
2624 24 : let mut res = [Ok(()), Ok(())];
2625 24 : REMOTE_TIMELINE_CLIENT_CALLS
2626 24 : .remove_label_values(&mut res, &[tenant_id, shard_id, timeline_id, a, b]);
2627 24 : // don't care about results
2628 24 : }
2629 19 : for ((a, b), _) in bytes_started_counter.get_mut().unwrap().drain() {
2630 6 : let _ = REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER.remove_label_values(&[
2631 6 : tenant_id,
2632 6 : shard_id,
2633 6 : timeline_id,
2634 6 : a,
2635 6 : b,
2636 6 : ]);
2637 6 : }
2638 19 : for ((a, b), _) in bytes_finished_counter.get_mut().unwrap().drain() {
2639 6 : let _ = REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER.remove_label_values(&[
2640 6 : tenant_id,
2641 6 : shard_id,
2642 6 : timeline_id,
2643 6 : a,
2644 6 : b,
2645 6 : ]);
2646 6 : }
2647 19 : {
2648 19 : let _ = remote_physical_size_gauge; // use to avoid 'unused' warning in desctructuring above
2649 19 : let _ = REMOTE_PHYSICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2650 19 : }
2651 19 : }
2652 : }
2653 :
2654 : /// Wrapper future that measures the time spent by a remote storage operation,
2655 : /// and records the time and success/failure as a prometheus metric.
2656 : pub(crate) trait MeasureRemoteOp: Sized {
2657 2728 : fn measure_remote_op(
2658 2728 : self,
2659 2728 : file_kind: RemoteOpFileKind,
2660 2728 : op: RemoteOpKind,
2661 2728 : metrics: Arc<RemoteTimelineClientMetrics>,
2662 2728 : ) -> MeasuredRemoteOp<Self> {
2663 2728 : let start = Instant::now();
2664 2728 : MeasuredRemoteOp {
2665 2728 : inner: self,
2666 2728 : file_kind,
2667 2728 : op,
2668 2728 : start,
2669 2728 : metrics,
2670 2728 : }
2671 2728 : }
2672 : }
2673 :
2674 : impl<T: Sized> MeasureRemoteOp for T {}
2675 :
2676 : pin_project! {
2677 : pub(crate) struct MeasuredRemoteOp<F>
2678 : {
2679 : #[pin]
2680 : inner: F,
2681 : file_kind: RemoteOpFileKind,
2682 : op: RemoteOpKind,
2683 : start: Instant,
2684 : metrics: Arc<RemoteTimelineClientMetrics>,
2685 : }
2686 : }
2687 :
2688 : impl<F: Future<Output = Result<O, E>>, O, E> Future for MeasuredRemoteOp<F> {
2689 : type Output = Result<O, E>;
2690 :
2691 38013 : fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
2692 38013 : let this = self.project();
2693 38013 : let poll_result = this.inner.poll(cx);
2694 38013 : if let Poll::Ready(ref res) = poll_result {
2695 2699 : let duration = this.start.elapsed();
2696 2699 : let status = if res.is_ok() { &"success" } else { &"failure" };
2697 2699 : this.metrics
2698 2699 : .remote_operation_time(this.file_kind, this.op, status)
2699 2699 : .observe(duration.as_secs_f64());
2700 35314 : }
2701 38013 : poll_result
2702 38013 : }
2703 : }
2704 :
2705 : pub mod tokio_epoll_uring {
2706 : use metrics::{register_int_counter, UIntGauge};
2707 : use once_cell::sync::Lazy;
2708 :
2709 : pub struct Collector {
2710 : descs: Vec<metrics::core::Desc>,
2711 : systems_created: UIntGauge,
2712 : systems_destroyed: UIntGauge,
2713 : }
2714 :
2715 : impl metrics::core::Collector for Collector {
2716 0 : fn desc(&self) -> Vec<&metrics::core::Desc> {
2717 0 : self.descs.iter().collect()
2718 0 : }
2719 :
2720 0 : fn collect(&self) -> Vec<metrics::proto::MetricFamily> {
2721 0 : let mut mfs = Vec::with_capacity(Self::NMETRICS);
2722 0 : let tokio_epoll_uring::metrics::Metrics {
2723 0 : systems_created,
2724 0 : systems_destroyed,
2725 0 : } = tokio_epoll_uring::metrics::global();
2726 0 : self.systems_created.set(systems_created);
2727 0 : mfs.extend(self.systems_created.collect());
2728 0 : self.systems_destroyed.set(systems_destroyed);
2729 0 : mfs.extend(self.systems_destroyed.collect());
2730 0 : mfs
2731 0 : }
2732 : }
2733 :
2734 : impl Collector {
2735 : const NMETRICS: usize = 2;
2736 :
2737 : #[allow(clippy::new_without_default)]
2738 0 : pub fn new() -> Self {
2739 0 : let mut descs = Vec::new();
2740 0 :
2741 0 : let systems_created = UIntGauge::new(
2742 0 : "pageserver_tokio_epoll_uring_systems_created",
2743 0 : "counter of tokio-epoll-uring systems that were created",
2744 0 : )
2745 0 : .unwrap();
2746 0 : descs.extend(
2747 0 : metrics::core::Collector::desc(&systems_created)
2748 0 : .into_iter()
2749 0 : .cloned(),
2750 0 : );
2751 0 :
2752 0 : let systems_destroyed = UIntGauge::new(
2753 0 : "pageserver_tokio_epoll_uring_systems_destroyed",
2754 0 : "counter of tokio-epoll-uring systems that were destroyed",
2755 0 : )
2756 0 : .unwrap();
2757 0 : descs.extend(
2758 0 : metrics::core::Collector::desc(&systems_destroyed)
2759 0 : .into_iter()
2760 0 : .cloned(),
2761 0 : );
2762 0 :
2763 0 : Self {
2764 0 : descs,
2765 0 : systems_created,
2766 0 : systems_destroyed,
2767 0 : }
2768 0 : }
2769 : }
2770 :
2771 81 : pub(crate) static THREAD_LOCAL_LAUNCH_SUCCESSES: Lazy<metrics::IntCounter> = Lazy::new(|| {
2772 : register_int_counter!(
2773 : "pageserver_tokio_epoll_uring_pageserver_thread_local_launch_success_count",
2774 : "Number of times where thread_local_system creation spanned multiple executor threads",
2775 : )
2776 81 : .unwrap()
2777 81 : });
2778 :
2779 0 : pub(crate) static THREAD_LOCAL_LAUNCH_FAILURES: Lazy<metrics::IntCounter> = Lazy::new(|| {
2780 : register_int_counter!(
2781 : "pageserver_tokio_epoll_uring_pageserver_thread_local_launch_failures_count",
2782 : "Number of times thread_local_system creation failed and was retried after back-off.",
2783 : )
2784 0 : .unwrap()
2785 0 : });
2786 : }
2787 :
2788 : pub(crate) mod tenant_throttling {
2789 : use metrics::{register_int_counter_vec, IntCounter};
2790 : use once_cell::sync::Lazy;
2791 :
2792 : use crate::tenant::{self, throttle::Metric};
2793 :
2794 : pub(crate) struct TimelineGet {
2795 : wait_time: IntCounter,
2796 : count: IntCounter,
2797 : }
2798 :
2799 143 : pub(crate) static TIMELINE_GET: Lazy<TimelineGet> = Lazy::new(|| {
2800 143 : static WAIT_USECS: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
2801 143 : register_int_counter_vec!(
2802 143 : "pageserver_tenant_throttling_wait_usecs_sum_global",
2803 143 : "Sum of microseconds that tenants spent waiting for a tenant throttle of a given kind.",
2804 143 : &["kind"]
2805 143 : )
2806 143 : .unwrap()
2807 143 : });
2808 143 :
2809 143 : static WAIT_COUNT: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
2810 143 : register_int_counter_vec!(
2811 143 : "pageserver_tenant_throttling_count_global",
2812 143 : "Count of tenant throttlings, by kind of throttle.",
2813 143 : &["kind"]
2814 143 : )
2815 143 : .unwrap()
2816 143 : });
2817 143 :
2818 143 : let kind = "timeline_get";
2819 143 : TimelineGet {
2820 143 : wait_time: WAIT_USECS.with_label_values(&[kind]),
2821 143 : count: WAIT_COUNT.with_label_values(&[kind]),
2822 143 : }
2823 143 : });
2824 :
2825 : impl Metric for &'static TimelineGet {
2826 : #[inline(always)]
2827 0 : fn observe_throttling(
2828 0 : &self,
2829 0 : tenant::throttle::Observation { wait_time }: &tenant::throttle::Observation,
2830 0 : ) {
2831 0 : let val = u64::try_from(wait_time.as_micros()).unwrap();
2832 0 : self.wait_time.inc_by(val);
2833 0 : self.count.inc();
2834 0 : }
2835 : }
2836 : }
2837 :
2838 : pub(crate) mod disk_usage_based_eviction {
2839 : use super::*;
2840 :
2841 : pub(crate) struct Metrics {
2842 : pub(crate) tenant_collection_time: Histogram,
2843 : pub(crate) tenant_layer_count: Histogram,
2844 : pub(crate) layers_collected: IntCounter,
2845 : pub(crate) layers_selected: IntCounter,
2846 : pub(crate) layers_evicted: IntCounter,
2847 : }
2848 :
2849 : impl Default for Metrics {
2850 0 : fn default() -> Self {
2851 0 : let tenant_collection_time = register_histogram!(
2852 : "pageserver_disk_usage_based_eviction_tenant_collection_seconds",
2853 : "Time spent collecting layers from a tenant -- not normalized by collected layer amount",
2854 : vec![0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0, 10.0]
2855 : )
2856 0 : .unwrap();
2857 0 :
2858 0 : let tenant_layer_count = register_histogram!(
2859 : "pageserver_disk_usage_based_eviction_tenant_collected_layers",
2860 : "Amount of layers gathered from a tenant",
2861 : vec![5.0, 50.0, 500.0, 5000.0, 50000.0]
2862 : )
2863 0 : .unwrap();
2864 0 :
2865 0 : let layers_collected = register_int_counter!(
2866 : "pageserver_disk_usage_based_eviction_collected_layers_total",
2867 : "Amount of layers collected"
2868 : )
2869 0 : .unwrap();
2870 0 :
2871 0 : let layers_selected = register_int_counter!(
2872 : "pageserver_disk_usage_based_eviction_select_layers_total",
2873 : "Amount of layers selected"
2874 : )
2875 0 : .unwrap();
2876 0 :
2877 0 : let layers_evicted = register_int_counter!(
2878 : "pageserver_disk_usage_based_eviction_evicted_layers_total",
2879 : "Amount of layers successfully evicted"
2880 : )
2881 0 : .unwrap();
2882 0 :
2883 0 : Self {
2884 0 : tenant_collection_time,
2885 0 : tenant_layer_count,
2886 0 : layers_collected,
2887 0 : layers_selected,
2888 0 : layers_evicted,
2889 0 : }
2890 0 : }
2891 : }
2892 :
2893 : pub(crate) static METRICS: Lazy<Metrics> = Lazy::new(Metrics::default);
2894 : }
2895 :
2896 136 : static TOKIO_EXECUTOR_THREAD_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
2897 : register_uint_gauge_vec!(
2898 : "pageserver_tokio_executor_thread_configured_count",
2899 : "Total number of configued tokio executor threads in the process.
2900 : The `setup` label denotes whether we're running with multiple runtimes or a single runtime.",
2901 : &["setup"],
2902 : )
2903 136 : .unwrap()
2904 136 : });
2905 :
2906 136 : pub(crate) fn set_tokio_runtime_setup(setup: &str, num_threads: NonZeroUsize) {
2907 136 : static SERIALIZE: std::sync::Mutex<()> = std::sync::Mutex::new(());
2908 136 : let _guard = SERIALIZE.lock().unwrap();
2909 136 : TOKIO_EXECUTOR_THREAD_COUNT.reset();
2910 136 : TOKIO_EXECUTOR_THREAD_COUNT
2911 136 : .get_metric_with_label_values(&[setup])
2912 136 : .unwrap()
2913 136 : .set(u64::try_from(num_threads.get()).unwrap());
2914 136 : }
2915 :
2916 0 : pub fn preinitialize_metrics() {
2917 0 : // Python tests need these and on some we do alerting.
2918 0 : //
2919 0 : // FIXME(4813): make it so that we have no top level metrics as this fn will easily fall out of
2920 0 : // order:
2921 0 : // - global metrics reside in a Lazy<PageserverMetrics>
2922 0 : // - access via crate::metrics::PS_METRICS.some_metric.inc()
2923 0 : // - could move the statics into TimelineMetrics::new()?
2924 0 :
2925 0 : // counters
2926 0 : [
2927 0 : &UNEXPECTED_ONDEMAND_DOWNLOADS,
2928 0 : &WALRECEIVER_STARTED_CONNECTIONS,
2929 0 : &WALRECEIVER_BROKER_UPDATES,
2930 0 : &WALRECEIVER_CANDIDATES_ADDED,
2931 0 : &WALRECEIVER_CANDIDATES_REMOVED,
2932 0 : &tokio_epoll_uring::THREAD_LOCAL_LAUNCH_FAILURES,
2933 0 : &tokio_epoll_uring::THREAD_LOCAL_LAUNCH_SUCCESSES,
2934 0 : &REMOTE_ONDEMAND_DOWNLOADED_LAYERS,
2935 0 : &REMOTE_ONDEMAND_DOWNLOADED_BYTES,
2936 0 : ]
2937 0 : .into_iter()
2938 0 : .for_each(|c| {
2939 0 : Lazy::force(c);
2940 0 : });
2941 0 :
2942 0 : // Deletion queue stats
2943 0 : Lazy::force(&DELETION_QUEUE);
2944 0 :
2945 0 : // Tenant stats
2946 0 : Lazy::force(&TENANT);
2947 0 :
2948 0 : // Tenant manager stats
2949 0 : Lazy::force(&TENANT_MANAGER);
2950 0 :
2951 0 : Lazy::force(&crate::tenant::storage_layer::layer::LAYER_IMPL_METRICS);
2952 0 : Lazy::force(&disk_usage_based_eviction::METRICS);
2953 :
2954 0 : for state_name in pageserver_api::models::TenantState::VARIANTS {
2955 0 : // initialize the metric for all gauges, otherwise the time series might seemingly show
2956 0 : // values from last restart.
2957 0 : TENANT_STATE_METRIC.with_label_values(&[state_name]).set(0);
2958 0 : }
2959 :
2960 : // countervecs
2961 0 : [&BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT]
2962 0 : .into_iter()
2963 0 : .for_each(|c| {
2964 0 : Lazy::force(c);
2965 0 : });
2966 0 :
2967 0 : // gauges
2968 0 : WALRECEIVER_ACTIVE_MANAGERS.get();
2969 0 :
2970 0 : // histograms
2971 0 : [
2972 0 : &READ_NUM_LAYERS_VISITED,
2973 0 : &VEC_READ_NUM_LAYERS_VISITED,
2974 0 : &WAIT_LSN_TIME,
2975 0 : &WAL_REDO_TIME,
2976 0 : &WAL_REDO_RECORDS_HISTOGRAM,
2977 0 : &WAL_REDO_BYTES_HISTOGRAM,
2978 0 : &WAL_REDO_PROCESS_LAUNCH_DURATION_HISTOGRAM,
2979 0 : ]
2980 0 : .into_iter()
2981 0 : .for_each(|h| {
2982 0 : Lazy::force(h);
2983 0 : });
2984 0 :
2985 0 : // Custom
2986 0 : Lazy::force(&RECONSTRUCT_TIME);
2987 0 : Lazy::force(&tenant_throttling::TIMELINE_GET);
2988 0 : Lazy::force(&BASEBACKUP_QUERY_TIME);
2989 0 : Lazy::force(&COMPUTE_COMMANDS_COUNTERS);
2990 0 : }
|
