Line data Source code
1 : use enum_map::EnumMap;
2 : use metrics::{
3 : register_counter_vec, register_gauge_vec, register_histogram, register_histogram_vec,
4 : register_int_counter, register_int_counter_pair_vec, register_int_counter_vec,
5 : register_int_gauge, register_int_gauge_vec, register_uint_gauge, register_uint_gauge_vec,
6 : Counter, CounterVec, GaugeVec, Histogram, HistogramVec, IntCounter, IntCounterPair,
7 : IntCounterPairVec, IntCounterVec, IntGauge, IntGaugeVec, UIntGauge, UIntGaugeVec,
8 : };
9 : use once_cell::sync::Lazy;
10 : use pageserver_api::shard::TenantShardId;
11 : use strum::{EnumCount, IntoEnumIterator, VariantNames};
12 : use strum_macros::{EnumVariantNames, IntoStaticStr};
13 : use tracing::warn;
14 : use utils::id::TimelineId;
15 :
16 : /// Prometheus histogram buckets (in seconds) for operations in the critical
17 : /// path. In other words, operations that directly affect that latency of user
18 : /// queries.
19 : ///
20 : /// The buckets capture the majority of latencies in the microsecond and
21 : /// millisecond range but also extend far enough up to distinguish "bad" from
22 : /// "really bad".
23 : const CRITICAL_OP_BUCKETS: &[f64] = &[
24 : 0.000_001, 0.000_010, 0.000_100, // 1 us, 10 us, 100 us
25 : 0.001_000, 0.010_000, 0.100_000, // 1 ms, 10 ms, 100 ms
26 : 1.0, 10.0, 100.0, // 1 s, 10 s, 100 s
27 : ];
28 :
29 : // Metrics collected on operations on the storage repository.
30 3064 : #[derive(Debug, EnumVariantNames, IntoStaticStr)]
31 : #[strum(serialize_all = "kebab_case")]
32 : pub(crate) enum StorageTimeOperation {
33 : #[strum(serialize = "layer flush")]
34 : LayerFlush,
35 :
36 : #[strum(serialize = "compact")]
37 : Compact,
38 :
39 : #[strum(serialize = "create images")]
40 : CreateImages,
41 :
42 : #[strum(serialize = "logical size")]
43 : LogicalSize,
44 :
45 : #[strum(serialize = "imitate logical size")]
46 : ImitateLogicalSize,
47 :
48 : #[strum(serialize = "load layer map")]
49 : LoadLayerMap,
50 :
51 : #[strum(serialize = "gc")]
52 : Gc,
53 :
54 : #[strum(serialize = "find gc cutoffs")]
55 : FindGcCutoffs,
56 :
57 : #[strum(serialize = "create tenant")]
58 : CreateTenant,
59 : }
60 :
61 141 : pub(crate) static STORAGE_TIME_SUM_PER_TIMELINE: Lazy<CounterVec> = Lazy::new(|| {
62 : register_counter_vec!(
63 : "pageserver_storage_operations_seconds_sum",
64 : "Total time spent on storage operations with operation, tenant and timeline dimensions",
65 : &["operation", "tenant_id", "shard_id", "timeline_id"],
66 : )
67 141 : .expect("failed to define a metric")
68 141 : });
69 :
70 141 : pub(crate) static STORAGE_TIME_COUNT_PER_TIMELINE: Lazy<IntCounterVec> = Lazy::new(|| {
71 : register_int_counter_vec!(
72 : "pageserver_storage_operations_seconds_count",
73 : "Count of storage operations with operation, tenant and timeline dimensions",
74 : &["operation", "tenant_id", "shard_id", "timeline_id"],
75 : )
76 141 : .expect("failed to define a metric")
77 141 : });
78 :
79 : // Buckets for background operations like compaction, GC, size calculation
80 : const STORAGE_OP_BUCKETS: &[f64] = &[0.010, 0.100, 1.0, 10.0, 100.0, 1000.0];
81 :
82 141 : pub(crate) static STORAGE_TIME_GLOBAL: Lazy<HistogramVec> = Lazy::new(|| {
83 : register_histogram_vec!(
84 : "pageserver_storage_operations_seconds_global",
85 : "Time spent on storage operations",
86 : &["operation"],
87 : STORAGE_OP_BUCKETS.into(),
88 : )
89 141 : .expect("failed to define a metric")
90 141 : });
91 :
92 138 : pub(crate) static READ_NUM_LAYERS_VISITED: Lazy<Histogram> = Lazy::new(|| {
93 : register_histogram!(
94 : "pageserver_layers_visited_per_read_global",
95 : "Number of layers visited to reconstruct one key",
96 : vec![1.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0],
97 : )
98 138 : .expect("failed to define a metric")
99 138 : });
100 :
101 24 : pub(crate) static VEC_READ_NUM_LAYERS_VISITED: Lazy<Histogram> = Lazy::new(|| {
102 : register_histogram!(
103 : "pageserver_layers_visited_per_vectored_read_global",
104 : "Average number of layers visited to reconstruct one key",
105 : vec![1.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0],
106 : )
107 24 : .expect("failed to define a metric")
108 24 : });
109 :
110 : // Metrics collected on operations on the storage repository.
111 : #[derive(
112 552 : Clone, Copy, enum_map::Enum, strum_macros::EnumString, strum_macros::Display, IntoStaticStr,
113 : )]
114 : pub(crate) enum GetKind {
115 : Singular,
116 : Vectored,
117 : }
118 :
119 : pub(crate) struct ReconstructTimeMetrics {
120 : singular: Histogram,
121 : vectored: Histogram,
122 : }
123 :
124 138 : pub(crate) static RECONSTRUCT_TIME: Lazy<ReconstructTimeMetrics> = Lazy::new(|| {
125 138 : let inner = register_histogram_vec!(
126 : "pageserver_getpage_reconstruct_seconds",
127 : "Time spent in reconstruct_value (reconstruct a page from deltas)",
128 : &["get_kind"],
129 : CRITICAL_OP_BUCKETS.into(),
130 : )
131 138 : .expect("failed to define a metric");
132 138 :
133 138 : ReconstructTimeMetrics {
134 138 : singular: inner.with_label_values(&[GetKind::Singular.into()]),
135 138 : vectored: inner.with_label_values(&[GetKind::Vectored.into()]),
136 138 : }
137 138 : });
138 :
139 : impl ReconstructTimeMetrics {
140 625839 : pub(crate) fn for_get_kind(&self, get_kind: GetKind) -> &Histogram {
141 625839 : match get_kind {
142 625769 : GetKind::Singular => &self.singular,
143 70 : GetKind::Vectored => &self.vectored,
144 : }
145 625839 : }
146 : }
147 :
148 : pub(crate) struct ReconstructDataTimeMetrics {
149 : singular: Histogram,
150 : vectored: Histogram,
151 : }
152 :
153 : impl ReconstructDataTimeMetrics {
154 625979 : pub(crate) fn for_get_kind(&self, get_kind: GetKind) -> &Histogram {
155 625979 : match get_kind {
156 625909 : GetKind::Singular => &self.singular,
157 70 : GetKind::Vectored => &self.vectored,
158 : }
159 625979 : }
160 : }
161 :
162 138 : pub(crate) static GET_RECONSTRUCT_DATA_TIME: Lazy<ReconstructDataTimeMetrics> = Lazy::new(|| {
163 138 : let inner = register_histogram_vec!(
164 : "pageserver_getpage_get_reconstruct_data_seconds",
165 : "Time spent in get_reconstruct_value_data",
166 : &["get_kind"],
167 : CRITICAL_OP_BUCKETS.into(),
168 : )
169 138 : .expect("failed to define a metric");
170 138 :
171 138 : ReconstructDataTimeMetrics {
172 138 : singular: inner.with_label_values(&[GetKind::Singular.into()]),
173 138 : vectored: inner.with_label_values(&[GetKind::Vectored.into()]),
174 138 : }
175 138 : });
176 :
177 : pub(crate) struct GetVectoredLatency {
178 : map: EnumMap<TaskKind, Option<Histogram>>,
179 : }
180 :
181 : #[allow(dead_code)]
182 : pub(crate) struct ScanLatency {
183 : map: EnumMap<TaskKind, Option<Histogram>>,
184 : }
185 :
186 : impl GetVectoredLatency {
187 : // Only these task types perform vectored gets. Filter all other tasks out to reduce total
188 : // cardinality of the metric.
189 : const TRACKED_TASK_KINDS: [TaskKind; 2] = [TaskKind::Compaction, TaskKind::PageRequestHandler];
190 :
191 952 : pub(crate) fn for_task_kind(&self, task_kind: TaskKind) -> Option<&Histogram> {
192 952 : self.map[task_kind].as_ref()
193 952 : }
194 : }
195 :
196 : impl ScanLatency {
197 : // Only these task types perform vectored gets. Filter all other tasks out to reduce total
198 : // cardinality of the metric.
199 : const TRACKED_TASK_KINDS: [TaskKind; 1] = [TaskKind::PageRequestHandler];
200 :
201 12 : pub(crate) fn for_task_kind(&self, task_kind: TaskKind) -> Option<&Histogram> {
202 12 : self.map[task_kind].as_ref()
203 12 : }
204 : }
205 :
206 : pub(crate) struct ScanLatencyOngoingRecording<'a> {
207 : parent: &'a Histogram,
208 : start: std::time::Instant,
209 : }
210 :
211 : impl<'a> ScanLatencyOngoingRecording<'a> {
212 0 : pub(crate) fn start_recording(parent: &'a Histogram) -> ScanLatencyOngoingRecording<'a> {
213 0 : let start = Instant::now();
214 0 : ScanLatencyOngoingRecording { parent, start }
215 0 : }
216 :
217 0 : pub(crate) fn observe(self, throttled: Option<Duration>) {
218 0 : let elapsed = self.start.elapsed();
219 0 : let ex_throttled = if let Some(throttled) = throttled {
220 0 : elapsed.checked_sub(throttled)
221 : } else {
222 0 : Some(elapsed)
223 : };
224 0 : if let Some(ex_throttled) = ex_throttled {
225 0 : self.parent.observe(ex_throttled.as_secs_f64());
226 0 : } else {
227 0 : use utils::rate_limit::RateLimit;
228 0 : static LOGGED: Lazy<Mutex<RateLimit>> =
229 0 : Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(10))));
230 0 : let mut rate_limit = LOGGED.lock().unwrap();
231 0 : rate_limit.call(|| {
232 0 : warn!("error deducting time spent throttled; this message is logged at a global rate limit");
233 0 : });
234 0 : }
235 0 : }
236 : }
237 :
238 132 : pub(crate) static GET_VECTORED_LATENCY: Lazy<GetVectoredLatency> = Lazy::new(|| {
239 132 : let inner = register_histogram_vec!(
240 : "pageserver_get_vectored_seconds",
241 : "Time spent in get_vectored, excluding time spent in timeline_get_throttle.",
242 : &["task_kind"],
243 : CRITICAL_OP_BUCKETS.into(),
244 : )
245 132 : .expect("failed to define a metric");
246 132 :
247 132 : GetVectoredLatency {
248 3960 : map: EnumMap::from_array(std::array::from_fn(|task_kind_idx| {
249 3960 : let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind_idx);
250 3960 :
251 3960 : if GetVectoredLatency::TRACKED_TASK_KINDS.contains(&task_kind) {
252 264 : let task_kind = task_kind.into();
253 264 : Some(inner.with_label_values(&[task_kind]))
254 : } else {
255 3696 : None
256 : }
257 3960 : })),
258 132 : }
259 132 : });
260 :
261 4 : pub(crate) static SCAN_LATENCY: Lazy<ScanLatency> = Lazy::new(|| {
262 4 : let inner = register_histogram_vec!(
263 : "pageserver_scan_seconds",
264 : "Time spent in scan, excluding time spent in timeline_get_throttle.",
265 : &["task_kind"],
266 : CRITICAL_OP_BUCKETS.into(),
267 : )
268 4 : .expect("failed to define a metric");
269 4 :
270 4 : ScanLatency {
271 120 : map: EnumMap::from_array(std::array::from_fn(|task_kind_idx| {
272 120 : let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind_idx);
273 120 :
274 120 : if ScanLatency::TRACKED_TASK_KINDS.contains(&task_kind) {
275 4 : let task_kind = task_kind.into();
276 4 : Some(inner.with_label_values(&[task_kind]))
277 : } else {
278 116 : None
279 : }
280 120 : })),
281 4 : }
282 4 : });
283 :
284 : pub(crate) struct PageCacheMetricsForTaskKind {
285 : pub read_accesses_immutable: IntCounter,
286 : pub read_hits_immutable: IntCounter,
287 : }
288 :
289 : pub(crate) struct PageCacheMetrics {
290 : map: EnumMap<TaskKind, EnumMap<PageContentKind, PageCacheMetricsForTaskKind>>,
291 : }
292 :
293 74 : static PAGE_CACHE_READ_HITS: Lazy<IntCounterVec> = Lazy::new(|| {
294 : register_int_counter_vec!(
295 : "pageserver_page_cache_read_hits_total",
296 : "Number of read accesses to the page cache that hit",
297 : &["task_kind", "key_kind", "content_kind", "hit_kind"]
298 : )
299 74 : .expect("failed to define a metric")
300 74 : });
301 :
302 74 : static PAGE_CACHE_READ_ACCESSES: Lazy<IntCounterVec> = Lazy::new(|| {
303 : register_int_counter_vec!(
304 : "pageserver_page_cache_read_accesses_total",
305 : "Number of read accesses to the page cache",
306 : &["task_kind", "key_kind", "content_kind"]
307 : )
308 74 : .expect("failed to define a metric")
309 74 : });
310 :
311 74 : pub(crate) static PAGE_CACHE: Lazy<PageCacheMetrics> = Lazy::new(|| PageCacheMetrics {
312 2220 : map: EnumMap::from_array(std::array::from_fn(|task_kind| {
313 2220 : let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind);
314 2220 : let task_kind: &'static str = task_kind.into();
315 13320 : EnumMap::from_array(std::array::from_fn(|content_kind| {
316 13320 : let content_kind = <PageContentKind as enum_map::Enum>::from_usize(content_kind);
317 13320 : let content_kind: &'static str = content_kind.into();
318 13320 : PageCacheMetricsForTaskKind {
319 13320 : read_accesses_immutable: {
320 13320 : PAGE_CACHE_READ_ACCESSES
321 13320 : .get_metric_with_label_values(&[task_kind, "immutable", content_kind])
322 13320 : .unwrap()
323 13320 : },
324 13320 :
325 13320 : read_hits_immutable: {
326 13320 : PAGE_CACHE_READ_HITS
327 13320 : .get_metric_with_label_values(&[task_kind, "immutable", content_kind, "-"])
328 13320 : .unwrap()
329 13320 : },
330 13320 : }
331 13320 : }))
332 2220 : })),
333 74 : });
334 :
335 : impl PageCacheMetrics {
336 14193692 : pub(crate) fn for_ctx(&self, ctx: &RequestContext) -> &PageCacheMetricsForTaskKind {
337 14193692 : &self.map[ctx.task_kind()][ctx.page_content_kind()]
338 14193692 : }
339 : }
340 :
341 : pub(crate) struct PageCacheSizeMetrics {
342 : pub max_bytes: UIntGauge,
343 :
344 : pub current_bytes_immutable: UIntGauge,
345 : }
346 :
347 74 : static PAGE_CACHE_SIZE_CURRENT_BYTES: Lazy<UIntGaugeVec> = Lazy::new(|| {
348 : register_uint_gauge_vec!(
349 : "pageserver_page_cache_size_current_bytes",
350 : "Current size of the page cache in bytes, by key kind",
351 : &["key_kind"]
352 : )
353 74 : .expect("failed to define a metric")
354 74 : });
355 :
356 : pub(crate) static PAGE_CACHE_SIZE: Lazy<PageCacheSizeMetrics> =
357 74 : Lazy::new(|| PageCacheSizeMetrics {
358 : max_bytes: {
359 : register_uint_gauge!(
360 : "pageserver_page_cache_size_max_bytes",
361 : "Maximum size of the page cache in bytes"
362 : )
363 74 : .expect("failed to define a metric")
364 74 : },
365 74 : current_bytes_immutable: {
366 74 : PAGE_CACHE_SIZE_CURRENT_BYTES
367 74 : .get_metric_with_label_values(&["immutable"])
368 74 : .unwrap()
369 74 : },
370 74 : });
371 :
372 : pub(crate) mod page_cache_eviction_metrics {
373 : use std::num::NonZeroUsize;
374 :
375 : use metrics::{register_int_counter_vec, IntCounter, IntCounterVec};
376 : use once_cell::sync::Lazy;
377 :
378 : #[derive(Clone, Copy)]
379 : pub(crate) enum Outcome {
380 : FoundSlotUnused { iters: NonZeroUsize },
381 : FoundSlotEvicted { iters: NonZeroUsize },
382 : ItersExceeded { iters: NonZeroUsize },
383 : }
384 :
385 74 : static ITERS_TOTAL_VEC: Lazy<IntCounterVec> = Lazy::new(|| {
386 : register_int_counter_vec!(
387 : "pageserver_page_cache_find_victim_iters_total",
388 : "Counter for the number of iterations in the find_victim loop",
389 : &["outcome"],
390 : )
391 74 : .expect("failed to define a metric")
392 74 : });
393 :
394 74 : static CALLS_VEC: Lazy<IntCounterVec> = Lazy::new(|| {
395 : register_int_counter_vec!(
396 : "pageserver_page_cache_find_victim_calls",
397 : "Incremented at the end of each find_victim() call.\
398 : Filter by outcome to get e.g., eviction rate.",
399 : &["outcome"]
400 : )
401 74 : .unwrap()
402 74 : });
403 :
404 166952 : pub(crate) fn observe(outcome: Outcome) {
405 166952 : macro_rules! dry {
406 166952 : ($label:literal, $iters:expr) => {{
407 166952 : static LABEL: &'static str = $label;
408 166952 : static ITERS_TOTAL: Lazy<IntCounter> =
409 166952 : Lazy::new(|| ITERS_TOTAL_VEC.with_label_values(&[LABEL]));
410 166952 : static CALLS: Lazy<IntCounter> =
411 166952 : Lazy::new(|| CALLS_VEC.with_label_values(&[LABEL]));
412 166952 : ITERS_TOTAL.inc_by(($iters.get()) as u64);
413 166952 : CALLS.inc();
414 166952 : }};
415 166952 : }
416 166952 : match outcome {
417 1892 : Outcome::FoundSlotUnused { iters } => dry!("found_empty", iters),
418 165060 : Outcome::FoundSlotEvicted { iters } => {
419 165060 : dry!("found_evicted", iters)
420 : }
421 0 : Outcome::ItersExceeded { iters } => {
422 0 : dry!("err_iters_exceeded", iters);
423 0 : super::page_cache_errors_inc(super::PageCacheErrorKind::EvictIterLimit);
424 0 : }
425 : }
426 166952 : }
427 : }
428 :
429 0 : static PAGE_CACHE_ERRORS: Lazy<IntCounterVec> = Lazy::new(|| {
430 : register_int_counter_vec!(
431 : "page_cache_errors_total",
432 : "Number of timeouts while acquiring a pinned slot in the page cache",
433 : &["error_kind"]
434 : )
435 0 : .expect("failed to define a metric")
436 0 : });
437 :
438 0 : #[derive(IntoStaticStr)]
439 : #[strum(serialize_all = "kebab_case")]
440 : pub(crate) enum PageCacheErrorKind {
441 : AcquirePinnedSlotTimeout,
442 : EvictIterLimit,
443 : }
444 :
445 0 : pub(crate) fn page_cache_errors_inc(error_kind: PageCacheErrorKind) {
446 0 : PAGE_CACHE_ERRORS
447 0 : .get_metric_with_label_values(&[error_kind.into()])
448 0 : .unwrap()
449 0 : .inc();
450 0 : }
451 :
452 16 : pub(crate) static WAIT_LSN_TIME: Lazy<Histogram> = Lazy::new(|| {
453 : register_histogram!(
454 : "pageserver_wait_lsn_seconds",
455 : "Time spent waiting for WAL to arrive",
456 : CRITICAL_OP_BUCKETS.into(),
457 : )
458 16 : .expect("failed to define a metric")
459 16 : });
460 :
461 141 : static LAST_RECORD_LSN: Lazy<IntGaugeVec> = Lazy::new(|| {
462 : register_int_gauge_vec!(
463 : "pageserver_last_record_lsn",
464 : "Last record LSN grouped by timeline",
465 : &["tenant_id", "shard_id", "timeline_id"]
466 : )
467 141 : .expect("failed to define a metric")
468 141 : });
469 :
470 141 : static STANDBY_HORIZON: Lazy<IntGaugeVec> = Lazy::new(|| {
471 : register_int_gauge_vec!(
472 : "pageserver_standby_horizon",
473 : "Standby apply LSN for which GC is hold off, by timeline.",
474 : &["tenant_id", "shard_id", "timeline_id"]
475 : )
476 141 : .expect("failed to define a metric")
477 141 : });
478 :
479 141 : static RESIDENT_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
480 : register_uint_gauge_vec!(
481 : "pageserver_resident_physical_size",
482 : "The size of the layer files present in the pageserver's filesystem.",
483 : &["tenant_id", "shard_id", "timeline_id"]
484 : )
485 141 : .expect("failed to define a metric")
486 141 : });
487 :
488 136 : pub(crate) static RESIDENT_PHYSICAL_SIZE_GLOBAL: Lazy<UIntGauge> = Lazy::new(|| {
489 : register_uint_gauge!(
490 : "pageserver_resident_physical_size_global",
491 : "Like `pageserver_resident_physical_size`, but without tenant/timeline dimensions."
492 : )
493 136 : .expect("failed to define a metric")
494 136 : });
495 :
496 141 : static REMOTE_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
497 : register_uint_gauge_vec!(
498 : "pageserver_remote_physical_size",
499 : "The size of the layer files present in the remote storage that are listed in the remote index_part.json.",
500 : // Corollary: If any files are missing from the index part, they won't be included here.
501 : &["tenant_id", "shard_id", "timeline_id"]
502 : )
503 141 : .expect("failed to define a metric")
504 141 : });
505 :
506 141 : static REMOTE_PHYSICAL_SIZE_GLOBAL: Lazy<UIntGauge> = Lazy::new(|| {
507 : register_uint_gauge!(
508 : "pageserver_remote_physical_size_global",
509 : "Like `pageserver_remote_physical_size`, but without tenant/timeline dimensions."
510 : )
511 141 : .expect("failed to define a metric")
512 141 : });
513 :
514 4 : pub(crate) static REMOTE_ONDEMAND_DOWNLOADED_LAYERS: Lazy<IntCounter> = Lazy::new(|| {
515 : register_int_counter!(
516 : "pageserver_remote_ondemand_downloaded_layers_total",
517 : "Total on-demand downloaded layers"
518 : )
519 4 : .unwrap()
520 4 : });
521 :
522 4 : pub(crate) static REMOTE_ONDEMAND_DOWNLOADED_BYTES: Lazy<IntCounter> = Lazy::new(|| {
523 : register_int_counter!(
524 : "pageserver_remote_ondemand_downloaded_bytes_total",
525 : "Total bytes of layers on-demand downloaded",
526 : )
527 4 : .unwrap()
528 4 : });
529 :
530 141 : static CURRENT_LOGICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
531 : register_uint_gauge_vec!(
532 : "pageserver_current_logical_size",
533 : "Current logical size grouped by timeline",
534 : &["tenant_id", "shard_id", "timeline_id"]
535 : )
536 141 : .expect("failed to define current logical size metric")
537 141 : });
538 :
539 141 : static AUX_FILE_SIZE: Lazy<IntGaugeVec> = Lazy::new(|| {
540 : register_int_gauge_vec!(
541 : "pageserver_aux_file_estimated_size",
542 : "The size of all aux files for a timeline in aux file v2 store.",
543 : &["tenant_id", "shard_id", "timeline_id"]
544 : )
545 141 : .expect("failed to define a metric")
546 141 : });
547 :
548 141 : static VALID_LSN_LEASE_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
549 : register_uint_gauge_vec!(
550 : "pageserver_valid_lsn_lease_count",
551 : "The number of valid leases after refreshing gc info.",
552 : &["tenant_id", "shard_id", "timeline_id"],
553 : )
554 141 : .expect("failed to define a metric")
555 141 : });
556 :
557 : pub(crate) mod initial_logical_size {
558 : use metrics::{register_int_counter, register_int_counter_vec, IntCounter, IntCounterVec};
559 : use once_cell::sync::Lazy;
560 :
561 : pub(crate) struct StartCalculation(IntCounterVec);
562 141 : pub(crate) static START_CALCULATION: Lazy<StartCalculation> = Lazy::new(|| {
563 141 : StartCalculation(
564 141 : register_int_counter_vec!(
565 : "pageserver_initial_logical_size_start_calculation",
566 : "Incremented each time we start an initial logical size calculation attempt. \
567 : The `circumstances` label provides some additional details.",
568 : &["attempt", "circumstances"]
569 141 : )
570 141 : .unwrap(),
571 141 : )
572 141 : });
573 :
574 : struct DropCalculation {
575 : first: IntCounter,
576 : retry: IntCounter,
577 : }
578 :
579 141 : static DROP_CALCULATION: Lazy<DropCalculation> = Lazy::new(|| {
580 141 : let vec = register_int_counter_vec!(
581 : "pageserver_initial_logical_size_drop_calculation",
582 : "Incremented each time we abort a started size calculation attmpt.",
583 : &["attempt"]
584 : )
585 141 : .unwrap();
586 141 : DropCalculation {
587 141 : first: vec.with_label_values(&["first"]),
588 141 : retry: vec.with_label_values(&["retry"]),
589 141 : }
590 141 : });
591 :
592 : pub(crate) struct Calculated {
593 : pub(crate) births: IntCounter,
594 : pub(crate) deaths: IntCounter,
595 : }
596 :
597 141 : pub(crate) static CALCULATED: Lazy<Calculated> = Lazy::new(|| Calculated {
598 : births: register_int_counter!(
599 : "pageserver_initial_logical_size_finish_calculation",
600 : "Incremented every time we finish calculation of initial logical size.\
601 : If everything is working well, this should happen at most once per Timeline object."
602 : )
603 141 : .unwrap(),
604 : deaths: register_int_counter!(
605 : "pageserver_initial_logical_size_drop_finished_calculation",
606 : "Incremented when we drop a finished initial logical size calculation result.\
607 : Mainly useful to turn pageserver_initial_logical_size_finish_calculation into a gauge."
608 : )
609 141 : .unwrap(),
610 141 : });
611 :
612 : pub(crate) struct OngoingCalculationGuard {
613 : inc_drop_calculation: Option<IntCounter>,
614 : }
615 :
616 153 : #[derive(strum_macros::IntoStaticStr)]
617 : pub(crate) enum StartCircumstances {
618 : EmptyInitial,
619 : SkippedConcurrencyLimiter,
620 : AfterBackgroundTasksRateLimit,
621 : }
622 :
623 : impl StartCalculation {
624 153 : pub(crate) fn first(&self, circumstances: StartCircumstances) -> OngoingCalculationGuard {
625 153 : let circumstances_label: &'static str = circumstances.into();
626 153 : self.0
627 153 : .with_label_values(&["first", circumstances_label])
628 153 : .inc();
629 153 : OngoingCalculationGuard {
630 153 : inc_drop_calculation: Some(DROP_CALCULATION.first.clone()),
631 153 : }
632 153 : }
633 0 : pub(crate) fn retry(&self, circumstances: StartCircumstances) -> OngoingCalculationGuard {
634 0 : let circumstances_label: &'static str = circumstances.into();
635 0 : self.0
636 0 : .with_label_values(&["retry", circumstances_label])
637 0 : .inc();
638 0 : OngoingCalculationGuard {
639 0 : inc_drop_calculation: Some(DROP_CALCULATION.retry.clone()),
640 0 : }
641 0 : }
642 : }
643 :
644 : impl Drop for OngoingCalculationGuard {
645 153 : fn drop(&mut self) {
646 153 : if let Some(counter) = self.inc_drop_calculation.take() {
647 0 : counter.inc();
648 153 : }
649 153 : }
650 : }
651 :
652 : impl OngoingCalculationGuard {
653 153 : pub(crate) fn calculation_result_saved(mut self) -> FinishedCalculationGuard {
654 153 : drop(self.inc_drop_calculation.take());
655 153 : CALCULATED.births.inc();
656 153 : FinishedCalculationGuard {
657 153 : inc_on_drop: CALCULATED.deaths.clone(),
658 153 : }
659 153 : }
660 : }
661 :
662 : pub(crate) struct FinishedCalculationGuard {
663 : inc_on_drop: IntCounter,
664 : }
665 :
666 : impl Drop for FinishedCalculationGuard {
667 7 : fn drop(&mut self) {
668 7 : self.inc_on_drop.inc();
669 7 : }
670 : }
671 :
672 : // context: https://github.com/neondatabase/neon/issues/5963
673 : pub(crate) static TIMELINES_WHERE_WALRECEIVER_GOT_APPROXIMATE_SIZE: Lazy<IntCounter> =
674 0 : Lazy::new(|| {
675 : register_int_counter!(
676 : "pageserver_initial_logical_size_timelines_where_walreceiver_got_approximate_size",
677 : "Counter for the following event: walreceiver calls\
678 : Timeline::get_current_logical_size() and it returns `Approximate` for the first time."
679 : )
680 0 : .unwrap()
681 0 : });
682 : }
683 :
684 0 : static DIRECTORY_ENTRIES_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
685 : register_uint_gauge_vec!(
686 : "pageserver_directory_entries_count",
687 : "Sum of the entries in pageserver-stored directory listings",
688 : &["tenant_id", "shard_id", "timeline_id"]
689 : )
690 0 : .expect("failed to define a metric")
691 0 : });
692 :
693 143 : pub(crate) static TENANT_STATE_METRIC: Lazy<UIntGaugeVec> = Lazy::new(|| {
694 : register_uint_gauge_vec!(
695 : "pageserver_tenant_states_count",
696 : "Count of tenants per state",
697 : &["state"]
698 : )
699 143 : .expect("Failed to register pageserver_tenant_states_count metric")
700 143 : });
701 :
702 : /// A set of broken tenants.
703 : ///
704 : /// These are expected to be so rare that a set is fine. Set as in a new timeseries per each broken
705 : /// tenant.
706 12 : pub(crate) static BROKEN_TENANTS_SET: Lazy<UIntGaugeVec> = Lazy::new(|| {
707 : register_uint_gauge_vec!(
708 : "pageserver_broken_tenants_count",
709 : "Set of broken tenants",
710 : &["tenant_id", "shard_id"]
711 : )
712 12 : .expect("Failed to register pageserver_tenant_states_count metric")
713 12 : });
714 :
715 6 : pub(crate) static TENANT_SYNTHETIC_SIZE_METRIC: Lazy<UIntGaugeVec> = Lazy::new(|| {
716 : register_uint_gauge_vec!(
717 : "pageserver_tenant_synthetic_cached_size_bytes",
718 : "Synthetic size of each tenant in bytes",
719 : &["tenant_id"]
720 : )
721 6 : .expect("Failed to register pageserver_tenant_synthetic_cached_size_bytes metric")
722 6 : });
723 :
724 0 : pub(crate) static EVICTION_ITERATION_DURATION: Lazy<HistogramVec> = Lazy::new(|| {
725 : register_histogram_vec!(
726 : "pageserver_eviction_iteration_duration_seconds_global",
727 : "Time spent on a single eviction iteration",
728 : &["period_secs", "threshold_secs"],
729 : STORAGE_OP_BUCKETS.into(),
730 : )
731 0 : .expect("failed to define a metric")
732 0 : });
733 :
734 141 : static EVICTIONS: Lazy<IntCounterVec> = Lazy::new(|| {
735 : register_int_counter_vec!(
736 : "pageserver_evictions",
737 : "Number of layers evicted from the pageserver",
738 : &["tenant_id", "shard_id", "timeline_id"]
739 : )
740 141 : .expect("failed to define a metric")
741 141 : });
742 :
743 141 : static EVICTIONS_WITH_LOW_RESIDENCE_DURATION: Lazy<IntCounterVec> = Lazy::new(|| {
744 : register_int_counter_vec!(
745 : "pageserver_evictions_with_low_residence_duration",
746 : "If a layer is evicted that was resident for less than `low_threshold`, it is counted to this counter. \
747 : Residence duration is determined using the `residence_duration_data_source`.",
748 : &["tenant_id", "shard_id", "timeline_id", "residence_duration_data_source", "low_threshold_secs"]
749 : )
750 141 : .expect("failed to define a metric")
751 141 : });
752 :
753 0 : pub(crate) static UNEXPECTED_ONDEMAND_DOWNLOADS: Lazy<IntCounter> = Lazy::new(|| {
754 : register_int_counter!(
755 : "pageserver_unexpected_ondemand_downloads_count",
756 : "Number of unexpected on-demand downloads. \
757 : We log more context for each increment, so, forgo any labels in this metric.",
758 : )
759 0 : .expect("failed to define a metric")
760 0 : });
761 :
762 : /// How long did we take to start up? Broken down by labels to describe
763 : /// different phases of startup.
764 0 : pub static STARTUP_DURATION: Lazy<GaugeVec> = Lazy::new(|| {
765 : register_gauge_vec!(
766 : "pageserver_startup_duration_seconds",
767 : "Time taken by phases of pageserver startup, in seconds",
768 : &["phase"]
769 : )
770 0 : .expect("Failed to register pageserver_startup_duration_seconds metric")
771 0 : });
772 :
773 0 : pub static STARTUP_IS_LOADING: Lazy<UIntGauge> = Lazy::new(|| {
774 : register_uint_gauge!(
775 : "pageserver_startup_is_loading",
776 : "1 while in initial startup load of tenants, 0 at other times"
777 : )
778 0 : .expect("Failed to register pageserver_startup_is_loading")
779 0 : });
780 :
781 136 : pub(crate) static TIMELINE_EPHEMERAL_BYTES: Lazy<UIntGauge> = Lazy::new(|| {
782 : register_uint_gauge!(
783 : "pageserver_timeline_ephemeral_bytes",
784 : "Total number of bytes in ephemeral layers, summed for all timelines. Approximate, lazily updated."
785 : )
786 136 : .expect("Failed to register metric")
787 136 : });
788 :
789 : /// Metrics related to the lifecycle of a [`crate::tenant::Tenant`] object: things
790 : /// like how long it took to load.
791 : ///
792 : /// Note that these are process-global metrics, _not_ per-tenant metrics. Per-tenant
793 : /// metrics are rather expensive, and usually fine grained stuff makes more sense
794 : /// at a timeline level than tenant level.
795 : pub(crate) struct TenantMetrics {
796 : /// How long did tenants take to go from construction to active state?
797 : pub(crate) activation: Histogram,
798 : pub(crate) preload: Histogram,
799 : pub(crate) attach: Histogram,
800 :
801 : /// How many tenants are included in the initial startup of the pagesrever?
802 : pub(crate) startup_scheduled: IntCounter,
803 : pub(crate) startup_complete: IntCounter,
804 : }
805 :
806 0 : pub(crate) static TENANT: Lazy<TenantMetrics> = Lazy::new(|| {
807 0 : TenantMetrics {
808 0 : activation: register_histogram!(
809 : "pageserver_tenant_activation_seconds",
810 : "Time taken by tenants to activate, in seconds",
811 : CRITICAL_OP_BUCKETS.into()
812 0 : )
813 0 : .expect("Failed to register metric"),
814 0 : preload: register_histogram!(
815 : "pageserver_tenant_preload_seconds",
816 : "Time taken by tenants to load remote metadata on startup/attach, in seconds",
817 : CRITICAL_OP_BUCKETS.into()
818 0 : )
819 0 : .expect("Failed to register metric"),
820 0 : attach: register_histogram!(
821 : "pageserver_tenant_attach_seconds",
822 : "Time taken by tenants to intialize, after remote metadata is already loaded",
823 : CRITICAL_OP_BUCKETS.into()
824 0 : )
825 0 : .expect("Failed to register metric"),
826 0 : startup_scheduled: register_int_counter!(
827 : "pageserver_tenant_startup_scheduled",
828 : "Number of tenants included in pageserver startup (doesn't count tenants attached later)"
829 0 : ).expect("Failed to register metric"),
830 0 : startup_complete: register_int_counter!(
831 : "pageserver_tenant_startup_complete",
832 : "Number of tenants that have completed warm-up, or activated on-demand during initial startup: \
833 : should eventually reach `pageserver_tenant_startup_scheduled_total`. Does not include broken \
834 : tenants: such cases will lead to this metric never reaching the scheduled count."
835 0 : ).expect("Failed to register metric"),
836 0 : }
837 0 : });
838 :
839 : /// Each `Timeline`'s [`EVICTIONS_WITH_LOW_RESIDENCE_DURATION`] metric.
840 : #[derive(Debug)]
841 : pub(crate) struct EvictionsWithLowResidenceDuration {
842 : data_source: &'static str,
843 : threshold: Duration,
844 : counter: Option<IntCounter>,
845 : }
846 :
847 : pub(crate) struct EvictionsWithLowResidenceDurationBuilder {
848 : data_source: &'static str,
849 : threshold: Duration,
850 : }
851 :
852 : impl EvictionsWithLowResidenceDurationBuilder {
853 383 : pub fn new(data_source: &'static str, threshold: Duration) -> Self {
854 383 : Self {
855 383 : data_source,
856 383 : threshold,
857 383 : }
858 383 : }
859 :
860 383 : fn build(
861 383 : &self,
862 383 : tenant_id: &str,
863 383 : shard_id: &str,
864 383 : timeline_id: &str,
865 383 : ) -> EvictionsWithLowResidenceDuration {
866 383 : let counter = EVICTIONS_WITH_LOW_RESIDENCE_DURATION
867 383 : .get_metric_with_label_values(&[
868 383 : tenant_id,
869 383 : shard_id,
870 383 : timeline_id,
871 383 : self.data_source,
872 383 : &EvictionsWithLowResidenceDuration::threshold_label_value(self.threshold),
873 383 : ])
874 383 : .unwrap();
875 383 : EvictionsWithLowResidenceDuration {
876 383 : data_source: self.data_source,
877 383 : threshold: self.threshold,
878 383 : counter: Some(counter),
879 383 : }
880 383 : }
881 : }
882 :
883 : impl EvictionsWithLowResidenceDuration {
884 391 : fn threshold_label_value(threshold: Duration) -> String {
885 391 : format!("{}", threshold.as_secs())
886 391 : }
887 :
888 18 : pub fn observe(&self, observed_value: Duration) {
889 18 : if observed_value < self.threshold {
890 18 : self.counter
891 18 : .as_ref()
892 18 : .expect("nobody calls this function after `remove_from_vec`")
893 18 : .inc();
894 18 : }
895 18 : }
896 :
897 8 : pub fn change_threshold(
898 8 : &mut self,
899 8 : tenant_id: &str,
900 8 : shard_id: &str,
901 8 : timeline_id: &str,
902 8 : new_threshold: Duration,
903 8 : ) {
904 8 : if new_threshold == self.threshold {
905 8 : return;
906 0 : }
907 0 : let mut with_new = EvictionsWithLowResidenceDurationBuilder::new(
908 0 : self.data_source,
909 0 : new_threshold,
910 0 : )
911 0 : .build(tenant_id, shard_id, timeline_id);
912 0 : std::mem::swap(self, &mut with_new);
913 0 : with_new.remove(tenant_id, shard_id, timeline_id);
914 8 : }
915 :
916 : // This could be a `Drop` impl, but, we need the `tenant_id` and `timeline_id`.
917 8 : fn remove(&mut self, tenant_id: &str, shard_id: &str, timeline_id: &str) {
918 8 : let Some(_counter) = self.counter.take() else {
919 0 : return;
920 : };
921 :
922 8 : let threshold = Self::threshold_label_value(self.threshold);
923 8 :
924 8 : let removed = EVICTIONS_WITH_LOW_RESIDENCE_DURATION.remove_label_values(&[
925 8 : tenant_id,
926 8 : shard_id,
927 8 : timeline_id,
928 8 : self.data_source,
929 8 : &threshold,
930 8 : ]);
931 8 :
932 8 : match removed {
933 0 : Err(e) => {
934 0 : // this has been hit in staging as
935 0 : // <https://neondatabase.sentry.io/issues/4142396994/>, but we don't know how.
936 0 : // because we can be in the drop path already, don't risk:
937 0 : // - "double-panic => illegal instruction" or
938 0 : // - future "drop panick => abort"
939 0 : //
940 0 : // so just nag: (the error has the labels)
941 0 : tracing::warn!("failed to remove EvictionsWithLowResidenceDuration, it was already removed? {e:#?}");
942 : }
943 : Ok(()) => {
944 : // to help identify cases where we double-remove the same values, let's log all
945 : // deletions?
946 8 : tracing::info!("removed EvictionsWithLowResidenceDuration with {tenant_id}, {timeline_id}, {}, {threshold}", self.data_source);
947 : }
948 : }
949 8 : }
950 : }
951 :
952 : // Metrics collected on disk IO operations
953 : //
954 : // Roughly logarithmic scale.
955 : const STORAGE_IO_TIME_BUCKETS: &[f64] = &[
956 : 0.000030, // 30 usec
957 : 0.001000, // 1000 usec
958 : 0.030, // 30 ms
959 : 1.000, // 1000 ms
960 : 30.000, // 30000 ms
961 : ];
962 :
963 : /// VirtualFile fs operation variants.
964 : ///
965 : /// Operations:
966 : /// - open ([`std::fs::OpenOptions::open`])
967 : /// - close (dropping [`crate::virtual_file::VirtualFile`])
968 : /// - close-by-replace (close by replacement algorithm)
969 : /// - read (`read_at`)
970 : /// - write (`write_at`)
971 : /// - seek (modify internal position or file length query)
972 : /// - fsync ([`std::fs::File::sync_all`])
973 : /// - metadata ([`std::fs::File::metadata`])
974 : #[derive(
975 1440 : Debug, Clone, Copy, strum_macros::EnumCount, strum_macros::EnumIter, strum_macros::FromRepr,
976 : )]
977 : pub(crate) enum StorageIoOperation {
978 : Open,
979 : OpenAfterReplace,
980 : Close,
981 : CloseByReplace,
982 : Read,
983 : Write,
984 : Seek,
985 : Fsync,
986 : Metadata,
987 : }
988 :
989 : impl StorageIoOperation {
990 1440 : pub fn as_str(&self) -> &'static str {
991 1440 : match self {
992 160 : StorageIoOperation::Open => "open",
993 160 : StorageIoOperation::OpenAfterReplace => "open-after-replace",
994 160 : StorageIoOperation::Close => "close",
995 160 : StorageIoOperation::CloseByReplace => "close-by-replace",
996 160 : StorageIoOperation::Read => "read",
997 160 : StorageIoOperation::Write => "write",
998 160 : StorageIoOperation::Seek => "seek",
999 160 : StorageIoOperation::Fsync => "fsync",
1000 160 : StorageIoOperation::Metadata => "metadata",
1001 : }
1002 1440 : }
1003 : }
1004 :
1005 : /// Tracks time taken by fs operations near VirtualFile.
1006 : #[derive(Debug)]
1007 : pub(crate) struct StorageIoTime {
1008 : metrics: [Histogram; StorageIoOperation::COUNT],
1009 : }
1010 :
1011 : impl StorageIoTime {
1012 160 : fn new() -> Self {
1013 160 : let storage_io_histogram_vec = register_histogram_vec!(
1014 : "pageserver_io_operations_seconds",
1015 : "Time spent in IO operations",
1016 : &["operation"],
1017 : STORAGE_IO_TIME_BUCKETS.into()
1018 : )
1019 160 : .expect("failed to define a metric");
1020 1440 : let metrics = std::array::from_fn(|i| {
1021 1440 : let op = StorageIoOperation::from_repr(i).unwrap();
1022 1440 : storage_io_histogram_vec
1023 1440 : .get_metric_with_label_values(&[op.as_str()])
1024 1440 : .unwrap()
1025 1440 : });
1026 160 : Self { metrics }
1027 160 : }
1028 :
1029 1903891 : pub(crate) fn get(&self, op: StorageIoOperation) -> &Histogram {
1030 1903891 : &self.metrics[op as usize]
1031 1903891 : }
1032 : }
1033 :
1034 : pub(crate) static STORAGE_IO_TIME_METRIC: Lazy<StorageIoTime> = Lazy::new(StorageIoTime::new);
1035 :
1036 : const STORAGE_IO_SIZE_OPERATIONS: &[&str] = &["read", "write"];
1037 :
1038 : // Needed for the https://neonprod.grafana.net/d/5uK9tHL4k/picking-tenant-for-relocation?orgId=1
1039 158 : pub(crate) static STORAGE_IO_SIZE: Lazy<IntGaugeVec> = Lazy::new(|| {
1040 : register_int_gauge_vec!(
1041 : "pageserver_io_operations_bytes_total",
1042 : "Total amount of bytes read/written in IO operations",
1043 : &["operation", "tenant_id", "shard_id", "timeline_id"]
1044 : )
1045 158 : .expect("failed to define a metric")
1046 158 : });
1047 :
1048 : #[cfg(not(test))]
1049 : pub(crate) mod virtual_file_descriptor_cache {
1050 : use super::*;
1051 :
1052 0 : pub(crate) static SIZE_MAX: Lazy<UIntGauge> = Lazy::new(|| {
1053 : register_uint_gauge!(
1054 : "pageserver_virtual_file_descriptor_cache_size_max",
1055 : "Maximum number of open file descriptors in the cache."
1056 : )
1057 0 : .unwrap()
1058 0 : });
1059 :
1060 : // SIZE_CURRENT: derive it like so:
1061 : // ```
1062 : // sum (pageserver_io_operations_seconds_count{operation=~"^(open|open-after-replace)$")
1063 : // -ignoring(operation)
1064 : // sum(pageserver_io_operations_seconds_count{operation=~"^(close|close-by-replace)$"}
1065 : // ```
1066 : }
1067 :
1068 : #[cfg(not(test))]
1069 : pub(crate) mod virtual_file_io_engine {
1070 : use super::*;
1071 :
1072 0 : pub(crate) static KIND: Lazy<UIntGaugeVec> = Lazy::new(|| {
1073 : register_uint_gauge_vec!(
1074 : "pageserver_virtual_file_io_engine_kind",
1075 : "The configured io engine for VirtualFile",
1076 : &["kind"],
1077 : )
1078 0 : .unwrap()
1079 0 : });
1080 : }
1081 :
1082 : #[derive(Debug)]
1083 : struct GlobalAndPerTimelineHistogram {
1084 : global: Histogram,
1085 : per_tenant_timeline: Histogram,
1086 : }
1087 :
1088 : impl GlobalAndPerTimelineHistogram {
1089 10 : fn observe(&self, value: f64) {
1090 10 : self.global.observe(value);
1091 10 : self.per_tenant_timeline.observe(value);
1092 10 : }
1093 : }
1094 :
1095 : struct GlobalAndPerTimelineHistogramTimer<'a, 'c> {
1096 : h: &'a GlobalAndPerTimelineHistogram,
1097 : ctx: &'c RequestContext,
1098 : start: std::time::Instant,
1099 : op: SmgrQueryType,
1100 : }
1101 :
1102 : impl<'a, 'c> Drop for GlobalAndPerTimelineHistogramTimer<'a, 'c> {
1103 10 : fn drop(&mut self) {
1104 10 : let elapsed = self.start.elapsed();
1105 10 : let ex_throttled = self
1106 10 : .ctx
1107 10 : .micros_spent_throttled
1108 10 : .close_and_checked_sub_from(elapsed);
1109 10 : let ex_throttled = match ex_throttled {
1110 10 : Ok(res) => res,
1111 0 : Err(error) => {
1112 0 : use utils::rate_limit::RateLimit;
1113 0 : static LOGGED: Lazy<Mutex<enum_map::EnumMap<SmgrQueryType, RateLimit>>> =
1114 0 : Lazy::new(|| {
1115 0 : Mutex::new(enum_map::EnumMap::from_array(std::array::from_fn(|_| {
1116 0 : RateLimit::new(Duration::from_secs(10))
1117 0 : })))
1118 0 : });
1119 0 : let mut guard = LOGGED.lock().unwrap();
1120 0 : let rate_limit = &mut guard[self.op];
1121 0 : rate_limit.call(|| {
1122 0 : warn!(op=?self.op, error, "error deducting time spent throttled; this message is logged at a global rate limit");
1123 0 : });
1124 0 : elapsed
1125 : }
1126 : };
1127 10 : self.h.observe(ex_throttled.as_secs_f64());
1128 10 : }
1129 : }
1130 :
1131 : #[derive(
1132 : Debug,
1133 : Clone,
1134 : Copy,
1135 3980 : IntoStaticStr,
1136 : strum_macros::EnumCount,
1137 112 : strum_macros::EnumIter,
1138 1965 : strum_macros::FromRepr,
1139 : enum_map::Enum,
1140 : )]
1141 : #[strum(serialize_all = "snake_case")]
1142 : pub enum SmgrQueryType {
1143 : GetRelExists,
1144 : GetRelSize,
1145 : GetPageAtLsn,
1146 : GetDbSize,
1147 : GetSlruSegment,
1148 : }
1149 :
1150 : #[derive(Debug)]
1151 : pub(crate) struct SmgrQueryTimePerTimeline {
1152 : metrics: [GlobalAndPerTimelineHistogram; SmgrQueryType::COUNT],
1153 : }
1154 :
1155 143 : static SMGR_QUERY_TIME_PER_TENANT_TIMELINE: Lazy<HistogramVec> = Lazy::new(|| {
1156 : register_histogram_vec!(
1157 : "pageserver_smgr_query_seconds",
1158 : "Time spent on smgr query handling, aggegated by query type and tenant/timeline.",
1159 : &["smgr_query_type", "tenant_id", "shard_id", "timeline_id"],
1160 : CRITICAL_OP_BUCKETS.into(),
1161 : )
1162 143 : .expect("failed to define a metric")
1163 143 : });
1164 :
1165 143 : static SMGR_QUERY_TIME_GLOBAL_BUCKETS: Lazy<Vec<f64>> = Lazy::new(|| {
1166 143 : [
1167 143 : 1,
1168 143 : 10,
1169 143 : 20,
1170 143 : 40,
1171 143 : 60,
1172 143 : 80,
1173 143 : 100,
1174 143 : 200,
1175 143 : 300,
1176 143 : 400,
1177 143 : 500,
1178 143 : 600,
1179 143 : 700,
1180 143 : 800,
1181 143 : 900,
1182 143 : 1_000, // 1ms
1183 143 : 2_000,
1184 143 : 4_000,
1185 143 : 6_000,
1186 143 : 8_000,
1187 143 : 10_000, // 10ms
1188 143 : 20_000,
1189 143 : 40_000,
1190 143 : 60_000,
1191 143 : 80_000,
1192 143 : 100_000,
1193 143 : 200_000,
1194 143 : 400_000,
1195 143 : 600_000,
1196 143 : 800_000,
1197 143 : 1_000_000, // 1s
1198 143 : 2_000_000,
1199 143 : 4_000_000,
1200 143 : 6_000_000,
1201 143 : 8_000_000,
1202 143 : 10_000_000, // 10s
1203 143 : 20_000_000,
1204 143 : 50_000_000,
1205 143 : 100_000_000,
1206 143 : 200_000_000,
1207 143 : 1_000_000_000, // 1000s
1208 143 : ]
1209 143 : .into_iter()
1210 143 : .map(Duration::from_micros)
1211 5863 : .map(|d| d.as_secs_f64())
1212 143 : .collect()
1213 143 : });
1214 :
1215 143 : static SMGR_QUERY_TIME_GLOBAL: Lazy<HistogramVec> = Lazy::new(|| {
1216 : register_histogram_vec!(
1217 : "pageserver_smgr_query_seconds_global",
1218 : "Time spent on smgr query handling, aggregated by query type.",
1219 : &["smgr_query_type"],
1220 : SMGR_QUERY_TIME_GLOBAL_BUCKETS.clone(),
1221 : )
1222 143 : .expect("failed to define a metric")
1223 143 : });
1224 :
1225 : impl SmgrQueryTimePerTimeline {
1226 393 : pub(crate) fn new(tenant_shard_id: &TenantShardId, timeline_id: &TimelineId) -> Self {
1227 393 : let tenant_id = tenant_shard_id.tenant_id.to_string();
1228 393 : let shard_slug = format!("{}", tenant_shard_id.shard_slug());
1229 393 : let timeline_id = timeline_id.to_string();
1230 1965 : let metrics = std::array::from_fn(|i| {
1231 1965 : let op = SmgrQueryType::from_repr(i).unwrap();
1232 1965 : let global = SMGR_QUERY_TIME_GLOBAL
1233 1965 : .get_metric_with_label_values(&[op.into()])
1234 1965 : .unwrap();
1235 1965 : let per_tenant_timeline = SMGR_QUERY_TIME_PER_TENANT_TIMELINE
1236 1965 : .get_metric_with_label_values(&[op.into(), &tenant_id, &shard_slug, &timeline_id])
1237 1965 : .unwrap();
1238 1965 : GlobalAndPerTimelineHistogram {
1239 1965 : global,
1240 1965 : per_tenant_timeline,
1241 1965 : }
1242 1965 : });
1243 393 : Self { metrics }
1244 393 : }
1245 10 : pub(crate) fn start_timer<'c: 'a, 'a>(
1246 10 : &'a self,
1247 10 : op: SmgrQueryType,
1248 10 : ctx: &'c RequestContext,
1249 10 : ) -> impl Drop + '_ {
1250 10 : let metric = &self.metrics[op as usize];
1251 10 : let start = Instant::now();
1252 10 : match ctx.micros_spent_throttled.open() {
1253 10 : Ok(()) => (),
1254 0 : Err(error) => {
1255 0 : use utils::rate_limit::RateLimit;
1256 0 : static LOGGED: Lazy<Mutex<enum_map::EnumMap<SmgrQueryType, RateLimit>>> =
1257 0 : Lazy::new(|| {
1258 0 : Mutex::new(enum_map::EnumMap::from_array(std::array::from_fn(|_| {
1259 0 : RateLimit::new(Duration::from_secs(10))
1260 0 : })))
1261 0 : });
1262 0 : let mut guard = LOGGED.lock().unwrap();
1263 0 : let rate_limit = &mut guard[op];
1264 0 : rate_limit.call(|| {
1265 0 : warn!(?op, error, "error opening micros_spent_throttled; this message is logged at a global rate limit");
1266 0 : });
1267 0 : }
1268 : }
1269 10 : GlobalAndPerTimelineHistogramTimer {
1270 10 : h: metric,
1271 10 : ctx,
1272 10 : start,
1273 10 : op,
1274 10 : }
1275 10 : }
1276 : }
1277 :
1278 : #[cfg(test)]
1279 : mod smgr_query_time_tests {
1280 : use pageserver_api::shard::TenantShardId;
1281 : use strum::IntoEnumIterator;
1282 : use utils::id::{TenantId, TimelineId};
1283 :
1284 : use crate::{
1285 : context::{DownloadBehavior, RequestContext},
1286 : task_mgr::TaskKind,
1287 : };
1288 :
1289 : // Regression test, we used hard-coded string constants before using an enum.
1290 : #[test]
1291 2 : fn op_label_name() {
1292 2 : use super::SmgrQueryType::*;
1293 2 : let expect: [(super::SmgrQueryType, &'static str); 5] = [
1294 2 : (GetRelExists, "get_rel_exists"),
1295 2 : (GetRelSize, "get_rel_size"),
1296 2 : (GetPageAtLsn, "get_page_at_lsn"),
1297 2 : (GetDbSize, "get_db_size"),
1298 2 : (GetSlruSegment, "get_slru_segment"),
1299 2 : ];
1300 12 : for (op, expect) in expect {
1301 10 : let actual: &'static str = op.into();
1302 10 : assert_eq!(actual, expect);
1303 : }
1304 2 : }
1305 :
1306 : #[test]
1307 2 : fn basic() {
1308 2 : let ops: Vec<_> = super::SmgrQueryType::iter().collect();
1309 :
1310 12 : for op in &ops {
1311 10 : let tenant_id = TenantId::generate();
1312 10 : let timeline_id = TimelineId::generate();
1313 10 : let metrics = super::SmgrQueryTimePerTimeline::new(
1314 10 : &TenantShardId::unsharded(tenant_id),
1315 10 : &timeline_id,
1316 10 : );
1317 10 :
1318 20 : let get_counts = || {
1319 20 : let global: u64 = ops
1320 20 : .iter()
1321 100 : .map(|op| metrics.metrics[*op as usize].global.get_sample_count())
1322 20 : .sum();
1323 20 : let per_tenant_timeline: u64 = ops
1324 20 : .iter()
1325 100 : .map(|op| {
1326 100 : metrics.metrics[*op as usize]
1327 100 : .per_tenant_timeline
1328 100 : .get_sample_count()
1329 100 : })
1330 20 : .sum();
1331 20 : (global, per_tenant_timeline)
1332 20 : };
1333 :
1334 10 : let (pre_global, pre_per_tenant_timeline) = get_counts();
1335 10 : assert_eq!(pre_per_tenant_timeline, 0);
1336 :
1337 10 : let ctx = RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Download);
1338 10 : let timer = metrics.start_timer(*op, &ctx);
1339 10 : drop(timer);
1340 10 :
1341 10 : let (post_global, post_per_tenant_timeline) = get_counts();
1342 10 : assert_eq!(post_per_tenant_timeline, 1);
1343 10 : assert!(post_global > pre_global);
1344 : }
1345 2 : }
1346 : }
1347 :
1348 : // keep in sync with control plane Go code so that we can validate
1349 : // compute's basebackup_ms metric with our perspective in the context of SLI/SLO.
1350 0 : static COMPUTE_STARTUP_BUCKETS: Lazy<[f64; 28]> = Lazy::new(|| {
1351 0 : // Go code uses milliseconds. Variable is called `computeStartupBuckets`
1352 0 : [
1353 0 : 5, 10, 20, 30, 50, 70, 100, 120, 150, 200, 250, 300, 350, 400, 450, 500, 600, 800, 1000,
1354 0 : 1500, 2000, 2500, 3000, 5000, 10000, 20000, 40000, 60000,
1355 0 : ]
1356 0 : .map(|ms| (ms as f64) / 1000.0)
1357 0 : });
1358 :
1359 : pub(crate) struct BasebackupQueryTime {
1360 : ok: Histogram,
1361 : error: Histogram,
1362 : }
1363 :
1364 0 : pub(crate) static BASEBACKUP_QUERY_TIME: Lazy<BasebackupQueryTime> = Lazy::new(|| {
1365 0 : let vec = register_histogram_vec!(
1366 : "pageserver_basebackup_query_seconds",
1367 : "Histogram of basebackup queries durations, by result type",
1368 : &["result"],
1369 : COMPUTE_STARTUP_BUCKETS.to_vec(),
1370 : )
1371 0 : .expect("failed to define a metric");
1372 0 : BasebackupQueryTime {
1373 0 : ok: vec.get_metric_with_label_values(&["ok"]).unwrap(),
1374 0 : error: vec.get_metric_with_label_values(&["error"]).unwrap(),
1375 0 : }
1376 0 : });
1377 :
1378 : pub(crate) struct BasebackupQueryTimeOngoingRecording<'a, 'c> {
1379 : parent: &'a BasebackupQueryTime,
1380 : ctx: &'c RequestContext,
1381 : start: std::time::Instant,
1382 : }
1383 :
1384 : impl BasebackupQueryTime {
1385 0 : pub(crate) fn start_recording<'c: 'a, 'a>(
1386 0 : &'a self,
1387 0 : ctx: &'c RequestContext,
1388 0 : ) -> BasebackupQueryTimeOngoingRecording<'_, '_> {
1389 0 : let start = Instant::now();
1390 0 : match ctx.micros_spent_throttled.open() {
1391 0 : Ok(()) => (),
1392 0 : Err(error) => {
1393 0 : use utils::rate_limit::RateLimit;
1394 0 : static LOGGED: Lazy<Mutex<RateLimit>> =
1395 0 : Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(10))));
1396 0 : let mut rate_limit = LOGGED.lock().unwrap();
1397 0 : rate_limit.call(|| {
1398 0 : warn!(error, "error opening micros_spent_throttled; this message is logged at a global rate limit");
1399 0 : });
1400 0 : }
1401 : }
1402 0 : BasebackupQueryTimeOngoingRecording {
1403 0 : parent: self,
1404 0 : ctx,
1405 0 : start,
1406 0 : }
1407 0 : }
1408 : }
1409 :
1410 : impl<'a, 'c> BasebackupQueryTimeOngoingRecording<'a, 'c> {
1411 0 : pub(crate) fn observe<T, E>(self, res: &Result<T, E>) {
1412 0 : let elapsed = self.start.elapsed();
1413 0 : let ex_throttled = self
1414 0 : .ctx
1415 0 : .micros_spent_throttled
1416 0 : .close_and_checked_sub_from(elapsed);
1417 0 : let ex_throttled = match ex_throttled {
1418 0 : Ok(ex_throttled) => ex_throttled,
1419 0 : Err(error) => {
1420 0 : use utils::rate_limit::RateLimit;
1421 0 : static LOGGED: Lazy<Mutex<RateLimit>> =
1422 0 : Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(10))));
1423 0 : let mut rate_limit = LOGGED.lock().unwrap();
1424 0 : rate_limit.call(|| {
1425 0 : warn!(error, "error deducting time spent throttled; this message is logged at a global rate limit");
1426 0 : });
1427 0 : elapsed
1428 : }
1429 : };
1430 0 : let metric = if res.is_ok() {
1431 0 : &self.parent.ok
1432 : } else {
1433 0 : &self.parent.error
1434 : };
1435 0 : metric.observe(ex_throttled.as_secs_f64());
1436 0 : }
1437 : }
1438 :
1439 0 : pub(crate) static LIVE_CONNECTIONS_COUNT: Lazy<IntGaugeVec> = Lazy::new(|| {
1440 : register_int_gauge_vec!(
1441 : "pageserver_live_connections",
1442 : "Number of live network connections",
1443 : &["pageserver_connection_kind"]
1444 : )
1445 0 : .expect("failed to define a metric")
1446 0 : });
1447 :
1448 0 : #[derive(Clone, Copy, enum_map::Enum, IntoStaticStr)]
1449 : pub(crate) enum ComputeCommandKind {
1450 : PageStreamV2,
1451 : PageStream,
1452 : Basebackup,
1453 : GetLastRecordRlsn,
1454 : Fullbackup,
1455 : ImportBasebackup,
1456 : ImportWal,
1457 : LeaseLsn,
1458 : Show,
1459 : }
1460 :
1461 : pub(crate) struct ComputeCommandCounters {
1462 : map: EnumMap<ComputeCommandKind, IntCounter>,
1463 : }
1464 :
1465 0 : pub(crate) static COMPUTE_COMMANDS_COUNTERS: Lazy<ComputeCommandCounters> = Lazy::new(|| {
1466 0 : let inner = register_int_counter_vec!(
1467 : "pageserver_compute_commands",
1468 : "Number of compute -> pageserver commands processed",
1469 : &["command"]
1470 : )
1471 0 : .expect("failed to define a metric");
1472 0 :
1473 0 : ComputeCommandCounters {
1474 0 : map: EnumMap::from_array(std::array::from_fn(|i| {
1475 0 : let command = <ComputeCommandKind as enum_map::Enum>::from_usize(i);
1476 0 : let command_str: &'static str = command.into();
1477 0 : inner.with_label_values(&[command_str])
1478 0 : })),
1479 0 : }
1480 0 : });
1481 :
1482 : impl ComputeCommandCounters {
1483 0 : pub(crate) fn for_command(&self, command: ComputeCommandKind) -> &IntCounter {
1484 0 : &self.map[command]
1485 0 : }
1486 : }
1487 :
1488 : // remote storage metrics
1489 :
1490 138 : static REMOTE_TIMELINE_CLIENT_CALLS: Lazy<IntCounterPairVec> = Lazy::new(|| {
1491 : register_int_counter_pair_vec!(
1492 : "pageserver_remote_timeline_client_calls_started",
1493 : "Number of started calls to remote timeline client.",
1494 : "pageserver_remote_timeline_client_calls_finished",
1495 : "Number of finshed calls to remote timeline client.",
1496 : &[
1497 : "tenant_id",
1498 : "shard_id",
1499 : "timeline_id",
1500 : "file_kind",
1501 : "op_kind"
1502 : ],
1503 : )
1504 138 : .unwrap()
1505 138 : });
1506 :
1507 : static REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER: Lazy<IntCounterVec> =
1508 134 : Lazy::new(|| {
1509 : register_int_counter_vec!(
1510 : "pageserver_remote_timeline_client_bytes_started",
1511 : "Incremented by the number of bytes associated with a remote timeline client operation. \
1512 : The increment happens when the operation is scheduled.",
1513 : &["tenant_id", "shard_id", "timeline_id", "file_kind", "op_kind"],
1514 : )
1515 134 : .expect("failed to define a metric")
1516 134 : });
1517 :
1518 134 : static REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER: Lazy<IntCounterVec> = Lazy::new(|| {
1519 : register_int_counter_vec!(
1520 : "pageserver_remote_timeline_client_bytes_finished",
1521 : "Incremented by the number of bytes associated with a remote timeline client operation. \
1522 : The increment happens when the operation finishes (regardless of success/failure/shutdown).",
1523 : &["tenant_id", "shard_id", "timeline_id", "file_kind", "op_kind"],
1524 : )
1525 134 : .expect("failed to define a metric")
1526 134 : });
1527 :
1528 : pub(crate) struct TenantManagerMetrics {
1529 : tenant_slots_attached: UIntGauge,
1530 : tenant_slots_secondary: UIntGauge,
1531 : tenant_slots_inprogress: UIntGauge,
1532 : pub(crate) tenant_slot_writes: IntCounter,
1533 : pub(crate) unexpected_errors: IntCounter,
1534 : }
1535 :
1536 : impl TenantManagerMetrics {
1537 : /// Helpers for tracking slots. Note that these do not track the lifetime of TenantSlot objects
1538 : /// exactly: they track the lifetime of the slots _in the tenant map_.
1539 2 : pub(crate) fn slot_inserted(&self, slot: &TenantSlot) {
1540 2 : match slot {
1541 0 : TenantSlot::Attached(_) => {
1542 0 : self.tenant_slots_attached.inc();
1543 0 : }
1544 0 : TenantSlot::Secondary(_) => {
1545 0 : self.tenant_slots_secondary.inc();
1546 0 : }
1547 2 : TenantSlot::InProgress(_) => {
1548 2 : self.tenant_slots_inprogress.inc();
1549 2 : }
1550 : }
1551 2 : }
1552 :
1553 2 : pub(crate) fn slot_removed(&self, slot: &TenantSlot) {
1554 2 : match slot {
1555 2 : TenantSlot::Attached(_) => {
1556 2 : self.tenant_slots_attached.dec();
1557 2 : }
1558 0 : TenantSlot::Secondary(_) => {
1559 0 : self.tenant_slots_secondary.dec();
1560 0 : }
1561 0 : TenantSlot::InProgress(_) => {
1562 0 : self.tenant_slots_inprogress.dec();
1563 0 : }
1564 : }
1565 2 : }
1566 :
1567 : #[cfg(all(debug_assertions, not(test)))]
1568 0 : pub(crate) fn slots_total(&self) -> u64 {
1569 0 : self.tenant_slots_attached.get()
1570 0 : + self.tenant_slots_secondary.get()
1571 0 : + self.tenant_slots_inprogress.get()
1572 0 : }
1573 : }
1574 :
1575 2 : pub(crate) static TENANT_MANAGER: Lazy<TenantManagerMetrics> = Lazy::new(|| {
1576 2 : let tenant_slots = register_uint_gauge_vec!(
1577 : "pageserver_tenant_manager_slots",
1578 : "How many slots currently exist, including all attached, secondary and in-progress operations",
1579 : &["mode"]
1580 : )
1581 2 : .expect("failed to define a metric");
1582 2 : TenantManagerMetrics {
1583 2 : tenant_slots_attached: tenant_slots
1584 2 : .get_metric_with_label_values(&["attached"])
1585 2 : .unwrap(),
1586 2 : tenant_slots_secondary: tenant_slots
1587 2 : .get_metric_with_label_values(&["secondary"])
1588 2 : .unwrap(),
1589 2 : tenant_slots_inprogress: tenant_slots
1590 2 : .get_metric_with_label_values(&["inprogress"])
1591 2 : .unwrap(),
1592 2 : tenant_slot_writes: register_int_counter!(
1593 : "pageserver_tenant_manager_slot_writes",
1594 : "Writes to a tenant slot, including all of create/attach/detach/delete"
1595 2 : )
1596 2 : .expect("failed to define a metric"),
1597 2 : unexpected_errors: register_int_counter!(
1598 : "pageserver_tenant_manager_unexpected_errors_total",
1599 : "Number of unexpected conditions encountered: nonzero value indicates a non-fatal bug."
1600 2 : )
1601 2 : .expect("failed to define a metric"),
1602 2 : }
1603 2 : });
1604 :
1605 : pub(crate) struct DeletionQueueMetrics {
1606 : pub(crate) keys_submitted: IntCounter,
1607 : pub(crate) keys_dropped: IntCounter,
1608 : pub(crate) keys_executed: IntCounter,
1609 : pub(crate) keys_validated: IntCounter,
1610 : pub(crate) dropped_lsn_updates: IntCounter,
1611 : pub(crate) unexpected_errors: IntCounter,
1612 : pub(crate) remote_errors: IntCounterVec,
1613 : }
1614 23 : pub(crate) static DELETION_QUEUE: Lazy<DeletionQueueMetrics> = Lazy::new(|| {
1615 23 : DeletionQueueMetrics{
1616 23 :
1617 23 : keys_submitted: register_int_counter!(
1618 : "pageserver_deletion_queue_submitted_total",
1619 : "Number of objects submitted for deletion"
1620 23 : )
1621 23 : .expect("failed to define a metric"),
1622 23 :
1623 23 : keys_dropped: register_int_counter!(
1624 : "pageserver_deletion_queue_dropped_total",
1625 : "Number of object deletions dropped due to stale generation."
1626 23 : )
1627 23 : .expect("failed to define a metric"),
1628 23 :
1629 23 : keys_executed: register_int_counter!(
1630 : "pageserver_deletion_queue_executed_total",
1631 : "Number of objects deleted. Only includes objects that we actually deleted, sum with pageserver_deletion_queue_dropped_total for the total number of keys processed to completion"
1632 23 : )
1633 23 : .expect("failed to define a metric"),
1634 23 :
1635 23 : keys_validated: register_int_counter!(
1636 : "pageserver_deletion_queue_validated_total",
1637 : "Number of keys validated for deletion. Sum with pageserver_deletion_queue_dropped_total for the total number of keys that have passed through the validation stage."
1638 23 : )
1639 23 : .expect("failed to define a metric"),
1640 23 :
1641 23 : dropped_lsn_updates: register_int_counter!(
1642 : "pageserver_deletion_queue_dropped_lsn_updates_total",
1643 : "Updates to remote_consistent_lsn dropped due to stale generation number."
1644 23 : )
1645 23 : .expect("failed to define a metric"),
1646 23 : unexpected_errors: register_int_counter!(
1647 : "pageserver_deletion_queue_unexpected_errors_total",
1648 : "Number of unexpected condiions that may stall the queue: any value above zero is unexpected."
1649 23 : )
1650 23 : .expect("failed to define a metric"),
1651 23 : remote_errors: register_int_counter_vec!(
1652 : "pageserver_deletion_queue_remote_errors_total",
1653 : "Retryable remote I/O errors while executing deletions, for example 503 responses to DeleteObjects",
1654 : &["op_kind"],
1655 23 : )
1656 23 : .expect("failed to define a metric")
1657 23 : }
1658 23 : });
1659 :
1660 : pub(crate) struct SecondaryModeMetrics {
1661 : pub(crate) upload_heatmap: IntCounter,
1662 : pub(crate) upload_heatmap_errors: IntCounter,
1663 : pub(crate) upload_heatmap_duration: Histogram,
1664 : pub(crate) download_heatmap: IntCounter,
1665 : pub(crate) download_layer: IntCounter,
1666 : }
1667 0 : pub(crate) static SECONDARY_MODE: Lazy<SecondaryModeMetrics> = Lazy::new(|| {
1668 0 : SecondaryModeMetrics {
1669 0 : upload_heatmap: register_int_counter!(
1670 : "pageserver_secondary_upload_heatmap",
1671 : "Number of heatmaps written to remote storage by attached tenants"
1672 0 : )
1673 0 : .expect("failed to define a metric"),
1674 0 : upload_heatmap_errors: register_int_counter!(
1675 : "pageserver_secondary_upload_heatmap_errors",
1676 : "Failures writing heatmap to remote storage"
1677 0 : )
1678 0 : .expect("failed to define a metric"),
1679 0 : upload_heatmap_duration: register_histogram!(
1680 : "pageserver_secondary_upload_heatmap_duration",
1681 : "Time to build and upload a heatmap, including any waiting inside the S3 client"
1682 0 : )
1683 0 : .expect("failed to define a metric"),
1684 0 : download_heatmap: register_int_counter!(
1685 : "pageserver_secondary_download_heatmap",
1686 : "Number of downloads of heatmaps by secondary mode locations, including when it hasn't changed"
1687 0 : )
1688 0 : .expect("failed to define a metric"),
1689 0 : download_layer: register_int_counter!(
1690 : "pageserver_secondary_download_layer",
1691 : "Number of downloads of layers by secondary mode locations"
1692 0 : )
1693 0 : .expect("failed to define a metric"),
1694 0 : }
1695 0 : });
1696 :
1697 : #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1698 : pub enum RemoteOpKind {
1699 : Upload,
1700 : Download,
1701 : Delete,
1702 : }
1703 : impl RemoteOpKind {
1704 12794 : pub fn as_str(&self) -> &'static str {
1705 12794 : match self {
1706 12087 : Self::Upload => "upload",
1707 52 : Self::Download => "download",
1708 655 : Self::Delete => "delete",
1709 : }
1710 12794 : }
1711 : }
1712 :
1713 : #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
1714 : pub enum RemoteOpFileKind {
1715 : Layer,
1716 : Index,
1717 : }
1718 : impl RemoteOpFileKind {
1719 12794 : pub fn as_str(&self) -> &'static str {
1720 12794 : match self {
1721 8638 : Self::Layer => "layer",
1722 4156 : Self::Index => "index",
1723 : }
1724 12794 : }
1725 : }
1726 :
1727 134 : pub(crate) static REMOTE_OPERATION_TIME: Lazy<HistogramVec> = Lazy::new(|| {
1728 : register_histogram_vec!(
1729 : "pageserver_remote_operation_seconds",
1730 : "Time spent on remote storage operations. \
1731 : Grouped by tenant, timeline, operation_kind and status. \
1732 : Does not account for time spent waiting in remote timeline client's queues.",
1733 : &["file_kind", "op_kind", "status"]
1734 : )
1735 134 : .expect("failed to define a metric")
1736 134 : });
1737 :
1738 0 : pub(crate) static TENANT_TASK_EVENTS: Lazy<IntCounterVec> = Lazy::new(|| {
1739 : register_int_counter_vec!(
1740 : "pageserver_tenant_task_events",
1741 : "Number of task start/stop/fail events.",
1742 : &["event"],
1743 : )
1744 0 : .expect("Failed to register tenant_task_events metric")
1745 0 : });
1746 :
1747 20 : pub(crate) static BACKGROUND_LOOP_SEMAPHORE_WAIT_GAUGE: Lazy<IntCounterPairVec> = Lazy::new(|| {
1748 : register_int_counter_pair_vec!(
1749 : "pageserver_background_loop_semaphore_wait_start_count",
1750 : "Counter for background loop concurrency-limiting semaphore acquire calls started",
1751 : "pageserver_background_loop_semaphore_wait_finish_count",
1752 : "Counter for background loop concurrency-limiting semaphore acquire calls finished",
1753 : &["task"],
1754 : )
1755 20 : .unwrap()
1756 20 : });
1757 :
1758 0 : pub(crate) static BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT: Lazy<IntCounterVec> = Lazy::new(|| {
1759 : register_int_counter_vec!(
1760 : "pageserver_background_loop_period_overrun_count",
1761 : "Incremented whenever warn_when_period_overrun() logs a warning.",
1762 : &["task", "period"],
1763 : )
1764 0 : .expect("failed to define a metric")
1765 0 : });
1766 :
1767 : // walreceiver metrics
1768 :
1769 0 : pub(crate) static WALRECEIVER_STARTED_CONNECTIONS: Lazy<IntCounter> = Lazy::new(|| {
1770 : register_int_counter!(
1771 : "pageserver_walreceiver_started_connections_total",
1772 : "Number of started walreceiver connections"
1773 : )
1774 0 : .expect("failed to define a metric")
1775 0 : });
1776 :
1777 0 : pub(crate) static WALRECEIVER_ACTIVE_MANAGERS: Lazy<IntGauge> = Lazy::new(|| {
1778 : register_int_gauge!(
1779 : "pageserver_walreceiver_active_managers",
1780 : "Number of active walreceiver managers"
1781 : )
1782 0 : .expect("failed to define a metric")
1783 0 : });
1784 :
1785 0 : pub(crate) static WALRECEIVER_SWITCHES: Lazy<IntCounterVec> = Lazy::new(|| {
1786 : register_int_counter_vec!(
1787 : "pageserver_walreceiver_switches_total",
1788 : "Number of walreceiver manager change_connection calls",
1789 : &["reason"]
1790 : )
1791 0 : .expect("failed to define a metric")
1792 0 : });
1793 :
1794 0 : pub(crate) static WALRECEIVER_BROKER_UPDATES: Lazy<IntCounter> = Lazy::new(|| {
1795 : register_int_counter!(
1796 : "pageserver_walreceiver_broker_updates_total",
1797 : "Number of received broker updates in walreceiver"
1798 : )
1799 0 : .expect("failed to define a metric")
1800 0 : });
1801 :
1802 2 : pub(crate) static WALRECEIVER_CANDIDATES_EVENTS: Lazy<IntCounterVec> = Lazy::new(|| {
1803 : register_int_counter_vec!(
1804 : "pageserver_walreceiver_candidates_events_total",
1805 : "Number of walreceiver candidate events",
1806 : &["event"]
1807 : )
1808 2 : .expect("failed to define a metric")
1809 2 : });
1810 :
1811 : pub(crate) static WALRECEIVER_CANDIDATES_ADDED: Lazy<IntCounter> =
1812 0 : Lazy::new(|| WALRECEIVER_CANDIDATES_EVENTS.with_label_values(&["add"]));
1813 :
1814 : pub(crate) static WALRECEIVER_CANDIDATES_REMOVED: Lazy<IntCounter> =
1815 2 : Lazy::new(|| WALRECEIVER_CANDIDATES_EVENTS.with_label_values(&["remove"]));
1816 :
1817 : // Metrics collected on WAL redo operations
1818 : //
1819 : // We collect the time spent in actual WAL redo ('redo'), and time waiting
1820 : // for access to the postgres process ('wait') since there is only one for
1821 : // each tenant.
1822 :
1823 : /// Time buckets are small because we want to be able to measure the
1824 : /// smallest redo processing times. These buckets allow us to measure down
1825 : /// to 5us, which equates to 200'000 pages/sec, which equates to 1.6GB/sec.
1826 : /// This is much better than the previous 5ms aka 200 pages/sec aka 1.6MB/sec.
1827 : ///
1828 : /// Values up to 1s are recorded because metrics show that we have redo
1829 : /// durations and lock times larger than 0.250s.
1830 : macro_rules! redo_histogram_time_buckets {
1831 : () => {
1832 : vec![
1833 : 0.000_005, 0.000_010, 0.000_025, 0.000_050, 0.000_100, 0.000_250, 0.000_500, 0.001_000,
1834 : 0.002_500, 0.005_000, 0.010_000, 0.025_000, 0.050_000, 0.100_000, 0.250_000, 0.500_000,
1835 : 1.000_000,
1836 : ]
1837 : };
1838 : }
1839 :
1840 : /// While we're at it, also measure the amount of records replayed in each
1841 : /// operation. We have a global 'total replayed' counter, but that's not
1842 : /// as useful as 'what is the skew for how many records we replay in one
1843 : /// operation'.
1844 : macro_rules! redo_histogram_count_buckets {
1845 : () => {
1846 : vec![0.0, 1.0, 2.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0]
1847 : };
1848 : }
1849 :
1850 : macro_rules! redo_bytes_histogram_count_buckets {
1851 : () => {
1852 : // powers of (2^.5), from 2^4.5 to 2^15 (22 buckets)
1853 : // rounded up to the next multiple of 8 to capture any MAXALIGNed record of that size, too.
1854 : vec![
1855 : 24.0, 32.0, 48.0, 64.0, 96.0, 128.0, 184.0, 256.0, 368.0, 512.0, 728.0, 1024.0, 1456.0,
1856 : 2048.0, 2904.0, 4096.0, 5800.0, 8192.0, 11592.0, 16384.0, 23176.0, 32768.0,
1857 : ]
1858 : };
1859 : }
1860 :
1861 : pub(crate) struct WalIngestMetrics {
1862 : pub(crate) bytes_received: IntCounter,
1863 : pub(crate) records_received: IntCounter,
1864 : pub(crate) records_committed: IntCounter,
1865 : pub(crate) records_filtered: IntCounter,
1866 : }
1867 :
1868 2 : pub(crate) static WAL_INGEST: Lazy<WalIngestMetrics> = Lazy::new(|| WalIngestMetrics {
1869 : bytes_received: register_int_counter!(
1870 : "pageserver_wal_ingest_bytes_received",
1871 : "Bytes of WAL ingested from safekeepers",
1872 : )
1873 2 : .unwrap(),
1874 : records_received: register_int_counter!(
1875 : "pageserver_wal_ingest_records_received",
1876 : "Number of WAL records received from safekeepers"
1877 : )
1878 2 : .expect("failed to define a metric"),
1879 : records_committed: register_int_counter!(
1880 : "pageserver_wal_ingest_records_committed",
1881 : "Number of WAL records which resulted in writes to pageserver storage"
1882 : )
1883 2 : .expect("failed to define a metric"),
1884 : records_filtered: register_int_counter!(
1885 : "pageserver_wal_ingest_records_filtered",
1886 : "Number of WAL records filtered out due to sharding"
1887 : )
1888 2 : .expect("failed to define a metric"),
1889 2 : });
1890 :
1891 6 : pub(crate) static WAL_REDO_TIME: Lazy<Histogram> = Lazy::new(|| {
1892 : register_histogram!(
1893 : "pageserver_wal_redo_seconds",
1894 : "Time spent on WAL redo",
1895 : redo_histogram_time_buckets!()
1896 : )
1897 6 : .expect("failed to define a metric")
1898 6 : });
1899 :
1900 6 : pub(crate) static WAL_REDO_RECORDS_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
1901 : register_histogram!(
1902 : "pageserver_wal_redo_records_histogram",
1903 : "Histogram of number of records replayed per redo in the Postgres WAL redo process",
1904 : redo_histogram_count_buckets!(),
1905 : )
1906 6 : .expect("failed to define a metric")
1907 6 : });
1908 :
1909 6 : pub(crate) static WAL_REDO_BYTES_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
1910 : register_histogram!(
1911 : "pageserver_wal_redo_bytes_histogram",
1912 : "Histogram of number of records replayed per redo sent to Postgres",
1913 : redo_bytes_histogram_count_buckets!(),
1914 : )
1915 6 : .expect("failed to define a metric")
1916 6 : });
1917 :
1918 : // FIXME: isn't this already included by WAL_REDO_RECORDS_HISTOGRAM which has _count?
1919 6 : pub(crate) static WAL_REDO_RECORD_COUNTER: Lazy<IntCounter> = Lazy::new(|| {
1920 : register_int_counter!(
1921 : "pageserver_replayed_wal_records_total",
1922 : "Number of WAL records replayed in WAL redo process"
1923 : )
1924 6 : .unwrap()
1925 6 : });
1926 :
1927 : #[rustfmt::skip]
1928 6 : pub(crate) static WAL_REDO_PROCESS_LAUNCH_DURATION_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
1929 : register_histogram!(
1930 : "pageserver_wal_redo_process_launch_duration",
1931 : "Histogram of the duration of successful WalRedoProcess::launch calls",
1932 : vec![
1933 : 0.0002, 0.0004, 0.0006, 0.0008, 0.0010,
1934 : 0.0020, 0.0040, 0.0060, 0.0080, 0.0100,
1935 : 0.0200, 0.0400, 0.0600, 0.0800, 0.1000,
1936 : 0.2000, 0.4000, 0.6000, 0.8000, 1.0000,
1937 : 1.5000, 2.0000, 2.5000, 3.0000, 4.0000, 10.0000
1938 : ],
1939 : )
1940 6 : .expect("failed to define a metric")
1941 6 : });
1942 :
1943 : pub(crate) struct WalRedoProcessCounters {
1944 : pub(crate) started: IntCounter,
1945 : pub(crate) killed_by_cause: enum_map::EnumMap<WalRedoKillCause, IntCounter>,
1946 : pub(crate) active_stderr_logger_tasks_started: IntCounter,
1947 : pub(crate) active_stderr_logger_tasks_finished: IntCounter,
1948 : }
1949 :
1950 18 : #[derive(Debug, enum_map::Enum, strum_macros::IntoStaticStr)]
1951 : pub(crate) enum WalRedoKillCause {
1952 : WalRedoProcessDrop,
1953 : NoLeakChildDrop,
1954 : Startup,
1955 : }
1956 :
1957 : impl Default for WalRedoProcessCounters {
1958 6 : fn default() -> Self {
1959 6 : let started = register_int_counter!(
1960 : "pageserver_wal_redo_process_started_total",
1961 : "Number of WAL redo processes started",
1962 : )
1963 6 : .unwrap();
1964 6 :
1965 6 : let killed = register_int_counter_vec!(
1966 : "pageserver_wal_redo_process_stopped_total",
1967 : "Number of WAL redo processes stopped",
1968 : &["cause"],
1969 : )
1970 6 : .unwrap();
1971 6 :
1972 6 : let active_stderr_logger_tasks_started = register_int_counter!(
1973 : "pageserver_walredo_stderr_logger_tasks_started_total",
1974 : "Number of active walredo stderr logger tasks that have started",
1975 : )
1976 6 : .unwrap();
1977 6 :
1978 6 : let active_stderr_logger_tasks_finished = register_int_counter!(
1979 : "pageserver_walredo_stderr_logger_tasks_finished_total",
1980 : "Number of active walredo stderr logger tasks that have finished",
1981 : )
1982 6 : .unwrap();
1983 6 :
1984 6 : Self {
1985 6 : started,
1986 18 : killed_by_cause: EnumMap::from_array(std::array::from_fn(|i| {
1987 18 : let cause = <WalRedoKillCause as enum_map::Enum>::from_usize(i);
1988 18 : let cause_str: &'static str = cause.into();
1989 18 : killed.with_label_values(&[cause_str])
1990 18 : })),
1991 6 : active_stderr_logger_tasks_started,
1992 6 : active_stderr_logger_tasks_finished,
1993 6 : }
1994 6 : }
1995 : }
1996 :
1997 : pub(crate) static WAL_REDO_PROCESS_COUNTERS: Lazy<WalRedoProcessCounters> =
1998 : Lazy::new(WalRedoProcessCounters::default);
1999 :
2000 : /// Similar to `prometheus::HistogramTimer` but does not record on drop.
2001 : pub(crate) struct StorageTimeMetricsTimer {
2002 : metrics: StorageTimeMetrics,
2003 : start: Instant,
2004 : }
2005 :
2006 : impl StorageTimeMetricsTimer {
2007 4602 : fn new(metrics: StorageTimeMetrics) -> Self {
2008 4602 : Self {
2009 4602 : metrics,
2010 4602 : start: Instant::now(),
2011 4602 : }
2012 4602 : }
2013 :
2014 : /// Record the time from creation to now.
2015 3497 : pub fn stop_and_record(self) {
2016 3497 : let duration = self.start.elapsed().as_secs_f64();
2017 3497 : self.metrics.timeline_sum.inc_by(duration);
2018 3497 : self.metrics.timeline_count.inc();
2019 3497 : self.metrics.global_histogram.observe(duration);
2020 3497 : }
2021 :
2022 : /// Turns this timer into a timer, which will always record -- usually this means recording
2023 : /// regardless an early `?` path was taken in a function.
2024 754 : pub(crate) fn record_on_drop(self) -> AlwaysRecordingStorageTimeMetricsTimer {
2025 754 : AlwaysRecordingStorageTimeMetricsTimer(Some(self))
2026 754 : }
2027 : }
2028 :
2029 : pub(crate) struct AlwaysRecordingStorageTimeMetricsTimer(Option<StorageTimeMetricsTimer>);
2030 :
2031 : impl Drop for AlwaysRecordingStorageTimeMetricsTimer {
2032 754 : fn drop(&mut self) {
2033 754 : if let Some(inner) = self.0.take() {
2034 754 : inner.stop_and_record();
2035 754 : }
2036 754 : }
2037 : }
2038 :
2039 : /// Timing facilities for an globally histogrammed metric, which is supported by per tenant and
2040 : /// timeline total sum and count.
2041 : #[derive(Clone, Debug)]
2042 : pub(crate) struct StorageTimeMetrics {
2043 : /// Sum of f64 seconds, per operation, tenant_id and timeline_id
2044 : timeline_sum: Counter,
2045 : /// Number of oeprations, per operation, tenant_id and timeline_id
2046 : timeline_count: IntCounter,
2047 : /// Global histogram having only the "operation" label.
2048 : global_histogram: Histogram,
2049 : }
2050 :
2051 : impl StorageTimeMetrics {
2052 3064 : pub fn new(
2053 3064 : operation: StorageTimeOperation,
2054 3064 : tenant_id: &str,
2055 3064 : shard_id: &str,
2056 3064 : timeline_id: &str,
2057 3064 : ) -> Self {
2058 3064 : let operation: &'static str = operation.into();
2059 3064 :
2060 3064 : let timeline_sum = STORAGE_TIME_SUM_PER_TIMELINE
2061 3064 : .get_metric_with_label_values(&[operation, tenant_id, shard_id, timeline_id])
2062 3064 : .unwrap();
2063 3064 : let timeline_count = STORAGE_TIME_COUNT_PER_TIMELINE
2064 3064 : .get_metric_with_label_values(&[operation, tenant_id, shard_id, timeline_id])
2065 3064 : .unwrap();
2066 3064 : let global_histogram = STORAGE_TIME_GLOBAL
2067 3064 : .get_metric_with_label_values(&[operation])
2068 3064 : .unwrap();
2069 3064 :
2070 3064 : StorageTimeMetrics {
2071 3064 : timeline_sum,
2072 3064 : timeline_count,
2073 3064 : global_histogram,
2074 3064 : }
2075 3064 : }
2076 :
2077 : /// Starts timing a new operation.
2078 : ///
2079 : /// Note: unlike `prometheus::HistogramTimer` the returned timer does not record on drop.
2080 4602 : pub fn start_timer(&self) -> StorageTimeMetricsTimer {
2081 4602 : StorageTimeMetricsTimer::new(self.clone())
2082 4602 : }
2083 : }
2084 :
2085 : #[derive(Debug)]
2086 : pub(crate) struct TimelineMetrics {
2087 : tenant_id: String,
2088 : shard_id: String,
2089 : timeline_id: String,
2090 : pub flush_time_histo: StorageTimeMetrics,
2091 : pub compact_time_histo: StorageTimeMetrics,
2092 : pub create_images_time_histo: StorageTimeMetrics,
2093 : pub logical_size_histo: StorageTimeMetrics,
2094 : pub imitate_logical_size_histo: StorageTimeMetrics,
2095 : pub load_layer_map_histo: StorageTimeMetrics,
2096 : pub garbage_collect_histo: StorageTimeMetrics,
2097 : pub find_gc_cutoffs_histo: StorageTimeMetrics,
2098 : pub last_record_gauge: IntGauge,
2099 : pub standby_horizon_gauge: IntGauge,
2100 : pub resident_physical_size_gauge: UIntGauge,
2101 : /// copy of LayeredTimeline.current_logical_size
2102 : pub current_logical_size_gauge: UIntGauge,
2103 : pub aux_file_size_gauge: IntGauge,
2104 : pub directory_entries_count_gauge: Lazy<UIntGauge, Box<dyn Send + Fn() -> UIntGauge>>,
2105 : pub evictions: IntCounter,
2106 : pub evictions_with_low_residence_duration: std::sync::RwLock<EvictionsWithLowResidenceDuration>,
2107 : /// Number of valid LSN leases.
2108 : pub valid_lsn_lease_count_gauge: UIntGauge,
2109 : shutdown: std::sync::atomic::AtomicBool,
2110 : }
2111 :
2112 : impl TimelineMetrics {
2113 383 : pub fn new(
2114 383 : tenant_shard_id: &TenantShardId,
2115 383 : timeline_id_raw: &TimelineId,
2116 383 : evictions_with_low_residence_duration_builder: EvictionsWithLowResidenceDurationBuilder,
2117 383 : ) -> Self {
2118 383 : let tenant_id = tenant_shard_id.tenant_id.to_string();
2119 383 : let shard_id = format!("{}", tenant_shard_id.shard_slug());
2120 383 : let timeline_id = timeline_id_raw.to_string();
2121 383 : let flush_time_histo = StorageTimeMetrics::new(
2122 383 : StorageTimeOperation::LayerFlush,
2123 383 : &tenant_id,
2124 383 : &shard_id,
2125 383 : &timeline_id,
2126 383 : );
2127 383 : let compact_time_histo = StorageTimeMetrics::new(
2128 383 : StorageTimeOperation::Compact,
2129 383 : &tenant_id,
2130 383 : &shard_id,
2131 383 : &timeline_id,
2132 383 : );
2133 383 : let create_images_time_histo = StorageTimeMetrics::new(
2134 383 : StorageTimeOperation::CreateImages,
2135 383 : &tenant_id,
2136 383 : &shard_id,
2137 383 : &timeline_id,
2138 383 : );
2139 383 : let logical_size_histo = StorageTimeMetrics::new(
2140 383 : StorageTimeOperation::LogicalSize,
2141 383 : &tenant_id,
2142 383 : &shard_id,
2143 383 : &timeline_id,
2144 383 : );
2145 383 : let imitate_logical_size_histo = StorageTimeMetrics::new(
2146 383 : StorageTimeOperation::ImitateLogicalSize,
2147 383 : &tenant_id,
2148 383 : &shard_id,
2149 383 : &timeline_id,
2150 383 : );
2151 383 : let load_layer_map_histo = StorageTimeMetrics::new(
2152 383 : StorageTimeOperation::LoadLayerMap,
2153 383 : &tenant_id,
2154 383 : &shard_id,
2155 383 : &timeline_id,
2156 383 : );
2157 383 : let garbage_collect_histo = StorageTimeMetrics::new(
2158 383 : StorageTimeOperation::Gc,
2159 383 : &tenant_id,
2160 383 : &shard_id,
2161 383 : &timeline_id,
2162 383 : );
2163 383 : let find_gc_cutoffs_histo = StorageTimeMetrics::new(
2164 383 : StorageTimeOperation::FindGcCutoffs,
2165 383 : &tenant_id,
2166 383 : &shard_id,
2167 383 : &timeline_id,
2168 383 : );
2169 383 : let last_record_gauge = LAST_RECORD_LSN
2170 383 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2171 383 : .unwrap();
2172 383 : let standby_horizon_gauge = STANDBY_HORIZON
2173 383 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2174 383 : .unwrap();
2175 383 : let resident_physical_size_gauge = RESIDENT_PHYSICAL_SIZE
2176 383 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2177 383 : .unwrap();
2178 383 : // TODO: we shouldn't expose this metric
2179 383 : let current_logical_size_gauge = CURRENT_LOGICAL_SIZE
2180 383 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2181 383 : .unwrap();
2182 383 : let aux_file_size_gauge = AUX_FILE_SIZE
2183 383 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2184 383 : .unwrap();
2185 383 : // TODO use impl Trait syntax here once we have ability to use it: https://github.com/rust-lang/rust/issues/63065
2186 383 : let directory_entries_count_gauge_closure = {
2187 383 : let tenant_shard_id = *tenant_shard_id;
2188 383 : let timeline_id_raw = *timeline_id_raw;
2189 0 : move || {
2190 0 : let tenant_id = tenant_shard_id.tenant_id.to_string();
2191 0 : let shard_id = format!("{}", tenant_shard_id.shard_slug());
2192 0 : let timeline_id = timeline_id_raw.to_string();
2193 0 : let gauge: UIntGauge = DIRECTORY_ENTRIES_COUNT
2194 0 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2195 0 : .unwrap();
2196 0 : gauge
2197 0 : }
2198 : };
2199 383 : let directory_entries_count_gauge: Lazy<UIntGauge, Box<dyn Send + Fn() -> UIntGauge>> =
2200 383 : Lazy::new(Box::new(directory_entries_count_gauge_closure));
2201 383 : let evictions = EVICTIONS
2202 383 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2203 383 : .unwrap();
2204 383 : let evictions_with_low_residence_duration = evictions_with_low_residence_duration_builder
2205 383 : .build(&tenant_id, &shard_id, &timeline_id);
2206 383 :
2207 383 : let valid_lsn_lease_count_gauge = VALID_LSN_LEASE_COUNT
2208 383 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2209 383 : .unwrap();
2210 383 :
2211 383 : TimelineMetrics {
2212 383 : tenant_id,
2213 383 : shard_id,
2214 383 : timeline_id,
2215 383 : flush_time_histo,
2216 383 : compact_time_histo,
2217 383 : create_images_time_histo,
2218 383 : logical_size_histo,
2219 383 : imitate_logical_size_histo,
2220 383 : garbage_collect_histo,
2221 383 : find_gc_cutoffs_histo,
2222 383 : load_layer_map_histo,
2223 383 : last_record_gauge,
2224 383 : standby_horizon_gauge,
2225 383 : resident_physical_size_gauge,
2226 383 : current_logical_size_gauge,
2227 383 : aux_file_size_gauge,
2228 383 : directory_entries_count_gauge,
2229 383 : evictions,
2230 383 : evictions_with_low_residence_duration: std::sync::RwLock::new(
2231 383 : evictions_with_low_residence_duration,
2232 383 : ),
2233 383 : valid_lsn_lease_count_gauge,
2234 383 : shutdown: std::sync::atomic::AtomicBool::default(),
2235 383 : }
2236 383 : }
2237 :
2238 1468 : pub(crate) fn record_new_file_metrics(&self, sz: u64) {
2239 1468 : self.resident_physical_size_add(sz);
2240 1468 : }
2241 :
2242 449 : pub(crate) fn resident_physical_size_sub(&self, sz: u64) {
2243 449 : self.resident_physical_size_gauge.sub(sz);
2244 449 : crate::metrics::RESIDENT_PHYSICAL_SIZE_GLOBAL.sub(sz);
2245 449 : }
2246 :
2247 1498 : pub(crate) fn resident_physical_size_add(&self, sz: u64) {
2248 1498 : self.resident_physical_size_gauge.add(sz);
2249 1498 : crate::metrics::RESIDENT_PHYSICAL_SIZE_GLOBAL.add(sz);
2250 1498 : }
2251 :
2252 8 : pub(crate) fn resident_physical_size_get(&self) -> u64 {
2253 8 : self.resident_physical_size_gauge.get()
2254 8 : }
2255 :
2256 8 : pub(crate) fn shutdown(&self) {
2257 8 : let was_shutdown = self
2258 8 : .shutdown
2259 8 : .swap(true, std::sync::atomic::Ordering::Relaxed);
2260 8 :
2261 8 : if was_shutdown {
2262 : // this happens on tenant deletion because tenant first shuts down timelines, then
2263 : // invokes timeline deletion which first shuts down the timeline again.
2264 : // TODO: this can be removed once https://github.com/neondatabase/neon/issues/5080
2265 0 : return;
2266 8 : }
2267 8 :
2268 8 : let tenant_id = &self.tenant_id;
2269 8 : let timeline_id = &self.timeline_id;
2270 8 : let shard_id = &self.shard_id;
2271 8 : let _ = LAST_RECORD_LSN.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2272 8 : let _ = STANDBY_HORIZON.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2273 8 : {
2274 8 : RESIDENT_PHYSICAL_SIZE_GLOBAL.sub(self.resident_physical_size_get());
2275 8 : let _ = RESIDENT_PHYSICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2276 8 : }
2277 8 : let _ = CURRENT_LOGICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2278 8 : if let Some(metric) = Lazy::get(&DIRECTORY_ENTRIES_COUNT) {
2279 0 : let _ = metric.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2280 8 : }
2281 8 : let _ = EVICTIONS.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2282 8 : let _ = AUX_FILE_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2283 8 : let _ = VALID_LSN_LEASE_COUNT.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2284 8 :
2285 8 : self.evictions_with_low_residence_duration
2286 8 : .write()
2287 8 : .unwrap()
2288 8 : .remove(tenant_id, shard_id, timeline_id);
2289 :
2290 : // The following metrics are born outside of the TimelineMetrics lifecycle but still
2291 : // removed at the end of it. The idea is to have the metrics outlive the
2292 : // entity during which they're observed, e.g., the smgr metrics shall
2293 : // outlive an individual smgr connection, but not the timeline.
2294 :
2295 80 : for op in StorageTimeOperation::VARIANTS {
2296 72 : let _ = STORAGE_TIME_SUM_PER_TIMELINE.remove_label_values(&[
2297 72 : op,
2298 72 : tenant_id,
2299 72 : shard_id,
2300 72 : timeline_id,
2301 72 : ]);
2302 72 : let _ = STORAGE_TIME_COUNT_PER_TIMELINE.remove_label_values(&[
2303 72 : op,
2304 72 : tenant_id,
2305 72 : shard_id,
2306 72 : timeline_id,
2307 72 : ]);
2308 72 : }
2309 :
2310 24 : for op in STORAGE_IO_SIZE_OPERATIONS {
2311 16 : let _ = STORAGE_IO_SIZE.remove_label_values(&[op, tenant_id, shard_id, timeline_id]);
2312 16 : }
2313 :
2314 48 : for op in SmgrQueryType::iter() {
2315 40 : let _ = SMGR_QUERY_TIME_PER_TENANT_TIMELINE.remove_label_values(&[
2316 40 : op.into(),
2317 40 : tenant_id,
2318 40 : shard_id,
2319 40 : timeline_id,
2320 40 : ]);
2321 40 : }
2322 8 : }
2323 : }
2324 :
2325 6 : pub(crate) fn remove_tenant_metrics(tenant_shard_id: &TenantShardId) {
2326 6 : // Only shard zero deals in synthetic sizes
2327 6 : if tenant_shard_id.is_shard_zero() {
2328 6 : let tid = tenant_shard_id.tenant_id.to_string();
2329 6 : let _ = TENANT_SYNTHETIC_SIZE_METRIC.remove_label_values(&[&tid]);
2330 6 : }
2331 :
2332 : // we leave the BROKEN_TENANTS_SET entry if any
2333 6 : }
2334 :
2335 : use futures::Future;
2336 : use pin_project_lite::pin_project;
2337 : use std::collections::HashMap;
2338 : use std::num::NonZeroUsize;
2339 : use std::pin::Pin;
2340 : use std::sync::atomic::AtomicU64;
2341 : use std::sync::{Arc, Mutex};
2342 : use std::task::{Context, Poll};
2343 : use std::time::{Duration, Instant};
2344 :
2345 : use crate::context::{PageContentKind, RequestContext};
2346 : use crate::task_mgr::TaskKind;
2347 : use crate::tenant::mgr::TenantSlot;
2348 :
2349 : /// Maintain a per timeline gauge in addition to the global gauge.
2350 : pub(crate) struct PerTimelineRemotePhysicalSizeGauge {
2351 : last_set: AtomicU64,
2352 : gauge: UIntGauge,
2353 : }
2354 :
2355 : impl PerTimelineRemotePhysicalSizeGauge {
2356 393 : fn new(per_timeline_gauge: UIntGauge) -> Self {
2357 393 : Self {
2358 393 : last_set: AtomicU64::new(0),
2359 393 : gauge: per_timeline_gauge,
2360 393 : }
2361 393 : }
2362 1734 : pub(crate) fn set(&self, sz: u64) {
2363 1734 : self.gauge.set(sz);
2364 1734 : let prev = self.last_set.swap(sz, std::sync::atomic::Ordering::Relaxed);
2365 1734 : if sz < prev {
2366 26 : REMOTE_PHYSICAL_SIZE_GLOBAL.sub(prev - sz);
2367 1708 : } else {
2368 1708 : REMOTE_PHYSICAL_SIZE_GLOBAL.add(sz - prev);
2369 1708 : };
2370 1734 : }
2371 2 : pub(crate) fn get(&self) -> u64 {
2372 2 : self.gauge.get()
2373 2 : }
2374 : }
2375 :
2376 : impl Drop for PerTimelineRemotePhysicalSizeGauge {
2377 19 : fn drop(&mut self) {
2378 19 : REMOTE_PHYSICAL_SIZE_GLOBAL.sub(self.last_set.load(std::sync::atomic::Ordering::Relaxed));
2379 19 : }
2380 : }
2381 :
2382 : pub(crate) struct RemoteTimelineClientMetrics {
2383 : tenant_id: String,
2384 : shard_id: String,
2385 : timeline_id: String,
2386 : pub(crate) remote_physical_size_gauge: PerTimelineRemotePhysicalSizeGauge,
2387 : calls: Mutex<HashMap<(&'static str, &'static str), IntCounterPair>>,
2388 : bytes_started_counter: Mutex<HashMap<(&'static str, &'static str), IntCounter>>,
2389 : bytes_finished_counter: Mutex<HashMap<(&'static str, &'static str), IntCounter>>,
2390 : }
2391 :
2392 : impl RemoteTimelineClientMetrics {
2393 393 : pub fn new(tenant_shard_id: &TenantShardId, timeline_id: &TimelineId) -> Self {
2394 393 : let tenant_id_str = tenant_shard_id.tenant_id.to_string();
2395 393 : let shard_id_str = format!("{}", tenant_shard_id.shard_slug());
2396 393 : let timeline_id_str = timeline_id.to_string();
2397 393 :
2398 393 : let remote_physical_size_gauge = PerTimelineRemotePhysicalSizeGauge::new(
2399 393 : REMOTE_PHYSICAL_SIZE
2400 393 : .get_metric_with_label_values(&[&tenant_id_str, &shard_id_str, &timeline_id_str])
2401 393 : .unwrap(),
2402 393 : );
2403 393 :
2404 393 : RemoteTimelineClientMetrics {
2405 393 : tenant_id: tenant_id_str,
2406 393 : shard_id: shard_id_str,
2407 393 : timeline_id: timeline_id_str,
2408 393 : calls: Mutex::new(HashMap::default()),
2409 393 : bytes_started_counter: Mutex::new(HashMap::default()),
2410 393 : bytes_finished_counter: Mutex::new(HashMap::default()),
2411 393 : remote_physical_size_gauge,
2412 393 : }
2413 393 : }
2414 :
2415 2554 : pub fn remote_operation_time(
2416 2554 : &self,
2417 2554 : file_kind: &RemoteOpFileKind,
2418 2554 : op_kind: &RemoteOpKind,
2419 2554 : status: &'static str,
2420 2554 : ) -> Histogram {
2421 2554 : let key = (file_kind.as_str(), op_kind.as_str(), status);
2422 2554 : REMOTE_OPERATION_TIME
2423 2554 : .get_metric_with_label_values(&[key.0, key.1, key.2])
2424 2554 : .unwrap()
2425 2554 : }
2426 :
2427 6099 : fn calls_counter_pair(
2428 6099 : &self,
2429 6099 : file_kind: &RemoteOpFileKind,
2430 6099 : op_kind: &RemoteOpKind,
2431 6099 : ) -> IntCounterPair {
2432 6099 : let mut guard = self.calls.lock().unwrap();
2433 6099 : let key = (file_kind.as_str(), op_kind.as_str());
2434 6099 : let metric = guard.entry(key).or_insert_with(move || {
2435 678 : REMOTE_TIMELINE_CLIENT_CALLS
2436 678 : .get_metric_with_label_values(&[
2437 678 : &self.tenant_id,
2438 678 : &self.shard_id,
2439 678 : &self.timeline_id,
2440 678 : key.0,
2441 678 : key.1,
2442 678 : ])
2443 678 : .unwrap()
2444 6099 : });
2445 6099 : metric.clone()
2446 6099 : }
2447 :
2448 1476 : fn bytes_started_counter(
2449 1476 : &self,
2450 1476 : file_kind: &RemoteOpFileKind,
2451 1476 : op_kind: &RemoteOpKind,
2452 1476 : ) -> IntCounter {
2453 1476 : let mut guard = self.bytes_started_counter.lock().unwrap();
2454 1476 : let key = (file_kind.as_str(), op_kind.as_str());
2455 1476 : let metric = guard.entry(key).or_insert_with(move || {
2456 256 : REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER
2457 256 : .get_metric_with_label_values(&[
2458 256 : &self.tenant_id,
2459 256 : &self.shard_id,
2460 256 : &self.timeline_id,
2461 256 : key.0,
2462 256 : key.1,
2463 256 : ])
2464 256 : .unwrap()
2465 1476 : });
2466 1476 : metric.clone()
2467 1476 : }
2468 :
2469 2653 : fn bytes_finished_counter(
2470 2653 : &self,
2471 2653 : file_kind: &RemoteOpFileKind,
2472 2653 : op_kind: &RemoteOpKind,
2473 2653 : ) -> IntCounter {
2474 2653 : let mut guard = self.bytes_finished_counter.lock().unwrap();
2475 2653 : let key = (file_kind.as_str(), op_kind.as_str());
2476 2653 : let metric = guard.entry(key).or_insert_with(move || {
2477 256 : REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER
2478 256 : .get_metric_with_label_values(&[
2479 256 : &self.tenant_id,
2480 256 : &self.shard_id,
2481 256 : &self.timeline_id,
2482 256 : key.0,
2483 256 : key.1,
2484 256 : ])
2485 256 : .unwrap()
2486 2653 : });
2487 2653 : metric.clone()
2488 2653 : }
2489 : }
2490 :
2491 : #[cfg(test)]
2492 : impl RemoteTimelineClientMetrics {
2493 6 : pub fn get_bytes_started_counter_value(
2494 6 : &self,
2495 6 : file_kind: &RemoteOpFileKind,
2496 6 : op_kind: &RemoteOpKind,
2497 6 : ) -> Option<u64> {
2498 6 : let guard = self.bytes_started_counter.lock().unwrap();
2499 6 : let key = (file_kind.as_str(), op_kind.as_str());
2500 6 : guard.get(&key).map(|counter| counter.get())
2501 6 : }
2502 :
2503 6 : pub fn get_bytes_finished_counter_value(
2504 6 : &self,
2505 6 : file_kind: &RemoteOpFileKind,
2506 6 : op_kind: &RemoteOpKind,
2507 6 : ) -> Option<u64> {
2508 6 : let guard = self.bytes_finished_counter.lock().unwrap();
2509 6 : let key = (file_kind.as_str(), op_kind.as_str());
2510 6 : guard.get(&key).map(|counter| counter.get())
2511 6 : }
2512 : }
2513 :
2514 : /// See [`RemoteTimelineClientMetrics::call_begin`].
2515 : #[must_use]
2516 : pub(crate) struct RemoteTimelineClientCallMetricGuard {
2517 : /// Decremented on drop.
2518 : calls_counter_pair: Option<IntCounterPair>,
2519 : /// If Some(), this references the bytes_finished metric, and we increment it by the given `u64` on drop.
2520 : bytes_finished: Option<(IntCounter, u64)>,
2521 : }
2522 :
2523 : impl RemoteTimelineClientCallMetricGuard {
2524 : /// Consume this guard object without performing the metric updates it would do on `drop()`.
2525 : /// The caller vouches to do the metric updates manually.
2526 3325 : pub fn will_decrement_manually(mut self) {
2527 3325 : let RemoteTimelineClientCallMetricGuard {
2528 3325 : calls_counter_pair,
2529 3325 : bytes_finished,
2530 3325 : } = &mut self;
2531 3325 : calls_counter_pair.take();
2532 3325 : bytes_finished.take();
2533 3325 : }
2534 : }
2535 :
2536 : impl Drop for RemoteTimelineClientCallMetricGuard {
2537 3351 : fn drop(&mut self) {
2538 3351 : let RemoteTimelineClientCallMetricGuard {
2539 3351 : calls_counter_pair,
2540 3351 : bytes_finished,
2541 3351 : } = self;
2542 3351 : if let Some(guard) = calls_counter_pair.take() {
2543 26 : guard.dec();
2544 3325 : }
2545 3351 : if let Some((bytes_finished_metric, value)) = bytes_finished {
2546 0 : bytes_finished_metric.inc_by(*value);
2547 3351 : }
2548 3351 : }
2549 : }
2550 :
2551 : /// The enum variants communicate to the [`RemoteTimelineClientMetrics`] whether to
2552 : /// track the byte size of this call in applicable metric(s).
2553 : pub(crate) enum RemoteTimelineClientMetricsCallTrackSize {
2554 : /// Do not account for this call's byte size in any metrics.
2555 : /// The `reason` field is there to make the call sites self-documenting
2556 : /// about why they don't need the metric.
2557 : DontTrackSize { reason: &'static str },
2558 : /// Track the byte size of the call in applicable metric(s).
2559 : Bytes(u64),
2560 : }
2561 :
2562 : impl RemoteTimelineClientMetrics {
2563 : /// Update the metrics that change when a call to the remote timeline client instance starts.
2564 : ///
2565 : /// Drop the returned guard object once the operation is finished to updates corresponding metrics that track completions.
2566 : /// Or, use [`RemoteTimelineClientCallMetricGuard::will_decrement_manually`] and [`call_end`](Self::call_end) if that
2567 : /// is more suitable.
2568 : /// Never do both.
2569 3351 : pub(crate) fn call_begin(
2570 3351 : &self,
2571 3351 : file_kind: &RemoteOpFileKind,
2572 3351 : op_kind: &RemoteOpKind,
2573 3351 : size: RemoteTimelineClientMetricsCallTrackSize,
2574 3351 : ) -> RemoteTimelineClientCallMetricGuard {
2575 3351 : let calls_counter_pair = self.calls_counter_pair(file_kind, op_kind);
2576 3351 : calls_counter_pair.inc();
2577 :
2578 3351 : let bytes_finished = match size {
2579 1875 : RemoteTimelineClientMetricsCallTrackSize::DontTrackSize { reason: _reason } => {
2580 1875 : // nothing to do
2581 1875 : None
2582 : }
2583 1476 : RemoteTimelineClientMetricsCallTrackSize::Bytes(size) => {
2584 1476 : self.bytes_started_counter(file_kind, op_kind).inc_by(size);
2585 1476 : let finished_counter = self.bytes_finished_counter(file_kind, op_kind);
2586 1476 : Some((finished_counter, size))
2587 : }
2588 : };
2589 3351 : RemoteTimelineClientCallMetricGuard {
2590 3351 : calls_counter_pair: Some(calls_counter_pair),
2591 3351 : bytes_finished,
2592 3351 : }
2593 3351 : }
2594 :
2595 : /// Manually udpate the metrics that track completions, instead of using the guard object.
2596 : /// Using the guard object is generally preferable.
2597 : /// See [`call_begin`](Self::call_begin) for more context.
2598 2748 : pub(crate) fn call_end(
2599 2748 : &self,
2600 2748 : file_kind: &RemoteOpFileKind,
2601 2748 : op_kind: &RemoteOpKind,
2602 2748 : size: RemoteTimelineClientMetricsCallTrackSize,
2603 2748 : ) {
2604 2748 : let calls_counter_pair = self.calls_counter_pair(file_kind, op_kind);
2605 2748 : calls_counter_pair.dec();
2606 2748 : match size {
2607 1571 : RemoteTimelineClientMetricsCallTrackSize::DontTrackSize { reason: _reason } => {}
2608 1177 : RemoteTimelineClientMetricsCallTrackSize::Bytes(size) => {
2609 1177 : self.bytes_finished_counter(file_kind, op_kind).inc_by(size);
2610 1177 : }
2611 : }
2612 2748 : }
2613 : }
2614 :
2615 : impl Drop for RemoteTimelineClientMetrics {
2616 19 : fn drop(&mut self) {
2617 19 : let RemoteTimelineClientMetrics {
2618 19 : tenant_id,
2619 19 : shard_id,
2620 19 : timeline_id,
2621 19 : remote_physical_size_gauge,
2622 19 : calls,
2623 19 : bytes_started_counter,
2624 19 : bytes_finished_counter,
2625 19 : } = self;
2626 24 : for ((a, b), _) in calls.get_mut().unwrap().drain() {
2627 24 : let mut res = [Ok(()), Ok(())];
2628 24 : REMOTE_TIMELINE_CLIENT_CALLS
2629 24 : .remove_label_values(&mut res, &[tenant_id, shard_id, timeline_id, a, b]);
2630 24 : // don't care about results
2631 24 : }
2632 19 : for ((a, b), _) in bytes_started_counter.get_mut().unwrap().drain() {
2633 6 : let _ = REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER.remove_label_values(&[
2634 6 : tenant_id,
2635 6 : shard_id,
2636 6 : timeline_id,
2637 6 : a,
2638 6 : b,
2639 6 : ]);
2640 6 : }
2641 19 : for ((a, b), _) in bytes_finished_counter.get_mut().unwrap().drain() {
2642 6 : let _ = REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER.remove_label_values(&[
2643 6 : tenant_id,
2644 6 : shard_id,
2645 6 : timeline_id,
2646 6 : a,
2647 6 : b,
2648 6 : ]);
2649 6 : }
2650 19 : {
2651 19 : let _ = remote_physical_size_gauge; // use to avoid 'unused' warning in desctructuring above
2652 19 : let _ = REMOTE_PHYSICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2653 19 : }
2654 19 : }
2655 : }
2656 :
2657 : /// Wrapper future that measures the time spent by a remote storage operation,
2658 : /// and records the time and success/failure as a prometheus metric.
2659 : pub(crate) trait MeasureRemoteOp: Sized {
2660 2714 : fn measure_remote_op(
2661 2714 : self,
2662 2714 : file_kind: RemoteOpFileKind,
2663 2714 : op: RemoteOpKind,
2664 2714 : metrics: Arc<RemoteTimelineClientMetrics>,
2665 2714 : ) -> MeasuredRemoteOp<Self> {
2666 2714 : let start = Instant::now();
2667 2714 : MeasuredRemoteOp {
2668 2714 : inner: self,
2669 2714 : file_kind,
2670 2714 : op,
2671 2714 : start,
2672 2714 : metrics,
2673 2714 : }
2674 2714 : }
2675 : }
2676 :
2677 : impl<T: Sized> MeasureRemoteOp for T {}
2678 :
2679 : pin_project! {
2680 : pub(crate) struct MeasuredRemoteOp<F>
2681 : {
2682 : #[pin]
2683 : inner: F,
2684 : file_kind: RemoteOpFileKind,
2685 : op: RemoteOpKind,
2686 : start: Instant,
2687 : metrics: Arc<RemoteTimelineClientMetrics>,
2688 : }
2689 : }
2690 :
2691 : impl<F: Future<Output = Result<O, E>>, O, E> Future for MeasuredRemoteOp<F> {
2692 : type Output = Result<O, E>;
2693 :
2694 40306 : fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
2695 40306 : let this = self.project();
2696 40306 : let poll_result = this.inner.poll(cx);
2697 40306 : if let Poll::Ready(ref res) = poll_result {
2698 2554 : let duration = this.start.elapsed();
2699 2554 : let status = if res.is_ok() { &"success" } else { &"failure" };
2700 2554 : this.metrics
2701 2554 : .remote_operation_time(this.file_kind, this.op, status)
2702 2554 : .observe(duration.as_secs_f64());
2703 37752 : }
2704 40306 : poll_result
2705 40306 : }
2706 : }
2707 :
2708 : pub mod tokio_epoll_uring {
2709 : use metrics::{register_int_counter, UIntGauge};
2710 : use once_cell::sync::Lazy;
2711 :
2712 : pub struct Collector {
2713 : descs: Vec<metrics::core::Desc>,
2714 : systems_created: UIntGauge,
2715 : systems_destroyed: UIntGauge,
2716 : }
2717 :
2718 : impl metrics::core::Collector for Collector {
2719 0 : fn desc(&self) -> Vec<&metrics::core::Desc> {
2720 0 : self.descs.iter().collect()
2721 0 : }
2722 :
2723 0 : fn collect(&self) -> Vec<metrics::proto::MetricFamily> {
2724 0 : let mut mfs = Vec::with_capacity(Self::NMETRICS);
2725 0 : let tokio_epoll_uring::metrics::Metrics {
2726 0 : systems_created,
2727 0 : systems_destroyed,
2728 0 : } = tokio_epoll_uring::metrics::global();
2729 0 : self.systems_created.set(systems_created);
2730 0 : mfs.extend(self.systems_created.collect());
2731 0 : self.systems_destroyed.set(systems_destroyed);
2732 0 : mfs.extend(self.systems_destroyed.collect());
2733 0 : mfs
2734 0 : }
2735 : }
2736 :
2737 : impl Collector {
2738 : const NMETRICS: usize = 2;
2739 :
2740 : #[allow(clippy::new_without_default)]
2741 0 : pub fn new() -> Self {
2742 0 : let mut descs = Vec::new();
2743 0 :
2744 0 : let systems_created = UIntGauge::new(
2745 0 : "pageserver_tokio_epoll_uring_systems_created",
2746 0 : "counter of tokio-epoll-uring systems that were created",
2747 0 : )
2748 0 : .unwrap();
2749 0 : descs.extend(
2750 0 : metrics::core::Collector::desc(&systems_created)
2751 0 : .into_iter()
2752 0 : .cloned(),
2753 0 : );
2754 0 :
2755 0 : let systems_destroyed = UIntGauge::new(
2756 0 : "pageserver_tokio_epoll_uring_systems_destroyed",
2757 0 : "counter of tokio-epoll-uring systems that were destroyed",
2758 0 : )
2759 0 : .unwrap();
2760 0 : descs.extend(
2761 0 : metrics::core::Collector::desc(&systems_destroyed)
2762 0 : .into_iter()
2763 0 : .cloned(),
2764 0 : );
2765 0 :
2766 0 : Self {
2767 0 : descs,
2768 0 : systems_created,
2769 0 : systems_destroyed,
2770 0 : }
2771 0 : }
2772 : }
2773 :
2774 81 : pub(crate) static THREAD_LOCAL_LAUNCH_SUCCESSES: Lazy<metrics::IntCounter> = Lazy::new(|| {
2775 : register_int_counter!(
2776 : "pageserver_tokio_epoll_uring_pageserver_thread_local_launch_success_count",
2777 : "Number of times where thread_local_system creation spanned multiple executor threads",
2778 : )
2779 81 : .unwrap()
2780 81 : });
2781 :
2782 0 : pub(crate) static THREAD_LOCAL_LAUNCH_FAILURES: Lazy<metrics::IntCounter> = Lazy::new(|| {
2783 : register_int_counter!(
2784 : "pageserver_tokio_epoll_uring_pageserver_thread_local_launch_failures_count",
2785 : "Number of times thread_local_system creation failed and was retried after back-off.",
2786 : )
2787 0 : .unwrap()
2788 0 : });
2789 : }
2790 :
2791 : pub(crate) mod tenant_throttling {
2792 : use metrics::{register_int_counter_vec, IntCounter};
2793 : use once_cell::sync::Lazy;
2794 :
2795 : use crate::tenant::{self, throttle::Metric};
2796 :
2797 : pub(crate) struct TimelineGet {
2798 : wait_time: IntCounter,
2799 : count: IntCounter,
2800 : }
2801 :
2802 143 : pub(crate) static TIMELINE_GET: Lazy<TimelineGet> = Lazy::new(|| {
2803 143 : static WAIT_USECS: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
2804 143 : register_int_counter_vec!(
2805 143 : "pageserver_tenant_throttling_wait_usecs_sum_global",
2806 143 : "Sum of microseconds that tenants spent waiting for a tenant throttle of a given kind.",
2807 143 : &["kind"]
2808 143 : )
2809 143 : .unwrap()
2810 143 : });
2811 143 :
2812 143 : static WAIT_COUNT: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
2813 143 : register_int_counter_vec!(
2814 143 : "pageserver_tenant_throttling_count_global",
2815 143 : "Count of tenant throttlings, by kind of throttle.",
2816 143 : &["kind"]
2817 143 : )
2818 143 : .unwrap()
2819 143 : });
2820 143 :
2821 143 : let kind = "timeline_get";
2822 143 : TimelineGet {
2823 143 : wait_time: WAIT_USECS.with_label_values(&[kind]),
2824 143 : count: WAIT_COUNT.with_label_values(&[kind]),
2825 143 : }
2826 143 : });
2827 :
2828 : impl Metric for &'static TimelineGet {
2829 : #[inline(always)]
2830 0 : fn observe_throttling(
2831 0 : &self,
2832 0 : tenant::throttle::Observation { wait_time }: &tenant::throttle::Observation,
2833 0 : ) {
2834 0 : let val = u64::try_from(wait_time.as_micros()).unwrap();
2835 0 : self.wait_time.inc_by(val);
2836 0 : self.count.inc();
2837 0 : }
2838 : }
2839 : }
2840 :
2841 : pub(crate) mod disk_usage_based_eviction {
2842 : use super::*;
2843 :
2844 : pub(crate) struct Metrics {
2845 : pub(crate) tenant_collection_time: Histogram,
2846 : pub(crate) tenant_layer_count: Histogram,
2847 : pub(crate) layers_collected: IntCounter,
2848 : pub(crate) layers_selected: IntCounter,
2849 : pub(crate) layers_evicted: IntCounter,
2850 : }
2851 :
2852 : impl Default for Metrics {
2853 0 : fn default() -> Self {
2854 0 : let tenant_collection_time = register_histogram!(
2855 : "pageserver_disk_usage_based_eviction_tenant_collection_seconds",
2856 : "Time spent collecting layers from a tenant -- not normalized by collected layer amount",
2857 : vec![0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0, 10.0]
2858 : )
2859 0 : .unwrap();
2860 0 :
2861 0 : let tenant_layer_count = register_histogram!(
2862 : "pageserver_disk_usage_based_eviction_tenant_collected_layers",
2863 : "Amount of layers gathered from a tenant",
2864 : vec![5.0, 50.0, 500.0, 5000.0, 50000.0]
2865 : )
2866 0 : .unwrap();
2867 0 :
2868 0 : let layers_collected = register_int_counter!(
2869 : "pageserver_disk_usage_based_eviction_collected_layers_total",
2870 : "Amount of layers collected"
2871 : )
2872 0 : .unwrap();
2873 0 :
2874 0 : let layers_selected = register_int_counter!(
2875 : "pageserver_disk_usage_based_eviction_select_layers_total",
2876 : "Amount of layers selected"
2877 : )
2878 0 : .unwrap();
2879 0 :
2880 0 : let layers_evicted = register_int_counter!(
2881 : "pageserver_disk_usage_based_eviction_evicted_layers_total",
2882 : "Amount of layers successfully evicted"
2883 : )
2884 0 : .unwrap();
2885 0 :
2886 0 : Self {
2887 0 : tenant_collection_time,
2888 0 : tenant_layer_count,
2889 0 : layers_collected,
2890 0 : layers_selected,
2891 0 : layers_evicted,
2892 0 : }
2893 0 : }
2894 : }
2895 :
2896 : pub(crate) static METRICS: Lazy<Metrics> = Lazy::new(Metrics::default);
2897 : }
2898 :
2899 136 : static TOKIO_EXECUTOR_THREAD_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
2900 : register_uint_gauge_vec!(
2901 : "pageserver_tokio_executor_thread_configured_count",
2902 : "Total number of configued tokio executor threads in the process.
2903 : The `setup` label denotes whether we're running with multiple runtimes or a single runtime.",
2904 : &["setup"],
2905 : )
2906 136 : .unwrap()
2907 136 : });
2908 :
2909 136 : pub(crate) fn set_tokio_runtime_setup(setup: &str, num_threads: NonZeroUsize) {
2910 136 : static SERIALIZE: std::sync::Mutex<()> = std::sync::Mutex::new(());
2911 136 : let _guard = SERIALIZE.lock().unwrap();
2912 136 : TOKIO_EXECUTOR_THREAD_COUNT.reset();
2913 136 : TOKIO_EXECUTOR_THREAD_COUNT
2914 136 : .get_metric_with_label_values(&[setup])
2915 136 : .unwrap()
2916 136 : .set(u64::try_from(num_threads.get()).unwrap());
2917 136 : }
2918 :
2919 0 : pub fn preinitialize_metrics() {
2920 0 : // Python tests need these and on some we do alerting.
2921 0 : //
2922 0 : // FIXME(4813): make it so that we have no top level metrics as this fn will easily fall out of
2923 0 : // order:
2924 0 : // - global metrics reside in a Lazy<PageserverMetrics>
2925 0 : // - access via crate::metrics::PS_METRICS.some_metric.inc()
2926 0 : // - could move the statics into TimelineMetrics::new()?
2927 0 :
2928 0 : // counters
2929 0 : [
2930 0 : &UNEXPECTED_ONDEMAND_DOWNLOADS,
2931 0 : &WALRECEIVER_STARTED_CONNECTIONS,
2932 0 : &WALRECEIVER_BROKER_UPDATES,
2933 0 : &WALRECEIVER_CANDIDATES_ADDED,
2934 0 : &WALRECEIVER_CANDIDATES_REMOVED,
2935 0 : &tokio_epoll_uring::THREAD_LOCAL_LAUNCH_FAILURES,
2936 0 : &tokio_epoll_uring::THREAD_LOCAL_LAUNCH_SUCCESSES,
2937 0 : &REMOTE_ONDEMAND_DOWNLOADED_LAYERS,
2938 0 : &REMOTE_ONDEMAND_DOWNLOADED_BYTES,
2939 0 : ]
2940 0 : .into_iter()
2941 0 : .for_each(|c| {
2942 0 : Lazy::force(c);
2943 0 : });
2944 0 :
2945 0 : // Deletion queue stats
2946 0 : Lazy::force(&DELETION_QUEUE);
2947 0 :
2948 0 : // Tenant stats
2949 0 : Lazy::force(&TENANT);
2950 0 :
2951 0 : // Tenant manager stats
2952 0 : Lazy::force(&TENANT_MANAGER);
2953 0 :
2954 0 : Lazy::force(&crate::tenant::storage_layer::layer::LAYER_IMPL_METRICS);
2955 0 : Lazy::force(&disk_usage_based_eviction::METRICS);
2956 :
2957 0 : for state_name in pageserver_api::models::TenantState::VARIANTS {
2958 0 : // initialize the metric for all gauges, otherwise the time series might seemingly show
2959 0 : // values from last restart.
2960 0 : TENANT_STATE_METRIC.with_label_values(&[state_name]).set(0);
2961 0 : }
2962 :
2963 : // countervecs
2964 0 : [&BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT]
2965 0 : .into_iter()
2966 0 : .for_each(|c| {
2967 0 : Lazy::force(c);
2968 0 : });
2969 0 :
2970 0 : // gauges
2971 0 : WALRECEIVER_ACTIVE_MANAGERS.get();
2972 0 :
2973 0 : // histograms
2974 0 : [
2975 0 : &READ_NUM_LAYERS_VISITED,
2976 0 : &VEC_READ_NUM_LAYERS_VISITED,
2977 0 : &WAIT_LSN_TIME,
2978 0 : &WAL_REDO_TIME,
2979 0 : &WAL_REDO_RECORDS_HISTOGRAM,
2980 0 : &WAL_REDO_BYTES_HISTOGRAM,
2981 0 : &WAL_REDO_PROCESS_LAUNCH_DURATION_HISTOGRAM,
2982 0 : ]
2983 0 : .into_iter()
2984 0 : .for_each(|h| {
2985 0 : Lazy::force(h);
2986 0 : });
2987 0 :
2988 0 : // Custom
2989 0 : Lazy::force(&RECONSTRUCT_TIME);
2990 0 : Lazy::force(&tenant_throttling::TIMELINE_GET);
2991 0 : Lazy::force(&BASEBACKUP_QUERY_TIME);
2992 0 : Lazy::force(&COMPUTE_COMMANDS_COUNTERS);
2993 0 : }
|
