Line data Source code
1 : use enum_map::EnumMap;
2 : use metrics::{
3 : register_counter_vec, register_gauge_vec, register_histogram, register_histogram_vec,
4 : register_int_counter, register_int_counter_pair_vec, register_int_counter_vec,
5 : register_int_gauge, register_int_gauge_vec, register_uint_gauge, register_uint_gauge_vec,
6 : Counter, CounterVec, GaugeVec, Histogram, HistogramVec, IntCounter, IntCounterPair,
7 : IntCounterPairVec, IntCounterVec, IntGauge, IntGaugeVec, UIntGauge, UIntGaugeVec,
8 : };
9 : use once_cell::sync::Lazy;
10 : use pageserver_api::shard::TenantShardId;
11 : use strum::{EnumCount, IntoEnumIterator, VariantNames};
12 : use strum_macros::{EnumVariantNames, IntoStaticStr};
13 : use tracing::warn;
14 : use utils::id::TimelineId;
15 :
16 : /// Prometheus histogram buckets (in seconds) for operations in the critical
17 : /// path. In other words, operations that directly affect that latency of user
18 : /// queries.
19 : ///
20 : /// The buckets capture the majority of latencies in the microsecond and
21 : /// millisecond range but also extend far enough up to distinguish "bad" from
22 : /// "really bad".
23 : const CRITICAL_OP_BUCKETS: &[f64] = &[
24 : 0.000_001, 0.000_010, 0.000_100, // 1 us, 10 us, 100 us
25 : 0.001_000, 0.010_000, 0.100_000, // 1 ms, 10 ms, 100 ms
26 : 1.0, 10.0, 100.0, // 1 s, 10 s, 100 s
27 : ];
28 :
29 : // Metrics collected on operations on the storage repository.
30 3032 : #[derive(Debug, EnumVariantNames, IntoStaticStr)]
31 : #[strum(serialize_all = "kebab_case")]
32 : pub(crate) enum StorageTimeOperation {
33 : #[strum(serialize = "layer flush")]
34 : LayerFlush,
35 :
36 : #[strum(serialize = "compact")]
37 : Compact,
38 :
39 : #[strum(serialize = "create images")]
40 : CreateImages,
41 :
42 : #[strum(serialize = "logical size")]
43 : LogicalSize,
44 :
45 : #[strum(serialize = "imitate logical size")]
46 : ImitateLogicalSize,
47 :
48 : #[strum(serialize = "load layer map")]
49 : LoadLayerMap,
50 :
51 : #[strum(serialize = "gc")]
52 : Gc,
53 :
54 : #[strum(serialize = "find gc cutoffs")]
55 : FindGcCutoffs,
56 :
57 : #[strum(serialize = "create tenant")]
58 : CreateTenant,
59 : }
60 :
61 137 : pub(crate) static STORAGE_TIME_SUM_PER_TIMELINE: Lazy<CounterVec> = Lazy::new(|| {
62 : register_counter_vec!(
63 : "pageserver_storage_operations_seconds_sum",
64 : "Total time spent on storage operations with operation, tenant and timeline dimensions",
65 : &["operation", "tenant_id", "shard_id", "timeline_id"],
66 : )
67 137 : .expect("failed to define a metric")
68 137 : });
69 :
70 137 : pub(crate) static STORAGE_TIME_COUNT_PER_TIMELINE: Lazy<IntCounterVec> = Lazy::new(|| {
71 : register_int_counter_vec!(
72 : "pageserver_storage_operations_seconds_count",
73 : "Count of storage operations with operation, tenant and timeline dimensions",
74 : &["operation", "tenant_id", "shard_id", "timeline_id"],
75 : )
76 137 : .expect("failed to define a metric")
77 137 : });
78 :
79 : // Buckets for background operations like compaction, GC, size calculation
80 : const STORAGE_OP_BUCKETS: &[f64] = &[0.010, 0.100, 1.0, 10.0, 100.0, 1000.0];
81 :
82 137 : pub(crate) static STORAGE_TIME_GLOBAL: Lazy<HistogramVec> = Lazy::new(|| {
83 : register_histogram_vec!(
84 : "pageserver_storage_operations_seconds_global",
85 : "Time spent on storage operations",
86 : &["operation"],
87 : STORAGE_OP_BUCKETS.into(),
88 : )
89 137 : .expect("failed to define a metric")
90 137 : });
91 :
92 135 : pub(crate) static READ_NUM_LAYERS_VISITED: Lazy<Histogram> = Lazy::new(|| {
93 : register_histogram!(
94 : "pageserver_layers_visited_per_read_global",
95 : "Number of layers visited to reconstruct one key",
96 : vec![1.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0],
97 : )
98 135 : .expect("failed to define a metric")
99 135 : });
100 :
101 24 : pub(crate) static VEC_READ_NUM_LAYERS_VISITED: Lazy<Histogram> = Lazy::new(|| {
102 : register_histogram!(
103 : "pageserver_layers_visited_per_vectored_read_global",
104 : "Average number of layers visited to reconstruct one key",
105 : vec![1.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0],
106 : )
107 24 : .expect("failed to define a metric")
108 24 : });
109 :
110 : // Metrics collected on operations on the storage repository.
111 : #[derive(
112 540 : Clone, Copy, enum_map::Enum, strum_macros::EnumString, strum_macros::Display, IntoStaticStr,
113 : )]
114 : pub(crate) enum GetKind {
115 : Singular,
116 : Vectored,
117 : }
118 :
119 : pub(crate) struct ReconstructTimeMetrics {
120 : singular: Histogram,
121 : vectored: Histogram,
122 : }
123 :
124 135 : pub(crate) static RECONSTRUCT_TIME: Lazy<ReconstructTimeMetrics> = Lazy::new(|| {
125 135 : let inner = register_histogram_vec!(
126 : "pageserver_getpage_reconstruct_seconds",
127 : "Time spent in reconstruct_value (reconstruct a page from deltas)",
128 : &["get_kind"],
129 : CRITICAL_OP_BUCKETS.into(),
130 : )
131 135 : .expect("failed to define a metric");
132 135 :
133 135 : ReconstructTimeMetrics {
134 135 : singular: inner.with_label_values(&[GetKind::Singular.into()]),
135 135 : vectored: inner.with_label_values(&[GetKind::Vectored.into()]),
136 135 : }
137 135 : });
138 :
139 : impl ReconstructTimeMetrics {
140 625726 : pub(crate) fn for_get_kind(&self, get_kind: GetKind) -> &Histogram {
141 625726 : match get_kind {
142 625656 : GetKind::Singular => &self.singular,
143 70 : GetKind::Vectored => &self.vectored,
144 : }
145 625726 : }
146 : }
147 :
148 : pub(crate) struct ReconstructDataTimeMetrics {
149 : singular: Histogram,
150 : vectored: Histogram,
151 : }
152 :
153 : impl ReconstructDataTimeMetrics {
154 625866 : pub(crate) fn for_get_kind(&self, get_kind: GetKind) -> &Histogram {
155 625866 : match get_kind {
156 625796 : GetKind::Singular => &self.singular,
157 70 : GetKind::Vectored => &self.vectored,
158 : }
159 625866 : }
160 : }
161 :
162 135 : pub(crate) static GET_RECONSTRUCT_DATA_TIME: Lazy<ReconstructDataTimeMetrics> = Lazy::new(|| {
163 135 : let inner = register_histogram_vec!(
164 : "pageserver_getpage_get_reconstruct_data_seconds",
165 : "Time spent in get_reconstruct_value_data",
166 : &["get_kind"],
167 : CRITICAL_OP_BUCKETS.into(),
168 : )
169 135 : .expect("failed to define a metric");
170 135 :
171 135 : ReconstructDataTimeMetrics {
172 135 : singular: inner.with_label_values(&[GetKind::Singular.into()]),
173 135 : vectored: inner.with_label_values(&[GetKind::Vectored.into()]),
174 135 : }
175 135 : });
176 :
177 : pub(crate) struct GetVectoredLatency {
178 : map: EnumMap<TaskKind, Option<Histogram>>,
179 : }
180 :
181 : #[allow(dead_code)]
182 : pub(crate) struct ScanLatency {
183 : map: EnumMap<TaskKind, Option<Histogram>>,
184 : }
185 :
186 : impl GetVectoredLatency {
187 : // Only these task types perform vectored gets. Filter all other tasks out to reduce total
188 : // cardinality of the metric.
189 : const TRACKED_TASK_KINDS: [TaskKind; 2] = [TaskKind::Compaction, TaskKind::PageRequestHandler];
190 :
191 934 : pub(crate) fn for_task_kind(&self, task_kind: TaskKind) -> Option<&Histogram> {
192 934 : self.map[task_kind].as_ref()
193 934 : }
194 : }
195 :
196 : impl ScanLatency {
197 : // Only these task types perform vectored gets. Filter all other tasks out to reduce total
198 : // cardinality of the metric.
199 : const TRACKED_TASK_KINDS: [TaskKind; 1] = [TaskKind::PageRequestHandler];
200 :
201 12 : pub(crate) fn for_task_kind(&self, task_kind: TaskKind) -> Option<&Histogram> {
202 12 : self.map[task_kind].as_ref()
203 12 : }
204 : }
205 :
206 : pub(crate) struct ScanLatencyOngoingRecording<'a> {
207 : parent: &'a Histogram,
208 : start: std::time::Instant,
209 : }
210 :
211 : impl<'a> ScanLatencyOngoingRecording<'a> {
212 0 : pub(crate) fn start_recording(parent: &'a Histogram) -> ScanLatencyOngoingRecording<'a> {
213 0 : let start = Instant::now();
214 0 : ScanLatencyOngoingRecording { parent, start }
215 0 : }
216 :
217 0 : pub(crate) fn observe(self, throttled: Option<Duration>) {
218 0 : let elapsed = self.start.elapsed();
219 0 : let ex_throttled = if let Some(throttled) = throttled {
220 0 : elapsed.checked_sub(throttled)
221 : } else {
222 0 : Some(elapsed)
223 : };
224 0 : if let Some(ex_throttled) = ex_throttled {
225 0 : self.parent.observe(ex_throttled.as_secs_f64());
226 0 : } else {
227 0 : use utils::rate_limit::RateLimit;
228 0 : static LOGGED: Lazy<Mutex<RateLimit>> =
229 0 : Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(10))));
230 0 : let mut rate_limit = LOGGED.lock().unwrap();
231 0 : rate_limit.call(|| {
232 0 : warn!("error deducting time spent throttled; this message is logged at a global rate limit");
233 0 : });
234 0 : }
235 0 : }
236 : }
237 :
238 129 : pub(crate) static GET_VECTORED_LATENCY: Lazy<GetVectoredLatency> = Lazy::new(|| {
239 129 : let inner = register_histogram_vec!(
240 : "pageserver_get_vectored_seconds",
241 : "Time spent in get_vectored, excluding time spent in timeline_get_throttle.",
242 : &["task_kind"],
243 : CRITICAL_OP_BUCKETS.into(),
244 : )
245 129 : .expect("failed to define a metric");
246 129 :
247 129 : GetVectoredLatency {
248 3870 : map: EnumMap::from_array(std::array::from_fn(|task_kind_idx| {
249 3870 : let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind_idx);
250 3870 :
251 3870 : if GetVectoredLatency::TRACKED_TASK_KINDS.contains(&task_kind) {
252 258 : let task_kind = task_kind.into();
253 258 : Some(inner.with_label_values(&[task_kind]))
254 : } else {
255 3612 : None
256 : }
257 3870 : })),
258 129 : }
259 129 : });
260 :
261 4 : pub(crate) static SCAN_LATENCY: Lazy<ScanLatency> = Lazy::new(|| {
262 4 : let inner = register_histogram_vec!(
263 : "pageserver_scan_seconds",
264 : "Time spent in scan, excluding time spent in timeline_get_throttle.",
265 : &["task_kind"],
266 : CRITICAL_OP_BUCKETS.into(),
267 : )
268 4 : .expect("failed to define a metric");
269 4 :
270 4 : ScanLatency {
271 120 : map: EnumMap::from_array(std::array::from_fn(|task_kind_idx| {
272 120 : let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind_idx);
273 120 :
274 120 : if ScanLatency::TRACKED_TASK_KINDS.contains(&task_kind) {
275 4 : let task_kind = task_kind.into();
276 4 : Some(inner.with_label_values(&[task_kind]))
277 : } else {
278 116 : None
279 : }
280 120 : })),
281 4 : }
282 4 : });
283 :
284 : pub(crate) struct PageCacheMetricsForTaskKind {
285 : pub read_accesses_immutable: IntCounter,
286 : pub read_hits_immutable: IntCounter,
287 : }
288 :
289 : pub(crate) struct PageCacheMetrics {
290 : map: EnumMap<TaskKind, EnumMap<PageContentKind, PageCacheMetricsForTaskKind>>,
291 : }
292 :
293 70 : static PAGE_CACHE_READ_HITS: Lazy<IntCounterVec> = Lazy::new(|| {
294 : register_int_counter_vec!(
295 : "pageserver_page_cache_read_hits_total",
296 : "Number of read accesses to the page cache that hit",
297 : &["task_kind", "key_kind", "content_kind", "hit_kind"]
298 : )
299 70 : .expect("failed to define a metric")
300 70 : });
301 :
302 70 : static PAGE_CACHE_READ_ACCESSES: Lazy<IntCounterVec> = Lazy::new(|| {
303 : register_int_counter_vec!(
304 : "pageserver_page_cache_read_accesses_total",
305 : "Number of read accesses to the page cache",
306 : &["task_kind", "key_kind", "content_kind"]
307 : )
308 70 : .expect("failed to define a metric")
309 70 : });
310 :
311 70 : pub(crate) static PAGE_CACHE: Lazy<PageCacheMetrics> = Lazy::new(|| PageCacheMetrics {
312 2100 : map: EnumMap::from_array(std::array::from_fn(|task_kind| {
313 2100 : let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind);
314 2100 : let task_kind: &'static str = task_kind.into();
315 12600 : EnumMap::from_array(std::array::from_fn(|content_kind| {
316 12600 : let content_kind = <PageContentKind as enum_map::Enum>::from_usize(content_kind);
317 12600 : let content_kind: &'static str = content_kind.into();
318 12600 : PageCacheMetricsForTaskKind {
319 12600 : read_accesses_immutable: {
320 12600 : PAGE_CACHE_READ_ACCESSES
321 12600 : .get_metric_with_label_values(&[task_kind, "immutable", content_kind])
322 12600 : .unwrap()
323 12600 : },
324 12600 :
325 12600 : read_hits_immutable: {
326 12600 : PAGE_CACHE_READ_HITS
327 12600 : .get_metric_with_label_values(&[task_kind, "immutable", content_kind, "-"])
328 12600 : .unwrap()
329 12600 : },
330 12600 : }
331 12600 : }))
332 2100 : })),
333 70 : });
334 :
335 : impl PageCacheMetrics {
336 14191712 : pub(crate) fn for_ctx(&self, ctx: &RequestContext) -> &PageCacheMetricsForTaskKind {
337 14191712 : &self.map[ctx.task_kind()][ctx.page_content_kind()]
338 14191712 : }
339 : }
340 :
341 : pub(crate) struct PageCacheSizeMetrics {
342 : pub max_bytes: UIntGauge,
343 :
344 : pub current_bytes_immutable: UIntGauge,
345 : }
346 :
347 70 : static PAGE_CACHE_SIZE_CURRENT_BYTES: Lazy<UIntGaugeVec> = Lazy::new(|| {
348 : register_uint_gauge_vec!(
349 : "pageserver_page_cache_size_current_bytes",
350 : "Current size of the page cache in bytes, by key kind",
351 : &["key_kind"]
352 : )
353 70 : .expect("failed to define a metric")
354 70 : });
355 :
356 : pub(crate) static PAGE_CACHE_SIZE: Lazy<PageCacheSizeMetrics> =
357 70 : Lazy::new(|| PageCacheSizeMetrics {
358 : max_bytes: {
359 : register_uint_gauge!(
360 : "pageserver_page_cache_size_max_bytes",
361 : "Maximum size of the page cache in bytes"
362 : )
363 70 : .expect("failed to define a metric")
364 70 : },
365 70 : current_bytes_immutable: {
366 70 : PAGE_CACHE_SIZE_CURRENT_BYTES
367 70 : .get_metric_with_label_values(&["immutable"])
368 70 : .unwrap()
369 70 : },
370 70 : });
371 :
372 : pub(crate) mod page_cache_eviction_metrics {
373 : use std::num::NonZeroUsize;
374 :
375 : use metrics::{register_int_counter_vec, IntCounter, IntCounterVec};
376 : use once_cell::sync::Lazy;
377 :
378 : #[derive(Clone, Copy)]
379 : pub(crate) enum Outcome {
380 : FoundSlotUnused { iters: NonZeroUsize },
381 : FoundSlotEvicted { iters: NonZeroUsize },
382 : ItersExceeded { iters: NonZeroUsize },
383 : }
384 :
385 70 : static ITERS_TOTAL_VEC: Lazy<IntCounterVec> = Lazy::new(|| {
386 : register_int_counter_vec!(
387 : "pageserver_page_cache_find_victim_iters_total",
388 : "Counter for the number of iterations in the find_victim loop",
389 : &["outcome"],
390 : )
391 70 : .expect("failed to define a metric")
392 70 : });
393 :
394 70 : static CALLS_VEC: Lazy<IntCounterVec> = Lazy::new(|| {
395 : register_int_counter_vec!(
396 : "pageserver_page_cache_find_victim_calls",
397 : "Incremented at the end of each find_victim() call.\
398 : Filter by outcome to get e.g., eviction rate.",
399 : &["outcome"]
400 : )
401 70 : .unwrap()
402 70 : });
403 :
404 166773 : pub(crate) fn observe(outcome: Outcome) {
405 166773 : macro_rules! dry {
406 166773 : ($label:literal, $iters:expr) => {{
407 166773 : static LABEL: &'static str = $label;
408 166773 : static ITERS_TOTAL: Lazy<IntCounter> =
409 166773 : Lazy::new(|| ITERS_TOTAL_VEC.with_label_values(&[LABEL]));
410 166773 : static CALLS: Lazy<IntCounter> =
411 166773 : Lazy::new(|| CALLS_VEC.with_label_values(&[LABEL]));
412 166773 : ITERS_TOTAL.inc_by(($iters.get()) as u64);
413 166773 : CALLS.inc();
414 166773 : }};
415 166773 : }
416 166773 : match outcome {
417 1832 : Outcome::FoundSlotUnused { iters } => dry!("found_empty", iters),
418 164941 : Outcome::FoundSlotEvicted { iters } => {
419 164941 : dry!("found_evicted", iters)
420 : }
421 0 : Outcome::ItersExceeded { iters } => {
422 0 : dry!("err_iters_exceeded", iters);
423 0 : super::page_cache_errors_inc(super::PageCacheErrorKind::EvictIterLimit);
424 0 : }
425 : }
426 166773 : }
427 : }
428 :
429 0 : static PAGE_CACHE_ERRORS: Lazy<IntCounterVec> = Lazy::new(|| {
430 : register_int_counter_vec!(
431 : "page_cache_errors_total",
432 : "Number of timeouts while acquiring a pinned slot in the page cache",
433 : &["error_kind"]
434 : )
435 0 : .expect("failed to define a metric")
436 0 : });
437 :
438 0 : #[derive(IntoStaticStr)]
439 : #[strum(serialize_all = "kebab_case")]
440 : pub(crate) enum PageCacheErrorKind {
441 : AcquirePinnedSlotTimeout,
442 : EvictIterLimit,
443 : }
444 :
445 0 : pub(crate) fn page_cache_errors_inc(error_kind: PageCacheErrorKind) {
446 0 : PAGE_CACHE_ERRORS
447 0 : .get_metric_with_label_values(&[error_kind.into()])
448 0 : .unwrap()
449 0 : .inc();
450 0 : }
451 :
452 16 : pub(crate) static WAIT_LSN_TIME: Lazy<Histogram> = Lazy::new(|| {
453 : register_histogram!(
454 : "pageserver_wait_lsn_seconds",
455 : "Time spent waiting for WAL to arrive",
456 : CRITICAL_OP_BUCKETS.into(),
457 : )
458 16 : .expect("failed to define a metric")
459 16 : });
460 :
461 137 : static LAST_RECORD_LSN: Lazy<IntGaugeVec> = Lazy::new(|| {
462 : register_int_gauge_vec!(
463 : "pageserver_last_record_lsn",
464 : "Last record LSN grouped by timeline",
465 : &["tenant_id", "shard_id", "timeline_id"]
466 : )
467 137 : .expect("failed to define a metric")
468 137 : });
469 :
470 137 : static STANDBY_HORIZON: Lazy<IntGaugeVec> = Lazy::new(|| {
471 : register_int_gauge_vec!(
472 : "pageserver_standby_horizon",
473 : "Standby apply LSN for which GC is hold off, by timeline.",
474 : &["tenant_id", "shard_id", "timeline_id"]
475 : )
476 137 : .expect("failed to define a metric")
477 137 : });
478 :
479 137 : static RESIDENT_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
480 : register_uint_gauge_vec!(
481 : "pageserver_resident_physical_size",
482 : "The size of the layer files present in the pageserver's filesystem.",
483 : &["tenant_id", "shard_id", "timeline_id"]
484 : )
485 137 : .expect("failed to define a metric")
486 137 : });
487 :
488 133 : pub(crate) static RESIDENT_PHYSICAL_SIZE_GLOBAL: Lazy<UIntGauge> = Lazy::new(|| {
489 : register_uint_gauge!(
490 : "pageserver_resident_physical_size_global",
491 : "Like `pageserver_resident_physical_size`, but without tenant/timeline dimensions."
492 : )
493 133 : .expect("failed to define a metric")
494 133 : });
495 :
496 137 : static REMOTE_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
497 : register_uint_gauge_vec!(
498 : "pageserver_remote_physical_size",
499 : "The size of the layer files present in the remote storage that are listed in the remote index_part.json.",
500 : // Corollary: If any files are missing from the index part, they won't be included here.
501 : &["tenant_id", "shard_id", "timeline_id"]
502 : )
503 137 : .expect("failed to define a metric")
504 137 : });
505 :
506 137 : static REMOTE_PHYSICAL_SIZE_GLOBAL: Lazy<UIntGauge> = Lazy::new(|| {
507 : register_uint_gauge!(
508 : "pageserver_remote_physical_size_global",
509 : "Like `pageserver_remote_physical_size`, but without tenant/timeline dimensions."
510 : )
511 137 : .expect("failed to define a metric")
512 137 : });
513 :
514 4 : pub(crate) static REMOTE_ONDEMAND_DOWNLOADED_LAYERS: Lazy<IntCounter> = Lazy::new(|| {
515 : register_int_counter!(
516 : "pageserver_remote_ondemand_downloaded_layers_total",
517 : "Total on-demand downloaded layers"
518 : )
519 4 : .unwrap()
520 4 : });
521 :
522 4 : pub(crate) static REMOTE_ONDEMAND_DOWNLOADED_BYTES: Lazy<IntCounter> = Lazy::new(|| {
523 : register_int_counter!(
524 : "pageserver_remote_ondemand_downloaded_bytes_total",
525 : "Total bytes of layers on-demand downloaded",
526 : )
527 4 : .unwrap()
528 4 : });
529 :
530 137 : static CURRENT_LOGICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
531 : register_uint_gauge_vec!(
532 : "pageserver_current_logical_size",
533 : "Current logical size grouped by timeline",
534 : &["tenant_id", "shard_id", "timeline_id"]
535 : )
536 137 : .expect("failed to define current logical size metric")
537 137 : });
538 :
539 137 : static AUX_FILE_SIZE: Lazy<IntGaugeVec> = Lazy::new(|| {
540 : register_int_gauge_vec!(
541 : "pageserver_aux_file_estimated_size",
542 : "The size of all aux files for a timeline in aux file v2 store.",
543 : &["tenant_id", "shard_id", "timeline_id"]
544 : )
545 137 : .expect("failed to define a metric")
546 137 : });
547 :
548 : pub(crate) mod initial_logical_size {
549 : use metrics::{register_int_counter, register_int_counter_vec, IntCounter, IntCounterVec};
550 : use once_cell::sync::Lazy;
551 :
552 : pub(crate) struct StartCalculation(IntCounterVec);
553 137 : pub(crate) static START_CALCULATION: Lazy<StartCalculation> = Lazy::new(|| {
554 137 : StartCalculation(
555 137 : register_int_counter_vec!(
556 : "pageserver_initial_logical_size_start_calculation",
557 : "Incremented each time we start an initial logical size calculation attempt. \
558 : The `circumstances` label provides some additional details.",
559 : &["attempt", "circumstances"]
560 137 : )
561 137 : .unwrap(),
562 137 : )
563 137 : });
564 :
565 : struct DropCalculation {
566 : first: IntCounter,
567 : retry: IntCounter,
568 : }
569 :
570 137 : static DROP_CALCULATION: Lazy<DropCalculation> = Lazy::new(|| {
571 137 : let vec = register_int_counter_vec!(
572 : "pageserver_initial_logical_size_drop_calculation",
573 : "Incremented each time we abort a started size calculation attmpt.",
574 : &["attempt"]
575 : )
576 137 : .unwrap();
577 137 : DropCalculation {
578 137 : first: vec.with_label_values(&["first"]),
579 137 : retry: vec.with_label_values(&["retry"]),
580 137 : }
581 137 : });
582 :
583 : pub(crate) struct Calculated {
584 : pub(crate) births: IntCounter,
585 : pub(crate) deaths: IntCounter,
586 : }
587 :
588 137 : pub(crate) static CALCULATED: Lazy<Calculated> = Lazy::new(|| Calculated {
589 : births: register_int_counter!(
590 : "pageserver_initial_logical_size_finish_calculation",
591 : "Incremented every time we finish calculation of initial logical size.\
592 : If everything is working well, this should happen at most once per Timeline object."
593 : )
594 137 : .unwrap(),
595 : deaths: register_int_counter!(
596 : "pageserver_initial_logical_size_drop_finished_calculation",
597 : "Incremented when we drop a finished initial logical size calculation result.\
598 : Mainly useful to turn pageserver_initial_logical_size_finish_calculation into a gauge."
599 : )
600 137 : .unwrap(),
601 137 : });
602 :
603 : pub(crate) struct OngoingCalculationGuard {
604 : inc_drop_calculation: Option<IntCounter>,
605 : }
606 :
607 149 : #[derive(strum_macros::IntoStaticStr)]
608 : pub(crate) enum StartCircumstances {
609 : EmptyInitial,
610 : SkippedConcurrencyLimiter,
611 : AfterBackgroundTasksRateLimit,
612 : }
613 :
614 : impl StartCalculation {
615 149 : pub(crate) fn first(&self, circumstances: StartCircumstances) -> OngoingCalculationGuard {
616 149 : let circumstances_label: &'static str = circumstances.into();
617 149 : self.0
618 149 : .with_label_values(&["first", circumstances_label])
619 149 : .inc();
620 149 : OngoingCalculationGuard {
621 149 : inc_drop_calculation: Some(DROP_CALCULATION.first.clone()),
622 149 : }
623 149 : }
624 0 : pub(crate) fn retry(&self, circumstances: StartCircumstances) -> OngoingCalculationGuard {
625 0 : let circumstances_label: &'static str = circumstances.into();
626 0 : self.0
627 0 : .with_label_values(&["retry", circumstances_label])
628 0 : .inc();
629 0 : OngoingCalculationGuard {
630 0 : inc_drop_calculation: Some(DROP_CALCULATION.retry.clone()),
631 0 : }
632 0 : }
633 : }
634 :
635 : impl Drop for OngoingCalculationGuard {
636 149 : fn drop(&mut self) {
637 149 : if let Some(counter) = self.inc_drop_calculation.take() {
638 0 : counter.inc();
639 149 : }
640 149 : }
641 : }
642 :
643 : impl OngoingCalculationGuard {
644 149 : pub(crate) fn calculation_result_saved(mut self) -> FinishedCalculationGuard {
645 149 : drop(self.inc_drop_calculation.take());
646 149 : CALCULATED.births.inc();
647 149 : FinishedCalculationGuard {
648 149 : inc_on_drop: CALCULATED.deaths.clone(),
649 149 : }
650 149 : }
651 : }
652 :
653 : pub(crate) struct FinishedCalculationGuard {
654 : inc_on_drop: IntCounter,
655 : }
656 :
657 : impl Drop for FinishedCalculationGuard {
658 6 : fn drop(&mut self) {
659 6 : self.inc_on_drop.inc();
660 6 : }
661 : }
662 :
663 : // context: https://github.com/neondatabase/neon/issues/5963
664 : pub(crate) static TIMELINES_WHERE_WALRECEIVER_GOT_APPROXIMATE_SIZE: Lazy<IntCounter> =
665 0 : Lazy::new(|| {
666 : register_int_counter!(
667 : "pageserver_initial_logical_size_timelines_where_walreceiver_got_approximate_size",
668 : "Counter for the following event: walreceiver calls\
669 : Timeline::get_current_logical_size() and it returns `Approximate` for the first time."
670 : )
671 0 : .unwrap()
672 0 : });
673 : }
674 :
675 0 : static DIRECTORY_ENTRIES_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
676 : register_uint_gauge_vec!(
677 : "pageserver_directory_entries_count",
678 : "Sum of the entries in pageserver-stored directory listings",
679 : &["tenant_id", "shard_id", "timeline_id"]
680 : )
681 0 : .expect("failed to define a metric")
682 0 : });
683 :
684 139 : pub(crate) static TENANT_STATE_METRIC: Lazy<UIntGaugeVec> = Lazy::new(|| {
685 : register_uint_gauge_vec!(
686 : "pageserver_tenant_states_count",
687 : "Count of tenants per state",
688 : &["state"]
689 : )
690 139 : .expect("Failed to register pageserver_tenant_states_count metric")
691 139 : });
692 :
693 : /// A set of broken tenants.
694 : ///
695 : /// These are expected to be so rare that a set is fine. Set as in a new timeseries per each broken
696 : /// tenant.
697 12 : pub(crate) static BROKEN_TENANTS_SET: Lazy<UIntGaugeVec> = Lazy::new(|| {
698 : register_uint_gauge_vec!(
699 : "pageserver_broken_tenants_count",
700 : "Set of broken tenants",
701 : &["tenant_id", "shard_id"]
702 : )
703 12 : .expect("Failed to register pageserver_tenant_states_count metric")
704 12 : });
705 :
706 6 : pub(crate) static TENANT_SYNTHETIC_SIZE_METRIC: Lazy<UIntGaugeVec> = Lazy::new(|| {
707 : register_uint_gauge_vec!(
708 : "pageserver_tenant_synthetic_cached_size_bytes",
709 : "Synthetic size of each tenant in bytes",
710 : &["tenant_id"]
711 : )
712 6 : .expect("Failed to register pageserver_tenant_synthetic_cached_size_bytes metric")
713 6 : });
714 :
715 0 : pub(crate) static EVICTION_ITERATION_DURATION: Lazy<HistogramVec> = Lazy::new(|| {
716 : register_histogram_vec!(
717 : "pageserver_eviction_iteration_duration_seconds_global",
718 : "Time spent on a single eviction iteration",
719 : &["period_secs", "threshold_secs"],
720 : STORAGE_OP_BUCKETS.into(),
721 : )
722 0 : .expect("failed to define a metric")
723 0 : });
724 :
725 137 : static EVICTIONS: Lazy<IntCounterVec> = Lazy::new(|| {
726 : register_int_counter_vec!(
727 : "pageserver_evictions",
728 : "Number of layers evicted from the pageserver",
729 : &["tenant_id", "shard_id", "timeline_id"]
730 : )
731 137 : .expect("failed to define a metric")
732 137 : });
733 :
734 137 : static EVICTIONS_WITH_LOW_RESIDENCE_DURATION: Lazy<IntCounterVec> = Lazy::new(|| {
735 : register_int_counter_vec!(
736 : "pageserver_evictions_with_low_residence_duration",
737 : "If a layer is evicted that was resident for less than `low_threshold`, it is counted to this counter. \
738 : Residence duration is determined using the `residence_duration_data_source`.",
739 : &["tenant_id", "shard_id", "timeline_id", "residence_duration_data_source", "low_threshold_secs"]
740 : )
741 137 : .expect("failed to define a metric")
742 137 : });
743 :
744 0 : pub(crate) static UNEXPECTED_ONDEMAND_DOWNLOADS: Lazy<IntCounter> = Lazy::new(|| {
745 : register_int_counter!(
746 : "pageserver_unexpected_ondemand_downloads_count",
747 : "Number of unexpected on-demand downloads. \
748 : We log more context for each increment, so, forgo any labels in this metric.",
749 : )
750 0 : .expect("failed to define a metric")
751 0 : });
752 :
753 : /// How long did we take to start up? Broken down by labels to describe
754 : /// different phases of startup.
755 0 : pub static STARTUP_DURATION: Lazy<GaugeVec> = Lazy::new(|| {
756 : register_gauge_vec!(
757 : "pageserver_startup_duration_seconds",
758 : "Time taken by phases of pageserver startup, in seconds",
759 : &["phase"]
760 : )
761 0 : .expect("Failed to register pageserver_startup_duration_seconds metric")
762 0 : });
763 :
764 0 : pub static STARTUP_IS_LOADING: Lazy<UIntGauge> = Lazy::new(|| {
765 : register_uint_gauge!(
766 : "pageserver_startup_is_loading",
767 : "1 while in initial startup load of tenants, 0 at other times"
768 : )
769 0 : .expect("Failed to register pageserver_startup_is_loading")
770 0 : });
771 :
772 133 : pub(crate) static TIMELINE_EPHEMERAL_BYTES: Lazy<UIntGauge> = Lazy::new(|| {
773 : register_uint_gauge!(
774 : "pageserver_timeline_ephemeral_bytes",
775 : "Total number of bytes in ephemeral layers, summed for all timelines. Approximate, lazily updated."
776 : )
777 133 : .expect("Failed to register metric")
778 133 : });
779 :
780 : /// Metrics related to the lifecycle of a [`crate::tenant::Tenant`] object: things
781 : /// like how long it took to load.
782 : ///
783 : /// Note that these are process-global metrics, _not_ per-tenant metrics. Per-tenant
784 : /// metrics are rather expensive, and usually fine grained stuff makes more sense
785 : /// at a timeline level than tenant level.
786 : pub(crate) struct TenantMetrics {
787 : /// How long did tenants take to go from construction to active state?
788 : pub(crate) activation: Histogram,
789 : pub(crate) preload: Histogram,
790 : pub(crate) attach: Histogram,
791 :
792 : /// How many tenants are included in the initial startup of the pagesrever?
793 : pub(crate) startup_scheduled: IntCounter,
794 : pub(crate) startup_complete: IntCounter,
795 : }
796 :
797 0 : pub(crate) static TENANT: Lazy<TenantMetrics> = Lazy::new(|| {
798 0 : TenantMetrics {
799 0 : activation: register_histogram!(
800 : "pageserver_tenant_activation_seconds",
801 : "Time taken by tenants to activate, in seconds",
802 : CRITICAL_OP_BUCKETS.into()
803 0 : )
804 0 : .expect("Failed to register metric"),
805 0 : preload: register_histogram!(
806 : "pageserver_tenant_preload_seconds",
807 : "Time taken by tenants to load remote metadata on startup/attach, in seconds",
808 : CRITICAL_OP_BUCKETS.into()
809 0 : )
810 0 : .expect("Failed to register metric"),
811 0 : attach: register_histogram!(
812 : "pageserver_tenant_attach_seconds",
813 : "Time taken by tenants to intialize, after remote metadata is already loaded",
814 : CRITICAL_OP_BUCKETS.into()
815 0 : )
816 0 : .expect("Failed to register metric"),
817 0 : startup_scheduled: register_int_counter!(
818 : "pageserver_tenant_startup_scheduled",
819 : "Number of tenants included in pageserver startup (doesn't count tenants attached later)"
820 0 : ).expect("Failed to register metric"),
821 0 : startup_complete: register_int_counter!(
822 : "pageserver_tenant_startup_complete",
823 : "Number of tenants that have completed warm-up, or activated on-demand during initial startup: \
824 : should eventually reach `pageserver_tenant_startup_scheduled_total`. Does not include broken \
825 : tenants: such cases will lead to this metric never reaching the scheduled count."
826 0 : ).expect("Failed to register metric"),
827 0 : }
828 0 : });
829 :
830 : /// Each `Timeline`'s [`EVICTIONS_WITH_LOW_RESIDENCE_DURATION`] metric.
831 : #[derive(Debug)]
832 : pub(crate) struct EvictionsWithLowResidenceDuration {
833 : data_source: &'static str,
834 : threshold: Duration,
835 : counter: Option<IntCounter>,
836 : }
837 :
838 : pub(crate) struct EvictionsWithLowResidenceDurationBuilder {
839 : data_source: &'static str,
840 : threshold: Duration,
841 : }
842 :
843 : impl EvictionsWithLowResidenceDurationBuilder {
844 379 : pub fn new(data_source: &'static str, threshold: Duration) -> Self {
845 379 : Self {
846 379 : data_source,
847 379 : threshold,
848 379 : }
849 379 : }
850 :
851 379 : fn build(
852 379 : &self,
853 379 : tenant_id: &str,
854 379 : shard_id: &str,
855 379 : timeline_id: &str,
856 379 : ) -> EvictionsWithLowResidenceDuration {
857 379 : let counter = EVICTIONS_WITH_LOW_RESIDENCE_DURATION
858 379 : .get_metric_with_label_values(&[
859 379 : tenant_id,
860 379 : shard_id,
861 379 : timeline_id,
862 379 : self.data_source,
863 379 : &EvictionsWithLowResidenceDuration::threshold_label_value(self.threshold),
864 379 : ])
865 379 : .unwrap();
866 379 : EvictionsWithLowResidenceDuration {
867 379 : data_source: self.data_source,
868 379 : threshold: self.threshold,
869 379 : counter: Some(counter),
870 379 : }
871 379 : }
872 : }
873 :
874 : impl EvictionsWithLowResidenceDuration {
875 387 : fn threshold_label_value(threshold: Duration) -> String {
876 387 : format!("{}", threshold.as_secs())
877 387 : }
878 :
879 18 : pub fn observe(&self, observed_value: Duration) {
880 18 : if observed_value < self.threshold {
881 18 : self.counter
882 18 : .as_ref()
883 18 : .expect("nobody calls this function after `remove_from_vec`")
884 18 : .inc();
885 18 : }
886 18 : }
887 :
888 8 : pub fn change_threshold(
889 8 : &mut self,
890 8 : tenant_id: &str,
891 8 : shard_id: &str,
892 8 : timeline_id: &str,
893 8 : new_threshold: Duration,
894 8 : ) {
895 8 : if new_threshold == self.threshold {
896 8 : return;
897 0 : }
898 0 : let mut with_new = EvictionsWithLowResidenceDurationBuilder::new(
899 0 : self.data_source,
900 0 : new_threshold,
901 0 : )
902 0 : .build(tenant_id, shard_id, timeline_id);
903 0 : std::mem::swap(self, &mut with_new);
904 0 : with_new.remove(tenant_id, shard_id, timeline_id);
905 8 : }
906 :
907 : // This could be a `Drop` impl, but, we need the `tenant_id` and `timeline_id`.
908 8 : fn remove(&mut self, tenant_id: &str, shard_id: &str, timeline_id: &str) {
909 8 : let Some(_counter) = self.counter.take() else {
910 0 : return;
911 : };
912 :
913 8 : let threshold = Self::threshold_label_value(self.threshold);
914 8 :
915 8 : let removed = EVICTIONS_WITH_LOW_RESIDENCE_DURATION.remove_label_values(&[
916 8 : tenant_id,
917 8 : shard_id,
918 8 : timeline_id,
919 8 : self.data_source,
920 8 : &threshold,
921 8 : ]);
922 8 :
923 8 : match removed {
924 0 : Err(e) => {
925 0 : // this has been hit in staging as
926 0 : // <https://neondatabase.sentry.io/issues/4142396994/>, but we don't know how.
927 0 : // because we can be in the drop path already, don't risk:
928 0 : // - "double-panic => illegal instruction" or
929 0 : // - future "drop panick => abort"
930 0 : //
931 0 : // so just nag: (the error has the labels)
932 0 : tracing::warn!("failed to remove EvictionsWithLowResidenceDuration, it was already removed? {e:#?}");
933 : }
934 : Ok(()) => {
935 : // to help identify cases where we double-remove the same values, let's log all
936 : // deletions?
937 8 : tracing::info!("removed EvictionsWithLowResidenceDuration with {tenant_id}, {timeline_id}, {}, {threshold}", self.data_source);
938 : }
939 : }
940 8 : }
941 : }
942 :
943 : // Metrics collected on disk IO operations
944 : //
945 : // Roughly logarithmic scale.
946 : const STORAGE_IO_TIME_BUCKETS: &[f64] = &[
947 : 0.000030, // 30 usec
948 : 0.001000, // 1000 usec
949 : 0.030, // 30 ms
950 : 1.000, // 1000 ms
951 : 30.000, // 30000 ms
952 : ];
953 :
954 : /// VirtualFile fs operation variants.
955 : ///
956 : /// Operations:
957 : /// - open ([`std::fs::OpenOptions::open`])
958 : /// - close (dropping [`crate::virtual_file::VirtualFile`])
959 : /// - close-by-replace (close by replacement algorithm)
960 : /// - read (`read_at`)
961 : /// - write (`write_at`)
962 : /// - seek (modify internal position or file length query)
963 : /// - fsync ([`std::fs::File::sync_all`])
964 : /// - metadata ([`std::fs::File::metadata`])
965 : #[derive(
966 1413 : Debug, Clone, Copy, strum_macros::EnumCount, strum_macros::EnumIter, strum_macros::FromRepr,
967 : )]
968 : pub(crate) enum StorageIoOperation {
969 : Open,
970 : OpenAfterReplace,
971 : Close,
972 : CloseByReplace,
973 : Read,
974 : Write,
975 : Seek,
976 : Fsync,
977 : Metadata,
978 : }
979 :
980 : impl StorageIoOperation {
981 1413 : pub fn as_str(&self) -> &'static str {
982 1413 : match self {
983 157 : StorageIoOperation::Open => "open",
984 157 : StorageIoOperation::OpenAfterReplace => "open-after-replace",
985 157 : StorageIoOperation::Close => "close",
986 157 : StorageIoOperation::CloseByReplace => "close-by-replace",
987 157 : StorageIoOperation::Read => "read",
988 157 : StorageIoOperation::Write => "write",
989 157 : StorageIoOperation::Seek => "seek",
990 157 : StorageIoOperation::Fsync => "fsync",
991 157 : StorageIoOperation::Metadata => "metadata",
992 : }
993 1413 : }
994 : }
995 :
996 : /// Tracks time taken by fs operations near VirtualFile.
997 : #[derive(Debug)]
998 : pub(crate) struct StorageIoTime {
999 : metrics: [Histogram; StorageIoOperation::COUNT],
1000 : }
1001 :
1002 : impl StorageIoTime {
1003 157 : fn new() -> Self {
1004 157 : let storage_io_histogram_vec = register_histogram_vec!(
1005 : "pageserver_io_operations_seconds",
1006 : "Time spent in IO operations",
1007 : &["operation"],
1008 : STORAGE_IO_TIME_BUCKETS.into()
1009 : )
1010 157 : .expect("failed to define a metric");
1011 1413 : let metrics = std::array::from_fn(|i| {
1012 1413 : let op = StorageIoOperation::from_repr(i).unwrap();
1013 1413 : storage_io_histogram_vec
1014 1413 : .get_metric_with_label_values(&[op.as_str()])
1015 1413 : .unwrap()
1016 1413 : });
1017 157 : Self { metrics }
1018 157 : }
1019 :
1020 1879237 : pub(crate) fn get(&self, op: StorageIoOperation) -> &Histogram {
1021 1879237 : &self.metrics[op as usize]
1022 1879237 : }
1023 : }
1024 :
1025 : pub(crate) static STORAGE_IO_TIME_METRIC: Lazy<StorageIoTime> = Lazy::new(StorageIoTime::new);
1026 :
1027 : const STORAGE_IO_SIZE_OPERATIONS: &[&str] = &["read", "write"];
1028 :
1029 : // Needed for the https://neonprod.grafana.net/d/5uK9tHL4k/picking-tenant-for-relocation?orgId=1
1030 155 : pub(crate) static STORAGE_IO_SIZE: Lazy<IntGaugeVec> = Lazy::new(|| {
1031 : register_int_gauge_vec!(
1032 : "pageserver_io_operations_bytes_total",
1033 : "Total amount of bytes read/written in IO operations",
1034 : &["operation", "tenant_id", "shard_id", "timeline_id"]
1035 : )
1036 155 : .expect("failed to define a metric")
1037 155 : });
1038 :
1039 : #[cfg(not(test))]
1040 : pub(crate) mod virtual_file_descriptor_cache {
1041 : use super::*;
1042 :
1043 0 : pub(crate) static SIZE_MAX: Lazy<UIntGauge> = Lazy::new(|| {
1044 : register_uint_gauge!(
1045 : "pageserver_virtual_file_descriptor_cache_size_max",
1046 : "Maximum number of open file descriptors in the cache."
1047 : )
1048 0 : .unwrap()
1049 0 : });
1050 :
1051 : // SIZE_CURRENT: derive it like so:
1052 : // ```
1053 : // sum (pageserver_io_operations_seconds_count{operation=~"^(open|open-after-replace)$")
1054 : // -ignoring(operation)
1055 : // sum(pageserver_io_operations_seconds_count{operation=~"^(close|close-by-replace)$"}
1056 : // ```
1057 : }
1058 :
1059 : #[cfg(not(test))]
1060 : pub(crate) mod virtual_file_io_engine {
1061 : use super::*;
1062 :
1063 0 : pub(crate) static KIND: Lazy<UIntGaugeVec> = Lazy::new(|| {
1064 : register_uint_gauge_vec!(
1065 : "pageserver_virtual_file_io_engine_kind",
1066 : "The configured io engine for VirtualFile",
1067 : &["kind"],
1068 : )
1069 0 : .unwrap()
1070 0 : });
1071 : }
1072 :
1073 : #[derive(Debug)]
1074 : struct GlobalAndPerTimelineHistogram {
1075 : global: Histogram,
1076 : per_tenant_timeline: Histogram,
1077 : }
1078 :
1079 : impl GlobalAndPerTimelineHistogram {
1080 10 : fn observe(&self, value: f64) {
1081 10 : self.global.observe(value);
1082 10 : self.per_tenant_timeline.observe(value);
1083 10 : }
1084 : }
1085 :
1086 : struct GlobalAndPerTimelineHistogramTimer<'a, 'c> {
1087 : h: &'a GlobalAndPerTimelineHistogram,
1088 : ctx: &'c RequestContext,
1089 : start: std::time::Instant,
1090 : op: SmgrQueryType,
1091 : }
1092 :
1093 : impl<'a, 'c> Drop for GlobalAndPerTimelineHistogramTimer<'a, 'c> {
1094 10 : fn drop(&mut self) {
1095 10 : let elapsed = self.start.elapsed();
1096 10 : let ex_throttled = self
1097 10 : .ctx
1098 10 : .micros_spent_throttled
1099 10 : .close_and_checked_sub_from(elapsed);
1100 10 : let ex_throttled = match ex_throttled {
1101 10 : Ok(res) => res,
1102 0 : Err(error) => {
1103 0 : use utils::rate_limit::RateLimit;
1104 0 : static LOGGED: Lazy<Mutex<enum_map::EnumMap<SmgrQueryType, RateLimit>>> =
1105 0 : Lazy::new(|| {
1106 0 : Mutex::new(enum_map::EnumMap::from_array(std::array::from_fn(|_| {
1107 0 : RateLimit::new(Duration::from_secs(10))
1108 0 : })))
1109 0 : });
1110 0 : let mut guard = LOGGED.lock().unwrap();
1111 0 : let rate_limit = &mut guard[self.op];
1112 0 : rate_limit.call(|| {
1113 0 : warn!(op=?self.op, error, "error deducting time spent throttled; this message is logged at a global rate limit");
1114 0 : });
1115 0 : elapsed
1116 : }
1117 : };
1118 10 : self.h.observe(ex_throttled.as_secs_f64());
1119 10 : }
1120 : }
1121 :
1122 : #[derive(
1123 : Debug,
1124 : Clone,
1125 : Copy,
1126 3940 : IntoStaticStr,
1127 : strum_macros::EnumCount,
1128 112 : strum_macros::EnumIter,
1129 1945 : strum_macros::FromRepr,
1130 : enum_map::Enum,
1131 : )]
1132 : #[strum(serialize_all = "snake_case")]
1133 : pub enum SmgrQueryType {
1134 : GetRelExists,
1135 : GetRelSize,
1136 : GetPageAtLsn,
1137 : GetDbSize,
1138 : GetSlruSegment,
1139 : }
1140 :
1141 : #[derive(Debug)]
1142 : pub(crate) struct SmgrQueryTimePerTimeline {
1143 : metrics: [GlobalAndPerTimelineHistogram; SmgrQueryType::COUNT],
1144 : }
1145 :
1146 139 : static SMGR_QUERY_TIME_PER_TENANT_TIMELINE: Lazy<HistogramVec> = Lazy::new(|| {
1147 : register_histogram_vec!(
1148 : "pageserver_smgr_query_seconds",
1149 : "Time spent on smgr query handling, aggegated by query type and tenant/timeline.",
1150 : &["smgr_query_type", "tenant_id", "shard_id", "timeline_id"],
1151 : CRITICAL_OP_BUCKETS.into(),
1152 : )
1153 139 : .expect("failed to define a metric")
1154 139 : });
1155 :
1156 139 : static SMGR_QUERY_TIME_GLOBAL_BUCKETS: Lazy<Vec<f64>> = Lazy::new(|| {
1157 139 : [
1158 139 : 1,
1159 139 : 10,
1160 139 : 20,
1161 139 : 40,
1162 139 : 60,
1163 139 : 80,
1164 139 : 100,
1165 139 : 200,
1166 139 : 300,
1167 139 : 400,
1168 139 : 500,
1169 139 : 600,
1170 139 : 700,
1171 139 : 800,
1172 139 : 900,
1173 139 : 1_000, // 1ms
1174 139 : 2_000,
1175 139 : 4_000,
1176 139 : 6_000,
1177 139 : 8_000,
1178 139 : 10_000, // 10ms
1179 139 : 20_000,
1180 139 : 40_000,
1181 139 : 60_000,
1182 139 : 80_000,
1183 139 : 100_000,
1184 139 : 200_000,
1185 139 : 400_000,
1186 139 : 600_000,
1187 139 : 800_000,
1188 139 : 1_000_000, // 1s
1189 139 : 2_000_000,
1190 139 : 4_000_000,
1191 139 : 6_000_000,
1192 139 : 8_000_000,
1193 139 : 10_000_000, // 10s
1194 139 : 20_000_000,
1195 139 : 50_000_000,
1196 139 : 100_000_000,
1197 139 : 200_000_000,
1198 139 : 1_000_000_000, // 1000s
1199 139 : ]
1200 139 : .into_iter()
1201 139 : .map(Duration::from_micros)
1202 5699 : .map(|d| d.as_secs_f64())
1203 139 : .collect()
1204 139 : });
1205 :
1206 139 : static SMGR_QUERY_TIME_GLOBAL: Lazy<HistogramVec> = Lazy::new(|| {
1207 : register_histogram_vec!(
1208 : "pageserver_smgr_query_seconds_global",
1209 : "Time spent on smgr query handling, aggregated by query type.",
1210 : &["smgr_query_type"],
1211 : SMGR_QUERY_TIME_GLOBAL_BUCKETS.clone(),
1212 : )
1213 139 : .expect("failed to define a metric")
1214 139 : });
1215 :
1216 : impl SmgrQueryTimePerTimeline {
1217 389 : pub(crate) fn new(tenant_shard_id: &TenantShardId, timeline_id: &TimelineId) -> Self {
1218 389 : let tenant_id = tenant_shard_id.tenant_id.to_string();
1219 389 : let shard_slug = format!("{}", tenant_shard_id.shard_slug());
1220 389 : let timeline_id = timeline_id.to_string();
1221 1945 : let metrics = std::array::from_fn(|i| {
1222 1945 : let op = SmgrQueryType::from_repr(i).unwrap();
1223 1945 : let global = SMGR_QUERY_TIME_GLOBAL
1224 1945 : .get_metric_with_label_values(&[op.into()])
1225 1945 : .unwrap();
1226 1945 : let per_tenant_timeline = SMGR_QUERY_TIME_PER_TENANT_TIMELINE
1227 1945 : .get_metric_with_label_values(&[op.into(), &tenant_id, &shard_slug, &timeline_id])
1228 1945 : .unwrap();
1229 1945 : GlobalAndPerTimelineHistogram {
1230 1945 : global,
1231 1945 : per_tenant_timeline,
1232 1945 : }
1233 1945 : });
1234 389 : Self { metrics }
1235 389 : }
1236 10 : pub(crate) fn start_timer<'c: 'a, 'a>(
1237 10 : &'a self,
1238 10 : op: SmgrQueryType,
1239 10 : ctx: &'c RequestContext,
1240 10 : ) -> impl Drop + '_ {
1241 10 : let metric = &self.metrics[op as usize];
1242 10 : let start = Instant::now();
1243 10 : match ctx.micros_spent_throttled.open() {
1244 10 : Ok(()) => (),
1245 0 : Err(error) => {
1246 0 : use utils::rate_limit::RateLimit;
1247 0 : static LOGGED: Lazy<Mutex<enum_map::EnumMap<SmgrQueryType, RateLimit>>> =
1248 0 : Lazy::new(|| {
1249 0 : Mutex::new(enum_map::EnumMap::from_array(std::array::from_fn(|_| {
1250 0 : RateLimit::new(Duration::from_secs(10))
1251 0 : })))
1252 0 : });
1253 0 : let mut guard = LOGGED.lock().unwrap();
1254 0 : let rate_limit = &mut guard[op];
1255 0 : rate_limit.call(|| {
1256 0 : warn!(?op, error, "error opening micros_spent_throttled; this message is logged at a global rate limit");
1257 0 : });
1258 0 : }
1259 : }
1260 10 : GlobalAndPerTimelineHistogramTimer {
1261 10 : h: metric,
1262 10 : ctx,
1263 10 : start,
1264 10 : op,
1265 10 : }
1266 10 : }
1267 : }
1268 :
1269 : #[cfg(test)]
1270 : mod smgr_query_time_tests {
1271 : use pageserver_api::shard::TenantShardId;
1272 : use strum::IntoEnumIterator;
1273 : use utils::id::{TenantId, TimelineId};
1274 :
1275 : use crate::{
1276 : context::{DownloadBehavior, RequestContext},
1277 : task_mgr::TaskKind,
1278 : };
1279 :
1280 : // Regression test, we used hard-coded string constants before using an enum.
1281 : #[test]
1282 2 : fn op_label_name() {
1283 2 : use super::SmgrQueryType::*;
1284 2 : let expect: [(super::SmgrQueryType, &'static str); 5] = [
1285 2 : (GetRelExists, "get_rel_exists"),
1286 2 : (GetRelSize, "get_rel_size"),
1287 2 : (GetPageAtLsn, "get_page_at_lsn"),
1288 2 : (GetDbSize, "get_db_size"),
1289 2 : (GetSlruSegment, "get_slru_segment"),
1290 2 : ];
1291 12 : for (op, expect) in expect {
1292 10 : let actual: &'static str = op.into();
1293 10 : assert_eq!(actual, expect);
1294 : }
1295 2 : }
1296 :
1297 : #[test]
1298 2 : fn basic() {
1299 2 : let ops: Vec<_> = super::SmgrQueryType::iter().collect();
1300 :
1301 12 : for op in &ops {
1302 10 : let tenant_id = TenantId::generate();
1303 10 : let timeline_id = TimelineId::generate();
1304 10 : let metrics = super::SmgrQueryTimePerTimeline::new(
1305 10 : &TenantShardId::unsharded(tenant_id),
1306 10 : &timeline_id,
1307 10 : );
1308 10 :
1309 20 : let get_counts = || {
1310 20 : let global: u64 = ops
1311 20 : .iter()
1312 100 : .map(|op| metrics.metrics[*op as usize].global.get_sample_count())
1313 20 : .sum();
1314 20 : let per_tenant_timeline: u64 = ops
1315 20 : .iter()
1316 100 : .map(|op| {
1317 100 : metrics.metrics[*op as usize]
1318 100 : .per_tenant_timeline
1319 100 : .get_sample_count()
1320 100 : })
1321 20 : .sum();
1322 20 : (global, per_tenant_timeline)
1323 20 : };
1324 :
1325 10 : let (pre_global, pre_per_tenant_timeline) = get_counts();
1326 10 : assert_eq!(pre_per_tenant_timeline, 0);
1327 :
1328 10 : let ctx = RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Download);
1329 10 : let timer = metrics.start_timer(*op, &ctx);
1330 10 : drop(timer);
1331 10 :
1332 10 : let (post_global, post_per_tenant_timeline) = get_counts();
1333 10 : assert_eq!(post_per_tenant_timeline, 1);
1334 10 : assert!(post_global > pre_global);
1335 : }
1336 2 : }
1337 : }
1338 :
1339 : // keep in sync with control plane Go code so that we can validate
1340 : // compute's basebackup_ms metric with our perspective in the context of SLI/SLO.
1341 0 : static COMPUTE_STARTUP_BUCKETS: Lazy<[f64; 28]> = Lazy::new(|| {
1342 0 : // Go code uses milliseconds. Variable is called `computeStartupBuckets`
1343 0 : [
1344 0 : 5, 10, 20, 30, 50, 70, 100, 120, 150, 200, 250, 300, 350, 400, 450, 500, 600, 800, 1000,
1345 0 : 1500, 2000, 2500, 3000, 5000, 10000, 20000, 40000, 60000,
1346 0 : ]
1347 0 : .map(|ms| (ms as f64) / 1000.0)
1348 0 : });
1349 :
1350 : pub(crate) struct BasebackupQueryTime {
1351 : ok: Histogram,
1352 : error: Histogram,
1353 : }
1354 :
1355 0 : pub(crate) static BASEBACKUP_QUERY_TIME: Lazy<BasebackupQueryTime> = Lazy::new(|| {
1356 0 : let vec = register_histogram_vec!(
1357 : "pageserver_basebackup_query_seconds",
1358 : "Histogram of basebackup queries durations, by result type",
1359 : &["result"],
1360 : COMPUTE_STARTUP_BUCKETS.to_vec(),
1361 : )
1362 0 : .expect("failed to define a metric");
1363 0 : BasebackupQueryTime {
1364 0 : ok: vec.get_metric_with_label_values(&["ok"]).unwrap(),
1365 0 : error: vec.get_metric_with_label_values(&["error"]).unwrap(),
1366 0 : }
1367 0 : });
1368 :
1369 : pub(crate) struct BasebackupQueryTimeOngoingRecording<'a, 'c> {
1370 : parent: &'a BasebackupQueryTime,
1371 : ctx: &'c RequestContext,
1372 : start: std::time::Instant,
1373 : }
1374 :
1375 : impl BasebackupQueryTime {
1376 0 : pub(crate) fn start_recording<'c: 'a, 'a>(
1377 0 : &'a self,
1378 0 : ctx: &'c RequestContext,
1379 0 : ) -> BasebackupQueryTimeOngoingRecording<'_, '_> {
1380 0 : let start = Instant::now();
1381 0 : match ctx.micros_spent_throttled.open() {
1382 0 : Ok(()) => (),
1383 0 : Err(error) => {
1384 0 : use utils::rate_limit::RateLimit;
1385 0 : static LOGGED: Lazy<Mutex<RateLimit>> =
1386 0 : Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(10))));
1387 0 : let mut rate_limit = LOGGED.lock().unwrap();
1388 0 : rate_limit.call(|| {
1389 0 : warn!(error, "error opening micros_spent_throttled; this message is logged at a global rate limit");
1390 0 : });
1391 0 : }
1392 : }
1393 0 : BasebackupQueryTimeOngoingRecording {
1394 0 : parent: self,
1395 0 : ctx,
1396 0 : start,
1397 0 : }
1398 0 : }
1399 : }
1400 :
1401 : impl<'a, 'c> BasebackupQueryTimeOngoingRecording<'a, 'c> {
1402 0 : pub(crate) fn observe<T, E>(self, res: &Result<T, E>) {
1403 0 : let elapsed = self.start.elapsed();
1404 0 : let ex_throttled = self
1405 0 : .ctx
1406 0 : .micros_spent_throttled
1407 0 : .close_and_checked_sub_from(elapsed);
1408 0 : let ex_throttled = match ex_throttled {
1409 0 : Ok(ex_throttled) => ex_throttled,
1410 0 : Err(error) => {
1411 0 : use utils::rate_limit::RateLimit;
1412 0 : static LOGGED: Lazy<Mutex<RateLimit>> =
1413 0 : Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(10))));
1414 0 : let mut rate_limit = LOGGED.lock().unwrap();
1415 0 : rate_limit.call(|| {
1416 0 : warn!(error, "error deducting time spent throttled; this message is logged at a global rate limit");
1417 0 : });
1418 0 : elapsed
1419 : }
1420 : };
1421 0 : let metric = if res.is_ok() {
1422 0 : &self.parent.ok
1423 : } else {
1424 0 : &self.parent.error
1425 : };
1426 0 : metric.observe(ex_throttled.as_secs_f64());
1427 0 : }
1428 : }
1429 :
1430 0 : pub(crate) static LIVE_CONNECTIONS_COUNT: Lazy<IntGaugeVec> = Lazy::new(|| {
1431 : register_int_gauge_vec!(
1432 : "pageserver_live_connections",
1433 : "Number of live network connections",
1434 : &["pageserver_connection_kind"]
1435 : )
1436 0 : .expect("failed to define a metric")
1437 0 : });
1438 :
1439 : // remote storage metrics
1440 :
1441 135 : static REMOTE_TIMELINE_CLIENT_CALLS: Lazy<IntCounterPairVec> = Lazy::new(|| {
1442 : register_int_counter_pair_vec!(
1443 : "pageserver_remote_timeline_client_calls_started",
1444 : "Number of started calls to remote timeline client.",
1445 : "pageserver_remote_timeline_client_calls_finished",
1446 : "Number of finshed calls to remote timeline client.",
1447 : &[
1448 : "tenant_id",
1449 : "shard_id",
1450 : "timeline_id",
1451 : "file_kind",
1452 : "op_kind"
1453 : ],
1454 : )
1455 135 : .unwrap()
1456 135 : });
1457 :
1458 : static REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER: Lazy<IntCounterVec> =
1459 131 : Lazy::new(|| {
1460 : register_int_counter_vec!(
1461 : "pageserver_remote_timeline_client_bytes_started",
1462 : "Incremented by the number of bytes associated with a remote timeline client operation. \
1463 : The increment happens when the operation is scheduled.",
1464 : &["tenant_id", "shard_id", "timeline_id", "file_kind", "op_kind"],
1465 : )
1466 131 : .expect("failed to define a metric")
1467 131 : });
1468 :
1469 131 : static REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER: Lazy<IntCounterVec> = Lazy::new(|| {
1470 : register_int_counter_vec!(
1471 : "pageserver_remote_timeline_client_bytes_finished",
1472 : "Incremented by the number of bytes associated with a remote timeline client operation. \
1473 : The increment happens when the operation finishes (regardless of success/failure/shutdown).",
1474 : &["tenant_id", "shard_id", "timeline_id", "file_kind", "op_kind"],
1475 : )
1476 131 : .expect("failed to define a metric")
1477 131 : });
1478 :
1479 : pub(crate) struct TenantManagerMetrics {
1480 : tenant_slots_attached: UIntGauge,
1481 : tenant_slots_secondary: UIntGauge,
1482 : tenant_slots_inprogress: UIntGauge,
1483 : pub(crate) tenant_slot_writes: IntCounter,
1484 : pub(crate) unexpected_errors: IntCounter,
1485 : }
1486 :
1487 : impl TenantManagerMetrics {
1488 : /// Helpers for tracking slots. Note that these do not track the lifetime of TenantSlot objects
1489 : /// exactly: they track the lifetime of the slots _in the tenant map_.
1490 2 : pub(crate) fn slot_inserted(&self, slot: &TenantSlot) {
1491 2 : match slot {
1492 0 : TenantSlot::Attached(_) => {
1493 0 : self.tenant_slots_attached.inc();
1494 0 : }
1495 0 : TenantSlot::Secondary(_) => {
1496 0 : self.tenant_slots_secondary.inc();
1497 0 : }
1498 2 : TenantSlot::InProgress(_) => {
1499 2 : self.tenant_slots_inprogress.inc();
1500 2 : }
1501 : }
1502 2 : }
1503 :
1504 2 : pub(crate) fn slot_removed(&self, slot: &TenantSlot) {
1505 2 : match slot {
1506 2 : TenantSlot::Attached(_) => {
1507 2 : self.tenant_slots_attached.dec();
1508 2 : }
1509 0 : TenantSlot::Secondary(_) => {
1510 0 : self.tenant_slots_secondary.dec();
1511 0 : }
1512 0 : TenantSlot::InProgress(_) => {
1513 0 : self.tenant_slots_inprogress.dec();
1514 0 : }
1515 : }
1516 2 : }
1517 :
1518 : #[cfg(all(debug_assertions, not(test)))]
1519 0 : pub(crate) fn slots_total(&self) -> u64 {
1520 0 : self.tenant_slots_attached.get()
1521 0 : + self.tenant_slots_secondary.get()
1522 0 : + self.tenant_slots_inprogress.get()
1523 0 : }
1524 : }
1525 :
1526 2 : pub(crate) static TENANT_MANAGER: Lazy<TenantManagerMetrics> = Lazy::new(|| {
1527 2 : let tenant_slots = register_uint_gauge_vec!(
1528 : "pageserver_tenant_manager_slots",
1529 : "How many slots currently exist, including all attached, secondary and in-progress operations",
1530 : &["mode"]
1531 : )
1532 2 : .expect("failed to define a metric");
1533 2 : TenantManagerMetrics {
1534 2 : tenant_slots_attached: tenant_slots
1535 2 : .get_metric_with_label_values(&["attached"])
1536 2 : .unwrap(),
1537 2 : tenant_slots_secondary: tenant_slots
1538 2 : .get_metric_with_label_values(&["secondary"])
1539 2 : .unwrap(),
1540 2 : tenant_slots_inprogress: tenant_slots
1541 2 : .get_metric_with_label_values(&["inprogress"])
1542 2 : .unwrap(),
1543 2 : tenant_slot_writes: register_int_counter!(
1544 : "pageserver_tenant_manager_slot_writes",
1545 : "Writes to a tenant slot, including all of create/attach/detach/delete"
1546 2 : )
1547 2 : .expect("failed to define a metric"),
1548 2 : unexpected_errors: register_int_counter!(
1549 : "pageserver_tenant_manager_unexpected_errors_total",
1550 : "Number of unexpected conditions encountered: nonzero value indicates a non-fatal bug."
1551 2 : )
1552 2 : .expect("failed to define a metric"),
1553 2 : }
1554 2 : });
1555 :
1556 : pub(crate) struct DeletionQueueMetrics {
1557 : pub(crate) keys_submitted: IntCounter,
1558 : pub(crate) keys_dropped: IntCounter,
1559 : pub(crate) keys_executed: IntCounter,
1560 : pub(crate) keys_validated: IntCounter,
1561 : pub(crate) dropped_lsn_updates: IntCounter,
1562 : pub(crate) unexpected_errors: IntCounter,
1563 : pub(crate) remote_errors: IntCounterVec,
1564 : }
1565 23 : pub(crate) static DELETION_QUEUE: Lazy<DeletionQueueMetrics> = Lazy::new(|| {
1566 23 : DeletionQueueMetrics{
1567 23 :
1568 23 : keys_submitted: register_int_counter!(
1569 : "pageserver_deletion_queue_submitted_total",
1570 : "Number of objects submitted for deletion"
1571 23 : )
1572 23 : .expect("failed to define a metric"),
1573 23 :
1574 23 : keys_dropped: register_int_counter!(
1575 : "pageserver_deletion_queue_dropped_total",
1576 : "Number of object deletions dropped due to stale generation."
1577 23 : )
1578 23 : .expect("failed to define a metric"),
1579 23 :
1580 23 : keys_executed: register_int_counter!(
1581 : "pageserver_deletion_queue_executed_total",
1582 : "Number of objects deleted. Only includes objects that we actually deleted, sum with pageserver_deletion_queue_dropped_total for the total number of keys processed to completion"
1583 23 : )
1584 23 : .expect("failed to define a metric"),
1585 23 :
1586 23 : keys_validated: register_int_counter!(
1587 : "pageserver_deletion_queue_validated_total",
1588 : "Number of keys validated for deletion. Sum with pageserver_deletion_queue_dropped_total for the total number of keys that have passed through the validation stage."
1589 23 : )
1590 23 : .expect("failed to define a metric"),
1591 23 :
1592 23 : dropped_lsn_updates: register_int_counter!(
1593 : "pageserver_deletion_queue_dropped_lsn_updates_total",
1594 : "Updates to remote_consistent_lsn dropped due to stale generation number."
1595 23 : )
1596 23 : .expect("failed to define a metric"),
1597 23 : unexpected_errors: register_int_counter!(
1598 : "pageserver_deletion_queue_unexpected_errors_total",
1599 : "Number of unexpected condiions that may stall the queue: any value above zero is unexpected."
1600 23 : )
1601 23 : .expect("failed to define a metric"),
1602 23 : remote_errors: register_int_counter_vec!(
1603 : "pageserver_deletion_queue_remote_errors_total",
1604 : "Retryable remote I/O errors while executing deletions, for example 503 responses to DeleteObjects",
1605 : &["op_kind"],
1606 23 : )
1607 23 : .expect("failed to define a metric")
1608 23 : }
1609 23 : });
1610 :
1611 : pub(crate) struct SecondaryModeMetrics {
1612 : pub(crate) upload_heatmap: IntCounter,
1613 : pub(crate) upload_heatmap_errors: IntCounter,
1614 : pub(crate) upload_heatmap_duration: Histogram,
1615 : pub(crate) download_heatmap: IntCounter,
1616 : pub(crate) download_layer: IntCounter,
1617 : }
1618 0 : pub(crate) static SECONDARY_MODE: Lazy<SecondaryModeMetrics> = Lazy::new(|| {
1619 0 : SecondaryModeMetrics {
1620 0 : upload_heatmap: register_int_counter!(
1621 : "pageserver_secondary_upload_heatmap",
1622 : "Number of heatmaps written to remote storage by attached tenants"
1623 0 : )
1624 0 : .expect("failed to define a metric"),
1625 0 : upload_heatmap_errors: register_int_counter!(
1626 : "pageserver_secondary_upload_heatmap_errors",
1627 : "Failures writing heatmap to remote storage"
1628 0 : )
1629 0 : .expect("failed to define a metric"),
1630 0 : upload_heatmap_duration: register_histogram!(
1631 : "pageserver_secondary_upload_heatmap_duration",
1632 : "Time to build and upload a heatmap, including any waiting inside the S3 client"
1633 0 : )
1634 0 : .expect("failed to define a metric"),
1635 0 : download_heatmap: register_int_counter!(
1636 : "pageserver_secondary_download_heatmap",
1637 : "Number of downloads of heatmaps by secondary mode locations, including when it hasn't changed"
1638 0 : )
1639 0 : .expect("failed to define a metric"),
1640 0 : download_layer: register_int_counter!(
1641 : "pageserver_secondary_download_layer",
1642 : "Number of downloads of layers by secondary mode locations"
1643 0 : )
1644 0 : .expect("failed to define a metric"),
1645 0 : }
1646 0 : });
1647 :
1648 : #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1649 : pub enum RemoteOpKind {
1650 : Upload,
1651 : Download,
1652 : Delete,
1653 : }
1654 : impl RemoteOpKind {
1655 13024 : pub fn as_str(&self) -> &'static str {
1656 13024 : match self {
1657 12325 : Self::Upload => "upload",
1658 52 : Self::Download => "download",
1659 647 : Self::Delete => "delete",
1660 : }
1661 13024 : }
1662 : }
1663 :
1664 : #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
1665 : pub enum RemoteOpFileKind {
1666 : Layer,
1667 : Index,
1668 : }
1669 : impl RemoteOpFileKind {
1670 13024 : pub fn as_str(&self) -> &'static str {
1671 13024 : match self {
1672 8873 : Self::Layer => "layer",
1673 4151 : Self::Index => "index",
1674 : }
1675 13024 : }
1676 : }
1677 :
1678 130 : pub(crate) static REMOTE_OPERATION_TIME: Lazy<HistogramVec> = Lazy::new(|| {
1679 : register_histogram_vec!(
1680 : "pageserver_remote_operation_seconds",
1681 : "Time spent on remote storage operations. \
1682 : Grouped by tenant, timeline, operation_kind and status. \
1683 : Does not account for time spent waiting in remote timeline client's queues.",
1684 : &["file_kind", "op_kind", "status"]
1685 : )
1686 130 : .expect("failed to define a metric")
1687 130 : });
1688 :
1689 0 : pub(crate) static TENANT_TASK_EVENTS: Lazy<IntCounterVec> = Lazy::new(|| {
1690 : register_int_counter_vec!(
1691 : "pageserver_tenant_task_events",
1692 : "Number of task start/stop/fail events.",
1693 : &["event"],
1694 : )
1695 0 : .expect("Failed to register tenant_task_events metric")
1696 0 : });
1697 :
1698 20 : pub(crate) static BACKGROUND_LOOP_SEMAPHORE_WAIT_GAUGE: Lazy<IntCounterPairVec> = Lazy::new(|| {
1699 : register_int_counter_pair_vec!(
1700 : "pageserver_background_loop_semaphore_wait_start_count",
1701 : "Counter for background loop concurrency-limiting semaphore acquire calls started",
1702 : "pageserver_background_loop_semaphore_wait_finish_count",
1703 : "Counter for background loop concurrency-limiting semaphore acquire calls finished",
1704 : &["task"],
1705 : )
1706 20 : .unwrap()
1707 20 : });
1708 :
1709 0 : pub(crate) static BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT: Lazy<IntCounterVec> = Lazy::new(|| {
1710 : register_int_counter_vec!(
1711 : "pageserver_background_loop_period_overrun_count",
1712 : "Incremented whenever warn_when_period_overrun() logs a warning.",
1713 : &["task", "period"],
1714 : )
1715 0 : .expect("failed to define a metric")
1716 0 : });
1717 :
1718 : // walreceiver metrics
1719 :
1720 0 : pub(crate) static WALRECEIVER_STARTED_CONNECTIONS: Lazy<IntCounter> = Lazy::new(|| {
1721 : register_int_counter!(
1722 : "pageserver_walreceiver_started_connections_total",
1723 : "Number of started walreceiver connections"
1724 : )
1725 0 : .expect("failed to define a metric")
1726 0 : });
1727 :
1728 0 : pub(crate) static WALRECEIVER_ACTIVE_MANAGERS: Lazy<IntGauge> = Lazy::new(|| {
1729 : register_int_gauge!(
1730 : "pageserver_walreceiver_active_managers",
1731 : "Number of active walreceiver managers"
1732 : )
1733 0 : .expect("failed to define a metric")
1734 0 : });
1735 :
1736 0 : pub(crate) static WALRECEIVER_SWITCHES: Lazy<IntCounterVec> = Lazy::new(|| {
1737 : register_int_counter_vec!(
1738 : "pageserver_walreceiver_switches_total",
1739 : "Number of walreceiver manager change_connection calls",
1740 : &["reason"]
1741 : )
1742 0 : .expect("failed to define a metric")
1743 0 : });
1744 :
1745 0 : pub(crate) static WALRECEIVER_BROKER_UPDATES: Lazy<IntCounter> = Lazy::new(|| {
1746 : register_int_counter!(
1747 : "pageserver_walreceiver_broker_updates_total",
1748 : "Number of received broker updates in walreceiver"
1749 : )
1750 0 : .expect("failed to define a metric")
1751 0 : });
1752 :
1753 2 : pub(crate) static WALRECEIVER_CANDIDATES_EVENTS: Lazy<IntCounterVec> = Lazy::new(|| {
1754 : register_int_counter_vec!(
1755 : "pageserver_walreceiver_candidates_events_total",
1756 : "Number of walreceiver candidate events",
1757 : &["event"]
1758 : )
1759 2 : .expect("failed to define a metric")
1760 2 : });
1761 :
1762 : pub(crate) static WALRECEIVER_CANDIDATES_ADDED: Lazy<IntCounter> =
1763 0 : Lazy::new(|| WALRECEIVER_CANDIDATES_EVENTS.with_label_values(&["add"]));
1764 :
1765 : pub(crate) static WALRECEIVER_CANDIDATES_REMOVED: Lazy<IntCounter> =
1766 2 : Lazy::new(|| WALRECEIVER_CANDIDATES_EVENTS.with_label_values(&["remove"]));
1767 :
1768 : // Metrics collected on WAL redo operations
1769 : //
1770 : // We collect the time spent in actual WAL redo ('redo'), and time waiting
1771 : // for access to the postgres process ('wait') since there is only one for
1772 : // each tenant.
1773 :
1774 : /// Time buckets are small because we want to be able to measure the
1775 : /// smallest redo processing times. These buckets allow us to measure down
1776 : /// to 5us, which equates to 200'000 pages/sec, which equates to 1.6GB/sec.
1777 : /// This is much better than the previous 5ms aka 200 pages/sec aka 1.6MB/sec.
1778 : ///
1779 : /// Values up to 1s are recorded because metrics show that we have redo
1780 : /// durations and lock times larger than 0.250s.
1781 : macro_rules! redo_histogram_time_buckets {
1782 : () => {
1783 : vec![
1784 : 0.000_005, 0.000_010, 0.000_025, 0.000_050, 0.000_100, 0.000_250, 0.000_500, 0.001_000,
1785 : 0.002_500, 0.005_000, 0.010_000, 0.025_000, 0.050_000, 0.100_000, 0.250_000, 0.500_000,
1786 : 1.000_000,
1787 : ]
1788 : };
1789 : }
1790 :
1791 : /// While we're at it, also measure the amount of records replayed in each
1792 : /// operation. We have a global 'total replayed' counter, but that's not
1793 : /// as useful as 'what is the skew for how many records we replay in one
1794 : /// operation'.
1795 : macro_rules! redo_histogram_count_buckets {
1796 : () => {
1797 : vec![0.0, 1.0, 2.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0]
1798 : };
1799 : }
1800 :
1801 : macro_rules! redo_bytes_histogram_count_buckets {
1802 : () => {
1803 : // powers of (2^.5), from 2^4.5 to 2^15 (22 buckets)
1804 : // rounded up to the next multiple of 8 to capture any MAXALIGNed record of that size, too.
1805 : vec![
1806 : 24.0, 32.0, 48.0, 64.0, 96.0, 128.0, 184.0, 256.0, 368.0, 512.0, 728.0, 1024.0, 1456.0,
1807 : 2048.0, 2904.0, 4096.0, 5800.0, 8192.0, 11592.0, 16384.0, 23176.0, 32768.0,
1808 : ]
1809 : };
1810 : }
1811 :
1812 : pub(crate) struct WalIngestMetrics {
1813 : pub(crate) bytes_received: IntCounter,
1814 : pub(crate) records_received: IntCounter,
1815 : pub(crate) records_committed: IntCounter,
1816 : pub(crate) records_filtered: IntCounter,
1817 : }
1818 :
1819 2 : pub(crate) static WAL_INGEST: Lazy<WalIngestMetrics> = Lazy::new(|| WalIngestMetrics {
1820 : bytes_received: register_int_counter!(
1821 : "pageserver_wal_ingest_bytes_received",
1822 : "Bytes of WAL ingested from safekeepers",
1823 : )
1824 2 : .unwrap(),
1825 : records_received: register_int_counter!(
1826 : "pageserver_wal_ingest_records_received",
1827 : "Number of WAL records received from safekeepers"
1828 : )
1829 2 : .expect("failed to define a metric"),
1830 : records_committed: register_int_counter!(
1831 : "pageserver_wal_ingest_records_committed",
1832 : "Number of WAL records which resulted in writes to pageserver storage"
1833 : )
1834 2 : .expect("failed to define a metric"),
1835 : records_filtered: register_int_counter!(
1836 : "pageserver_wal_ingest_records_filtered",
1837 : "Number of WAL records filtered out due to sharding"
1838 : )
1839 2 : .expect("failed to define a metric"),
1840 2 : });
1841 :
1842 6 : pub(crate) static WAL_REDO_TIME: Lazy<Histogram> = Lazy::new(|| {
1843 : register_histogram!(
1844 : "pageserver_wal_redo_seconds",
1845 : "Time spent on WAL redo",
1846 : redo_histogram_time_buckets!()
1847 : )
1848 6 : .expect("failed to define a metric")
1849 6 : });
1850 :
1851 6 : pub(crate) static WAL_REDO_RECORDS_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
1852 : register_histogram!(
1853 : "pageserver_wal_redo_records_histogram",
1854 : "Histogram of number of records replayed per redo in the Postgres WAL redo process",
1855 : redo_histogram_count_buckets!(),
1856 : )
1857 6 : .expect("failed to define a metric")
1858 6 : });
1859 :
1860 6 : pub(crate) static WAL_REDO_BYTES_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
1861 : register_histogram!(
1862 : "pageserver_wal_redo_bytes_histogram",
1863 : "Histogram of number of records replayed per redo sent to Postgres",
1864 : redo_bytes_histogram_count_buckets!(),
1865 : )
1866 6 : .expect("failed to define a metric")
1867 6 : });
1868 :
1869 : // FIXME: isn't this already included by WAL_REDO_RECORDS_HISTOGRAM which has _count?
1870 6 : pub(crate) static WAL_REDO_RECORD_COUNTER: Lazy<IntCounter> = Lazy::new(|| {
1871 : register_int_counter!(
1872 : "pageserver_replayed_wal_records_total",
1873 : "Number of WAL records replayed in WAL redo process"
1874 : )
1875 6 : .unwrap()
1876 6 : });
1877 :
1878 : #[rustfmt::skip]
1879 6 : pub(crate) static WAL_REDO_PROCESS_LAUNCH_DURATION_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
1880 : register_histogram!(
1881 : "pageserver_wal_redo_process_launch_duration",
1882 : "Histogram of the duration of successful WalRedoProcess::launch calls",
1883 : vec![
1884 : 0.0002, 0.0004, 0.0006, 0.0008, 0.0010,
1885 : 0.0020, 0.0040, 0.0060, 0.0080, 0.0100,
1886 : 0.0200, 0.0400, 0.0600, 0.0800, 0.1000,
1887 : 0.2000, 0.4000, 0.6000, 0.8000, 1.0000,
1888 : 1.5000, 2.0000, 2.5000, 3.0000, 4.0000, 10.0000
1889 : ],
1890 : )
1891 6 : .expect("failed to define a metric")
1892 6 : });
1893 :
1894 : pub(crate) struct WalRedoProcessCounters {
1895 : pub(crate) started: IntCounter,
1896 : pub(crate) killed_by_cause: enum_map::EnumMap<WalRedoKillCause, IntCounter>,
1897 : pub(crate) active_stderr_logger_tasks_started: IntCounter,
1898 : pub(crate) active_stderr_logger_tasks_finished: IntCounter,
1899 : }
1900 :
1901 18 : #[derive(Debug, enum_map::Enum, strum_macros::IntoStaticStr)]
1902 : pub(crate) enum WalRedoKillCause {
1903 : WalRedoProcessDrop,
1904 : NoLeakChildDrop,
1905 : Startup,
1906 : }
1907 :
1908 : impl Default for WalRedoProcessCounters {
1909 6 : fn default() -> Self {
1910 6 : let started = register_int_counter!(
1911 : "pageserver_wal_redo_process_started_total",
1912 : "Number of WAL redo processes started",
1913 : )
1914 6 : .unwrap();
1915 6 :
1916 6 : let killed = register_int_counter_vec!(
1917 : "pageserver_wal_redo_process_stopped_total",
1918 : "Number of WAL redo processes stopped",
1919 : &["cause"],
1920 : )
1921 6 : .unwrap();
1922 6 :
1923 6 : let active_stderr_logger_tasks_started = register_int_counter!(
1924 : "pageserver_walredo_stderr_logger_tasks_started_total",
1925 : "Number of active walredo stderr logger tasks that have started",
1926 : )
1927 6 : .unwrap();
1928 6 :
1929 6 : let active_stderr_logger_tasks_finished = register_int_counter!(
1930 : "pageserver_walredo_stderr_logger_tasks_finished_total",
1931 : "Number of active walredo stderr logger tasks that have finished",
1932 : )
1933 6 : .unwrap();
1934 6 :
1935 6 : Self {
1936 6 : started,
1937 18 : killed_by_cause: EnumMap::from_array(std::array::from_fn(|i| {
1938 18 : let cause = <WalRedoKillCause as enum_map::Enum>::from_usize(i);
1939 18 : let cause_str: &'static str = cause.into();
1940 18 : killed.with_label_values(&[cause_str])
1941 18 : })),
1942 6 : active_stderr_logger_tasks_started,
1943 6 : active_stderr_logger_tasks_finished,
1944 6 : }
1945 6 : }
1946 : }
1947 :
1948 : pub(crate) static WAL_REDO_PROCESS_COUNTERS: Lazy<WalRedoProcessCounters> =
1949 : Lazy::new(WalRedoProcessCounters::default);
1950 :
1951 : /// Similar to `prometheus::HistogramTimer` but does not record on drop.
1952 : pub(crate) struct StorageTimeMetricsTimer {
1953 : metrics: StorageTimeMetrics,
1954 : start: Instant,
1955 : }
1956 :
1957 : impl StorageTimeMetricsTimer {
1958 4593 : fn new(metrics: StorageTimeMetrics) -> Self {
1959 4593 : Self {
1960 4593 : metrics,
1961 4593 : start: Instant::now(),
1962 4593 : }
1963 4593 : }
1964 :
1965 : /// Record the time from creation to now.
1966 3490 : pub fn stop_and_record(self) {
1967 3490 : let duration = self.start.elapsed().as_secs_f64();
1968 3490 : self.metrics.timeline_sum.inc_by(duration);
1969 3490 : self.metrics.timeline_count.inc();
1970 3490 : self.metrics.global_histogram.observe(duration);
1971 3490 : }
1972 :
1973 : /// Turns this timer into a timer, which will always record -- usually this means recording
1974 : /// regardless an early `?` path was taken in a function.
1975 754 : pub(crate) fn record_on_drop(self) -> AlwaysRecordingStorageTimeMetricsTimer {
1976 754 : AlwaysRecordingStorageTimeMetricsTimer(Some(self))
1977 754 : }
1978 : }
1979 :
1980 : pub(crate) struct AlwaysRecordingStorageTimeMetricsTimer(Option<StorageTimeMetricsTimer>);
1981 :
1982 : impl Drop for AlwaysRecordingStorageTimeMetricsTimer {
1983 754 : fn drop(&mut self) {
1984 754 : if let Some(inner) = self.0.take() {
1985 754 : inner.stop_and_record();
1986 754 : }
1987 754 : }
1988 : }
1989 :
1990 : /// Timing facilities for an globally histogrammed metric, which is supported by per tenant and
1991 : /// timeline total sum and count.
1992 : #[derive(Clone, Debug)]
1993 : pub(crate) struct StorageTimeMetrics {
1994 : /// Sum of f64 seconds, per operation, tenant_id and timeline_id
1995 : timeline_sum: Counter,
1996 : /// Number of oeprations, per operation, tenant_id and timeline_id
1997 : timeline_count: IntCounter,
1998 : /// Global histogram having only the "operation" label.
1999 : global_histogram: Histogram,
2000 : }
2001 :
2002 : impl StorageTimeMetrics {
2003 3032 : pub fn new(
2004 3032 : operation: StorageTimeOperation,
2005 3032 : tenant_id: &str,
2006 3032 : shard_id: &str,
2007 3032 : timeline_id: &str,
2008 3032 : ) -> Self {
2009 3032 : let operation: &'static str = operation.into();
2010 3032 :
2011 3032 : let timeline_sum = STORAGE_TIME_SUM_PER_TIMELINE
2012 3032 : .get_metric_with_label_values(&[operation, tenant_id, shard_id, timeline_id])
2013 3032 : .unwrap();
2014 3032 : let timeline_count = STORAGE_TIME_COUNT_PER_TIMELINE
2015 3032 : .get_metric_with_label_values(&[operation, tenant_id, shard_id, timeline_id])
2016 3032 : .unwrap();
2017 3032 : let global_histogram = STORAGE_TIME_GLOBAL
2018 3032 : .get_metric_with_label_values(&[operation])
2019 3032 : .unwrap();
2020 3032 :
2021 3032 : StorageTimeMetrics {
2022 3032 : timeline_sum,
2023 3032 : timeline_count,
2024 3032 : global_histogram,
2025 3032 : }
2026 3032 : }
2027 :
2028 : /// Starts timing a new operation.
2029 : ///
2030 : /// Note: unlike `prometheus::HistogramTimer` the returned timer does not record on drop.
2031 4593 : pub fn start_timer(&self) -> StorageTimeMetricsTimer {
2032 4593 : StorageTimeMetricsTimer::new(self.clone())
2033 4593 : }
2034 : }
2035 :
2036 : #[derive(Debug)]
2037 : pub(crate) struct TimelineMetrics {
2038 : tenant_id: String,
2039 : shard_id: String,
2040 : timeline_id: String,
2041 : pub flush_time_histo: StorageTimeMetrics,
2042 : pub compact_time_histo: StorageTimeMetrics,
2043 : pub create_images_time_histo: StorageTimeMetrics,
2044 : pub logical_size_histo: StorageTimeMetrics,
2045 : pub imitate_logical_size_histo: StorageTimeMetrics,
2046 : pub load_layer_map_histo: StorageTimeMetrics,
2047 : pub garbage_collect_histo: StorageTimeMetrics,
2048 : pub find_gc_cutoffs_histo: StorageTimeMetrics,
2049 : pub last_record_gauge: IntGauge,
2050 : pub standby_horizon_gauge: IntGauge,
2051 : pub resident_physical_size_gauge: UIntGauge,
2052 : /// copy of LayeredTimeline.current_logical_size
2053 : pub current_logical_size_gauge: UIntGauge,
2054 : pub aux_file_size_gauge: IntGauge,
2055 : pub directory_entries_count_gauge: Lazy<UIntGauge, Box<dyn Send + Fn() -> UIntGauge>>,
2056 : pub evictions: IntCounter,
2057 : pub evictions_with_low_residence_duration: std::sync::RwLock<EvictionsWithLowResidenceDuration>,
2058 : shutdown: std::sync::atomic::AtomicBool,
2059 : }
2060 :
2061 : impl TimelineMetrics {
2062 379 : pub fn new(
2063 379 : tenant_shard_id: &TenantShardId,
2064 379 : timeline_id_raw: &TimelineId,
2065 379 : evictions_with_low_residence_duration_builder: EvictionsWithLowResidenceDurationBuilder,
2066 379 : ) -> Self {
2067 379 : let tenant_id = tenant_shard_id.tenant_id.to_string();
2068 379 : let shard_id = format!("{}", tenant_shard_id.shard_slug());
2069 379 : let timeline_id = timeline_id_raw.to_string();
2070 379 : let flush_time_histo = StorageTimeMetrics::new(
2071 379 : StorageTimeOperation::LayerFlush,
2072 379 : &tenant_id,
2073 379 : &shard_id,
2074 379 : &timeline_id,
2075 379 : );
2076 379 : let compact_time_histo = StorageTimeMetrics::new(
2077 379 : StorageTimeOperation::Compact,
2078 379 : &tenant_id,
2079 379 : &shard_id,
2080 379 : &timeline_id,
2081 379 : );
2082 379 : let create_images_time_histo = StorageTimeMetrics::new(
2083 379 : StorageTimeOperation::CreateImages,
2084 379 : &tenant_id,
2085 379 : &shard_id,
2086 379 : &timeline_id,
2087 379 : );
2088 379 : let logical_size_histo = StorageTimeMetrics::new(
2089 379 : StorageTimeOperation::LogicalSize,
2090 379 : &tenant_id,
2091 379 : &shard_id,
2092 379 : &timeline_id,
2093 379 : );
2094 379 : let imitate_logical_size_histo = StorageTimeMetrics::new(
2095 379 : StorageTimeOperation::ImitateLogicalSize,
2096 379 : &tenant_id,
2097 379 : &shard_id,
2098 379 : &timeline_id,
2099 379 : );
2100 379 : let load_layer_map_histo = StorageTimeMetrics::new(
2101 379 : StorageTimeOperation::LoadLayerMap,
2102 379 : &tenant_id,
2103 379 : &shard_id,
2104 379 : &timeline_id,
2105 379 : );
2106 379 : let garbage_collect_histo = StorageTimeMetrics::new(
2107 379 : StorageTimeOperation::Gc,
2108 379 : &tenant_id,
2109 379 : &shard_id,
2110 379 : &timeline_id,
2111 379 : );
2112 379 : let find_gc_cutoffs_histo = StorageTimeMetrics::new(
2113 379 : StorageTimeOperation::FindGcCutoffs,
2114 379 : &tenant_id,
2115 379 : &shard_id,
2116 379 : &timeline_id,
2117 379 : );
2118 379 : let last_record_gauge = LAST_RECORD_LSN
2119 379 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2120 379 : .unwrap();
2121 379 : let standby_horizon_gauge = STANDBY_HORIZON
2122 379 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2123 379 : .unwrap();
2124 379 : let resident_physical_size_gauge = RESIDENT_PHYSICAL_SIZE
2125 379 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2126 379 : .unwrap();
2127 379 : // TODO: we shouldn't expose this metric
2128 379 : let current_logical_size_gauge = CURRENT_LOGICAL_SIZE
2129 379 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2130 379 : .unwrap();
2131 379 : let aux_file_size_gauge = AUX_FILE_SIZE
2132 379 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2133 379 : .unwrap();
2134 379 : // TODO use impl Trait syntax here once we have ability to use it: https://github.com/rust-lang/rust/issues/63065
2135 379 : let directory_entries_count_gauge_closure = {
2136 379 : let tenant_shard_id = *tenant_shard_id;
2137 379 : let timeline_id_raw = *timeline_id_raw;
2138 0 : move || {
2139 0 : let tenant_id = tenant_shard_id.tenant_id.to_string();
2140 0 : let shard_id = format!("{}", tenant_shard_id.shard_slug());
2141 0 : let timeline_id = timeline_id_raw.to_string();
2142 0 : let gauge: UIntGauge = DIRECTORY_ENTRIES_COUNT
2143 0 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2144 0 : .unwrap();
2145 0 : gauge
2146 0 : }
2147 : };
2148 379 : let directory_entries_count_gauge: Lazy<UIntGauge, Box<dyn Send + Fn() -> UIntGauge>> =
2149 379 : Lazy::new(Box::new(directory_entries_count_gauge_closure));
2150 379 : let evictions = EVICTIONS
2151 379 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2152 379 : .unwrap();
2153 379 : let evictions_with_low_residence_duration = evictions_with_low_residence_duration_builder
2154 379 : .build(&tenant_id, &shard_id, &timeline_id);
2155 379 :
2156 379 : TimelineMetrics {
2157 379 : tenant_id,
2158 379 : shard_id,
2159 379 : timeline_id,
2160 379 : flush_time_histo,
2161 379 : compact_time_histo,
2162 379 : create_images_time_histo,
2163 379 : logical_size_histo,
2164 379 : imitate_logical_size_histo,
2165 379 : garbage_collect_histo,
2166 379 : find_gc_cutoffs_histo,
2167 379 : load_layer_map_histo,
2168 379 : last_record_gauge,
2169 379 : standby_horizon_gauge,
2170 379 : resident_physical_size_gauge,
2171 379 : current_logical_size_gauge,
2172 379 : aux_file_size_gauge,
2173 379 : directory_entries_count_gauge,
2174 379 : evictions,
2175 379 : evictions_with_low_residence_duration: std::sync::RwLock::new(
2176 379 : evictions_with_low_residence_duration,
2177 379 : ),
2178 379 : shutdown: std::sync::atomic::AtomicBool::default(),
2179 379 : }
2180 379 : }
2181 :
2182 1461 : pub(crate) fn record_new_file_metrics(&self, sz: u64) {
2183 1461 : self.resident_physical_size_add(sz);
2184 1461 : }
2185 :
2186 441 : pub(crate) fn resident_physical_size_sub(&self, sz: u64) {
2187 441 : self.resident_physical_size_gauge.sub(sz);
2188 441 : crate::metrics::RESIDENT_PHYSICAL_SIZE_GLOBAL.sub(sz);
2189 441 : }
2190 :
2191 1491 : pub(crate) fn resident_physical_size_add(&self, sz: u64) {
2192 1491 : self.resident_physical_size_gauge.add(sz);
2193 1491 : crate::metrics::RESIDENT_PHYSICAL_SIZE_GLOBAL.add(sz);
2194 1491 : }
2195 :
2196 8 : pub(crate) fn resident_physical_size_get(&self) -> u64 {
2197 8 : self.resident_physical_size_gauge.get()
2198 8 : }
2199 :
2200 8 : pub(crate) fn shutdown(&self) {
2201 8 : let was_shutdown = self
2202 8 : .shutdown
2203 8 : .swap(true, std::sync::atomic::Ordering::Relaxed);
2204 8 :
2205 8 : if was_shutdown {
2206 : // this happens on tenant deletion because tenant first shuts down timelines, then
2207 : // invokes timeline deletion which first shuts down the timeline again.
2208 : // TODO: this can be removed once https://github.com/neondatabase/neon/issues/5080
2209 0 : return;
2210 8 : }
2211 8 :
2212 8 : let tenant_id = &self.tenant_id;
2213 8 : let timeline_id = &self.timeline_id;
2214 8 : let shard_id = &self.shard_id;
2215 8 : let _ = LAST_RECORD_LSN.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2216 8 : let _ = STANDBY_HORIZON.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2217 8 : {
2218 8 : RESIDENT_PHYSICAL_SIZE_GLOBAL.sub(self.resident_physical_size_get());
2219 8 : let _ = RESIDENT_PHYSICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2220 8 : }
2221 8 : let _ = CURRENT_LOGICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2222 8 : if let Some(metric) = Lazy::get(&DIRECTORY_ENTRIES_COUNT) {
2223 0 : let _ = metric.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2224 8 : }
2225 8 : let _ = EVICTIONS.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2226 8 : let _ = AUX_FILE_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2227 8 :
2228 8 : self.evictions_with_low_residence_duration
2229 8 : .write()
2230 8 : .unwrap()
2231 8 : .remove(tenant_id, shard_id, timeline_id);
2232 :
2233 : // The following metrics are born outside of the TimelineMetrics lifecycle but still
2234 : // removed at the end of it. The idea is to have the metrics outlive the
2235 : // entity during which they're observed, e.g., the smgr metrics shall
2236 : // outlive an individual smgr connection, but not the timeline.
2237 :
2238 80 : for op in StorageTimeOperation::VARIANTS {
2239 72 : let _ = STORAGE_TIME_SUM_PER_TIMELINE.remove_label_values(&[
2240 72 : op,
2241 72 : tenant_id,
2242 72 : shard_id,
2243 72 : timeline_id,
2244 72 : ]);
2245 72 : let _ = STORAGE_TIME_COUNT_PER_TIMELINE.remove_label_values(&[
2246 72 : op,
2247 72 : tenant_id,
2248 72 : shard_id,
2249 72 : timeline_id,
2250 72 : ]);
2251 72 : }
2252 :
2253 24 : for op in STORAGE_IO_SIZE_OPERATIONS {
2254 16 : let _ = STORAGE_IO_SIZE.remove_label_values(&[op, tenant_id, shard_id, timeline_id]);
2255 16 : }
2256 :
2257 48 : for op in SmgrQueryType::iter() {
2258 40 : let _ = SMGR_QUERY_TIME_PER_TENANT_TIMELINE.remove_label_values(&[
2259 40 : op.into(),
2260 40 : tenant_id,
2261 40 : shard_id,
2262 40 : timeline_id,
2263 40 : ]);
2264 40 : }
2265 8 : }
2266 : }
2267 :
2268 6 : pub(crate) fn remove_tenant_metrics(tenant_shard_id: &TenantShardId) {
2269 6 : // Only shard zero deals in synthetic sizes
2270 6 : if tenant_shard_id.is_shard_zero() {
2271 6 : let tid = tenant_shard_id.tenant_id.to_string();
2272 6 : let _ = TENANT_SYNTHETIC_SIZE_METRIC.remove_label_values(&[&tid]);
2273 6 : }
2274 :
2275 : // we leave the BROKEN_TENANTS_SET entry if any
2276 6 : }
2277 :
2278 : use futures::Future;
2279 : use pin_project_lite::pin_project;
2280 : use std::collections::HashMap;
2281 : use std::num::NonZeroUsize;
2282 : use std::pin::Pin;
2283 : use std::sync::atomic::AtomicU64;
2284 : use std::sync::{Arc, Mutex};
2285 : use std::task::{Context, Poll};
2286 : use std::time::{Duration, Instant};
2287 :
2288 : use crate::context::{PageContentKind, RequestContext};
2289 : use crate::task_mgr::TaskKind;
2290 : use crate::tenant::mgr::TenantSlot;
2291 :
2292 : /// Maintain a per timeline gauge in addition to the global gauge.
2293 : pub(crate) struct PerTimelineRemotePhysicalSizeGauge {
2294 : last_set: AtomicU64,
2295 : gauge: UIntGauge,
2296 : }
2297 :
2298 : impl PerTimelineRemotePhysicalSizeGauge {
2299 389 : fn new(per_timeline_gauge: UIntGauge) -> Self {
2300 389 : Self {
2301 389 : last_set: AtomicU64::new(0),
2302 389 : gauge: per_timeline_gauge,
2303 389 : }
2304 389 : }
2305 1731 : pub(crate) fn set(&self, sz: u64) {
2306 1731 : self.gauge.set(sz);
2307 1731 : let prev = self.last_set.swap(sz, std::sync::atomic::Ordering::Relaxed);
2308 1731 : if sz < prev {
2309 26 : REMOTE_PHYSICAL_SIZE_GLOBAL.sub(prev - sz);
2310 1705 : } else {
2311 1705 : REMOTE_PHYSICAL_SIZE_GLOBAL.add(sz - prev);
2312 1705 : };
2313 1731 : }
2314 2 : pub(crate) fn get(&self) -> u64 {
2315 2 : self.gauge.get()
2316 2 : }
2317 : }
2318 :
2319 : impl Drop for PerTimelineRemotePhysicalSizeGauge {
2320 18 : fn drop(&mut self) {
2321 18 : REMOTE_PHYSICAL_SIZE_GLOBAL.sub(self.last_set.load(std::sync::atomic::Ordering::Relaxed));
2322 18 : }
2323 : }
2324 :
2325 : pub(crate) struct RemoteTimelineClientMetrics {
2326 : tenant_id: String,
2327 : shard_id: String,
2328 : timeline_id: String,
2329 : pub(crate) remote_physical_size_gauge: PerTimelineRemotePhysicalSizeGauge,
2330 : calls: Mutex<HashMap<(&'static str, &'static str), IntCounterPair>>,
2331 : bytes_started_counter: Mutex<HashMap<(&'static str, &'static str), IntCounter>>,
2332 : bytes_finished_counter: Mutex<HashMap<(&'static str, &'static str), IntCounter>>,
2333 : }
2334 :
2335 : impl RemoteTimelineClientMetrics {
2336 389 : pub fn new(tenant_shard_id: &TenantShardId, timeline_id: &TimelineId) -> Self {
2337 389 : let tenant_id_str = tenant_shard_id.tenant_id.to_string();
2338 389 : let shard_id_str = format!("{}", tenant_shard_id.shard_slug());
2339 389 : let timeline_id_str = timeline_id.to_string();
2340 389 :
2341 389 : let remote_physical_size_gauge = PerTimelineRemotePhysicalSizeGauge::new(
2342 389 : REMOTE_PHYSICAL_SIZE
2343 389 : .get_metric_with_label_values(&[&tenant_id_str, &shard_id_str, &timeline_id_str])
2344 389 : .unwrap(),
2345 389 : );
2346 389 :
2347 389 : RemoteTimelineClientMetrics {
2348 389 : tenant_id: tenant_id_str,
2349 389 : shard_id: shard_id_str,
2350 389 : timeline_id: timeline_id_str,
2351 389 : calls: Mutex::new(HashMap::default()),
2352 389 : bytes_started_counter: Mutex::new(HashMap::default()),
2353 389 : bytes_finished_counter: Mutex::new(HashMap::default()),
2354 389 : remote_physical_size_gauge,
2355 389 : }
2356 389 : }
2357 :
2358 2647 : pub fn remote_operation_time(
2359 2647 : &self,
2360 2647 : file_kind: &RemoteOpFileKind,
2361 2647 : op_kind: &RemoteOpKind,
2362 2647 : status: &'static str,
2363 2647 : ) -> Histogram {
2364 2647 : let key = (file_kind.as_str(), op_kind.as_str(), status);
2365 2647 : REMOTE_OPERATION_TIME
2366 2647 : .get_metric_with_label_values(&[key.0, key.1, key.2])
2367 2647 : .unwrap()
2368 2647 : }
2369 :
2370 6166 : fn calls_counter_pair(
2371 6166 : &self,
2372 6166 : file_kind: &RemoteOpFileKind,
2373 6166 : op_kind: &RemoteOpKind,
2374 6166 : ) -> IntCounterPair {
2375 6166 : let mut guard = self.calls.lock().unwrap();
2376 6166 : let key = (file_kind.as_str(), op_kind.as_str());
2377 6166 : let metric = guard.entry(key).or_insert_with(move || {
2378 670 : REMOTE_TIMELINE_CLIENT_CALLS
2379 670 : .get_metric_with_label_values(&[
2380 670 : &self.tenant_id,
2381 670 : &self.shard_id,
2382 670 : &self.timeline_id,
2383 670 : key.0,
2384 670 : key.1,
2385 670 : ])
2386 670 : .unwrap()
2387 6166 : });
2388 6166 : metric.clone()
2389 6166 : }
2390 :
2391 1465 : fn bytes_started_counter(
2392 1465 : &self,
2393 1465 : file_kind: &RemoteOpFileKind,
2394 1465 : op_kind: &RemoteOpKind,
2395 1465 : ) -> IntCounter {
2396 1465 : let mut guard = self.bytes_started_counter.lock().unwrap();
2397 1465 : let key = (file_kind.as_str(), op_kind.as_str());
2398 1465 : let metric = guard.entry(key).or_insert_with(move || {
2399 253 : REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER
2400 253 : .get_metric_with_label_values(&[
2401 253 : &self.tenant_id,
2402 253 : &self.shard_id,
2403 253 : &self.timeline_id,
2404 253 : key.0,
2405 253 : key.1,
2406 253 : ])
2407 253 : .unwrap()
2408 1465 : });
2409 1465 : metric.clone()
2410 1465 : }
2411 :
2412 2734 : fn bytes_finished_counter(
2413 2734 : &self,
2414 2734 : file_kind: &RemoteOpFileKind,
2415 2734 : op_kind: &RemoteOpKind,
2416 2734 : ) -> IntCounter {
2417 2734 : let mut guard = self.bytes_finished_counter.lock().unwrap();
2418 2734 : let key = (file_kind.as_str(), op_kind.as_str());
2419 2734 : let metric = guard.entry(key).or_insert_with(move || {
2420 253 : REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER
2421 253 : .get_metric_with_label_values(&[
2422 253 : &self.tenant_id,
2423 253 : &self.shard_id,
2424 253 : &self.timeline_id,
2425 253 : key.0,
2426 253 : key.1,
2427 253 : ])
2428 253 : .unwrap()
2429 2734 : });
2430 2734 : metric.clone()
2431 2734 : }
2432 : }
2433 :
2434 : #[cfg(test)]
2435 : impl RemoteTimelineClientMetrics {
2436 6 : pub fn get_bytes_started_counter_value(
2437 6 : &self,
2438 6 : file_kind: &RemoteOpFileKind,
2439 6 : op_kind: &RemoteOpKind,
2440 6 : ) -> Option<u64> {
2441 6 : let guard = self.bytes_started_counter.lock().unwrap();
2442 6 : let key = (file_kind.as_str(), op_kind.as_str());
2443 6 : guard.get(&key).map(|counter| counter.get())
2444 6 : }
2445 :
2446 6 : pub fn get_bytes_finished_counter_value(
2447 6 : &self,
2448 6 : file_kind: &RemoteOpFileKind,
2449 6 : op_kind: &RemoteOpKind,
2450 6 : ) -> Option<u64> {
2451 6 : let guard = self.bytes_finished_counter.lock().unwrap();
2452 6 : let key = (file_kind.as_str(), op_kind.as_str());
2453 6 : guard.get(&key).map(|counter| counter.get())
2454 6 : }
2455 : }
2456 :
2457 : /// See [`RemoteTimelineClientMetrics::call_begin`].
2458 : #[must_use]
2459 : pub(crate) struct RemoteTimelineClientCallMetricGuard {
2460 : /// Decremented on drop.
2461 : calls_counter_pair: Option<IntCounterPair>,
2462 : /// If Some(), this references the bytes_finished metric, and we increment it by the given `u64` on drop.
2463 : bytes_finished: Option<(IntCounter, u64)>,
2464 : }
2465 :
2466 : impl RemoteTimelineClientCallMetricGuard {
2467 : /// Consume this guard object without performing the metric updates it would do on `drop()`.
2468 : /// The caller vouches to do the metric updates manually.
2469 3299 : pub fn will_decrement_manually(mut self) {
2470 3299 : let RemoteTimelineClientCallMetricGuard {
2471 3299 : calls_counter_pair,
2472 3299 : bytes_finished,
2473 3299 : } = &mut self;
2474 3299 : calls_counter_pair.take();
2475 3299 : bytes_finished.take();
2476 3299 : }
2477 : }
2478 :
2479 : impl Drop for RemoteTimelineClientCallMetricGuard {
2480 3325 : fn drop(&mut self) {
2481 3325 : let RemoteTimelineClientCallMetricGuard {
2482 3325 : calls_counter_pair,
2483 3325 : bytes_finished,
2484 3325 : } = self;
2485 3325 : if let Some(guard) = calls_counter_pair.take() {
2486 26 : guard.dec();
2487 3299 : }
2488 3325 : if let Some((bytes_finished_metric, value)) = bytes_finished {
2489 0 : bytes_finished_metric.inc_by(*value);
2490 3325 : }
2491 3325 : }
2492 : }
2493 :
2494 : /// The enum variants communicate to the [`RemoteTimelineClientMetrics`] whether to
2495 : /// track the byte size of this call in applicable metric(s).
2496 : pub(crate) enum RemoteTimelineClientMetricsCallTrackSize {
2497 : /// Do not account for this call's byte size in any metrics.
2498 : /// The `reason` field is there to make the call sites self-documenting
2499 : /// about why they don't need the metric.
2500 : DontTrackSize { reason: &'static str },
2501 : /// Track the byte size of the call in applicable metric(s).
2502 : Bytes(u64),
2503 : }
2504 :
2505 : impl RemoteTimelineClientMetrics {
2506 : /// Update the metrics that change when a call to the remote timeline client instance starts.
2507 : ///
2508 : /// Drop the returned guard object once the operation is finished to updates corresponding metrics that track completions.
2509 : /// Or, use [`RemoteTimelineClientCallMetricGuard::will_decrement_manually`] and [`call_end`](Self::call_end) if that
2510 : /// is more suitable.
2511 : /// Never do both.
2512 3325 : pub(crate) fn call_begin(
2513 3325 : &self,
2514 3325 : file_kind: &RemoteOpFileKind,
2515 3325 : op_kind: &RemoteOpKind,
2516 3325 : size: RemoteTimelineClientMetricsCallTrackSize,
2517 3325 : ) -> RemoteTimelineClientCallMetricGuard {
2518 3325 : let calls_counter_pair = self.calls_counter_pair(file_kind, op_kind);
2519 3325 : calls_counter_pair.inc();
2520 :
2521 3325 : let bytes_finished = match size {
2522 1860 : RemoteTimelineClientMetricsCallTrackSize::DontTrackSize { reason: _reason } => {
2523 1860 : // nothing to do
2524 1860 : None
2525 : }
2526 1465 : RemoteTimelineClientMetricsCallTrackSize::Bytes(size) => {
2527 1465 : self.bytes_started_counter(file_kind, op_kind).inc_by(size);
2528 1465 : let finished_counter = self.bytes_finished_counter(file_kind, op_kind);
2529 1465 : Some((finished_counter, size))
2530 : }
2531 : };
2532 3325 : RemoteTimelineClientCallMetricGuard {
2533 3325 : calls_counter_pair: Some(calls_counter_pair),
2534 3325 : bytes_finished,
2535 3325 : }
2536 3325 : }
2537 :
2538 : /// Manually udpate the metrics that track completions, instead of using the guard object.
2539 : /// Using the guard object is generally preferable.
2540 : /// See [`call_begin`](Self::call_begin) for more context.
2541 2841 : pub(crate) fn call_end(
2542 2841 : &self,
2543 2841 : file_kind: &RemoteOpFileKind,
2544 2841 : op_kind: &RemoteOpKind,
2545 2841 : size: RemoteTimelineClientMetricsCallTrackSize,
2546 2841 : ) {
2547 2841 : let calls_counter_pair = self.calls_counter_pair(file_kind, op_kind);
2548 2841 : calls_counter_pair.dec();
2549 2841 : match size {
2550 1572 : RemoteTimelineClientMetricsCallTrackSize::DontTrackSize { reason: _reason } => {}
2551 1269 : RemoteTimelineClientMetricsCallTrackSize::Bytes(size) => {
2552 1269 : self.bytes_finished_counter(file_kind, op_kind).inc_by(size);
2553 1269 : }
2554 : }
2555 2841 : }
2556 : }
2557 :
2558 : impl Drop for RemoteTimelineClientMetrics {
2559 18 : fn drop(&mut self) {
2560 18 : let RemoteTimelineClientMetrics {
2561 18 : tenant_id,
2562 18 : shard_id,
2563 18 : timeline_id,
2564 18 : remote_physical_size_gauge,
2565 18 : calls,
2566 18 : bytes_started_counter,
2567 18 : bytes_finished_counter,
2568 18 : } = self;
2569 24 : for ((a, b), _) in calls.get_mut().unwrap().drain() {
2570 24 : let mut res = [Ok(()), Ok(())];
2571 24 : REMOTE_TIMELINE_CLIENT_CALLS
2572 24 : .remove_label_values(&mut res, &[tenant_id, shard_id, timeline_id, a, b]);
2573 24 : // don't care about results
2574 24 : }
2575 18 : for ((a, b), _) in bytes_started_counter.get_mut().unwrap().drain() {
2576 6 : let _ = REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER.remove_label_values(&[
2577 6 : tenant_id,
2578 6 : shard_id,
2579 6 : timeline_id,
2580 6 : a,
2581 6 : b,
2582 6 : ]);
2583 6 : }
2584 18 : for ((a, b), _) in bytes_finished_counter.get_mut().unwrap().drain() {
2585 6 : let _ = REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER.remove_label_values(&[
2586 6 : tenant_id,
2587 6 : shard_id,
2588 6 : timeline_id,
2589 6 : a,
2590 6 : b,
2591 6 : ]);
2592 6 : }
2593 18 : {
2594 18 : let _ = remote_physical_size_gauge; // use to avoid 'unused' warning in desctructuring above
2595 18 : let _ = REMOTE_PHYSICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2596 18 : }
2597 18 : }
2598 : }
2599 :
2600 : /// Wrapper future that measures the time spent by a remote storage operation,
2601 : /// and records the time and success/failure as a prometheus metric.
2602 : pub(crate) trait MeasureRemoteOp: Sized {
2603 2709 : fn measure_remote_op(
2604 2709 : self,
2605 2709 : file_kind: RemoteOpFileKind,
2606 2709 : op: RemoteOpKind,
2607 2709 : metrics: Arc<RemoteTimelineClientMetrics>,
2608 2709 : ) -> MeasuredRemoteOp<Self> {
2609 2709 : let start = Instant::now();
2610 2709 : MeasuredRemoteOp {
2611 2709 : inner: self,
2612 2709 : file_kind,
2613 2709 : op,
2614 2709 : start,
2615 2709 : metrics,
2616 2709 : }
2617 2709 : }
2618 : }
2619 :
2620 : impl<T: Sized> MeasureRemoteOp for T {}
2621 :
2622 : pin_project! {
2623 : pub(crate) struct MeasuredRemoteOp<F>
2624 : {
2625 : #[pin]
2626 : inner: F,
2627 : file_kind: RemoteOpFileKind,
2628 : op: RemoteOpKind,
2629 : start: Instant,
2630 : metrics: Arc<RemoteTimelineClientMetrics>,
2631 : }
2632 : }
2633 :
2634 : impl<F: Future<Output = Result<O, E>>, O, E> Future for MeasuredRemoteOp<F> {
2635 : type Output = Result<O, E>;
2636 :
2637 40291 : fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
2638 40291 : let this = self.project();
2639 40291 : let poll_result = this.inner.poll(cx);
2640 40291 : if let Poll::Ready(ref res) = poll_result {
2641 2647 : let duration = this.start.elapsed();
2642 2647 : let status = if res.is_ok() { &"success" } else { &"failure" };
2643 2647 : this.metrics
2644 2647 : .remote_operation_time(this.file_kind, this.op, status)
2645 2647 : .observe(duration.as_secs_f64());
2646 37644 : }
2647 40291 : poll_result
2648 40291 : }
2649 : }
2650 :
2651 : pub mod tokio_epoll_uring {
2652 : use metrics::{register_int_counter, UIntGauge};
2653 : use once_cell::sync::Lazy;
2654 :
2655 : pub struct Collector {
2656 : descs: Vec<metrics::core::Desc>,
2657 : systems_created: UIntGauge,
2658 : systems_destroyed: UIntGauge,
2659 : }
2660 :
2661 : impl metrics::core::Collector for Collector {
2662 0 : fn desc(&self) -> Vec<&metrics::core::Desc> {
2663 0 : self.descs.iter().collect()
2664 0 : }
2665 :
2666 0 : fn collect(&self) -> Vec<metrics::proto::MetricFamily> {
2667 0 : let mut mfs = Vec::with_capacity(Self::NMETRICS);
2668 0 : let tokio_epoll_uring::metrics::Metrics {
2669 0 : systems_created,
2670 0 : systems_destroyed,
2671 0 : } = tokio_epoll_uring::metrics::global();
2672 0 : self.systems_created.set(systems_created);
2673 0 : mfs.extend(self.systems_created.collect());
2674 0 : self.systems_destroyed.set(systems_destroyed);
2675 0 : mfs.extend(self.systems_destroyed.collect());
2676 0 : mfs
2677 0 : }
2678 : }
2679 :
2680 : impl Collector {
2681 : const NMETRICS: usize = 2;
2682 :
2683 : #[allow(clippy::new_without_default)]
2684 0 : pub fn new() -> Self {
2685 0 : let mut descs = Vec::new();
2686 0 :
2687 0 : let systems_created = UIntGauge::new(
2688 0 : "pageserver_tokio_epoll_uring_systems_created",
2689 0 : "counter of tokio-epoll-uring systems that were created",
2690 0 : )
2691 0 : .unwrap();
2692 0 : descs.extend(
2693 0 : metrics::core::Collector::desc(&systems_created)
2694 0 : .into_iter()
2695 0 : .cloned(),
2696 0 : );
2697 0 :
2698 0 : let systems_destroyed = UIntGauge::new(
2699 0 : "pageserver_tokio_epoll_uring_systems_destroyed",
2700 0 : "counter of tokio-epoll-uring systems that were destroyed",
2701 0 : )
2702 0 : .unwrap();
2703 0 : descs.extend(
2704 0 : metrics::core::Collector::desc(&systems_destroyed)
2705 0 : .into_iter()
2706 0 : .cloned(),
2707 0 : );
2708 0 :
2709 0 : Self {
2710 0 : descs,
2711 0 : systems_created,
2712 0 : systems_destroyed,
2713 0 : }
2714 0 : }
2715 : }
2716 :
2717 80 : pub(crate) static THREAD_LOCAL_LAUNCH_SUCCESSES: Lazy<metrics::IntCounter> = Lazy::new(|| {
2718 : register_int_counter!(
2719 : "pageserver_tokio_epoll_uring_pageserver_thread_local_launch_success_count",
2720 : "Number of times where thread_local_system creation spanned multiple executor threads",
2721 : )
2722 80 : .unwrap()
2723 80 : });
2724 :
2725 0 : pub(crate) static THREAD_LOCAL_LAUNCH_FAILURES: Lazy<metrics::IntCounter> = Lazy::new(|| {
2726 : register_int_counter!(
2727 : "pageserver_tokio_epoll_uring_pageserver_thread_local_launch_failures_count",
2728 : "Number of times thread_local_system creation failed and was retried after back-off.",
2729 : )
2730 0 : .unwrap()
2731 0 : });
2732 : }
2733 :
2734 : pub(crate) mod tenant_throttling {
2735 : use metrics::{register_int_counter_vec, IntCounter};
2736 : use once_cell::sync::Lazy;
2737 :
2738 : use crate::tenant::{self, throttle::Metric};
2739 :
2740 : pub(crate) struct TimelineGet {
2741 : wait_time: IntCounter,
2742 : count: IntCounter,
2743 : }
2744 :
2745 139 : pub(crate) static TIMELINE_GET: Lazy<TimelineGet> = Lazy::new(|| {
2746 139 : static WAIT_USECS: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
2747 139 : register_int_counter_vec!(
2748 139 : "pageserver_tenant_throttling_wait_usecs_sum_global",
2749 139 : "Sum of microseconds that tenants spent waiting for a tenant throttle of a given kind.",
2750 139 : &["kind"]
2751 139 : )
2752 139 : .unwrap()
2753 139 : });
2754 139 :
2755 139 : static WAIT_COUNT: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
2756 139 : register_int_counter_vec!(
2757 139 : "pageserver_tenant_throttling_count_global",
2758 139 : "Count of tenant throttlings, by kind of throttle.",
2759 139 : &["kind"]
2760 139 : )
2761 139 : .unwrap()
2762 139 : });
2763 139 :
2764 139 : let kind = "timeline_get";
2765 139 : TimelineGet {
2766 139 : wait_time: WAIT_USECS.with_label_values(&[kind]),
2767 139 : count: WAIT_COUNT.with_label_values(&[kind]),
2768 139 : }
2769 139 : });
2770 :
2771 : impl Metric for &'static TimelineGet {
2772 : #[inline(always)]
2773 0 : fn observe_throttling(
2774 0 : &self,
2775 0 : tenant::throttle::Observation { wait_time }: &tenant::throttle::Observation,
2776 0 : ) {
2777 0 : let val = u64::try_from(wait_time.as_micros()).unwrap();
2778 0 : self.wait_time.inc_by(val);
2779 0 : self.count.inc();
2780 0 : }
2781 : }
2782 : }
2783 :
2784 : pub(crate) mod disk_usage_based_eviction {
2785 : use super::*;
2786 :
2787 : pub(crate) struct Metrics {
2788 : pub(crate) tenant_collection_time: Histogram,
2789 : pub(crate) tenant_layer_count: Histogram,
2790 : pub(crate) layers_collected: IntCounter,
2791 : pub(crate) layers_selected: IntCounter,
2792 : pub(crate) layers_evicted: IntCounter,
2793 : }
2794 :
2795 : impl Default for Metrics {
2796 0 : fn default() -> Self {
2797 0 : let tenant_collection_time = register_histogram!(
2798 : "pageserver_disk_usage_based_eviction_tenant_collection_seconds",
2799 : "Time spent collecting layers from a tenant -- not normalized by collected layer amount",
2800 : vec![0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0, 10.0]
2801 : )
2802 0 : .unwrap();
2803 0 :
2804 0 : let tenant_layer_count = register_histogram!(
2805 : "pageserver_disk_usage_based_eviction_tenant_collected_layers",
2806 : "Amount of layers gathered from a tenant",
2807 : vec![5.0, 50.0, 500.0, 5000.0, 50000.0]
2808 : )
2809 0 : .unwrap();
2810 0 :
2811 0 : let layers_collected = register_int_counter!(
2812 : "pageserver_disk_usage_based_eviction_collected_layers_total",
2813 : "Amount of layers collected"
2814 : )
2815 0 : .unwrap();
2816 0 :
2817 0 : let layers_selected = register_int_counter!(
2818 : "pageserver_disk_usage_based_eviction_select_layers_total",
2819 : "Amount of layers selected"
2820 : )
2821 0 : .unwrap();
2822 0 :
2823 0 : let layers_evicted = register_int_counter!(
2824 : "pageserver_disk_usage_based_eviction_evicted_layers_total",
2825 : "Amount of layers successfully evicted"
2826 : )
2827 0 : .unwrap();
2828 0 :
2829 0 : Self {
2830 0 : tenant_collection_time,
2831 0 : tenant_layer_count,
2832 0 : layers_collected,
2833 0 : layers_selected,
2834 0 : layers_evicted,
2835 0 : }
2836 0 : }
2837 : }
2838 :
2839 : pub(crate) static METRICS: Lazy<Metrics> = Lazy::new(Metrics::default);
2840 : }
2841 :
2842 133 : static TOKIO_EXECUTOR_THREAD_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
2843 : register_uint_gauge_vec!(
2844 : "pageserver_tokio_executor_thread_configured_count",
2845 : "Total number of configued tokio executor threads in the process.
2846 : The `setup` label denotes whether we're running with multiple runtimes or a single runtime.",
2847 : &["setup"],
2848 : )
2849 133 : .unwrap()
2850 133 : });
2851 :
2852 133 : pub(crate) fn set_tokio_runtime_setup(setup: &str, num_threads: NonZeroUsize) {
2853 133 : static SERIALIZE: std::sync::Mutex<()> = std::sync::Mutex::new(());
2854 133 : let _guard = SERIALIZE.lock().unwrap();
2855 133 : TOKIO_EXECUTOR_THREAD_COUNT.reset();
2856 133 : TOKIO_EXECUTOR_THREAD_COUNT
2857 133 : .get_metric_with_label_values(&[setup])
2858 133 : .unwrap()
2859 133 : .set(u64::try_from(num_threads.get()).unwrap());
2860 133 : }
2861 :
2862 0 : pub fn preinitialize_metrics() {
2863 0 : // Python tests need these and on some we do alerting.
2864 0 : //
2865 0 : // FIXME(4813): make it so that we have no top level metrics as this fn will easily fall out of
2866 0 : // order:
2867 0 : // - global metrics reside in a Lazy<PageserverMetrics>
2868 0 : // - access via crate::metrics::PS_METRICS.some_metric.inc()
2869 0 : // - could move the statics into TimelineMetrics::new()?
2870 0 :
2871 0 : // counters
2872 0 : [
2873 0 : &UNEXPECTED_ONDEMAND_DOWNLOADS,
2874 0 : &WALRECEIVER_STARTED_CONNECTIONS,
2875 0 : &WALRECEIVER_BROKER_UPDATES,
2876 0 : &WALRECEIVER_CANDIDATES_ADDED,
2877 0 : &WALRECEIVER_CANDIDATES_REMOVED,
2878 0 : &tokio_epoll_uring::THREAD_LOCAL_LAUNCH_FAILURES,
2879 0 : &tokio_epoll_uring::THREAD_LOCAL_LAUNCH_SUCCESSES,
2880 0 : &REMOTE_ONDEMAND_DOWNLOADED_LAYERS,
2881 0 : &REMOTE_ONDEMAND_DOWNLOADED_BYTES,
2882 0 : ]
2883 0 : .into_iter()
2884 0 : .for_each(|c| {
2885 0 : Lazy::force(c);
2886 0 : });
2887 0 :
2888 0 : // Deletion queue stats
2889 0 : Lazy::force(&DELETION_QUEUE);
2890 0 :
2891 0 : // Tenant stats
2892 0 : Lazy::force(&TENANT);
2893 0 :
2894 0 : // Tenant manager stats
2895 0 : Lazy::force(&TENANT_MANAGER);
2896 0 :
2897 0 : Lazy::force(&crate::tenant::storage_layer::layer::LAYER_IMPL_METRICS);
2898 0 : Lazy::force(&disk_usage_based_eviction::METRICS);
2899 :
2900 0 : for state_name in pageserver_api::models::TenantState::VARIANTS {
2901 0 : // initialize the metric for all gauges, otherwise the time series might seemingly show
2902 0 : // values from last restart.
2903 0 : TENANT_STATE_METRIC.with_label_values(&[state_name]).set(0);
2904 0 : }
2905 :
2906 : // countervecs
2907 0 : [&BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT]
2908 0 : .into_iter()
2909 0 : .for_each(|c| {
2910 0 : Lazy::force(c);
2911 0 : });
2912 0 :
2913 0 : // gauges
2914 0 : WALRECEIVER_ACTIVE_MANAGERS.get();
2915 0 :
2916 0 : // histograms
2917 0 : [
2918 0 : &READ_NUM_LAYERS_VISITED,
2919 0 : &VEC_READ_NUM_LAYERS_VISITED,
2920 0 : &WAIT_LSN_TIME,
2921 0 : &WAL_REDO_TIME,
2922 0 : &WAL_REDO_RECORDS_HISTOGRAM,
2923 0 : &WAL_REDO_BYTES_HISTOGRAM,
2924 0 : &WAL_REDO_PROCESS_LAUNCH_DURATION_HISTOGRAM,
2925 0 : ]
2926 0 : .into_iter()
2927 0 : .for_each(|h| {
2928 0 : Lazy::force(h);
2929 0 : });
2930 0 :
2931 0 : // Custom
2932 0 : Lazy::force(&RECONSTRUCT_TIME);
2933 0 : Lazy::force(&tenant_throttling::TIMELINE_GET);
2934 0 : Lazy::force(&BASEBACKUP_QUERY_TIME);
2935 0 : }
|
