Line data Source code
1 : use enum_map::EnumMap;
2 : use metrics::{
3 : register_counter_vec, register_gauge_vec, register_histogram, register_histogram_vec,
4 : register_int_counter, register_int_counter_pair_vec, register_int_counter_vec,
5 : register_int_gauge, register_int_gauge_vec, register_uint_gauge, register_uint_gauge_vec,
6 : Counter, CounterVec, GaugeVec, Histogram, HistogramVec, IntCounter, IntCounterPair,
7 : IntCounterPairVec, IntCounterVec, IntGauge, IntGaugeVec, UIntGauge, UIntGaugeVec,
8 : };
9 : use once_cell::sync::Lazy;
10 : use pageserver_api::shard::TenantShardId;
11 : use postgres_backend::{is_expected_io_error, QueryError};
12 : use pq_proto::framed::ConnectionError;
13 : use strum::{EnumCount, VariantNames};
14 : use strum_macros::{IntoStaticStr, VariantNames};
15 : use tracing::warn;
16 : use utils::id::TimelineId;
17 :
18 : /// Prometheus histogram buckets (in seconds) for operations in the critical
19 : /// path. In other words, operations that directly affect that latency of user
20 : /// queries.
21 : ///
22 : /// The buckets capture the majority of latencies in the microsecond and
23 : /// millisecond range but also extend far enough up to distinguish "bad" from
24 : /// "really bad".
25 : const CRITICAL_OP_BUCKETS: &[f64] = &[
26 : 0.000_001, 0.000_010, 0.000_100, // 1 us, 10 us, 100 us
27 : 0.001_000, 0.010_000, 0.100_000, // 1 ms, 10 ms, 100 ms
28 : 1.0, 10.0, 100.0, // 1 s, 10 s, 100 s
29 : ];
30 :
31 : // Metrics collected on operations on the storage repository.
32 3312 : #[derive(Debug, VariantNames, IntoStaticStr)]
33 : #[strum(serialize_all = "kebab_case")]
34 : pub(crate) enum StorageTimeOperation {
35 : #[strum(serialize = "layer flush")]
36 : LayerFlush,
37 :
38 : #[strum(serialize = "compact")]
39 : Compact,
40 :
41 : #[strum(serialize = "create images")]
42 : CreateImages,
43 :
44 : #[strum(serialize = "logical size")]
45 : LogicalSize,
46 :
47 : #[strum(serialize = "imitate logical size")]
48 : ImitateLogicalSize,
49 :
50 : #[strum(serialize = "load layer map")]
51 : LoadLayerMap,
52 :
53 : #[strum(serialize = "gc")]
54 : Gc,
55 :
56 : #[strum(serialize = "find gc cutoffs")]
57 : FindGcCutoffs,
58 : }
59 :
60 170 : pub(crate) static STORAGE_TIME_SUM_PER_TIMELINE: Lazy<CounterVec> = Lazy::new(|| {
61 170 : register_counter_vec!(
62 170 : "pageserver_storage_operations_seconds_sum",
63 170 : "Total time spent on storage operations with operation, tenant and timeline dimensions",
64 170 : &["operation", "tenant_id", "shard_id", "timeline_id"],
65 170 : )
66 170 : .expect("failed to define a metric")
67 170 : });
68 :
69 170 : pub(crate) static STORAGE_TIME_COUNT_PER_TIMELINE: Lazy<IntCounterVec> = Lazy::new(|| {
70 170 : register_int_counter_vec!(
71 170 : "pageserver_storage_operations_seconds_count",
72 170 : "Count of storage operations with operation, tenant and timeline dimensions",
73 170 : &["operation", "tenant_id", "shard_id", "timeline_id"],
74 170 : )
75 170 : .expect("failed to define a metric")
76 170 : });
77 :
78 : // Buckets for background operations like compaction, GC, size calculation
79 : const STORAGE_OP_BUCKETS: &[f64] = &[0.010, 0.100, 1.0, 10.0, 100.0, 1000.0];
80 :
81 170 : pub(crate) static STORAGE_TIME_GLOBAL: Lazy<HistogramVec> = Lazy::new(|| {
82 170 : register_histogram_vec!(
83 170 : "pageserver_storage_operations_seconds_global",
84 170 : "Time spent on storage operations",
85 170 : &["operation"],
86 170 : STORAGE_OP_BUCKETS.into(),
87 170 : )
88 170 : .expect("failed to define a metric")
89 170 : });
90 :
91 0 : pub(crate) static READ_NUM_LAYERS_VISITED: Lazy<Histogram> = Lazy::new(|| {
92 0 : register_histogram!(
93 0 : "pageserver_layers_visited_per_read_global",
94 0 : "Number of layers visited to reconstruct one key",
95 0 : vec![1.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0],
96 0 : )
97 0 : .expect("failed to define a metric")
98 0 : });
99 :
100 166 : pub(crate) static VEC_READ_NUM_LAYERS_VISITED: Lazy<Histogram> = Lazy::new(|| {
101 166 : register_histogram!(
102 166 : "pageserver_layers_visited_per_vectored_read_global",
103 166 : "Average number of layers visited to reconstruct one key",
104 166 : vec![1.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0],
105 166 : )
106 166 : .expect("failed to define a metric")
107 166 : });
108 :
109 : // Metrics collected on operations on the storage repository.
110 : #[derive(
111 672 : Clone, Copy, enum_map::Enum, strum_macros::EnumString, strum_macros::Display, IntoStaticStr,
112 : )]
113 : pub(crate) enum GetKind {
114 : Singular,
115 : Vectored,
116 : }
117 :
118 : pub(crate) struct ReconstructTimeMetrics {
119 : singular: Histogram,
120 : vectored: Histogram,
121 : }
122 :
123 168 : pub(crate) static RECONSTRUCT_TIME: Lazy<ReconstructTimeMetrics> = Lazy::new(|| {
124 168 : let inner = register_histogram_vec!(
125 168 : "pageserver_getpage_reconstruct_seconds",
126 168 : "Time spent in reconstruct_value (reconstruct a page from deltas)",
127 168 : &["get_kind"],
128 168 : CRITICAL_OP_BUCKETS.into(),
129 168 : )
130 168 : .expect("failed to define a metric");
131 168 :
132 168 : ReconstructTimeMetrics {
133 168 : singular: inner.with_label_values(&[GetKind::Singular.into()]),
134 168 : vectored: inner.with_label_values(&[GetKind::Vectored.into()]),
135 168 : }
136 168 : });
137 :
138 : impl ReconstructTimeMetrics {
139 626558 : pub(crate) fn for_get_kind(&self, get_kind: GetKind) -> &Histogram {
140 626558 : match get_kind {
141 626126 : GetKind::Singular => &self.singular,
142 432 : GetKind::Vectored => &self.vectored,
143 : }
144 626558 : }
145 : }
146 :
147 : pub(crate) struct ReconstructDataTimeMetrics {
148 : singular: Histogram,
149 : vectored: Histogram,
150 : }
151 :
152 : impl ReconstructDataTimeMetrics {
153 626574 : pub(crate) fn for_get_kind(&self, get_kind: GetKind) -> &Histogram {
154 626574 : match get_kind {
155 626142 : GetKind::Singular => &self.singular,
156 432 : GetKind::Vectored => &self.vectored,
157 : }
158 626574 : }
159 : }
160 :
161 168 : pub(crate) static GET_RECONSTRUCT_DATA_TIME: Lazy<ReconstructDataTimeMetrics> = Lazy::new(|| {
162 168 : let inner = register_histogram_vec!(
163 168 : "pageserver_getpage_get_reconstruct_data_seconds",
164 168 : "Time spent in get_reconstruct_value_data",
165 168 : &["get_kind"],
166 168 : CRITICAL_OP_BUCKETS.into(),
167 168 : )
168 168 : .expect("failed to define a metric");
169 168 :
170 168 : ReconstructDataTimeMetrics {
171 168 : singular: inner.with_label_values(&[GetKind::Singular.into()]),
172 168 : vectored: inner.with_label_values(&[GetKind::Vectored.into()]),
173 168 : }
174 168 : });
175 :
176 : pub(crate) struct GetVectoredLatency {
177 : map: EnumMap<TaskKind, Option<Histogram>>,
178 : }
179 :
180 : #[allow(dead_code)]
181 : pub(crate) struct ScanLatency {
182 : map: EnumMap<TaskKind, Option<Histogram>>,
183 : }
184 :
185 : impl GetVectoredLatency {
186 : // Only these task types perform vectored gets. Filter all other tasks out to reduce total
187 : // cardinality of the metric.
188 : const TRACKED_TASK_KINDS: [TaskKind; 2] = [TaskKind::Compaction, TaskKind::PageRequestHandler];
189 :
190 1132 : pub(crate) fn for_task_kind(&self, task_kind: TaskKind) -> Option<&Histogram> {
191 1132 : self.map[task_kind].as_ref()
192 1132 : }
193 : }
194 :
195 : impl ScanLatency {
196 : // Only these task types perform vectored gets. Filter all other tasks out to reduce total
197 : // cardinality of the metric.
198 : const TRACKED_TASK_KINDS: [TaskKind; 1] = [TaskKind::PageRequestHandler];
199 :
200 12 : pub(crate) fn for_task_kind(&self, task_kind: TaskKind) -> Option<&Histogram> {
201 12 : self.map[task_kind].as_ref()
202 12 : }
203 : }
204 :
205 : pub(crate) struct ScanLatencyOngoingRecording<'a> {
206 : parent: &'a Histogram,
207 : start: std::time::Instant,
208 : }
209 :
210 : impl<'a> ScanLatencyOngoingRecording<'a> {
211 0 : pub(crate) fn start_recording(parent: &'a Histogram) -> ScanLatencyOngoingRecording<'a> {
212 0 : let start = Instant::now();
213 0 : ScanLatencyOngoingRecording { parent, start }
214 0 : }
215 :
216 0 : pub(crate) fn observe(self, throttled: Option<Duration>) {
217 0 : let elapsed = self.start.elapsed();
218 0 : let ex_throttled = if let Some(throttled) = throttled {
219 0 : elapsed.checked_sub(throttled)
220 : } else {
221 0 : Some(elapsed)
222 : };
223 0 : if let Some(ex_throttled) = ex_throttled {
224 0 : self.parent.observe(ex_throttled.as_secs_f64());
225 0 : } else {
226 0 : use utils::rate_limit::RateLimit;
227 0 : static LOGGED: Lazy<Mutex<RateLimit>> =
228 0 : Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(10))));
229 0 : let mut rate_limit = LOGGED.lock().unwrap();
230 0 : rate_limit.call(|| {
231 0 : warn!("error deducting time spent throttled; this message is logged at a global rate limit");
232 0 : });
233 0 : }
234 0 : }
235 : }
236 :
237 162 : pub(crate) static GET_VECTORED_LATENCY: Lazy<GetVectoredLatency> = Lazy::new(|| {
238 162 : let inner = register_histogram_vec!(
239 162 : "pageserver_get_vectored_seconds",
240 162 : "Time spent in get_vectored, excluding time spent in timeline_get_throttle.",
241 162 : &["task_kind"],
242 162 : CRITICAL_OP_BUCKETS.into(),
243 162 : )
244 162 : .expect("failed to define a metric");
245 162 :
246 162 : GetVectoredLatency {
247 4860 : map: EnumMap::from_array(std::array::from_fn(|task_kind_idx| {
248 4860 : let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind_idx);
249 4860 :
250 4860 : if GetVectoredLatency::TRACKED_TASK_KINDS.contains(&task_kind) {
251 324 : let task_kind = task_kind.into();
252 324 : Some(inner.with_label_values(&[task_kind]))
253 : } else {
254 4536 : None
255 : }
256 4860 : })),
257 162 : }
258 162 : });
259 :
260 4 : pub(crate) static SCAN_LATENCY: Lazy<ScanLatency> = Lazy::new(|| {
261 4 : let inner = register_histogram_vec!(
262 4 : "pageserver_scan_seconds",
263 4 : "Time spent in scan, excluding time spent in timeline_get_throttle.",
264 4 : &["task_kind"],
265 4 : CRITICAL_OP_BUCKETS.into(),
266 4 : )
267 4 : .expect("failed to define a metric");
268 4 :
269 4 : ScanLatency {
270 120 : map: EnumMap::from_array(std::array::from_fn(|task_kind_idx| {
271 120 : let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind_idx);
272 120 :
273 120 : if ScanLatency::TRACKED_TASK_KINDS.contains(&task_kind) {
274 4 : let task_kind = task_kind.into();
275 4 : Some(inner.with_label_values(&[task_kind]))
276 : } else {
277 116 : None
278 : }
279 120 : })),
280 4 : }
281 4 : });
282 :
283 : pub(crate) struct PageCacheMetricsForTaskKind {
284 : pub read_accesses_immutable: IntCounter,
285 : pub read_hits_immutable: IntCounter,
286 : }
287 :
288 : pub(crate) struct PageCacheMetrics {
289 : map: EnumMap<TaskKind, EnumMap<PageContentKind, PageCacheMetricsForTaskKind>>,
290 : }
291 :
292 88 : static PAGE_CACHE_READ_HITS: Lazy<IntCounterVec> = Lazy::new(|| {
293 88 : register_int_counter_vec!(
294 88 : "pageserver_page_cache_read_hits_total",
295 88 : "Number of read accesses to the page cache that hit",
296 88 : &["task_kind", "key_kind", "content_kind", "hit_kind"]
297 88 : )
298 88 : .expect("failed to define a metric")
299 88 : });
300 :
301 88 : static PAGE_CACHE_READ_ACCESSES: Lazy<IntCounterVec> = Lazy::new(|| {
302 88 : register_int_counter_vec!(
303 88 : "pageserver_page_cache_read_accesses_total",
304 88 : "Number of read accesses to the page cache",
305 88 : &["task_kind", "key_kind", "content_kind"]
306 88 : )
307 88 : .expect("failed to define a metric")
308 88 : });
309 :
310 88 : pub(crate) static PAGE_CACHE: Lazy<PageCacheMetrics> = Lazy::new(|| PageCacheMetrics {
311 2640 : map: EnumMap::from_array(std::array::from_fn(|task_kind| {
312 2640 : let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind);
313 2640 : let task_kind: &'static str = task_kind.into();
314 21120 : EnumMap::from_array(std::array::from_fn(|content_kind| {
315 21120 : let content_kind = <PageContentKind as enum_map::Enum>::from_usize(content_kind);
316 21120 : let content_kind: &'static str = content_kind.into();
317 21120 : PageCacheMetricsForTaskKind {
318 21120 : read_accesses_immutable: {
319 21120 : PAGE_CACHE_READ_ACCESSES
320 21120 : .get_metric_with_label_values(&[task_kind, "immutable", content_kind])
321 21120 : .unwrap()
322 21120 : },
323 21120 :
324 21120 : read_hits_immutable: {
325 21120 : PAGE_CACHE_READ_HITS
326 21120 : .get_metric_with_label_values(&[task_kind, "immutable", content_kind, "-"])
327 21120 : .unwrap()
328 21120 : },
329 21120 : }
330 21120 : }))
331 2640 : })),
332 88 : });
333 :
334 : impl PageCacheMetrics {
335 971188 : pub(crate) fn for_ctx(&self, ctx: &RequestContext) -> &PageCacheMetricsForTaskKind {
336 971188 : &self.map[ctx.task_kind()][ctx.page_content_kind()]
337 971188 : }
338 : }
339 :
340 : pub(crate) struct PageCacheSizeMetrics {
341 : pub max_bytes: UIntGauge,
342 :
343 : pub current_bytes_immutable: UIntGauge,
344 : }
345 :
346 88 : static PAGE_CACHE_SIZE_CURRENT_BYTES: Lazy<UIntGaugeVec> = Lazy::new(|| {
347 88 : register_uint_gauge_vec!(
348 88 : "pageserver_page_cache_size_current_bytes",
349 88 : "Current size of the page cache in bytes, by key kind",
350 88 : &["key_kind"]
351 88 : )
352 88 : .expect("failed to define a metric")
353 88 : });
354 :
355 : pub(crate) static PAGE_CACHE_SIZE: Lazy<PageCacheSizeMetrics> =
356 88 : Lazy::new(|| PageCacheSizeMetrics {
357 88 : max_bytes: {
358 88 : register_uint_gauge!(
359 88 : "pageserver_page_cache_size_max_bytes",
360 88 : "Maximum size of the page cache in bytes"
361 88 : )
362 88 : .expect("failed to define a metric")
363 88 : },
364 88 : current_bytes_immutable: {
365 88 : PAGE_CACHE_SIZE_CURRENT_BYTES
366 88 : .get_metric_with_label_values(&["immutable"])
367 88 : .unwrap()
368 88 : },
369 88 : });
370 :
371 : pub(crate) mod page_cache_eviction_metrics {
372 : use std::num::NonZeroUsize;
373 :
374 : use metrics::{register_int_counter_vec, IntCounter, IntCounterVec};
375 : use once_cell::sync::Lazy;
376 :
377 : #[derive(Clone, Copy)]
378 : pub(crate) enum Outcome {
379 : FoundSlotUnused { iters: NonZeroUsize },
380 : FoundSlotEvicted { iters: NonZeroUsize },
381 : ItersExceeded { iters: NonZeroUsize },
382 : }
383 :
384 88 : static ITERS_TOTAL_VEC: Lazy<IntCounterVec> = Lazy::new(|| {
385 88 : register_int_counter_vec!(
386 88 : "pageserver_page_cache_find_victim_iters_total",
387 88 : "Counter for the number of iterations in the find_victim loop",
388 88 : &["outcome"],
389 88 : )
390 88 : .expect("failed to define a metric")
391 88 : });
392 :
393 88 : static CALLS_VEC: Lazy<IntCounterVec> = Lazy::new(|| {
394 88 : register_int_counter_vec!(
395 88 : "pageserver_page_cache_find_victim_calls",
396 88 : "Incremented at the end of each find_victim() call.\
397 88 : Filter by outcome to get e.g., eviction rate.",
398 88 : &["outcome"]
399 88 : )
400 88 : .unwrap()
401 88 : });
402 :
403 32049 : pub(crate) fn observe(outcome: Outcome) {
404 : macro_rules! dry {
405 : ($label:literal, $iters:expr) => {{
406 : static LABEL: &'static str = $label;
407 : static ITERS_TOTAL: Lazy<IntCounter> =
408 108 : Lazy::new(|| ITERS_TOTAL_VEC.with_label_values(&[LABEL]));
409 : static CALLS: Lazy<IntCounter> =
410 108 : Lazy::new(|| CALLS_VEC.with_label_values(&[LABEL]));
411 : ITERS_TOTAL.inc_by(($iters.get()) as u64);
412 : CALLS.inc();
413 : }};
414 : }
415 32049 : match outcome {
416 1560 : Outcome::FoundSlotUnused { iters } => dry!("found_empty", iters),
417 30489 : Outcome::FoundSlotEvicted { iters } => {
418 30489 : dry!("found_evicted", iters)
419 : }
420 0 : Outcome::ItersExceeded { iters } => {
421 0 : dry!("err_iters_exceeded", iters);
422 0 : super::page_cache_errors_inc(super::PageCacheErrorKind::EvictIterLimit);
423 0 : }
424 : }
425 32049 : }
426 : }
427 :
428 0 : static PAGE_CACHE_ERRORS: Lazy<IntCounterVec> = Lazy::new(|| {
429 0 : register_int_counter_vec!(
430 0 : "page_cache_errors_total",
431 0 : "Number of timeouts while acquiring a pinned slot in the page cache",
432 0 : &["error_kind"]
433 0 : )
434 0 : .expect("failed to define a metric")
435 0 : });
436 :
437 0 : #[derive(IntoStaticStr)]
438 : #[strum(serialize_all = "kebab_case")]
439 : pub(crate) enum PageCacheErrorKind {
440 : AcquirePinnedSlotTimeout,
441 : EvictIterLimit,
442 : }
443 :
444 0 : pub(crate) fn page_cache_errors_inc(error_kind: PageCacheErrorKind) {
445 0 : PAGE_CACHE_ERRORS
446 0 : .get_metric_with_label_values(&[error_kind.into()])
447 0 : .unwrap()
448 0 : .inc();
449 0 : }
450 :
451 18 : pub(crate) static WAIT_LSN_TIME: Lazy<Histogram> = Lazy::new(|| {
452 18 : register_histogram!(
453 18 : "pageserver_wait_lsn_seconds",
454 18 : "Time spent waiting for WAL to arrive",
455 18 : CRITICAL_OP_BUCKETS.into(),
456 18 : )
457 18 : .expect("failed to define a metric")
458 18 : });
459 :
460 170 : static LAST_RECORD_LSN: Lazy<IntGaugeVec> = Lazy::new(|| {
461 170 : register_int_gauge_vec!(
462 170 : "pageserver_last_record_lsn",
463 170 : "Last record LSN grouped by timeline",
464 170 : &["tenant_id", "shard_id", "timeline_id"]
465 170 : )
466 170 : .expect("failed to define a metric")
467 170 : });
468 :
469 170 : static PITR_HISTORY_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
470 170 : register_uint_gauge_vec!(
471 170 : "pageserver_pitr_history_size",
472 170 : "Data written since PITR cutoff on this timeline",
473 170 : &["tenant_id", "shard_id", "timeline_id"]
474 170 : )
475 170 : .expect("failed to define a metric")
476 170 : });
477 :
478 1688 : #[derive(strum_macros::EnumString, strum_macros::Display, strum_macros::IntoStaticStr)]
479 : #[strum(serialize_all = "kebab_case")]
480 : pub(crate) enum MetricLayerKind {
481 : Delta,
482 : Image,
483 : }
484 :
485 170 : static TIMELINE_LAYER_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
486 170 : register_uint_gauge_vec!(
487 170 : "pageserver_layer_bytes",
488 170 : "Sum of layer physical sizes in bytes",
489 170 : &["tenant_id", "shard_id", "timeline_id", "kind"]
490 170 : )
491 170 : .expect("failed to define a metric")
492 170 : });
493 :
494 170 : static TIMELINE_LAYER_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
495 170 : register_uint_gauge_vec!(
496 170 : "pageserver_layer_count",
497 170 : "Number of layers that exist",
498 170 : &["tenant_id", "shard_id", "timeline_id", "kind"]
499 170 : )
500 170 : .expect("failed to define a metric")
501 170 : });
502 :
503 170 : static TIMELINE_ARCHIVE_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
504 170 : register_uint_gauge_vec!(
505 170 : "pageserver_archive_size",
506 170 : "Timeline's logical size if it is considered eligible for archival (outside PITR window), else zero",
507 170 : &["tenant_id", "shard_id", "timeline_id"]
508 170 : )
509 170 : .expect("failed to define a metric")
510 170 : });
511 :
512 170 : static STANDBY_HORIZON: Lazy<IntGaugeVec> = Lazy::new(|| {
513 170 : register_int_gauge_vec!(
514 170 : "pageserver_standby_horizon",
515 170 : "Standby apply LSN for which GC is hold off, by timeline.",
516 170 : &["tenant_id", "shard_id", "timeline_id"]
517 170 : )
518 170 : .expect("failed to define a metric")
519 170 : });
520 :
521 170 : static RESIDENT_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
522 170 : register_uint_gauge_vec!(
523 170 : "pageserver_resident_physical_size",
524 170 : "The size of the layer files present in the pageserver's filesystem, for attached locations.",
525 170 : &["tenant_id", "shard_id", "timeline_id"]
526 170 : )
527 170 : .expect("failed to define a metric")
528 170 : });
529 :
530 170 : static VISIBLE_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
531 170 : register_uint_gauge_vec!(
532 170 : "pageserver_visible_physical_size",
533 170 : "The size of the layer files present in the pageserver's filesystem.",
534 170 : &["tenant_id", "shard_id", "timeline_id"]
535 170 : )
536 170 : .expect("failed to define a metric")
537 170 : });
538 :
539 166 : pub(crate) static RESIDENT_PHYSICAL_SIZE_GLOBAL: Lazy<UIntGauge> = Lazy::new(|| {
540 166 : register_uint_gauge!(
541 166 : "pageserver_resident_physical_size_global",
542 166 : "Like `pageserver_resident_physical_size`, but without tenant/timeline dimensions."
543 166 : )
544 166 : .expect("failed to define a metric")
545 166 : });
546 :
547 170 : static REMOTE_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
548 170 : register_uint_gauge_vec!(
549 170 : "pageserver_remote_physical_size",
550 170 : "The size of the layer files present in the remote storage that are listed in the remote index_part.json.",
551 170 : // Corollary: If any files are missing from the index part, they won't be included here.
552 170 : &["tenant_id", "shard_id", "timeline_id"]
553 170 : )
554 170 : .expect("failed to define a metric")
555 170 : });
556 :
557 170 : static REMOTE_PHYSICAL_SIZE_GLOBAL: Lazy<UIntGauge> = Lazy::new(|| {
558 170 : register_uint_gauge!(
559 170 : "pageserver_remote_physical_size_global",
560 170 : "Like `pageserver_remote_physical_size`, but without tenant/timeline dimensions."
561 170 : )
562 170 : .expect("failed to define a metric")
563 170 : });
564 :
565 4 : pub(crate) static REMOTE_ONDEMAND_DOWNLOADED_LAYERS: Lazy<IntCounter> = Lazy::new(|| {
566 4 : register_int_counter!(
567 4 : "pageserver_remote_ondemand_downloaded_layers_total",
568 4 : "Total on-demand downloaded layers"
569 4 : )
570 4 : .unwrap()
571 4 : });
572 :
573 4 : pub(crate) static REMOTE_ONDEMAND_DOWNLOADED_BYTES: Lazy<IntCounter> = Lazy::new(|| {
574 4 : register_int_counter!(
575 4 : "pageserver_remote_ondemand_downloaded_bytes_total",
576 4 : "Total bytes of layers on-demand downloaded",
577 4 : )
578 4 : .unwrap()
579 4 : });
580 :
581 170 : static CURRENT_LOGICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
582 170 : register_uint_gauge_vec!(
583 170 : "pageserver_current_logical_size",
584 170 : "Current logical size grouped by timeline",
585 170 : &["tenant_id", "shard_id", "timeline_id"]
586 170 : )
587 170 : .expect("failed to define current logical size metric")
588 170 : });
589 :
590 170 : static AUX_FILE_SIZE: Lazy<IntGaugeVec> = Lazy::new(|| {
591 170 : register_int_gauge_vec!(
592 170 : "pageserver_aux_file_estimated_size",
593 170 : "The size of all aux files for a timeline in aux file v2 store.",
594 170 : &["tenant_id", "shard_id", "timeline_id"]
595 170 : )
596 170 : .expect("failed to define a metric")
597 170 : });
598 :
599 170 : static VALID_LSN_LEASE_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
600 170 : register_uint_gauge_vec!(
601 170 : "pageserver_valid_lsn_lease_count",
602 170 : "The number of valid leases after refreshing gc info.",
603 170 : &["tenant_id", "shard_id", "timeline_id"],
604 170 : )
605 170 : .expect("failed to define a metric")
606 170 : });
607 :
608 0 : pub(crate) static CIRCUIT_BREAKERS_BROKEN: Lazy<IntCounter> = Lazy::new(|| {
609 0 : register_int_counter!(
610 0 : "pageserver_circuit_breaker_broken",
611 0 : "How many times a circuit breaker has broken"
612 0 : )
613 0 : .expect("failed to define a metric")
614 0 : });
615 :
616 0 : pub(crate) static CIRCUIT_BREAKERS_UNBROKEN: Lazy<IntCounter> = Lazy::new(|| {
617 0 : register_int_counter!(
618 0 : "pageserver_circuit_breaker_unbroken",
619 0 : "How many times a circuit breaker has been un-broken (recovered)"
620 0 : )
621 0 : .expect("failed to define a metric")
622 0 : });
623 :
624 162 : pub(crate) static COMPRESSION_IMAGE_INPUT_BYTES: Lazy<IntCounter> = Lazy::new(|| {
625 162 : register_int_counter!(
626 162 : "pageserver_compression_image_in_bytes_total",
627 162 : "Size of data written into image layers before compression"
628 162 : )
629 162 : .expect("failed to define a metric")
630 162 : });
631 :
632 162 : pub(crate) static COMPRESSION_IMAGE_INPUT_BYTES_CONSIDERED: Lazy<IntCounter> = Lazy::new(|| {
633 162 : register_int_counter!(
634 162 : "pageserver_compression_image_in_bytes_considered",
635 162 : "Size of potentially compressible data written into image layers before compression"
636 162 : )
637 162 : .expect("failed to define a metric")
638 162 : });
639 :
640 162 : pub(crate) static COMPRESSION_IMAGE_INPUT_BYTES_CHOSEN: Lazy<IntCounter> = Lazy::new(|| {
641 162 : register_int_counter!(
642 162 : "pageserver_compression_image_in_bytes_chosen",
643 162 : "Size of data whose compressed form was written into image layers"
644 162 : )
645 162 : .expect("failed to define a metric")
646 162 : });
647 :
648 162 : pub(crate) static COMPRESSION_IMAGE_OUTPUT_BYTES: Lazy<IntCounter> = Lazy::new(|| {
649 162 : register_int_counter!(
650 162 : "pageserver_compression_image_out_bytes_total",
651 162 : "Size of compressed image layer written"
652 162 : )
653 162 : .expect("failed to define a metric")
654 162 : });
655 :
656 : pub(crate) mod initial_logical_size {
657 : use metrics::{register_int_counter, register_int_counter_vec, IntCounter, IntCounterVec};
658 : use once_cell::sync::Lazy;
659 :
660 : pub(crate) struct StartCalculation(IntCounterVec);
661 170 : pub(crate) static START_CALCULATION: Lazy<StartCalculation> = Lazy::new(|| {
662 170 : StartCalculation(
663 170 : register_int_counter_vec!(
664 170 : "pageserver_initial_logical_size_start_calculation",
665 170 : "Incremented each time we start an initial logical size calculation attempt. \
666 170 : The `circumstances` label provides some additional details.",
667 170 : &["attempt", "circumstances"]
668 170 : )
669 170 : .unwrap(),
670 170 : )
671 170 : });
672 :
673 : struct DropCalculation {
674 : first: IntCounter,
675 : retry: IntCounter,
676 : }
677 :
678 170 : static DROP_CALCULATION: Lazy<DropCalculation> = Lazy::new(|| {
679 170 : let vec = register_int_counter_vec!(
680 170 : "pageserver_initial_logical_size_drop_calculation",
681 170 : "Incremented each time we abort a started size calculation attmpt.",
682 170 : &["attempt"]
683 170 : )
684 170 : .unwrap();
685 170 : DropCalculation {
686 170 : first: vec.with_label_values(&["first"]),
687 170 : retry: vec.with_label_values(&["retry"]),
688 170 : }
689 170 : });
690 :
691 : pub(crate) struct Calculated {
692 : pub(crate) births: IntCounter,
693 : pub(crate) deaths: IntCounter,
694 : }
695 :
696 170 : pub(crate) static CALCULATED: Lazy<Calculated> = Lazy::new(|| Calculated {
697 170 : births: register_int_counter!(
698 170 : "pageserver_initial_logical_size_finish_calculation",
699 170 : "Incremented every time we finish calculation of initial logical size.\
700 170 : If everything is working well, this should happen at most once per Timeline object."
701 170 : )
702 170 : .unwrap(),
703 170 : deaths: register_int_counter!(
704 170 : "pageserver_initial_logical_size_drop_finished_calculation",
705 170 : "Incremented when we drop a finished initial logical size calculation result.\
706 170 : Mainly useful to turn pageserver_initial_logical_size_finish_calculation into a gauge."
707 170 : )
708 170 : .unwrap(),
709 170 : });
710 :
711 : pub(crate) struct OngoingCalculationGuard {
712 : inc_drop_calculation: Option<IntCounter>,
713 : }
714 :
715 182 : #[derive(strum_macros::IntoStaticStr)]
716 : pub(crate) enum StartCircumstances {
717 : EmptyInitial,
718 : SkippedConcurrencyLimiter,
719 : AfterBackgroundTasksRateLimit,
720 : }
721 :
722 : impl StartCalculation {
723 182 : pub(crate) fn first(&self, circumstances: StartCircumstances) -> OngoingCalculationGuard {
724 182 : let circumstances_label: &'static str = circumstances.into();
725 182 : self.0
726 182 : .with_label_values(&["first", circumstances_label])
727 182 : .inc();
728 182 : OngoingCalculationGuard {
729 182 : inc_drop_calculation: Some(DROP_CALCULATION.first.clone()),
730 182 : }
731 182 : }
732 0 : pub(crate) fn retry(&self, circumstances: StartCircumstances) -> OngoingCalculationGuard {
733 0 : let circumstances_label: &'static str = circumstances.into();
734 0 : self.0
735 0 : .with_label_values(&["retry", circumstances_label])
736 0 : .inc();
737 0 : OngoingCalculationGuard {
738 0 : inc_drop_calculation: Some(DROP_CALCULATION.retry.clone()),
739 0 : }
740 0 : }
741 : }
742 :
743 : impl Drop for OngoingCalculationGuard {
744 182 : fn drop(&mut self) {
745 182 : if let Some(counter) = self.inc_drop_calculation.take() {
746 0 : counter.inc();
747 182 : }
748 182 : }
749 : }
750 :
751 : impl OngoingCalculationGuard {
752 182 : pub(crate) fn calculation_result_saved(mut self) -> FinishedCalculationGuard {
753 182 : drop(self.inc_drop_calculation.take());
754 182 : CALCULATED.births.inc();
755 182 : FinishedCalculationGuard {
756 182 : inc_on_drop: CALCULATED.deaths.clone(),
757 182 : }
758 182 : }
759 : }
760 :
761 : pub(crate) struct FinishedCalculationGuard {
762 : inc_on_drop: IntCounter,
763 : }
764 :
765 : impl Drop for FinishedCalculationGuard {
766 6 : fn drop(&mut self) {
767 6 : self.inc_on_drop.inc();
768 6 : }
769 : }
770 :
771 : // context: https://github.com/neondatabase/neon/issues/5963
772 : pub(crate) static TIMELINES_WHERE_WALRECEIVER_GOT_APPROXIMATE_SIZE: Lazy<IntCounter> =
773 0 : Lazy::new(|| {
774 0 : register_int_counter!(
775 0 : "pageserver_initial_logical_size_timelines_where_walreceiver_got_approximate_size",
776 0 : "Counter for the following event: walreceiver calls\
777 0 : Timeline::get_current_logical_size() and it returns `Approximate` for the first time."
778 0 : )
779 0 : .unwrap()
780 0 : });
781 : }
782 :
783 0 : static DIRECTORY_ENTRIES_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
784 0 : register_uint_gauge_vec!(
785 0 : "pageserver_directory_entries_count",
786 0 : "Sum of the entries in pageserver-stored directory listings",
787 0 : &["tenant_id", "shard_id", "timeline_id"]
788 0 : )
789 0 : .expect("failed to define a metric")
790 0 : });
791 :
792 172 : pub(crate) static TENANT_STATE_METRIC: Lazy<UIntGaugeVec> = Lazy::new(|| {
793 172 : register_uint_gauge_vec!(
794 172 : "pageserver_tenant_states_count",
795 172 : "Count of tenants per state",
796 172 : &["state"]
797 172 : )
798 172 : .expect("Failed to register pageserver_tenant_states_count metric")
799 172 : });
800 :
801 : /// A set of broken tenants.
802 : ///
803 : /// These are expected to be so rare that a set is fine. Set as in a new timeseries per each broken
804 : /// tenant.
805 12 : pub(crate) static BROKEN_TENANTS_SET: Lazy<UIntGaugeVec> = Lazy::new(|| {
806 12 : register_uint_gauge_vec!(
807 12 : "pageserver_broken_tenants_count",
808 12 : "Set of broken tenants",
809 12 : &["tenant_id", "shard_id"]
810 12 : )
811 12 : .expect("Failed to register pageserver_tenant_states_count metric")
812 12 : });
813 :
814 6 : pub(crate) static TENANT_SYNTHETIC_SIZE_METRIC: Lazy<UIntGaugeVec> = Lazy::new(|| {
815 6 : register_uint_gauge_vec!(
816 6 : "pageserver_tenant_synthetic_cached_size_bytes",
817 6 : "Synthetic size of each tenant in bytes",
818 6 : &["tenant_id"]
819 6 : )
820 6 : .expect("Failed to register pageserver_tenant_synthetic_cached_size_bytes metric")
821 6 : });
822 :
823 0 : pub(crate) static EVICTION_ITERATION_DURATION: Lazy<HistogramVec> = Lazy::new(|| {
824 0 : register_histogram_vec!(
825 0 : "pageserver_eviction_iteration_duration_seconds_global",
826 0 : "Time spent on a single eviction iteration",
827 0 : &["period_secs", "threshold_secs"],
828 0 : STORAGE_OP_BUCKETS.into(),
829 0 : )
830 0 : .expect("failed to define a metric")
831 0 : });
832 :
833 170 : static EVICTIONS: Lazy<IntCounterVec> = Lazy::new(|| {
834 170 : register_int_counter_vec!(
835 170 : "pageserver_evictions",
836 170 : "Number of layers evicted from the pageserver",
837 170 : &["tenant_id", "shard_id", "timeline_id"]
838 170 : )
839 170 : .expect("failed to define a metric")
840 170 : });
841 :
842 170 : static EVICTIONS_WITH_LOW_RESIDENCE_DURATION: Lazy<IntCounterVec> = Lazy::new(|| {
843 170 : register_int_counter_vec!(
844 170 : "pageserver_evictions_with_low_residence_duration",
845 170 : "If a layer is evicted that was resident for less than `low_threshold`, it is counted to this counter. \
846 170 : Residence duration is determined using the `residence_duration_data_source`.",
847 170 : &["tenant_id", "shard_id", "timeline_id", "residence_duration_data_source", "low_threshold_secs"]
848 170 : )
849 170 : .expect("failed to define a metric")
850 170 : });
851 :
852 0 : pub(crate) static UNEXPECTED_ONDEMAND_DOWNLOADS: Lazy<IntCounter> = Lazy::new(|| {
853 0 : register_int_counter!(
854 0 : "pageserver_unexpected_ondemand_downloads_count",
855 0 : "Number of unexpected on-demand downloads. \
856 0 : We log more context for each increment, so, forgo any labels in this metric.",
857 0 : )
858 0 : .expect("failed to define a metric")
859 0 : });
860 :
861 : /// How long did we take to start up? Broken down by labels to describe
862 : /// different phases of startup.
863 0 : pub static STARTUP_DURATION: Lazy<GaugeVec> = Lazy::new(|| {
864 0 : register_gauge_vec!(
865 0 : "pageserver_startup_duration_seconds",
866 0 : "Time taken by phases of pageserver startup, in seconds",
867 0 : &["phase"]
868 0 : )
869 0 : .expect("Failed to register pageserver_startup_duration_seconds metric")
870 0 : });
871 :
872 0 : pub static STARTUP_IS_LOADING: Lazy<UIntGauge> = Lazy::new(|| {
873 0 : register_uint_gauge!(
874 0 : "pageserver_startup_is_loading",
875 0 : "1 while in initial startup load of tenants, 0 at other times"
876 0 : )
877 0 : .expect("Failed to register pageserver_startup_is_loading")
878 0 : });
879 :
880 166 : pub(crate) static TIMELINE_EPHEMERAL_BYTES: Lazy<UIntGauge> = Lazy::new(|| {
881 166 : register_uint_gauge!(
882 166 : "pageserver_timeline_ephemeral_bytes",
883 166 : "Total number of bytes in ephemeral layers, summed for all timelines. Approximate, lazily updated."
884 166 : )
885 166 : .expect("Failed to register metric")
886 166 : });
887 :
888 : /// Metrics related to the lifecycle of a [`crate::tenant::Tenant`] object: things
889 : /// like how long it took to load.
890 : ///
891 : /// Note that these are process-global metrics, _not_ per-tenant metrics. Per-tenant
892 : /// metrics are rather expensive, and usually fine grained stuff makes more sense
893 : /// at a timeline level than tenant level.
894 : pub(crate) struct TenantMetrics {
895 : /// How long did tenants take to go from construction to active state?
896 : pub(crate) activation: Histogram,
897 : pub(crate) preload: Histogram,
898 : pub(crate) attach: Histogram,
899 :
900 : /// How many tenants are included in the initial startup of the pagesrever?
901 : pub(crate) startup_scheduled: IntCounter,
902 : pub(crate) startup_complete: IntCounter,
903 : }
904 :
905 0 : pub(crate) static TENANT: Lazy<TenantMetrics> = Lazy::new(|| {
906 0 : TenantMetrics {
907 0 : activation: register_histogram!(
908 0 : "pageserver_tenant_activation_seconds",
909 0 : "Time taken by tenants to activate, in seconds",
910 0 : CRITICAL_OP_BUCKETS.into()
911 0 : )
912 0 : .expect("Failed to register metric"),
913 0 : preload: register_histogram!(
914 0 : "pageserver_tenant_preload_seconds",
915 0 : "Time taken by tenants to load remote metadata on startup/attach, in seconds",
916 0 : CRITICAL_OP_BUCKETS.into()
917 0 : )
918 0 : .expect("Failed to register metric"),
919 0 : attach: register_histogram!(
920 0 : "pageserver_tenant_attach_seconds",
921 0 : "Time taken by tenants to intialize, after remote metadata is already loaded",
922 0 : CRITICAL_OP_BUCKETS.into()
923 0 : )
924 0 : .expect("Failed to register metric"),
925 0 : startup_scheduled: register_int_counter!(
926 0 : "pageserver_tenant_startup_scheduled",
927 0 : "Number of tenants included in pageserver startup (doesn't count tenants attached later)"
928 0 : ).expect("Failed to register metric"),
929 0 : startup_complete: register_int_counter!(
930 0 : "pageserver_tenant_startup_complete",
931 0 : "Number of tenants that have completed warm-up, or activated on-demand during initial startup: \
932 0 : should eventually reach `pageserver_tenant_startup_scheduled_total`. Does not include broken \
933 0 : tenants: such cases will lead to this metric never reaching the scheduled count."
934 0 : ).expect("Failed to register metric"),
935 0 : }
936 0 : });
937 :
938 : /// Each `Timeline`'s [`EVICTIONS_WITH_LOW_RESIDENCE_DURATION`] metric.
939 : #[derive(Debug)]
940 : pub(crate) struct EvictionsWithLowResidenceDuration {
941 : data_source: &'static str,
942 : threshold: Duration,
943 : counter: Option<IntCounter>,
944 : }
945 :
946 : pub(crate) struct EvictionsWithLowResidenceDurationBuilder {
947 : data_source: &'static str,
948 : threshold: Duration,
949 : }
950 :
951 : impl EvictionsWithLowResidenceDurationBuilder {
952 414 : pub fn new(data_source: &'static str, threshold: Duration) -> Self {
953 414 : Self {
954 414 : data_source,
955 414 : threshold,
956 414 : }
957 414 : }
958 :
959 414 : fn build(
960 414 : &self,
961 414 : tenant_id: &str,
962 414 : shard_id: &str,
963 414 : timeline_id: &str,
964 414 : ) -> EvictionsWithLowResidenceDuration {
965 414 : let counter = EVICTIONS_WITH_LOW_RESIDENCE_DURATION
966 414 : .get_metric_with_label_values(&[
967 414 : tenant_id,
968 414 : shard_id,
969 414 : timeline_id,
970 414 : self.data_source,
971 414 : &EvictionsWithLowResidenceDuration::threshold_label_value(self.threshold),
972 414 : ])
973 414 : .unwrap();
974 414 : EvictionsWithLowResidenceDuration {
975 414 : data_source: self.data_source,
976 414 : threshold: self.threshold,
977 414 : counter: Some(counter),
978 414 : }
979 414 : }
980 : }
981 :
982 : impl EvictionsWithLowResidenceDuration {
983 422 : fn threshold_label_value(threshold: Duration) -> String {
984 422 : format!("{}", threshold.as_secs())
985 422 : }
986 :
987 4 : pub fn observe(&self, observed_value: Duration) {
988 4 : if observed_value < self.threshold {
989 4 : self.counter
990 4 : .as_ref()
991 4 : .expect("nobody calls this function after `remove_from_vec`")
992 4 : .inc();
993 4 : }
994 4 : }
995 :
996 0 : pub fn change_threshold(
997 0 : &mut self,
998 0 : tenant_id: &str,
999 0 : shard_id: &str,
1000 0 : timeline_id: &str,
1001 0 : new_threshold: Duration,
1002 0 : ) {
1003 0 : if new_threshold == self.threshold {
1004 0 : return;
1005 0 : }
1006 0 : let mut with_new = EvictionsWithLowResidenceDurationBuilder::new(
1007 0 : self.data_source,
1008 0 : new_threshold,
1009 0 : )
1010 0 : .build(tenant_id, shard_id, timeline_id);
1011 0 : std::mem::swap(self, &mut with_new);
1012 0 : with_new.remove(tenant_id, shard_id, timeline_id);
1013 0 : }
1014 :
1015 : // This could be a `Drop` impl, but, we need the `tenant_id` and `timeline_id`.
1016 8 : fn remove(&mut self, tenant_id: &str, shard_id: &str, timeline_id: &str) {
1017 8 : let Some(_counter) = self.counter.take() else {
1018 0 : return;
1019 : };
1020 :
1021 8 : let threshold = Self::threshold_label_value(self.threshold);
1022 8 :
1023 8 : let removed = EVICTIONS_WITH_LOW_RESIDENCE_DURATION.remove_label_values(&[
1024 8 : tenant_id,
1025 8 : shard_id,
1026 8 : timeline_id,
1027 8 : self.data_source,
1028 8 : &threshold,
1029 8 : ]);
1030 8 :
1031 8 : match removed {
1032 0 : Err(e) => {
1033 0 : // this has been hit in staging as
1034 0 : // <https://neondatabase.sentry.io/issues/4142396994/>, but we don't know how.
1035 0 : // because we can be in the drop path already, don't risk:
1036 0 : // - "double-panic => illegal instruction" or
1037 0 : // - future "drop panick => abort"
1038 0 : //
1039 0 : // so just nag: (the error has the labels)
1040 0 : tracing::warn!("failed to remove EvictionsWithLowResidenceDuration, it was already removed? {e:#?}");
1041 : }
1042 : Ok(()) => {
1043 : // to help identify cases where we double-remove the same values, let's log all
1044 : // deletions?
1045 8 : tracing::info!("removed EvictionsWithLowResidenceDuration with {tenant_id}, {timeline_id}, {}, {threshold}", self.data_source);
1046 : }
1047 : }
1048 8 : }
1049 : }
1050 :
1051 : // Metrics collected on disk IO operations
1052 : //
1053 : // Roughly logarithmic scale.
1054 : const STORAGE_IO_TIME_BUCKETS: &[f64] = &[
1055 : 0.000030, // 30 usec
1056 : 0.001000, // 1000 usec
1057 : 0.030, // 30 ms
1058 : 1.000, // 1000 ms
1059 : 30.000, // 30000 ms
1060 : ];
1061 :
1062 : /// VirtualFile fs operation variants.
1063 : ///
1064 : /// Operations:
1065 : /// - open ([`std::fs::OpenOptions::open`])
1066 : /// - close (dropping [`crate::virtual_file::VirtualFile`])
1067 : /// - close-by-replace (close by replacement algorithm)
1068 : /// - read (`read_at`)
1069 : /// - write (`write_at`)
1070 : /// - seek (modify internal position or file length query)
1071 : /// - fsync ([`std::fs::File::sync_all`])
1072 : /// - metadata ([`std::fs::File::metadata`])
1073 : #[derive(
1074 1800 : Debug, Clone, Copy, strum_macros::EnumCount, strum_macros::EnumIter, strum_macros::FromRepr,
1075 : )]
1076 : pub(crate) enum StorageIoOperation {
1077 : Open,
1078 : OpenAfterReplace,
1079 : Close,
1080 : CloseByReplace,
1081 : Read,
1082 : Write,
1083 : Seek,
1084 : Fsync,
1085 : Metadata,
1086 : }
1087 :
1088 : impl StorageIoOperation {
1089 1800 : pub fn as_str(&self) -> &'static str {
1090 1800 : match self {
1091 200 : StorageIoOperation::Open => "open",
1092 200 : StorageIoOperation::OpenAfterReplace => "open-after-replace",
1093 200 : StorageIoOperation::Close => "close",
1094 200 : StorageIoOperation::CloseByReplace => "close-by-replace",
1095 200 : StorageIoOperation::Read => "read",
1096 200 : StorageIoOperation::Write => "write",
1097 200 : StorageIoOperation::Seek => "seek",
1098 200 : StorageIoOperation::Fsync => "fsync",
1099 200 : StorageIoOperation::Metadata => "metadata",
1100 : }
1101 1800 : }
1102 : }
1103 :
1104 : /// Tracks time taken by fs operations near VirtualFile.
1105 : #[derive(Debug)]
1106 : pub(crate) struct StorageIoTime {
1107 : metrics: [Histogram; StorageIoOperation::COUNT],
1108 : }
1109 :
1110 : impl StorageIoTime {
1111 200 : fn new() -> Self {
1112 200 : let storage_io_histogram_vec = register_histogram_vec!(
1113 200 : "pageserver_io_operations_seconds",
1114 200 : "Time spent in IO operations",
1115 200 : &["operation"],
1116 200 : STORAGE_IO_TIME_BUCKETS.into()
1117 200 : )
1118 200 : .expect("failed to define a metric");
1119 1800 : let metrics = std::array::from_fn(|i| {
1120 1800 : let op = StorageIoOperation::from_repr(i).unwrap();
1121 1800 : storage_io_histogram_vec
1122 1800 : .get_metric_with_label_values(&[op.as_str()])
1123 1800 : .unwrap()
1124 1800 : });
1125 200 : Self { metrics }
1126 200 : }
1127 :
1128 2292992 : pub(crate) fn get(&self, op: StorageIoOperation) -> &Histogram {
1129 2292992 : &self.metrics[op as usize]
1130 2292992 : }
1131 : }
1132 :
1133 : pub(crate) static STORAGE_IO_TIME_METRIC: Lazy<StorageIoTime> = Lazy::new(StorageIoTime::new);
1134 :
1135 : const STORAGE_IO_SIZE_OPERATIONS: &[&str] = &["read", "write"];
1136 :
1137 : // Needed for the https://neonprod.grafana.net/d/5uK9tHL4k/picking-tenant-for-relocation?orgId=1
1138 196 : pub(crate) static STORAGE_IO_SIZE: Lazy<IntGaugeVec> = Lazy::new(|| {
1139 196 : register_int_gauge_vec!(
1140 196 : "pageserver_io_operations_bytes_total",
1141 196 : "Total amount of bytes read/written in IO operations",
1142 196 : &["operation", "tenant_id", "shard_id", "timeline_id"]
1143 196 : )
1144 196 : .expect("failed to define a metric")
1145 196 : });
1146 :
1147 : #[cfg(not(test))]
1148 : pub(crate) mod virtual_file_descriptor_cache {
1149 : use super::*;
1150 :
1151 0 : pub(crate) static SIZE_MAX: Lazy<UIntGauge> = Lazy::new(|| {
1152 0 : register_uint_gauge!(
1153 0 : "pageserver_virtual_file_descriptor_cache_size_max",
1154 0 : "Maximum number of open file descriptors in the cache."
1155 0 : )
1156 0 : .unwrap()
1157 0 : });
1158 :
1159 : // SIZE_CURRENT: derive it like so:
1160 : // ```
1161 : // sum (pageserver_io_operations_seconds_count{operation=~"^(open|open-after-replace)$")
1162 : // -ignoring(operation)
1163 : // sum(pageserver_io_operations_seconds_count{operation=~"^(close|close-by-replace)$"}
1164 : // ```
1165 : }
1166 :
1167 : #[cfg(not(test))]
1168 : pub(crate) mod virtual_file_io_engine {
1169 : use super::*;
1170 :
1171 0 : pub(crate) static KIND: Lazy<UIntGaugeVec> = Lazy::new(|| {
1172 0 : register_uint_gauge_vec!(
1173 0 : "pageserver_virtual_file_io_engine_kind",
1174 0 : "The configured io engine for VirtualFile",
1175 0 : &["kind"],
1176 0 : )
1177 0 : .unwrap()
1178 0 : });
1179 : }
1180 :
1181 : struct GlobalAndPerTimelineHistogramTimer<'a, 'c> {
1182 : global_latency_histo: &'a Histogram,
1183 :
1184 : // Optional because not all op types are tracked per-timeline
1185 : per_timeline_latency_histo: Option<&'a Histogram>,
1186 :
1187 : ctx: &'c RequestContext,
1188 : start: std::time::Instant,
1189 : op: SmgrQueryType,
1190 : }
1191 :
1192 : impl Drop for GlobalAndPerTimelineHistogramTimer<'_, '_> {
1193 10 : fn drop(&mut self) {
1194 10 : let elapsed = self.start.elapsed();
1195 10 : let ex_throttled = self
1196 10 : .ctx
1197 10 : .micros_spent_throttled
1198 10 : .close_and_checked_sub_from(elapsed);
1199 10 : let ex_throttled = match ex_throttled {
1200 10 : Ok(res) => res,
1201 0 : Err(error) => {
1202 : use utils::rate_limit::RateLimit;
1203 : static LOGGED: Lazy<Mutex<enum_map::EnumMap<SmgrQueryType, RateLimit>>> =
1204 0 : Lazy::new(|| {
1205 0 : Mutex::new(enum_map::EnumMap::from_array(std::array::from_fn(|_| {
1206 0 : RateLimit::new(Duration::from_secs(10))
1207 0 : })))
1208 0 : });
1209 0 : let mut guard = LOGGED.lock().unwrap();
1210 0 : let rate_limit = &mut guard[self.op];
1211 0 : rate_limit.call(|| {
1212 0 : warn!(op=?self.op, error, "error deducting time spent throttled; this message is logged at a global rate limit");
1213 0 : });
1214 0 : elapsed
1215 : }
1216 : };
1217 10 : self.global_latency_histo
1218 10 : .observe(ex_throttled.as_secs_f64());
1219 10 : if let Some(per_timeline_getpage_histo) = self.per_timeline_latency_histo {
1220 2 : per_timeline_getpage_histo.observe(ex_throttled.as_secs_f64());
1221 8 : }
1222 10 : }
1223 : }
1224 :
1225 : #[derive(
1226 : Debug,
1227 : Clone,
1228 : Copy,
1229 5114 : IntoStaticStr,
1230 : strum_macros::EnumCount,
1231 24 : strum_macros::EnumIter,
1232 4240 : strum_macros::FromRepr,
1233 : enum_map::Enum,
1234 : )]
1235 : #[strum(serialize_all = "snake_case")]
1236 : pub enum SmgrQueryType {
1237 : GetRelExists,
1238 : GetRelSize,
1239 : GetPageAtLsn,
1240 : GetDbSize,
1241 : GetSlruSegment,
1242 : }
1243 :
1244 : #[derive(Debug)]
1245 : pub(crate) struct SmgrQueryTimePerTimeline {
1246 : global_started: [IntCounter; SmgrQueryType::COUNT],
1247 : global_latency: [Histogram; SmgrQueryType::COUNT],
1248 : per_timeline_getpage_started: IntCounter,
1249 : per_timeline_getpage_latency: Histogram,
1250 : }
1251 :
1252 172 : static SMGR_QUERY_STARTED_GLOBAL: Lazy<IntCounterVec> = Lazy::new(|| {
1253 172 : register_int_counter_vec!(
1254 172 : // it's a counter, but, name is prepared to extend it to a histogram of queue depth
1255 172 : "pageserver_smgr_query_started_global_count",
1256 172 : "Number of smgr queries started, aggregated by query type.",
1257 172 : &["smgr_query_type"],
1258 172 : )
1259 172 : .expect("failed to define a metric")
1260 172 : });
1261 :
1262 172 : static SMGR_QUERY_STARTED_PER_TENANT_TIMELINE: Lazy<IntCounterVec> = Lazy::new(|| {
1263 172 : register_int_counter_vec!(
1264 172 : // it's a counter, but, name is prepared to extend it to a histogram of queue depth
1265 172 : "pageserver_smgr_query_started_count",
1266 172 : "Number of smgr queries started, aggregated by query type and tenant/timeline.",
1267 172 : &["smgr_query_type", "tenant_id", "shard_id", "timeline_id"],
1268 172 : )
1269 172 : .expect("failed to define a metric")
1270 172 : });
1271 :
1272 172 : static SMGR_QUERY_TIME_PER_TENANT_TIMELINE: Lazy<HistogramVec> = Lazy::new(|| {
1273 172 : register_histogram_vec!(
1274 172 : "pageserver_smgr_query_seconds",
1275 172 : "Time spent on smgr query handling, aggegated by query type and tenant/timeline.",
1276 172 : &["smgr_query_type", "tenant_id", "shard_id", "timeline_id"],
1277 172 : CRITICAL_OP_BUCKETS.into(),
1278 172 : )
1279 172 : .expect("failed to define a metric")
1280 172 : });
1281 :
1282 172 : static SMGR_QUERY_TIME_GLOBAL_BUCKETS: Lazy<Vec<f64>> = Lazy::new(|| {
1283 172 : [
1284 172 : 1,
1285 172 : 10,
1286 172 : 20,
1287 172 : 40,
1288 172 : 60,
1289 172 : 80,
1290 172 : 100,
1291 172 : 200,
1292 172 : 300,
1293 172 : 400,
1294 172 : 500,
1295 172 : 600,
1296 172 : 700,
1297 172 : 800,
1298 172 : 900,
1299 172 : 1_000, // 1ms
1300 172 : 2_000,
1301 172 : 4_000,
1302 172 : 6_000,
1303 172 : 8_000,
1304 172 : 10_000, // 10ms
1305 172 : 20_000,
1306 172 : 40_000,
1307 172 : 60_000,
1308 172 : 80_000,
1309 172 : 100_000,
1310 172 : 200_000,
1311 172 : 400_000,
1312 172 : 600_000,
1313 172 : 800_000,
1314 172 : 1_000_000, // 1s
1315 172 : 2_000_000,
1316 172 : 4_000_000,
1317 172 : 6_000_000,
1318 172 : 8_000_000,
1319 172 : 10_000_000, // 10s
1320 172 : 20_000_000,
1321 172 : 50_000_000,
1322 172 : 100_000_000,
1323 172 : 200_000_000,
1324 172 : 1_000_000_000, // 1000s
1325 172 : ]
1326 172 : .into_iter()
1327 172 : .map(Duration::from_micros)
1328 7052 : .map(|d| d.as_secs_f64())
1329 172 : .collect()
1330 172 : });
1331 :
1332 172 : static SMGR_QUERY_TIME_GLOBAL: Lazy<HistogramVec> = Lazy::new(|| {
1333 172 : register_histogram_vec!(
1334 172 : "pageserver_smgr_query_seconds_global",
1335 172 : "Time spent on smgr query handling, aggregated by query type.",
1336 172 : &["smgr_query_type"],
1337 172 : SMGR_QUERY_TIME_GLOBAL_BUCKETS.clone(),
1338 172 : )
1339 172 : .expect("failed to define a metric")
1340 172 : });
1341 :
1342 : impl SmgrQueryTimePerTimeline {
1343 424 : pub(crate) fn new(tenant_shard_id: &TenantShardId, timeline_id: &TimelineId) -> Self {
1344 424 : let tenant_id = tenant_shard_id.tenant_id.to_string();
1345 424 : let shard_slug = format!("{}", tenant_shard_id.shard_slug());
1346 424 : let timeline_id = timeline_id.to_string();
1347 2120 : let global_started = std::array::from_fn(|i| {
1348 2120 : let op = SmgrQueryType::from_repr(i).unwrap();
1349 2120 : SMGR_QUERY_STARTED_GLOBAL
1350 2120 : .get_metric_with_label_values(&[op.into()])
1351 2120 : .unwrap()
1352 2120 : });
1353 2120 : let global_latency = std::array::from_fn(|i| {
1354 2120 : let op = SmgrQueryType::from_repr(i).unwrap();
1355 2120 : SMGR_QUERY_TIME_GLOBAL
1356 2120 : .get_metric_with_label_values(&[op.into()])
1357 2120 : .unwrap()
1358 2120 : });
1359 424 :
1360 424 : let per_timeline_getpage_started = SMGR_QUERY_STARTED_PER_TENANT_TIMELINE
1361 424 : .get_metric_with_label_values(&[
1362 424 : SmgrQueryType::GetPageAtLsn.into(),
1363 424 : &tenant_id,
1364 424 : &shard_slug,
1365 424 : &timeline_id,
1366 424 : ])
1367 424 : .unwrap();
1368 424 : let per_timeline_getpage_latency = SMGR_QUERY_TIME_PER_TENANT_TIMELINE
1369 424 : .get_metric_with_label_values(&[
1370 424 : SmgrQueryType::GetPageAtLsn.into(),
1371 424 : &tenant_id,
1372 424 : &shard_slug,
1373 424 : &timeline_id,
1374 424 : ])
1375 424 : .unwrap();
1376 424 :
1377 424 : Self {
1378 424 : global_started,
1379 424 : global_latency,
1380 424 : per_timeline_getpage_latency,
1381 424 : per_timeline_getpage_started,
1382 424 : }
1383 424 : }
1384 10 : pub(crate) fn start_timer<'c: 'a, 'a>(
1385 10 : &'a self,
1386 10 : op: SmgrQueryType,
1387 10 : ctx: &'c RequestContext,
1388 10 : ) -> Option<impl Drop + 'a> {
1389 10 : let start = Instant::now();
1390 10 :
1391 10 : self.global_started[op as usize].inc();
1392 10 :
1393 10 : // We subtract time spent throttled from the observed latency.
1394 10 : match ctx.micros_spent_throttled.open() {
1395 10 : Ok(()) => (),
1396 0 : Err(error) => {
1397 0 : use utils::rate_limit::RateLimit;
1398 0 : static LOGGED: Lazy<Mutex<enum_map::EnumMap<SmgrQueryType, RateLimit>>> =
1399 0 : Lazy::new(|| {
1400 0 : Mutex::new(enum_map::EnumMap::from_array(std::array::from_fn(|_| {
1401 0 : RateLimit::new(Duration::from_secs(10))
1402 0 : })))
1403 0 : });
1404 0 : let mut guard = LOGGED.lock().unwrap();
1405 0 : let rate_limit = &mut guard[op];
1406 0 : rate_limit.call(|| {
1407 0 : warn!(?op, error, "error opening micros_spent_throttled; this message is logged at a global rate limit");
1408 0 : });
1409 0 : }
1410 : }
1411 :
1412 10 : let per_timeline_latency_histo = if matches!(op, SmgrQueryType::GetPageAtLsn) {
1413 2 : self.per_timeline_getpage_started.inc();
1414 2 : Some(&self.per_timeline_getpage_latency)
1415 : } else {
1416 8 : None
1417 : };
1418 :
1419 10 : Some(GlobalAndPerTimelineHistogramTimer {
1420 10 : global_latency_histo: &self.global_latency[op as usize],
1421 10 : per_timeline_latency_histo,
1422 10 : ctx,
1423 10 : start,
1424 10 : op,
1425 10 : })
1426 10 : }
1427 : }
1428 :
1429 : #[cfg(test)]
1430 : mod smgr_query_time_tests {
1431 : use pageserver_api::shard::TenantShardId;
1432 : use strum::IntoEnumIterator;
1433 : use utils::id::{TenantId, TimelineId};
1434 :
1435 : use crate::{
1436 : context::{DownloadBehavior, RequestContext},
1437 : task_mgr::TaskKind,
1438 : };
1439 :
1440 : // Regression test, we used hard-coded string constants before using an enum.
1441 : #[test]
1442 2 : fn op_label_name() {
1443 : use super::SmgrQueryType::*;
1444 2 : let expect: [(super::SmgrQueryType, &'static str); 5] = [
1445 2 : (GetRelExists, "get_rel_exists"),
1446 2 : (GetRelSize, "get_rel_size"),
1447 2 : (GetPageAtLsn, "get_page_at_lsn"),
1448 2 : (GetDbSize, "get_db_size"),
1449 2 : (GetSlruSegment, "get_slru_segment"),
1450 2 : ];
1451 12 : for (op, expect) in expect {
1452 10 : let actual: &'static str = op.into();
1453 10 : assert_eq!(actual, expect);
1454 : }
1455 2 : }
1456 :
1457 : #[test]
1458 2 : fn basic() {
1459 2 : let ops: Vec<_> = super::SmgrQueryType::iter().collect();
1460 :
1461 12 : for op in &ops {
1462 10 : let tenant_id = TenantId::generate();
1463 10 : let timeline_id = TimelineId::generate();
1464 10 : let metrics = super::SmgrQueryTimePerTimeline::new(
1465 10 : &TenantShardId::unsharded(tenant_id),
1466 10 : &timeline_id,
1467 10 : );
1468 10 :
1469 20 : let get_counts = || {
1470 20 : let global: u64 = ops
1471 20 : .iter()
1472 100 : .map(|op| metrics.global_latency[*op as usize].get_sample_count())
1473 20 : .sum();
1474 20 : (
1475 20 : global,
1476 20 : metrics.per_timeline_getpage_latency.get_sample_count(),
1477 20 : )
1478 20 : };
1479 :
1480 10 : let (pre_global, pre_per_tenant_timeline) = get_counts();
1481 10 : assert_eq!(pre_per_tenant_timeline, 0);
1482 :
1483 10 : let ctx = RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Download);
1484 10 : let timer = metrics.start_timer(*op, &ctx);
1485 10 : drop(timer);
1486 10 :
1487 10 : let (post_global, post_per_tenant_timeline) = get_counts();
1488 10 : if matches!(op, super::SmgrQueryType::GetPageAtLsn) {
1489 : // getpage ops are tracked per-timeline, others aren't
1490 2 : assert_eq!(post_per_tenant_timeline, 1);
1491 : } else {
1492 8 : assert_eq!(post_per_tenant_timeline, 0);
1493 : }
1494 10 : assert!(post_global > pre_global);
1495 : }
1496 2 : }
1497 : }
1498 :
1499 : // keep in sync with control plane Go code so that we can validate
1500 : // compute's basebackup_ms metric with our perspective in the context of SLI/SLO.
1501 0 : static COMPUTE_STARTUP_BUCKETS: Lazy<[f64; 28]> = Lazy::new(|| {
1502 0 : // Go code uses milliseconds. Variable is called `computeStartupBuckets`
1503 0 : [
1504 0 : 5, 10, 20, 30, 50, 70, 100, 120, 150, 200, 250, 300, 350, 400, 450, 500, 600, 800, 1000,
1505 0 : 1500, 2000, 2500, 3000, 5000, 10000, 20000, 40000, 60000,
1506 0 : ]
1507 0 : .map(|ms| (ms as f64) / 1000.0)
1508 0 : });
1509 :
1510 : pub(crate) struct BasebackupQueryTime {
1511 : ok: Histogram,
1512 : error: Histogram,
1513 : client_error: Histogram,
1514 : }
1515 :
1516 0 : pub(crate) static BASEBACKUP_QUERY_TIME: Lazy<BasebackupQueryTime> = Lazy::new(|| {
1517 0 : let vec = register_histogram_vec!(
1518 0 : "pageserver_basebackup_query_seconds",
1519 0 : "Histogram of basebackup queries durations, by result type",
1520 0 : &["result"],
1521 0 : COMPUTE_STARTUP_BUCKETS.to_vec(),
1522 0 : )
1523 0 : .expect("failed to define a metric");
1524 0 : BasebackupQueryTime {
1525 0 : ok: vec.get_metric_with_label_values(&["ok"]).unwrap(),
1526 0 : error: vec.get_metric_with_label_values(&["error"]).unwrap(),
1527 0 : client_error: vec.get_metric_with_label_values(&["client_error"]).unwrap(),
1528 0 : }
1529 0 : });
1530 :
1531 : pub(crate) struct BasebackupQueryTimeOngoingRecording<'a, 'c> {
1532 : parent: &'a BasebackupQueryTime,
1533 : ctx: &'c RequestContext,
1534 : start: std::time::Instant,
1535 : }
1536 :
1537 : impl BasebackupQueryTime {
1538 0 : pub(crate) fn start_recording<'c: 'a, 'a>(
1539 0 : &'a self,
1540 0 : ctx: &'c RequestContext,
1541 0 : ) -> BasebackupQueryTimeOngoingRecording<'a, 'a> {
1542 0 : let start = Instant::now();
1543 0 : match ctx.micros_spent_throttled.open() {
1544 0 : Ok(()) => (),
1545 0 : Err(error) => {
1546 0 : use utils::rate_limit::RateLimit;
1547 0 : static LOGGED: Lazy<Mutex<RateLimit>> =
1548 0 : Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(10))));
1549 0 : let mut rate_limit = LOGGED.lock().unwrap();
1550 0 : rate_limit.call(|| {
1551 0 : warn!(error, "error opening micros_spent_throttled; this message is logged at a global rate limit");
1552 0 : });
1553 0 : }
1554 : }
1555 0 : BasebackupQueryTimeOngoingRecording {
1556 0 : parent: self,
1557 0 : ctx,
1558 0 : start,
1559 0 : }
1560 0 : }
1561 : }
1562 :
1563 : impl BasebackupQueryTimeOngoingRecording<'_, '_> {
1564 0 : pub(crate) fn observe<T>(self, res: &Result<T, QueryError>) {
1565 0 : let elapsed = self.start.elapsed();
1566 0 : let ex_throttled = self
1567 0 : .ctx
1568 0 : .micros_spent_throttled
1569 0 : .close_and_checked_sub_from(elapsed);
1570 0 : let ex_throttled = match ex_throttled {
1571 0 : Ok(ex_throttled) => ex_throttled,
1572 0 : Err(error) => {
1573 : use utils::rate_limit::RateLimit;
1574 : static LOGGED: Lazy<Mutex<RateLimit>> =
1575 0 : Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(10))));
1576 0 : let mut rate_limit = LOGGED.lock().unwrap();
1577 0 : rate_limit.call(|| {
1578 0 : warn!(error, "error deducting time spent throttled; this message is logged at a global rate limit");
1579 0 : });
1580 0 : elapsed
1581 : }
1582 : };
1583 : // If you want to change categorize of a specific error, also change it in `log_query_error`.
1584 0 : let metric = match res {
1585 0 : Ok(_) => &self.parent.ok,
1586 0 : Err(QueryError::Disconnected(ConnectionError::Io(io_error)))
1587 0 : if is_expected_io_error(io_error) =>
1588 0 : {
1589 0 : &self.parent.client_error
1590 : }
1591 0 : Err(_) => &self.parent.error,
1592 : };
1593 0 : metric.observe(ex_throttled.as_secs_f64());
1594 0 : }
1595 : }
1596 :
1597 0 : pub(crate) static LIVE_CONNECTIONS: Lazy<IntCounterPairVec> = Lazy::new(|| {
1598 0 : register_int_counter_pair_vec!(
1599 0 : "pageserver_live_connections_started",
1600 0 : "Number of network connections that we started handling",
1601 0 : "pageserver_live_connections_finished",
1602 0 : "Number of network connections that we finished handling",
1603 0 : &["pageserver_connection_kind"]
1604 0 : )
1605 0 : .expect("failed to define a metric")
1606 0 : });
1607 :
1608 0 : #[derive(Clone, Copy, enum_map::Enum, IntoStaticStr)]
1609 : pub(crate) enum ComputeCommandKind {
1610 : PageStreamV2,
1611 : Basebackup,
1612 : Fullbackup,
1613 : LeaseLsn,
1614 : }
1615 :
1616 : pub(crate) struct ComputeCommandCounters {
1617 : map: EnumMap<ComputeCommandKind, IntCounter>,
1618 : }
1619 :
1620 0 : pub(crate) static COMPUTE_COMMANDS_COUNTERS: Lazy<ComputeCommandCounters> = Lazy::new(|| {
1621 0 : let inner = register_int_counter_vec!(
1622 0 : "pageserver_compute_commands",
1623 0 : "Number of compute -> pageserver commands processed",
1624 0 : &["command"]
1625 0 : )
1626 0 : .expect("failed to define a metric");
1627 0 :
1628 0 : ComputeCommandCounters {
1629 0 : map: EnumMap::from_array(std::array::from_fn(|i| {
1630 0 : let command = <ComputeCommandKind as enum_map::Enum>::from_usize(i);
1631 0 : let command_str: &'static str = command.into();
1632 0 : inner.with_label_values(&[command_str])
1633 0 : })),
1634 0 : }
1635 0 : });
1636 :
1637 : impl ComputeCommandCounters {
1638 0 : pub(crate) fn for_command(&self, command: ComputeCommandKind) -> &IntCounter {
1639 0 : &self.map[command]
1640 0 : }
1641 : }
1642 :
1643 : // remote storage metrics
1644 :
1645 166 : static REMOTE_TIMELINE_CLIENT_CALLS: Lazy<IntCounterPairVec> = Lazy::new(|| {
1646 166 : register_int_counter_pair_vec!(
1647 166 : "pageserver_remote_timeline_client_calls_started",
1648 166 : "Number of started calls to remote timeline client.",
1649 166 : "pageserver_remote_timeline_client_calls_finished",
1650 166 : "Number of finshed calls to remote timeline client.",
1651 166 : &[
1652 166 : "tenant_id",
1653 166 : "shard_id",
1654 166 : "timeline_id",
1655 166 : "file_kind",
1656 166 : "op_kind"
1657 166 : ],
1658 166 : )
1659 166 : .unwrap()
1660 166 : });
1661 :
1662 : static REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER: Lazy<IntCounterVec> =
1663 164 : Lazy::new(|| {
1664 164 : register_int_counter_vec!(
1665 164 : "pageserver_remote_timeline_client_bytes_started",
1666 164 : "Incremented by the number of bytes associated with a remote timeline client operation. \
1667 164 : The increment happens when the operation is scheduled.",
1668 164 : &["tenant_id", "shard_id", "timeline_id", "file_kind", "op_kind"],
1669 164 : )
1670 164 : .expect("failed to define a metric")
1671 164 : });
1672 :
1673 164 : static REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER: Lazy<IntCounterVec> = Lazy::new(|| {
1674 164 : register_int_counter_vec!(
1675 164 : "pageserver_remote_timeline_client_bytes_finished",
1676 164 : "Incremented by the number of bytes associated with a remote timeline client operation. \
1677 164 : The increment happens when the operation finishes (regardless of success/failure/shutdown).",
1678 164 : &["tenant_id", "shard_id", "timeline_id", "file_kind", "op_kind"],
1679 164 : )
1680 164 : .expect("failed to define a metric")
1681 164 : });
1682 :
1683 : pub(crate) struct TenantManagerMetrics {
1684 : tenant_slots_attached: UIntGauge,
1685 : tenant_slots_secondary: UIntGauge,
1686 : tenant_slots_inprogress: UIntGauge,
1687 : pub(crate) tenant_slot_writes: IntCounter,
1688 : pub(crate) unexpected_errors: IntCounter,
1689 : }
1690 :
1691 : impl TenantManagerMetrics {
1692 : /// Helpers for tracking slots. Note that these do not track the lifetime of TenantSlot objects
1693 : /// exactly: they track the lifetime of the slots _in the tenant map_.
1694 2 : pub(crate) fn slot_inserted(&self, slot: &TenantSlot) {
1695 2 : match slot {
1696 0 : TenantSlot::Attached(_) => {
1697 0 : self.tenant_slots_attached.inc();
1698 0 : }
1699 0 : TenantSlot::Secondary(_) => {
1700 0 : self.tenant_slots_secondary.inc();
1701 0 : }
1702 2 : TenantSlot::InProgress(_) => {
1703 2 : self.tenant_slots_inprogress.inc();
1704 2 : }
1705 : }
1706 2 : }
1707 :
1708 2 : pub(crate) fn slot_removed(&self, slot: &TenantSlot) {
1709 2 : match slot {
1710 2 : TenantSlot::Attached(_) => {
1711 2 : self.tenant_slots_attached.dec();
1712 2 : }
1713 0 : TenantSlot::Secondary(_) => {
1714 0 : self.tenant_slots_secondary.dec();
1715 0 : }
1716 0 : TenantSlot::InProgress(_) => {
1717 0 : self.tenant_slots_inprogress.dec();
1718 0 : }
1719 : }
1720 2 : }
1721 :
1722 : #[cfg(all(debug_assertions, not(test)))]
1723 0 : pub(crate) fn slots_total(&self) -> u64 {
1724 0 : self.tenant_slots_attached.get()
1725 0 : + self.tenant_slots_secondary.get()
1726 0 : + self.tenant_slots_inprogress.get()
1727 0 : }
1728 : }
1729 :
1730 2 : pub(crate) static TENANT_MANAGER: Lazy<TenantManagerMetrics> = Lazy::new(|| {
1731 2 : let tenant_slots = register_uint_gauge_vec!(
1732 2 : "pageserver_tenant_manager_slots",
1733 2 : "How many slots currently exist, including all attached, secondary and in-progress operations",
1734 2 : &["mode"]
1735 2 : )
1736 2 : .expect("failed to define a metric");
1737 2 : TenantManagerMetrics {
1738 2 : tenant_slots_attached: tenant_slots
1739 2 : .get_metric_with_label_values(&["attached"])
1740 2 : .unwrap(),
1741 2 : tenant_slots_secondary: tenant_slots
1742 2 : .get_metric_with_label_values(&["secondary"])
1743 2 : .unwrap(),
1744 2 : tenant_slots_inprogress: tenant_slots
1745 2 : .get_metric_with_label_values(&["inprogress"])
1746 2 : .unwrap(),
1747 2 : tenant_slot_writes: register_int_counter!(
1748 2 : "pageserver_tenant_manager_slot_writes",
1749 2 : "Writes to a tenant slot, including all of create/attach/detach/delete"
1750 2 : )
1751 2 : .expect("failed to define a metric"),
1752 2 : unexpected_errors: register_int_counter!(
1753 2 : "pageserver_tenant_manager_unexpected_errors_total",
1754 2 : "Number of unexpected conditions encountered: nonzero value indicates a non-fatal bug."
1755 2 : )
1756 2 : .expect("failed to define a metric"),
1757 2 : }
1758 2 : });
1759 :
1760 : pub(crate) struct DeletionQueueMetrics {
1761 : pub(crate) keys_submitted: IntCounter,
1762 : pub(crate) keys_dropped: IntCounter,
1763 : pub(crate) keys_executed: IntCounter,
1764 : pub(crate) keys_validated: IntCounter,
1765 : pub(crate) dropped_lsn_updates: IntCounter,
1766 : pub(crate) unexpected_errors: IntCounter,
1767 : pub(crate) remote_errors: IntCounterVec,
1768 : }
1769 29 : pub(crate) static DELETION_QUEUE: Lazy<DeletionQueueMetrics> = Lazy::new(|| {
1770 29 : DeletionQueueMetrics{
1771 29 :
1772 29 : keys_submitted: register_int_counter!(
1773 29 : "pageserver_deletion_queue_submitted_total",
1774 29 : "Number of objects submitted for deletion"
1775 29 : )
1776 29 : .expect("failed to define a metric"),
1777 29 :
1778 29 : keys_dropped: register_int_counter!(
1779 29 : "pageserver_deletion_queue_dropped_total",
1780 29 : "Number of object deletions dropped due to stale generation."
1781 29 : )
1782 29 : .expect("failed to define a metric"),
1783 29 :
1784 29 : keys_executed: register_int_counter!(
1785 29 : "pageserver_deletion_queue_executed_total",
1786 29 : "Number of objects deleted. Only includes objects that we actually deleted, sum with pageserver_deletion_queue_dropped_total for the total number of keys processed to completion"
1787 29 : )
1788 29 : .expect("failed to define a metric"),
1789 29 :
1790 29 : keys_validated: register_int_counter!(
1791 29 : "pageserver_deletion_queue_validated_total",
1792 29 : "Number of keys validated for deletion. Sum with pageserver_deletion_queue_dropped_total for the total number of keys that have passed through the validation stage."
1793 29 : )
1794 29 : .expect("failed to define a metric"),
1795 29 :
1796 29 : dropped_lsn_updates: register_int_counter!(
1797 29 : "pageserver_deletion_queue_dropped_lsn_updates_total",
1798 29 : "Updates to remote_consistent_lsn dropped due to stale generation number."
1799 29 : )
1800 29 : .expect("failed to define a metric"),
1801 29 : unexpected_errors: register_int_counter!(
1802 29 : "pageserver_deletion_queue_unexpected_errors_total",
1803 29 : "Number of unexpected condiions that may stall the queue: any value above zero is unexpected."
1804 29 : )
1805 29 : .expect("failed to define a metric"),
1806 29 : remote_errors: register_int_counter_vec!(
1807 29 : "pageserver_deletion_queue_remote_errors_total",
1808 29 : "Retryable remote I/O errors while executing deletions, for example 503 responses to DeleteObjects",
1809 29 : &["op_kind"],
1810 29 : )
1811 29 : .expect("failed to define a metric")
1812 29 : }
1813 29 : });
1814 :
1815 : pub(crate) struct SecondaryModeMetrics {
1816 : pub(crate) upload_heatmap: IntCounter,
1817 : pub(crate) upload_heatmap_errors: IntCounter,
1818 : pub(crate) upload_heatmap_duration: Histogram,
1819 : pub(crate) download_heatmap: IntCounter,
1820 : pub(crate) download_layer: IntCounter,
1821 : }
1822 0 : pub(crate) static SECONDARY_MODE: Lazy<SecondaryModeMetrics> = Lazy::new(|| {
1823 0 : SecondaryModeMetrics {
1824 0 : upload_heatmap: register_int_counter!(
1825 0 : "pageserver_secondary_upload_heatmap",
1826 0 : "Number of heatmaps written to remote storage by attached tenants"
1827 0 : )
1828 0 : .expect("failed to define a metric"),
1829 0 : upload_heatmap_errors: register_int_counter!(
1830 0 : "pageserver_secondary_upload_heatmap_errors",
1831 0 : "Failures writing heatmap to remote storage"
1832 0 : )
1833 0 : .expect("failed to define a metric"),
1834 0 : upload_heatmap_duration: register_histogram!(
1835 0 : "pageserver_secondary_upload_heatmap_duration",
1836 0 : "Time to build and upload a heatmap, including any waiting inside the remote storage client"
1837 0 : )
1838 0 : .expect("failed to define a metric"),
1839 0 : download_heatmap: register_int_counter!(
1840 0 : "pageserver_secondary_download_heatmap",
1841 0 : "Number of downloads of heatmaps by secondary mode locations, including when it hasn't changed"
1842 0 : )
1843 0 : .expect("failed to define a metric"),
1844 0 : download_layer: register_int_counter!(
1845 0 : "pageserver_secondary_download_layer",
1846 0 : "Number of downloads of layers by secondary mode locations"
1847 0 : )
1848 0 : .expect("failed to define a metric"),
1849 0 : }
1850 0 : });
1851 :
1852 0 : pub(crate) static SECONDARY_RESIDENT_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
1853 0 : register_uint_gauge_vec!(
1854 0 : "pageserver_secondary_resident_physical_size",
1855 0 : "The size of the layer files present in the pageserver's filesystem, for secondary locations.",
1856 0 : &["tenant_id", "shard_id"]
1857 0 : )
1858 0 : .expect("failed to define a metric")
1859 0 : });
1860 :
1861 0 : pub(crate) static NODE_UTILIZATION_SCORE: Lazy<UIntGauge> = Lazy::new(|| {
1862 0 : register_uint_gauge!(
1863 0 : "pageserver_utilization_score",
1864 0 : "The utilization score we report to the storage controller for scheduling, where 0 is empty, 1000000 is full, and anything above is considered overloaded",
1865 0 : )
1866 0 : .expect("failed to define a metric")
1867 0 : });
1868 :
1869 0 : pub(crate) static SECONDARY_HEATMAP_TOTAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
1870 0 : register_uint_gauge_vec!(
1871 0 : "pageserver_secondary_heatmap_total_size",
1872 0 : "The total size in bytes of all layers in the most recently downloaded heatmap.",
1873 0 : &["tenant_id", "shard_id"]
1874 0 : )
1875 0 : .expect("failed to define a metric")
1876 0 : });
1877 :
1878 : #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1879 : pub enum RemoteOpKind {
1880 : Upload,
1881 : Download,
1882 : Delete,
1883 : }
1884 : impl RemoteOpKind {
1885 13761 : pub fn as_str(&self) -> &'static str {
1886 13761 : match self {
1887 12945 : Self::Upload => "upload",
1888 52 : Self::Download => "download",
1889 764 : Self::Delete => "delete",
1890 : }
1891 13761 : }
1892 : }
1893 :
1894 : #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
1895 : pub enum RemoteOpFileKind {
1896 : Layer,
1897 : Index,
1898 : }
1899 : impl RemoteOpFileKind {
1900 13761 : pub fn as_str(&self) -> &'static str {
1901 13761 : match self {
1902 9479 : Self::Layer => "layer",
1903 4282 : Self::Index => "index",
1904 : }
1905 13761 : }
1906 : }
1907 :
1908 164 : pub(crate) static REMOTE_OPERATION_TIME: Lazy<HistogramVec> = Lazy::new(|| {
1909 164 : register_histogram_vec!(
1910 164 : "pageserver_remote_operation_seconds",
1911 164 : "Time spent on remote storage operations. \
1912 164 : Grouped by tenant, timeline, operation_kind and status. \
1913 164 : Does not account for time spent waiting in remote timeline client's queues.",
1914 164 : &["file_kind", "op_kind", "status"]
1915 164 : )
1916 164 : .expect("failed to define a metric")
1917 164 : });
1918 :
1919 0 : pub(crate) static TENANT_TASK_EVENTS: Lazy<IntCounterVec> = Lazy::new(|| {
1920 0 : register_int_counter_vec!(
1921 0 : "pageserver_tenant_task_events",
1922 0 : "Number of task start/stop/fail events.",
1923 0 : &["event"],
1924 0 : )
1925 0 : .expect("Failed to register tenant_task_events metric")
1926 0 : });
1927 :
1928 : pub struct BackgroundLoopSemaphoreMetrics {
1929 : counters: EnumMap<BackgroundLoopKind, IntCounterPair>,
1930 : durations: EnumMap<BackgroundLoopKind, Counter>,
1931 : }
1932 :
1933 : pub(crate) static BACKGROUND_LOOP_SEMAPHORE: Lazy<BackgroundLoopSemaphoreMetrics> = Lazy::new(
1934 20 : || {
1935 20 : let counters = register_int_counter_pair_vec!(
1936 20 : "pageserver_background_loop_semaphore_wait_start_count",
1937 20 : "Counter for background loop concurrency-limiting semaphore acquire calls started",
1938 20 : "pageserver_background_loop_semaphore_wait_finish_count",
1939 20 : "Counter for background loop concurrency-limiting semaphore acquire calls finished",
1940 20 : &["task"],
1941 20 : )
1942 20 : .unwrap();
1943 20 :
1944 20 : let durations = register_counter_vec!(
1945 20 : "pageserver_background_loop_semaphore_wait_duration_seconds",
1946 20 : "Sum of wall clock time spent waiting on the background loop concurrency-limiting semaphore acquire calls",
1947 20 : &["task"],
1948 20 : )
1949 20 : .unwrap();
1950 20 :
1951 20 : BackgroundLoopSemaphoreMetrics {
1952 180 : counters: enum_map::EnumMap::from_array(std::array::from_fn(|i| {
1953 180 : let kind = <BackgroundLoopKind as enum_map::Enum>::from_usize(i);
1954 180 : counters.with_label_values(&[kind.into()])
1955 180 : })),
1956 180 : durations: enum_map::EnumMap::from_array(std::array::from_fn(|i| {
1957 180 : let kind = <BackgroundLoopKind as enum_map::Enum>::from_usize(i);
1958 180 : durations.with_label_values(&[kind.into()])
1959 180 : })),
1960 20 : }
1961 20 : },
1962 : );
1963 :
1964 : impl BackgroundLoopSemaphoreMetrics {
1965 364 : pub(crate) fn measure_acquisition(&self, task: BackgroundLoopKind) -> impl Drop + '_ {
1966 : struct Record<'a> {
1967 : metrics: &'a BackgroundLoopSemaphoreMetrics,
1968 : task: BackgroundLoopKind,
1969 : _counter_guard: metrics::IntCounterPairGuard,
1970 : start: Instant,
1971 : }
1972 : impl Drop for Record<'_> {
1973 364 : fn drop(&mut self) {
1974 364 : let elapsed = self.start.elapsed().as_secs_f64();
1975 364 : self.metrics.durations[self.task].inc_by(elapsed);
1976 364 : }
1977 : }
1978 364 : Record {
1979 364 : metrics: self,
1980 364 : task,
1981 364 : _counter_guard: self.counters[task].guard(),
1982 364 : start: Instant::now(),
1983 364 : }
1984 364 : }
1985 : }
1986 :
1987 0 : pub(crate) static BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT: Lazy<IntCounterVec> = Lazy::new(|| {
1988 0 : register_int_counter_vec!(
1989 0 : "pageserver_background_loop_period_overrun_count",
1990 0 : "Incremented whenever warn_when_period_overrun() logs a warning.",
1991 0 : &["task", "period"],
1992 0 : )
1993 0 : .expect("failed to define a metric")
1994 0 : });
1995 :
1996 : // walreceiver metrics
1997 :
1998 0 : pub(crate) static WALRECEIVER_STARTED_CONNECTIONS: Lazy<IntCounter> = Lazy::new(|| {
1999 0 : register_int_counter!(
2000 0 : "pageserver_walreceiver_started_connections_total",
2001 0 : "Number of started walreceiver connections"
2002 0 : )
2003 0 : .expect("failed to define a metric")
2004 0 : });
2005 :
2006 0 : pub(crate) static WALRECEIVER_ACTIVE_MANAGERS: Lazy<IntGauge> = Lazy::new(|| {
2007 0 : register_int_gauge!(
2008 0 : "pageserver_walreceiver_active_managers",
2009 0 : "Number of active walreceiver managers"
2010 0 : )
2011 0 : .expect("failed to define a metric")
2012 0 : });
2013 :
2014 0 : pub(crate) static WALRECEIVER_SWITCHES: Lazy<IntCounterVec> = Lazy::new(|| {
2015 0 : register_int_counter_vec!(
2016 0 : "pageserver_walreceiver_switches_total",
2017 0 : "Number of walreceiver manager change_connection calls",
2018 0 : &["reason"]
2019 0 : )
2020 0 : .expect("failed to define a metric")
2021 0 : });
2022 :
2023 0 : pub(crate) static WALRECEIVER_BROKER_UPDATES: Lazy<IntCounter> = Lazy::new(|| {
2024 0 : register_int_counter!(
2025 0 : "pageserver_walreceiver_broker_updates_total",
2026 0 : "Number of received broker updates in walreceiver"
2027 0 : )
2028 0 : .expect("failed to define a metric")
2029 0 : });
2030 :
2031 2 : pub(crate) static WALRECEIVER_CANDIDATES_EVENTS: Lazy<IntCounterVec> = Lazy::new(|| {
2032 2 : register_int_counter_vec!(
2033 2 : "pageserver_walreceiver_candidates_events_total",
2034 2 : "Number of walreceiver candidate events",
2035 2 : &["event"]
2036 2 : )
2037 2 : .expect("failed to define a metric")
2038 2 : });
2039 :
2040 : pub(crate) static WALRECEIVER_CANDIDATES_ADDED: Lazy<IntCounter> =
2041 0 : Lazy::new(|| WALRECEIVER_CANDIDATES_EVENTS.with_label_values(&["add"]));
2042 :
2043 : pub(crate) static WALRECEIVER_CANDIDATES_REMOVED: Lazy<IntCounter> =
2044 2 : Lazy::new(|| WALRECEIVER_CANDIDATES_EVENTS.with_label_values(&["remove"]));
2045 :
2046 : // Metrics collected on WAL redo operations
2047 : //
2048 : // We collect the time spent in actual WAL redo ('redo'), and time waiting
2049 : // for access to the postgres process ('wait') since there is only one for
2050 : // each tenant.
2051 :
2052 : /// Time buckets are small because we want to be able to measure the
2053 : /// smallest redo processing times. These buckets allow us to measure down
2054 : /// to 5us, which equates to 200'000 pages/sec, which equates to 1.6GB/sec.
2055 : /// This is much better than the previous 5ms aka 200 pages/sec aka 1.6MB/sec.
2056 : ///
2057 : /// Values up to 1s are recorded because metrics show that we have redo
2058 : /// durations and lock times larger than 0.250s.
2059 : macro_rules! redo_histogram_time_buckets {
2060 : () => {
2061 : vec![
2062 : 0.000_005, 0.000_010, 0.000_025, 0.000_050, 0.000_100, 0.000_250, 0.000_500, 0.001_000,
2063 : 0.002_500, 0.005_000, 0.010_000, 0.025_000, 0.050_000, 0.100_000, 0.250_000, 0.500_000,
2064 : 1.000_000,
2065 : ]
2066 : };
2067 : }
2068 :
2069 : /// While we're at it, also measure the amount of records replayed in each
2070 : /// operation. We have a global 'total replayed' counter, but that's not
2071 : /// as useful as 'what is the skew for how many records we replay in one
2072 : /// operation'.
2073 : macro_rules! redo_histogram_count_buckets {
2074 : () => {
2075 : vec![0.0, 1.0, 2.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0]
2076 : };
2077 : }
2078 :
2079 : macro_rules! redo_bytes_histogram_count_buckets {
2080 : () => {
2081 : // powers of (2^.5), from 2^4.5 to 2^15 (22 buckets)
2082 : // rounded up to the next multiple of 8 to capture any MAXALIGNed record of that size, too.
2083 : vec![
2084 : 24.0, 32.0, 48.0, 64.0, 96.0, 128.0, 184.0, 256.0, 368.0, 512.0, 728.0, 1024.0, 1456.0,
2085 : 2048.0, 2904.0, 4096.0, 5800.0, 8192.0, 11592.0, 16384.0, 23176.0, 32768.0,
2086 : ]
2087 : };
2088 : }
2089 :
2090 : pub(crate) struct WalIngestMetrics {
2091 : pub(crate) bytes_received: IntCounter,
2092 : pub(crate) records_received: IntCounter,
2093 : pub(crate) records_committed: IntCounter,
2094 : pub(crate) records_filtered: IntCounter,
2095 : pub(crate) gap_blocks_zeroed_on_rel_extend: IntCounter,
2096 : }
2097 :
2098 10 : pub(crate) static WAL_INGEST: Lazy<WalIngestMetrics> = Lazy::new(|| WalIngestMetrics {
2099 10 : bytes_received: register_int_counter!(
2100 10 : "pageserver_wal_ingest_bytes_received",
2101 10 : "Bytes of WAL ingested from safekeepers",
2102 10 : )
2103 10 : .unwrap(),
2104 10 : records_received: register_int_counter!(
2105 10 : "pageserver_wal_ingest_records_received",
2106 10 : "Number of WAL records received from safekeepers"
2107 10 : )
2108 10 : .expect("failed to define a metric"),
2109 10 : records_committed: register_int_counter!(
2110 10 : "pageserver_wal_ingest_records_committed",
2111 10 : "Number of WAL records which resulted in writes to pageserver storage"
2112 10 : )
2113 10 : .expect("failed to define a metric"),
2114 10 : records_filtered: register_int_counter!(
2115 10 : "pageserver_wal_ingest_records_filtered",
2116 10 : "Number of WAL records filtered out due to sharding"
2117 10 : )
2118 10 : .expect("failed to define a metric"),
2119 10 : gap_blocks_zeroed_on_rel_extend: register_int_counter!(
2120 10 : "pageserver_gap_blocks_zeroed_on_rel_extend",
2121 10 : "Total number of zero gap blocks written on relation extends"
2122 10 : )
2123 10 : .expect("failed to define a metric"),
2124 10 : });
2125 :
2126 6 : pub(crate) static WAL_REDO_TIME: Lazy<Histogram> = Lazy::new(|| {
2127 6 : register_histogram!(
2128 6 : "pageserver_wal_redo_seconds",
2129 6 : "Time spent on WAL redo",
2130 6 : redo_histogram_time_buckets!()
2131 6 : )
2132 6 : .expect("failed to define a metric")
2133 6 : });
2134 :
2135 6 : pub(crate) static WAL_REDO_RECORDS_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
2136 6 : register_histogram!(
2137 6 : "pageserver_wal_redo_records_histogram",
2138 6 : "Histogram of number of records replayed per redo in the Postgres WAL redo process",
2139 6 : redo_histogram_count_buckets!(),
2140 6 : )
2141 6 : .expect("failed to define a metric")
2142 6 : });
2143 :
2144 6 : pub(crate) static WAL_REDO_BYTES_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
2145 6 : register_histogram!(
2146 6 : "pageserver_wal_redo_bytes_histogram",
2147 6 : "Histogram of number of records replayed per redo sent to Postgres",
2148 6 : redo_bytes_histogram_count_buckets!(),
2149 6 : )
2150 6 : .expect("failed to define a metric")
2151 6 : });
2152 :
2153 : // FIXME: isn't this already included by WAL_REDO_RECORDS_HISTOGRAM which has _count?
2154 6 : pub(crate) static WAL_REDO_RECORD_COUNTER: Lazy<IntCounter> = Lazy::new(|| {
2155 6 : register_int_counter!(
2156 6 : "pageserver_replayed_wal_records_total",
2157 6 : "Number of WAL records replayed in WAL redo process"
2158 6 : )
2159 6 : .unwrap()
2160 6 : });
2161 :
2162 : #[rustfmt::skip]
2163 8 : pub(crate) static WAL_REDO_PROCESS_LAUNCH_DURATION_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
2164 8 : register_histogram!(
2165 8 : "pageserver_wal_redo_process_launch_duration",
2166 8 : "Histogram of the duration of successful WalRedoProcess::launch calls",
2167 8 : vec![
2168 8 : 0.0002, 0.0004, 0.0006, 0.0008, 0.0010,
2169 8 : 0.0020, 0.0040, 0.0060, 0.0080, 0.0100,
2170 8 : 0.0200, 0.0400, 0.0600, 0.0800, 0.1000,
2171 8 : 0.2000, 0.4000, 0.6000, 0.8000, 1.0000,
2172 8 : 1.5000, 2.0000, 2.5000, 3.0000, 4.0000, 10.0000
2173 8 : ],
2174 8 : )
2175 8 : .expect("failed to define a metric")
2176 8 : });
2177 :
2178 : pub(crate) struct WalRedoProcessCounters {
2179 : pub(crate) started: IntCounter,
2180 : pub(crate) killed_by_cause: enum_map::EnumMap<WalRedoKillCause, IntCounter>,
2181 : pub(crate) active_stderr_logger_tasks_started: IntCounter,
2182 : pub(crate) active_stderr_logger_tasks_finished: IntCounter,
2183 : }
2184 :
2185 24 : #[derive(Debug, enum_map::Enum, strum_macros::IntoStaticStr)]
2186 : pub(crate) enum WalRedoKillCause {
2187 : WalRedoProcessDrop,
2188 : NoLeakChildDrop,
2189 : Startup,
2190 : }
2191 :
2192 : impl Default for WalRedoProcessCounters {
2193 8 : fn default() -> Self {
2194 8 : let started = register_int_counter!(
2195 8 : "pageserver_wal_redo_process_started_total",
2196 8 : "Number of WAL redo processes started",
2197 8 : )
2198 8 : .unwrap();
2199 8 :
2200 8 : let killed = register_int_counter_vec!(
2201 8 : "pageserver_wal_redo_process_stopped_total",
2202 8 : "Number of WAL redo processes stopped",
2203 8 : &["cause"],
2204 8 : )
2205 8 : .unwrap();
2206 8 :
2207 8 : let active_stderr_logger_tasks_started = register_int_counter!(
2208 8 : "pageserver_walredo_stderr_logger_tasks_started_total",
2209 8 : "Number of active walredo stderr logger tasks that have started",
2210 8 : )
2211 8 : .unwrap();
2212 8 :
2213 8 : let active_stderr_logger_tasks_finished = register_int_counter!(
2214 8 : "pageserver_walredo_stderr_logger_tasks_finished_total",
2215 8 : "Number of active walredo stderr logger tasks that have finished",
2216 8 : )
2217 8 : .unwrap();
2218 8 :
2219 8 : Self {
2220 8 : started,
2221 24 : killed_by_cause: EnumMap::from_array(std::array::from_fn(|i| {
2222 24 : let cause = <WalRedoKillCause as enum_map::Enum>::from_usize(i);
2223 24 : let cause_str: &'static str = cause.into();
2224 24 : killed.with_label_values(&[cause_str])
2225 24 : })),
2226 8 : active_stderr_logger_tasks_started,
2227 8 : active_stderr_logger_tasks_finished,
2228 8 : }
2229 8 : }
2230 : }
2231 :
2232 : pub(crate) static WAL_REDO_PROCESS_COUNTERS: Lazy<WalRedoProcessCounters> =
2233 : Lazy::new(WalRedoProcessCounters::default);
2234 :
2235 : /// Similar to `prometheus::HistogramTimer` but does not record on drop.
2236 : pub(crate) struct StorageTimeMetricsTimer {
2237 : metrics: StorageTimeMetrics,
2238 : start: Instant,
2239 : }
2240 :
2241 : impl StorageTimeMetricsTimer {
2242 3363 : fn new(metrics: StorageTimeMetrics) -> Self {
2243 3363 : Self {
2244 3363 : metrics,
2245 3363 : start: Instant::now(),
2246 3363 : }
2247 3363 : }
2248 :
2249 : /// Record the time from creation to now.
2250 2232 : pub fn stop_and_record(self) {
2251 2232 : let duration = self.start.elapsed().as_secs_f64();
2252 2232 : self.metrics.timeline_sum.inc_by(duration);
2253 2232 : self.metrics.timeline_count.inc();
2254 2232 : self.metrics.global_histogram.observe(duration);
2255 2232 : }
2256 :
2257 : /// Turns this timer into a timer, which will always record -- usually this means recording
2258 : /// regardless an early `?` path was taken in a function.
2259 4 : pub(crate) fn record_on_drop(self) -> AlwaysRecordingStorageTimeMetricsTimer {
2260 4 : AlwaysRecordingStorageTimeMetricsTimer(Some(self))
2261 4 : }
2262 : }
2263 :
2264 : pub(crate) struct AlwaysRecordingStorageTimeMetricsTimer(Option<StorageTimeMetricsTimer>);
2265 :
2266 : impl Drop for AlwaysRecordingStorageTimeMetricsTimer {
2267 4 : fn drop(&mut self) {
2268 4 : if let Some(inner) = self.0.take() {
2269 4 : inner.stop_and_record();
2270 4 : }
2271 4 : }
2272 : }
2273 :
2274 : /// Timing facilities for an globally histogrammed metric, which is supported by per tenant and
2275 : /// timeline total sum and count.
2276 : #[derive(Clone, Debug)]
2277 : pub(crate) struct StorageTimeMetrics {
2278 : /// Sum of f64 seconds, per operation, tenant_id and timeline_id
2279 : timeline_sum: Counter,
2280 : /// Number of oeprations, per operation, tenant_id and timeline_id
2281 : timeline_count: IntCounter,
2282 : /// Global histogram having only the "operation" label.
2283 : global_histogram: Histogram,
2284 : }
2285 :
2286 : impl StorageTimeMetrics {
2287 3312 : pub fn new(
2288 3312 : operation: StorageTimeOperation,
2289 3312 : tenant_id: &str,
2290 3312 : shard_id: &str,
2291 3312 : timeline_id: &str,
2292 3312 : ) -> Self {
2293 3312 : let operation: &'static str = operation.into();
2294 3312 :
2295 3312 : let timeline_sum = STORAGE_TIME_SUM_PER_TIMELINE
2296 3312 : .get_metric_with_label_values(&[operation, tenant_id, shard_id, timeline_id])
2297 3312 : .unwrap();
2298 3312 : let timeline_count = STORAGE_TIME_COUNT_PER_TIMELINE
2299 3312 : .get_metric_with_label_values(&[operation, tenant_id, shard_id, timeline_id])
2300 3312 : .unwrap();
2301 3312 : let global_histogram = STORAGE_TIME_GLOBAL
2302 3312 : .get_metric_with_label_values(&[operation])
2303 3312 : .unwrap();
2304 3312 :
2305 3312 : StorageTimeMetrics {
2306 3312 : timeline_sum,
2307 3312 : timeline_count,
2308 3312 : global_histogram,
2309 3312 : }
2310 3312 : }
2311 :
2312 : /// Starts timing a new operation.
2313 : ///
2314 : /// Note: unlike `prometheus::HistogramTimer` the returned timer does not record on drop.
2315 3363 : pub fn start_timer(&self) -> StorageTimeMetricsTimer {
2316 3363 : StorageTimeMetricsTimer::new(self.clone())
2317 3363 : }
2318 : }
2319 :
2320 : #[derive(Debug)]
2321 : pub(crate) struct TimelineMetrics {
2322 : tenant_id: String,
2323 : shard_id: String,
2324 : timeline_id: String,
2325 : pub flush_time_histo: StorageTimeMetrics,
2326 : pub compact_time_histo: StorageTimeMetrics,
2327 : pub create_images_time_histo: StorageTimeMetrics,
2328 : pub logical_size_histo: StorageTimeMetrics,
2329 : pub imitate_logical_size_histo: StorageTimeMetrics,
2330 : pub load_layer_map_histo: StorageTimeMetrics,
2331 : pub garbage_collect_histo: StorageTimeMetrics,
2332 : pub find_gc_cutoffs_histo: StorageTimeMetrics,
2333 : pub last_record_gauge: IntGauge,
2334 : pub pitr_history_size: UIntGauge,
2335 : pub archival_size: UIntGauge,
2336 : pub(crate) layer_size_image: UIntGauge,
2337 : pub(crate) layer_count_image: UIntGauge,
2338 : pub(crate) layer_size_delta: UIntGauge,
2339 : pub(crate) layer_count_delta: UIntGauge,
2340 : pub standby_horizon_gauge: IntGauge,
2341 : pub resident_physical_size_gauge: UIntGauge,
2342 : pub visible_physical_size_gauge: UIntGauge,
2343 : /// copy of LayeredTimeline.current_logical_size
2344 : pub current_logical_size_gauge: UIntGauge,
2345 : pub aux_file_size_gauge: IntGauge,
2346 : pub directory_entries_count_gauge: Lazy<UIntGauge, Box<dyn Send + Fn() -> UIntGauge>>,
2347 : pub evictions: IntCounter,
2348 : pub evictions_with_low_residence_duration: std::sync::RwLock<EvictionsWithLowResidenceDuration>,
2349 : /// Number of valid LSN leases.
2350 : pub valid_lsn_lease_count_gauge: UIntGauge,
2351 : shutdown: std::sync::atomic::AtomicBool,
2352 : }
2353 :
2354 : impl TimelineMetrics {
2355 414 : pub fn new(
2356 414 : tenant_shard_id: &TenantShardId,
2357 414 : timeline_id_raw: &TimelineId,
2358 414 : evictions_with_low_residence_duration_builder: EvictionsWithLowResidenceDurationBuilder,
2359 414 : ) -> Self {
2360 414 : let tenant_id = tenant_shard_id.tenant_id.to_string();
2361 414 : let shard_id = format!("{}", tenant_shard_id.shard_slug());
2362 414 : let timeline_id = timeline_id_raw.to_string();
2363 414 : let flush_time_histo = StorageTimeMetrics::new(
2364 414 : StorageTimeOperation::LayerFlush,
2365 414 : &tenant_id,
2366 414 : &shard_id,
2367 414 : &timeline_id,
2368 414 : );
2369 414 : let compact_time_histo = StorageTimeMetrics::new(
2370 414 : StorageTimeOperation::Compact,
2371 414 : &tenant_id,
2372 414 : &shard_id,
2373 414 : &timeline_id,
2374 414 : );
2375 414 : let create_images_time_histo = StorageTimeMetrics::new(
2376 414 : StorageTimeOperation::CreateImages,
2377 414 : &tenant_id,
2378 414 : &shard_id,
2379 414 : &timeline_id,
2380 414 : );
2381 414 : let logical_size_histo = StorageTimeMetrics::new(
2382 414 : StorageTimeOperation::LogicalSize,
2383 414 : &tenant_id,
2384 414 : &shard_id,
2385 414 : &timeline_id,
2386 414 : );
2387 414 : let imitate_logical_size_histo = StorageTimeMetrics::new(
2388 414 : StorageTimeOperation::ImitateLogicalSize,
2389 414 : &tenant_id,
2390 414 : &shard_id,
2391 414 : &timeline_id,
2392 414 : );
2393 414 : let load_layer_map_histo = StorageTimeMetrics::new(
2394 414 : StorageTimeOperation::LoadLayerMap,
2395 414 : &tenant_id,
2396 414 : &shard_id,
2397 414 : &timeline_id,
2398 414 : );
2399 414 : let garbage_collect_histo = StorageTimeMetrics::new(
2400 414 : StorageTimeOperation::Gc,
2401 414 : &tenant_id,
2402 414 : &shard_id,
2403 414 : &timeline_id,
2404 414 : );
2405 414 : let find_gc_cutoffs_histo = StorageTimeMetrics::new(
2406 414 : StorageTimeOperation::FindGcCutoffs,
2407 414 : &tenant_id,
2408 414 : &shard_id,
2409 414 : &timeline_id,
2410 414 : );
2411 414 : let last_record_gauge = LAST_RECORD_LSN
2412 414 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2413 414 : .unwrap();
2414 414 :
2415 414 : let pitr_history_size = PITR_HISTORY_SIZE
2416 414 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2417 414 : .unwrap();
2418 414 :
2419 414 : let archival_size = TIMELINE_ARCHIVE_SIZE
2420 414 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2421 414 : .unwrap();
2422 414 :
2423 414 : let layer_size_image = TIMELINE_LAYER_SIZE
2424 414 : .get_metric_with_label_values(&[
2425 414 : &tenant_id,
2426 414 : &shard_id,
2427 414 : &timeline_id,
2428 414 : MetricLayerKind::Image.into(),
2429 414 : ])
2430 414 : .unwrap();
2431 414 :
2432 414 : let layer_count_image = TIMELINE_LAYER_COUNT
2433 414 : .get_metric_with_label_values(&[
2434 414 : &tenant_id,
2435 414 : &shard_id,
2436 414 : &timeline_id,
2437 414 : MetricLayerKind::Image.into(),
2438 414 : ])
2439 414 : .unwrap();
2440 414 :
2441 414 : let layer_size_delta = TIMELINE_LAYER_SIZE
2442 414 : .get_metric_with_label_values(&[
2443 414 : &tenant_id,
2444 414 : &shard_id,
2445 414 : &timeline_id,
2446 414 : MetricLayerKind::Delta.into(),
2447 414 : ])
2448 414 : .unwrap();
2449 414 :
2450 414 : let layer_count_delta = TIMELINE_LAYER_COUNT
2451 414 : .get_metric_with_label_values(&[
2452 414 : &tenant_id,
2453 414 : &shard_id,
2454 414 : &timeline_id,
2455 414 : MetricLayerKind::Delta.into(),
2456 414 : ])
2457 414 : .unwrap();
2458 414 :
2459 414 : let standby_horizon_gauge = STANDBY_HORIZON
2460 414 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2461 414 : .unwrap();
2462 414 : let resident_physical_size_gauge = RESIDENT_PHYSICAL_SIZE
2463 414 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2464 414 : .unwrap();
2465 414 : let visible_physical_size_gauge = VISIBLE_PHYSICAL_SIZE
2466 414 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2467 414 : .unwrap();
2468 414 : // TODO: we shouldn't expose this metric
2469 414 : let current_logical_size_gauge = CURRENT_LOGICAL_SIZE
2470 414 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2471 414 : .unwrap();
2472 414 : let aux_file_size_gauge = AUX_FILE_SIZE
2473 414 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2474 414 : .unwrap();
2475 414 : // TODO use impl Trait syntax here once we have ability to use it: https://github.com/rust-lang/rust/issues/63065
2476 414 : let directory_entries_count_gauge_closure = {
2477 414 : let tenant_shard_id = *tenant_shard_id;
2478 414 : let timeline_id_raw = *timeline_id_raw;
2479 0 : move || {
2480 0 : let tenant_id = tenant_shard_id.tenant_id.to_string();
2481 0 : let shard_id = format!("{}", tenant_shard_id.shard_slug());
2482 0 : let timeline_id = timeline_id_raw.to_string();
2483 0 : let gauge: UIntGauge = DIRECTORY_ENTRIES_COUNT
2484 0 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2485 0 : .unwrap();
2486 0 : gauge
2487 0 : }
2488 : };
2489 414 : let directory_entries_count_gauge: Lazy<UIntGauge, Box<dyn Send + Fn() -> UIntGauge>> =
2490 414 : Lazy::new(Box::new(directory_entries_count_gauge_closure));
2491 414 : let evictions = EVICTIONS
2492 414 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2493 414 : .unwrap();
2494 414 : let evictions_with_low_residence_duration = evictions_with_low_residence_duration_builder
2495 414 : .build(&tenant_id, &shard_id, &timeline_id);
2496 414 :
2497 414 : let valid_lsn_lease_count_gauge = VALID_LSN_LEASE_COUNT
2498 414 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2499 414 : .unwrap();
2500 414 :
2501 414 : TimelineMetrics {
2502 414 : tenant_id,
2503 414 : shard_id,
2504 414 : timeline_id,
2505 414 : flush_time_histo,
2506 414 : compact_time_histo,
2507 414 : create_images_time_histo,
2508 414 : logical_size_histo,
2509 414 : imitate_logical_size_histo,
2510 414 : garbage_collect_histo,
2511 414 : find_gc_cutoffs_histo,
2512 414 : load_layer_map_histo,
2513 414 : last_record_gauge,
2514 414 : pitr_history_size,
2515 414 : archival_size,
2516 414 : layer_size_image,
2517 414 : layer_count_image,
2518 414 : layer_size_delta,
2519 414 : layer_count_delta,
2520 414 : standby_horizon_gauge,
2521 414 : resident_physical_size_gauge,
2522 414 : visible_physical_size_gauge,
2523 414 : current_logical_size_gauge,
2524 414 : aux_file_size_gauge,
2525 414 : directory_entries_count_gauge,
2526 414 : evictions,
2527 414 : evictions_with_low_residence_duration: std::sync::RwLock::new(
2528 414 : evictions_with_low_residence_duration,
2529 414 : ),
2530 414 : valid_lsn_lease_count_gauge,
2531 414 : shutdown: std::sync::atomic::AtomicBool::default(),
2532 414 : }
2533 414 : }
2534 :
2535 1528 : pub(crate) fn record_new_file_metrics(&self, sz: u64) {
2536 1528 : self.resident_physical_size_add(sz);
2537 1528 : }
2538 :
2539 491 : pub(crate) fn resident_physical_size_sub(&self, sz: u64) {
2540 491 : self.resident_physical_size_gauge.sub(sz);
2541 491 : crate::metrics::RESIDENT_PHYSICAL_SIZE_GLOBAL.sub(sz);
2542 491 : }
2543 :
2544 1558 : pub(crate) fn resident_physical_size_add(&self, sz: u64) {
2545 1558 : self.resident_physical_size_gauge.add(sz);
2546 1558 : crate::metrics::RESIDENT_PHYSICAL_SIZE_GLOBAL.add(sz);
2547 1558 : }
2548 :
2549 8 : pub(crate) fn resident_physical_size_get(&self) -> u64 {
2550 8 : self.resident_physical_size_gauge.get()
2551 8 : }
2552 :
2553 8 : pub(crate) fn shutdown(&self) {
2554 8 : let was_shutdown = self
2555 8 : .shutdown
2556 8 : .swap(true, std::sync::atomic::Ordering::Relaxed);
2557 8 :
2558 8 : if was_shutdown {
2559 : // this happens on tenant deletion because tenant first shuts down timelines, then
2560 : // invokes timeline deletion which first shuts down the timeline again.
2561 : // TODO: this can be removed once https://github.com/neondatabase/neon/issues/5080
2562 0 : return;
2563 8 : }
2564 8 :
2565 8 : let tenant_id = &self.tenant_id;
2566 8 : let timeline_id = &self.timeline_id;
2567 8 : let shard_id = &self.shard_id;
2568 8 : let _ = LAST_RECORD_LSN.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2569 8 : let _ = STANDBY_HORIZON.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2570 8 : {
2571 8 : RESIDENT_PHYSICAL_SIZE_GLOBAL.sub(self.resident_physical_size_get());
2572 8 : let _ = RESIDENT_PHYSICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2573 8 : }
2574 8 : let _ = VISIBLE_PHYSICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2575 8 : let _ = CURRENT_LOGICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2576 8 : if let Some(metric) = Lazy::get(&DIRECTORY_ENTRIES_COUNT) {
2577 0 : let _ = metric.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2578 8 : }
2579 :
2580 8 : let _ = TIMELINE_ARCHIVE_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2581 8 : let _ = PITR_HISTORY_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2582 8 :
2583 8 : let _ = TIMELINE_LAYER_SIZE.remove_label_values(&[
2584 8 : tenant_id,
2585 8 : shard_id,
2586 8 : timeline_id,
2587 8 : MetricLayerKind::Image.into(),
2588 8 : ]);
2589 8 : let _ = TIMELINE_LAYER_COUNT.remove_label_values(&[
2590 8 : tenant_id,
2591 8 : shard_id,
2592 8 : timeline_id,
2593 8 : MetricLayerKind::Image.into(),
2594 8 : ]);
2595 8 : let _ = TIMELINE_LAYER_SIZE.remove_label_values(&[
2596 8 : tenant_id,
2597 8 : shard_id,
2598 8 : timeline_id,
2599 8 : MetricLayerKind::Delta.into(),
2600 8 : ]);
2601 8 : let _ = TIMELINE_LAYER_COUNT.remove_label_values(&[
2602 8 : tenant_id,
2603 8 : shard_id,
2604 8 : timeline_id,
2605 8 : MetricLayerKind::Delta.into(),
2606 8 : ]);
2607 8 :
2608 8 : let _ = EVICTIONS.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2609 8 : let _ = AUX_FILE_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2610 8 : let _ = VALID_LSN_LEASE_COUNT.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2611 8 :
2612 8 : self.evictions_with_low_residence_duration
2613 8 : .write()
2614 8 : .unwrap()
2615 8 : .remove(tenant_id, shard_id, timeline_id);
2616 :
2617 : // The following metrics are born outside of the TimelineMetrics lifecycle but still
2618 : // removed at the end of it. The idea is to have the metrics outlive the
2619 : // entity during which they're observed, e.g., the smgr metrics shall
2620 : // outlive an individual smgr connection, but not the timeline.
2621 :
2622 72 : for op in StorageTimeOperation::VARIANTS {
2623 64 : let _ = STORAGE_TIME_SUM_PER_TIMELINE.remove_label_values(&[
2624 64 : op,
2625 64 : tenant_id,
2626 64 : shard_id,
2627 64 : timeline_id,
2628 64 : ]);
2629 64 : let _ = STORAGE_TIME_COUNT_PER_TIMELINE.remove_label_values(&[
2630 64 : op,
2631 64 : tenant_id,
2632 64 : shard_id,
2633 64 : timeline_id,
2634 64 : ]);
2635 64 : }
2636 :
2637 24 : for op in STORAGE_IO_SIZE_OPERATIONS {
2638 16 : let _ = STORAGE_IO_SIZE.remove_label_values(&[op, tenant_id, shard_id, timeline_id]);
2639 16 : }
2640 :
2641 8 : let _ = SMGR_QUERY_STARTED_PER_TENANT_TIMELINE.remove_label_values(&[
2642 8 : SmgrQueryType::GetPageAtLsn.into(),
2643 8 : tenant_id,
2644 8 : shard_id,
2645 8 : timeline_id,
2646 8 : ]);
2647 8 : let _ = SMGR_QUERY_TIME_PER_TENANT_TIMELINE.remove_label_values(&[
2648 8 : SmgrQueryType::GetPageAtLsn.into(),
2649 8 : tenant_id,
2650 8 : shard_id,
2651 8 : timeline_id,
2652 8 : ]);
2653 8 : }
2654 : }
2655 :
2656 6 : pub(crate) fn remove_tenant_metrics(tenant_shard_id: &TenantShardId) {
2657 6 : // Only shard zero deals in synthetic sizes
2658 6 : if tenant_shard_id.is_shard_zero() {
2659 6 : let tid = tenant_shard_id.tenant_id.to_string();
2660 6 : let _ = TENANT_SYNTHETIC_SIZE_METRIC.remove_label_values(&[&tid]);
2661 6 : }
2662 :
2663 6 : tenant_throttling::remove_tenant_metrics(tenant_shard_id);
2664 6 :
2665 6 : // we leave the BROKEN_TENANTS_SET entry if any
2666 6 : }
2667 :
2668 : use futures::Future;
2669 : use pin_project_lite::pin_project;
2670 : use std::collections::HashMap;
2671 : use std::num::NonZeroUsize;
2672 : use std::pin::Pin;
2673 : use std::sync::atomic::AtomicU64;
2674 : use std::sync::{Arc, Mutex};
2675 : use std::task::{Context, Poll};
2676 : use std::time::{Duration, Instant};
2677 :
2678 : use crate::context::{PageContentKind, RequestContext};
2679 : use crate::task_mgr::TaskKind;
2680 : use crate::tenant::mgr::TenantSlot;
2681 : use crate::tenant::tasks::BackgroundLoopKind;
2682 :
2683 : /// Maintain a per timeline gauge in addition to the global gauge.
2684 : pub(crate) struct PerTimelineRemotePhysicalSizeGauge {
2685 : last_set: AtomicU64,
2686 : gauge: UIntGauge,
2687 : }
2688 :
2689 : impl PerTimelineRemotePhysicalSizeGauge {
2690 424 : fn new(per_timeline_gauge: UIntGauge) -> Self {
2691 424 : Self {
2692 424 : last_set: AtomicU64::new(0),
2693 424 : gauge: per_timeline_gauge,
2694 424 : }
2695 424 : }
2696 1812 : pub(crate) fn set(&self, sz: u64) {
2697 1812 : self.gauge.set(sz);
2698 1812 : let prev = self.last_set.swap(sz, std::sync::atomic::Ordering::Relaxed);
2699 1812 : if sz < prev {
2700 21 : REMOTE_PHYSICAL_SIZE_GLOBAL.sub(prev - sz);
2701 1791 : } else {
2702 1791 : REMOTE_PHYSICAL_SIZE_GLOBAL.add(sz - prev);
2703 1791 : };
2704 1812 : }
2705 2 : pub(crate) fn get(&self) -> u64 {
2706 2 : self.gauge.get()
2707 2 : }
2708 : }
2709 :
2710 : impl Drop for PerTimelineRemotePhysicalSizeGauge {
2711 18 : fn drop(&mut self) {
2712 18 : REMOTE_PHYSICAL_SIZE_GLOBAL.sub(self.last_set.load(std::sync::atomic::Ordering::Relaxed));
2713 18 : }
2714 : }
2715 :
2716 : pub(crate) struct RemoteTimelineClientMetrics {
2717 : tenant_id: String,
2718 : shard_id: String,
2719 : timeline_id: String,
2720 : pub(crate) remote_physical_size_gauge: PerTimelineRemotePhysicalSizeGauge,
2721 : calls: Mutex<HashMap<(&'static str, &'static str), IntCounterPair>>,
2722 : bytes_started_counter: Mutex<HashMap<(&'static str, &'static str), IntCounter>>,
2723 : bytes_finished_counter: Mutex<HashMap<(&'static str, &'static str), IntCounter>>,
2724 : }
2725 :
2726 : impl RemoteTimelineClientMetrics {
2727 424 : pub fn new(tenant_shard_id: &TenantShardId, timeline_id: &TimelineId) -> Self {
2728 424 : let tenant_id_str = tenant_shard_id.tenant_id.to_string();
2729 424 : let shard_id_str = format!("{}", tenant_shard_id.shard_slug());
2730 424 : let timeline_id_str = timeline_id.to_string();
2731 424 :
2732 424 : let remote_physical_size_gauge = PerTimelineRemotePhysicalSizeGauge::new(
2733 424 : REMOTE_PHYSICAL_SIZE
2734 424 : .get_metric_with_label_values(&[&tenant_id_str, &shard_id_str, &timeline_id_str])
2735 424 : .unwrap(),
2736 424 : );
2737 424 :
2738 424 : RemoteTimelineClientMetrics {
2739 424 : tenant_id: tenant_id_str,
2740 424 : shard_id: shard_id_str,
2741 424 : timeline_id: timeline_id_str,
2742 424 : calls: Mutex::new(HashMap::default()),
2743 424 : bytes_started_counter: Mutex::new(HashMap::default()),
2744 424 : bytes_finished_counter: Mutex::new(HashMap::default()),
2745 424 : remote_physical_size_gauge,
2746 424 : }
2747 424 : }
2748 :
2749 2785 : pub fn remote_operation_time(
2750 2785 : &self,
2751 2785 : file_kind: &RemoteOpFileKind,
2752 2785 : op_kind: &RemoteOpKind,
2753 2785 : status: &'static str,
2754 2785 : ) -> Histogram {
2755 2785 : let key = (file_kind.as_str(), op_kind.as_str(), status);
2756 2785 : REMOTE_OPERATION_TIME
2757 2785 : .get_metric_with_label_values(&[key.0, key.1, key.2])
2758 2785 : .unwrap()
2759 2785 : }
2760 :
2761 6531 : fn calls_counter_pair(
2762 6531 : &self,
2763 6531 : file_kind: &RemoteOpFileKind,
2764 6531 : op_kind: &RemoteOpKind,
2765 6531 : ) -> IntCounterPair {
2766 6531 : let mut guard = self.calls.lock().unwrap();
2767 6531 : let key = (file_kind.as_str(), op_kind.as_str());
2768 6531 : let metric = guard.entry(key).or_insert_with(move || {
2769 748 : REMOTE_TIMELINE_CLIENT_CALLS
2770 748 : .get_metric_with_label_values(&[
2771 748 : &self.tenant_id,
2772 748 : &self.shard_id,
2773 748 : &self.timeline_id,
2774 748 : key.0,
2775 748 : key.1,
2776 748 : ])
2777 748 : .unwrap()
2778 6531 : });
2779 6531 : metric.clone()
2780 6531 : }
2781 :
2782 1536 : fn bytes_started_counter(
2783 1536 : &self,
2784 1536 : file_kind: &RemoteOpFileKind,
2785 1536 : op_kind: &RemoteOpKind,
2786 1536 : ) -> IntCounter {
2787 1536 : let mut guard = self.bytes_started_counter.lock().unwrap();
2788 1536 : let key = (file_kind.as_str(), op_kind.as_str());
2789 1536 : let metric = guard.entry(key).or_insert_with(move || {
2790 288 : REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER
2791 288 : .get_metric_with_label_values(&[
2792 288 : &self.tenant_id,
2793 288 : &self.shard_id,
2794 288 : &self.timeline_id,
2795 288 : key.0,
2796 288 : key.1,
2797 288 : ])
2798 288 : .unwrap()
2799 1536 : });
2800 1536 : metric.clone()
2801 1536 : }
2802 :
2803 2897 : fn bytes_finished_counter(
2804 2897 : &self,
2805 2897 : file_kind: &RemoteOpFileKind,
2806 2897 : op_kind: &RemoteOpKind,
2807 2897 : ) -> IntCounter {
2808 2897 : let mut guard = self.bytes_finished_counter.lock().unwrap();
2809 2897 : let key = (file_kind.as_str(), op_kind.as_str());
2810 2897 : let metric = guard.entry(key).or_insert_with(move || {
2811 288 : REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER
2812 288 : .get_metric_with_label_values(&[
2813 288 : &self.tenant_id,
2814 288 : &self.shard_id,
2815 288 : &self.timeline_id,
2816 288 : key.0,
2817 288 : key.1,
2818 288 : ])
2819 288 : .unwrap()
2820 2897 : });
2821 2897 : metric.clone()
2822 2897 : }
2823 : }
2824 :
2825 : #[cfg(test)]
2826 : impl RemoteTimelineClientMetrics {
2827 6 : pub fn get_bytes_started_counter_value(
2828 6 : &self,
2829 6 : file_kind: &RemoteOpFileKind,
2830 6 : op_kind: &RemoteOpKind,
2831 6 : ) -> Option<u64> {
2832 6 : let guard = self.bytes_started_counter.lock().unwrap();
2833 6 : let key = (file_kind.as_str(), op_kind.as_str());
2834 6 : guard.get(&key).map(|counter| counter.get())
2835 6 : }
2836 :
2837 6 : pub fn get_bytes_finished_counter_value(
2838 6 : &self,
2839 6 : file_kind: &RemoteOpFileKind,
2840 6 : op_kind: &RemoteOpKind,
2841 6 : ) -> Option<u64> {
2842 6 : let guard = self.bytes_finished_counter.lock().unwrap();
2843 6 : let key = (file_kind.as_str(), op_kind.as_str());
2844 6 : guard.get(&key).map(|counter| counter.get())
2845 6 : }
2846 : }
2847 :
2848 : /// See [`RemoteTimelineClientMetrics::call_begin`].
2849 : #[must_use]
2850 : pub(crate) struct RemoteTimelineClientCallMetricGuard {
2851 : /// Decremented on drop.
2852 : calls_counter_pair: Option<IntCounterPair>,
2853 : /// If Some(), this references the bytes_finished metric, and we increment it by the given `u64` on drop.
2854 : bytes_finished: Option<(IntCounter, u64)>,
2855 : }
2856 :
2857 : impl RemoteTimelineClientCallMetricGuard {
2858 : /// Consume this guard object without performing the metric updates it would do on `drop()`.
2859 : /// The caller vouches to do the metric updates manually.
2860 3459 : pub fn will_decrement_manually(mut self) {
2861 3459 : let RemoteTimelineClientCallMetricGuard {
2862 3459 : calls_counter_pair,
2863 3459 : bytes_finished,
2864 3459 : } = &mut self;
2865 3459 : calls_counter_pair.take();
2866 3459 : bytes_finished.take();
2867 3459 : }
2868 : }
2869 :
2870 : impl Drop for RemoteTimelineClientCallMetricGuard {
2871 3485 : fn drop(&mut self) {
2872 3485 : let RemoteTimelineClientCallMetricGuard {
2873 3485 : calls_counter_pair,
2874 3485 : bytes_finished,
2875 3485 : } = self;
2876 3485 : if let Some(guard) = calls_counter_pair.take() {
2877 26 : guard.dec();
2878 3459 : }
2879 3485 : if let Some((bytes_finished_metric, value)) = bytes_finished {
2880 0 : bytes_finished_metric.inc_by(*value);
2881 3485 : }
2882 3485 : }
2883 : }
2884 :
2885 : /// The enum variants communicate to the [`RemoteTimelineClientMetrics`] whether to
2886 : /// track the byte size of this call in applicable metric(s).
2887 : pub(crate) enum RemoteTimelineClientMetricsCallTrackSize {
2888 : /// Do not account for this call's byte size in any metrics.
2889 : /// The `reason` field is there to make the call sites self-documenting
2890 : /// about why they don't need the metric.
2891 : DontTrackSize { reason: &'static str },
2892 : /// Track the byte size of the call in applicable metric(s).
2893 : Bytes(u64),
2894 : }
2895 :
2896 : impl RemoteTimelineClientMetrics {
2897 : /// Update the metrics that change when a call to the remote timeline client instance starts.
2898 : ///
2899 : /// Drop the returned guard object once the operation is finished to updates corresponding metrics that track completions.
2900 : /// Or, use [`RemoteTimelineClientCallMetricGuard::will_decrement_manually`] and [`call_end`](Self::call_end) if that
2901 : /// is more suitable.
2902 : /// Never do both.
2903 3485 : pub(crate) fn call_begin(
2904 3485 : &self,
2905 3485 : file_kind: &RemoteOpFileKind,
2906 3485 : op_kind: &RemoteOpKind,
2907 3485 : size: RemoteTimelineClientMetricsCallTrackSize,
2908 3485 : ) -> RemoteTimelineClientCallMetricGuard {
2909 3485 : let calls_counter_pair = self.calls_counter_pair(file_kind, op_kind);
2910 3485 : calls_counter_pair.inc();
2911 :
2912 3485 : let bytes_finished = match size {
2913 1949 : RemoteTimelineClientMetricsCallTrackSize::DontTrackSize { reason: _reason } => {
2914 1949 : // nothing to do
2915 1949 : None
2916 : }
2917 1536 : RemoteTimelineClientMetricsCallTrackSize::Bytes(size) => {
2918 1536 : self.bytes_started_counter(file_kind, op_kind).inc_by(size);
2919 1536 : let finished_counter = self.bytes_finished_counter(file_kind, op_kind);
2920 1536 : Some((finished_counter, size))
2921 : }
2922 : };
2923 3485 : RemoteTimelineClientCallMetricGuard {
2924 3485 : calls_counter_pair: Some(calls_counter_pair),
2925 3485 : bytes_finished,
2926 3485 : }
2927 3485 : }
2928 :
2929 : /// Manually udpate the metrics that track completions, instead of using the guard object.
2930 : /// Using the guard object is generally preferable.
2931 : /// See [`call_begin`](Self::call_begin) for more context.
2932 3046 : pub(crate) fn call_end(
2933 3046 : &self,
2934 3046 : file_kind: &RemoteOpFileKind,
2935 3046 : op_kind: &RemoteOpKind,
2936 3046 : size: RemoteTimelineClientMetricsCallTrackSize,
2937 3046 : ) {
2938 3046 : let calls_counter_pair = self.calls_counter_pair(file_kind, op_kind);
2939 3046 : calls_counter_pair.dec();
2940 3046 : match size {
2941 1685 : RemoteTimelineClientMetricsCallTrackSize::DontTrackSize { reason: _reason } => {}
2942 1361 : RemoteTimelineClientMetricsCallTrackSize::Bytes(size) => {
2943 1361 : self.bytes_finished_counter(file_kind, op_kind).inc_by(size);
2944 1361 : }
2945 : }
2946 3046 : }
2947 : }
2948 :
2949 : impl Drop for RemoteTimelineClientMetrics {
2950 18 : fn drop(&mut self) {
2951 18 : let RemoteTimelineClientMetrics {
2952 18 : tenant_id,
2953 18 : shard_id,
2954 18 : timeline_id,
2955 18 : remote_physical_size_gauge,
2956 18 : calls,
2957 18 : bytes_started_counter,
2958 18 : bytes_finished_counter,
2959 18 : } = self;
2960 22 : for ((a, b), _) in calls.get_mut().unwrap().drain() {
2961 22 : let mut res = [Ok(()), Ok(())];
2962 22 : REMOTE_TIMELINE_CLIENT_CALLS
2963 22 : .remove_label_values(&mut res, &[tenant_id, shard_id, timeline_id, a, b]);
2964 22 : // don't care about results
2965 22 : }
2966 18 : for ((a, b), _) in bytes_started_counter.get_mut().unwrap().drain() {
2967 6 : let _ = REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER.remove_label_values(&[
2968 6 : tenant_id,
2969 6 : shard_id,
2970 6 : timeline_id,
2971 6 : a,
2972 6 : b,
2973 6 : ]);
2974 6 : }
2975 18 : for ((a, b), _) in bytes_finished_counter.get_mut().unwrap().drain() {
2976 6 : let _ = REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER.remove_label_values(&[
2977 6 : tenant_id,
2978 6 : shard_id,
2979 6 : timeline_id,
2980 6 : a,
2981 6 : b,
2982 6 : ]);
2983 6 : }
2984 18 : {
2985 18 : let _ = remote_physical_size_gauge; // use to avoid 'unused' warning in desctructuring above
2986 18 : let _ = REMOTE_PHYSICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2987 18 : }
2988 18 : }
2989 : }
2990 :
2991 : /// Wrapper future that measures the time spent by a remote storage operation,
2992 : /// and records the time and success/failure as a prometheus metric.
2993 : pub(crate) trait MeasureRemoteOp: Sized {
2994 2868 : fn measure_remote_op(
2995 2868 : self,
2996 2868 : file_kind: RemoteOpFileKind,
2997 2868 : op: RemoteOpKind,
2998 2868 : metrics: Arc<RemoteTimelineClientMetrics>,
2999 2868 : ) -> MeasuredRemoteOp<Self> {
3000 2868 : let start = Instant::now();
3001 2868 : MeasuredRemoteOp {
3002 2868 : inner: self,
3003 2868 : file_kind,
3004 2868 : op,
3005 2868 : start,
3006 2868 : metrics,
3007 2868 : }
3008 2868 : }
3009 : }
3010 :
3011 : impl<T: Sized> MeasureRemoteOp for T {}
3012 :
3013 : pin_project! {
3014 : pub(crate) struct MeasuredRemoteOp<F>
3015 : {
3016 : #[pin]
3017 : inner: F,
3018 : file_kind: RemoteOpFileKind,
3019 : op: RemoteOpKind,
3020 : start: Instant,
3021 : metrics: Arc<RemoteTimelineClientMetrics>,
3022 : }
3023 : }
3024 :
3025 : impl<F: Future<Output = Result<O, E>>, O, E> Future for MeasuredRemoteOp<F> {
3026 : type Output = Result<O, E>;
3027 :
3028 46099 : fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
3029 46099 : let this = self.project();
3030 46099 : let poll_result = this.inner.poll(cx);
3031 46099 : if let Poll::Ready(ref res) = poll_result {
3032 2785 : let duration = this.start.elapsed();
3033 2785 : let status = if res.is_ok() { &"success" } else { &"failure" };
3034 2785 : this.metrics
3035 2785 : .remote_operation_time(this.file_kind, this.op, status)
3036 2785 : .observe(duration.as_secs_f64());
3037 43314 : }
3038 46099 : poll_result
3039 46099 : }
3040 : }
3041 :
3042 : pub mod tokio_epoll_uring {
3043 : use std::{
3044 : collections::HashMap,
3045 : sync::{Arc, Mutex},
3046 : };
3047 :
3048 : use metrics::{register_histogram, register_int_counter, Histogram, LocalHistogram, UIntGauge};
3049 : use once_cell::sync::Lazy;
3050 :
3051 : /// Shared storage for tokio-epoll-uring thread local metrics.
3052 : pub(crate) static THREAD_LOCAL_METRICS_STORAGE: Lazy<ThreadLocalMetricsStorage> =
3053 101 : Lazy::new(|| {
3054 101 : let slots_submission_queue_depth = register_histogram!(
3055 101 : "pageserver_tokio_epoll_uring_slots_submission_queue_depth",
3056 101 : "The slots waiters queue depth of each tokio_epoll_uring system",
3057 101 : vec![1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0],
3058 101 : )
3059 101 : .expect("failed to define a metric");
3060 101 : ThreadLocalMetricsStorage {
3061 101 : observers: Mutex::new(HashMap::new()),
3062 101 : slots_submission_queue_depth,
3063 101 : }
3064 101 : });
3065 :
3066 : pub struct ThreadLocalMetricsStorage {
3067 : /// List of thread local metrics observers.
3068 : observers: Mutex<HashMap<u64, Arc<ThreadLocalMetrics>>>,
3069 : /// A histogram shared between all thread local systems
3070 : /// for collecting slots submission queue depth.
3071 : slots_submission_queue_depth: Histogram,
3072 : }
3073 :
3074 : /// Each thread-local [`tokio_epoll_uring::System`] gets one of these as its
3075 : /// [`tokio_epoll_uring::metrics::PerSystemMetrics`] generic.
3076 : ///
3077 : /// The System makes observations into [`Self`] and periodically, the collector
3078 : /// comes along and flushes [`Self`] into the shared storage [`THREAD_LOCAL_METRICS_STORAGE`].
3079 : ///
3080 : /// [`LocalHistogram`] is `!Send`, so, we need to put it behind a [`Mutex`].
3081 : /// But except for the periodic flush, the lock is uncontended so there's no waiting
3082 : /// for cache coherence protocol to get an exclusive cache line.
3083 : pub struct ThreadLocalMetrics {
3084 : /// Local observer of thread local tokio-epoll-uring system's slots waiters queue depth.
3085 : slots_submission_queue_depth: Mutex<LocalHistogram>,
3086 : }
3087 :
3088 : impl ThreadLocalMetricsStorage {
3089 : /// Registers a new thread local system. Returns a thread local metrics observer.
3090 383 : pub fn register_system(&self, id: u64) -> Arc<ThreadLocalMetrics> {
3091 383 : let per_system_metrics = Arc::new(ThreadLocalMetrics::new(
3092 383 : self.slots_submission_queue_depth.local(),
3093 383 : ));
3094 383 : let mut g = self.observers.lock().unwrap();
3095 383 : g.insert(id, Arc::clone(&per_system_metrics));
3096 383 : per_system_metrics
3097 383 : }
3098 :
3099 : /// Removes metrics observer for a thread local system.
3100 : /// This should be called before dropping a thread local system.
3101 101 : pub fn remove_system(&self, id: u64) {
3102 101 : let mut g = self.observers.lock().unwrap();
3103 101 : g.remove(&id);
3104 101 : }
3105 :
3106 : /// Flush all thread local metrics to the shared storage.
3107 0 : pub fn flush_thread_local_metrics(&self) {
3108 0 : let g = self.observers.lock().unwrap();
3109 0 : g.values().for_each(|local| {
3110 0 : local.flush();
3111 0 : });
3112 0 : }
3113 : }
3114 :
3115 : impl ThreadLocalMetrics {
3116 383 : pub fn new(slots_submission_queue_depth: LocalHistogram) -> Self {
3117 383 : ThreadLocalMetrics {
3118 383 : slots_submission_queue_depth: Mutex::new(slots_submission_queue_depth),
3119 383 : }
3120 383 : }
3121 :
3122 : /// Flushes the thread local metrics to shared aggregator.
3123 0 : pub fn flush(&self) {
3124 0 : let Self {
3125 0 : slots_submission_queue_depth,
3126 0 : } = self;
3127 0 : slots_submission_queue_depth.lock().unwrap().flush();
3128 0 : }
3129 : }
3130 :
3131 : impl tokio_epoll_uring::metrics::PerSystemMetrics for ThreadLocalMetrics {
3132 1048409 : fn observe_slots_submission_queue_depth(&self, queue_depth: u64) {
3133 1048409 : let Self {
3134 1048409 : slots_submission_queue_depth,
3135 1048409 : } = self;
3136 1048409 : slots_submission_queue_depth
3137 1048409 : .lock()
3138 1048409 : .unwrap()
3139 1048409 : .observe(queue_depth as f64);
3140 1048409 : }
3141 : }
3142 :
3143 : pub struct Collector {
3144 : descs: Vec<metrics::core::Desc>,
3145 : systems_created: UIntGauge,
3146 : systems_destroyed: UIntGauge,
3147 : thread_local_metrics_storage: &'static ThreadLocalMetricsStorage,
3148 : }
3149 :
3150 : impl metrics::core::Collector for Collector {
3151 0 : fn desc(&self) -> Vec<&metrics::core::Desc> {
3152 0 : self.descs.iter().collect()
3153 0 : }
3154 :
3155 0 : fn collect(&self) -> Vec<metrics::proto::MetricFamily> {
3156 0 : let mut mfs = Vec::with_capacity(Self::NMETRICS);
3157 0 : let tokio_epoll_uring::metrics::GlobalMetrics {
3158 0 : systems_created,
3159 0 : systems_destroyed,
3160 0 : } = tokio_epoll_uring::metrics::global();
3161 0 : self.systems_created.set(systems_created);
3162 0 : mfs.extend(self.systems_created.collect());
3163 0 : self.systems_destroyed.set(systems_destroyed);
3164 0 : mfs.extend(self.systems_destroyed.collect());
3165 0 :
3166 0 : self.thread_local_metrics_storage
3167 0 : .flush_thread_local_metrics();
3168 0 :
3169 0 : mfs.extend(
3170 0 : self.thread_local_metrics_storage
3171 0 : .slots_submission_queue_depth
3172 0 : .collect(),
3173 0 : );
3174 0 : mfs
3175 0 : }
3176 : }
3177 :
3178 : impl Collector {
3179 : const NMETRICS: usize = 3;
3180 :
3181 : #[allow(clippy::new_without_default)]
3182 0 : pub fn new() -> Self {
3183 0 : let mut descs = Vec::new();
3184 0 :
3185 0 : let systems_created = UIntGauge::new(
3186 0 : "pageserver_tokio_epoll_uring_systems_created",
3187 0 : "counter of tokio-epoll-uring systems that were created",
3188 0 : )
3189 0 : .unwrap();
3190 0 : descs.extend(
3191 0 : metrics::core::Collector::desc(&systems_created)
3192 0 : .into_iter()
3193 0 : .cloned(),
3194 0 : );
3195 0 :
3196 0 : let systems_destroyed = UIntGauge::new(
3197 0 : "pageserver_tokio_epoll_uring_systems_destroyed",
3198 0 : "counter of tokio-epoll-uring systems that were destroyed",
3199 0 : )
3200 0 : .unwrap();
3201 0 : descs.extend(
3202 0 : metrics::core::Collector::desc(&systems_destroyed)
3203 0 : .into_iter()
3204 0 : .cloned(),
3205 0 : );
3206 0 :
3207 0 : Self {
3208 0 : descs,
3209 0 : systems_created,
3210 0 : systems_destroyed,
3211 0 : thread_local_metrics_storage: &THREAD_LOCAL_METRICS_STORAGE,
3212 0 : }
3213 0 : }
3214 : }
3215 :
3216 101 : pub(crate) static THREAD_LOCAL_LAUNCH_SUCCESSES: Lazy<metrics::IntCounter> = Lazy::new(|| {
3217 101 : register_int_counter!(
3218 101 : "pageserver_tokio_epoll_uring_pageserver_thread_local_launch_success_count",
3219 101 : "Number of times where thread_local_system creation spanned multiple executor threads",
3220 101 : )
3221 101 : .unwrap()
3222 101 : });
3223 :
3224 0 : pub(crate) static THREAD_LOCAL_LAUNCH_FAILURES: Lazy<metrics::IntCounter> = Lazy::new(|| {
3225 0 : register_int_counter!(
3226 0 : "pageserver_tokio_epoll_uring_pageserver_thread_local_launch_failures_count",
3227 0 : "Number of times thread_local_system creation failed and was retried after back-off.",
3228 0 : )
3229 0 : .unwrap()
3230 0 : });
3231 : }
3232 :
3233 : pub(crate) mod tenant_throttling {
3234 : use metrics::{register_int_counter_vec, IntCounter};
3235 : use once_cell::sync::Lazy;
3236 : use utils::shard::TenantShardId;
3237 :
3238 : use crate::tenant::{self, throttle::Metric};
3239 :
3240 : struct GlobalAndPerTenantIntCounter {
3241 : global: IntCounter,
3242 : per_tenant: IntCounter,
3243 : }
3244 :
3245 : impl GlobalAndPerTenantIntCounter {
3246 : #[inline(always)]
3247 0 : pub(crate) fn inc(&self) {
3248 0 : self.inc_by(1)
3249 0 : }
3250 : #[inline(always)]
3251 0 : pub(crate) fn inc_by(&self, n: u64) {
3252 0 : self.global.inc_by(n);
3253 0 : self.per_tenant.inc_by(n);
3254 0 : }
3255 : }
3256 :
3257 : pub(crate) struct TimelineGet {
3258 : count_accounted_start: GlobalAndPerTenantIntCounter,
3259 : count_accounted_finish: GlobalAndPerTenantIntCounter,
3260 : wait_time: GlobalAndPerTenantIntCounter,
3261 : count_throttled: GlobalAndPerTenantIntCounter,
3262 : }
3263 :
3264 172 : static COUNT_ACCOUNTED_START: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
3265 172 : register_int_counter_vec!(
3266 172 : "pageserver_tenant_throttling_count_accounted_start_global",
3267 172 : "Count of tenant throttling starts, by kind of throttle.",
3268 172 : &["kind"]
3269 172 : )
3270 172 : .unwrap()
3271 172 : });
3272 172 : static COUNT_ACCOUNTED_START_PER_TENANT: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
3273 172 : register_int_counter_vec!(
3274 172 : "pageserver_tenant_throttling_count_accounted_start",
3275 172 : "Count of tenant throttling starts, by kind of throttle.",
3276 172 : &["kind", "tenant_id", "shard_id"]
3277 172 : )
3278 172 : .unwrap()
3279 172 : });
3280 172 : static COUNT_ACCOUNTED_FINISH: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
3281 172 : register_int_counter_vec!(
3282 172 : "pageserver_tenant_throttling_count_accounted_finish_global",
3283 172 : "Count of tenant throttling finishes, by kind of throttle.",
3284 172 : &["kind"]
3285 172 : )
3286 172 : .unwrap()
3287 172 : });
3288 172 : static COUNT_ACCOUNTED_FINISH_PER_TENANT: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
3289 172 : register_int_counter_vec!(
3290 172 : "pageserver_tenant_throttling_count_accounted_finish",
3291 172 : "Count of tenant throttling finishes, by kind of throttle.",
3292 172 : &["kind", "tenant_id", "shard_id"]
3293 172 : )
3294 172 : .unwrap()
3295 172 : });
3296 172 : static WAIT_USECS: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
3297 172 : register_int_counter_vec!(
3298 172 : "pageserver_tenant_throttling_wait_usecs_sum_global",
3299 172 : "Sum of microseconds that spent waiting throttle by kind of throttle.",
3300 172 : &["kind"]
3301 172 : )
3302 172 : .unwrap()
3303 172 : });
3304 172 : static WAIT_USECS_PER_TENANT: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
3305 172 : register_int_counter_vec!(
3306 172 : "pageserver_tenant_throttling_wait_usecs_sum",
3307 172 : "Sum of microseconds that spent waiting throttle by kind of throttle.",
3308 172 : &["kind", "tenant_id", "shard_id"]
3309 172 : )
3310 172 : .unwrap()
3311 172 : });
3312 :
3313 172 : static WAIT_COUNT: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
3314 172 : register_int_counter_vec!(
3315 172 : "pageserver_tenant_throttling_count_global",
3316 172 : "Count of tenant throttlings, by kind of throttle.",
3317 172 : &["kind"]
3318 172 : )
3319 172 : .unwrap()
3320 172 : });
3321 172 : static WAIT_COUNT_PER_TENANT: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
3322 172 : register_int_counter_vec!(
3323 172 : "pageserver_tenant_throttling_count",
3324 172 : "Count of tenant throttlings, by kind of throttle.",
3325 172 : &["kind", "tenant_id", "shard_id"]
3326 172 : )
3327 172 : .unwrap()
3328 172 : });
3329 :
3330 : const KIND: &str = "timeline_get";
3331 :
3332 : impl TimelineGet {
3333 190 : pub(crate) fn new(tenant_shard_id: &TenantShardId) -> Self {
3334 190 : let per_tenant_label_values = &[
3335 190 : KIND,
3336 190 : &tenant_shard_id.tenant_id.to_string(),
3337 190 : &tenant_shard_id.shard_slug().to_string(),
3338 190 : ];
3339 190 : TimelineGet {
3340 190 : count_accounted_start: {
3341 190 : GlobalAndPerTenantIntCounter {
3342 190 : global: COUNT_ACCOUNTED_START.with_label_values(&[KIND]),
3343 190 : per_tenant: COUNT_ACCOUNTED_START_PER_TENANT
3344 190 : .with_label_values(per_tenant_label_values),
3345 190 : }
3346 190 : },
3347 190 : count_accounted_finish: {
3348 190 : GlobalAndPerTenantIntCounter {
3349 190 : global: COUNT_ACCOUNTED_FINISH.with_label_values(&[KIND]),
3350 190 : per_tenant: COUNT_ACCOUNTED_FINISH_PER_TENANT
3351 190 : .with_label_values(per_tenant_label_values),
3352 190 : }
3353 190 : },
3354 190 : wait_time: {
3355 190 : GlobalAndPerTenantIntCounter {
3356 190 : global: WAIT_USECS.with_label_values(&[KIND]),
3357 190 : per_tenant: WAIT_USECS_PER_TENANT
3358 190 : .with_label_values(per_tenant_label_values),
3359 190 : }
3360 190 : },
3361 190 : count_throttled: {
3362 190 : GlobalAndPerTenantIntCounter {
3363 190 : global: WAIT_COUNT.with_label_values(&[KIND]),
3364 190 : per_tenant: WAIT_COUNT_PER_TENANT
3365 190 : .with_label_values(per_tenant_label_values),
3366 190 : }
3367 190 : },
3368 190 : }
3369 190 : }
3370 : }
3371 :
3372 0 : pub(crate) fn preinitialize_global_metrics() {
3373 0 : Lazy::force(&COUNT_ACCOUNTED_START);
3374 0 : Lazy::force(&COUNT_ACCOUNTED_FINISH);
3375 0 : Lazy::force(&WAIT_USECS);
3376 0 : Lazy::force(&WAIT_COUNT);
3377 0 : }
3378 :
3379 6 : pub(crate) fn remove_tenant_metrics(tenant_shard_id: &TenantShardId) {
3380 24 : for m in &[
3381 6 : &COUNT_ACCOUNTED_START_PER_TENANT,
3382 6 : &COUNT_ACCOUNTED_FINISH_PER_TENANT,
3383 6 : &WAIT_USECS_PER_TENANT,
3384 6 : &WAIT_COUNT_PER_TENANT,
3385 24 : ] {
3386 24 : let _ = m.remove_label_values(&[
3387 24 : KIND,
3388 24 : &tenant_shard_id.tenant_id.to_string(),
3389 24 : &tenant_shard_id.shard_slug().to_string(),
3390 24 : ]);
3391 24 : }
3392 6 : }
3393 :
3394 : impl Metric for TimelineGet {
3395 : #[inline(always)]
3396 0 : fn accounting_start(&self) {
3397 0 : self.count_accounted_start.inc();
3398 0 : }
3399 : #[inline(always)]
3400 0 : fn accounting_finish(&self) {
3401 0 : self.count_accounted_finish.inc();
3402 0 : }
3403 : #[inline(always)]
3404 0 : fn observe_throttling(
3405 0 : &self,
3406 0 : tenant::throttle::Observation { wait_time }: &tenant::throttle::Observation,
3407 0 : ) {
3408 0 : let val = u64::try_from(wait_time.as_micros()).unwrap();
3409 0 : self.wait_time.inc_by(val);
3410 0 : self.count_throttled.inc();
3411 0 : }
3412 : }
3413 : }
3414 :
3415 : pub(crate) mod disk_usage_based_eviction {
3416 : use super::*;
3417 :
3418 : pub(crate) struct Metrics {
3419 : pub(crate) tenant_collection_time: Histogram,
3420 : pub(crate) tenant_layer_count: Histogram,
3421 : pub(crate) layers_collected: IntCounter,
3422 : pub(crate) layers_selected: IntCounter,
3423 : pub(crate) layers_evicted: IntCounter,
3424 : }
3425 :
3426 : impl Default for Metrics {
3427 0 : fn default() -> Self {
3428 0 : let tenant_collection_time = register_histogram!(
3429 0 : "pageserver_disk_usage_based_eviction_tenant_collection_seconds",
3430 0 : "Time spent collecting layers from a tenant -- not normalized by collected layer amount",
3431 0 : vec![0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0, 10.0]
3432 0 : )
3433 0 : .unwrap();
3434 0 :
3435 0 : let tenant_layer_count = register_histogram!(
3436 0 : "pageserver_disk_usage_based_eviction_tenant_collected_layers",
3437 0 : "Amount of layers gathered from a tenant",
3438 0 : vec![5.0, 50.0, 500.0, 5000.0, 50000.0]
3439 0 : )
3440 0 : .unwrap();
3441 0 :
3442 0 : let layers_collected = register_int_counter!(
3443 0 : "pageserver_disk_usage_based_eviction_collected_layers_total",
3444 0 : "Amount of layers collected"
3445 0 : )
3446 0 : .unwrap();
3447 0 :
3448 0 : let layers_selected = register_int_counter!(
3449 0 : "pageserver_disk_usage_based_eviction_select_layers_total",
3450 0 : "Amount of layers selected"
3451 0 : )
3452 0 : .unwrap();
3453 0 :
3454 0 : let layers_evicted = register_int_counter!(
3455 0 : "pageserver_disk_usage_based_eviction_evicted_layers_total",
3456 0 : "Amount of layers successfully evicted"
3457 0 : )
3458 0 : .unwrap();
3459 0 :
3460 0 : Self {
3461 0 : tenant_collection_time,
3462 0 : tenant_layer_count,
3463 0 : layers_collected,
3464 0 : layers_selected,
3465 0 : layers_evicted,
3466 0 : }
3467 0 : }
3468 : }
3469 :
3470 : pub(crate) static METRICS: Lazy<Metrics> = Lazy::new(Metrics::default);
3471 : }
3472 :
3473 166 : static TOKIO_EXECUTOR_THREAD_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
3474 166 : register_uint_gauge_vec!(
3475 166 : "pageserver_tokio_executor_thread_configured_count",
3476 166 : "Total number of configued tokio executor threads in the process.
3477 166 : The `setup` label denotes whether we're running with multiple runtimes or a single runtime.",
3478 166 : &["setup"],
3479 166 : )
3480 166 : .unwrap()
3481 166 : });
3482 :
3483 166 : pub(crate) fn set_tokio_runtime_setup(setup: &str, num_threads: NonZeroUsize) {
3484 : static SERIALIZE: std::sync::Mutex<()> = std::sync::Mutex::new(());
3485 166 : let _guard = SERIALIZE.lock().unwrap();
3486 166 : TOKIO_EXECUTOR_THREAD_COUNT.reset();
3487 166 : TOKIO_EXECUTOR_THREAD_COUNT
3488 166 : .get_metric_with_label_values(&[setup])
3489 166 : .unwrap()
3490 166 : .set(u64::try_from(num_threads.get()).unwrap());
3491 166 : }
3492 :
3493 0 : pub fn preinitialize_metrics() {
3494 0 : // Python tests need these and on some we do alerting.
3495 0 : //
3496 0 : // FIXME(4813): make it so that we have no top level metrics as this fn will easily fall out of
3497 0 : // order:
3498 0 : // - global metrics reside in a Lazy<PageserverMetrics>
3499 0 : // - access via crate::metrics::PS_METRICS.some_metric.inc()
3500 0 : // - could move the statics into TimelineMetrics::new()?
3501 0 :
3502 0 : // counters
3503 0 : [
3504 0 : &UNEXPECTED_ONDEMAND_DOWNLOADS,
3505 0 : &WALRECEIVER_STARTED_CONNECTIONS,
3506 0 : &WALRECEIVER_BROKER_UPDATES,
3507 0 : &WALRECEIVER_CANDIDATES_ADDED,
3508 0 : &WALRECEIVER_CANDIDATES_REMOVED,
3509 0 : &tokio_epoll_uring::THREAD_LOCAL_LAUNCH_FAILURES,
3510 0 : &tokio_epoll_uring::THREAD_LOCAL_LAUNCH_SUCCESSES,
3511 0 : &REMOTE_ONDEMAND_DOWNLOADED_LAYERS,
3512 0 : &REMOTE_ONDEMAND_DOWNLOADED_BYTES,
3513 0 : &CIRCUIT_BREAKERS_BROKEN,
3514 0 : &CIRCUIT_BREAKERS_UNBROKEN,
3515 0 : ]
3516 0 : .into_iter()
3517 0 : .for_each(|c| {
3518 0 : Lazy::force(c);
3519 0 : });
3520 0 :
3521 0 : // Deletion queue stats
3522 0 : Lazy::force(&DELETION_QUEUE);
3523 0 :
3524 0 : // Tenant stats
3525 0 : Lazy::force(&TENANT);
3526 0 :
3527 0 : // Tenant manager stats
3528 0 : Lazy::force(&TENANT_MANAGER);
3529 0 :
3530 0 : Lazy::force(&crate::tenant::storage_layer::layer::LAYER_IMPL_METRICS);
3531 0 : Lazy::force(&disk_usage_based_eviction::METRICS);
3532 :
3533 0 : for state_name in pageserver_api::models::TenantState::VARIANTS {
3534 0 : // initialize the metric for all gauges, otherwise the time series might seemingly show
3535 0 : // values from last restart.
3536 0 : TENANT_STATE_METRIC.with_label_values(&[state_name]).set(0);
3537 0 : }
3538 :
3539 : // countervecs
3540 0 : [
3541 0 : &BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT,
3542 0 : &SMGR_QUERY_STARTED_GLOBAL,
3543 0 : ]
3544 0 : .into_iter()
3545 0 : .for_each(|c| {
3546 0 : Lazy::force(c);
3547 0 : });
3548 0 :
3549 0 : // gauges
3550 0 : WALRECEIVER_ACTIVE_MANAGERS.get();
3551 0 :
3552 0 : // histograms
3553 0 : [
3554 0 : &READ_NUM_LAYERS_VISITED,
3555 0 : &VEC_READ_NUM_LAYERS_VISITED,
3556 0 : &WAIT_LSN_TIME,
3557 0 : &WAL_REDO_TIME,
3558 0 : &WAL_REDO_RECORDS_HISTOGRAM,
3559 0 : &WAL_REDO_BYTES_HISTOGRAM,
3560 0 : &WAL_REDO_PROCESS_LAUNCH_DURATION_HISTOGRAM,
3561 0 : ]
3562 0 : .into_iter()
3563 0 : .for_each(|h| {
3564 0 : Lazy::force(h);
3565 0 : });
3566 0 :
3567 0 : // Custom
3568 0 : Lazy::force(&RECONSTRUCT_TIME);
3569 0 : Lazy::force(&BASEBACKUP_QUERY_TIME);
3570 0 : Lazy::force(&COMPUTE_COMMANDS_COUNTERS);
3571 0 : Lazy::force(&tokio_epoll_uring::THREAD_LOCAL_METRICS_STORAGE);
3572 0 :
3573 0 : tenant_throttling::preinitialize_global_metrics();
3574 0 : }
|
