Line data Source code
1 : use enum_map::EnumMap;
2 : use metrics::{
3 : register_counter_vec, register_gauge_vec, register_histogram, register_histogram_vec,
4 : register_int_counter, register_int_counter_pair_vec, register_int_counter_vec,
5 : register_int_gauge, register_int_gauge_vec, register_uint_gauge, register_uint_gauge_vec,
6 : Counter, CounterVec, Gauge, GaugeVec, Histogram, HistogramVec, IntCounter, IntCounterPair,
7 : IntCounterPairVec, IntCounterVec, IntGauge, IntGaugeVec, UIntGauge, UIntGaugeVec,
8 : };
9 : use once_cell::sync::Lazy;
10 : use pageserver_api::config::{
11 : PageServicePipeliningConfig, PageServicePipeliningConfigPipelined,
12 : PageServiceProtocolPipelinedExecutionStrategy,
13 : };
14 : use pageserver_api::shard::TenantShardId;
15 : use postgres_backend::{is_expected_io_error, QueryError};
16 : use pq_proto::framed::ConnectionError;
17 : use strum::{EnumCount, VariantNames};
18 : use strum_macros::{IntoStaticStr, VariantNames};
19 : use utils::id::TimelineId;
20 :
21 : /// Prometheus histogram buckets (in seconds) for operations in the critical
22 : /// path. In other words, operations that directly affect that latency of user
23 : /// queries.
24 : ///
25 : /// The buckets capture the majority of latencies in the microsecond and
26 : /// millisecond range but also extend far enough up to distinguish "bad" from
27 : /// "really bad".
28 : const CRITICAL_OP_BUCKETS: &[f64] = &[
29 : 0.000_001, 0.000_010, 0.000_100, // 1 us, 10 us, 100 us
30 : 0.001_000, 0.010_000, 0.100_000, // 1 ms, 10 ms, 100 ms
31 : 1.0, 10.0, 100.0, // 1 s, 10 s, 100 s
32 : ];
33 :
34 : // Metrics collected on operations on the storage repository.
35 3376 : #[derive(Debug, VariantNames, IntoStaticStr)]
36 : #[strum(serialize_all = "kebab_case")]
37 : pub(crate) enum StorageTimeOperation {
38 : #[strum(serialize = "layer flush")]
39 : LayerFlush,
40 :
41 : #[strum(serialize = "compact")]
42 : Compact,
43 :
44 : #[strum(serialize = "create images")]
45 : CreateImages,
46 :
47 : #[strum(serialize = "logical size")]
48 : LogicalSize,
49 :
50 : #[strum(serialize = "imitate logical size")]
51 : ImitateLogicalSize,
52 :
53 : #[strum(serialize = "load layer map")]
54 : LoadLayerMap,
55 :
56 : #[strum(serialize = "gc")]
57 : Gc,
58 :
59 : #[strum(serialize = "find gc cutoffs")]
60 : FindGcCutoffs,
61 : }
62 :
63 176 : pub(crate) static STORAGE_TIME_SUM_PER_TIMELINE: Lazy<CounterVec> = Lazy::new(|| {
64 176 : register_counter_vec!(
65 176 : "pageserver_storage_operations_seconds_sum",
66 176 : "Total time spent on storage operations with operation, tenant and timeline dimensions",
67 176 : &["operation", "tenant_id", "shard_id", "timeline_id"],
68 176 : )
69 176 : .expect("failed to define a metric")
70 176 : });
71 :
72 176 : pub(crate) static STORAGE_TIME_COUNT_PER_TIMELINE: Lazy<IntCounterVec> = Lazy::new(|| {
73 176 : register_int_counter_vec!(
74 176 : "pageserver_storage_operations_seconds_count",
75 176 : "Count of storage operations with operation, tenant and timeline dimensions",
76 176 : &["operation", "tenant_id", "shard_id", "timeline_id"],
77 176 : )
78 176 : .expect("failed to define a metric")
79 176 : });
80 :
81 : // Buckets for background operations like compaction, GC, size calculation
82 : const STORAGE_OP_BUCKETS: &[f64] = &[0.010, 0.100, 1.0, 10.0, 100.0, 1000.0];
83 :
84 176 : pub(crate) static STORAGE_TIME_GLOBAL: Lazy<HistogramVec> = Lazy::new(|| {
85 176 : register_histogram_vec!(
86 176 : "pageserver_storage_operations_seconds_global",
87 176 : "Time spent on storage operations",
88 176 : &["operation"],
89 176 : STORAGE_OP_BUCKETS.into(),
90 176 : )
91 176 : .expect("failed to define a metric")
92 176 : });
93 :
94 0 : pub(crate) static READ_NUM_LAYERS_VISITED: Lazy<Histogram> = Lazy::new(|| {
95 0 : register_histogram!(
96 0 : "pageserver_layers_visited_per_read_global",
97 0 : "Number of layers visited to reconstruct one key",
98 0 : vec![1.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0],
99 0 : )
100 0 : .expect("failed to define a metric")
101 0 : });
102 :
103 172 : pub(crate) static VEC_READ_NUM_LAYERS_VISITED: Lazy<Histogram> = Lazy::new(|| {
104 172 : register_histogram!(
105 172 : "pageserver_layers_visited_per_vectored_read_global",
106 172 : "Average number of layers visited to reconstruct one key",
107 172 : vec![1.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0],
108 172 : )
109 172 : .expect("failed to define a metric")
110 172 : });
111 :
112 : // Metrics collected on operations on the storage repository.
113 : #[derive(
114 696 : Clone, Copy, enum_map::Enum, strum_macros::EnumString, strum_macros::Display, IntoStaticStr,
115 : )]
116 : pub(crate) enum GetKind {
117 : Singular,
118 : Vectored,
119 : }
120 :
121 : pub(crate) struct ReconstructTimeMetrics {
122 : singular: Histogram,
123 : vectored: Histogram,
124 : }
125 :
126 174 : pub(crate) static RECONSTRUCT_TIME: Lazy<ReconstructTimeMetrics> = Lazy::new(|| {
127 174 : let inner = register_histogram_vec!(
128 174 : "pageserver_getpage_reconstruct_seconds",
129 174 : "Time spent in reconstruct_value (reconstruct a page from deltas)",
130 174 : &["get_kind"],
131 174 : CRITICAL_OP_BUCKETS.into(),
132 174 : )
133 174 : .expect("failed to define a metric");
134 174 :
135 174 : ReconstructTimeMetrics {
136 174 : singular: inner.with_label_values(&[GetKind::Singular.into()]),
137 174 : vectored: inner.with_label_values(&[GetKind::Vectored.into()]),
138 174 : }
139 174 : });
140 :
141 : impl ReconstructTimeMetrics {
142 627401 : pub(crate) fn for_get_kind(&self, get_kind: GetKind) -> &Histogram {
143 627401 : match get_kind {
144 626957 : GetKind::Singular => &self.singular,
145 444 : GetKind::Vectored => &self.vectored,
146 : }
147 627401 : }
148 : }
149 :
150 : pub(crate) struct ReconstructDataTimeMetrics {
151 : singular: Histogram,
152 : vectored: Histogram,
153 : }
154 :
155 : impl ReconstructDataTimeMetrics {
156 627417 : pub(crate) fn for_get_kind(&self, get_kind: GetKind) -> &Histogram {
157 627417 : match get_kind {
158 626973 : GetKind::Singular => &self.singular,
159 444 : GetKind::Vectored => &self.vectored,
160 : }
161 627417 : }
162 : }
163 :
164 174 : pub(crate) static GET_RECONSTRUCT_DATA_TIME: Lazy<ReconstructDataTimeMetrics> = Lazy::new(|| {
165 174 : let inner = register_histogram_vec!(
166 174 : "pageserver_getpage_get_reconstruct_data_seconds",
167 174 : "Time spent in get_reconstruct_value_data",
168 174 : &["get_kind"],
169 174 : CRITICAL_OP_BUCKETS.into(),
170 174 : )
171 174 : .expect("failed to define a metric");
172 174 :
173 174 : ReconstructDataTimeMetrics {
174 174 : singular: inner.with_label_values(&[GetKind::Singular.into()]),
175 174 : vectored: inner.with_label_values(&[GetKind::Vectored.into()]),
176 174 : }
177 174 : });
178 :
179 : pub(crate) struct GetVectoredLatency {
180 : map: EnumMap<TaskKind, Option<Histogram>>,
181 : }
182 :
183 : #[allow(dead_code)]
184 : pub(crate) struct ScanLatency {
185 : map: EnumMap<TaskKind, Option<Histogram>>,
186 : }
187 :
188 : impl GetVectoredLatency {
189 : // Only these task types perform vectored gets. Filter all other tasks out to reduce total
190 : // cardinality of the metric.
191 : const TRACKED_TASK_KINDS: [TaskKind; 2] = [TaskKind::Compaction, TaskKind::PageRequestHandler];
192 :
193 19534 : pub(crate) fn for_task_kind(&self, task_kind: TaskKind) -> Option<&Histogram> {
194 19534 : self.map[task_kind].as_ref()
195 19534 : }
196 : }
197 :
198 : impl ScanLatency {
199 : // Only these task types perform vectored gets. Filter all other tasks out to reduce total
200 : // cardinality of the metric.
201 : const TRACKED_TASK_KINDS: [TaskKind; 1] = [TaskKind::PageRequestHandler];
202 :
203 12 : pub(crate) fn for_task_kind(&self, task_kind: TaskKind) -> Option<&Histogram> {
204 12 : self.map[task_kind].as_ref()
205 12 : }
206 : }
207 :
208 : pub(crate) struct ScanLatencyOngoingRecording<'a> {
209 : parent: &'a Histogram,
210 : start: std::time::Instant,
211 : }
212 :
213 : impl<'a> ScanLatencyOngoingRecording<'a> {
214 0 : pub(crate) fn start_recording(parent: &'a Histogram) -> ScanLatencyOngoingRecording<'a> {
215 0 : let start = Instant::now();
216 0 : ScanLatencyOngoingRecording { parent, start }
217 0 : }
218 :
219 0 : pub(crate) fn observe(self) {
220 0 : let elapsed = self.start.elapsed();
221 0 : self.parent.observe(elapsed.as_secs_f64());
222 0 : }
223 : }
224 :
225 168 : pub(crate) static GET_VECTORED_LATENCY: Lazy<GetVectoredLatency> = Lazy::new(|| {
226 168 : let inner = register_histogram_vec!(
227 168 : "pageserver_get_vectored_seconds",
228 168 : "Time spent in get_vectored.",
229 168 : &["task_kind"],
230 168 : CRITICAL_OP_BUCKETS.into(),
231 168 : )
232 168 : .expect("failed to define a metric");
233 168 :
234 168 : GetVectoredLatency {
235 5208 : map: EnumMap::from_array(std::array::from_fn(|task_kind_idx| {
236 5208 : let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind_idx);
237 5208 :
238 5208 : if GetVectoredLatency::TRACKED_TASK_KINDS.contains(&task_kind) {
239 336 : let task_kind = task_kind.into();
240 336 : Some(inner.with_label_values(&[task_kind]))
241 : } else {
242 4872 : None
243 : }
244 5208 : })),
245 168 : }
246 168 : });
247 :
248 4 : pub(crate) static SCAN_LATENCY: Lazy<ScanLatency> = Lazy::new(|| {
249 4 : let inner = register_histogram_vec!(
250 4 : "pageserver_scan_seconds",
251 4 : "Time spent in scan.",
252 4 : &["task_kind"],
253 4 : CRITICAL_OP_BUCKETS.into(),
254 4 : )
255 4 : .expect("failed to define a metric");
256 4 :
257 4 : ScanLatency {
258 124 : map: EnumMap::from_array(std::array::from_fn(|task_kind_idx| {
259 124 : let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind_idx);
260 124 :
261 124 : if ScanLatency::TRACKED_TASK_KINDS.contains(&task_kind) {
262 4 : let task_kind = task_kind.into();
263 4 : Some(inner.with_label_values(&[task_kind]))
264 : } else {
265 120 : None
266 : }
267 124 : })),
268 4 : }
269 4 : });
270 :
271 : pub(crate) struct PageCacheMetricsForTaskKind {
272 : pub read_accesses_immutable: IntCounter,
273 : pub read_hits_immutable: IntCounter,
274 : }
275 :
276 : pub(crate) struct PageCacheMetrics {
277 : map: EnumMap<TaskKind, EnumMap<PageContentKind, PageCacheMetricsForTaskKind>>,
278 : }
279 :
280 92 : static PAGE_CACHE_READ_HITS: Lazy<IntCounterVec> = Lazy::new(|| {
281 92 : register_int_counter_vec!(
282 92 : "pageserver_page_cache_read_hits_total",
283 92 : "Number of read accesses to the page cache that hit",
284 92 : &["task_kind", "key_kind", "content_kind", "hit_kind"]
285 92 : )
286 92 : .expect("failed to define a metric")
287 92 : });
288 :
289 92 : static PAGE_CACHE_READ_ACCESSES: Lazy<IntCounterVec> = Lazy::new(|| {
290 92 : register_int_counter_vec!(
291 92 : "pageserver_page_cache_read_accesses_total",
292 92 : "Number of read accesses to the page cache",
293 92 : &["task_kind", "key_kind", "content_kind"]
294 92 : )
295 92 : .expect("failed to define a metric")
296 92 : });
297 :
298 92 : pub(crate) static PAGE_CACHE: Lazy<PageCacheMetrics> = Lazy::new(|| PageCacheMetrics {
299 2852 : map: EnumMap::from_array(std::array::from_fn(|task_kind| {
300 2852 : let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind);
301 2852 : let task_kind: &'static str = task_kind.into();
302 22816 : EnumMap::from_array(std::array::from_fn(|content_kind| {
303 22816 : let content_kind = <PageContentKind as enum_map::Enum>::from_usize(content_kind);
304 22816 : let content_kind: &'static str = content_kind.into();
305 22816 : PageCacheMetricsForTaskKind {
306 22816 : read_accesses_immutable: {
307 22816 : PAGE_CACHE_READ_ACCESSES
308 22816 : .get_metric_with_label_values(&[task_kind, "immutable", content_kind])
309 22816 : .unwrap()
310 22816 : },
311 22816 :
312 22816 : read_hits_immutable: {
313 22816 : PAGE_CACHE_READ_HITS
314 22816 : .get_metric_with_label_values(&[task_kind, "immutable", content_kind, "-"])
315 22816 : .unwrap()
316 22816 : },
317 22816 : }
318 22816 : }))
319 2852 : })),
320 92 : });
321 :
322 : impl PageCacheMetrics {
323 977082 : pub(crate) fn for_ctx(&self, ctx: &RequestContext) -> &PageCacheMetricsForTaskKind {
324 977082 : &self.map[ctx.task_kind()][ctx.page_content_kind()]
325 977082 : }
326 : }
327 :
328 : pub(crate) struct PageCacheSizeMetrics {
329 : pub max_bytes: UIntGauge,
330 :
331 : pub current_bytes_immutable: UIntGauge,
332 : }
333 :
334 92 : static PAGE_CACHE_SIZE_CURRENT_BYTES: Lazy<UIntGaugeVec> = Lazy::new(|| {
335 92 : register_uint_gauge_vec!(
336 92 : "pageserver_page_cache_size_current_bytes",
337 92 : "Current size of the page cache in bytes, by key kind",
338 92 : &["key_kind"]
339 92 : )
340 92 : .expect("failed to define a metric")
341 92 : });
342 :
343 : pub(crate) static PAGE_CACHE_SIZE: Lazy<PageCacheSizeMetrics> =
344 92 : Lazy::new(|| PageCacheSizeMetrics {
345 92 : max_bytes: {
346 92 : register_uint_gauge!(
347 92 : "pageserver_page_cache_size_max_bytes",
348 92 : "Maximum size of the page cache in bytes"
349 92 : )
350 92 : .expect("failed to define a metric")
351 92 : },
352 92 : current_bytes_immutable: {
353 92 : PAGE_CACHE_SIZE_CURRENT_BYTES
354 92 : .get_metric_with_label_values(&["immutable"])
355 92 : .unwrap()
356 92 : },
357 92 : });
358 :
359 : pub(crate) mod page_cache_eviction_metrics {
360 : use std::num::NonZeroUsize;
361 :
362 : use metrics::{register_int_counter_vec, IntCounter, IntCounterVec};
363 : use once_cell::sync::Lazy;
364 :
365 : #[derive(Clone, Copy)]
366 : pub(crate) enum Outcome {
367 : FoundSlotUnused { iters: NonZeroUsize },
368 : FoundSlotEvicted { iters: NonZeroUsize },
369 : ItersExceeded { iters: NonZeroUsize },
370 : }
371 :
372 92 : static ITERS_TOTAL_VEC: Lazy<IntCounterVec> = Lazy::new(|| {
373 92 : register_int_counter_vec!(
374 92 : "pageserver_page_cache_find_victim_iters_total",
375 92 : "Counter for the number of iterations in the find_victim loop",
376 92 : &["outcome"],
377 92 : )
378 92 : .expect("failed to define a metric")
379 92 : });
380 :
381 92 : static CALLS_VEC: Lazy<IntCounterVec> = Lazy::new(|| {
382 92 : register_int_counter_vec!(
383 92 : "pageserver_page_cache_find_victim_calls",
384 92 : "Incremented at the end of each find_victim() call.\
385 92 : Filter by outcome to get e.g., eviction rate.",
386 92 : &["outcome"]
387 92 : )
388 92 : .unwrap()
389 92 : });
390 :
391 32163 : pub(crate) fn observe(outcome: Outcome) {
392 : macro_rules! dry {
393 : ($label:literal, $iters:expr) => {{
394 : static LABEL: &'static str = $label;
395 : static ITERS_TOTAL: Lazy<IntCounter> =
396 112 : Lazy::new(|| ITERS_TOTAL_VEC.with_label_values(&[LABEL]));
397 : static CALLS: Lazy<IntCounter> =
398 112 : Lazy::new(|| CALLS_VEC.with_label_values(&[LABEL]));
399 : ITERS_TOTAL.inc_by(($iters.get()) as u64);
400 : CALLS.inc();
401 : }};
402 : }
403 32163 : match outcome {
404 1636 : Outcome::FoundSlotUnused { iters } => dry!("found_empty", iters),
405 30527 : Outcome::FoundSlotEvicted { iters } => {
406 30527 : dry!("found_evicted", iters)
407 : }
408 0 : Outcome::ItersExceeded { iters } => {
409 0 : dry!("err_iters_exceeded", iters);
410 0 : super::page_cache_errors_inc(super::PageCacheErrorKind::EvictIterLimit);
411 0 : }
412 : }
413 32163 : }
414 : }
415 :
416 0 : static PAGE_CACHE_ERRORS: Lazy<IntCounterVec> = Lazy::new(|| {
417 0 : register_int_counter_vec!(
418 0 : "page_cache_errors_total",
419 0 : "Number of timeouts while acquiring a pinned slot in the page cache",
420 0 : &["error_kind"]
421 0 : )
422 0 : .expect("failed to define a metric")
423 0 : });
424 :
425 0 : #[derive(IntoStaticStr)]
426 : #[strum(serialize_all = "kebab_case")]
427 : pub(crate) enum PageCacheErrorKind {
428 : AcquirePinnedSlotTimeout,
429 : EvictIterLimit,
430 : }
431 :
432 0 : pub(crate) fn page_cache_errors_inc(error_kind: PageCacheErrorKind) {
433 0 : PAGE_CACHE_ERRORS
434 0 : .get_metric_with_label_values(&[error_kind.into()])
435 0 : .unwrap()
436 0 : .inc();
437 0 : }
438 :
439 18 : pub(crate) static WAIT_LSN_TIME: Lazy<Histogram> = Lazy::new(|| {
440 18 : register_histogram!(
441 18 : "pageserver_wait_lsn_seconds",
442 18 : "Time spent waiting for WAL to arrive",
443 18 : CRITICAL_OP_BUCKETS.into(),
444 18 : )
445 18 : .expect("failed to define a metric")
446 18 : });
447 :
448 176 : static FLUSH_WAIT_UPLOAD_TIME: Lazy<GaugeVec> = Lazy::new(|| {
449 176 : register_gauge_vec!(
450 176 : "pageserver_flush_wait_upload_seconds",
451 176 : "Time spent waiting for preceding uploads during layer flush",
452 176 : &["tenant_id", "shard_id", "timeline_id"]
453 176 : )
454 176 : .expect("failed to define a metric")
455 176 : });
456 :
457 176 : static LAST_RECORD_LSN: Lazy<IntGaugeVec> = Lazy::new(|| {
458 176 : register_int_gauge_vec!(
459 176 : "pageserver_last_record_lsn",
460 176 : "Last record LSN grouped by timeline",
461 176 : &["tenant_id", "shard_id", "timeline_id"]
462 176 : )
463 176 : .expect("failed to define a metric")
464 176 : });
465 :
466 176 : static DISK_CONSISTENT_LSN: Lazy<IntGaugeVec> = Lazy::new(|| {
467 176 : register_int_gauge_vec!(
468 176 : "pageserver_disk_consistent_lsn",
469 176 : "Disk consistent LSN grouped by timeline",
470 176 : &["tenant_id", "shard_id", "timeline_id"]
471 176 : )
472 176 : .expect("failed to define a metric")
473 176 : });
474 :
475 176 : pub(crate) static PROJECTED_REMOTE_CONSISTENT_LSN: Lazy<UIntGaugeVec> = Lazy::new(|| {
476 176 : register_uint_gauge_vec!(
477 176 : "pageserver_projected_remote_consistent_lsn",
478 176 : "Projected remote consistent LSN grouped by timeline",
479 176 : &["tenant_id", "shard_id", "timeline_id"]
480 176 : )
481 176 : .expect("failed to define a metric")
482 176 : });
483 :
484 176 : static PITR_HISTORY_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
485 176 : register_uint_gauge_vec!(
486 176 : "pageserver_pitr_history_size",
487 176 : "Data written since PITR cutoff on this timeline",
488 176 : &["tenant_id", "shard_id", "timeline_id"]
489 176 : )
490 176 : .expect("failed to define a metric")
491 176 : });
492 :
493 1728 : #[derive(strum_macros::EnumString, strum_macros::Display, strum_macros::IntoStaticStr)]
494 : #[strum(serialize_all = "kebab_case")]
495 : pub(crate) enum MetricLayerKind {
496 : Delta,
497 : Image,
498 : }
499 :
500 176 : static TIMELINE_LAYER_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
501 176 : register_uint_gauge_vec!(
502 176 : "pageserver_layer_bytes",
503 176 : "Sum of layer physical sizes in bytes",
504 176 : &["tenant_id", "shard_id", "timeline_id", "kind"]
505 176 : )
506 176 : .expect("failed to define a metric")
507 176 : });
508 :
509 176 : static TIMELINE_LAYER_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
510 176 : register_uint_gauge_vec!(
511 176 : "pageserver_layer_count",
512 176 : "Number of layers that exist",
513 176 : &["tenant_id", "shard_id", "timeline_id", "kind"]
514 176 : )
515 176 : .expect("failed to define a metric")
516 176 : });
517 :
518 176 : static TIMELINE_ARCHIVE_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
519 176 : register_uint_gauge_vec!(
520 176 : "pageserver_archive_size",
521 176 : "Timeline's logical size if it is considered eligible for archival (outside PITR window), else zero",
522 176 : &["tenant_id", "shard_id", "timeline_id"]
523 176 : )
524 176 : .expect("failed to define a metric")
525 176 : });
526 :
527 176 : static STANDBY_HORIZON: Lazy<IntGaugeVec> = Lazy::new(|| {
528 176 : register_int_gauge_vec!(
529 176 : "pageserver_standby_horizon",
530 176 : "Standby apply LSN for which GC is hold off, by timeline.",
531 176 : &["tenant_id", "shard_id", "timeline_id"]
532 176 : )
533 176 : .expect("failed to define a metric")
534 176 : });
535 :
536 176 : static RESIDENT_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
537 176 : register_uint_gauge_vec!(
538 176 : "pageserver_resident_physical_size",
539 176 : "The size of the layer files present in the pageserver's filesystem, for attached locations.",
540 176 : &["tenant_id", "shard_id", "timeline_id"]
541 176 : )
542 176 : .expect("failed to define a metric")
543 176 : });
544 :
545 176 : static VISIBLE_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
546 176 : register_uint_gauge_vec!(
547 176 : "pageserver_visible_physical_size",
548 176 : "The size of the layer files present in the pageserver's filesystem.",
549 176 : &["tenant_id", "shard_id", "timeline_id"]
550 176 : )
551 176 : .expect("failed to define a metric")
552 176 : });
553 :
554 172 : pub(crate) static RESIDENT_PHYSICAL_SIZE_GLOBAL: Lazy<UIntGauge> = Lazy::new(|| {
555 172 : register_uint_gauge!(
556 172 : "pageserver_resident_physical_size_global",
557 172 : "Like `pageserver_resident_physical_size`, but without tenant/timeline dimensions."
558 172 : )
559 172 : .expect("failed to define a metric")
560 172 : });
561 :
562 176 : static REMOTE_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
563 176 : register_uint_gauge_vec!(
564 176 : "pageserver_remote_physical_size",
565 176 : "The size of the layer files present in the remote storage that are listed in the remote index_part.json.",
566 176 : // Corollary: If any files are missing from the index part, they won't be included here.
567 176 : &["tenant_id", "shard_id", "timeline_id"]
568 176 : )
569 176 : .expect("failed to define a metric")
570 176 : });
571 :
572 176 : static REMOTE_PHYSICAL_SIZE_GLOBAL: Lazy<UIntGauge> = Lazy::new(|| {
573 176 : register_uint_gauge!(
574 176 : "pageserver_remote_physical_size_global",
575 176 : "Like `pageserver_remote_physical_size`, but without tenant/timeline dimensions."
576 176 : )
577 176 : .expect("failed to define a metric")
578 176 : });
579 :
580 4 : pub(crate) static REMOTE_ONDEMAND_DOWNLOADED_LAYERS: Lazy<IntCounter> = Lazy::new(|| {
581 4 : register_int_counter!(
582 4 : "pageserver_remote_ondemand_downloaded_layers_total",
583 4 : "Total on-demand downloaded layers"
584 4 : )
585 4 : .unwrap()
586 4 : });
587 :
588 4 : pub(crate) static REMOTE_ONDEMAND_DOWNLOADED_BYTES: Lazy<IntCounter> = Lazy::new(|| {
589 4 : register_int_counter!(
590 4 : "pageserver_remote_ondemand_downloaded_bytes_total",
591 4 : "Total bytes of layers on-demand downloaded",
592 4 : )
593 4 : .unwrap()
594 4 : });
595 :
596 176 : static CURRENT_LOGICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
597 176 : register_uint_gauge_vec!(
598 176 : "pageserver_current_logical_size",
599 176 : "Current logical size grouped by timeline",
600 176 : &["tenant_id", "shard_id", "timeline_id"]
601 176 : )
602 176 : .expect("failed to define current logical size metric")
603 176 : });
604 :
605 176 : static AUX_FILE_SIZE: Lazy<IntGaugeVec> = Lazy::new(|| {
606 176 : register_int_gauge_vec!(
607 176 : "pageserver_aux_file_estimated_size",
608 176 : "The size of all aux files for a timeline in aux file v2 store.",
609 176 : &["tenant_id", "shard_id", "timeline_id"]
610 176 : )
611 176 : .expect("failed to define a metric")
612 176 : });
613 :
614 176 : static VALID_LSN_LEASE_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
615 176 : register_uint_gauge_vec!(
616 176 : "pageserver_valid_lsn_lease_count",
617 176 : "The number of valid leases after refreshing gc info.",
618 176 : &["tenant_id", "shard_id", "timeline_id"],
619 176 : )
620 176 : .expect("failed to define a metric")
621 176 : });
622 :
623 0 : pub(crate) static CIRCUIT_BREAKERS_BROKEN: Lazy<IntCounter> = Lazy::new(|| {
624 0 : register_int_counter!(
625 0 : "pageserver_circuit_breaker_broken",
626 0 : "How many times a circuit breaker has broken"
627 0 : )
628 0 : .expect("failed to define a metric")
629 0 : });
630 :
631 0 : pub(crate) static CIRCUIT_BREAKERS_UNBROKEN: Lazy<IntCounter> = Lazy::new(|| {
632 0 : register_int_counter!(
633 0 : "pageserver_circuit_breaker_unbroken",
634 0 : "How many times a circuit breaker has been un-broken (recovered)"
635 0 : )
636 0 : .expect("failed to define a metric")
637 0 : });
638 :
639 168 : pub(crate) static COMPRESSION_IMAGE_INPUT_BYTES: Lazy<IntCounter> = Lazy::new(|| {
640 168 : register_int_counter!(
641 168 : "pageserver_compression_image_in_bytes_total",
642 168 : "Size of data written into image layers before compression"
643 168 : )
644 168 : .expect("failed to define a metric")
645 168 : });
646 :
647 168 : pub(crate) static COMPRESSION_IMAGE_INPUT_BYTES_CONSIDERED: Lazy<IntCounter> = Lazy::new(|| {
648 168 : register_int_counter!(
649 168 : "pageserver_compression_image_in_bytes_considered",
650 168 : "Size of potentially compressible data written into image layers before compression"
651 168 : )
652 168 : .expect("failed to define a metric")
653 168 : });
654 :
655 168 : pub(crate) static COMPRESSION_IMAGE_INPUT_BYTES_CHOSEN: Lazy<IntCounter> = Lazy::new(|| {
656 168 : register_int_counter!(
657 168 : "pageserver_compression_image_in_bytes_chosen",
658 168 : "Size of data whose compressed form was written into image layers"
659 168 : )
660 168 : .expect("failed to define a metric")
661 168 : });
662 :
663 168 : pub(crate) static COMPRESSION_IMAGE_OUTPUT_BYTES: Lazy<IntCounter> = Lazy::new(|| {
664 168 : register_int_counter!(
665 168 : "pageserver_compression_image_out_bytes_total",
666 168 : "Size of compressed image layer written"
667 168 : )
668 168 : .expect("failed to define a metric")
669 168 : });
670 :
671 10 : pub(crate) static RELSIZE_CACHE_ENTRIES: Lazy<UIntGauge> = Lazy::new(|| {
672 10 : register_uint_gauge!(
673 10 : "pageserver_relsize_cache_entries",
674 10 : "Number of entries in the relation size cache",
675 10 : )
676 10 : .expect("failed to define a metric")
677 10 : });
678 :
679 10 : pub(crate) static RELSIZE_CACHE_HITS: Lazy<IntCounter> = Lazy::new(|| {
680 10 : register_int_counter!("pageserver_relsize_cache_hits", "Relation size cache hits",)
681 10 : .expect("failed to define a metric")
682 10 : });
683 :
684 10 : pub(crate) static RELSIZE_CACHE_MISSES: Lazy<IntCounter> = Lazy::new(|| {
685 10 : register_int_counter!(
686 10 : "pageserver_relsize_cache_misses",
687 10 : "Relation size cache misses",
688 10 : )
689 10 : .expect("failed to define a metric")
690 10 : });
691 :
692 4 : pub(crate) static RELSIZE_CACHE_MISSES_OLD: Lazy<IntCounter> = Lazy::new(|| {
693 4 : register_int_counter!(
694 4 : "pageserver_relsize_cache_misses_old",
695 4 : "Relation size cache misses where the lookup LSN is older than the last relation update"
696 4 : )
697 4 : .expect("failed to define a metric")
698 4 : });
699 :
700 : pub(crate) mod initial_logical_size {
701 : use metrics::{register_int_counter, register_int_counter_vec, IntCounter, IntCounterVec};
702 : use once_cell::sync::Lazy;
703 :
704 : pub(crate) struct StartCalculation(IntCounterVec);
705 176 : pub(crate) static START_CALCULATION: Lazy<StartCalculation> = Lazy::new(|| {
706 176 : StartCalculation(
707 176 : register_int_counter_vec!(
708 176 : "pageserver_initial_logical_size_start_calculation",
709 176 : "Incremented each time we start an initial logical size calculation attempt. \
710 176 : The `circumstances` label provides some additional details.",
711 176 : &["attempt", "circumstances"]
712 176 : )
713 176 : .unwrap(),
714 176 : )
715 176 : });
716 :
717 : struct DropCalculation {
718 : first: IntCounter,
719 : retry: IntCounter,
720 : }
721 :
722 176 : static DROP_CALCULATION: Lazy<DropCalculation> = Lazy::new(|| {
723 176 : let vec = register_int_counter_vec!(
724 176 : "pageserver_initial_logical_size_drop_calculation",
725 176 : "Incremented each time we abort a started size calculation attmpt.",
726 176 : &["attempt"]
727 176 : )
728 176 : .unwrap();
729 176 : DropCalculation {
730 176 : first: vec.with_label_values(&["first"]),
731 176 : retry: vec.with_label_values(&["retry"]),
732 176 : }
733 176 : });
734 :
735 : pub(crate) struct Calculated {
736 : pub(crate) births: IntCounter,
737 : pub(crate) deaths: IntCounter,
738 : }
739 :
740 176 : pub(crate) static CALCULATED: Lazy<Calculated> = Lazy::new(|| Calculated {
741 176 : births: register_int_counter!(
742 176 : "pageserver_initial_logical_size_finish_calculation",
743 176 : "Incremented every time we finish calculation of initial logical size.\
744 176 : If everything is working well, this should happen at most once per Timeline object."
745 176 : )
746 176 : .unwrap(),
747 176 : deaths: register_int_counter!(
748 176 : "pageserver_initial_logical_size_drop_finished_calculation",
749 176 : "Incremented when we drop a finished initial logical size calculation result.\
750 176 : Mainly useful to turn pageserver_initial_logical_size_finish_calculation into a gauge."
751 176 : )
752 176 : .unwrap(),
753 176 : });
754 :
755 : pub(crate) struct OngoingCalculationGuard {
756 : inc_drop_calculation: Option<IntCounter>,
757 : }
758 :
759 188 : #[derive(strum_macros::IntoStaticStr)]
760 : pub(crate) enum StartCircumstances {
761 : EmptyInitial,
762 : SkippedConcurrencyLimiter,
763 : AfterBackgroundTasksRateLimit,
764 : }
765 :
766 : impl StartCalculation {
767 188 : pub(crate) fn first(&self, circumstances: StartCircumstances) -> OngoingCalculationGuard {
768 188 : let circumstances_label: &'static str = circumstances.into();
769 188 : self.0
770 188 : .with_label_values(&["first", circumstances_label])
771 188 : .inc();
772 188 : OngoingCalculationGuard {
773 188 : inc_drop_calculation: Some(DROP_CALCULATION.first.clone()),
774 188 : }
775 188 : }
776 0 : pub(crate) fn retry(&self, circumstances: StartCircumstances) -> OngoingCalculationGuard {
777 0 : let circumstances_label: &'static str = circumstances.into();
778 0 : self.0
779 0 : .with_label_values(&["retry", circumstances_label])
780 0 : .inc();
781 0 : OngoingCalculationGuard {
782 0 : inc_drop_calculation: Some(DROP_CALCULATION.retry.clone()),
783 0 : }
784 0 : }
785 : }
786 :
787 : impl Drop for OngoingCalculationGuard {
788 188 : fn drop(&mut self) {
789 188 : if let Some(counter) = self.inc_drop_calculation.take() {
790 0 : counter.inc();
791 188 : }
792 188 : }
793 : }
794 :
795 : impl OngoingCalculationGuard {
796 188 : pub(crate) fn calculation_result_saved(mut self) -> FinishedCalculationGuard {
797 188 : drop(self.inc_drop_calculation.take());
798 188 : CALCULATED.births.inc();
799 188 : FinishedCalculationGuard {
800 188 : inc_on_drop: CALCULATED.deaths.clone(),
801 188 : }
802 188 : }
803 : }
804 :
805 : pub(crate) struct FinishedCalculationGuard {
806 : inc_on_drop: IntCounter,
807 : }
808 :
809 : impl Drop for FinishedCalculationGuard {
810 6 : fn drop(&mut self) {
811 6 : self.inc_on_drop.inc();
812 6 : }
813 : }
814 :
815 : // context: https://github.com/neondatabase/neon/issues/5963
816 : pub(crate) static TIMELINES_WHERE_WALRECEIVER_GOT_APPROXIMATE_SIZE: Lazy<IntCounter> =
817 0 : Lazy::new(|| {
818 0 : register_int_counter!(
819 0 : "pageserver_initial_logical_size_timelines_where_walreceiver_got_approximate_size",
820 0 : "Counter for the following event: walreceiver calls\
821 0 : Timeline::get_current_logical_size() and it returns `Approximate` for the first time."
822 0 : )
823 0 : .unwrap()
824 0 : });
825 : }
826 :
827 0 : static DIRECTORY_ENTRIES_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
828 0 : register_uint_gauge_vec!(
829 0 : "pageserver_directory_entries_count",
830 0 : "Sum of the entries in pageserver-stored directory listings",
831 0 : &["tenant_id", "shard_id", "timeline_id"]
832 0 : )
833 0 : .expect("failed to define a metric")
834 0 : });
835 :
836 178 : pub(crate) static TENANT_STATE_METRIC: Lazy<UIntGaugeVec> = Lazy::new(|| {
837 178 : register_uint_gauge_vec!(
838 178 : "pageserver_tenant_states_count",
839 178 : "Count of tenants per state",
840 178 : &["state"]
841 178 : )
842 178 : .expect("Failed to register pageserver_tenant_states_count metric")
843 178 : });
844 :
845 : /// A set of broken tenants.
846 : ///
847 : /// These are expected to be so rare that a set is fine. Set as in a new timeseries per each broken
848 : /// tenant.
849 10 : pub(crate) static BROKEN_TENANTS_SET: Lazy<UIntGaugeVec> = Lazy::new(|| {
850 10 : register_uint_gauge_vec!(
851 10 : "pageserver_broken_tenants_count",
852 10 : "Set of broken tenants",
853 10 : &["tenant_id", "shard_id"]
854 10 : )
855 10 : .expect("Failed to register pageserver_tenant_states_count metric")
856 10 : });
857 :
858 6 : pub(crate) static TENANT_SYNTHETIC_SIZE_METRIC: Lazy<UIntGaugeVec> = Lazy::new(|| {
859 6 : register_uint_gauge_vec!(
860 6 : "pageserver_tenant_synthetic_cached_size_bytes",
861 6 : "Synthetic size of each tenant in bytes",
862 6 : &["tenant_id"]
863 6 : )
864 6 : .expect("Failed to register pageserver_tenant_synthetic_cached_size_bytes metric")
865 6 : });
866 :
867 0 : pub(crate) static EVICTION_ITERATION_DURATION: Lazy<HistogramVec> = Lazy::new(|| {
868 0 : register_histogram_vec!(
869 0 : "pageserver_eviction_iteration_duration_seconds_global",
870 0 : "Time spent on a single eviction iteration",
871 0 : &["period_secs", "threshold_secs"],
872 0 : STORAGE_OP_BUCKETS.into(),
873 0 : )
874 0 : .expect("failed to define a metric")
875 0 : });
876 :
877 176 : static EVICTIONS: Lazy<IntCounterVec> = Lazy::new(|| {
878 176 : register_int_counter_vec!(
879 176 : "pageserver_evictions",
880 176 : "Number of layers evicted from the pageserver",
881 176 : &["tenant_id", "shard_id", "timeline_id"]
882 176 : )
883 176 : .expect("failed to define a metric")
884 176 : });
885 :
886 176 : static EVICTIONS_WITH_LOW_RESIDENCE_DURATION: Lazy<IntCounterVec> = Lazy::new(|| {
887 176 : register_int_counter_vec!(
888 176 : "pageserver_evictions_with_low_residence_duration",
889 176 : "If a layer is evicted that was resident for less than `low_threshold`, it is counted to this counter. \
890 176 : Residence duration is determined using the `residence_duration_data_source`.",
891 176 : &["tenant_id", "shard_id", "timeline_id", "residence_duration_data_source", "low_threshold_secs"]
892 176 : )
893 176 : .expect("failed to define a metric")
894 176 : });
895 :
896 0 : pub(crate) static UNEXPECTED_ONDEMAND_DOWNLOADS: Lazy<IntCounter> = Lazy::new(|| {
897 0 : register_int_counter!(
898 0 : "pageserver_unexpected_ondemand_downloads_count",
899 0 : "Number of unexpected on-demand downloads. \
900 0 : We log more context for each increment, so, forgo any labels in this metric.",
901 0 : )
902 0 : .expect("failed to define a metric")
903 0 : });
904 :
905 : /// How long did we take to start up? Broken down by labels to describe
906 : /// different phases of startup.
907 0 : pub static STARTUP_DURATION: Lazy<GaugeVec> = Lazy::new(|| {
908 0 : register_gauge_vec!(
909 0 : "pageserver_startup_duration_seconds",
910 0 : "Time taken by phases of pageserver startup, in seconds",
911 0 : &["phase"]
912 0 : )
913 0 : .expect("Failed to register pageserver_startup_duration_seconds metric")
914 0 : });
915 :
916 0 : pub static STARTUP_IS_LOADING: Lazy<UIntGauge> = Lazy::new(|| {
917 0 : register_uint_gauge!(
918 0 : "pageserver_startup_is_loading",
919 0 : "1 while in initial startup load of tenants, 0 at other times"
920 0 : )
921 0 : .expect("Failed to register pageserver_startup_is_loading")
922 0 : });
923 :
924 172 : pub(crate) static TIMELINE_EPHEMERAL_BYTES: Lazy<UIntGauge> = Lazy::new(|| {
925 172 : register_uint_gauge!(
926 172 : "pageserver_timeline_ephemeral_bytes",
927 172 : "Total number of bytes in ephemeral layers, summed for all timelines. Approximate, lazily updated."
928 172 : )
929 172 : .expect("Failed to register metric")
930 172 : });
931 :
932 : /// Metrics related to the lifecycle of a [`crate::tenant::Tenant`] object: things
933 : /// like how long it took to load.
934 : ///
935 : /// Note that these are process-global metrics, _not_ per-tenant metrics. Per-tenant
936 : /// metrics are rather expensive, and usually fine grained stuff makes more sense
937 : /// at a timeline level than tenant level.
938 : pub(crate) struct TenantMetrics {
939 : /// How long did tenants take to go from construction to active state?
940 : pub(crate) activation: Histogram,
941 : pub(crate) preload: Histogram,
942 : pub(crate) attach: Histogram,
943 :
944 : /// How many tenants are included in the initial startup of the pagesrever?
945 : pub(crate) startup_scheduled: IntCounter,
946 : pub(crate) startup_complete: IntCounter,
947 : }
948 :
949 0 : pub(crate) static TENANT: Lazy<TenantMetrics> = Lazy::new(|| {
950 0 : TenantMetrics {
951 0 : activation: register_histogram!(
952 0 : "pageserver_tenant_activation_seconds",
953 0 : "Time taken by tenants to activate, in seconds",
954 0 : CRITICAL_OP_BUCKETS.into()
955 0 : )
956 0 : .expect("Failed to register metric"),
957 0 : preload: register_histogram!(
958 0 : "pageserver_tenant_preload_seconds",
959 0 : "Time taken by tenants to load remote metadata on startup/attach, in seconds",
960 0 : CRITICAL_OP_BUCKETS.into()
961 0 : )
962 0 : .expect("Failed to register metric"),
963 0 : attach: register_histogram!(
964 0 : "pageserver_tenant_attach_seconds",
965 0 : "Time taken by tenants to intialize, after remote metadata is already loaded",
966 0 : CRITICAL_OP_BUCKETS.into()
967 0 : )
968 0 : .expect("Failed to register metric"),
969 0 : startup_scheduled: register_int_counter!(
970 0 : "pageserver_tenant_startup_scheduled",
971 0 : "Number of tenants included in pageserver startup (doesn't count tenants attached later)"
972 0 : ).expect("Failed to register metric"),
973 0 : startup_complete: register_int_counter!(
974 0 : "pageserver_tenant_startup_complete",
975 0 : "Number of tenants that have completed warm-up, or activated on-demand during initial startup: \
976 0 : should eventually reach `pageserver_tenant_startup_scheduled_total`. Does not include broken \
977 0 : tenants: such cases will lead to this metric never reaching the scheduled count."
978 0 : ).expect("Failed to register metric"),
979 0 : }
980 0 : });
981 :
982 : /// Each `Timeline`'s [`EVICTIONS_WITH_LOW_RESIDENCE_DURATION`] metric.
983 : #[derive(Debug)]
984 : pub(crate) struct EvictionsWithLowResidenceDuration {
985 : data_source: &'static str,
986 : threshold: Duration,
987 : counter: Option<IntCounter>,
988 : }
989 :
990 : pub(crate) struct EvictionsWithLowResidenceDurationBuilder {
991 : data_source: &'static str,
992 : threshold: Duration,
993 : }
994 :
995 : impl EvictionsWithLowResidenceDurationBuilder {
996 422 : pub fn new(data_source: &'static str, threshold: Duration) -> Self {
997 422 : Self {
998 422 : data_source,
999 422 : threshold,
1000 422 : }
1001 422 : }
1002 :
1003 422 : fn build(
1004 422 : &self,
1005 422 : tenant_id: &str,
1006 422 : shard_id: &str,
1007 422 : timeline_id: &str,
1008 422 : ) -> EvictionsWithLowResidenceDuration {
1009 422 : let counter = EVICTIONS_WITH_LOW_RESIDENCE_DURATION
1010 422 : .get_metric_with_label_values(&[
1011 422 : tenant_id,
1012 422 : shard_id,
1013 422 : timeline_id,
1014 422 : self.data_source,
1015 422 : &EvictionsWithLowResidenceDuration::threshold_label_value(self.threshold),
1016 422 : ])
1017 422 : .unwrap();
1018 422 : EvictionsWithLowResidenceDuration {
1019 422 : data_source: self.data_source,
1020 422 : threshold: self.threshold,
1021 422 : counter: Some(counter),
1022 422 : }
1023 422 : }
1024 : }
1025 :
1026 : impl EvictionsWithLowResidenceDuration {
1027 432 : fn threshold_label_value(threshold: Duration) -> String {
1028 432 : format!("{}", threshold.as_secs())
1029 432 : }
1030 :
1031 4 : pub fn observe(&self, observed_value: Duration) {
1032 4 : if observed_value < self.threshold {
1033 4 : self.counter
1034 4 : .as_ref()
1035 4 : .expect("nobody calls this function after `remove_from_vec`")
1036 4 : .inc();
1037 4 : }
1038 4 : }
1039 :
1040 0 : pub fn change_threshold(
1041 0 : &mut self,
1042 0 : tenant_id: &str,
1043 0 : shard_id: &str,
1044 0 : timeline_id: &str,
1045 0 : new_threshold: Duration,
1046 0 : ) {
1047 0 : if new_threshold == self.threshold {
1048 0 : return;
1049 0 : }
1050 0 : let mut with_new = EvictionsWithLowResidenceDurationBuilder::new(
1051 0 : self.data_source,
1052 0 : new_threshold,
1053 0 : )
1054 0 : .build(tenant_id, shard_id, timeline_id);
1055 0 : std::mem::swap(self, &mut with_new);
1056 0 : with_new.remove(tenant_id, shard_id, timeline_id);
1057 0 : }
1058 :
1059 : // This could be a `Drop` impl, but, we need the `tenant_id` and `timeline_id`.
1060 10 : fn remove(&mut self, tenant_id: &str, shard_id: &str, timeline_id: &str) {
1061 10 : let Some(_counter) = self.counter.take() else {
1062 0 : return;
1063 : };
1064 :
1065 10 : let threshold = Self::threshold_label_value(self.threshold);
1066 10 :
1067 10 : let removed = EVICTIONS_WITH_LOW_RESIDENCE_DURATION.remove_label_values(&[
1068 10 : tenant_id,
1069 10 : shard_id,
1070 10 : timeline_id,
1071 10 : self.data_source,
1072 10 : &threshold,
1073 10 : ]);
1074 10 :
1075 10 : match removed {
1076 0 : Err(e) => {
1077 0 : // this has been hit in staging as
1078 0 : // <https://neondatabase.sentry.io/issues/4142396994/>, but we don't know how.
1079 0 : // because we can be in the drop path already, don't risk:
1080 0 : // - "double-panic => illegal instruction" or
1081 0 : // - future "drop panick => abort"
1082 0 : //
1083 0 : // so just nag: (the error has the labels)
1084 0 : tracing::warn!("failed to remove EvictionsWithLowResidenceDuration, it was already removed? {e:#?}");
1085 : }
1086 : Ok(()) => {
1087 : // to help identify cases where we double-remove the same values, let's log all
1088 : // deletions?
1089 10 : tracing::info!("removed EvictionsWithLowResidenceDuration with {tenant_id}, {timeline_id}, {}, {threshold}", self.data_source);
1090 : }
1091 : }
1092 10 : }
1093 : }
1094 :
1095 : // Metrics collected on disk IO operations
1096 : //
1097 : // Roughly logarithmic scale.
1098 : const STORAGE_IO_TIME_BUCKETS: &[f64] = &[
1099 : 0.000030, // 30 usec
1100 : 0.001000, // 1000 usec
1101 : 0.030, // 30 ms
1102 : 1.000, // 1000 ms
1103 : 30.000, // 30000 ms
1104 : ];
1105 :
1106 : /// VirtualFile fs operation variants.
1107 : ///
1108 : /// Operations:
1109 : /// - open ([`std::fs::OpenOptions::open`])
1110 : /// - close (dropping [`crate::virtual_file::VirtualFile`])
1111 : /// - close-by-replace (close by replacement algorithm)
1112 : /// - read (`read_at`)
1113 : /// - write (`write_at`)
1114 : /// - seek (modify internal position or file length query)
1115 : /// - fsync ([`std::fs::File::sync_all`])
1116 : /// - metadata ([`std::fs::File::metadata`])
1117 : #[derive(
1118 1854 : Debug, Clone, Copy, strum_macros::EnumCount, strum_macros::EnumIter, strum_macros::FromRepr,
1119 : )]
1120 : pub(crate) enum StorageIoOperation {
1121 : Open,
1122 : OpenAfterReplace,
1123 : Close,
1124 : CloseByReplace,
1125 : Read,
1126 : Write,
1127 : Seek,
1128 : Fsync,
1129 : Metadata,
1130 : }
1131 :
1132 : impl StorageIoOperation {
1133 1854 : pub fn as_str(&self) -> &'static str {
1134 1854 : match self {
1135 206 : StorageIoOperation::Open => "open",
1136 206 : StorageIoOperation::OpenAfterReplace => "open-after-replace",
1137 206 : StorageIoOperation::Close => "close",
1138 206 : StorageIoOperation::CloseByReplace => "close-by-replace",
1139 206 : StorageIoOperation::Read => "read",
1140 206 : StorageIoOperation::Write => "write",
1141 206 : StorageIoOperation::Seek => "seek",
1142 206 : StorageIoOperation::Fsync => "fsync",
1143 206 : StorageIoOperation::Metadata => "metadata",
1144 : }
1145 1854 : }
1146 : }
1147 :
1148 : /// Tracks time taken by fs operations near VirtualFile.
1149 : #[derive(Debug)]
1150 : pub(crate) struct StorageIoTime {
1151 : metrics: [Histogram; StorageIoOperation::COUNT],
1152 : }
1153 :
1154 : impl StorageIoTime {
1155 206 : fn new() -> Self {
1156 206 : let storage_io_histogram_vec = register_histogram_vec!(
1157 206 : "pageserver_io_operations_seconds",
1158 206 : "Time spent in IO operations",
1159 206 : &["operation"],
1160 206 : STORAGE_IO_TIME_BUCKETS.into()
1161 206 : )
1162 206 : .expect("failed to define a metric");
1163 1854 : let metrics = std::array::from_fn(|i| {
1164 1854 : let op = StorageIoOperation::from_repr(i).unwrap();
1165 1854 : storage_io_histogram_vec
1166 1854 : .get_metric_with_label_values(&[op.as_str()])
1167 1854 : .unwrap()
1168 1854 : });
1169 206 : Self { metrics }
1170 206 : }
1171 :
1172 2002180 : pub(crate) fn get(&self, op: StorageIoOperation) -> &Histogram {
1173 2002180 : &self.metrics[op as usize]
1174 2002180 : }
1175 : }
1176 :
1177 : pub(crate) static STORAGE_IO_TIME_METRIC: Lazy<StorageIoTime> = Lazy::new(StorageIoTime::new);
1178 :
1179 : const STORAGE_IO_SIZE_OPERATIONS: &[&str] = &["read", "write"];
1180 :
1181 : // Needed for the https://neonprod.grafana.net/d/5uK9tHL4k/picking-tenant-for-relocation?orgId=1
1182 202 : pub(crate) static STORAGE_IO_SIZE: Lazy<IntGaugeVec> = Lazy::new(|| {
1183 202 : register_int_gauge_vec!(
1184 202 : "pageserver_io_operations_bytes_total",
1185 202 : "Total amount of bytes read/written in IO operations",
1186 202 : &["operation", "tenant_id", "shard_id", "timeline_id"]
1187 202 : )
1188 202 : .expect("failed to define a metric")
1189 202 : });
1190 :
1191 : #[cfg(not(test))]
1192 : pub(crate) mod virtual_file_descriptor_cache {
1193 : use super::*;
1194 :
1195 0 : pub(crate) static SIZE_MAX: Lazy<UIntGauge> = Lazy::new(|| {
1196 0 : register_uint_gauge!(
1197 0 : "pageserver_virtual_file_descriptor_cache_size_max",
1198 0 : "Maximum number of open file descriptors in the cache."
1199 0 : )
1200 0 : .unwrap()
1201 0 : });
1202 :
1203 : // SIZE_CURRENT: derive it like so:
1204 : // ```
1205 : // sum (pageserver_io_operations_seconds_count{operation=~"^(open|open-after-replace)$")
1206 : // -ignoring(operation)
1207 : // sum(pageserver_io_operations_seconds_count{operation=~"^(close|close-by-replace)$"}
1208 : // ```
1209 : }
1210 :
1211 : #[cfg(not(test))]
1212 : pub(crate) mod virtual_file_io_engine {
1213 : use super::*;
1214 :
1215 0 : pub(crate) static KIND: Lazy<UIntGaugeVec> = Lazy::new(|| {
1216 0 : register_uint_gauge_vec!(
1217 0 : "pageserver_virtual_file_io_engine_kind",
1218 0 : "The configured io engine for VirtualFile",
1219 0 : &["kind"],
1220 0 : )
1221 0 : .unwrap()
1222 0 : });
1223 : }
1224 :
1225 : pub(crate) struct SmgrOpTimer(Option<SmgrOpTimerInner>);
1226 : pub(crate) struct SmgrOpTimerInner {
1227 : global_execution_latency_histo: Histogram,
1228 : per_timeline_execution_latency_histo: Option<Histogram>,
1229 :
1230 : global_batch_wait_time: Histogram,
1231 : per_timeline_batch_wait_time: Histogram,
1232 :
1233 : global_flush_in_progress_micros: IntCounter,
1234 : per_timeline_flush_in_progress_micros: IntCounter,
1235 :
1236 : timings: SmgrOpTimerState,
1237 : }
1238 :
1239 : #[derive(Debug)]
1240 : enum SmgrOpTimerState {
1241 : Received {
1242 : received_at: Instant,
1243 : },
1244 : ThrottleDoneExecutionStarting {
1245 : received_at: Instant,
1246 : throttle_started_at: Instant,
1247 : started_execution_at: Instant,
1248 : },
1249 : }
1250 :
1251 : pub(crate) struct SmgrOpFlushInProgress {
1252 : flush_started_at: Instant,
1253 : global_micros: IntCounter,
1254 : per_timeline_micros: IntCounter,
1255 : }
1256 :
1257 : impl SmgrOpTimer {
1258 0 : pub(crate) fn observe_throttle_done_execution_starting(&mut self, throttle: &ThrottleResult) {
1259 0 : let inner = self.0.as_mut().expect("other public methods consume self");
1260 0 : match (&mut inner.timings, throttle) {
1261 0 : (SmgrOpTimerState::Received { received_at }, throttle) => match throttle {
1262 0 : ThrottleResult::NotThrottled { start } => {
1263 0 : inner.timings = SmgrOpTimerState::ThrottleDoneExecutionStarting {
1264 0 : received_at: *received_at,
1265 0 : throttle_started_at: *start,
1266 0 : started_execution_at: *start,
1267 0 : };
1268 0 : }
1269 0 : ThrottleResult::Throttled { start, end } => {
1270 0 : inner.timings = SmgrOpTimerState::ThrottleDoneExecutionStarting {
1271 0 : received_at: *start,
1272 0 : throttle_started_at: *start,
1273 0 : started_execution_at: *end,
1274 0 : };
1275 0 : }
1276 : },
1277 0 : (x, _) => panic!("called in unexpected state: {x:?}"),
1278 : }
1279 0 : }
1280 :
1281 0 : pub(crate) fn observe_smgr_op_completion_and_start_flushing(mut self) -> SmgrOpFlushInProgress {
1282 0 : let (flush_start, inner) = self
1283 0 : .smgr_op_end()
1284 0 : .expect("this method consume self, and the only other caller is drop handler");
1285 0 : let SmgrOpTimerInner {
1286 0 : global_flush_in_progress_micros,
1287 0 : per_timeline_flush_in_progress_micros,
1288 0 : ..
1289 0 : } = inner;
1290 0 : SmgrOpFlushInProgress {
1291 0 : flush_started_at: flush_start,
1292 0 : global_micros: global_flush_in_progress_micros,
1293 0 : per_timeline_micros: per_timeline_flush_in_progress_micros,
1294 0 : }
1295 0 : }
1296 :
1297 : /// Returns `None`` if this method has already been called, `Some` otherwise.
1298 10 : fn smgr_op_end(&mut self) -> Option<(Instant, SmgrOpTimerInner)> {
1299 10 : let inner = self.0.take()?;
1300 :
1301 10 : let now = Instant::now();
1302 10 :
1303 10 : let batch;
1304 10 : let execution;
1305 10 : let throttle;
1306 10 : match inner.timings {
1307 10 : SmgrOpTimerState::Received { received_at } => {
1308 10 : batch = (now - received_at).as_secs_f64();
1309 10 : // TODO: use label for dropped requests.
1310 10 : // This is quite rare in practice, only during tenant/pageservers shutdown.
1311 10 : throttle = Duration::ZERO;
1312 10 : execution = Duration::ZERO.as_secs_f64();
1313 10 : }
1314 : SmgrOpTimerState::ThrottleDoneExecutionStarting {
1315 0 : received_at,
1316 0 : throttle_started_at,
1317 0 : started_execution_at,
1318 0 : } => {
1319 0 : batch = (throttle_started_at - received_at).as_secs_f64();
1320 0 : throttle = started_execution_at - throttle_started_at;
1321 0 : execution = (now - started_execution_at).as_secs_f64();
1322 0 : }
1323 : }
1324 :
1325 : // update time spent in batching
1326 10 : inner.global_batch_wait_time.observe(batch);
1327 10 : inner.per_timeline_batch_wait_time.observe(batch);
1328 10 :
1329 10 : // time spent in throttle metric is updated by throttle impl
1330 10 : let _ = throttle;
1331 10 :
1332 10 : // update metrics for execution latency
1333 10 : inner.global_execution_latency_histo.observe(execution);
1334 2 : if let Some(per_timeline_execution_latency_histo) =
1335 10 : &inner.per_timeline_execution_latency_histo
1336 2 : {
1337 2 : per_timeline_execution_latency_histo.observe(execution);
1338 8 : }
1339 :
1340 10 : Some((now, inner))
1341 10 : }
1342 : }
1343 :
1344 : impl Drop for SmgrOpTimer {
1345 10 : fn drop(&mut self) {
1346 10 : self.smgr_op_end();
1347 10 : }
1348 : }
1349 :
1350 : impl SmgrOpFlushInProgress {
1351 0 : pub(crate) async fn measure<Fut, O>(mut self, mut fut: Fut) -> O
1352 0 : where
1353 0 : Fut: std::future::Future<Output = O>,
1354 0 : {
1355 0 : let mut fut = std::pin::pin!(fut);
1356 0 :
1357 0 : let now = Instant::now();
1358 0 : // Whenever observe_guard gets called, or dropped,
1359 0 : // it adds the time elapsed since its last call to metrics.
1360 0 : // Last call is tracked in `now`.
1361 0 : let mut observe_guard = scopeguard::guard(
1362 0 : || {
1363 0 : let elapsed = now - self.flush_started_at;
1364 0 : self.global_micros
1365 0 : .inc_by(u64::try_from(elapsed.as_micros()).unwrap());
1366 0 : self.per_timeline_micros
1367 0 : .inc_by(u64::try_from(elapsed.as_micros()).unwrap());
1368 0 : self.flush_started_at = now;
1369 0 : },
1370 0 : |mut observe| {
1371 0 : observe();
1372 0 : },
1373 0 : );
1374 :
1375 : loop {
1376 0 : match tokio::time::timeout(Duration::from_secs(10), &mut fut).await {
1377 0 : Ok(v) => return v,
1378 0 : Err(_timeout) => {
1379 0 : (*observe_guard)();
1380 0 : }
1381 : }
1382 : }
1383 0 : }
1384 : }
1385 :
1386 : #[derive(
1387 : Debug,
1388 : Clone,
1389 : Copy,
1390 5214 : IntoStaticStr,
1391 : strum_macros::EnumCount,
1392 24 : strum_macros::EnumIter,
1393 4320 : strum_macros::FromRepr,
1394 : enum_map::Enum,
1395 : )]
1396 : #[strum(serialize_all = "snake_case")]
1397 : pub enum SmgrQueryType {
1398 : GetRelExists,
1399 : GetRelSize,
1400 : GetPageAtLsn,
1401 : GetDbSize,
1402 : GetSlruSegment,
1403 : }
1404 :
1405 : #[derive(Debug)]
1406 : pub(crate) struct SmgrQueryTimePerTimeline {
1407 : global_started: [IntCounter; SmgrQueryType::COUNT],
1408 : global_latency: [Histogram; SmgrQueryType::COUNT],
1409 : per_timeline_getpage_started: IntCounter,
1410 : per_timeline_getpage_latency: Histogram,
1411 : global_batch_size: Histogram,
1412 : per_timeline_batch_size: Histogram,
1413 : global_flush_in_progress_micros: IntCounter,
1414 : per_timeline_flush_in_progress_micros: IntCounter,
1415 : global_batch_wait_time: Histogram,
1416 : per_timeline_batch_wait_time: Histogram,
1417 : }
1418 :
1419 178 : static SMGR_QUERY_STARTED_GLOBAL: Lazy<IntCounterVec> = Lazy::new(|| {
1420 178 : register_int_counter_vec!(
1421 178 : // it's a counter, but, name is prepared to extend it to a histogram of queue depth
1422 178 : "pageserver_smgr_query_started_global_count",
1423 178 : "Number of smgr queries started, aggregated by query type.",
1424 178 : &["smgr_query_type"],
1425 178 : )
1426 178 : .expect("failed to define a metric")
1427 178 : });
1428 :
1429 178 : static SMGR_QUERY_STARTED_PER_TENANT_TIMELINE: Lazy<IntCounterVec> = Lazy::new(|| {
1430 178 : register_int_counter_vec!(
1431 178 : // it's a counter, but, name is prepared to extend it to a histogram of queue depth
1432 178 : "pageserver_smgr_query_started_count",
1433 178 : "Number of smgr queries started, aggregated by query type and tenant/timeline.",
1434 178 : &["smgr_query_type", "tenant_id", "shard_id", "timeline_id"],
1435 178 : )
1436 178 : .expect("failed to define a metric")
1437 178 : });
1438 :
1439 : // Alias so all histograms recording per-timeline smgr timings use the same buckets.
1440 : static SMGR_QUERY_TIME_PER_TENANT_TIMELINE_BUCKETS: &[f64] = CRITICAL_OP_BUCKETS;
1441 :
1442 178 : static SMGR_QUERY_TIME_PER_TENANT_TIMELINE: Lazy<HistogramVec> = Lazy::new(|| {
1443 178 : register_histogram_vec!(
1444 178 : "pageserver_smgr_query_seconds",
1445 178 : "Time spent _executing_ smgr query handling, excluding batch and throttle delays.",
1446 178 : &["smgr_query_type", "tenant_id", "shard_id", "timeline_id"],
1447 178 : SMGR_QUERY_TIME_PER_TENANT_TIMELINE_BUCKETS.into(),
1448 178 : )
1449 178 : .expect("failed to define a metric")
1450 178 : });
1451 :
1452 178 : static SMGR_QUERY_TIME_GLOBAL_BUCKETS: Lazy<Vec<f64>> = Lazy::new(|| {
1453 178 : [
1454 178 : 1,
1455 178 : 10,
1456 178 : 20,
1457 178 : 40,
1458 178 : 60,
1459 178 : 80,
1460 178 : 100,
1461 178 : 200,
1462 178 : 300,
1463 178 : 400,
1464 178 : 500,
1465 178 : 600,
1466 178 : 700,
1467 178 : 800,
1468 178 : 900,
1469 178 : 1_000, // 1ms
1470 178 : 2_000,
1471 178 : 4_000,
1472 178 : 6_000,
1473 178 : 8_000,
1474 178 : 10_000, // 10ms
1475 178 : 20_000,
1476 178 : 40_000,
1477 178 : 60_000,
1478 178 : 80_000,
1479 178 : 100_000,
1480 178 : 200_000,
1481 178 : 400_000,
1482 178 : 600_000,
1483 178 : 800_000,
1484 178 : 1_000_000, // 1s
1485 178 : 2_000_000,
1486 178 : 4_000_000,
1487 178 : 6_000_000,
1488 178 : 8_000_000,
1489 178 : 10_000_000, // 10s
1490 178 : 20_000_000,
1491 178 : 50_000_000,
1492 178 : 100_000_000,
1493 178 : 200_000_000,
1494 178 : 1_000_000_000, // 1000s
1495 178 : ]
1496 178 : .into_iter()
1497 178 : .map(Duration::from_micros)
1498 7298 : .map(|d| d.as_secs_f64())
1499 178 : .collect()
1500 178 : });
1501 :
1502 178 : static SMGR_QUERY_TIME_GLOBAL: Lazy<HistogramVec> = Lazy::new(|| {
1503 178 : register_histogram_vec!(
1504 178 : "pageserver_smgr_query_seconds_global",
1505 178 : "Like pageserver_smgr_query_seconds, but aggregated to instance level.",
1506 178 : &["smgr_query_type"],
1507 178 : SMGR_QUERY_TIME_GLOBAL_BUCKETS.clone(),
1508 178 : )
1509 178 : .expect("failed to define a metric")
1510 178 : });
1511 :
1512 178 : static PAGE_SERVICE_BATCH_SIZE_BUCKETS_GLOBAL: Lazy<Vec<f64>> = Lazy::new(|| {
1513 178 : (1..=u32::try_from(Timeline::MAX_GET_VECTORED_KEYS).unwrap())
1514 5696 : .map(|v| v.into())
1515 178 : .collect()
1516 178 : });
1517 :
1518 178 : static PAGE_SERVICE_BATCH_SIZE_GLOBAL: Lazy<Histogram> = Lazy::new(|| {
1519 178 : register_histogram!(
1520 178 : "pageserver_page_service_batch_size_global",
1521 178 : "Batch size of pageserver page service requests",
1522 178 : PAGE_SERVICE_BATCH_SIZE_BUCKETS_GLOBAL.clone(),
1523 178 : )
1524 178 : .expect("failed to define a metric")
1525 178 : });
1526 :
1527 178 : static PAGE_SERVICE_BATCH_SIZE_BUCKETS_PER_TIMELINE: Lazy<Vec<f64>> = Lazy::new(|| {
1528 178 : let mut buckets = Vec::new();
1529 1246 : for i in 0.. {
1530 1246 : let bucket = 1 << i;
1531 1246 : if bucket > u32::try_from(Timeline::MAX_GET_VECTORED_KEYS).unwrap() {
1532 178 : break;
1533 1068 : }
1534 1068 : buckets.push(bucket.into());
1535 : }
1536 178 : buckets
1537 178 : });
1538 :
1539 178 : static PAGE_SERVICE_BATCH_SIZE_PER_TENANT_TIMELINE: Lazy<HistogramVec> = Lazy::new(|| {
1540 178 : register_histogram_vec!(
1541 178 : "pageserver_page_service_batch_size",
1542 178 : "Batch size of pageserver page service requests",
1543 178 : &["tenant_id", "shard_id", "timeline_id"],
1544 178 : PAGE_SERVICE_BATCH_SIZE_BUCKETS_PER_TIMELINE.clone()
1545 178 : )
1546 178 : .expect("failed to define a metric")
1547 178 : });
1548 :
1549 0 : pub(crate) static PAGE_SERVICE_CONFIG_MAX_BATCH_SIZE: Lazy<IntGaugeVec> = Lazy::new(|| {
1550 0 : register_int_gauge_vec!(
1551 0 : "pageserver_page_service_config_max_batch_size",
1552 0 : "Configured maximum batch size for the server-side batching functionality of page_service. \
1553 0 : Labels expose more of the configuration parameters.",
1554 0 : &["mode", "execution"]
1555 0 : )
1556 0 : .expect("failed to define a metric")
1557 0 : });
1558 :
1559 0 : fn set_page_service_config_max_batch_size(conf: &PageServicePipeliningConfig) {
1560 0 : PAGE_SERVICE_CONFIG_MAX_BATCH_SIZE.reset();
1561 0 : let (label_values, value) = match conf {
1562 0 : PageServicePipeliningConfig::Serial => (["serial", "-"], 1),
1563 : PageServicePipeliningConfig::Pipelined(PageServicePipeliningConfigPipelined {
1564 0 : max_batch_size,
1565 0 : execution,
1566 0 : }) => {
1567 0 : let mode = "pipelined";
1568 0 : let execution = match execution {
1569 : PageServiceProtocolPipelinedExecutionStrategy::ConcurrentFutures => {
1570 0 : "concurrent-futures"
1571 : }
1572 0 : PageServiceProtocolPipelinedExecutionStrategy::Tasks => "tasks",
1573 : };
1574 0 : ([mode, execution], max_batch_size.get())
1575 : }
1576 : };
1577 0 : PAGE_SERVICE_CONFIG_MAX_BATCH_SIZE
1578 0 : .with_label_values(&label_values)
1579 0 : .set(value.try_into().unwrap());
1580 0 : }
1581 :
1582 178 : static PAGE_SERVICE_SMGR_FLUSH_INPROGRESS_MICROS: Lazy<IntCounterVec> = Lazy::new(|| {
1583 178 : register_int_counter_vec!(
1584 178 : "pageserver_page_service_pagestream_flush_in_progress_micros",
1585 178 : "Counter that sums up the microseconds that a pagestream response was being flushed into the TCP connection. \
1586 178 : If the flush is particularly slow, this counter will be updated periodically to make slow flushes \
1587 178 : easily discoverable in monitoring. \
1588 178 : Hence, this is NOT a completion latency historgram.",
1589 178 : &["tenant_id", "shard_id", "timeline_id"],
1590 178 : )
1591 178 : .expect("failed to define a metric")
1592 178 : });
1593 :
1594 178 : static PAGE_SERVICE_SMGR_FLUSH_INPROGRESS_MICROS_GLOBAL: Lazy<IntCounter> = Lazy::new(|| {
1595 178 : register_int_counter!(
1596 178 : "pageserver_page_service_pagestream_flush_in_progress_micros_global",
1597 178 : "Like pageserver_page_service_pagestream_flush_in_progress_seconds, but instance-wide.",
1598 178 : )
1599 178 : .expect("failed to define a metric")
1600 178 : });
1601 :
1602 178 : static PAGE_SERVICE_SMGR_BATCH_WAIT_TIME: Lazy<HistogramVec> = Lazy::new(|| {
1603 178 : register_histogram_vec!(
1604 178 : "pageserver_page_service_pagestream_batch_wait_time_seconds",
1605 178 : "Time a request spent waiting in its batch until the batch moved to throttle&execution.",
1606 178 : &["tenant_id", "shard_id", "timeline_id"],
1607 178 : SMGR_QUERY_TIME_PER_TENANT_TIMELINE_BUCKETS.into(),
1608 178 : )
1609 178 : .expect("failed to define a metric")
1610 178 : });
1611 :
1612 178 : static PAGE_SERVICE_SMGR_BATCH_WAIT_TIME_GLOBAL: Lazy<Histogram> = Lazy::new(|| {
1613 178 : register_histogram!(
1614 178 : "pageserver_page_service_pagestream_batch_wait_time_seconds_global",
1615 178 : "Like pageserver_page_service_pagestream_batch_wait_time_seconds, but aggregated to instance level.",
1616 178 : SMGR_QUERY_TIME_GLOBAL_BUCKETS.to_vec(),
1617 178 : )
1618 178 : .expect("failed to define a metric")
1619 178 : });
1620 :
1621 : impl SmgrQueryTimePerTimeline {
1622 432 : pub(crate) fn new(tenant_shard_id: &TenantShardId, timeline_id: &TimelineId) -> Self {
1623 432 : let tenant_id = tenant_shard_id.tenant_id.to_string();
1624 432 : let shard_slug = format!("{}", tenant_shard_id.shard_slug());
1625 432 : let timeline_id = timeline_id.to_string();
1626 2160 : let global_started = std::array::from_fn(|i| {
1627 2160 : let op = SmgrQueryType::from_repr(i).unwrap();
1628 2160 : SMGR_QUERY_STARTED_GLOBAL
1629 2160 : .get_metric_with_label_values(&[op.into()])
1630 2160 : .unwrap()
1631 2160 : });
1632 2160 : let global_latency = std::array::from_fn(|i| {
1633 2160 : let op = SmgrQueryType::from_repr(i).unwrap();
1634 2160 : SMGR_QUERY_TIME_GLOBAL
1635 2160 : .get_metric_with_label_values(&[op.into()])
1636 2160 : .unwrap()
1637 2160 : });
1638 432 :
1639 432 : let per_timeline_getpage_started = SMGR_QUERY_STARTED_PER_TENANT_TIMELINE
1640 432 : .get_metric_with_label_values(&[
1641 432 : SmgrQueryType::GetPageAtLsn.into(),
1642 432 : &tenant_id,
1643 432 : &shard_slug,
1644 432 : &timeline_id,
1645 432 : ])
1646 432 : .unwrap();
1647 432 : let per_timeline_getpage_latency = SMGR_QUERY_TIME_PER_TENANT_TIMELINE
1648 432 : .get_metric_with_label_values(&[
1649 432 : SmgrQueryType::GetPageAtLsn.into(),
1650 432 : &tenant_id,
1651 432 : &shard_slug,
1652 432 : &timeline_id,
1653 432 : ])
1654 432 : .unwrap();
1655 432 :
1656 432 : let global_batch_size = PAGE_SERVICE_BATCH_SIZE_GLOBAL.clone();
1657 432 : let per_timeline_batch_size = PAGE_SERVICE_BATCH_SIZE_PER_TENANT_TIMELINE
1658 432 : .get_metric_with_label_values(&[&tenant_id, &shard_slug, &timeline_id])
1659 432 : .unwrap();
1660 432 :
1661 432 : let global_batch_wait_time = PAGE_SERVICE_SMGR_BATCH_WAIT_TIME_GLOBAL.clone();
1662 432 : let per_timeline_batch_wait_time = PAGE_SERVICE_SMGR_BATCH_WAIT_TIME
1663 432 : .get_metric_with_label_values(&[&tenant_id, &shard_slug, &timeline_id])
1664 432 : .unwrap();
1665 432 :
1666 432 : let global_flush_in_progress_micros =
1667 432 : PAGE_SERVICE_SMGR_FLUSH_INPROGRESS_MICROS_GLOBAL.clone();
1668 432 : let per_timeline_flush_in_progress_micros = PAGE_SERVICE_SMGR_FLUSH_INPROGRESS_MICROS
1669 432 : .get_metric_with_label_values(&[&tenant_id, &shard_slug, &timeline_id])
1670 432 : .unwrap();
1671 432 :
1672 432 : Self {
1673 432 : global_started,
1674 432 : global_latency,
1675 432 : per_timeline_getpage_latency,
1676 432 : per_timeline_getpage_started,
1677 432 : global_batch_size,
1678 432 : per_timeline_batch_size,
1679 432 : global_flush_in_progress_micros,
1680 432 : per_timeline_flush_in_progress_micros,
1681 432 : global_batch_wait_time,
1682 432 : per_timeline_batch_wait_time,
1683 432 : }
1684 432 : }
1685 10 : pub(crate) fn start_smgr_op(&self, op: SmgrQueryType, received_at: Instant) -> SmgrOpTimer {
1686 10 : self.global_started[op as usize].inc();
1687 :
1688 10 : let per_timeline_latency_histo = if matches!(op, SmgrQueryType::GetPageAtLsn) {
1689 2 : self.per_timeline_getpage_started.inc();
1690 2 : Some(self.per_timeline_getpage_latency.clone())
1691 : } else {
1692 8 : None
1693 : };
1694 :
1695 10 : SmgrOpTimer(Some(SmgrOpTimerInner {
1696 10 : global_execution_latency_histo: self.global_latency[op as usize].clone(),
1697 10 : per_timeline_execution_latency_histo: per_timeline_latency_histo,
1698 10 : timings: SmgrOpTimerState::Received { received_at },
1699 10 : global_flush_in_progress_micros: self.global_flush_in_progress_micros.clone(),
1700 10 : per_timeline_flush_in_progress_micros: self
1701 10 : .per_timeline_flush_in_progress_micros
1702 10 : .clone(),
1703 10 : global_batch_wait_time: self.global_batch_wait_time.clone(),
1704 10 : per_timeline_batch_wait_time: self.per_timeline_batch_wait_time.clone(),
1705 10 : }))
1706 10 : }
1707 :
1708 0 : pub(crate) fn observe_getpage_batch_start(&self, batch_size: usize) {
1709 0 : self.global_batch_size.observe(batch_size as f64);
1710 0 : self.per_timeline_batch_size.observe(batch_size as f64);
1711 0 : }
1712 : }
1713 :
1714 : #[cfg(test)]
1715 : mod smgr_query_time_tests {
1716 : use std::time::Instant;
1717 :
1718 : use pageserver_api::shard::TenantShardId;
1719 : use strum::IntoEnumIterator;
1720 : use utils::id::{TenantId, TimelineId};
1721 :
1722 : // Regression test, we used hard-coded string constants before using an enum.
1723 : #[test]
1724 2 : fn op_label_name() {
1725 : use super::SmgrQueryType::*;
1726 2 : let expect: [(super::SmgrQueryType, &'static str); 5] = [
1727 2 : (GetRelExists, "get_rel_exists"),
1728 2 : (GetRelSize, "get_rel_size"),
1729 2 : (GetPageAtLsn, "get_page_at_lsn"),
1730 2 : (GetDbSize, "get_db_size"),
1731 2 : (GetSlruSegment, "get_slru_segment"),
1732 2 : ];
1733 12 : for (op, expect) in expect {
1734 10 : let actual: &'static str = op.into();
1735 10 : assert_eq!(actual, expect);
1736 : }
1737 2 : }
1738 :
1739 : #[test]
1740 2 : fn basic() {
1741 2 : let ops: Vec<_> = super::SmgrQueryType::iter().collect();
1742 :
1743 12 : for op in &ops {
1744 10 : let tenant_id = TenantId::generate();
1745 10 : let timeline_id = TimelineId::generate();
1746 10 : let metrics = super::SmgrQueryTimePerTimeline::new(
1747 10 : &TenantShardId::unsharded(tenant_id),
1748 10 : &timeline_id,
1749 10 : );
1750 10 :
1751 20 : let get_counts = || {
1752 20 : let global: u64 = ops
1753 20 : .iter()
1754 100 : .map(|op| metrics.global_latency[*op as usize].get_sample_count())
1755 20 : .sum();
1756 20 : (
1757 20 : global,
1758 20 : metrics.per_timeline_getpage_latency.get_sample_count(),
1759 20 : )
1760 20 : };
1761 :
1762 10 : let (pre_global, pre_per_tenant_timeline) = get_counts();
1763 10 : assert_eq!(pre_per_tenant_timeline, 0);
1764 :
1765 10 : let timer = metrics.start_smgr_op(*op, Instant::now());
1766 10 : drop(timer);
1767 10 :
1768 10 : let (post_global, post_per_tenant_timeline) = get_counts();
1769 10 : if matches!(op, super::SmgrQueryType::GetPageAtLsn) {
1770 : // getpage ops are tracked per-timeline, others aren't
1771 2 : assert_eq!(post_per_tenant_timeline, 1);
1772 : } else {
1773 8 : assert_eq!(post_per_tenant_timeline, 0);
1774 : }
1775 10 : assert!(post_global > pre_global);
1776 : }
1777 2 : }
1778 : }
1779 :
1780 : // keep in sync with control plane Go code so that we can validate
1781 : // compute's basebackup_ms metric with our perspective in the context of SLI/SLO.
1782 0 : static COMPUTE_STARTUP_BUCKETS: Lazy<[f64; 28]> = Lazy::new(|| {
1783 0 : // Go code uses milliseconds. Variable is called `computeStartupBuckets`
1784 0 : [
1785 0 : 5, 10, 20, 30, 50, 70, 100, 120, 150, 200, 250, 300, 350, 400, 450, 500, 600, 800, 1000,
1786 0 : 1500, 2000, 2500, 3000, 5000, 10000, 20000, 40000, 60000,
1787 0 : ]
1788 0 : .map(|ms| (ms as f64) / 1000.0)
1789 0 : });
1790 :
1791 : pub(crate) struct BasebackupQueryTime {
1792 : ok: Histogram,
1793 : error: Histogram,
1794 : client_error: Histogram,
1795 : }
1796 :
1797 0 : pub(crate) static BASEBACKUP_QUERY_TIME: Lazy<BasebackupQueryTime> = Lazy::new(|| {
1798 0 : let vec = register_histogram_vec!(
1799 0 : "pageserver_basebackup_query_seconds",
1800 0 : "Histogram of basebackup queries durations, by result type",
1801 0 : &["result"],
1802 0 : COMPUTE_STARTUP_BUCKETS.to_vec(),
1803 0 : )
1804 0 : .expect("failed to define a metric");
1805 0 : BasebackupQueryTime {
1806 0 : ok: vec.get_metric_with_label_values(&["ok"]).unwrap(),
1807 0 : error: vec.get_metric_with_label_values(&["error"]).unwrap(),
1808 0 : client_error: vec.get_metric_with_label_values(&["client_error"]).unwrap(),
1809 0 : }
1810 0 : });
1811 :
1812 : pub(crate) struct BasebackupQueryTimeOngoingRecording<'a> {
1813 : parent: &'a BasebackupQueryTime,
1814 : start: std::time::Instant,
1815 : }
1816 :
1817 : impl BasebackupQueryTime {
1818 0 : pub(crate) fn start_recording(&self) -> BasebackupQueryTimeOngoingRecording<'_> {
1819 0 : let start = Instant::now();
1820 0 : BasebackupQueryTimeOngoingRecording {
1821 0 : parent: self,
1822 0 : start,
1823 0 : }
1824 0 : }
1825 : }
1826 :
1827 : impl BasebackupQueryTimeOngoingRecording<'_> {
1828 0 : pub(crate) fn observe<T>(self, res: &Result<T, QueryError>) {
1829 0 : let elapsed = self.start.elapsed().as_secs_f64();
1830 : // If you want to change categorize of a specific error, also change it in `log_query_error`.
1831 0 : let metric = match res {
1832 0 : Ok(_) => &self.parent.ok,
1833 0 : Err(QueryError::Disconnected(ConnectionError::Io(io_error)))
1834 0 : if is_expected_io_error(io_error) =>
1835 0 : {
1836 0 : &self.parent.client_error
1837 : }
1838 0 : Err(_) => &self.parent.error,
1839 : };
1840 0 : metric.observe(elapsed);
1841 0 : }
1842 : }
1843 :
1844 0 : pub(crate) static LIVE_CONNECTIONS: Lazy<IntCounterPairVec> = Lazy::new(|| {
1845 0 : register_int_counter_pair_vec!(
1846 0 : "pageserver_live_connections_started",
1847 0 : "Number of network connections that we started handling",
1848 0 : "pageserver_live_connections_finished",
1849 0 : "Number of network connections that we finished handling",
1850 0 : &["pageserver_connection_kind"]
1851 0 : )
1852 0 : .expect("failed to define a metric")
1853 0 : });
1854 :
1855 0 : #[derive(Clone, Copy, enum_map::Enum, IntoStaticStr)]
1856 : pub(crate) enum ComputeCommandKind {
1857 : PageStreamV2,
1858 : Basebackup,
1859 : Fullbackup,
1860 : LeaseLsn,
1861 : }
1862 :
1863 : pub(crate) struct ComputeCommandCounters {
1864 : map: EnumMap<ComputeCommandKind, IntCounter>,
1865 : }
1866 :
1867 0 : pub(crate) static COMPUTE_COMMANDS_COUNTERS: Lazy<ComputeCommandCounters> = Lazy::new(|| {
1868 0 : let inner = register_int_counter_vec!(
1869 0 : "pageserver_compute_commands",
1870 0 : "Number of compute -> pageserver commands processed",
1871 0 : &["command"]
1872 0 : )
1873 0 : .expect("failed to define a metric");
1874 0 :
1875 0 : ComputeCommandCounters {
1876 0 : map: EnumMap::from_array(std::array::from_fn(|i| {
1877 0 : let command = <ComputeCommandKind as enum_map::Enum>::from_usize(i);
1878 0 : let command_str: &'static str = command.into();
1879 0 : inner.with_label_values(&[command_str])
1880 0 : })),
1881 0 : }
1882 0 : });
1883 :
1884 : impl ComputeCommandCounters {
1885 0 : pub(crate) fn for_command(&self, command: ComputeCommandKind) -> &IntCounter {
1886 0 : &self.map[command]
1887 0 : }
1888 : }
1889 :
1890 : // remote storage metrics
1891 :
1892 172 : static REMOTE_TIMELINE_CLIENT_CALLS: Lazy<IntCounterPairVec> = Lazy::new(|| {
1893 172 : register_int_counter_pair_vec!(
1894 172 : "pageserver_remote_timeline_client_calls_started",
1895 172 : "Number of started calls to remote timeline client.",
1896 172 : "pageserver_remote_timeline_client_calls_finished",
1897 172 : "Number of finshed calls to remote timeline client.",
1898 172 : &[
1899 172 : "tenant_id",
1900 172 : "shard_id",
1901 172 : "timeline_id",
1902 172 : "file_kind",
1903 172 : "op_kind"
1904 172 : ],
1905 172 : )
1906 172 : .unwrap()
1907 172 : });
1908 :
1909 : static REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER: Lazy<IntCounterVec> =
1910 170 : Lazy::new(|| {
1911 170 : register_int_counter_vec!(
1912 170 : "pageserver_remote_timeline_client_bytes_started",
1913 170 : "Incremented by the number of bytes associated with a remote timeline client operation. \
1914 170 : The increment happens when the operation is scheduled.",
1915 170 : &["tenant_id", "shard_id", "timeline_id", "file_kind", "op_kind"],
1916 170 : )
1917 170 : .expect("failed to define a metric")
1918 170 : });
1919 :
1920 170 : static REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER: Lazy<IntCounterVec> = Lazy::new(|| {
1921 170 : register_int_counter_vec!(
1922 170 : "pageserver_remote_timeline_client_bytes_finished",
1923 170 : "Incremented by the number of bytes associated with a remote timeline client operation. \
1924 170 : The increment happens when the operation finishes (regardless of success/failure/shutdown).",
1925 170 : &["tenant_id", "shard_id", "timeline_id", "file_kind", "op_kind"],
1926 170 : )
1927 170 : .expect("failed to define a metric")
1928 170 : });
1929 :
1930 : pub(crate) struct TenantManagerMetrics {
1931 : tenant_slots_attached: UIntGauge,
1932 : tenant_slots_secondary: UIntGauge,
1933 : tenant_slots_inprogress: UIntGauge,
1934 : pub(crate) tenant_slot_writes: IntCounter,
1935 : pub(crate) unexpected_errors: IntCounter,
1936 : }
1937 :
1938 : impl TenantManagerMetrics {
1939 : /// Helpers for tracking slots. Note that these do not track the lifetime of TenantSlot objects
1940 : /// exactly: they track the lifetime of the slots _in the tenant map_.
1941 2 : pub(crate) fn slot_inserted(&self, slot: &TenantSlot) {
1942 2 : match slot {
1943 0 : TenantSlot::Attached(_) => {
1944 0 : self.tenant_slots_attached.inc();
1945 0 : }
1946 0 : TenantSlot::Secondary(_) => {
1947 0 : self.tenant_slots_secondary.inc();
1948 0 : }
1949 2 : TenantSlot::InProgress(_) => {
1950 2 : self.tenant_slots_inprogress.inc();
1951 2 : }
1952 : }
1953 2 : }
1954 :
1955 2 : pub(crate) fn slot_removed(&self, slot: &TenantSlot) {
1956 2 : match slot {
1957 2 : TenantSlot::Attached(_) => {
1958 2 : self.tenant_slots_attached.dec();
1959 2 : }
1960 0 : TenantSlot::Secondary(_) => {
1961 0 : self.tenant_slots_secondary.dec();
1962 0 : }
1963 0 : TenantSlot::InProgress(_) => {
1964 0 : self.tenant_slots_inprogress.dec();
1965 0 : }
1966 : }
1967 2 : }
1968 :
1969 : #[cfg(all(debug_assertions, not(test)))]
1970 0 : pub(crate) fn slots_total(&self) -> u64 {
1971 0 : self.tenant_slots_attached.get()
1972 0 : + self.tenant_slots_secondary.get()
1973 0 : + self.tenant_slots_inprogress.get()
1974 0 : }
1975 : }
1976 :
1977 2 : pub(crate) static TENANT_MANAGER: Lazy<TenantManagerMetrics> = Lazy::new(|| {
1978 2 : let tenant_slots = register_uint_gauge_vec!(
1979 2 : "pageserver_tenant_manager_slots",
1980 2 : "How many slots currently exist, including all attached, secondary and in-progress operations",
1981 2 : &["mode"]
1982 2 : )
1983 2 : .expect("failed to define a metric");
1984 2 : TenantManagerMetrics {
1985 2 : tenant_slots_attached: tenant_slots
1986 2 : .get_metric_with_label_values(&["attached"])
1987 2 : .unwrap(),
1988 2 : tenant_slots_secondary: tenant_slots
1989 2 : .get_metric_with_label_values(&["secondary"])
1990 2 : .unwrap(),
1991 2 : tenant_slots_inprogress: tenant_slots
1992 2 : .get_metric_with_label_values(&["inprogress"])
1993 2 : .unwrap(),
1994 2 : tenant_slot_writes: register_int_counter!(
1995 2 : "pageserver_tenant_manager_slot_writes",
1996 2 : "Writes to a tenant slot, including all of create/attach/detach/delete"
1997 2 : )
1998 2 : .expect("failed to define a metric"),
1999 2 : unexpected_errors: register_int_counter!(
2000 2 : "pageserver_tenant_manager_unexpected_errors_total",
2001 2 : "Number of unexpected conditions encountered: nonzero value indicates a non-fatal bug."
2002 2 : )
2003 2 : .expect("failed to define a metric"),
2004 2 : }
2005 2 : });
2006 :
2007 : pub(crate) struct DeletionQueueMetrics {
2008 : pub(crate) keys_submitted: IntCounter,
2009 : pub(crate) keys_dropped: IntCounter,
2010 : pub(crate) keys_executed: IntCounter,
2011 : pub(crate) keys_validated: IntCounter,
2012 : pub(crate) dropped_lsn_updates: IntCounter,
2013 : pub(crate) unexpected_errors: IntCounter,
2014 : pub(crate) remote_errors: IntCounterVec,
2015 : }
2016 31 : pub(crate) static DELETION_QUEUE: Lazy<DeletionQueueMetrics> = Lazy::new(|| {
2017 31 : DeletionQueueMetrics{
2018 31 :
2019 31 : keys_submitted: register_int_counter!(
2020 31 : "pageserver_deletion_queue_submitted_total",
2021 31 : "Number of objects submitted for deletion"
2022 31 : )
2023 31 : .expect("failed to define a metric"),
2024 31 :
2025 31 : keys_dropped: register_int_counter!(
2026 31 : "pageserver_deletion_queue_dropped_total",
2027 31 : "Number of object deletions dropped due to stale generation."
2028 31 : )
2029 31 : .expect("failed to define a metric"),
2030 31 :
2031 31 : keys_executed: register_int_counter!(
2032 31 : "pageserver_deletion_queue_executed_total",
2033 31 : "Number of objects deleted. Only includes objects that we actually deleted, sum with pageserver_deletion_queue_dropped_total for the total number of keys processed to completion"
2034 31 : )
2035 31 : .expect("failed to define a metric"),
2036 31 :
2037 31 : keys_validated: register_int_counter!(
2038 31 : "pageserver_deletion_queue_validated_total",
2039 31 : "Number of keys validated for deletion. Sum with pageserver_deletion_queue_dropped_total for the total number of keys that have passed through the validation stage."
2040 31 : )
2041 31 : .expect("failed to define a metric"),
2042 31 :
2043 31 : dropped_lsn_updates: register_int_counter!(
2044 31 : "pageserver_deletion_queue_dropped_lsn_updates_total",
2045 31 : "Updates to remote_consistent_lsn dropped due to stale generation number."
2046 31 : )
2047 31 : .expect("failed to define a metric"),
2048 31 : unexpected_errors: register_int_counter!(
2049 31 : "pageserver_deletion_queue_unexpected_errors_total",
2050 31 : "Number of unexpected condiions that may stall the queue: any value above zero is unexpected."
2051 31 : )
2052 31 : .expect("failed to define a metric"),
2053 31 : remote_errors: register_int_counter_vec!(
2054 31 : "pageserver_deletion_queue_remote_errors_total",
2055 31 : "Retryable remote I/O errors while executing deletions, for example 503 responses to DeleteObjects",
2056 31 : &["op_kind"],
2057 31 : )
2058 31 : .expect("failed to define a metric")
2059 31 : }
2060 31 : });
2061 :
2062 : pub(crate) struct SecondaryModeMetrics {
2063 : pub(crate) upload_heatmap: IntCounter,
2064 : pub(crate) upload_heatmap_errors: IntCounter,
2065 : pub(crate) upload_heatmap_duration: Histogram,
2066 : pub(crate) download_heatmap: IntCounter,
2067 : pub(crate) download_layer: IntCounter,
2068 : }
2069 0 : pub(crate) static SECONDARY_MODE: Lazy<SecondaryModeMetrics> = Lazy::new(|| {
2070 0 : SecondaryModeMetrics {
2071 0 : upload_heatmap: register_int_counter!(
2072 0 : "pageserver_secondary_upload_heatmap",
2073 0 : "Number of heatmaps written to remote storage by attached tenants"
2074 0 : )
2075 0 : .expect("failed to define a metric"),
2076 0 : upload_heatmap_errors: register_int_counter!(
2077 0 : "pageserver_secondary_upload_heatmap_errors",
2078 0 : "Failures writing heatmap to remote storage"
2079 0 : )
2080 0 : .expect("failed to define a metric"),
2081 0 : upload_heatmap_duration: register_histogram!(
2082 0 : "pageserver_secondary_upload_heatmap_duration",
2083 0 : "Time to build and upload a heatmap, including any waiting inside the remote storage client"
2084 0 : )
2085 0 : .expect("failed to define a metric"),
2086 0 : download_heatmap: register_int_counter!(
2087 0 : "pageserver_secondary_download_heatmap",
2088 0 : "Number of downloads of heatmaps by secondary mode locations, including when it hasn't changed"
2089 0 : )
2090 0 : .expect("failed to define a metric"),
2091 0 : download_layer: register_int_counter!(
2092 0 : "pageserver_secondary_download_layer",
2093 0 : "Number of downloads of layers by secondary mode locations"
2094 0 : )
2095 0 : .expect("failed to define a metric"),
2096 0 : }
2097 0 : });
2098 :
2099 0 : pub(crate) static SECONDARY_RESIDENT_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
2100 0 : register_uint_gauge_vec!(
2101 0 : "pageserver_secondary_resident_physical_size",
2102 0 : "The size of the layer files present in the pageserver's filesystem, for secondary locations.",
2103 0 : &["tenant_id", "shard_id"]
2104 0 : )
2105 0 : .expect("failed to define a metric")
2106 0 : });
2107 :
2108 0 : pub(crate) static NODE_UTILIZATION_SCORE: Lazy<UIntGauge> = Lazy::new(|| {
2109 0 : register_uint_gauge!(
2110 0 : "pageserver_utilization_score",
2111 0 : "The utilization score we report to the storage controller for scheduling, where 0 is empty, 1000000 is full, and anything above is considered overloaded",
2112 0 : )
2113 0 : .expect("failed to define a metric")
2114 0 : });
2115 :
2116 0 : pub(crate) static SECONDARY_HEATMAP_TOTAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
2117 0 : register_uint_gauge_vec!(
2118 0 : "pageserver_secondary_heatmap_total_size",
2119 0 : "The total size in bytes of all layers in the most recently downloaded heatmap.",
2120 0 : &["tenant_id", "shard_id"]
2121 0 : )
2122 0 : .expect("failed to define a metric")
2123 0 : });
2124 :
2125 : #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
2126 : pub enum RemoteOpKind {
2127 : Upload,
2128 : Download,
2129 : Delete,
2130 : }
2131 : impl RemoteOpKind {
2132 13700 : pub fn as_str(&self) -> &'static str {
2133 13700 : match self {
2134 12886 : Self::Upload => "upload",
2135 52 : Self::Download => "download",
2136 762 : Self::Delete => "delete",
2137 : }
2138 13700 : }
2139 : }
2140 :
2141 : #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
2142 : pub enum RemoteOpFileKind {
2143 : Layer,
2144 : Index,
2145 : }
2146 : impl RemoteOpFileKind {
2147 13700 : pub fn as_str(&self) -> &'static str {
2148 13700 : match self {
2149 9234 : Self::Layer => "layer",
2150 4466 : Self::Index => "index",
2151 : }
2152 13700 : }
2153 : }
2154 :
2155 170 : pub(crate) static REMOTE_OPERATION_TIME: Lazy<HistogramVec> = Lazy::new(|| {
2156 170 : register_histogram_vec!(
2157 170 : "pageserver_remote_operation_seconds",
2158 170 : "Time spent on remote storage operations. \
2159 170 : Grouped by tenant, timeline, operation_kind and status. \
2160 170 : Does not account for time spent waiting in remote timeline client's queues.",
2161 170 : &["file_kind", "op_kind", "status"]
2162 170 : )
2163 170 : .expect("failed to define a metric")
2164 170 : });
2165 :
2166 0 : pub(crate) static TENANT_TASK_EVENTS: Lazy<IntCounterVec> = Lazy::new(|| {
2167 0 : register_int_counter_vec!(
2168 0 : "pageserver_tenant_task_events",
2169 0 : "Number of task start/stop/fail events.",
2170 0 : &["event"],
2171 0 : )
2172 0 : .expect("Failed to register tenant_task_events metric")
2173 0 : });
2174 :
2175 : pub struct BackgroundLoopSemaphoreMetrics {
2176 : counters: EnumMap<BackgroundLoopKind, IntCounterPair>,
2177 : durations: EnumMap<BackgroundLoopKind, Counter>,
2178 : }
2179 :
2180 : pub(crate) static BACKGROUND_LOOP_SEMAPHORE: Lazy<BackgroundLoopSemaphoreMetrics> = Lazy::new(
2181 20 : || {
2182 20 : let counters = register_int_counter_pair_vec!(
2183 20 : "pageserver_background_loop_semaphore_wait_start_count",
2184 20 : "Counter for background loop concurrency-limiting semaphore acquire calls started",
2185 20 : "pageserver_background_loop_semaphore_wait_finish_count",
2186 20 : "Counter for background loop concurrency-limiting semaphore acquire calls finished",
2187 20 : &["task"],
2188 20 : )
2189 20 : .unwrap();
2190 20 :
2191 20 : let durations = register_counter_vec!(
2192 20 : "pageserver_background_loop_semaphore_wait_duration_seconds",
2193 20 : "Sum of wall clock time spent waiting on the background loop concurrency-limiting semaphore acquire calls",
2194 20 : &["task"],
2195 20 : )
2196 20 : .unwrap();
2197 20 :
2198 20 : BackgroundLoopSemaphoreMetrics {
2199 180 : counters: enum_map::EnumMap::from_array(std::array::from_fn(|i| {
2200 180 : let kind = <BackgroundLoopKind as enum_map::Enum>::from_usize(i);
2201 180 : counters.with_label_values(&[kind.into()])
2202 180 : })),
2203 180 : durations: enum_map::EnumMap::from_array(std::array::from_fn(|i| {
2204 180 : let kind = <BackgroundLoopKind as enum_map::Enum>::from_usize(i);
2205 180 : durations.with_label_values(&[kind.into()])
2206 180 : })),
2207 20 : }
2208 20 : },
2209 : );
2210 :
2211 : impl BackgroundLoopSemaphoreMetrics {
2212 364 : pub(crate) fn measure_acquisition(&self, task: BackgroundLoopKind) -> impl Drop + '_ {
2213 : struct Record<'a> {
2214 : metrics: &'a BackgroundLoopSemaphoreMetrics,
2215 : task: BackgroundLoopKind,
2216 : _counter_guard: metrics::IntCounterPairGuard,
2217 : start: Instant,
2218 : }
2219 : impl Drop for Record<'_> {
2220 364 : fn drop(&mut self) {
2221 364 : let elapsed = self.start.elapsed().as_secs_f64();
2222 364 : self.metrics.durations[self.task].inc_by(elapsed);
2223 364 : }
2224 : }
2225 364 : Record {
2226 364 : metrics: self,
2227 364 : task,
2228 364 : _counter_guard: self.counters[task].guard(),
2229 364 : start: Instant::now(),
2230 364 : }
2231 364 : }
2232 : }
2233 :
2234 0 : pub(crate) static BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT: Lazy<IntCounterVec> = Lazy::new(|| {
2235 0 : register_int_counter_vec!(
2236 0 : "pageserver_background_loop_period_overrun_count",
2237 0 : "Incremented whenever warn_when_period_overrun() logs a warning.",
2238 0 : &["task", "period"],
2239 0 : )
2240 0 : .expect("failed to define a metric")
2241 0 : });
2242 :
2243 : // walreceiver metrics
2244 :
2245 0 : pub(crate) static WALRECEIVER_STARTED_CONNECTIONS: Lazy<IntCounter> = Lazy::new(|| {
2246 0 : register_int_counter!(
2247 0 : "pageserver_walreceiver_started_connections_total",
2248 0 : "Number of started walreceiver connections"
2249 0 : )
2250 0 : .expect("failed to define a metric")
2251 0 : });
2252 :
2253 0 : pub(crate) static WALRECEIVER_ACTIVE_MANAGERS: Lazy<IntGauge> = Lazy::new(|| {
2254 0 : register_int_gauge!(
2255 0 : "pageserver_walreceiver_active_managers",
2256 0 : "Number of active walreceiver managers"
2257 0 : )
2258 0 : .expect("failed to define a metric")
2259 0 : });
2260 :
2261 0 : pub(crate) static WALRECEIVER_SWITCHES: Lazy<IntCounterVec> = Lazy::new(|| {
2262 0 : register_int_counter_vec!(
2263 0 : "pageserver_walreceiver_switches_total",
2264 0 : "Number of walreceiver manager change_connection calls",
2265 0 : &["reason"]
2266 0 : )
2267 0 : .expect("failed to define a metric")
2268 0 : });
2269 :
2270 0 : pub(crate) static WALRECEIVER_BROKER_UPDATES: Lazy<IntCounter> = Lazy::new(|| {
2271 0 : register_int_counter!(
2272 0 : "pageserver_walreceiver_broker_updates_total",
2273 0 : "Number of received broker updates in walreceiver"
2274 0 : )
2275 0 : .expect("failed to define a metric")
2276 0 : });
2277 :
2278 2 : pub(crate) static WALRECEIVER_CANDIDATES_EVENTS: Lazy<IntCounterVec> = Lazy::new(|| {
2279 2 : register_int_counter_vec!(
2280 2 : "pageserver_walreceiver_candidates_events_total",
2281 2 : "Number of walreceiver candidate events",
2282 2 : &["event"]
2283 2 : )
2284 2 : .expect("failed to define a metric")
2285 2 : });
2286 :
2287 : pub(crate) static WALRECEIVER_CANDIDATES_ADDED: Lazy<IntCounter> =
2288 0 : Lazy::new(|| WALRECEIVER_CANDIDATES_EVENTS.with_label_values(&["add"]));
2289 :
2290 : pub(crate) static WALRECEIVER_CANDIDATES_REMOVED: Lazy<IntCounter> =
2291 2 : Lazy::new(|| WALRECEIVER_CANDIDATES_EVENTS.with_label_values(&["remove"]));
2292 :
2293 : // Metrics collected on WAL redo operations
2294 : //
2295 : // We collect the time spent in actual WAL redo ('redo'), and time waiting
2296 : // for access to the postgres process ('wait') since there is only one for
2297 : // each tenant.
2298 :
2299 : /// Time buckets are small because we want to be able to measure the
2300 : /// smallest redo processing times. These buckets allow us to measure down
2301 : /// to 5us, which equates to 200'000 pages/sec, which equates to 1.6GB/sec.
2302 : /// This is much better than the previous 5ms aka 200 pages/sec aka 1.6MB/sec.
2303 : ///
2304 : /// Values up to 1s are recorded because metrics show that we have redo
2305 : /// durations and lock times larger than 0.250s.
2306 : macro_rules! redo_histogram_time_buckets {
2307 : () => {
2308 : vec![
2309 : 0.000_005, 0.000_010, 0.000_025, 0.000_050, 0.000_100, 0.000_250, 0.000_500, 0.001_000,
2310 : 0.002_500, 0.005_000, 0.010_000, 0.025_000, 0.050_000, 0.100_000, 0.250_000, 0.500_000,
2311 : 1.000_000,
2312 : ]
2313 : };
2314 : }
2315 :
2316 : /// While we're at it, also measure the amount of records replayed in each
2317 : /// operation. We have a global 'total replayed' counter, but that's not
2318 : /// as useful as 'what is the skew for how many records we replay in one
2319 : /// operation'.
2320 : macro_rules! redo_histogram_count_buckets {
2321 : () => {
2322 : vec![0.0, 1.0, 2.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0]
2323 : };
2324 : }
2325 :
2326 : macro_rules! redo_bytes_histogram_count_buckets {
2327 : () => {
2328 : // powers of (2^.5), from 2^4.5 to 2^15 (22 buckets)
2329 : // rounded up to the next multiple of 8 to capture any MAXALIGNed record of that size, too.
2330 : vec![
2331 : 24.0, 32.0, 48.0, 64.0, 96.0, 128.0, 184.0, 256.0, 368.0, 512.0, 728.0, 1024.0, 1456.0,
2332 : 2048.0, 2904.0, 4096.0, 5800.0, 8192.0, 11592.0, 16384.0, 23176.0, 32768.0,
2333 : ]
2334 : };
2335 : }
2336 :
2337 : pub(crate) struct WalIngestMetrics {
2338 : pub(crate) bytes_received: IntCounter,
2339 : pub(crate) records_received: IntCounter,
2340 : pub(crate) records_committed: IntCounter,
2341 : pub(crate) records_filtered: IntCounter,
2342 : pub(crate) gap_blocks_zeroed_on_rel_extend: IntCounter,
2343 : pub(crate) clear_vm_bits_unknown: IntCounterVec,
2344 : }
2345 :
2346 10 : pub(crate) static WAL_INGEST: Lazy<WalIngestMetrics> = Lazy::new(|| WalIngestMetrics {
2347 10 : bytes_received: register_int_counter!(
2348 10 : "pageserver_wal_ingest_bytes_received",
2349 10 : "Bytes of WAL ingested from safekeepers",
2350 10 : )
2351 10 : .unwrap(),
2352 10 : records_received: register_int_counter!(
2353 10 : "pageserver_wal_ingest_records_received",
2354 10 : "Number of WAL records received from safekeepers"
2355 10 : )
2356 10 : .expect("failed to define a metric"),
2357 10 : records_committed: register_int_counter!(
2358 10 : "pageserver_wal_ingest_records_committed",
2359 10 : "Number of WAL records which resulted in writes to pageserver storage"
2360 10 : )
2361 10 : .expect("failed to define a metric"),
2362 10 : records_filtered: register_int_counter!(
2363 10 : "pageserver_wal_ingest_records_filtered",
2364 10 : "Number of WAL records filtered out due to sharding"
2365 10 : )
2366 10 : .expect("failed to define a metric"),
2367 10 : gap_blocks_zeroed_on_rel_extend: register_int_counter!(
2368 10 : "pageserver_gap_blocks_zeroed_on_rel_extend",
2369 10 : "Total number of zero gap blocks written on relation extends"
2370 10 : )
2371 10 : .expect("failed to define a metric"),
2372 10 : clear_vm_bits_unknown: register_int_counter_vec!(
2373 10 : "pageserver_wal_ingest_clear_vm_bits_unknown",
2374 10 : "Number of ignored ClearVmBits operations due to unknown pages/relations",
2375 10 : &["entity"],
2376 10 : )
2377 10 : .expect("failed to define a metric"),
2378 10 : });
2379 :
2380 176 : pub(crate) static PAGESERVER_TIMELINE_WAL_RECORDS_RECEIVED: Lazy<IntCounterVec> = Lazy::new(|| {
2381 176 : register_int_counter_vec!(
2382 176 : "pageserver_timeline_wal_records_received",
2383 176 : "Number of WAL records received per shard",
2384 176 : &["tenant_id", "shard_id", "timeline_id"]
2385 176 : )
2386 176 : .expect("failed to define a metric")
2387 176 : });
2388 :
2389 6 : pub(crate) static WAL_REDO_TIME: Lazy<Histogram> = Lazy::new(|| {
2390 6 : register_histogram!(
2391 6 : "pageserver_wal_redo_seconds",
2392 6 : "Time spent on WAL redo",
2393 6 : redo_histogram_time_buckets!()
2394 6 : )
2395 6 : .expect("failed to define a metric")
2396 6 : });
2397 :
2398 6 : pub(crate) static WAL_REDO_RECORDS_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
2399 6 : register_histogram!(
2400 6 : "pageserver_wal_redo_records_histogram",
2401 6 : "Histogram of number of records replayed per redo in the Postgres WAL redo process",
2402 6 : redo_histogram_count_buckets!(),
2403 6 : )
2404 6 : .expect("failed to define a metric")
2405 6 : });
2406 :
2407 6 : pub(crate) static WAL_REDO_BYTES_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
2408 6 : register_histogram!(
2409 6 : "pageserver_wal_redo_bytes_histogram",
2410 6 : "Histogram of number of records replayed per redo sent to Postgres",
2411 6 : redo_bytes_histogram_count_buckets!(),
2412 6 : )
2413 6 : .expect("failed to define a metric")
2414 6 : });
2415 :
2416 : // FIXME: isn't this already included by WAL_REDO_RECORDS_HISTOGRAM which has _count?
2417 6 : pub(crate) static WAL_REDO_RECORD_COUNTER: Lazy<IntCounter> = Lazy::new(|| {
2418 6 : register_int_counter!(
2419 6 : "pageserver_replayed_wal_records_total",
2420 6 : "Number of WAL records replayed in WAL redo process"
2421 6 : )
2422 6 : .unwrap()
2423 6 : });
2424 :
2425 : #[rustfmt::skip]
2426 8 : pub(crate) static WAL_REDO_PROCESS_LAUNCH_DURATION_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
2427 8 : register_histogram!(
2428 8 : "pageserver_wal_redo_process_launch_duration",
2429 8 : "Histogram of the duration of successful WalRedoProcess::launch calls",
2430 8 : vec![
2431 8 : 0.0002, 0.0004, 0.0006, 0.0008, 0.0010,
2432 8 : 0.0020, 0.0040, 0.0060, 0.0080, 0.0100,
2433 8 : 0.0200, 0.0400, 0.0600, 0.0800, 0.1000,
2434 8 : 0.2000, 0.4000, 0.6000, 0.8000, 1.0000,
2435 8 : 1.5000, 2.0000, 2.5000, 3.0000, 4.0000, 10.0000
2436 8 : ],
2437 8 : )
2438 8 : .expect("failed to define a metric")
2439 8 : });
2440 :
2441 : pub(crate) struct WalRedoProcessCounters {
2442 : pub(crate) started: IntCounter,
2443 : pub(crate) killed_by_cause: enum_map::EnumMap<WalRedoKillCause, IntCounter>,
2444 : pub(crate) active_stderr_logger_tasks_started: IntCounter,
2445 : pub(crate) active_stderr_logger_tasks_finished: IntCounter,
2446 : }
2447 :
2448 24 : #[derive(Debug, enum_map::Enum, strum_macros::IntoStaticStr)]
2449 : pub(crate) enum WalRedoKillCause {
2450 : WalRedoProcessDrop,
2451 : NoLeakChildDrop,
2452 : Startup,
2453 : }
2454 :
2455 : impl Default for WalRedoProcessCounters {
2456 8 : fn default() -> Self {
2457 8 : let started = register_int_counter!(
2458 8 : "pageserver_wal_redo_process_started_total",
2459 8 : "Number of WAL redo processes started",
2460 8 : )
2461 8 : .unwrap();
2462 8 :
2463 8 : let killed = register_int_counter_vec!(
2464 8 : "pageserver_wal_redo_process_stopped_total",
2465 8 : "Number of WAL redo processes stopped",
2466 8 : &["cause"],
2467 8 : )
2468 8 : .unwrap();
2469 8 :
2470 8 : let active_stderr_logger_tasks_started = register_int_counter!(
2471 8 : "pageserver_walredo_stderr_logger_tasks_started_total",
2472 8 : "Number of active walredo stderr logger tasks that have started",
2473 8 : )
2474 8 : .unwrap();
2475 8 :
2476 8 : let active_stderr_logger_tasks_finished = register_int_counter!(
2477 8 : "pageserver_walredo_stderr_logger_tasks_finished_total",
2478 8 : "Number of active walredo stderr logger tasks that have finished",
2479 8 : )
2480 8 : .unwrap();
2481 8 :
2482 8 : Self {
2483 8 : started,
2484 24 : killed_by_cause: EnumMap::from_array(std::array::from_fn(|i| {
2485 24 : let cause = <WalRedoKillCause as enum_map::Enum>::from_usize(i);
2486 24 : let cause_str: &'static str = cause.into();
2487 24 : killed.with_label_values(&[cause_str])
2488 24 : })),
2489 8 : active_stderr_logger_tasks_started,
2490 8 : active_stderr_logger_tasks_finished,
2491 8 : }
2492 8 : }
2493 : }
2494 :
2495 : pub(crate) static WAL_REDO_PROCESS_COUNTERS: Lazy<WalRedoProcessCounters> =
2496 : Lazy::new(WalRedoProcessCounters::default);
2497 :
2498 : /// Similar to `prometheus::HistogramTimer` but does not record on drop.
2499 : pub(crate) struct StorageTimeMetricsTimer {
2500 : metrics: StorageTimeMetrics,
2501 : start: Instant,
2502 : }
2503 :
2504 : impl StorageTimeMetricsTimer {
2505 3390 : fn new(metrics: StorageTimeMetrics) -> Self {
2506 3390 : Self {
2507 3390 : metrics,
2508 3390 : start: Instant::now(),
2509 3390 : }
2510 3390 : }
2511 :
2512 : /// Record the time from creation to now.
2513 2250 : pub fn stop_and_record(self) {
2514 2250 : let duration = self.start.elapsed().as_secs_f64();
2515 2250 : self.metrics.timeline_sum.inc_by(duration);
2516 2250 : self.metrics.timeline_count.inc();
2517 2250 : self.metrics.global_histogram.observe(duration);
2518 2250 : }
2519 :
2520 : /// Turns this timer into a timer, which will always record -- usually this means recording
2521 : /// regardless an early `?` path was taken in a function.
2522 4 : pub(crate) fn record_on_drop(self) -> AlwaysRecordingStorageTimeMetricsTimer {
2523 4 : AlwaysRecordingStorageTimeMetricsTimer(Some(self))
2524 4 : }
2525 : }
2526 :
2527 : pub(crate) struct AlwaysRecordingStorageTimeMetricsTimer(Option<StorageTimeMetricsTimer>);
2528 :
2529 : impl Drop for AlwaysRecordingStorageTimeMetricsTimer {
2530 4 : fn drop(&mut self) {
2531 4 : if let Some(inner) = self.0.take() {
2532 4 : inner.stop_and_record();
2533 4 : }
2534 4 : }
2535 : }
2536 :
2537 : /// Timing facilities for an globally histogrammed metric, which is supported by per tenant and
2538 : /// timeline total sum and count.
2539 : #[derive(Clone, Debug)]
2540 : pub(crate) struct StorageTimeMetrics {
2541 : /// Sum of f64 seconds, per operation, tenant_id and timeline_id
2542 : timeline_sum: Counter,
2543 : /// Number of oeprations, per operation, tenant_id and timeline_id
2544 : timeline_count: IntCounter,
2545 : /// Global histogram having only the "operation" label.
2546 : global_histogram: Histogram,
2547 : }
2548 :
2549 : impl StorageTimeMetrics {
2550 3376 : pub fn new(
2551 3376 : operation: StorageTimeOperation,
2552 3376 : tenant_id: &str,
2553 3376 : shard_id: &str,
2554 3376 : timeline_id: &str,
2555 3376 : ) -> Self {
2556 3376 : let operation: &'static str = operation.into();
2557 3376 :
2558 3376 : let timeline_sum = STORAGE_TIME_SUM_PER_TIMELINE
2559 3376 : .get_metric_with_label_values(&[operation, tenant_id, shard_id, timeline_id])
2560 3376 : .unwrap();
2561 3376 : let timeline_count = STORAGE_TIME_COUNT_PER_TIMELINE
2562 3376 : .get_metric_with_label_values(&[operation, tenant_id, shard_id, timeline_id])
2563 3376 : .unwrap();
2564 3376 : let global_histogram = STORAGE_TIME_GLOBAL
2565 3376 : .get_metric_with_label_values(&[operation])
2566 3376 : .unwrap();
2567 3376 :
2568 3376 : StorageTimeMetrics {
2569 3376 : timeline_sum,
2570 3376 : timeline_count,
2571 3376 : global_histogram,
2572 3376 : }
2573 3376 : }
2574 :
2575 : /// Starts timing a new operation.
2576 : ///
2577 : /// Note: unlike `prometheus::HistogramTimer` the returned timer does not record on drop.
2578 3390 : pub fn start_timer(&self) -> StorageTimeMetricsTimer {
2579 3390 : StorageTimeMetricsTimer::new(self.clone())
2580 3390 : }
2581 : }
2582 :
2583 : #[derive(Debug)]
2584 : pub(crate) struct TimelineMetrics {
2585 : tenant_id: String,
2586 : shard_id: String,
2587 : timeline_id: String,
2588 : pub flush_time_histo: StorageTimeMetrics,
2589 : pub flush_wait_upload_time_gauge: Gauge,
2590 : pub compact_time_histo: StorageTimeMetrics,
2591 : pub create_images_time_histo: StorageTimeMetrics,
2592 : pub logical_size_histo: StorageTimeMetrics,
2593 : pub imitate_logical_size_histo: StorageTimeMetrics,
2594 : pub load_layer_map_histo: StorageTimeMetrics,
2595 : pub garbage_collect_histo: StorageTimeMetrics,
2596 : pub find_gc_cutoffs_histo: StorageTimeMetrics,
2597 : pub last_record_lsn_gauge: IntGauge,
2598 : pub disk_consistent_lsn_gauge: IntGauge,
2599 : pub pitr_history_size: UIntGauge,
2600 : pub archival_size: UIntGauge,
2601 : pub(crate) layer_size_image: UIntGauge,
2602 : pub(crate) layer_count_image: UIntGauge,
2603 : pub(crate) layer_size_delta: UIntGauge,
2604 : pub(crate) layer_count_delta: UIntGauge,
2605 : pub standby_horizon_gauge: IntGauge,
2606 : pub resident_physical_size_gauge: UIntGauge,
2607 : pub visible_physical_size_gauge: UIntGauge,
2608 : /// copy of LayeredTimeline.current_logical_size
2609 : pub current_logical_size_gauge: UIntGauge,
2610 : pub aux_file_size_gauge: IntGauge,
2611 : pub directory_entries_count_gauge: Lazy<UIntGauge, Box<dyn Send + Fn() -> UIntGauge>>,
2612 : pub evictions: IntCounter,
2613 : pub evictions_with_low_residence_duration: std::sync::RwLock<EvictionsWithLowResidenceDuration>,
2614 : /// Number of valid LSN leases.
2615 : pub valid_lsn_lease_count_gauge: UIntGauge,
2616 : pub wal_records_received: IntCounter,
2617 : shutdown: std::sync::atomic::AtomicBool,
2618 : }
2619 :
2620 : impl TimelineMetrics {
2621 422 : pub fn new(
2622 422 : tenant_shard_id: &TenantShardId,
2623 422 : timeline_id_raw: &TimelineId,
2624 422 : evictions_with_low_residence_duration_builder: EvictionsWithLowResidenceDurationBuilder,
2625 422 : ) -> Self {
2626 422 : let tenant_id = tenant_shard_id.tenant_id.to_string();
2627 422 : let shard_id = format!("{}", tenant_shard_id.shard_slug());
2628 422 : let timeline_id = timeline_id_raw.to_string();
2629 422 : let flush_time_histo = StorageTimeMetrics::new(
2630 422 : StorageTimeOperation::LayerFlush,
2631 422 : &tenant_id,
2632 422 : &shard_id,
2633 422 : &timeline_id,
2634 422 : );
2635 422 : let flush_wait_upload_time_gauge = FLUSH_WAIT_UPLOAD_TIME
2636 422 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2637 422 : .unwrap();
2638 422 : let compact_time_histo = StorageTimeMetrics::new(
2639 422 : StorageTimeOperation::Compact,
2640 422 : &tenant_id,
2641 422 : &shard_id,
2642 422 : &timeline_id,
2643 422 : );
2644 422 : let create_images_time_histo = StorageTimeMetrics::new(
2645 422 : StorageTimeOperation::CreateImages,
2646 422 : &tenant_id,
2647 422 : &shard_id,
2648 422 : &timeline_id,
2649 422 : );
2650 422 : let logical_size_histo = StorageTimeMetrics::new(
2651 422 : StorageTimeOperation::LogicalSize,
2652 422 : &tenant_id,
2653 422 : &shard_id,
2654 422 : &timeline_id,
2655 422 : );
2656 422 : let imitate_logical_size_histo = StorageTimeMetrics::new(
2657 422 : StorageTimeOperation::ImitateLogicalSize,
2658 422 : &tenant_id,
2659 422 : &shard_id,
2660 422 : &timeline_id,
2661 422 : );
2662 422 : let load_layer_map_histo = StorageTimeMetrics::new(
2663 422 : StorageTimeOperation::LoadLayerMap,
2664 422 : &tenant_id,
2665 422 : &shard_id,
2666 422 : &timeline_id,
2667 422 : );
2668 422 : let garbage_collect_histo = StorageTimeMetrics::new(
2669 422 : StorageTimeOperation::Gc,
2670 422 : &tenant_id,
2671 422 : &shard_id,
2672 422 : &timeline_id,
2673 422 : );
2674 422 : let find_gc_cutoffs_histo = StorageTimeMetrics::new(
2675 422 : StorageTimeOperation::FindGcCutoffs,
2676 422 : &tenant_id,
2677 422 : &shard_id,
2678 422 : &timeline_id,
2679 422 : );
2680 422 : let last_record_lsn_gauge = LAST_RECORD_LSN
2681 422 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2682 422 : .unwrap();
2683 422 :
2684 422 : let disk_consistent_lsn_gauge = DISK_CONSISTENT_LSN
2685 422 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2686 422 : .unwrap();
2687 422 :
2688 422 : let pitr_history_size = PITR_HISTORY_SIZE
2689 422 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2690 422 : .unwrap();
2691 422 :
2692 422 : let archival_size = TIMELINE_ARCHIVE_SIZE
2693 422 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2694 422 : .unwrap();
2695 422 :
2696 422 : let layer_size_image = TIMELINE_LAYER_SIZE
2697 422 : .get_metric_with_label_values(&[
2698 422 : &tenant_id,
2699 422 : &shard_id,
2700 422 : &timeline_id,
2701 422 : MetricLayerKind::Image.into(),
2702 422 : ])
2703 422 : .unwrap();
2704 422 :
2705 422 : let layer_count_image = TIMELINE_LAYER_COUNT
2706 422 : .get_metric_with_label_values(&[
2707 422 : &tenant_id,
2708 422 : &shard_id,
2709 422 : &timeline_id,
2710 422 : MetricLayerKind::Image.into(),
2711 422 : ])
2712 422 : .unwrap();
2713 422 :
2714 422 : let layer_size_delta = TIMELINE_LAYER_SIZE
2715 422 : .get_metric_with_label_values(&[
2716 422 : &tenant_id,
2717 422 : &shard_id,
2718 422 : &timeline_id,
2719 422 : MetricLayerKind::Delta.into(),
2720 422 : ])
2721 422 : .unwrap();
2722 422 :
2723 422 : let layer_count_delta = TIMELINE_LAYER_COUNT
2724 422 : .get_metric_with_label_values(&[
2725 422 : &tenant_id,
2726 422 : &shard_id,
2727 422 : &timeline_id,
2728 422 : MetricLayerKind::Delta.into(),
2729 422 : ])
2730 422 : .unwrap();
2731 422 :
2732 422 : let standby_horizon_gauge = STANDBY_HORIZON
2733 422 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2734 422 : .unwrap();
2735 422 : let resident_physical_size_gauge = RESIDENT_PHYSICAL_SIZE
2736 422 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2737 422 : .unwrap();
2738 422 : let visible_physical_size_gauge = VISIBLE_PHYSICAL_SIZE
2739 422 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2740 422 : .unwrap();
2741 422 : // TODO: we shouldn't expose this metric
2742 422 : let current_logical_size_gauge = CURRENT_LOGICAL_SIZE
2743 422 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2744 422 : .unwrap();
2745 422 : let aux_file_size_gauge = AUX_FILE_SIZE
2746 422 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2747 422 : .unwrap();
2748 422 : // TODO use impl Trait syntax here once we have ability to use it: https://github.com/rust-lang/rust/issues/63065
2749 422 : let directory_entries_count_gauge_closure = {
2750 422 : let tenant_shard_id = *tenant_shard_id;
2751 422 : let timeline_id_raw = *timeline_id_raw;
2752 0 : move || {
2753 0 : let tenant_id = tenant_shard_id.tenant_id.to_string();
2754 0 : let shard_id = format!("{}", tenant_shard_id.shard_slug());
2755 0 : let timeline_id = timeline_id_raw.to_string();
2756 0 : let gauge: UIntGauge = DIRECTORY_ENTRIES_COUNT
2757 0 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2758 0 : .unwrap();
2759 0 : gauge
2760 0 : }
2761 : };
2762 422 : let directory_entries_count_gauge: Lazy<UIntGauge, Box<dyn Send + Fn() -> UIntGauge>> =
2763 422 : Lazy::new(Box::new(directory_entries_count_gauge_closure));
2764 422 : let evictions = EVICTIONS
2765 422 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2766 422 : .unwrap();
2767 422 : let evictions_with_low_residence_duration = evictions_with_low_residence_duration_builder
2768 422 : .build(&tenant_id, &shard_id, &timeline_id);
2769 422 :
2770 422 : let valid_lsn_lease_count_gauge = VALID_LSN_LEASE_COUNT
2771 422 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2772 422 : .unwrap();
2773 422 :
2774 422 : let wal_records_received = PAGESERVER_TIMELINE_WAL_RECORDS_RECEIVED
2775 422 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2776 422 : .unwrap();
2777 422 :
2778 422 : TimelineMetrics {
2779 422 : tenant_id,
2780 422 : shard_id,
2781 422 : timeline_id,
2782 422 : flush_time_histo,
2783 422 : flush_wait_upload_time_gauge,
2784 422 : compact_time_histo,
2785 422 : create_images_time_histo,
2786 422 : logical_size_histo,
2787 422 : imitate_logical_size_histo,
2788 422 : garbage_collect_histo,
2789 422 : find_gc_cutoffs_histo,
2790 422 : load_layer_map_histo,
2791 422 : last_record_lsn_gauge,
2792 422 : disk_consistent_lsn_gauge,
2793 422 : pitr_history_size,
2794 422 : archival_size,
2795 422 : layer_size_image,
2796 422 : layer_count_image,
2797 422 : layer_size_delta,
2798 422 : layer_count_delta,
2799 422 : standby_horizon_gauge,
2800 422 : resident_physical_size_gauge,
2801 422 : visible_physical_size_gauge,
2802 422 : current_logical_size_gauge,
2803 422 : aux_file_size_gauge,
2804 422 : directory_entries_count_gauge,
2805 422 : evictions,
2806 422 : evictions_with_low_residence_duration: std::sync::RwLock::new(
2807 422 : evictions_with_low_residence_duration,
2808 422 : ),
2809 422 : valid_lsn_lease_count_gauge,
2810 422 : wal_records_received,
2811 422 : shutdown: std::sync::atomic::AtomicBool::default(),
2812 422 : }
2813 422 : }
2814 :
2815 1548 : pub(crate) fn record_new_file_metrics(&self, sz: u64) {
2816 1548 : self.resident_physical_size_add(sz);
2817 1548 : }
2818 :
2819 527 : pub(crate) fn resident_physical_size_sub(&self, sz: u64) {
2820 527 : self.resident_physical_size_gauge.sub(sz);
2821 527 : crate::metrics::RESIDENT_PHYSICAL_SIZE_GLOBAL.sub(sz);
2822 527 : }
2823 :
2824 1578 : pub(crate) fn resident_physical_size_add(&self, sz: u64) {
2825 1578 : self.resident_physical_size_gauge.add(sz);
2826 1578 : crate::metrics::RESIDENT_PHYSICAL_SIZE_GLOBAL.add(sz);
2827 1578 : }
2828 :
2829 10 : pub(crate) fn resident_physical_size_get(&self) -> u64 {
2830 10 : self.resident_physical_size_gauge.get()
2831 10 : }
2832 :
2833 1148 : pub(crate) fn flush_wait_upload_time_gauge_add(&self, duration: f64) {
2834 1148 : self.flush_wait_upload_time_gauge.add(duration);
2835 1148 : crate::metrics::FLUSH_WAIT_UPLOAD_TIME
2836 1148 : .get_metric_with_label_values(&[&self.tenant_id, &self.shard_id, &self.timeline_id])
2837 1148 : .unwrap()
2838 1148 : .add(duration);
2839 1148 : }
2840 :
2841 10 : pub(crate) fn shutdown(&self) {
2842 10 : let was_shutdown = self
2843 10 : .shutdown
2844 10 : .swap(true, std::sync::atomic::Ordering::Relaxed);
2845 10 :
2846 10 : if was_shutdown {
2847 : // this happens on tenant deletion because tenant first shuts down timelines, then
2848 : // invokes timeline deletion which first shuts down the timeline again.
2849 : // TODO: this can be removed once https://github.com/neondatabase/neon/issues/5080
2850 0 : return;
2851 10 : }
2852 10 :
2853 10 : let tenant_id = &self.tenant_id;
2854 10 : let timeline_id = &self.timeline_id;
2855 10 : let shard_id = &self.shard_id;
2856 10 : let _ = LAST_RECORD_LSN.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2857 10 : let _ = DISK_CONSISTENT_LSN.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2858 10 : let _ = FLUSH_WAIT_UPLOAD_TIME.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2859 10 : let _ = STANDBY_HORIZON.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2860 10 : {
2861 10 : RESIDENT_PHYSICAL_SIZE_GLOBAL.sub(self.resident_physical_size_get());
2862 10 : let _ = RESIDENT_PHYSICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2863 10 : }
2864 10 : let _ = VISIBLE_PHYSICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2865 10 : let _ = CURRENT_LOGICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2866 10 : if let Some(metric) = Lazy::get(&DIRECTORY_ENTRIES_COUNT) {
2867 0 : let _ = metric.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2868 10 : }
2869 :
2870 10 : let _ = TIMELINE_ARCHIVE_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2871 10 : let _ = PITR_HISTORY_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2872 10 :
2873 10 : let _ = TIMELINE_LAYER_SIZE.remove_label_values(&[
2874 10 : tenant_id,
2875 10 : shard_id,
2876 10 : timeline_id,
2877 10 : MetricLayerKind::Image.into(),
2878 10 : ]);
2879 10 : let _ = TIMELINE_LAYER_COUNT.remove_label_values(&[
2880 10 : tenant_id,
2881 10 : shard_id,
2882 10 : timeline_id,
2883 10 : MetricLayerKind::Image.into(),
2884 10 : ]);
2885 10 : let _ = TIMELINE_LAYER_SIZE.remove_label_values(&[
2886 10 : tenant_id,
2887 10 : shard_id,
2888 10 : timeline_id,
2889 10 : MetricLayerKind::Delta.into(),
2890 10 : ]);
2891 10 : let _ = TIMELINE_LAYER_COUNT.remove_label_values(&[
2892 10 : tenant_id,
2893 10 : shard_id,
2894 10 : timeline_id,
2895 10 : MetricLayerKind::Delta.into(),
2896 10 : ]);
2897 10 :
2898 10 : let _ = EVICTIONS.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2899 10 : let _ = AUX_FILE_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2900 10 : let _ = VALID_LSN_LEASE_COUNT.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2901 10 :
2902 10 : self.evictions_with_low_residence_duration
2903 10 : .write()
2904 10 : .unwrap()
2905 10 : .remove(tenant_id, shard_id, timeline_id);
2906 :
2907 : // The following metrics are born outside of the TimelineMetrics lifecycle but still
2908 : // removed at the end of it. The idea is to have the metrics outlive the
2909 : // entity during which they're observed, e.g., the smgr metrics shall
2910 : // outlive an individual smgr connection, but not the timeline.
2911 :
2912 90 : for op in StorageTimeOperation::VARIANTS {
2913 80 : let _ = STORAGE_TIME_SUM_PER_TIMELINE.remove_label_values(&[
2914 80 : op,
2915 80 : tenant_id,
2916 80 : shard_id,
2917 80 : timeline_id,
2918 80 : ]);
2919 80 : let _ = STORAGE_TIME_COUNT_PER_TIMELINE.remove_label_values(&[
2920 80 : op,
2921 80 : tenant_id,
2922 80 : shard_id,
2923 80 : timeline_id,
2924 80 : ]);
2925 80 : }
2926 :
2927 30 : for op in STORAGE_IO_SIZE_OPERATIONS {
2928 20 : let _ = STORAGE_IO_SIZE.remove_label_values(&[op, tenant_id, shard_id, timeline_id]);
2929 20 : }
2930 :
2931 10 : let _ = SMGR_QUERY_STARTED_PER_TENANT_TIMELINE.remove_label_values(&[
2932 10 : SmgrQueryType::GetPageAtLsn.into(),
2933 10 : tenant_id,
2934 10 : shard_id,
2935 10 : timeline_id,
2936 10 : ]);
2937 10 : let _ = SMGR_QUERY_TIME_PER_TENANT_TIMELINE.remove_label_values(&[
2938 10 : SmgrQueryType::GetPageAtLsn.into(),
2939 10 : tenant_id,
2940 10 : shard_id,
2941 10 : timeline_id,
2942 10 : ]);
2943 10 : let _ = PAGE_SERVICE_BATCH_SIZE_PER_TENANT_TIMELINE.remove_label_values(&[
2944 10 : tenant_id,
2945 10 : shard_id,
2946 10 : timeline_id,
2947 10 : ]);
2948 10 : let _ = PAGESERVER_TIMELINE_WAL_RECORDS_RECEIVED.remove_label_values(&[
2949 10 : tenant_id,
2950 10 : shard_id,
2951 10 : timeline_id,
2952 10 : ]);
2953 10 : let _ = PAGE_SERVICE_SMGR_FLUSH_INPROGRESS_MICROS.remove_label_values(&[
2954 10 : tenant_id,
2955 10 : shard_id,
2956 10 : timeline_id,
2957 10 : ]);
2958 10 : let _ = PAGE_SERVICE_SMGR_BATCH_WAIT_TIME.remove_label_values(&[
2959 10 : tenant_id,
2960 10 : shard_id,
2961 10 : timeline_id,
2962 10 : ]);
2963 10 : }
2964 : }
2965 :
2966 6 : pub(crate) fn remove_tenant_metrics(tenant_shard_id: &TenantShardId) {
2967 6 : // Only shard zero deals in synthetic sizes
2968 6 : if tenant_shard_id.is_shard_zero() {
2969 6 : let tid = tenant_shard_id.tenant_id.to_string();
2970 6 : let _ = TENANT_SYNTHETIC_SIZE_METRIC.remove_label_values(&[&tid]);
2971 6 : }
2972 :
2973 6 : tenant_throttling::remove_tenant_metrics(tenant_shard_id);
2974 6 :
2975 6 : // we leave the BROKEN_TENANTS_SET entry if any
2976 6 : }
2977 :
2978 : use futures::Future;
2979 : use pin_project_lite::pin_project;
2980 : use std::collections::HashMap;
2981 : use std::num::NonZeroUsize;
2982 : use std::pin::Pin;
2983 : use std::sync::atomic::AtomicU64;
2984 : use std::sync::{Arc, Mutex};
2985 : use std::task::{Context, Poll};
2986 : use std::time::{Duration, Instant};
2987 :
2988 : use crate::config::PageServerConf;
2989 : use crate::context::{PageContentKind, RequestContext};
2990 : use crate::task_mgr::TaskKind;
2991 : use crate::tenant::mgr::TenantSlot;
2992 : use crate::tenant::tasks::BackgroundLoopKind;
2993 : use crate::tenant::throttle::ThrottleResult;
2994 : use crate::tenant::Timeline;
2995 :
2996 : /// Maintain a per timeline gauge in addition to the global gauge.
2997 : pub(crate) struct PerTimelineRemotePhysicalSizeGauge {
2998 : last_set: AtomicU64,
2999 : gauge: UIntGauge,
3000 : }
3001 :
3002 : impl PerTimelineRemotePhysicalSizeGauge {
3003 432 : fn new(per_timeline_gauge: UIntGauge) -> Self {
3004 432 : Self {
3005 432 : last_set: AtomicU64::new(0),
3006 432 : gauge: per_timeline_gauge,
3007 432 : }
3008 432 : }
3009 1878 : pub(crate) fn set(&self, sz: u64) {
3010 1878 : self.gauge.set(sz);
3011 1878 : let prev = self.last_set.swap(sz, std::sync::atomic::Ordering::Relaxed);
3012 1878 : if sz < prev {
3013 21 : REMOTE_PHYSICAL_SIZE_GLOBAL.sub(prev - sz);
3014 1857 : } else {
3015 1857 : REMOTE_PHYSICAL_SIZE_GLOBAL.add(sz - prev);
3016 1857 : };
3017 1878 : }
3018 2 : pub(crate) fn get(&self) -> u64 {
3019 2 : self.gauge.get()
3020 2 : }
3021 : }
3022 :
3023 : impl Drop for PerTimelineRemotePhysicalSizeGauge {
3024 20 : fn drop(&mut self) {
3025 20 : REMOTE_PHYSICAL_SIZE_GLOBAL.sub(self.last_set.load(std::sync::atomic::Ordering::Relaxed));
3026 20 : }
3027 : }
3028 :
3029 : pub(crate) struct RemoteTimelineClientMetrics {
3030 : tenant_id: String,
3031 : shard_id: String,
3032 : timeline_id: String,
3033 : pub(crate) remote_physical_size_gauge: PerTimelineRemotePhysicalSizeGauge,
3034 : calls: Mutex<HashMap<(&'static str, &'static str), IntCounterPair>>,
3035 : bytes_started_counter: Mutex<HashMap<(&'static str, &'static str), IntCounter>>,
3036 : bytes_finished_counter: Mutex<HashMap<(&'static str, &'static str), IntCounter>>,
3037 : pub(crate) projected_remote_consistent_lsn_gauge: UIntGauge,
3038 : }
3039 :
3040 : impl RemoteTimelineClientMetrics {
3041 432 : pub fn new(tenant_shard_id: &TenantShardId, timeline_id: &TimelineId) -> Self {
3042 432 : let tenant_id_str = tenant_shard_id.tenant_id.to_string();
3043 432 : let shard_id_str = format!("{}", tenant_shard_id.shard_slug());
3044 432 : let timeline_id_str = timeline_id.to_string();
3045 432 :
3046 432 : let remote_physical_size_gauge = PerTimelineRemotePhysicalSizeGauge::new(
3047 432 : REMOTE_PHYSICAL_SIZE
3048 432 : .get_metric_with_label_values(&[&tenant_id_str, &shard_id_str, &timeline_id_str])
3049 432 : .unwrap(),
3050 432 : );
3051 432 :
3052 432 : let projected_remote_consistent_lsn_gauge = PROJECTED_REMOTE_CONSISTENT_LSN
3053 432 : .get_metric_with_label_values(&[&tenant_id_str, &shard_id_str, &timeline_id_str])
3054 432 : .unwrap();
3055 432 :
3056 432 : RemoteTimelineClientMetrics {
3057 432 : tenant_id: tenant_id_str,
3058 432 : shard_id: shard_id_str,
3059 432 : timeline_id: timeline_id_str,
3060 432 : calls: Mutex::new(HashMap::default()),
3061 432 : bytes_started_counter: Mutex::new(HashMap::default()),
3062 432 : bytes_finished_counter: Mutex::new(HashMap::default()),
3063 432 : remote_physical_size_gauge,
3064 432 : projected_remote_consistent_lsn_gauge,
3065 432 : }
3066 432 : }
3067 :
3068 2742 : pub fn remote_operation_time(
3069 2742 : &self,
3070 2742 : file_kind: &RemoteOpFileKind,
3071 2742 : op_kind: &RemoteOpKind,
3072 2742 : status: &'static str,
3073 2742 : ) -> Histogram {
3074 2742 : let key = (file_kind.as_str(), op_kind.as_str(), status);
3075 2742 : REMOTE_OPERATION_TIME
3076 2742 : .get_metric_with_label_values(&[key.0, key.1, key.2])
3077 2742 : .unwrap()
3078 2742 : }
3079 :
3080 6574 : fn calls_counter_pair(
3081 6574 : &self,
3082 6574 : file_kind: &RemoteOpFileKind,
3083 6574 : op_kind: &RemoteOpKind,
3084 6574 : ) -> IntCounterPair {
3085 6574 : let mut guard = self.calls.lock().unwrap();
3086 6574 : let key = (file_kind.as_str(), op_kind.as_str());
3087 6574 : let metric = guard.entry(key).or_insert_with(move || {
3088 766 : REMOTE_TIMELINE_CLIENT_CALLS
3089 766 : .get_metric_with_label_values(&[
3090 766 : &self.tenant_id,
3091 766 : &self.shard_id,
3092 766 : &self.timeline_id,
3093 766 : key.0,
3094 766 : key.1,
3095 766 : ])
3096 766 : .unwrap()
3097 6574 : });
3098 6574 : metric.clone()
3099 6574 : }
3100 :
3101 1556 : fn bytes_started_counter(
3102 1556 : &self,
3103 1556 : file_kind: &RemoteOpFileKind,
3104 1556 : op_kind: &RemoteOpKind,
3105 1556 : ) -> IntCounter {
3106 1556 : let mut guard = self.bytes_started_counter.lock().unwrap();
3107 1556 : let key = (file_kind.as_str(), op_kind.as_str());
3108 1556 : let metric = guard.entry(key).or_insert_with(move || {
3109 294 : REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER
3110 294 : .get_metric_with_label_values(&[
3111 294 : &self.tenant_id,
3112 294 : &self.shard_id,
3113 294 : &self.timeline_id,
3114 294 : key.0,
3115 294 : key.1,
3116 294 : ])
3117 294 : .unwrap()
3118 1556 : });
3119 1556 : metric.clone()
3120 1556 : }
3121 :
3122 2816 : fn bytes_finished_counter(
3123 2816 : &self,
3124 2816 : file_kind: &RemoteOpFileKind,
3125 2816 : op_kind: &RemoteOpKind,
3126 2816 : ) -> IntCounter {
3127 2816 : let mut guard = self.bytes_finished_counter.lock().unwrap();
3128 2816 : let key = (file_kind.as_str(), op_kind.as_str());
3129 2816 : let metric = guard.entry(key).or_insert_with(move || {
3130 294 : REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER
3131 294 : .get_metric_with_label_values(&[
3132 294 : &self.tenant_id,
3133 294 : &self.shard_id,
3134 294 : &self.timeline_id,
3135 294 : key.0,
3136 294 : key.1,
3137 294 : ])
3138 294 : .unwrap()
3139 2816 : });
3140 2816 : metric.clone()
3141 2816 : }
3142 : }
3143 :
3144 : #[cfg(test)]
3145 : impl RemoteTimelineClientMetrics {
3146 6 : pub fn get_bytes_started_counter_value(
3147 6 : &self,
3148 6 : file_kind: &RemoteOpFileKind,
3149 6 : op_kind: &RemoteOpKind,
3150 6 : ) -> Option<u64> {
3151 6 : let guard = self.bytes_started_counter.lock().unwrap();
3152 6 : let key = (file_kind.as_str(), op_kind.as_str());
3153 6 : guard.get(&key).map(|counter| counter.get())
3154 6 : }
3155 :
3156 6 : pub fn get_bytes_finished_counter_value(
3157 6 : &self,
3158 6 : file_kind: &RemoteOpFileKind,
3159 6 : op_kind: &RemoteOpKind,
3160 6 : ) -> Option<u64> {
3161 6 : let guard = self.bytes_finished_counter.lock().unwrap();
3162 6 : let key = (file_kind.as_str(), op_kind.as_str());
3163 6 : guard.get(&key).map(|counter| counter.get())
3164 6 : }
3165 : }
3166 :
3167 : /// See [`RemoteTimelineClientMetrics::call_begin`].
3168 : #[must_use]
3169 : pub(crate) struct RemoteTimelineClientCallMetricGuard {
3170 : /// Decremented on drop.
3171 : calls_counter_pair: Option<IntCounterPair>,
3172 : /// If Some(), this references the bytes_finished metric, and we increment it by the given `u64` on drop.
3173 : bytes_finished: Option<(IntCounter, u64)>,
3174 : }
3175 :
3176 : impl RemoteTimelineClientCallMetricGuard {
3177 : /// Consume this guard object without performing the metric updates it would do on `drop()`.
3178 : /// The caller vouches to do the metric updates manually.
3179 3581 : pub fn will_decrement_manually(mut self) {
3180 3581 : let RemoteTimelineClientCallMetricGuard {
3181 3581 : calls_counter_pair,
3182 3581 : bytes_finished,
3183 3581 : } = &mut self;
3184 3581 : calls_counter_pair.take();
3185 3581 : bytes_finished.take();
3186 3581 : }
3187 : }
3188 :
3189 : impl Drop for RemoteTimelineClientCallMetricGuard {
3190 3607 : fn drop(&mut self) {
3191 3607 : let RemoteTimelineClientCallMetricGuard {
3192 3607 : calls_counter_pair,
3193 3607 : bytes_finished,
3194 3607 : } = self;
3195 3607 : if let Some(guard) = calls_counter_pair.take() {
3196 26 : guard.dec();
3197 3581 : }
3198 3607 : if let Some((bytes_finished_metric, value)) = bytes_finished {
3199 0 : bytes_finished_metric.inc_by(*value);
3200 3607 : }
3201 3607 : }
3202 : }
3203 :
3204 : /// The enum variants communicate to the [`RemoteTimelineClientMetrics`] whether to
3205 : /// track the byte size of this call in applicable metric(s).
3206 : pub(crate) enum RemoteTimelineClientMetricsCallTrackSize {
3207 : /// Do not account for this call's byte size in any metrics.
3208 : /// The `reason` field is there to make the call sites self-documenting
3209 : /// about why they don't need the metric.
3210 : DontTrackSize { reason: &'static str },
3211 : /// Track the byte size of the call in applicable metric(s).
3212 : Bytes(u64),
3213 : }
3214 :
3215 : impl RemoteTimelineClientMetrics {
3216 : /// Update the metrics that change when a call to the remote timeline client instance starts.
3217 : ///
3218 : /// Drop the returned guard object once the operation is finished to updates corresponding metrics that track completions.
3219 : /// Or, use [`RemoteTimelineClientCallMetricGuard::will_decrement_manually`] and [`call_end`](Self::call_end) if that
3220 : /// is more suitable.
3221 : /// Never do both.
3222 3607 : pub(crate) fn call_begin(
3223 3607 : &self,
3224 3607 : file_kind: &RemoteOpFileKind,
3225 3607 : op_kind: &RemoteOpKind,
3226 3607 : size: RemoteTimelineClientMetricsCallTrackSize,
3227 3607 : ) -> RemoteTimelineClientCallMetricGuard {
3228 3607 : let calls_counter_pair = self.calls_counter_pair(file_kind, op_kind);
3229 3607 : calls_counter_pair.inc();
3230 :
3231 3607 : let bytes_finished = match size {
3232 2051 : RemoteTimelineClientMetricsCallTrackSize::DontTrackSize { reason: _reason } => {
3233 2051 : // nothing to do
3234 2051 : None
3235 : }
3236 1556 : RemoteTimelineClientMetricsCallTrackSize::Bytes(size) => {
3237 1556 : self.bytes_started_counter(file_kind, op_kind).inc_by(size);
3238 1556 : let finished_counter = self.bytes_finished_counter(file_kind, op_kind);
3239 1556 : Some((finished_counter, size))
3240 : }
3241 : };
3242 3607 : RemoteTimelineClientCallMetricGuard {
3243 3607 : calls_counter_pair: Some(calls_counter_pair),
3244 3607 : bytes_finished,
3245 3607 : }
3246 3607 : }
3247 :
3248 : /// Manually udpate the metrics that track completions, instead of using the guard object.
3249 : /// Using the guard object is generally preferable.
3250 : /// See [`call_begin`](Self::call_begin) for more context.
3251 2967 : pub(crate) fn call_end(
3252 2967 : &self,
3253 2967 : file_kind: &RemoteOpFileKind,
3254 2967 : op_kind: &RemoteOpKind,
3255 2967 : size: RemoteTimelineClientMetricsCallTrackSize,
3256 2967 : ) {
3257 2967 : let calls_counter_pair = self.calls_counter_pair(file_kind, op_kind);
3258 2967 : calls_counter_pair.dec();
3259 2967 : match size {
3260 1707 : RemoteTimelineClientMetricsCallTrackSize::DontTrackSize { reason: _reason } => {}
3261 1260 : RemoteTimelineClientMetricsCallTrackSize::Bytes(size) => {
3262 1260 : self.bytes_finished_counter(file_kind, op_kind).inc_by(size);
3263 1260 : }
3264 : }
3265 2967 : }
3266 : }
3267 :
3268 : impl Drop for RemoteTimelineClientMetrics {
3269 20 : fn drop(&mut self) {
3270 20 : let RemoteTimelineClientMetrics {
3271 20 : tenant_id,
3272 20 : shard_id,
3273 20 : timeline_id,
3274 20 : remote_physical_size_gauge,
3275 20 : calls,
3276 20 : bytes_started_counter,
3277 20 : bytes_finished_counter,
3278 20 : projected_remote_consistent_lsn_gauge,
3279 20 : } = self;
3280 24 : for ((a, b), _) in calls.get_mut().unwrap().drain() {
3281 24 : let mut res = [Ok(()), Ok(())];
3282 24 : REMOTE_TIMELINE_CLIENT_CALLS
3283 24 : .remove_label_values(&mut res, &[tenant_id, shard_id, timeline_id, a, b]);
3284 24 : // don't care about results
3285 24 : }
3286 20 : for ((a, b), _) in bytes_started_counter.get_mut().unwrap().drain() {
3287 6 : let _ = REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER.remove_label_values(&[
3288 6 : tenant_id,
3289 6 : shard_id,
3290 6 : timeline_id,
3291 6 : a,
3292 6 : b,
3293 6 : ]);
3294 6 : }
3295 20 : for ((a, b), _) in bytes_finished_counter.get_mut().unwrap().drain() {
3296 6 : let _ = REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER.remove_label_values(&[
3297 6 : tenant_id,
3298 6 : shard_id,
3299 6 : timeline_id,
3300 6 : a,
3301 6 : b,
3302 6 : ]);
3303 6 : }
3304 20 : {
3305 20 : let _ = remote_physical_size_gauge; // use to avoid 'unused' warning in desctructuring above
3306 20 : let _ = REMOTE_PHYSICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
3307 20 : }
3308 20 : {
3309 20 : let _ = projected_remote_consistent_lsn_gauge;
3310 20 : let _ = PROJECTED_REMOTE_CONSISTENT_LSN.remove_label_values(&[
3311 20 : tenant_id,
3312 20 : shard_id,
3313 20 : timeline_id,
3314 20 : ]);
3315 20 : }
3316 20 : }
3317 : }
3318 :
3319 : /// Wrapper future that measures the time spent by a remote storage operation,
3320 : /// and records the time and success/failure as a prometheus metric.
3321 : pub(crate) trait MeasureRemoteOp: Sized {
3322 2945 : fn measure_remote_op(
3323 2945 : self,
3324 2945 : file_kind: RemoteOpFileKind,
3325 2945 : op: RemoteOpKind,
3326 2945 : metrics: Arc<RemoteTimelineClientMetrics>,
3327 2945 : ) -> MeasuredRemoteOp<Self> {
3328 2945 : let start = Instant::now();
3329 2945 : MeasuredRemoteOp {
3330 2945 : inner: self,
3331 2945 : file_kind,
3332 2945 : op,
3333 2945 : start,
3334 2945 : metrics,
3335 2945 : }
3336 2945 : }
3337 : }
3338 :
3339 : impl<T: Sized> MeasureRemoteOp for T {}
3340 :
3341 : pin_project! {
3342 : pub(crate) struct MeasuredRemoteOp<F>
3343 : {
3344 : #[pin]
3345 : inner: F,
3346 : file_kind: RemoteOpFileKind,
3347 : op: RemoteOpKind,
3348 : start: Instant,
3349 : metrics: Arc<RemoteTimelineClientMetrics>,
3350 : }
3351 : }
3352 :
3353 : impl<F: Future<Output = Result<O, E>>, O, E> Future for MeasuredRemoteOp<F> {
3354 : type Output = Result<O, E>;
3355 :
3356 45196 : fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
3357 45196 : let this = self.project();
3358 45196 : let poll_result = this.inner.poll(cx);
3359 45196 : if let Poll::Ready(ref res) = poll_result {
3360 2742 : let duration = this.start.elapsed();
3361 2742 : let status = if res.is_ok() { &"success" } else { &"failure" };
3362 2742 : this.metrics
3363 2742 : .remote_operation_time(this.file_kind, this.op, status)
3364 2742 : .observe(duration.as_secs_f64());
3365 42454 : }
3366 45196 : poll_result
3367 45196 : }
3368 : }
3369 :
3370 : pub mod tokio_epoll_uring {
3371 : use std::{
3372 : collections::HashMap,
3373 : sync::{Arc, Mutex},
3374 : };
3375 :
3376 : use metrics::{register_histogram, register_int_counter, Histogram, LocalHistogram, UIntGauge};
3377 : use once_cell::sync::Lazy;
3378 :
3379 : /// Shared storage for tokio-epoll-uring thread local metrics.
3380 : pub(crate) static THREAD_LOCAL_METRICS_STORAGE: Lazy<ThreadLocalMetricsStorage> =
3381 104 : Lazy::new(|| {
3382 104 : let slots_submission_queue_depth = register_histogram!(
3383 104 : "pageserver_tokio_epoll_uring_slots_submission_queue_depth",
3384 104 : "The slots waiters queue depth of each tokio_epoll_uring system",
3385 104 : vec![1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0],
3386 104 : )
3387 104 : .expect("failed to define a metric");
3388 104 : ThreadLocalMetricsStorage {
3389 104 : observers: Mutex::new(HashMap::new()),
3390 104 : slots_submission_queue_depth,
3391 104 : }
3392 104 : });
3393 :
3394 : pub struct ThreadLocalMetricsStorage {
3395 : /// List of thread local metrics observers.
3396 : observers: Mutex<HashMap<u64, Arc<ThreadLocalMetrics>>>,
3397 : /// A histogram shared between all thread local systems
3398 : /// for collecting slots submission queue depth.
3399 : slots_submission_queue_depth: Histogram,
3400 : }
3401 :
3402 : /// Each thread-local [`tokio_epoll_uring::System`] gets one of these as its
3403 : /// [`tokio_epoll_uring::metrics::PerSystemMetrics`] generic.
3404 : ///
3405 : /// The System makes observations into [`Self`] and periodically, the collector
3406 : /// comes along and flushes [`Self`] into the shared storage [`THREAD_LOCAL_METRICS_STORAGE`].
3407 : ///
3408 : /// [`LocalHistogram`] is `!Send`, so, we need to put it behind a [`Mutex`].
3409 : /// But except for the periodic flush, the lock is uncontended so there's no waiting
3410 : /// for cache coherence protocol to get an exclusive cache line.
3411 : pub struct ThreadLocalMetrics {
3412 : /// Local observer of thread local tokio-epoll-uring system's slots waiters queue depth.
3413 : slots_submission_queue_depth: Mutex<LocalHistogram>,
3414 : }
3415 :
3416 : impl ThreadLocalMetricsStorage {
3417 : /// Registers a new thread local system. Returns a thread local metrics observer.
3418 457 : pub fn register_system(&self, id: u64) -> Arc<ThreadLocalMetrics> {
3419 457 : let per_system_metrics = Arc::new(ThreadLocalMetrics::new(
3420 457 : self.slots_submission_queue_depth.local(),
3421 457 : ));
3422 457 : let mut g = self.observers.lock().unwrap();
3423 457 : g.insert(id, Arc::clone(&per_system_metrics));
3424 457 : per_system_metrics
3425 457 : }
3426 :
3427 : /// Removes metrics observer for a thread local system.
3428 : /// This should be called before dropping a thread local system.
3429 104 : pub fn remove_system(&self, id: u64) {
3430 104 : let mut g = self.observers.lock().unwrap();
3431 104 : g.remove(&id);
3432 104 : }
3433 :
3434 : /// Flush all thread local metrics to the shared storage.
3435 0 : pub fn flush_thread_local_metrics(&self) {
3436 0 : let g = self.observers.lock().unwrap();
3437 0 : g.values().for_each(|local| {
3438 0 : local.flush();
3439 0 : });
3440 0 : }
3441 : }
3442 :
3443 : impl ThreadLocalMetrics {
3444 457 : pub fn new(slots_submission_queue_depth: LocalHistogram) -> Self {
3445 457 : ThreadLocalMetrics {
3446 457 : slots_submission_queue_depth: Mutex::new(slots_submission_queue_depth),
3447 457 : }
3448 457 : }
3449 :
3450 : /// Flushes the thread local metrics to shared aggregator.
3451 0 : pub fn flush(&self) {
3452 0 : let Self {
3453 0 : slots_submission_queue_depth,
3454 0 : } = self;
3455 0 : slots_submission_queue_depth.lock().unwrap().flush();
3456 0 : }
3457 : }
3458 :
3459 : impl tokio_epoll_uring::metrics::PerSystemMetrics for ThreadLocalMetrics {
3460 903896 : fn observe_slots_submission_queue_depth(&self, queue_depth: u64) {
3461 903896 : let Self {
3462 903896 : slots_submission_queue_depth,
3463 903896 : } = self;
3464 903896 : slots_submission_queue_depth
3465 903896 : .lock()
3466 903896 : .unwrap()
3467 903896 : .observe(queue_depth as f64);
3468 903896 : }
3469 : }
3470 :
3471 : pub struct Collector {
3472 : descs: Vec<metrics::core::Desc>,
3473 : systems_created: UIntGauge,
3474 : systems_destroyed: UIntGauge,
3475 : thread_local_metrics_storage: &'static ThreadLocalMetricsStorage,
3476 : }
3477 :
3478 : impl metrics::core::Collector for Collector {
3479 0 : fn desc(&self) -> Vec<&metrics::core::Desc> {
3480 0 : self.descs.iter().collect()
3481 0 : }
3482 :
3483 0 : fn collect(&self) -> Vec<metrics::proto::MetricFamily> {
3484 0 : let mut mfs = Vec::with_capacity(Self::NMETRICS);
3485 0 : let tokio_epoll_uring::metrics::GlobalMetrics {
3486 0 : systems_created,
3487 0 : systems_destroyed,
3488 0 : } = tokio_epoll_uring::metrics::global();
3489 0 : self.systems_created.set(systems_created);
3490 0 : mfs.extend(self.systems_created.collect());
3491 0 : self.systems_destroyed.set(systems_destroyed);
3492 0 : mfs.extend(self.systems_destroyed.collect());
3493 0 :
3494 0 : self.thread_local_metrics_storage
3495 0 : .flush_thread_local_metrics();
3496 0 :
3497 0 : mfs.extend(
3498 0 : self.thread_local_metrics_storage
3499 0 : .slots_submission_queue_depth
3500 0 : .collect(),
3501 0 : );
3502 0 : mfs
3503 0 : }
3504 : }
3505 :
3506 : impl Collector {
3507 : const NMETRICS: usize = 3;
3508 :
3509 : #[allow(clippy::new_without_default)]
3510 0 : pub fn new() -> Self {
3511 0 : let mut descs = Vec::new();
3512 0 :
3513 0 : let systems_created = UIntGauge::new(
3514 0 : "pageserver_tokio_epoll_uring_systems_created",
3515 0 : "counter of tokio-epoll-uring systems that were created",
3516 0 : )
3517 0 : .unwrap();
3518 0 : descs.extend(
3519 0 : metrics::core::Collector::desc(&systems_created)
3520 0 : .into_iter()
3521 0 : .cloned(),
3522 0 : );
3523 0 :
3524 0 : let systems_destroyed = UIntGauge::new(
3525 0 : "pageserver_tokio_epoll_uring_systems_destroyed",
3526 0 : "counter of tokio-epoll-uring systems that were destroyed",
3527 0 : )
3528 0 : .unwrap();
3529 0 : descs.extend(
3530 0 : metrics::core::Collector::desc(&systems_destroyed)
3531 0 : .into_iter()
3532 0 : .cloned(),
3533 0 : );
3534 0 :
3535 0 : Self {
3536 0 : descs,
3537 0 : systems_created,
3538 0 : systems_destroyed,
3539 0 : thread_local_metrics_storage: &THREAD_LOCAL_METRICS_STORAGE,
3540 0 : }
3541 0 : }
3542 : }
3543 :
3544 104 : pub(crate) static THREAD_LOCAL_LAUNCH_SUCCESSES: Lazy<metrics::IntCounter> = Lazy::new(|| {
3545 104 : register_int_counter!(
3546 104 : "pageserver_tokio_epoll_uring_pageserver_thread_local_launch_success_count",
3547 104 : "Number of times where thread_local_system creation spanned multiple executor threads",
3548 104 : )
3549 104 : .unwrap()
3550 104 : });
3551 :
3552 0 : pub(crate) static THREAD_LOCAL_LAUNCH_FAILURES: Lazy<metrics::IntCounter> = Lazy::new(|| {
3553 0 : register_int_counter!(
3554 0 : "pageserver_tokio_epoll_uring_pageserver_thread_local_launch_failures_count",
3555 0 : "Number of times thread_local_system creation failed and was retried after back-off.",
3556 0 : )
3557 0 : .unwrap()
3558 0 : });
3559 : }
3560 :
3561 : pub(crate) mod tenant_throttling {
3562 : use metrics::{register_int_counter_vec, IntCounter};
3563 : use once_cell::sync::Lazy;
3564 : use utils::shard::TenantShardId;
3565 :
3566 : use crate::tenant::{self};
3567 :
3568 : struct GlobalAndPerTenantIntCounter {
3569 : global: IntCounter,
3570 : per_tenant: IntCounter,
3571 : }
3572 :
3573 : impl GlobalAndPerTenantIntCounter {
3574 : #[inline(always)]
3575 0 : pub(crate) fn inc(&self) {
3576 0 : self.inc_by(1)
3577 0 : }
3578 : #[inline(always)]
3579 0 : pub(crate) fn inc_by(&self, n: u64) {
3580 0 : self.global.inc_by(n);
3581 0 : self.per_tenant.inc_by(n);
3582 0 : }
3583 : }
3584 :
3585 : pub(crate) struct Metrics<const KIND: usize> {
3586 : count_accounted_start: GlobalAndPerTenantIntCounter,
3587 : count_accounted_finish: GlobalAndPerTenantIntCounter,
3588 : wait_time: GlobalAndPerTenantIntCounter,
3589 : count_throttled: GlobalAndPerTenantIntCounter,
3590 : }
3591 :
3592 178 : static COUNT_ACCOUNTED_START: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
3593 178 : register_int_counter_vec!(
3594 178 : "pageserver_tenant_throttling_count_accounted_start_global",
3595 178 : "Count of tenant throttling starts, by kind of throttle.",
3596 178 : &["kind"]
3597 178 : )
3598 178 : .unwrap()
3599 178 : });
3600 178 : static COUNT_ACCOUNTED_START_PER_TENANT: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
3601 178 : register_int_counter_vec!(
3602 178 : "pageserver_tenant_throttling_count_accounted_start",
3603 178 : "Count of tenant throttling starts, by kind of throttle.",
3604 178 : &["kind", "tenant_id", "shard_id"]
3605 178 : )
3606 178 : .unwrap()
3607 178 : });
3608 178 : static COUNT_ACCOUNTED_FINISH: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
3609 178 : register_int_counter_vec!(
3610 178 : "pageserver_tenant_throttling_count_accounted_finish_global",
3611 178 : "Count of tenant throttling finishes, by kind of throttle.",
3612 178 : &["kind"]
3613 178 : )
3614 178 : .unwrap()
3615 178 : });
3616 178 : static COUNT_ACCOUNTED_FINISH_PER_TENANT: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
3617 178 : register_int_counter_vec!(
3618 178 : "pageserver_tenant_throttling_count_accounted_finish",
3619 178 : "Count of tenant throttling finishes, by kind of throttle.",
3620 178 : &["kind", "tenant_id", "shard_id"]
3621 178 : )
3622 178 : .unwrap()
3623 178 : });
3624 178 : static WAIT_USECS: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
3625 178 : register_int_counter_vec!(
3626 178 : "pageserver_tenant_throttling_wait_usecs_sum_global",
3627 178 : "Sum of microseconds that spent waiting throttle by kind of throttle.",
3628 178 : &["kind"]
3629 178 : )
3630 178 : .unwrap()
3631 178 : });
3632 178 : static WAIT_USECS_PER_TENANT: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
3633 178 : register_int_counter_vec!(
3634 178 : "pageserver_tenant_throttling_wait_usecs_sum",
3635 178 : "Sum of microseconds that spent waiting throttle by kind of throttle.",
3636 178 : &["kind", "tenant_id", "shard_id"]
3637 178 : )
3638 178 : .unwrap()
3639 178 : });
3640 :
3641 178 : static WAIT_COUNT: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
3642 178 : register_int_counter_vec!(
3643 178 : "pageserver_tenant_throttling_count_global",
3644 178 : "Count of tenant throttlings, by kind of throttle.",
3645 178 : &["kind"]
3646 178 : )
3647 178 : .unwrap()
3648 178 : });
3649 178 : static WAIT_COUNT_PER_TENANT: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
3650 178 : register_int_counter_vec!(
3651 178 : "pageserver_tenant_throttling_count",
3652 178 : "Count of tenant throttlings, by kind of throttle.",
3653 178 : &["kind", "tenant_id", "shard_id"]
3654 178 : )
3655 178 : .unwrap()
3656 178 : });
3657 :
3658 : const KINDS: &[&str] = &["pagestream"];
3659 : pub type Pagestream = Metrics<0>;
3660 :
3661 : impl<const KIND: usize> Metrics<KIND> {
3662 196 : pub(crate) fn new(tenant_shard_id: &TenantShardId) -> Self {
3663 196 : let per_tenant_label_values = &[
3664 196 : KINDS[KIND],
3665 196 : &tenant_shard_id.tenant_id.to_string(),
3666 196 : &tenant_shard_id.shard_slug().to_string(),
3667 196 : ];
3668 196 : Metrics {
3669 196 : count_accounted_start: {
3670 196 : GlobalAndPerTenantIntCounter {
3671 196 : global: COUNT_ACCOUNTED_START.with_label_values(&[KINDS[KIND]]),
3672 196 : per_tenant: COUNT_ACCOUNTED_START_PER_TENANT
3673 196 : .with_label_values(per_tenant_label_values),
3674 196 : }
3675 196 : },
3676 196 : count_accounted_finish: {
3677 196 : GlobalAndPerTenantIntCounter {
3678 196 : global: COUNT_ACCOUNTED_FINISH.with_label_values(&[KINDS[KIND]]),
3679 196 : per_tenant: COUNT_ACCOUNTED_FINISH_PER_TENANT
3680 196 : .with_label_values(per_tenant_label_values),
3681 196 : }
3682 196 : },
3683 196 : wait_time: {
3684 196 : GlobalAndPerTenantIntCounter {
3685 196 : global: WAIT_USECS.with_label_values(&[KINDS[KIND]]),
3686 196 : per_tenant: WAIT_USECS_PER_TENANT
3687 196 : .with_label_values(per_tenant_label_values),
3688 196 : }
3689 196 : },
3690 196 : count_throttled: {
3691 196 : GlobalAndPerTenantIntCounter {
3692 196 : global: WAIT_COUNT.with_label_values(&[KINDS[KIND]]),
3693 196 : per_tenant: WAIT_COUNT_PER_TENANT
3694 196 : .with_label_values(per_tenant_label_values),
3695 196 : }
3696 196 : },
3697 196 : }
3698 196 : }
3699 : }
3700 :
3701 0 : pub(crate) fn preinitialize_global_metrics() {
3702 0 : Lazy::force(&COUNT_ACCOUNTED_START);
3703 0 : Lazy::force(&COUNT_ACCOUNTED_FINISH);
3704 0 : Lazy::force(&WAIT_USECS);
3705 0 : Lazy::force(&WAIT_COUNT);
3706 0 : }
3707 :
3708 6 : pub(crate) fn remove_tenant_metrics(tenant_shard_id: &TenantShardId) {
3709 24 : for m in &[
3710 6 : &COUNT_ACCOUNTED_START_PER_TENANT,
3711 6 : &COUNT_ACCOUNTED_FINISH_PER_TENANT,
3712 6 : &WAIT_USECS_PER_TENANT,
3713 6 : &WAIT_COUNT_PER_TENANT,
3714 6 : ] {
3715 48 : for kind in KINDS {
3716 24 : let _ = m.remove_label_values(&[
3717 24 : kind,
3718 24 : &tenant_shard_id.tenant_id.to_string(),
3719 24 : &tenant_shard_id.shard_slug().to_string(),
3720 24 : ]);
3721 24 : }
3722 : }
3723 6 : }
3724 :
3725 : impl<const KIND: usize> tenant::throttle::Metric for Metrics<KIND> {
3726 : #[inline(always)]
3727 0 : fn accounting_start(&self) {
3728 0 : self.count_accounted_start.inc();
3729 0 : }
3730 : #[inline(always)]
3731 0 : fn accounting_finish(&self) {
3732 0 : self.count_accounted_finish.inc();
3733 0 : }
3734 : #[inline(always)]
3735 0 : fn observe_throttling(
3736 0 : &self,
3737 0 : tenant::throttle::Observation { wait_time }: &tenant::throttle::Observation,
3738 0 : ) {
3739 0 : let val = u64::try_from(wait_time.as_micros()).unwrap();
3740 0 : self.wait_time.inc_by(val);
3741 0 : self.count_throttled.inc();
3742 0 : }
3743 : }
3744 : }
3745 :
3746 : pub(crate) mod disk_usage_based_eviction {
3747 : use super::*;
3748 :
3749 : pub(crate) struct Metrics {
3750 : pub(crate) tenant_collection_time: Histogram,
3751 : pub(crate) tenant_layer_count: Histogram,
3752 : pub(crate) layers_collected: IntCounter,
3753 : pub(crate) layers_selected: IntCounter,
3754 : pub(crate) layers_evicted: IntCounter,
3755 : }
3756 :
3757 : impl Default for Metrics {
3758 0 : fn default() -> Self {
3759 0 : let tenant_collection_time = register_histogram!(
3760 0 : "pageserver_disk_usage_based_eviction_tenant_collection_seconds",
3761 0 : "Time spent collecting layers from a tenant -- not normalized by collected layer amount",
3762 0 : vec![0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0, 10.0]
3763 0 : )
3764 0 : .unwrap();
3765 0 :
3766 0 : let tenant_layer_count = register_histogram!(
3767 0 : "pageserver_disk_usage_based_eviction_tenant_collected_layers",
3768 0 : "Amount of layers gathered from a tenant",
3769 0 : vec![5.0, 50.0, 500.0, 5000.0, 50000.0]
3770 0 : )
3771 0 : .unwrap();
3772 0 :
3773 0 : let layers_collected = register_int_counter!(
3774 0 : "pageserver_disk_usage_based_eviction_collected_layers_total",
3775 0 : "Amount of layers collected"
3776 0 : )
3777 0 : .unwrap();
3778 0 :
3779 0 : let layers_selected = register_int_counter!(
3780 0 : "pageserver_disk_usage_based_eviction_select_layers_total",
3781 0 : "Amount of layers selected"
3782 0 : )
3783 0 : .unwrap();
3784 0 :
3785 0 : let layers_evicted = register_int_counter!(
3786 0 : "pageserver_disk_usage_based_eviction_evicted_layers_total",
3787 0 : "Amount of layers successfully evicted"
3788 0 : )
3789 0 : .unwrap();
3790 0 :
3791 0 : Self {
3792 0 : tenant_collection_time,
3793 0 : tenant_layer_count,
3794 0 : layers_collected,
3795 0 : layers_selected,
3796 0 : layers_evicted,
3797 0 : }
3798 0 : }
3799 : }
3800 :
3801 : pub(crate) static METRICS: Lazy<Metrics> = Lazy::new(Metrics::default);
3802 : }
3803 :
3804 172 : static TOKIO_EXECUTOR_THREAD_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
3805 172 : register_uint_gauge_vec!(
3806 172 : "pageserver_tokio_executor_thread_configured_count",
3807 172 : "Total number of configued tokio executor threads in the process.
3808 172 : The `setup` label denotes whether we're running with multiple runtimes or a single runtime.",
3809 172 : &["setup"],
3810 172 : )
3811 172 : .unwrap()
3812 172 : });
3813 :
3814 172 : pub(crate) fn set_tokio_runtime_setup(setup: &str, num_threads: NonZeroUsize) {
3815 : static SERIALIZE: std::sync::Mutex<()> = std::sync::Mutex::new(());
3816 172 : let _guard = SERIALIZE.lock().unwrap();
3817 172 : TOKIO_EXECUTOR_THREAD_COUNT.reset();
3818 172 : TOKIO_EXECUTOR_THREAD_COUNT
3819 172 : .get_metric_with_label_values(&[setup])
3820 172 : .unwrap()
3821 172 : .set(u64::try_from(num_threads.get()).unwrap());
3822 172 : }
3823 :
3824 0 : pub fn preinitialize_metrics(conf: &'static PageServerConf) {
3825 0 : set_page_service_config_max_batch_size(&conf.page_service_pipelining);
3826 0 :
3827 0 : // Python tests need these and on some we do alerting.
3828 0 : //
3829 0 : // FIXME(4813): make it so that we have no top level metrics as this fn will easily fall out of
3830 0 : // order:
3831 0 : // - global metrics reside in a Lazy<PageserverMetrics>
3832 0 : // - access via crate::metrics::PS_METRICS.some_metric.inc()
3833 0 : // - could move the statics into TimelineMetrics::new()?
3834 0 :
3835 0 : // counters
3836 0 : [
3837 0 : &UNEXPECTED_ONDEMAND_DOWNLOADS,
3838 0 : &WALRECEIVER_STARTED_CONNECTIONS,
3839 0 : &WALRECEIVER_BROKER_UPDATES,
3840 0 : &WALRECEIVER_CANDIDATES_ADDED,
3841 0 : &WALRECEIVER_CANDIDATES_REMOVED,
3842 0 : &tokio_epoll_uring::THREAD_LOCAL_LAUNCH_FAILURES,
3843 0 : &tokio_epoll_uring::THREAD_LOCAL_LAUNCH_SUCCESSES,
3844 0 : &REMOTE_ONDEMAND_DOWNLOADED_LAYERS,
3845 0 : &REMOTE_ONDEMAND_DOWNLOADED_BYTES,
3846 0 : &CIRCUIT_BREAKERS_BROKEN,
3847 0 : &CIRCUIT_BREAKERS_UNBROKEN,
3848 0 : &PAGE_SERVICE_SMGR_FLUSH_INPROGRESS_MICROS_GLOBAL,
3849 0 : ]
3850 0 : .into_iter()
3851 0 : .for_each(|c| {
3852 0 : Lazy::force(c);
3853 0 : });
3854 0 :
3855 0 : // Deletion queue stats
3856 0 : Lazy::force(&DELETION_QUEUE);
3857 0 :
3858 0 : // Tenant stats
3859 0 : Lazy::force(&TENANT);
3860 0 :
3861 0 : // Tenant manager stats
3862 0 : Lazy::force(&TENANT_MANAGER);
3863 0 :
3864 0 : Lazy::force(&crate::tenant::storage_layer::layer::LAYER_IMPL_METRICS);
3865 0 : Lazy::force(&disk_usage_based_eviction::METRICS);
3866 :
3867 0 : for state_name in pageserver_api::models::TenantState::VARIANTS {
3868 0 : // initialize the metric for all gauges, otherwise the time series might seemingly show
3869 0 : // values from last restart.
3870 0 : TENANT_STATE_METRIC.with_label_values(&[state_name]).set(0);
3871 0 : }
3872 :
3873 : // countervecs
3874 0 : [
3875 0 : &BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT,
3876 0 : &SMGR_QUERY_STARTED_GLOBAL,
3877 0 : ]
3878 0 : .into_iter()
3879 0 : .for_each(|c| {
3880 0 : Lazy::force(c);
3881 0 : });
3882 0 :
3883 0 : // gauges
3884 0 : WALRECEIVER_ACTIVE_MANAGERS.get();
3885 0 :
3886 0 : // histograms
3887 0 : [
3888 0 : &READ_NUM_LAYERS_VISITED,
3889 0 : &VEC_READ_NUM_LAYERS_VISITED,
3890 0 : &WAIT_LSN_TIME,
3891 0 : &WAL_REDO_TIME,
3892 0 : &WAL_REDO_RECORDS_HISTOGRAM,
3893 0 : &WAL_REDO_BYTES_HISTOGRAM,
3894 0 : &WAL_REDO_PROCESS_LAUNCH_DURATION_HISTOGRAM,
3895 0 : &PAGE_SERVICE_BATCH_SIZE_GLOBAL,
3896 0 : &PAGE_SERVICE_SMGR_BATCH_WAIT_TIME_GLOBAL,
3897 0 : ]
3898 0 : .into_iter()
3899 0 : .for_each(|h| {
3900 0 : Lazy::force(h);
3901 0 : });
3902 0 :
3903 0 : // Custom
3904 0 : Lazy::force(&RECONSTRUCT_TIME);
3905 0 : Lazy::force(&BASEBACKUP_QUERY_TIME);
3906 0 : Lazy::force(&COMPUTE_COMMANDS_COUNTERS);
3907 0 : Lazy::force(&tokio_epoll_uring::THREAD_LOCAL_METRICS_STORAGE);
3908 0 :
3909 0 : tenant_throttling::preinitialize_global_metrics();
3910 0 : }
|
