Line data Source code
1 : use enum_map::EnumMap;
2 : use metrics::{
3 : register_counter_vec, register_gauge_vec, register_histogram, register_histogram_vec,
4 : register_int_counter, register_int_counter_pair_vec, register_int_counter_vec,
5 : register_int_gauge, register_int_gauge_vec, register_uint_gauge, register_uint_gauge_vec,
6 : Counter, CounterVec, Gauge, GaugeVec, Histogram, HistogramVec, IntCounter, IntCounterPair,
7 : IntCounterPairVec, IntCounterVec, IntGauge, IntGaugeVec, UIntGauge, UIntGaugeVec,
8 : };
9 : use once_cell::sync::Lazy;
10 : use pageserver_api::config::{
11 : PageServicePipeliningConfig, PageServicePipeliningConfigPipelined,
12 : PageServiceProtocolPipelinedExecutionStrategy,
13 : };
14 : use pageserver_api::shard::TenantShardId;
15 : use postgres_backend::{is_expected_io_error, QueryError};
16 : use pq_proto::framed::ConnectionError;
17 : use strum::{EnumCount, VariantNames};
18 : use strum_macros::{IntoStaticStr, VariantNames};
19 : use utils::id::TimelineId;
20 :
21 : /// Prometheus histogram buckets (in seconds) for operations in the critical
22 : /// path. In other words, operations that directly affect that latency of user
23 : /// queries.
24 : ///
25 : /// The buckets capture the majority of latencies in the microsecond and
26 : /// millisecond range but also extend far enough up to distinguish "bad" from
27 : /// "really bad".
28 : const CRITICAL_OP_BUCKETS: &[f64] = &[
29 : 0.000_001, 0.000_010, 0.000_100, // 1 us, 10 us, 100 us
30 : 0.001_000, 0.010_000, 0.100_000, // 1 ms, 10 ms, 100 ms
31 : 1.0, 10.0, 100.0, // 1 s, 10 s, 100 s
32 : ];
33 :
34 : // Metrics collected on operations on the storage repository.
35 : #[derive(Debug, VariantNames, IntoStaticStr)]
36 : #[strum(serialize_all = "kebab_case")]
37 : pub(crate) enum StorageTimeOperation {
38 : #[strum(serialize = "layer flush")]
39 : LayerFlush,
40 :
41 : #[strum(serialize = "compact")]
42 : Compact,
43 :
44 : #[strum(serialize = "create images")]
45 : CreateImages,
46 :
47 : #[strum(serialize = "logical size")]
48 : LogicalSize,
49 :
50 : #[strum(serialize = "imitate logical size")]
51 : ImitateLogicalSize,
52 :
53 : #[strum(serialize = "load layer map")]
54 : LoadLayerMap,
55 :
56 : #[strum(serialize = "gc")]
57 : Gc,
58 :
59 : #[strum(serialize = "find gc cutoffs")]
60 : FindGcCutoffs,
61 : }
62 :
63 200 : pub(crate) static STORAGE_TIME_SUM_PER_TIMELINE: Lazy<CounterVec> = Lazy::new(|| {
64 200 : register_counter_vec!(
65 200 : "pageserver_storage_operations_seconds_sum",
66 200 : "Total time spent on storage operations with operation, tenant and timeline dimensions",
67 200 : &["operation", "tenant_id", "shard_id", "timeline_id"],
68 200 : )
69 200 : .expect("failed to define a metric")
70 200 : });
71 :
72 200 : pub(crate) static STORAGE_TIME_COUNT_PER_TIMELINE: Lazy<IntCounterVec> = Lazy::new(|| {
73 200 : register_int_counter_vec!(
74 200 : "pageserver_storage_operations_seconds_count",
75 200 : "Count of storage operations with operation, tenant and timeline dimensions",
76 200 : &["operation", "tenant_id", "shard_id", "timeline_id"],
77 200 : )
78 200 : .expect("failed to define a metric")
79 200 : });
80 :
81 : // Buckets for background operations like compaction, GC, size calculation
82 : const STORAGE_OP_BUCKETS: &[f64] = &[0.010, 0.100, 1.0, 10.0, 100.0, 1000.0];
83 :
84 200 : pub(crate) static STORAGE_TIME_GLOBAL: Lazy<HistogramVec> = Lazy::new(|| {
85 200 : register_histogram_vec!(
86 200 : "pageserver_storage_operations_seconds_global",
87 200 : "Time spent on storage operations",
88 200 : &["operation"],
89 200 : STORAGE_OP_BUCKETS.into(),
90 200 : )
91 200 : .expect("failed to define a metric")
92 200 : });
93 :
94 196 : pub(crate) static VEC_READ_NUM_LAYERS_VISITED: Lazy<Histogram> = Lazy::new(|| {
95 196 : register_histogram!(
96 196 : "pageserver_layers_visited_per_vectored_read_global",
97 196 : "Average number of layers visited to reconstruct one key",
98 196 : vec![1.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0],
99 196 : )
100 196 : .expect("failed to define a metric")
101 196 : });
102 :
103 : // Metrics collected on operations on the storage repository.
104 : #[derive(
105 0 : Clone, Copy, enum_map::Enum, strum_macros::EnumString, strum_macros::Display, IntoStaticStr,
106 : )]
107 : pub(crate) enum GetKind {
108 : Singular,
109 : Vectored,
110 : }
111 :
112 : pub(crate) struct ReconstructTimeMetrics {
113 : singular: Histogram,
114 : vectored: Histogram,
115 : }
116 :
117 198 : pub(crate) static RECONSTRUCT_TIME: Lazy<ReconstructTimeMetrics> = Lazy::new(|| {
118 198 : let inner = register_histogram_vec!(
119 198 : "pageserver_getpage_reconstruct_seconds",
120 198 : "Time spent in reconstruct_value (reconstruct a page from deltas)",
121 198 : &["get_kind"],
122 198 : CRITICAL_OP_BUCKETS.into(),
123 198 : )
124 198 : .expect("failed to define a metric");
125 198 :
126 198 : ReconstructTimeMetrics {
127 198 : singular: inner.with_label_values(&[GetKind::Singular.into()]),
128 198 : vectored: inner.with_label_values(&[GetKind::Vectored.into()]),
129 198 : }
130 198 : });
131 :
132 : impl ReconstructTimeMetrics {
133 627593 : pub(crate) fn for_get_kind(&self, get_kind: GetKind) -> &Histogram {
134 627593 : match get_kind {
135 627097 : GetKind::Singular => &self.singular,
136 496 : GetKind::Vectored => &self.vectored,
137 : }
138 627593 : }
139 : }
140 :
141 : pub(crate) struct ReconstructDataTimeMetrics {
142 : singular: Histogram,
143 : vectored: Histogram,
144 : }
145 :
146 : impl ReconstructDataTimeMetrics {
147 627609 : pub(crate) fn for_get_kind(&self, get_kind: GetKind) -> &Histogram {
148 627609 : match get_kind {
149 627113 : GetKind::Singular => &self.singular,
150 496 : GetKind::Vectored => &self.vectored,
151 : }
152 627609 : }
153 : }
154 :
155 198 : pub(crate) static GET_RECONSTRUCT_DATA_TIME: Lazy<ReconstructDataTimeMetrics> = Lazy::new(|| {
156 198 : let inner = register_histogram_vec!(
157 198 : "pageserver_getpage_get_reconstruct_data_seconds",
158 198 : "Time spent in get_reconstruct_value_data",
159 198 : &["get_kind"],
160 198 : CRITICAL_OP_BUCKETS.into(),
161 198 : )
162 198 : .expect("failed to define a metric");
163 198 :
164 198 : ReconstructDataTimeMetrics {
165 198 : singular: inner.with_label_values(&[GetKind::Singular.into()]),
166 198 : vectored: inner.with_label_values(&[GetKind::Vectored.into()]),
167 198 : }
168 198 : });
169 :
170 : pub(crate) struct GetVectoredLatency {
171 : map: EnumMap<TaskKind, Option<Histogram>>,
172 : }
173 :
174 : #[allow(dead_code)]
175 : pub(crate) struct ScanLatency {
176 : map: EnumMap<TaskKind, Option<Histogram>>,
177 : }
178 :
179 : impl GetVectoredLatency {
180 : // Only these task types perform vectored gets. Filter all other tasks out to reduce total
181 : // cardinality of the metric.
182 : const TRACKED_TASK_KINDS: [TaskKind; 2] = [TaskKind::Compaction, TaskKind::PageRequestHandler];
183 :
184 19678 : pub(crate) fn for_task_kind(&self, task_kind: TaskKind) -> Option<&Histogram> {
185 19678 : self.map[task_kind].as_ref()
186 19678 : }
187 : }
188 :
189 : impl ScanLatency {
190 : // Only these task types perform vectored gets. Filter all other tasks out to reduce total
191 : // cardinality of the metric.
192 : const TRACKED_TASK_KINDS: [TaskKind; 1] = [TaskKind::PageRequestHandler];
193 :
194 12 : pub(crate) fn for_task_kind(&self, task_kind: TaskKind) -> Option<&Histogram> {
195 12 : self.map[task_kind].as_ref()
196 12 : }
197 : }
198 :
199 : pub(crate) struct ScanLatencyOngoingRecording<'a> {
200 : parent: &'a Histogram,
201 : start: std::time::Instant,
202 : }
203 :
204 : impl<'a> ScanLatencyOngoingRecording<'a> {
205 0 : pub(crate) fn start_recording(parent: &'a Histogram) -> ScanLatencyOngoingRecording<'a> {
206 0 : let start = Instant::now();
207 0 : ScanLatencyOngoingRecording { parent, start }
208 0 : }
209 :
210 0 : pub(crate) fn observe(self) {
211 0 : let elapsed = self.start.elapsed();
212 0 : self.parent.observe(elapsed.as_secs_f64());
213 0 : }
214 : }
215 :
216 192 : pub(crate) static GET_VECTORED_LATENCY: Lazy<GetVectoredLatency> = Lazy::new(|| {
217 192 : let inner = register_histogram_vec!(
218 192 : "pageserver_get_vectored_seconds",
219 192 : "Time spent in get_vectored.",
220 192 : &["task_kind"],
221 192 : CRITICAL_OP_BUCKETS.into(),
222 192 : )
223 192 : .expect("failed to define a metric");
224 192 :
225 192 : GetVectoredLatency {
226 5952 : map: EnumMap::from_array(std::array::from_fn(|task_kind_idx| {
227 5952 : let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind_idx);
228 5952 :
229 5952 : if GetVectoredLatency::TRACKED_TASK_KINDS.contains(&task_kind) {
230 384 : let task_kind = task_kind.into();
231 384 : Some(inner.with_label_values(&[task_kind]))
232 : } else {
233 5568 : None
234 : }
235 5952 : })),
236 192 : }
237 192 : });
238 :
239 4 : pub(crate) static SCAN_LATENCY: Lazy<ScanLatency> = Lazy::new(|| {
240 4 : let inner = register_histogram_vec!(
241 4 : "pageserver_scan_seconds",
242 4 : "Time spent in scan.",
243 4 : &["task_kind"],
244 4 : CRITICAL_OP_BUCKETS.into(),
245 4 : )
246 4 : .expect("failed to define a metric");
247 4 :
248 4 : ScanLatency {
249 124 : map: EnumMap::from_array(std::array::from_fn(|task_kind_idx| {
250 124 : let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind_idx);
251 124 :
252 124 : if ScanLatency::TRACKED_TASK_KINDS.contains(&task_kind) {
253 4 : let task_kind = task_kind.into();
254 4 : Some(inner.with_label_values(&[task_kind]))
255 : } else {
256 120 : None
257 : }
258 124 : })),
259 4 : }
260 4 : });
261 :
262 : pub(crate) struct PageCacheMetricsForTaskKind {
263 : pub read_accesses_immutable: IntCounter,
264 : pub read_hits_immutable: IntCounter,
265 : }
266 :
267 : pub(crate) struct PageCacheMetrics {
268 : map: EnumMap<TaskKind, EnumMap<PageContentKind, PageCacheMetricsForTaskKind>>,
269 : }
270 :
271 92 : static PAGE_CACHE_READ_HITS: Lazy<IntCounterVec> = Lazy::new(|| {
272 92 : register_int_counter_vec!(
273 92 : "pageserver_page_cache_read_hits_total",
274 92 : "Number of read accesses to the page cache that hit",
275 92 : &["task_kind", "key_kind", "content_kind", "hit_kind"]
276 92 : )
277 92 : .expect("failed to define a metric")
278 92 : });
279 :
280 92 : static PAGE_CACHE_READ_ACCESSES: Lazy<IntCounterVec> = Lazy::new(|| {
281 92 : register_int_counter_vec!(
282 92 : "pageserver_page_cache_read_accesses_total",
283 92 : "Number of read accesses to the page cache",
284 92 : &["task_kind", "key_kind", "content_kind"]
285 92 : )
286 92 : .expect("failed to define a metric")
287 92 : });
288 :
289 92 : pub(crate) static PAGE_CACHE: Lazy<PageCacheMetrics> = Lazy::new(|| PageCacheMetrics {
290 2852 : map: EnumMap::from_array(std::array::from_fn(|task_kind| {
291 2852 : let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind);
292 2852 : let task_kind: &'static str = task_kind.into();
293 22816 : EnumMap::from_array(std::array::from_fn(|content_kind| {
294 22816 : let content_kind = <PageContentKind as enum_map::Enum>::from_usize(content_kind);
295 22816 : let content_kind: &'static str = content_kind.into();
296 22816 : PageCacheMetricsForTaskKind {
297 22816 : read_accesses_immutable: {
298 22816 : PAGE_CACHE_READ_ACCESSES
299 22816 : .get_metric_with_label_values(&[task_kind, "immutable", content_kind])
300 22816 : .unwrap()
301 22816 : },
302 22816 :
303 22816 : read_hits_immutable: {
304 22816 : PAGE_CACHE_READ_HITS
305 22816 : .get_metric_with_label_values(&[task_kind, "immutable", content_kind, "-"])
306 22816 : .unwrap()
307 22816 : },
308 22816 : }
309 22816 : }))
310 2852 : })),
311 92 : });
312 :
313 : impl PageCacheMetrics {
314 972060 : pub(crate) fn for_ctx(&self, ctx: &RequestContext) -> &PageCacheMetricsForTaskKind {
315 972060 : &self.map[ctx.task_kind()][ctx.page_content_kind()]
316 972060 : }
317 : }
318 :
319 : pub(crate) struct PageCacheSizeMetrics {
320 : pub max_bytes: UIntGauge,
321 :
322 : pub current_bytes_immutable: UIntGauge,
323 : }
324 :
325 92 : static PAGE_CACHE_SIZE_CURRENT_BYTES: Lazy<UIntGaugeVec> = Lazy::new(|| {
326 92 : register_uint_gauge_vec!(
327 92 : "pageserver_page_cache_size_current_bytes",
328 92 : "Current size of the page cache in bytes, by key kind",
329 92 : &["key_kind"]
330 92 : )
331 92 : .expect("failed to define a metric")
332 92 : });
333 :
334 : pub(crate) static PAGE_CACHE_SIZE: Lazy<PageCacheSizeMetrics> =
335 92 : Lazy::new(|| PageCacheSizeMetrics {
336 92 : max_bytes: {
337 92 : register_uint_gauge!(
338 92 : "pageserver_page_cache_size_max_bytes",
339 92 : "Maximum size of the page cache in bytes"
340 92 : )
341 92 : .expect("failed to define a metric")
342 92 : },
343 92 : current_bytes_immutable: {
344 92 : PAGE_CACHE_SIZE_CURRENT_BYTES
345 92 : .get_metric_with_label_values(&["immutable"])
346 92 : .unwrap()
347 92 : },
348 92 : });
349 :
350 : pub(crate) mod page_cache_eviction_metrics {
351 : use std::num::NonZeroUsize;
352 :
353 : use metrics::{register_int_counter_vec, IntCounter, IntCounterVec};
354 : use once_cell::sync::Lazy;
355 :
356 : #[derive(Clone, Copy)]
357 : pub(crate) enum Outcome {
358 : FoundSlotUnused { iters: NonZeroUsize },
359 : FoundSlotEvicted { iters: NonZeroUsize },
360 : ItersExceeded { iters: NonZeroUsize },
361 : }
362 :
363 92 : static ITERS_TOTAL_VEC: Lazy<IntCounterVec> = Lazy::new(|| {
364 92 : register_int_counter_vec!(
365 92 : "pageserver_page_cache_find_victim_iters_total",
366 92 : "Counter for the number of iterations in the find_victim loop",
367 92 : &["outcome"],
368 92 : )
369 92 : .expect("failed to define a metric")
370 92 : });
371 :
372 92 : static CALLS_VEC: Lazy<IntCounterVec> = Lazy::new(|| {
373 92 : register_int_counter_vec!(
374 92 : "pageserver_page_cache_find_victim_calls",
375 92 : "Incremented at the end of each find_victim() call.\
376 92 : Filter by outcome to get e.g., eviction rate.",
377 92 : &["outcome"]
378 92 : )
379 92 : .unwrap()
380 92 : });
381 :
382 32181 : pub(crate) fn observe(outcome: Outcome) {
383 : macro_rules! dry {
384 : ($label:literal, $iters:expr) => {{
385 : static LABEL: &'static str = $label;
386 : static ITERS_TOTAL: Lazy<IntCounter> =
387 112 : Lazy::new(|| ITERS_TOTAL_VEC.with_label_values(&[LABEL]));
388 : static CALLS: Lazy<IntCounter> =
389 112 : Lazy::new(|| CALLS_VEC.with_label_values(&[LABEL]));
390 : ITERS_TOTAL.inc_by(($iters.get()) as u64);
391 : CALLS.inc();
392 : }};
393 : }
394 32181 : match outcome {
395 1636 : Outcome::FoundSlotUnused { iters } => dry!("found_empty", iters),
396 30545 : Outcome::FoundSlotEvicted { iters } => {
397 30545 : dry!("found_evicted", iters)
398 : }
399 0 : Outcome::ItersExceeded { iters } => {
400 0 : dry!("err_iters_exceeded", iters);
401 0 : super::page_cache_errors_inc(super::PageCacheErrorKind::EvictIterLimit);
402 0 : }
403 : }
404 32181 : }
405 : }
406 :
407 0 : static PAGE_CACHE_ERRORS: Lazy<IntCounterVec> = Lazy::new(|| {
408 0 : register_int_counter_vec!(
409 0 : "page_cache_errors_total",
410 0 : "Number of timeouts while acquiring a pinned slot in the page cache",
411 0 : &["error_kind"]
412 0 : )
413 0 : .expect("failed to define a metric")
414 0 : });
415 :
416 : #[derive(IntoStaticStr)]
417 : #[strum(serialize_all = "kebab_case")]
418 : pub(crate) enum PageCacheErrorKind {
419 : AcquirePinnedSlotTimeout,
420 : EvictIterLimit,
421 : }
422 :
423 0 : pub(crate) fn page_cache_errors_inc(error_kind: PageCacheErrorKind) {
424 0 : PAGE_CACHE_ERRORS
425 0 : .get_metric_with_label_values(&[error_kind.into()])
426 0 : .unwrap()
427 0 : .inc();
428 0 : }
429 :
430 20 : pub(crate) static WAIT_LSN_TIME: Lazy<Histogram> = Lazy::new(|| {
431 20 : register_histogram!(
432 20 : "pageserver_wait_lsn_seconds",
433 20 : "Time spent waiting for WAL to arrive",
434 20 : CRITICAL_OP_BUCKETS.into(),
435 20 : )
436 20 : .expect("failed to define a metric")
437 20 : });
438 :
439 200 : static FLUSH_WAIT_UPLOAD_TIME: Lazy<GaugeVec> = Lazy::new(|| {
440 200 : register_gauge_vec!(
441 200 : "pageserver_flush_wait_upload_seconds",
442 200 : "Time spent waiting for preceding uploads during layer flush",
443 200 : &["tenant_id", "shard_id", "timeline_id"]
444 200 : )
445 200 : .expect("failed to define a metric")
446 200 : });
447 :
448 200 : static LAST_RECORD_LSN: Lazy<IntGaugeVec> = Lazy::new(|| {
449 200 : register_int_gauge_vec!(
450 200 : "pageserver_last_record_lsn",
451 200 : "Last record LSN grouped by timeline",
452 200 : &["tenant_id", "shard_id", "timeline_id"]
453 200 : )
454 200 : .expect("failed to define a metric")
455 200 : });
456 :
457 200 : static DISK_CONSISTENT_LSN: Lazy<IntGaugeVec> = Lazy::new(|| {
458 200 : register_int_gauge_vec!(
459 200 : "pageserver_disk_consistent_lsn",
460 200 : "Disk consistent LSN grouped by timeline",
461 200 : &["tenant_id", "shard_id", "timeline_id"]
462 200 : )
463 200 : .expect("failed to define a metric")
464 200 : });
465 :
466 200 : pub(crate) static PROJECTED_REMOTE_CONSISTENT_LSN: Lazy<UIntGaugeVec> = Lazy::new(|| {
467 200 : register_uint_gauge_vec!(
468 200 : "pageserver_projected_remote_consistent_lsn",
469 200 : "Projected remote consistent LSN grouped by timeline",
470 200 : &["tenant_id", "shard_id", "timeline_id"]
471 200 : )
472 200 : .expect("failed to define a metric")
473 200 : });
474 :
475 200 : static PITR_HISTORY_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
476 200 : register_uint_gauge_vec!(
477 200 : "pageserver_pitr_history_size",
478 200 : "Data written since PITR cutoff on this timeline",
479 200 : &["tenant_id", "shard_id", "timeline_id"]
480 200 : )
481 200 : .expect("failed to define a metric")
482 200 : });
483 :
484 0 : #[derive(strum_macros::EnumString, strum_macros::Display, strum_macros::IntoStaticStr)]
485 : #[strum(serialize_all = "kebab_case")]
486 : pub(crate) enum MetricLayerKind {
487 : Delta,
488 : Image,
489 : }
490 :
491 200 : static TIMELINE_LAYER_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
492 200 : register_uint_gauge_vec!(
493 200 : "pageserver_layer_bytes",
494 200 : "Sum of layer physical sizes in bytes",
495 200 : &["tenant_id", "shard_id", "timeline_id", "kind"]
496 200 : )
497 200 : .expect("failed to define a metric")
498 200 : });
499 :
500 200 : static TIMELINE_LAYER_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
501 200 : register_uint_gauge_vec!(
502 200 : "pageserver_layer_count",
503 200 : "Number of layers that exist",
504 200 : &["tenant_id", "shard_id", "timeline_id", "kind"]
505 200 : )
506 200 : .expect("failed to define a metric")
507 200 : });
508 :
509 200 : static TIMELINE_ARCHIVE_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
510 200 : register_uint_gauge_vec!(
511 200 : "pageserver_archive_size",
512 200 : "Timeline's logical size if it is considered eligible for archival (outside PITR window), else zero",
513 200 : &["tenant_id", "shard_id", "timeline_id"]
514 200 : )
515 200 : .expect("failed to define a metric")
516 200 : });
517 :
518 200 : static STANDBY_HORIZON: Lazy<IntGaugeVec> = Lazy::new(|| {
519 200 : register_int_gauge_vec!(
520 200 : "pageserver_standby_horizon",
521 200 : "Standby apply LSN for which GC is hold off, by timeline.",
522 200 : &["tenant_id", "shard_id", "timeline_id"]
523 200 : )
524 200 : .expect("failed to define a metric")
525 200 : });
526 :
527 200 : static RESIDENT_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
528 200 : register_uint_gauge_vec!(
529 200 : "pageserver_resident_physical_size",
530 200 : "The size of the layer files present in the pageserver's filesystem, for attached locations.",
531 200 : &["tenant_id", "shard_id", "timeline_id"]
532 200 : )
533 200 : .expect("failed to define a metric")
534 200 : });
535 :
536 200 : static VISIBLE_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
537 200 : register_uint_gauge_vec!(
538 200 : "pageserver_visible_physical_size",
539 200 : "The size of the layer files present in the pageserver's filesystem.",
540 200 : &["tenant_id", "shard_id", "timeline_id"]
541 200 : )
542 200 : .expect("failed to define a metric")
543 200 : });
544 :
545 196 : pub(crate) static RESIDENT_PHYSICAL_SIZE_GLOBAL: Lazy<UIntGauge> = Lazy::new(|| {
546 196 : register_uint_gauge!(
547 196 : "pageserver_resident_physical_size_global",
548 196 : "Like `pageserver_resident_physical_size`, but without tenant/timeline dimensions."
549 196 : )
550 196 : .expect("failed to define a metric")
551 196 : });
552 :
553 200 : static REMOTE_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
554 200 : register_uint_gauge_vec!(
555 200 : "pageserver_remote_physical_size",
556 200 : "The size of the layer files present in the remote storage that are listed in the remote index_part.json.",
557 200 : // Corollary: If any files are missing from the index part, they won't be included here.
558 200 : &["tenant_id", "shard_id", "timeline_id"]
559 200 : )
560 200 : .expect("failed to define a metric")
561 200 : });
562 :
563 200 : static REMOTE_PHYSICAL_SIZE_GLOBAL: Lazy<UIntGauge> = Lazy::new(|| {
564 200 : register_uint_gauge!(
565 200 : "pageserver_remote_physical_size_global",
566 200 : "Like `pageserver_remote_physical_size`, but without tenant/timeline dimensions."
567 200 : )
568 200 : .expect("failed to define a metric")
569 200 : });
570 :
571 4 : pub(crate) static REMOTE_ONDEMAND_DOWNLOADED_LAYERS: Lazy<IntCounter> = Lazy::new(|| {
572 4 : register_int_counter!(
573 4 : "pageserver_remote_ondemand_downloaded_layers_total",
574 4 : "Total on-demand downloaded layers"
575 4 : )
576 4 : .unwrap()
577 4 : });
578 :
579 4 : pub(crate) static REMOTE_ONDEMAND_DOWNLOADED_BYTES: Lazy<IntCounter> = Lazy::new(|| {
580 4 : register_int_counter!(
581 4 : "pageserver_remote_ondemand_downloaded_bytes_total",
582 4 : "Total bytes of layers on-demand downloaded",
583 4 : )
584 4 : .unwrap()
585 4 : });
586 :
587 200 : static CURRENT_LOGICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
588 200 : register_uint_gauge_vec!(
589 200 : "pageserver_current_logical_size",
590 200 : "Current logical size grouped by timeline",
591 200 : &["tenant_id", "shard_id", "timeline_id"]
592 200 : )
593 200 : .expect("failed to define current logical size metric")
594 200 : });
595 :
596 200 : static AUX_FILE_SIZE: Lazy<IntGaugeVec> = Lazy::new(|| {
597 200 : register_int_gauge_vec!(
598 200 : "pageserver_aux_file_estimated_size",
599 200 : "The size of all aux files for a timeline in aux file v2 store.",
600 200 : &["tenant_id", "shard_id", "timeline_id"]
601 200 : )
602 200 : .expect("failed to define a metric")
603 200 : });
604 :
605 200 : static VALID_LSN_LEASE_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
606 200 : register_uint_gauge_vec!(
607 200 : "pageserver_valid_lsn_lease_count",
608 200 : "The number of valid leases after refreshing gc info.",
609 200 : &["tenant_id", "shard_id", "timeline_id"],
610 200 : )
611 200 : .expect("failed to define a metric")
612 200 : });
613 :
614 0 : pub(crate) static CIRCUIT_BREAKERS_BROKEN: Lazy<IntCounter> = Lazy::new(|| {
615 0 : register_int_counter!(
616 0 : "pageserver_circuit_breaker_broken",
617 0 : "How many times a circuit breaker has broken"
618 0 : )
619 0 : .expect("failed to define a metric")
620 0 : });
621 :
622 0 : pub(crate) static CIRCUIT_BREAKERS_UNBROKEN: Lazy<IntCounter> = Lazy::new(|| {
623 0 : register_int_counter!(
624 0 : "pageserver_circuit_breaker_unbroken",
625 0 : "How many times a circuit breaker has been un-broken (recovered)"
626 0 : )
627 0 : .expect("failed to define a metric")
628 0 : });
629 :
630 192 : pub(crate) static COMPRESSION_IMAGE_INPUT_BYTES: Lazy<IntCounter> = Lazy::new(|| {
631 192 : register_int_counter!(
632 192 : "pageserver_compression_image_in_bytes_total",
633 192 : "Size of data written into image layers before compression"
634 192 : )
635 192 : .expect("failed to define a metric")
636 192 : });
637 :
638 192 : pub(crate) static COMPRESSION_IMAGE_INPUT_BYTES_CONSIDERED: Lazy<IntCounter> = Lazy::new(|| {
639 192 : register_int_counter!(
640 192 : "pageserver_compression_image_in_bytes_considered",
641 192 : "Size of potentially compressible data written into image layers before compression"
642 192 : )
643 192 : .expect("failed to define a metric")
644 192 : });
645 :
646 192 : pub(crate) static COMPRESSION_IMAGE_INPUT_BYTES_CHOSEN: Lazy<IntCounter> = Lazy::new(|| {
647 192 : register_int_counter!(
648 192 : "pageserver_compression_image_in_bytes_chosen",
649 192 : "Size of data whose compressed form was written into image layers"
650 192 : )
651 192 : .expect("failed to define a metric")
652 192 : });
653 :
654 192 : pub(crate) static COMPRESSION_IMAGE_OUTPUT_BYTES: Lazy<IntCounter> = Lazy::new(|| {
655 192 : register_int_counter!(
656 192 : "pageserver_compression_image_out_bytes_total",
657 192 : "Size of compressed image layer written"
658 192 : )
659 192 : .expect("failed to define a metric")
660 192 : });
661 :
662 10 : pub(crate) static RELSIZE_CACHE_ENTRIES: Lazy<UIntGauge> = Lazy::new(|| {
663 10 : register_uint_gauge!(
664 10 : "pageserver_relsize_cache_entries",
665 10 : "Number of entries in the relation size cache",
666 10 : )
667 10 : .expect("failed to define a metric")
668 10 : });
669 :
670 10 : pub(crate) static RELSIZE_CACHE_HITS: Lazy<IntCounter> = Lazy::new(|| {
671 10 : register_int_counter!("pageserver_relsize_cache_hits", "Relation size cache hits",)
672 10 : .expect("failed to define a metric")
673 10 : });
674 :
675 10 : pub(crate) static RELSIZE_CACHE_MISSES: Lazy<IntCounter> = Lazy::new(|| {
676 10 : register_int_counter!(
677 10 : "pageserver_relsize_cache_misses",
678 10 : "Relation size cache misses",
679 10 : )
680 10 : .expect("failed to define a metric")
681 10 : });
682 :
683 4 : pub(crate) static RELSIZE_CACHE_MISSES_OLD: Lazy<IntCounter> = Lazy::new(|| {
684 4 : register_int_counter!(
685 4 : "pageserver_relsize_cache_misses_old",
686 4 : "Relation size cache misses where the lookup LSN is older than the last relation update"
687 4 : )
688 4 : .expect("failed to define a metric")
689 4 : });
690 :
691 : pub(crate) mod initial_logical_size {
692 : use metrics::{register_int_counter, register_int_counter_vec, IntCounter, IntCounterVec};
693 : use once_cell::sync::Lazy;
694 :
695 : pub(crate) struct StartCalculation(IntCounterVec);
696 200 : pub(crate) static START_CALCULATION: Lazy<StartCalculation> = Lazy::new(|| {
697 200 : StartCalculation(
698 200 : register_int_counter_vec!(
699 200 : "pageserver_initial_logical_size_start_calculation",
700 200 : "Incremented each time we start an initial logical size calculation attempt. \
701 200 : The `circumstances` label provides some additional details.",
702 200 : &["attempt", "circumstances"]
703 200 : )
704 200 : .unwrap(),
705 200 : )
706 200 : });
707 :
708 : struct DropCalculation {
709 : first: IntCounter,
710 : retry: IntCounter,
711 : }
712 :
713 200 : static DROP_CALCULATION: Lazy<DropCalculation> = Lazy::new(|| {
714 200 : let vec = register_int_counter_vec!(
715 200 : "pageserver_initial_logical_size_drop_calculation",
716 200 : "Incremented each time we abort a started size calculation attmpt.",
717 200 : &["attempt"]
718 200 : )
719 200 : .unwrap();
720 200 : DropCalculation {
721 200 : first: vec.with_label_values(&["first"]),
722 200 : retry: vec.with_label_values(&["retry"]),
723 200 : }
724 200 : });
725 :
726 : pub(crate) struct Calculated {
727 : pub(crate) births: IntCounter,
728 : pub(crate) deaths: IntCounter,
729 : }
730 :
731 200 : pub(crate) static CALCULATED: Lazy<Calculated> = Lazy::new(|| Calculated {
732 200 : births: register_int_counter!(
733 200 : "pageserver_initial_logical_size_finish_calculation",
734 200 : "Incremented every time we finish calculation of initial logical size.\
735 200 : If everything is working well, this should happen at most once per Timeline object."
736 200 : )
737 200 : .unwrap(),
738 200 : deaths: register_int_counter!(
739 200 : "pageserver_initial_logical_size_drop_finished_calculation",
740 200 : "Incremented when we drop a finished initial logical size calculation result.\
741 200 : Mainly useful to turn pageserver_initial_logical_size_finish_calculation into a gauge."
742 200 : )
743 200 : .unwrap(),
744 200 : });
745 :
746 : pub(crate) struct OngoingCalculationGuard {
747 : inc_drop_calculation: Option<IntCounter>,
748 : }
749 :
750 : #[derive(strum_macros::IntoStaticStr)]
751 : pub(crate) enum StartCircumstances {
752 : EmptyInitial,
753 : SkippedConcurrencyLimiter,
754 : AfterBackgroundTasksRateLimit,
755 : }
756 :
757 : impl StartCalculation {
758 212 : pub(crate) fn first(&self, circumstances: StartCircumstances) -> OngoingCalculationGuard {
759 212 : let circumstances_label: &'static str = circumstances.into();
760 212 : self.0
761 212 : .with_label_values(&["first", circumstances_label])
762 212 : .inc();
763 212 : OngoingCalculationGuard {
764 212 : inc_drop_calculation: Some(DROP_CALCULATION.first.clone()),
765 212 : }
766 212 : }
767 0 : pub(crate) fn retry(&self, circumstances: StartCircumstances) -> OngoingCalculationGuard {
768 0 : let circumstances_label: &'static str = circumstances.into();
769 0 : self.0
770 0 : .with_label_values(&["retry", circumstances_label])
771 0 : .inc();
772 0 : OngoingCalculationGuard {
773 0 : inc_drop_calculation: Some(DROP_CALCULATION.retry.clone()),
774 0 : }
775 0 : }
776 : }
777 :
778 : impl Drop for OngoingCalculationGuard {
779 212 : fn drop(&mut self) {
780 212 : if let Some(counter) = self.inc_drop_calculation.take() {
781 0 : counter.inc();
782 212 : }
783 212 : }
784 : }
785 :
786 : impl OngoingCalculationGuard {
787 212 : pub(crate) fn calculation_result_saved(mut self) -> FinishedCalculationGuard {
788 212 : drop(self.inc_drop_calculation.take());
789 212 : CALCULATED.births.inc();
790 212 : FinishedCalculationGuard {
791 212 : inc_on_drop: CALCULATED.deaths.clone(),
792 212 : }
793 212 : }
794 : }
795 :
796 : pub(crate) struct FinishedCalculationGuard {
797 : inc_on_drop: IntCounter,
798 : }
799 :
800 : impl Drop for FinishedCalculationGuard {
801 6 : fn drop(&mut self) {
802 6 : self.inc_on_drop.inc();
803 6 : }
804 : }
805 :
806 : // context: https://github.com/neondatabase/neon/issues/5963
807 : pub(crate) static TIMELINES_WHERE_WALRECEIVER_GOT_APPROXIMATE_SIZE: Lazy<IntCounter> =
808 0 : Lazy::new(|| {
809 0 : register_int_counter!(
810 0 : "pageserver_initial_logical_size_timelines_where_walreceiver_got_approximate_size",
811 0 : "Counter for the following event: walreceiver calls\
812 0 : Timeline::get_current_logical_size() and it returns `Approximate` for the first time."
813 0 : )
814 0 : .unwrap()
815 0 : });
816 : }
817 :
818 0 : static DIRECTORY_ENTRIES_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
819 0 : register_uint_gauge_vec!(
820 0 : "pageserver_directory_entries_count",
821 0 : "Sum of the entries in pageserver-stored directory listings",
822 0 : &["tenant_id", "shard_id", "timeline_id"]
823 0 : )
824 0 : .expect("failed to define a metric")
825 0 : });
826 :
827 202 : pub(crate) static TENANT_STATE_METRIC: Lazy<UIntGaugeVec> = Lazy::new(|| {
828 202 : register_uint_gauge_vec!(
829 202 : "pageserver_tenant_states_count",
830 202 : "Count of tenants per state",
831 202 : &["state"]
832 202 : )
833 202 : .expect("Failed to register pageserver_tenant_states_count metric")
834 202 : });
835 :
836 : /// A set of broken tenants.
837 : ///
838 : /// These are expected to be so rare that a set is fine. Set as in a new timeseries per each broken
839 : /// tenant.
840 10 : pub(crate) static BROKEN_TENANTS_SET: Lazy<UIntGaugeVec> = Lazy::new(|| {
841 10 : register_uint_gauge_vec!(
842 10 : "pageserver_broken_tenants_count",
843 10 : "Set of broken tenants",
844 10 : &["tenant_id", "shard_id"]
845 10 : )
846 10 : .expect("Failed to register pageserver_tenant_states_count metric")
847 10 : });
848 :
849 6 : pub(crate) static TENANT_SYNTHETIC_SIZE_METRIC: Lazy<UIntGaugeVec> = Lazy::new(|| {
850 6 : register_uint_gauge_vec!(
851 6 : "pageserver_tenant_synthetic_cached_size_bytes",
852 6 : "Synthetic size of each tenant in bytes",
853 6 : &["tenant_id"]
854 6 : )
855 6 : .expect("Failed to register pageserver_tenant_synthetic_cached_size_bytes metric")
856 6 : });
857 :
858 0 : pub(crate) static EVICTION_ITERATION_DURATION: Lazy<HistogramVec> = Lazy::new(|| {
859 0 : register_histogram_vec!(
860 0 : "pageserver_eviction_iteration_duration_seconds_global",
861 0 : "Time spent on a single eviction iteration",
862 0 : &["period_secs", "threshold_secs"],
863 0 : STORAGE_OP_BUCKETS.into(),
864 0 : )
865 0 : .expect("failed to define a metric")
866 0 : });
867 :
868 200 : static EVICTIONS: Lazy<IntCounterVec> = Lazy::new(|| {
869 200 : register_int_counter_vec!(
870 200 : "pageserver_evictions",
871 200 : "Number of layers evicted from the pageserver",
872 200 : &["tenant_id", "shard_id", "timeline_id"]
873 200 : )
874 200 : .expect("failed to define a metric")
875 200 : });
876 :
877 200 : static EVICTIONS_WITH_LOW_RESIDENCE_DURATION: Lazy<IntCounterVec> = Lazy::new(|| {
878 200 : register_int_counter_vec!(
879 200 : "pageserver_evictions_with_low_residence_duration",
880 200 : "If a layer is evicted that was resident for less than `low_threshold`, it is counted to this counter. \
881 200 : Residence duration is determined using the `residence_duration_data_source`.",
882 200 : &["tenant_id", "shard_id", "timeline_id", "residence_duration_data_source", "low_threshold_secs"]
883 200 : )
884 200 : .expect("failed to define a metric")
885 200 : });
886 :
887 0 : pub(crate) static UNEXPECTED_ONDEMAND_DOWNLOADS: Lazy<IntCounter> = Lazy::new(|| {
888 0 : register_int_counter!(
889 0 : "pageserver_unexpected_ondemand_downloads_count",
890 0 : "Number of unexpected on-demand downloads. \
891 0 : We log more context for each increment, so, forgo any labels in this metric.",
892 0 : )
893 0 : .expect("failed to define a metric")
894 0 : });
895 :
896 : /// How long did we take to start up? Broken down by labels to describe
897 : /// different phases of startup.
898 0 : pub static STARTUP_DURATION: Lazy<GaugeVec> = Lazy::new(|| {
899 0 : register_gauge_vec!(
900 0 : "pageserver_startup_duration_seconds",
901 0 : "Time taken by phases of pageserver startup, in seconds",
902 0 : &["phase"]
903 0 : )
904 0 : .expect("Failed to register pageserver_startup_duration_seconds metric")
905 0 : });
906 :
907 0 : pub static STARTUP_IS_LOADING: Lazy<UIntGauge> = Lazy::new(|| {
908 0 : register_uint_gauge!(
909 0 : "pageserver_startup_is_loading",
910 0 : "1 while in initial startup load of tenants, 0 at other times"
911 0 : )
912 0 : .expect("Failed to register pageserver_startup_is_loading")
913 0 : });
914 :
915 196 : pub(crate) static TIMELINE_EPHEMERAL_BYTES: Lazy<UIntGauge> = Lazy::new(|| {
916 196 : register_uint_gauge!(
917 196 : "pageserver_timeline_ephemeral_bytes",
918 196 : "Total number of bytes in ephemeral layers, summed for all timelines. Approximate, lazily updated."
919 196 : )
920 196 : .expect("Failed to register metric")
921 196 : });
922 :
923 : /// Metrics related to the lifecycle of a [`crate::tenant::Tenant`] object: things
924 : /// like how long it took to load.
925 : ///
926 : /// Note that these are process-global metrics, _not_ per-tenant metrics. Per-tenant
927 : /// metrics are rather expensive, and usually fine grained stuff makes more sense
928 : /// at a timeline level than tenant level.
929 : pub(crate) struct TenantMetrics {
930 : /// How long did tenants take to go from construction to active state?
931 : pub(crate) activation: Histogram,
932 : pub(crate) preload: Histogram,
933 : pub(crate) attach: Histogram,
934 :
935 : /// How many tenants are included in the initial startup of the pagesrever?
936 : pub(crate) startup_scheduled: IntCounter,
937 : pub(crate) startup_complete: IntCounter,
938 : }
939 :
940 0 : pub(crate) static TENANT: Lazy<TenantMetrics> = Lazy::new(|| {
941 0 : TenantMetrics {
942 0 : activation: register_histogram!(
943 0 : "pageserver_tenant_activation_seconds",
944 0 : "Time taken by tenants to activate, in seconds",
945 0 : CRITICAL_OP_BUCKETS.into()
946 0 : )
947 0 : .expect("Failed to register metric"),
948 0 : preload: register_histogram!(
949 0 : "pageserver_tenant_preload_seconds",
950 0 : "Time taken by tenants to load remote metadata on startup/attach, in seconds",
951 0 : CRITICAL_OP_BUCKETS.into()
952 0 : )
953 0 : .expect("Failed to register metric"),
954 0 : attach: register_histogram!(
955 0 : "pageserver_tenant_attach_seconds",
956 0 : "Time taken by tenants to intialize, after remote metadata is already loaded",
957 0 : CRITICAL_OP_BUCKETS.into()
958 0 : )
959 0 : .expect("Failed to register metric"),
960 0 : startup_scheduled: register_int_counter!(
961 0 : "pageserver_tenant_startup_scheduled",
962 0 : "Number of tenants included in pageserver startup (doesn't count tenants attached later)"
963 0 : ).expect("Failed to register metric"),
964 0 : startup_complete: register_int_counter!(
965 0 : "pageserver_tenant_startup_complete",
966 0 : "Number of tenants that have completed warm-up, or activated on-demand during initial startup: \
967 0 : should eventually reach `pageserver_tenant_startup_scheduled_total`. Does not include broken \
968 0 : tenants: such cases will lead to this metric never reaching the scheduled count."
969 0 : ).expect("Failed to register metric"),
970 0 : }
971 0 : });
972 :
973 : /// Each `Timeline`'s [`EVICTIONS_WITH_LOW_RESIDENCE_DURATION`] metric.
974 : #[derive(Debug)]
975 : pub(crate) struct EvictionsWithLowResidenceDuration {
976 : data_source: &'static str,
977 : threshold: Duration,
978 : counter: Option<IntCounter>,
979 : }
980 :
981 : pub(crate) struct EvictionsWithLowResidenceDurationBuilder {
982 : data_source: &'static str,
983 : threshold: Duration,
984 : }
985 :
986 : impl EvictionsWithLowResidenceDurationBuilder {
987 446 : pub fn new(data_source: &'static str, threshold: Duration) -> Self {
988 446 : Self {
989 446 : data_source,
990 446 : threshold,
991 446 : }
992 446 : }
993 :
994 446 : fn build(
995 446 : &self,
996 446 : tenant_id: &str,
997 446 : shard_id: &str,
998 446 : timeline_id: &str,
999 446 : ) -> EvictionsWithLowResidenceDuration {
1000 446 : let counter = EVICTIONS_WITH_LOW_RESIDENCE_DURATION
1001 446 : .get_metric_with_label_values(&[
1002 446 : tenant_id,
1003 446 : shard_id,
1004 446 : timeline_id,
1005 446 : self.data_source,
1006 446 : &EvictionsWithLowResidenceDuration::threshold_label_value(self.threshold),
1007 446 : ])
1008 446 : .unwrap();
1009 446 : EvictionsWithLowResidenceDuration {
1010 446 : data_source: self.data_source,
1011 446 : threshold: self.threshold,
1012 446 : counter: Some(counter),
1013 446 : }
1014 446 : }
1015 : }
1016 :
1017 : impl EvictionsWithLowResidenceDuration {
1018 456 : fn threshold_label_value(threshold: Duration) -> String {
1019 456 : format!("{}", threshold.as_secs())
1020 456 : }
1021 :
1022 4 : pub fn observe(&self, observed_value: Duration) {
1023 4 : if observed_value < self.threshold {
1024 4 : self.counter
1025 4 : .as_ref()
1026 4 : .expect("nobody calls this function after `remove_from_vec`")
1027 4 : .inc();
1028 4 : }
1029 4 : }
1030 :
1031 0 : pub fn change_threshold(
1032 0 : &mut self,
1033 0 : tenant_id: &str,
1034 0 : shard_id: &str,
1035 0 : timeline_id: &str,
1036 0 : new_threshold: Duration,
1037 0 : ) {
1038 0 : if new_threshold == self.threshold {
1039 0 : return;
1040 0 : }
1041 0 : let mut with_new = EvictionsWithLowResidenceDurationBuilder::new(
1042 0 : self.data_source,
1043 0 : new_threshold,
1044 0 : )
1045 0 : .build(tenant_id, shard_id, timeline_id);
1046 0 : std::mem::swap(self, &mut with_new);
1047 0 : with_new.remove(tenant_id, shard_id, timeline_id);
1048 0 : }
1049 :
1050 : // This could be a `Drop` impl, but, we need the `tenant_id` and `timeline_id`.
1051 10 : fn remove(&mut self, tenant_id: &str, shard_id: &str, timeline_id: &str) {
1052 10 : let Some(_counter) = self.counter.take() else {
1053 0 : return;
1054 : };
1055 :
1056 10 : let threshold = Self::threshold_label_value(self.threshold);
1057 10 :
1058 10 : let removed = EVICTIONS_WITH_LOW_RESIDENCE_DURATION.remove_label_values(&[
1059 10 : tenant_id,
1060 10 : shard_id,
1061 10 : timeline_id,
1062 10 : self.data_source,
1063 10 : &threshold,
1064 10 : ]);
1065 10 :
1066 10 : match removed {
1067 0 : Err(e) => {
1068 0 : // this has been hit in staging as
1069 0 : // <https://neondatabase.sentry.io/issues/4142396994/>, but we don't know how.
1070 0 : // because we can be in the drop path already, don't risk:
1071 0 : // - "double-panic => illegal instruction" or
1072 0 : // - future "drop panick => abort"
1073 0 : //
1074 0 : // so just nag: (the error has the labels)
1075 0 : tracing::warn!("failed to remove EvictionsWithLowResidenceDuration, it was already removed? {e:#?}");
1076 : }
1077 : Ok(()) => {
1078 : // to help identify cases where we double-remove the same values, let's log all
1079 : // deletions?
1080 10 : tracing::info!("removed EvictionsWithLowResidenceDuration with {tenant_id}, {timeline_id}, {}, {threshold}", self.data_source);
1081 : }
1082 : }
1083 10 : }
1084 : }
1085 :
1086 : // Metrics collected on disk IO operations
1087 : //
1088 : // Roughly logarithmic scale.
1089 : const STORAGE_IO_TIME_BUCKETS: &[f64] = &[
1090 : 0.000030, // 30 usec
1091 : 0.001000, // 1000 usec
1092 : 0.030, // 30 ms
1093 : 1.000, // 1000 ms
1094 : 30.000, // 30000 ms
1095 : ];
1096 :
1097 : /// VirtualFile fs operation variants.
1098 : ///
1099 : /// Operations:
1100 : /// - open ([`std::fs::OpenOptions::open`])
1101 : /// - close (dropping [`crate::virtual_file::VirtualFile`])
1102 : /// - close-by-replace (close by replacement algorithm)
1103 : /// - read (`read_at`)
1104 : /// - write (`write_at`)
1105 : /// - seek (modify internal position or file length query)
1106 : /// - fsync ([`std::fs::File::sync_all`])
1107 : /// - metadata ([`std::fs::File::metadata`])
1108 : #[derive(
1109 0 : Debug, Clone, Copy, strum_macros::EnumCount, strum_macros::EnumIter, strum_macros::FromRepr,
1110 : )]
1111 : pub(crate) enum StorageIoOperation {
1112 : Open,
1113 : OpenAfterReplace,
1114 : Close,
1115 : CloseByReplace,
1116 : Read,
1117 : Write,
1118 : Seek,
1119 : Fsync,
1120 : Metadata,
1121 : }
1122 :
1123 : impl StorageIoOperation {
1124 2070 : pub fn as_str(&self) -> &'static str {
1125 2070 : match self {
1126 230 : StorageIoOperation::Open => "open",
1127 230 : StorageIoOperation::OpenAfterReplace => "open-after-replace",
1128 230 : StorageIoOperation::Close => "close",
1129 230 : StorageIoOperation::CloseByReplace => "close-by-replace",
1130 230 : StorageIoOperation::Read => "read",
1131 230 : StorageIoOperation::Write => "write",
1132 230 : StorageIoOperation::Seek => "seek",
1133 230 : StorageIoOperation::Fsync => "fsync",
1134 230 : StorageIoOperation::Metadata => "metadata",
1135 : }
1136 2070 : }
1137 : }
1138 :
1139 : /// Tracks time taken by fs operations near VirtualFile.
1140 : #[derive(Debug)]
1141 : pub(crate) struct StorageIoTime {
1142 : metrics: [Histogram; StorageIoOperation::COUNT],
1143 : }
1144 :
1145 : impl StorageIoTime {
1146 230 : fn new() -> Self {
1147 230 : let storage_io_histogram_vec = register_histogram_vec!(
1148 230 : "pageserver_io_operations_seconds",
1149 230 : "Time spent in IO operations",
1150 230 : &["operation"],
1151 230 : STORAGE_IO_TIME_BUCKETS.into()
1152 230 : )
1153 230 : .expect("failed to define a metric");
1154 2070 : let metrics = std::array::from_fn(|i| {
1155 2070 : let op = StorageIoOperation::from_repr(i).unwrap();
1156 2070 : storage_io_histogram_vec
1157 2070 : .get_metric_with_label_values(&[op.as_str()])
1158 2070 : .unwrap()
1159 2070 : });
1160 230 : Self { metrics }
1161 230 : }
1162 :
1163 2006649 : pub(crate) fn get(&self, op: StorageIoOperation) -> &Histogram {
1164 2006649 : &self.metrics[op as usize]
1165 2006649 : }
1166 : }
1167 :
1168 : pub(crate) static STORAGE_IO_TIME_METRIC: Lazy<StorageIoTime> = Lazy::new(StorageIoTime::new);
1169 :
1170 : const STORAGE_IO_SIZE_OPERATIONS: &[&str] = &["read", "write"];
1171 :
1172 : // Needed for the https://neonprod.grafana.net/d/5uK9tHL4k/picking-tenant-for-relocation?orgId=1
1173 226 : pub(crate) static STORAGE_IO_SIZE: Lazy<IntGaugeVec> = Lazy::new(|| {
1174 226 : register_int_gauge_vec!(
1175 226 : "pageserver_io_operations_bytes_total",
1176 226 : "Total amount of bytes read/written in IO operations",
1177 226 : &["operation", "tenant_id", "shard_id", "timeline_id"]
1178 226 : )
1179 226 : .expect("failed to define a metric")
1180 226 : });
1181 :
1182 : #[cfg(not(test))]
1183 : pub(crate) mod virtual_file_descriptor_cache {
1184 : use super::*;
1185 :
1186 0 : pub(crate) static SIZE_MAX: Lazy<UIntGauge> = Lazy::new(|| {
1187 0 : register_uint_gauge!(
1188 0 : "pageserver_virtual_file_descriptor_cache_size_max",
1189 0 : "Maximum number of open file descriptors in the cache."
1190 0 : )
1191 0 : .unwrap()
1192 0 : });
1193 :
1194 : // SIZE_CURRENT: derive it like so:
1195 : // ```
1196 : // sum (pageserver_io_operations_seconds_count{operation=~"^(open|open-after-replace)$")
1197 : // -ignoring(operation)
1198 : // sum(pageserver_io_operations_seconds_count{operation=~"^(close|close-by-replace)$"}
1199 : // ```
1200 : }
1201 :
1202 : #[cfg(not(test))]
1203 : pub(crate) mod virtual_file_io_engine {
1204 : use super::*;
1205 :
1206 0 : pub(crate) static KIND: Lazy<UIntGaugeVec> = Lazy::new(|| {
1207 0 : register_uint_gauge_vec!(
1208 0 : "pageserver_virtual_file_io_engine_kind",
1209 0 : "The configured io engine for VirtualFile",
1210 0 : &["kind"],
1211 0 : )
1212 0 : .unwrap()
1213 0 : });
1214 : }
1215 :
1216 : pub(crate) struct SmgrOpTimer(Option<SmgrOpTimerInner>);
1217 : pub(crate) struct SmgrOpTimerInner {
1218 : global_execution_latency_histo: Histogram,
1219 : per_timeline_execution_latency_histo: Option<Histogram>,
1220 :
1221 : global_batch_wait_time: Histogram,
1222 : per_timeline_batch_wait_time: Histogram,
1223 :
1224 : global_flush_in_progress_micros: IntCounter,
1225 : per_timeline_flush_in_progress_micros: IntCounter,
1226 :
1227 : throttling: Arc<tenant_throttling::Pagestream>,
1228 :
1229 : timings: SmgrOpTimerState,
1230 : }
1231 :
1232 : /// The stages of request processing are represented by the enum variants.
1233 : /// Used as part of [`SmgrOpTimerInner::timings`].
1234 : ///
1235 : /// Request processing calls into the `SmgrOpTimer::observe_*` methods at the
1236 : /// transition points.
1237 : /// These methods bump relevant counters and then update [`SmgrOpTimerInner::timings`]
1238 : /// to the next state.
1239 : ///
1240 : /// Each request goes through every stage, in all configurations.
1241 : ///
1242 : #[derive(Debug)]
1243 : enum SmgrOpTimerState {
1244 : Received {
1245 : // In the future, we may want to track the full time the request spent
1246 : // inside pageserver process (time spent in kernel buffers can't be tracked).
1247 : // `received_at` would be used for that.
1248 : #[allow(dead_code)]
1249 : received_at: Instant,
1250 : },
1251 : Throttling {
1252 : throttle_started_at: Instant,
1253 : },
1254 : Batching {
1255 : throttle_done_at: Instant,
1256 : },
1257 : Executing {
1258 : execution_started_at: Instant,
1259 : },
1260 : Flushing,
1261 : // NB: when adding observation points, remember to update the Drop impl.
1262 : }
1263 :
1264 : // NB: when adding observation points, remember to update the Drop impl.
1265 : impl SmgrOpTimer {
1266 : /// See [`SmgrOpTimerState`] for more context.
1267 0 : pub(crate) fn observe_throttle_start(&mut self, at: Instant) {
1268 0 : let Some(inner) = self.0.as_mut() else {
1269 0 : return;
1270 : };
1271 0 : let SmgrOpTimerState::Received { received_at: _ } = &mut inner.timings else {
1272 0 : return;
1273 : };
1274 0 : inner.throttling.count_accounted_start.inc();
1275 0 : inner.timings = SmgrOpTimerState::Throttling {
1276 0 : throttle_started_at: at,
1277 0 : };
1278 0 : }
1279 :
1280 : /// See [`SmgrOpTimerState`] for more context.
1281 0 : pub(crate) fn observe_throttle_done(&mut self, throttle: ThrottleResult) {
1282 0 : let Some(inner) = self.0.as_mut() else {
1283 0 : return;
1284 : };
1285 : let SmgrOpTimerState::Throttling {
1286 0 : throttle_started_at,
1287 0 : } = &inner.timings
1288 : else {
1289 0 : return;
1290 : };
1291 0 : inner.throttling.count_accounted_finish.inc();
1292 0 : match throttle {
1293 0 : ThrottleResult::NotThrottled { end } => {
1294 0 : inner.timings = SmgrOpTimerState::Batching {
1295 0 : throttle_done_at: end,
1296 0 : };
1297 0 : }
1298 0 : ThrottleResult::Throttled { end } => {
1299 0 : // update metrics
1300 0 : inner.throttling.count_throttled.inc();
1301 0 : inner
1302 0 : .throttling
1303 0 : .wait_time
1304 0 : .inc_by((end - *throttle_started_at).as_micros().try_into().unwrap());
1305 0 : // state transition
1306 0 : inner.timings = SmgrOpTimerState::Batching {
1307 0 : throttle_done_at: end,
1308 0 : };
1309 0 : }
1310 : }
1311 0 : }
1312 :
1313 : /// See [`SmgrOpTimerState`] for more context.
1314 0 : pub(crate) fn observe_execution_start(&mut self, at: Instant) {
1315 0 : let Some(inner) = self.0.as_mut() else {
1316 0 : return;
1317 : };
1318 0 : let SmgrOpTimerState::Batching { throttle_done_at } = &inner.timings else {
1319 0 : return;
1320 : };
1321 : // update metrics
1322 0 : let batch = at - *throttle_done_at;
1323 0 : inner.global_batch_wait_time.observe(batch.as_secs_f64());
1324 0 : inner
1325 0 : .per_timeline_batch_wait_time
1326 0 : .observe(batch.as_secs_f64());
1327 0 : // state transition
1328 0 : inner.timings = SmgrOpTimerState::Executing {
1329 0 : execution_started_at: at,
1330 0 : }
1331 0 : }
1332 :
1333 : /// For all but the first caller, this is a no-op.
1334 : /// The first callers receives Some, subsequent ones None.
1335 : ///
1336 : /// See [`SmgrOpTimerState`] for more context.
1337 0 : pub(crate) fn observe_execution_end_flush_start(
1338 0 : &mut self,
1339 0 : at: Instant,
1340 0 : ) -> Option<SmgrOpFlushInProgress> {
1341 : // NB: unlike the other observe_* methods, this one take()s.
1342 : #[allow(clippy::question_mark)] // maintain similar code pattern.
1343 0 : let Some(mut inner) = self.0.take() else {
1344 0 : return None;
1345 : };
1346 : let SmgrOpTimerState::Executing {
1347 0 : execution_started_at,
1348 0 : } = &inner.timings
1349 : else {
1350 0 : return None;
1351 : };
1352 : // update metrics
1353 0 : let execution = at - *execution_started_at;
1354 0 : inner
1355 0 : .global_execution_latency_histo
1356 0 : .observe(execution.as_secs_f64());
1357 0 : if let Some(per_timeline_execution_latency_histo) =
1358 0 : &inner.per_timeline_execution_latency_histo
1359 0 : {
1360 0 : per_timeline_execution_latency_histo.observe(execution.as_secs_f64());
1361 0 : }
1362 :
1363 : // state transition
1364 0 : inner.timings = SmgrOpTimerState::Flushing;
1365 0 :
1366 0 : // return the flush in progress object which
1367 0 : // will do the remaining metrics updates
1368 0 : let SmgrOpTimerInner {
1369 0 : global_flush_in_progress_micros,
1370 0 : per_timeline_flush_in_progress_micros,
1371 0 : ..
1372 0 : } = inner;
1373 0 : Some(SmgrOpFlushInProgress {
1374 0 : flush_started_at: at,
1375 0 : global_micros: global_flush_in_progress_micros,
1376 0 : per_timeline_micros: per_timeline_flush_in_progress_micros,
1377 0 : })
1378 0 : }
1379 : }
1380 :
1381 : /// The last stage of request processing is serializing and flushing the request
1382 : /// into the TCP connection. We want to make slow flushes observable
1383 : /// _while they are occuring_, so this struct provides a wrapper method [`Self::measure`]
1384 : /// to periodically bump the metric.
1385 : ///
1386 : /// If in the future we decide that we're not interested in live updates, we can
1387 : /// add another `observe_*` method to [`SmgrOpTimer`], follow the existing pattern there,
1388 : /// and remove this struct from the code base.
1389 : pub(crate) struct SmgrOpFlushInProgress {
1390 : flush_started_at: Instant,
1391 : global_micros: IntCounter,
1392 : per_timeline_micros: IntCounter,
1393 : }
1394 :
1395 : impl Drop for SmgrOpTimer {
1396 0 : fn drop(&mut self) {
1397 0 : // In case of early drop, update any of the remaining metrics with
1398 0 : // observations so that (started,finished) counter pairs balance out
1399 0 : // and all counters on the latency path have the the same number of
1400 0 : // observations.
1401 0 : // It's technically lying and it would be better if each metric had
1402 0 : // a separate label or similar for cancelled requests.
1403 0 : // But we don't have that right now and counter pairs balancing
1404 0 : // out is useful when using the metrics in panels and whatnot.
1405 0 : let now = Instant::now();
1406 0 : self.observe_throttle_start(now);
1407 0 : self.observe_throttle_done(ThrottleResult::NotThrottled { end: now });
1408 0 : self.observe_execution_start(now);
1409 0 : self.observe_execution_end_flush_start(now);
1410 0 : }
1411 : }
1412 :
1413 : impl SmgrOpFlushInProgress {
1414 0 : pub(crate) async fn measure<Fut, O>(mut self, mut fut: Fut) -> O
1415 0 : where
1416 0 : Fut: std::future::Future<Output = O>,
1417 0 : {
1418 0 : let mut fut = std::pin::pin!(fut);
1419 0 :
1420 0 : // Whenever observe_guard gets called, or dropped,
1421 0 : // it adds the time elapsed since its last call to metrics.
1422 0 : // Last call is tracked in `now`.
1423 0 : let mut observe_guard = scopeguard::guard(
1424 0 : || {
1425 0 : let now = Instant::now();
1426 0 : let elapsed = now - self.flush_started_at;
1427 0 : self.global_micros
1428 0 : .inc_by(u64::try_from(elapsed.as_micros()).unwrap());
1429 0 : self.per_timeline_micros
1430 0 : .inc_by(u64::try_from(elapsed.as_micros()).unwrap());
1431 0 : self.flush_started_at = now;
1432 0 : },
1433 0 : |mut observe| {
1434 0 : observe();
1435 0 : },
1436 0 : );
1437 :
1438 : loop {
1439 0 : match tokio::time::timeout(Duration::from_secs(10), &mut fut).await {
1440 0 : Ok(v) => return v,
1441 0 : Err(_timeout) => {
1442 0 : (*observe_guard)();
1443 0 : }
1444 : }
1445 : }
1446 0 : }
1447 : }
1448 :
1449 : #[derive(
1450 : Debug,
1451 : Clone,
1452 : Copy,
1453 : IntoStaticStr,
1454 : strum_macros::EnumCount,
1455 0 : strum_macros::EnumIter,
1456 : strum_macros::FromRepr,
1457 : enum_map::Enum,
1458 : )]
1459 : #[strum(serialize_all = "snake_case")]
1460 : pub enum SmgrQueryType {
1461 : GetRelExists,
1462 : GetRelSize,
1463 : GetPageAtLsn,
1464 : GetDbSize,
1465 : GetSlruSegment,
1466 : }
1467 :
1468 : pub(crate) struct SmgrQueryTimePerTimeline {
1469 : global_started: [IntCounter; SmgrQueryType::COUNT],
1470 : global_latency: [Histogram; SmgrQueryType::COUNT],
1471 : per_timeline_getpage_started: IntCounter,
1472 : per_timeline_getpage_latency: Histogram,
1473 : global_batch_size: Histogram,
1474 : per_timeline_batch_size: Histogram,
1475 : global_flush_in_progress_micros: IntCounter,
1476 : per_timeline_flush_in_progress_micros: IntCounter,
1477 : global_batch_wait_time: Histogram,
1478 : per_timeline_batch_wait_time: Histogram,
1479 : throttling: Arc<tenant_throttling::Pagestream>,
1480 : }
1481 :
1482 200 : static SMGR_QUERY_STARTED_GLOBAL: Lazy<IntCounterVec> = Lazy::new(|| {
1483 200 : register_int_counter_vec!(
1484 200 : // it's a counter, but, name is prepared to extend it to a histogram of queue depth
1485 200 : "pageserver_smgr_query_started_global_count",
1486 200 : "Number of smgr queries started, aggregated by query type.",
1487 200 : &["smgr_query_type"],
1488 200 : )
1489 200 : .expect("failed to define a metric")
1490 200 : });
1491 :
1492 200 : static SMGR_QUERY_STARTED_PER_TENANT_TIMELINE: Lazy<IntCounterVec> = Lazy::new(|| {
1493 200 : register_int_counter_vec!(
1494 200 : // it's a counter, but, name is prepared to extend it to a histogram of queue depth
1495 200 : "pageserver_smgr_query_started_count",
1496 200 : "Number of smgr queries started, aggregated by query type and tenant/timeline.",
1497 200 : &["smgr_query_type", "tenant_id", "shard_id", "timeline_id"],
1498 200 : )
1499 200 : .expect("failed to define a metric")
1500 200 : });
1501 :
1502 : // Alias so all histograms recording per-timeline smgr timings use the same buckets.
1503 : static SMGR_QUERY_TIME_PER_TENANT_TIMELINE_BUCKETS: &[f64] = CRITICAL_OP_BUCKETS;
1504 :
1505 200 : static SMGR_QUERY_TIME_PER_TENANT_TIMELINE: Lazy<HistogramVec> = Lazy::new(|| {
1506 200 : register_histogram_vec!(
1507 200 : "pageserver_smgr_query_seconds",
1508 200 : "Time spent _executing_ smgr query handling, excluding batch and throttle delays.",
1509 200 : &["smgr_query_type", "tenant_id", "shard_id", "timeline_id"],
1510 200 : SMGR_QUERY_TIME_PER_TENANT_TIMELINE_BUCKETS.into(),
1511 200 : )
1512 200 : .expect("failed to define a metric")
1513 200 : });
1514 :
1515 200 : static SMGR_QUERY_TIME_GLOBAL_BUCKETS: Lazy<Vec<f64>> = Lazy::new(|| {
1516 200 : [
1517 200 : 1,
1518 200 : 10,
1519 200 : 20,
1520 200 : 40,
1521 200 : 60,
1522 200 : 80,
1523 200 : 100,
1524 200 : 200,
1525 200 : 300,
1526 200 : 400,
1527 200 : 500,
1528 200 : 600,
1529 200 : 700,
1530 200 : 800,
1531 200 : 900,
1532 200 : 1_000, // 1ms
1533 200 : 2_000,
1534 200 : 4_000,
1535 200 : 6_000,
1536 200 : 8_000,
1537 200 : 10_000, // 10ms
1538 200 : 20_000,
1539 200 : 40_000,
1540 200 : 60_000,
1541 200 : 80_000,
1542 200 : 100_000,
1543 200 : 200_000,
1544 200 : 400_000,
1545 200 : 600_000,
1546 200 : 800_000,
1547 200 : 1_000_000, // 1s
1548 200 : 2_000_000,
1549 200 : 4_000_000,
1550 200 : 6_000_000,
1551 200 : 8_000_000,
1552 200 : 10_000_000, // 10s
1553 200 : 20_000_000,
1554 200 : 50_000_000,
1555 200 : 100_000_000,
1556 200 : 200_000_000,
1557 200 : 1_000_000_000, // 1000s
1558 200 : ]
1559 200 : .into_iter()
1560 200 : .map(Duration::from_micros)
1561 8200 : .map(|d| d.as_secs_f64())
1562 200 : .collect()
1563 200 : });
1564 :
1565 200 : static SMGR_QUERY_TIME_GLOBAL: Lazy<HistogramVec> = Lazy::new(|| {
1566 200 : register_histogram_vec!(
1567 200 : "pageserver_smgr_query_seconds_global",
1568 200 : "Like pageserver_smgr_query_seconds, but aggregated to instance level.",
1569 200 : &["smgr_query_type"],
1570 200 : SMGR_QUERY_TIME_GLOBAL_BUCKETS.clone(),
1571 200 : )
1572 200 : .expect("failed to define a metric")
1573 200 : });
1574 :
1575 200 : static PAGE_SERVICE_BATCH_SIZE_BUCKETS_GLOBAL: Lazy<Vec<f64>> = Lazy::new(|| {
1576 200 : (1..=u32::try_from(Timeline::MAX_GET_VECTORED_KEYS).unwrap())
1577 6400 : .map(|v| v.into())
1578 200 : .collect()
1579 200 : });
1580 :
1581 200 : static PAGE_SERVICE_BATCH_SIZE_GLOBAL: Lazy<Histogram> = Lazy::new(|| {
1582 200 : register_histogram!(
1583 200 : "pageserver_page_service_batch_size_global",
1584 200 : "Batch size of pageserver page service requests",
1585 200 : PAGE_SERVICE_BATCH_SIZE_BUCKETS_GLOBAL.clone(),
1586 200 : )
1587 200 : .expect("failed to define a metric")
1588 200 : });
1589 :
1590 200 : static PAGE_SERVICE_BATCH_SIZE_BUCKETS_PER_TIMELINE: Lazy<Vec<f64>> = Lazy::new(|| {
1591 200 : let mut buckets = Vec::new();
1592 1400 : for i in 0.. {
1593 1400 : let bucket = 1 << i;
1594 1400 : if bucket > u32::try_from(Timeline::MAX_GET_VECTORED_KEYS).unwrap() {
1595 200 : break;
1596 1200 : }
1597 1200 : buckets.push(bucket.into());
1598 : }
1599 200 : buckets
1600 200 : });
1601 :
1602 200 : static PAGE_SERVICE_BATCH_SIZE_PER_TENANT_TIMELINE: Lazy<HistogramVec> = Lazy::new(|| {
1603 200 : register_histogram_vec!(
1604 200 : "pageserver_page_service_batch_size",
1605 200 : "Batch size of pageserver page service requests",
1606 200 : &["tenant_id", "shard_id", "timeline_id"],
1607 200 : PAGE_SERVICE_BATCH_SIZE_BUCKETS_PER_TIMELINE.clone()
1608 200 : )
1609 200 : .expect("failed to define a metric")
1610 200 : });
1611 :
1612 0 : pub(crate) static PAGE_SERVICE_CONFIG_MAX_BATCH_SIZE: Lazy<IntGaugeVec> = Lazy::new(|| {
1613 0 : register_int_gauge_vec!(
1614 0 : "pageserver_page_service_config_max_batch_size",
1615 0 : "Configured maximum batch size for the server-side batching functionality of page_service. \
1616 0 : Labels expose more of the configuration parameters.",
1617 0 : &["mode", "execution"]
1618 0 : )
1619 0 : .expect("failed to define a metric")
1620 0 : });
1621 :
1622 0 : fn set_page_service_config_max_batch_size(conf: &PageServicePipeliningConfig) {
1623 0 : PAGE_SERVICE_CONFIG_MAX_BATCH_SIZE.reset();
1624 0 : let (label_values, value) = match conf {
1625 0 : PageServicePipeliningConfig::Serial => (["serial", "-"], 1),
1626 : PageServicePipeliningConfig::Pipelined(PageServicePipeliningConfigPipelined {
1627 0 : max_batch_size,
1628 0 : execution,
1629 0 : }) => {
1630 0 : let mode = "pipelined";
1631 0 : let execution = match execution {
1632 : PageServiceProtocolPipelinedExecutionStrategy::ConcurrentFutures => {
1633 0 : "concurrent-futures"
1634 : }
1635 0 : PageServiceProtocolPipelinedExecutionStrategy::Tasks => "tasks",
1636 : };
1637 0 : ([mode, execution], max_batch_size.get())
1638 : }
1639 : };
1640 0 : PAGE_SERVICE_CONFIG_MAX_BATCH_SIZE
1641 0 : .with_label_values(&label_values)
1642 0 : .set(value.try_into().unwrap());
1643 0 : }
1644 :
1645 200 : static PAGE_SERVICE_SMGR_FLUSH_INPROGRESS_MICROS: Lazy<IntCounterVec> = Lazy::new(|| {
1646 200 : register_int_counter_vec!(
1647 200 : "pageserver_page_service_pagestream_flush_in_progress_micros",
1648 200 : "Counter that sums up the microseconds that a pagestream response was being flushed into the TCP connection. \
1649 200 : If the flush is particularly slow, this counter will be updated periodically to make slow flushes \
1650 200 : easily discoverable in monitoring. \
1651 200 : Hence, this is NOT a completion latency historgram.",
1652 200 : &["tenant_id", "shard_id", "timeline_id"],
1653 200 : )
1654 200 : .expect("failed to define a metric")
1655 200 : });
1656 :
1657 200 : static PAGE_SERVICE_SMGR_FLUSH_INPROGRESS_MICROS_GLOBAL: Lazy<IntCounter> = Lazy::new(|| {
1658 200 : register_int_counter!(
1659 200 : "pageserver_page_service_pagestream_flush_in_progress_micros_global",
1660 200 : "Like pageserver_page_service_pagestream_flush_in_progress_seconds, but instance-wide.",
1661 200 : )
1662 200 : .expect("failed to define a metric")
1663 200 : });
1664 :
1665 200 : static PAGE_SERVICE_SMGR_BATCH_WAIT_TIME: Lazy<HistogramVec> = Lazy::new(|| {
1666 200 : register_histogram_vec!(
1667 200 : "pageserver_page_service_pagestream_batch_wait_time_seconds",
1668 200 : "Time a request spent waiting in its batch until the batch moved to throttle&execution.",
1669 200 : &["tenant_id", "shard_id", "timeline_id"],
1670 200 : SMGR_QUERY_TIME_PER_TENANT_TIMELINE_BUCKETS.into(),
1671 200 : )
1672 200 : .expect("failed to define a metric")
1673 200 : });
1674 :
1675 200 : static PAGE_SERVICE_SMGR_BATCH_WAIT_TIME_GLOBAL: Lazy<Histogram> = Lazy::new(|| {
1676 200 : register_histogram!(
1677 200 : "pageserver_page_service_pagestream_batch_wait_time_seconds_global",
1678 200 : "Like pageserver_page_service_pagestream_batch_wait_time_seconds, but aggregated to instance level.",
1679 200 : SMGR_QUERY_TIME_GLOBAL_BUCKETS.to_vec(),
1680 200 : )
1681 200 : .expect("failed to define a metric")
1682 200 : });
1683 :
1684 : impl SmgrQueryTimePerTimeline {
1685 446 : pub(crate) fn new(
1686 446 : tenant_shard_id: &TenantShardId,
1687 446 : timeline_id: &TimelineId,
1688 446 : pagestream_throttle_metrics: Arc<tenant_throttling::Pagestream>,
1689 446 : ) -> Self {
1690 446 : let tenant_id = tenant_shard_id.tenant_id.to_string();
1691 446 : let shard_slug = format!("{}", tenant_shard_id.shard_slug());
1692 446 : let timeline_id = timeline_id.to_string();
1693 2230 : let global_started = std::array::from_fn(|i| {
1694 2230 : let op = SmgrQueryType::from_repr(i).unwrap();
1695 2230 : SMGR_QUERY_STARTED_GLOBAL
1696 2230 : .get_metric_with_label_values(&[op.into()])
1697 2230 : .unwrap()
1698 2230 : });
1699 2230 : let global_latency = std::array::from_fn(|i| {
1700 2230 : let op = SmgrQueryType::from_repr(i).unwrap();
1701 2230 : SMGR_QUERY_TIME_GLOBAL
1702 2230 : .get_metric_with_label_values(&[op.into()])
1703 2230 : .unwrap()
1704 2230 : });
1705 446 :
1706 446 : let per_timeline_getpage_started = SMGR_QUERY_STARTED_PER_TENANT_TIMELINE
1707 446 : .get_metric_with_label_values(&[
1708 446 : SmgrQueryType::GetPageAtLsn.into(),
1709 446 : &tenant_id,
1710 446 : &shard_slug,
1711 446 : &timeline_id,
1712 446 : ])
1713 446 : .unwrap();
1714 446 : let per_timeline_getpage_latency = SMGR_QUERY_TIME_PER_TENANT_TIMELINE
1715 446 : .get_metric_with_label_values(&[
1716 446 : SmgrQueryType::GetPageAtLsn.into(),
1717 446 : &tenant_id,
1718 446 : &shard_slug,
1719 446 : &timeline_id,
1720 446 : ])
1721 446 : .unwrap();
1722 446 :
1723 446 : let global_batch_size = PAGE_SERVICE_BATCH_SIZE_GLOBAL.clone();
1724 446 : let per_timeline_batch_size = PAGE_SERVICE_BATCH_SIZE_PER_TENANT_TIMELINE
1725 446 : .get_metric_with_label_values(&[&tenant_id, &shard_slug, &timeline_id])
1726 446 : .unwrap();
1727 446 :
1728 446 : let global_batch_wait_time = PAGE_SERVICE_SMGR_BATCH_WAIT_TIME_GLOBAL.clone();
1729 446 : let per_timeline_batch_wait_time = PAGE_SERVICE_SMGR_BATCH_WAIT_TIME
1730 446 : .get_metric_with_label_values(&[&tenant_id, &shard_slug, &timeline_id])
1731 446 : .unwrap();
1732 446 :
1733 446 : let global_flush_in_progress_micros =
1734 446 : PAGE_SERVICE_SMGR_FLUSH_INPROGRESS_MICROS_GLOBAL.clone();
1735 446 : let per_timeline_flush_in_progress_micros = PAGE_SERVICE_SMGR_FLUSH_INPROGRESS_MICROS
1736 446 : .get_metric_with_label_values(&[&tenant_id, &shard_slug, &timeline_id])
1737 446 : .unwrap();
1738 446 :
1739 446 : Self {
1740 446 : global_started,
1741 446 : global_latency,
1742 446 : per_timeline_getpage_latency,
1743 446 : per_timeline_getpage_started,
1744 446 : global_batch_size,
1745 446 : per_timeline_batch_size,
1746 446 : global_flush_in_progress_micros,
1747 446 : per_timeline_flush_in_progress_micros,
1748 446 : global_batch_wait_time,
1749 446 : per_timeline_batch_wait_time,
1750 446 : throttling: pagestream_throttle_metrics,
1751 446 : }
1752 446 : }
1753 0 : pub(crate) fn start_smgr_op(&self, op: SmgrQueryType, received_at: Instant) -> SmgrOpTimer {
1754 0 : self.global_started[op as usize].inc();
1755 :
1756 0 : let per_timeline_latency_histo = if matches!(op, SmgrQueryType::GetPageAtLsn) {
1757 0 : self.per_timeline_getpage_started.inc();
1758 0 : Some(self.per_timeline_getpage_latency.clone())
1759 : } else {
1760 0 : None
1761 : };
1762 :
1763 0 : SmgrOpTimer(Some(SmgrOpTimerInner {
1764 0 : global_execution_latency_histo: self.global_latency[op as usize].clone(),
1765 0 : per_timeline_execution_latency_histo: per_timeline_latency_histo,
1766 0 : global_flush_in_progress_micros: self.global_flush_in_progress_micros.clone(),
1767 0 : per_timeline_flush_in_progress_micros: self
1768 0 : .per_timeline_flush_in_progress_micros
1769 0 : .clone(),
1770 0 : global_batch_wait_time: self.global_batch_wait_time.clone(),
1771 0 : per_timeline_batch_wait_time: self.per_timeline_batch_wait_time.clone(),
1772 0 : throttling: self.throttling.clone(),
1773 0 : timings: SmgrOpTimerState::Received { received_at },
1774 0 : }))
1775 0 : }
1776 :
1777 : /// TODO: do something about this? seems odd, we have a similar call on SmgrOpTimer
1778 0 : pub(crate) fn observe_getpage_batch_start(&self, batch_size: usize) {
1779 0 : self.global_batch_size.observe(batch_size as f64);
1780 0 : self.per_timeline_batch_size.observe(batch_size as f64);
1781 0 : }
1782 : }
1783 :
1784 : // keep in sync with control plane Go code so that we can validate
1785 : // compute's basebackup_ms metric with our perspective in the context of SLI/SLO.
1786 0 : static COMPUTE_STARTUP_BUCKETS: Lazy<[f64; 28]> = Lazy::new(|| {
1787 0 : // Go code uses milliseconds. Variable is called `computeStartupBuckets`
1788 0 : [
1789 0 : 5, 10, 20, 30, 50, 70, 100, 120, 150, 200, 250, 300, 350, 400, 450, 500, 600, 800, 1000,
1790 0 : 1500, 2000, 2500, 3000, 5000, 10000, 20000, 40000, 60000,
1791 0 : ]
1792 0 : .map(|ms| (ms as f64) / 1000.0)
1793 0 : });
1794 :
1795 : pub(crate) struct BasebackupQueryTime {
1796 : ok: Histogram,
1797 : error: Histogram,
1798 : client_error: Histogram,
1799 : }
1800 :
1801 0 : pub(crate) static BASEBACKUP_QUERY_TIME: Lazy<BasebackupQueryTime> = Lazy::new(|| {
1802 0 : let vec = register_histogram_vec!(
1803 0 : "pageserver_basebackup_query_seconds",
1804 0 : "Histogram of basebackup queries durations, by result type",
1805 0 : &["result"],
1806 0 : COMPUTE_STARTUP_BUCKETS.to_vec(),
1807 0 : )
1808 0 : .expect("failed to define a metric");
1809 0 : BasebackupQueryTime {
1810 0 : ok: vec.get_metric_with_label_values(&["ok"]).unwrap(),
1811 0 : error: vec.get_metric_with_label_values(&["error"]).unwrap(),
1812 0 : client_error: vec.get_metric_with_label_values(&["client_error"]).unwrap(),
1813 0 : }
1814 0 : });
1815 :
1816 : pub(crate) struct BasebackupQueryTimeOngoingRecording<'a> {
1817 : parent: &'a BasebackupQueryTime,
1818 : start: std::time::Instant,
1819 : }
1820 :
1821 : impl BasebackupQueryTime {
1822 0 : pub(crate) fn start_recording(&self) -> BasebackupQueryTimeOngoingRecording<'_> {
1823 0 : let start = Instant::now();
1824 0 : BasebackupQueryTimeOngoingRecording {
1825 0 : parent: self,
1826 0 : start,
1827 0 : }
1828 0 : }
1829 : }
1830 :
1831 : impl BasebackupQueryTimeOngoingRecording<'_> {
1832 0 : pub(crate) fn observe<T>(self, res: &Result<T, QueryError>) {
1833 0 : let elapsed = self.start.elapsed().as_secs_f64();
1834 : // If you want to change categorize of a specific error, also change it in `log_query_error`.
1835 0 : let metric = match res {
1836 0 : Ok(_) => &self.parent.ok,
1837 0 : Err(QueryError::Disconnected(ConnectionError::Io(io_error)))
1838 0 : if is_expected_io_error(io_error) =>
1839 0 : {
1840 0 : &self.parent.client_error
1841 : }
1842 0 : Err(_) => &self.parent.error,
1843 : };
1844 0 : metric.observe(elapsed);
1845 0 : }
1846 : }
1847 :
1848 0 : pub(crate) static LIVE_CONNECTIONS: Lazy<IntCounterPairVec> = Lazy::new(|| {
1849 0 : register_int_counter_pair_vec!(
1850 0 : "pageserver_live_connections_started",
1851 0 : "Number of network connections that we started handling",
1852 0 : "pageserver_live_connections_finished",
1853 0 : "Number of network connections that we finished handling",
1854 0 : &["pageserver_connection_kind"]
1855 0 : )
1856 0 : .expect("failed to define a metric")
1857 0 : });
1858 :
1859 : #[derive(Clone, Copy, enum_map::Enum, IntoStaticStr)]
1860 : pub(crate) enum ComputeCommandKind {
1861 : PageStreamV3,
1862 : PageStreamV2,
1863 : Basebackup,
1864 : Fullbackup,
1865 : LeaseLsn,
1866 : }
1867 :
1868 : pub(crate) struct ComputeCommandCounters {
1869 : map: EnumMap<ComputeCommandKind, IntCounter>,
1870 : }
1871 :
1872 0 : pub(crate) static COMPUTE_COMMANDS_COUNTERS: Lazy<ComputeCommandCounters> = Lazy::new(|| {
1873 0 : let inner = register_int_counter_vec!(
1874 0 : "pageserver_compute_commands",
1875 0 : "Number of compute -> pageserver commands processed",
1876 0 : &["command"]
1877 0 : )
1878 0 : .expect("failed to define a metric");
1879 0 :
1880 0 : ComputeCommandCounters {
1881 0 : map: EnumMap::from_array(std::array::from_fn(|i| {
1882 0 : let command = <ComputeCommandKind as enum_map::Enum>::from_usize(i);
1883 0 : let command_str: &'static str = command.into();
1884 0 : inner.with_label_values(&[command_str])
1885 0 : })),
1886 0 : }
1887 0 : });
1888 :
1889 : impl ComputeCommandCounters {
1890 0 : pub(crate) fn for_command(&self, command: ComputeCommandKind) -> &IntCounter {
1891 0 : &self.map[command]
1892 0 : }
1893 : }
1894 :
1895 : // remote storage metrics
1896 :
1897 196 : static REMOTE_TIMELINE_CLIENT_CALLS: Lazy<IntCounterPairVec> = Lazy::new(|| {
1898 196 : register_int_counter_pair_vec!(
1899 196 : "pageserver_remote_timeline_client_calls_started",
1900 196 : "Number of started calls to remote timeline client.",
1901 196 : "pageserver_remote_timeline_client_calls_finished",
1902 196 : "Number of finshed calls to remote timeline client.",
1903 196 : &[
1904 196 : "tenant_id",
1905 196 : "shard_id",
1906 196 : "timeline_id",
1907 196 : "file_kind",
1908 196 : "op_kind"
1909 196 : ],
1910 196 : )
1911 196 : .unwrap()
1912 196 : });
1913 :
1914 : static REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER: Lazy<IntCounterVec> =
1915 194 : Lazy::new(|| {
1916 194 : register_int_counter_vec!(
1917 194 : "pageserver_remote_timeline_client_bytes_started",
1918 194 : "Incremented by the number of bytes associated with a remote timeline client operation. \
1919 194 : The increment happens when the operation is scheduled.",
1920 194 : &["tenant_id", "shard_id", "timeline_id", "file_kind", "op_kind"],
1921 194 : )
1922 194 : .expect("failed to define a metric")
1923 194 : });
1924 :
1925 194 : static REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER: Lazy<IntCounterVec> = Lazy::new(|| {
1926 194 : register_int_counter_vec!(
1927 194 : "pageserver_remote_timeline_client_bytes_finished",
1928 194 : "Incremented by the number of bytes associated with a remote timeline client operation. \
1929 194 : The increment happens when the operation finishes (regardless of success/failure/shutdown).",
1930 194 : &["tenant_id", "shard_id", "timeline_id", "file_kind", "op_kind"],
1931 194 : )
1932 194 : .expect("failed to define a metric")
1933 194 : });
1934 :
1935 : pub(crate) struct TenantManagerMetrics {
1936 : tenant_slots_attached: UIntGauge,
1937 : tenant_slots_secondary: UIntGauge,
1938 : tenant_slots_inprogress: UIntGauge,
1939 : pub(crate) tenant_slot_writes: IntCounter,
1940 : pub(crate) unexpected_errors: IntCounter,
1941 : }
1942 :
1943 : impl TenantManagerMetrics {
1944 : /// Helpers for tracking slots. Note that these do not track the lifetime of TenantSlot objects
1945 : /// exactly: they track the lifetime of the slots _in the tenant map_.
1946 2 : pub(crate) fn slot_inserted(&self, slot: &TenantSlot) {
1947 2 : match slot {
1948 0 : TenantSlot::Attached(_) => {
1949 0 : self.tenant_slots_attached.inc();
1950 0 : }
1951 0 : TenantSlot::Secondary(_) => {
1952 0 : self.tenant_slots_secondary.inc();
1953 0 : }
1954 2 : TenantSlot::InProgress(_) => {
1955 2 : self.tenant_slots_inprogress.inc();
1956 2 : }
1957 : }
1958 2 : }
1959 :
1960 2 : pub(crate) fn slot_removed(&self, slot: &TenantSlot) {
1961 2 : match slot {
1962 2 : TenantSlot::Attached(_) => {
1963 2 : self.tenant_slots_attached.dec();
1964 2 : }
1965 0 : TenantSlot::Secondary(_) => {
1966 0 : self.tenant_slots_secondary.dec();
1967 0 : }
1968 0 : TenantSlot::InProgress(_) => {
1969 0 : self.tenant_slots_inprogress.dec();
1970 0 : }
1971 : }
1972 2 : }
1973 :
1974 : #[cfg(all(debug_assertions, not(test)))]
1975 0 : pub(crate) fn slots_total(&self) -> u64 {
1976 0 : self.tenant_slots_attached.get()
1977 0 : + self.tenant_slots_secondary.get()
1978 0 : + self.tenant_slots_inprogress.get()
1979 0 : }
1980 : }
1981 :
1982 2 : pub(crate) static TENANT_MANAGER: Lazy<TenantManagerMetrics> = Lazy::new(|| {
1983 2 : let tenant_slots = register_uint_gauge_vec!(
1984 2 : "pageserver_tenant_manager_slots",
1985 2 : "How many slots currently exist, including all attached, secondary and in-progress operations",
1986 2 : &["mode"]
1987 2 : )
1988 2 : .expect("failed to define a metric");
1989 2 : TenantManagerMetrics {
1990 2 : tenant_slots_attached: tenant_slots
1991 2 : .get_metric_with_label_values(&["attached"])
1992 2 : .unwrap(),
1993 2 : tenant_slots_secondary: tenant_slots
1994 2 : .get_metric_with_label_values(&["secondary"])
1995 2 : .unwrap(),
1996 2 : tenant_slots_inprogress: tenant_slots
1997 2 : .get_metric_with_label_values(&["inprogress"])
1998 2 : .unwrap(),
1999 2 : tenant_slot_writes: register_int_counter!(
2000 2 : "pageserver_tenant_manager_slot_writes",
2001 2 : "Writes to a tenant slot, including all of create/attach/detach/delete"
2002 2 : )
2003 2 : .expect("failed to define a metric"),
2004 2 : unexpected_errors: register_int_counter!(
2005 2 : "pageserver_tenant_manager_unexpected_errors_total",
2006 2 : "Number of unexpected conditions encountered: nonzero value indicates a non-fatal bug."
2007 2 : )
2008 2 : .expect("failed to define a metric"),
2009 2 : }
2010 2 : });
2011 :
2012 : pub(crate) struct DeletionQueueMetrics {
2013 : pub(crate) keys_submitted: IntCounter,
2014 : pub(crate) keys_dropped: IntCounter,
2015 : pub(crate) keys_executed: IntCounter,
2016 : pub(crate) keys_validated: IntCounter,
2017 : pub(crate) dropped_lsn_updates: IntCounter,
2018 : pub(crate) unexpected_errors: IntCounter,
2019 : pub(crate) remote_errors: IntCounterVec,
2020 : }
2021 35 : pub(crate) static DELETION_QUEUE: Lazy<DeletionQueueMetrics> = Lazy::new(|| {
2022 35 : DeletionQueueMetrics{
2023 35 :
2024 35 : keys_submitted: register_int_counter!(
2025 35 : "pageserver_deletion_queue_submitted_total",
2026 35 : "Number of objects submitted for deletion"
2027 35 : )
2028 35 : .expect("failed to define a metric"),
2029 35 :
2030 35 : keys_dropped: register_int_counter!(
2031 35 : "pageserver_deletion_queue_dropped_total",
2032 35 : "Number of object deletions dropped due to stale generation."
2033 35 : )
2034 35 : .expect("failed to define a metric"),
2035 35 :
2036 35 : keys_executed: register_int_counter!(
2037 35 : "pageserver_deletion_queue_executed_total",
2038 35 : "Number of objects deleted. Only includes objects that we actually deleted, sum with pageserver_deletion_queue_dropped_total for the total number of keys processed to completion"
2039 35 : )
2040 35 : .expect("failed to define a metric"),
2041 35 :
2042 35 : keys_validated: register_int_counter!(
2043 35 : "pageserver_deletion_queue_validated_total",
2044 35 : "Number of keys validated for deletion. Sum with pageserver_deletion_queue_dropped_total for the total number of keys that have passed through the validation stage."
2045 35 : )
2046 35 : .expect("failed to define a metric"),
2047 35 :
2048 35 : dropped_lsn_updates: register_int_counter!(
2049 35 : "pageserver_deletion_queue_dropped_lsn_updates_total",
2050 35 : "Updates to remote_consistent_lsn dropped due to stale generation number."
2051 35 : )
2052 35 : .expect("failed to define a metric"),
2053 35 : unexpected_errors: register_int_counter!(
2054 35 : "pageserver_deletion_queue_unexpected_errors_total",
2055 35 : "Number of unexpected condiions that may stall the queue: any value above zero is unexpected."
2056 35 : )
2057 35 : .expect("failed to define a metric"),
2058 35 : remote_errors: register_int_counter_vec!(
2059 35 : "pageserver_deletion_queue_remote_errors_total",
2060 35 : "Retryable remote I/O errors while executing deletions, for example 503 responses to DeleteObjects",
2061 35 : &["op_kind"],
2062 35 : )
2063 35 : .expect("failed to define a metric")
2064 35 : }
2065 35 : });
2066 :
2067 : pub(crate) struct SecondaryModeMetrics {
2068 : pub(crate) upload_heatmap: IntCounter,
2069 : pub(crate) upload_heatmap_errors: IntCounter,
2070 : pub(crate) upload_heatmap_duration: Histogram,
2071 : pub(crate) download_heatmap: IntCounter,
2072 : pub(crate) download_layer: IntCounter,
2073 : }
2074 0 : pub(crate) static SECONDARY_MODE: Lazy<SecondaryModeMetrics> = Lazy::new(|| {
2075 0 : SecondaryModeMetrics {
2076 0 : upload_heatmap: register_int_counter!(
2077 0 : "pageserver_secondary_upload_heatmap",
2078 0 : "Number of heatmaps written to remote storage by attached tenants"
2079 0 : )
2080 0 : .expect("failed to define a metric"),
2081 0 : upload_heatmap_errors: register_int_counter!(
2082 0 : "pageserver_secondary_upload_heatmap_errors",
2083 0 : "Failures writing heatmap to remote storage"
2084 0 : )
2085 0 : .expect("failed to define a metric"),
2086 0 : upload_heatmap_duration: register_histogram!(
2087 0 : "pageserver_secondary_upload_heatmap_duration",
2088 0 : "Time to build and upload a heatmap, including any waiting inside the remote storage client"
2089 0 : )
2090 0 : .expect("failed to define a metric"),
2091 0 : download_heatmap: register_int_counter!(
2092 0 : "pageserver_secondary_download_heatmap",
2093 0 : "Number of downloads of heatmaps by secondary mode locations, including when it hasn't changed"
2094 0 : )
2095 0 : .expect("failed to define a metric"),
2096 0 : download_layer: register_int_counter!(
2097 0 : "pageserver_secondary_download_layer",
2098 0 : "Number of downloads of layers by secondary mode locations"
2099 0 : )
2100 0 : .expect("failed to define a metric"),
2101 0 : }
2102 0 : });
2103 :
2104 0 : pub(crate) static SECONDARY_RESIDENT_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
2105 0 : register_uint_gauge_vec!(
2106 0 : "pageserver_secondary_resident_physical_size",
2107 0 : "The size of the layer files present in the pageserver's filesystem, for secondary locations.",
2108 0 : &["tenant_id", "shard_id"]
2109 0 : )
2110 0 : .expect("failed to define a metric")
2111 0 : });
2112 :
2113 0 : pub(crate) static NODE_UTILIZATION_SCORE: Lazy<UIntGauge> = Lazy::new(|| {
2114 0 : register_uint_gauge!(
2115 0 : "pageserver_utilization_score",
2116 0 : "The utilization score we report to the storage controller for scheduling, where 0 is empty, 1000000 is full, and anything above is considered overloaded",
2117 0 : )
2118 0 : .expect("failed to define a metric")
2119 0 : });
2120 :
2121 0 : pub(crate) static SECONDARY_HEATMAP_TOTAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
2122 0 : register_uint_gauge_vec!(
2123 0 : "pageserver_secondary_heatmap_total_size",
2124 0 : "The total size in bytes of all layers in the most recently downloaded heatmap.",
2125 0 : &["tenant_id", "shard_id"]
2126 0 : )
2127 0 : .expect("failed to define a metric")
2128 0 : });
2129 :
2130 : #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
2131 : pub enum RemoteOpKind {
2132 : Upload,
2133 : Download,
2134 : Delete,
2135 : }
2136 : impl RemoteOpKind {
2137 14358 : pub fn as_str(&self) -> &'static str {
2138 14358 : match self {
2139 13463 : Self::Upload => "upload",
2140 52 : Self::Download => "download",
2141 843 : Self::Delete => "delete",
2142 : }
2143 14358 : }
2144 : }
2145 :
2146 : #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
2147 : pub enum RemoteOpFileKind {
2148 : Layer,
2149 : Index,
2150 : }
2151 : impl RemoteOpFileKind {
2152 14358 : pub fn as_str(&self) -> &'static str {
2153 14358 : match self {
2154 9822 : Self::Layer => "layer",
2155 4536 : Self::Index => "index",
2156 : }
2157 14358 : }
2158 : }
2159 :
2160 194 : pub(crate) static REMOTE_OPERATION_TIME: Lazy<HistogramVec> = Lazy::new(|| {
2161 194 : register_histogram_vec!(
2162 194 : "pageserver_remote_operation_seconds",
2163 194 : "Time spent on remote storage operations. \
2164 194 : Grouped by tenant, timeline, operation_kind and status. \
2165 194 : Does not account for time spent waiting in remote timeline client's queues.",
2166 194 : &["file_kind", "op_kind", "status"]
2167 194 : )
2168 194 : .expect("failed to define a metric")
2169 194 : });
2170 :
2171 0 : pub(crate) static TENANT_TASK_EVENTS: Lazy<IntCounterVec> = Lazy::new(|| {
2172 0 : register_int_counter_vec!(
2173 0 : "pageserver_tenant_task_events",
2174 0 : "Number of task start/stop/fail events.",
2175 0 : &["event"],
2176 0 : )
2177 0 : .expect("Failed to register tenant_task_events metric")
2178 0 : });
2179 :
2180 : pub struct BackgroundLoopSemaphoreMetrics {
2181 : counters: EnumMap<BackgroundLoopKind, IntCounterPair>,
2182 : durations: EnumMap<BackgroundLoopKind, Counter>,
2183 : }
2184 :
2185 : pub(crate) static BACKGROUND_LOOP_SEMAPHORE: Lazy<BackgroundLoopSemaphoreMetrics> = Lazy::new(
2186 20 : || {
2187 20 : let counters = register_int_counter_pair_vec!(
2188 20 : "pageserver_background_loop_semaphore_wait_start_count",
2189 20 : "Counter for background loop concurrency-limiting semaphore acquire calls started",
2190 20 : "pageserver_background_loop_semaphore_wait_finish_count",
2191 20 : "Counter for background loop concurrency-limiting semaphore acquire calls finished",
2192 20 : &["task"],
2193 20 : )
2194 20 : .unwrap();
2195 20 :
2196 20 : let durations = register_counter_vec!(
2197 20 : "pageserver_background_loop_semaphore_wait_duration_seconds",
2198 20 : "Sum of wall clock time spent waiting on the background loop concurrency-limiting semaphore acquire calls",
2199 20 : &["task"],
2200 20 : )
2201 20 : .unwrap();
2202 20 :
2203 20 : BackgroundLoopSemaphoreMetrics {
2204 180 : counters: enum_map::EnumMap::from_array(std::array::from_fn(|i| {
2205 180 : let kind = <BackgroundLoopKind as enum_map::Enum>::from_usize(i);
2206 180 : counters.with_label_values(&[kind.into()])
2207 180 : })),
2208 180 : durations: enum_map::EnumMap::from_array(std::array::from_fn(|i| {
2209 180 : let kind = <BackgroundLoopKind as enum_map::Enum>::from_usize(i);
2210 180 : durations.with_label_values(&[kind.into()])
2211 180 : })),
2212 20 : }
2213 20 : },
2214 : );
2215 :
2216 : impl BackgroundLoopSemaphoreMetrics {
2217 364 : pub(crate) fn measure_acquisition(&self, task: BackgroundLoopKind) -> impl Drop + '_ {
2218 : struct Record<'a> {
2219 : metrics: &'a BackgroundLoopSemaphoreMetrics,
2220 : task: BackgroundLoopKind,
2221 : _counter_guard: metrics::IntCounterPairGuard,
2222 : start: Instant,
2223 : }
2224 : impl Drop for Record<'_> {
2225 364 : fn drop(&mut self) {
2226 364 : let elapsed = self.start.elapsed().as_secs_f64();
2227 364 : self.metrics.durations[self.task].inc_by(elapsed);
2228 364 : }
2229 : }
2230 364 : Record {
2231 364 : metrics: self,
2232 364 : task,
2233 364 : _counter_guard: self.counters[task].guard(),
2234 364 : start: Instant::now(),
2235 364 : }
2236 364 : }
2237 : }
2238 :
2239 0 : pub(crate) static BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT: Lazy<IntCounterVec> = Lazy::new(|| {
2240 0 : register_int_counter_vec!(
2241 0 : "pageserver_background_loop_period_overrun_count",
2242 0 : "Incremented whenever warn_when_period_overrun() logs a warning.",
2243 0 : &["task", "period"],
2244 0 : )
2245 0 : .expect("failed to define a metric")
2246 0 : });
2247 :
2248 : // walreceiver metrics
2249 :
2250 0 : pub(crate) static WALRECEIVER_STARTED_CONNECTIONS: Lazy<IntCounter> = Lazy::new(|| {
2251 0 : register_int_counter!(
2252 0 : "pageserver_walreceiver_started_connections_total",
2253 0 : "Number of started walreceiver connections"
2254 0 : )
2255 0 : .expect("failed to define a metric")
2256 0 : });
2257 :
2258 0 : pub(crate) static WALRECEIVER_ACTIVE_MANAGERS: Lazy<IntGauge> = Lazy::new(|| {
2259 0 : register_int_gauge!(
2260 0 : "pageserver_walreceiver_active_managers",
2261 0 : "Number of active walreceiver managers"
2262 0 : )
2263 0 : .expect("failed to define a metric")
2264 0 : });
2265 :
2266 0 : pub(crate) static WALRECEIVER_SWITCHES: Lazy<IntCounterVec> = Lazy::new(|| {
2267 0 : register_int_counter_vec!(
2268 0 : "pageserver_walreceiver_switches_total",
2269 0 : "Number of walreceiver manager change_connection calls",
2270 0 : &["reason"]
2271 0 : )
2272 0 : .expect("failed to define a metric")
2273 0 : });
2274 :
2275 0 : pub(crate) static WALRECEIVER_BROKER_UPDATES: Lazy<IntCounter> = Lazy::new(|| {
2276 0 : register_int_counter!(
2277 0 : "pageserver_walreceiver_broker_updates_total",
2278 0 : "Number of received broker updates in walreceiver"
2279 0 : )
2280 0 : .expect("failed to define a metric")
2281 0 : });
2282 :
2283 2 : pub(crate) static WALRECEIVER_CANDIDATES_EVENTS: Lazy<IntCounterVec> = Lazy::new(|| {
2284 2 : register_int_counter_vec!(
2285 2 : "pageserver_walreceiver_candidates_events_total",
2286 2 : "Number of walreceiver candidate events",
2287 2 : &["event"]
2288 2 : )
2289 2 : .expect("failed to define a metric")
2290 2 : });
2291 :
2292 : pub(crate) static WALRECEIVER_CANDIDATES_ADDED: Lazy<IntCounter> =
2293 0 : Lazy::new(|| WALRECEIVER_CANDIDATES_EVENTS.with_label_values(&["add"]));
2294 :
2295 : pub(crate) static WALRECEIVER_CANDIDATES_REMOVED: Lazy<IntCounter> =
2296 2 : Lazy::new(|| WALRECEIVER_CANDIDATES_EVENTS.with_label_values(&["remove"]));
2297 :
2298 : // Metrics collected on WAL redo operations
2299 : //
2300 : // We collect the time spent in actual WAL redo ('redo'), and time waiting
2301 : // for access to the postgres process ('wait') since there is only one for
2302 : // each tenant.
2303 :
2304 : /// Time buckets are small because we want to be able to measure the
2305 : /// smallest redo processing times. These buckets allow us to measure down
2306 : /// to 5us, which equates to 200'000 pages/sec, which equates to 1.6GB/sec.
2307 : /// This is much better than the previous 5ms aka 200 pages/sec aka 1.6MB/sec.
2308 : ///
2309 : /// Values up to 1s are recorded because metrics show that we have redo
2310 : /// durations and lock times larger than 0.250s.
2311 : macro_rules! redo_histogram_time_buckets {
2312 : () => {
2313 : vec![
2314 : 0.000_005, 0.000_010, 0.000_025, 0.000_050, 0.000_100, 0.000_250, 0.000_500, 0.001_000,
2315 : 0.002_500, 0.005_000, 0.010_000, 0.025_000, 0.050_000, 0.100_000, 0.250_000, 0.500_000,
2316 : 1.000_000,
2317 : ]
2318 : };
2319 : }
2320 :
2321 : /// While we're at it, also measure the amount of records replayed in each
2322 : /// operation. We have a global 'total replayed' counter, but that's not
2323 : /// as useful as 'what is the skew for how many records we replay in one
2324 : /// operation'.
2325 : macro_rules! redo_histogram_count_buckets {
2326 : () => {
2327 : vec![0.0, 1.0, 2.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0]
2328 : };
2329 : }
2330 :
2331 : macro_rules! redo_bytes_histogram_count_buckets {
2332 : () => {
2333 : // powers of (2^.5), from 2^4.5 to 2^15 (22 buckets)
2334 : // rounded up to the next multiple of 8 to capture any MAXALIGNed record of that size, too.
2335 : vec![
2336 : 24.0, 32.0, 48.0, 64.0, 96.0, 128.0, 184.0, 256.0, 368.0, 512.0, 728.0, 1024.0, 1456.0,
2337 : 2048.0, 2904.0, 4096.0, 5800.0, 8192.0, 11592.0, 16384.0, 23176.0, 32768.0,
2338 : ]
2339 : };
2340 : }
2341 :
2342 : pub(crate) struct WalIngestMetrics {
2343 : pub(crate) bytes_received: IntCounter,
2344 : pub(crate) records_received: IntCounter,
2345 : pub(crate) records_observed: IntCounter,
2346 : pub(crate) records_committed: IntCounter,
2347 : pub(crate) records_filtered: IntCounter,
2348 : pub(crate) gap_blocks_zeroed_on_rel_extend: IntCounter,
2349 : pub(crate) clear_vm_bits_unknown: IntCounterVec,
2350 : }
2351 :
2352 10 : pub(crate) static WAL_INGEST: Lazy<WalIngestMetrics> = Lazy::new(|| {
2353 10 : WalIngestMetrics {
2354 10 : bytes_received: register_int_counter!(
2355 10 : "pageserver_wal_ingest_bytes_received",
2356 10 : "Bytes of WAL ingested from safekeepers",
2357 10 : )
2358 10 : .unwrap(),
2359 10 : records_received: register_int_counter!(
2360 10 : "pageserver_wal_ingest_records_received",
2361 10 : "Number of WAL records received from safekeepers"
2362 10 : )
2363 10 : .expect("failed to define a metric"),
2364 10 : records_observed: register_int_counter!(
2365 10 : "pageserver_wal_ingest_records_observed",
2366 10 : "Number of WAL records observed from safekeepers. These are metadata only records for shard 0."
2367 10 : )
2368 10 : .expect("failed to define a metric"),
2369 10 : records_committed: register_int_counter!(
2370 10 : "pageserver_wal_ingest_records_committed",
2371 10 : "Number of WAL records which resulted in writes to pageserver storage"
2372 10 : )
2373 10 : .expect("failed to define a metric"),
2374 10 : records_filtered: register_int_counter!(
2375 10 : "pageserver_wal_ingest_records_filtered",
2376 10 : "Number of WAL records filtered out due to sharding"
2377 10 : )
2378 10 : .expect("failed to define a metric"),
2379 10 : gap_blocks_zeroed_on_rel_extend: register_int_counter!(
2380 10 : "pageserver_gap_blocks_zeroed_on_rel_extend",
2381 10 : "Total number of zero gap blocks written on relation extends"
2382 10 : )
2383 10 : .expect("failed to define a metric"),
2384 10 : clear_vm_bits_unknown: register_int_counter_vec!(
2385 10 : "pageserver_wal_ingest_clear_vm_bits_unknown",
2386 10 : "Number of ignored ClearVmBits operations due to unknown pages/relations",
2387 10 : &["entity"],
2388 10 : )
2389 10 : .expect("failed to define a metric"),
2390 10 : }
2391 10 : });
2392 :
2393 200 : pub(crate) static PAGESERVER_TIMELINE_WAL_RECORDS_RECEIVED: Lazy<IntCounterVec> = Lazy::new(|| {
2394 200 : register_int_counter_vec!(
2395 200 : "pageserver_timeline_wal_records_received",
2396 200 : "Number of WAL records received per shard",
2397 200 : &["tenant_id", "shard_id", "timeline_id"]
2398 200 : )
2399 200 : .expect("failed to define a metric")
2400 200 : });
2401 :
2402 6 : pub(crate) static WAL_REDO_TIME: Lazy<Histogram> = Lazy::new(|| {
2403 6 : register_histogram!(
2404 6 : "pageserver_wal_redo_seconds",
2405 6 : "Time spent on WAL redo",
2406 6 : redo_histogram_time_buckets!()
2407 6 : )
2408 6 : .expect("failed to define a metric")
2409 6 : });
2410 :
2411 6 : pub(crate) static WAL_REDO_RECORDS_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
2412 6 : register_histogram!(
2413 6 : "pageserver_wal_redo_records_histogram",
2414 6 : "Histogram of number of records replayed per redo in the Postgres WAL redo process",
2415 6 : redo_histogram_count_buckets!(),
2416 6 : )
2417 6 : .expect("failed to define a metric")
2418 6 : });
2419 :
2420 6 : pub(crate) static WAL_REDO_BYTES_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
2421 6 : register_histogram!(
2422 6 : "pageserver_wal_redo_bytes_histogram",
2423 6 : "Histogram of number of records replayed per redo sent to Postgres",
2424 6 : redo_bytes_histogram_count_buckets!(),
2425 6 : )
2426 6 : .expect("failed to define a metric")
2427 6 : });
2428 :
2429 : // FIXME: isn't this already included by WAL_REDO_RECORDS_HISTOGRAM which has _count?
2430 6 : pub(crate) static WAL_REDO_RECORD_COUNTER: Lazy<IntCounter> = Lazy::new(|| {
2431 6 : register_int_counter!(
2432 6 : "pageserver_replayed_wal_records_total",
2433 6 : "Number of WAL records replayed in WAL redo process"
2434 6 : )
2435 6 : .unwrap()
2436 6 : });
2437 :
2438 : #[rustfmt::skip]
2439 8 : pub(crate) static WAL_REDO_PROCESS_LAUNCH_DURATION_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
2440 8 : register_histogram!(
2441 8 : "pageserver_wal_redo_process_launch_duration",
2442 8 : "Histogram of the duration of successful WalRedoProcess::launch calls",
2443 8 : vec![
2444 8 : 0.0002, 0.0004, 0.0006, 0.0008, 0.0010,
2445 8 : 0.0020, 0.0040, 0.0060, 0.0080, 0.0100,
2446 8 : 0.0200, 0.0400, 0.0600, 0.0800, 0.1000,
2447 8 : 0.2000, 0.4000, 0.6000, 0.8000, 1.0000,
2448 8 : 1.5000, 2.0000, 2.5000, 3.0000, 4.0000, 10.0000
2449 8 : ],
2450 8 : )
2451 8 : .expect("failed to define a metric")
2452 8 : });
2453 :
2454 : pub(crate) struct WalRedoProcessCounters {
2455 : pub(crate) started: IntCounter,
2456 : pub(crate) killed_by_cause: enum_map::EnumMap<WalRedoKillCause, IntCounter>,
2457 : pub(crate) active_stderr_logger_tasks_started: IntCounter,
2458 : pub(crate) active_stderr_logger_tasks_finished: IntCounter,
2459 : }
2460 :
2461 : #[derive(Debug, enum_map::Enum, strum_macros::IntoStaticStr)]
2462 : pub(crate) enum WalRedoKillCause {
2463 : WalRedoProcessDrop,
2464 : NoLeakChildDrop,
2465 : Startup,
2466 : }
2467 :
2468 : impl Default for WalRedoProcessCounters {
2469 8 : fn default() -> Self {
2470 8 : let started = register_int_counter!(
2471 8 : "pageserver_wal_redo_process_started_total",
2472 8 : "Number of WAL redo processes started",
2473 8 : )
2474 8 : .unwrap();
2475 8 :
2476 8 : let killed = register_int_counter_vec!(
2477 8 : "pageserver_wal_redo_process_stopped_total",
2478 8 : "Number of WAL redo processes stopped",
2479 8 : &["cause"],
2480 8 : )
2481 8 : .unwrap();
2482 8 :
2483 8 : let active_stderr_logger_tasks_started = register_int_counter!(
2484 8 : "pageserver_walredo_stderr_logger_tasks_started_total",
2485 8 : "Number of active walredo stderr logger tasks that have started",
2486 8 : )
2487 8 : .unwrap();
2488 8 :
2489 8 : let active_stderr_logger_tasks_finished = register_int_counter!(
2490 8 : "pageserver_walredo_stderr_logger_tasks_finished_total",
2491 8 : "Number of active walredo stderr logger tasks that have finished",
2492 8 : )
2493 8 : .unwrap();
2494 8 :
2495 8 : Self {
2496 8 : started,
2497 24 : killed_by_cause: EnumMap::from_array(std::array::from_fn(|i| {
2498 24 : let cause = <WalRedoKillCause as enum_map::Enum>::from_usize(i);
2499 24 : let cause_str: &'static str = cause.into();
2500 24 : killed.with_label_values(&[cause_str])
2501 24 : })),
2502 8 : active_stderr_logger_tasks_started,
2503 8 : active_stderr_logger_tasks_finished,
2504 8 : }
2505 8 : }
2506 : }
2507 :
2508 : pub(crate) static WAL_REDO_PROCESS_COUNTERS: Lazy<WalRedoProcessCounters> =
2509 : Lazy::new(WalRedoProcessCounters::default);
2510 :
2511 : /// Similar to `prometheus::HistogramTimer` but does not record on drop.
2512 : pub(crate) struct StorageTimeMetricsTimer {
2513 : metrics: StorageTimeMetrics,
2514 : start: Instant,
2515 : }
2516 :
2517 : impl StorageTimeMetricsTimer {
2518 3276 : fn new(metrics: StorageTimeMetrics) -> Self {
2519 3276 : Self {
2520 3276 : metrics,
2521 3276 : start: Instant::now(),
2522 3276 : }
2523 3276 : }
2524 :
2525 : /// Record the time from creation to now.
2526 2118 : pub fn stop_and_record(self) {
2527 2118 : let duration = self.start.elapsed().as_secs_f64();
2528 2118 : self.metrics.timeline_sum.inc_by(duration);
2529 2118 : self.metrics.timeline_count.inc();
2530 2118 : self.metrics.global_histogram.observe(duration);
2531 2118 : }
2532 :
2533 : /// Turns this timer into a timer, which will always record -- usually this means recording
2534 : /// regardless an early `?` path was taken in a function.
2535 4 : pub(crate) fn record_on_drop(self) -> AlwaysRecordingStorageTimeMetricsTimer {
2536 4 : AlwaysRecordingStorageTimeMetricsTimer(Some(self))
2537 4 : }
2538 : }
2539 :
2540 : pub(crate) struct AlwaysRecordingStorageTimeMetricsTimer(Option<StorageTimeMetricsTimer>);
2541 :
2542 : impl Drop for AlwaysRecordingStorageTimeMetricsTimer {
2543 4 : fn drop(&mut self) {
2544 4 : if let Some(inner) = self.0.take() {
2545 4 : inner.stop_and_record();
2546 4 : }
2547 4 : }
2548 : }
2549 :
2550 : /// Timing facilities for an globally histogrammed metric, which is supported by per tenant and
2551 : /// timeline total sum and count.
2552 : #[derive(Clone, Debug)]
2553 : pub(crate) struct StorageTimeMetrics {
2554 : /// Sum of f64 seconds, per operation, tenant_id and timeline_id
2555 : timeline_sum: Counter,
2556 : /// Number of oeprations, per operation, tenant_id and timeline_id
2557 : timeline_count: IntCounter,
2558 : /// Global histogram having only the "operation" label.
2559 : global_histogram: Histogram,
2560 : }
2561 :
2562 : impl StorageTimeMetrics {
2563 3568 : pub fn new(
2564 3568 : operation: StorageTimeOperation,
2565 3568 : tenant_id: &str,
2566 3568 : shard_id: &str,
2567 3568 : timeline_id: &str,
2568 3568 : ) -> Self {
2569 3568 : let operation: &'static str = operation.into();
2570 3568 :
2571 3568 : let timeline_sum = STORAGE_TIME_SUM_PER_TIMELINE
2572 3568 : .get_metric_with_label_values(&[operation, tenant_id, shard_id, timeline_id])
2573 3568 : .unwrap();
2574 3568 : let timeline_count = STORAGE_TIME_COUNT_PER_TIMELINE
2575 3568 : .get_metric_with_label_values(&[operation, tenant_id, shard_id, timeline_id])
2576 3568 : .unwrap();
2577 3568 : let global_histogram = STORAGE_TIME_GLOBAL
2578 3568 : .get_metric_with_label_values(&[operation])
2579 3568 : .unwrap();
2580 3568 :
2581 3568 : StorageTimeMetrics {
2582 3568 : timeline_sum,
2583 3568 : timeline_count,
2584 3568 : global_histogram,
2585 3568 : }
2586 3568 : }
2587 :
2588 : /// Starts timing a new operation.
2589 : ///
2590 : /// Note: unlike `prometheus::HistogramTimer` the returned timer does not record on drop.
2591 3276 : pub fn start_timer(&self) -> StorageTimeMetricsTimer {
2592 3276 : StorageTimeMetricsTimer::new(self.clone())
2593 3276 : }
2594 : }
2595 :
2596 : #[derive(Debug)]
2597 : pub(crate) struct TimelineMetrics {
2598 : tenant_id: String,
2599 : shard_id: String,
2600 : timeline_id: String,
2601 : pub flush_time_histo: StorageTimeMetrics,
2602 : pub flush_wait_upload_time_gauge: Gauge,
2603 : pub compact_time_histo: StorageTimeMetrics,
2604 : pub create_images_time_histo: StorageTimeMetrics,
2605 : pub logical_size_histo: StorageTimeMetrics,
2606 : pub imitate_logical_size_histo: StorageTimeMetrics,
2607 : pub load_layer_map_histo: StorageTimeMetrics,
2608 : pub garbage_collect_histo: StorageTimeMetrics,
2609 : pub find_gc_cutoffs_histo: StorageTimeMetrics,
2610 : pub last_record_lsn_gauge: IntGauge,
2611 : pub disk_consistent_lsn_gauge: IntGauge,
2612 : pub pitr_history_size: UIntGauge,
2613 : pub archival_size: UIntGauge,
2614 : pub(crate) layer_size_image: UIntGauge,
2615 : pub(crate) layer_count_image: UIntGauge,
2616 : pub(crate) layer_size_delta: UIntGauge,
2617 : pub(crate) layer_count_delta: UIntGauge,
2618 : pub standby_horizon_gauge: IntGauge,
2619 : pub resident_physical_size_gauge: UIntGauge,
2620 : pub visible_physical_size_gauge: UIntGauge,
2621 : /// copy of LayeredTimeline.current_logical_size
2622 : pub current_logical_size_gauge: UIntGauge,
2623 : pub aux_file_size_gauge: IntGauge,
2624 : pub directory_entries_count_gauge: Lazy<UIntGauge, Box<dyn Send + Fn() -> UIntGauge>>,
2625 : pub evictions: IntCounter,
2626 : pub evictions_with_low_residence_duration: std::sync::RwLock<EvictionsWithLowResidenceDuration>,
2627 : /// Number of valid LSN leases.
2628 : pub valid_lsn_lease_count_gauge: UIntGauge,
2629 : pub wal_records_received: IntCounter,
2630 : shutdown: std::sync::atomic::AtomicBool,
2631 : }
2632 :
2633 : impl TimelineMetrics {
2634 446 : pub fn new(
2635 446 : tenant_shard_id: &TenantShardId,
2636 446 : timeline_id_raw: &TimelineId,
2637 446 : evictions_with_low_residence_duration_builder: EvictionsWithLowResidenceDurationBuilder,
2638 446 : ) -> Self {
2639 446 : let tenant_id = tenant_shard_id.tenant_id.to_string();
2640 446 : let shard_id = format!("{}", tenant_shard_id.shard_slug());
2641 446 : let timeline_id = timeline_id_raw.to_string();
2642 446 : let flush_time_histo = StorageTimeMetrics::new(
2643 446 : StorageTimeOperation::LayerFlush,
2644 446 : &tenant_id,
2645 446 : &shard_id,
2646 446 : &timeline_id,
2647 446 : );
2648 446 : let flush_wait_upload_time_gauge = FLUSH_WAIT_UPLOAD_TIME
2649 446 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2650 446 : .unwrap();
2651 446 : let compact_time_histo = StorageTimeMetrics::new(
2652 446 : StorageTimeOperation::Compact,
2653 446 : &tenant_id,
2654 446 : &shard_id,
2655 446 : &timeline_id,
2656 446 : );
2657 446 : let create_images_time_histo = StorageTimeMetrics::new(
2658 446 : StorageTimeOperation::CreateImages,
2659 446 : &tenant_id,
2660 446 : &shard_id,
2661 446 : &timeline_id,
2662 446 : );
2663 446 : let logical_size_histo = StorageTimeMetrics::new(
2664 446 : StorageTimeOperation::LogicalSize,
2665 446 : &tenant_id,
2666 446 : &shard_id,
2667 446 : &timeline_id,
2668 446 : );
2669 446 : let imitate_logical_size_histo = StorageTimeMetrics::new(
2670 446 : StorageTimeOperation::ImitateLogicalSize,
2671 446 : &tenant_id,
2672 446 : &shard_id,
2673 446 : &timeline_id,
2674 446 : );
2675 446 : let load_layer_map_histo = StorageTimeMetrics::new(
2676 446 : StorageTimeOperation::LoadLayerMap,
2677 446 : &tenant_id,
2678 446 : &shard_id,
2679 446 : &timeline_id,
2680 446 : );
2681 446 : let garbage_collect_histo = StorageTimeMetrics::new(
2682 446 : StorageTimeOperation::Gc,
2683 446 : &tenant_id,
2684 446 : &shard_id,
2685 446 : &timeline_id,
2686 446 : );
2687 446 : let find_gc_cutoffs_histo = StorageTimeMetrics::new(
2688 446 : StorageTimeOperation::FindGcCutoffs,
2689 446 : &tenant_id,
2690 446 : &shard_id,
2691 446 : &timeline_id,
2692 446 : );
2693 446 : let last_record_lsn_gauge = LAST_RECORD_LSN
2694 446 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2695 446 : .unwrap();
2696 446 :
2697 446 : let disk_consistent_lsn_gauge = DISK_CONSISTENT_LSN
2698 446 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2699 446 : .unwrap();
2700 446 :
2701 446 : let pitr_history_size = PITR_HISTORY_SIZE
2702 446 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2703 446 : .unwrap();
2704 446 :
2705 446 : let archival_size = TIMELINE_ARCHIVE_SIZE
2706 446 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2707 446 : .unwrap();
2708 446 :
2709 446 : let layer_size_image = TIMELINE_LAYER_SIZE
2710 446 : .get_metric_with_label_values(&[
2711 446 : &tenant_id,
2712 446 : &shard_id,
2713 446 : &timeline_id,
2714 446 : MetricLayerKind::Image.into(),
2715 446 : ])
2716 446 : .unwrap();
2717 446 :
2718 446 : let layer_count_image = TIMELINE_LAYER_COUNT
2719 446 : .get_metric_with_label_values(&[
2720 446 : &tenant_id,
2721 446 : &shard_id,
2722 446 : &timeline_id,
2723 446 : MetricLayerKind::Image.into(),
2724 446 : ])
2725 446 : .unwrap();
2726 446 :
2727 446 : let layer_size_delta = TIMELINE_LAYER_SIZE
2728 446 : .get_metric_with_label_values(&[
2729 446 : &tenant_id,
2730 446 : &shard_id,
2731 446 : &timeline_id,
2732 446 : MetricLayerKind::Delta.into(),
2733 446 : ])
2734 446 : .unwrap();
2735 446 :
2736 446 : let layer_count_delta = TIMELINE_LAYER_COUNT
2737 446 : .get_metric_with_label_values(&[
2738 446 : &tenant_id,
2739 446 : &shard_id,
2740 446 : &timeline_id,
2741 446 : MetricLayerKind::Delta.into(),
2742 446 : ])
2743 446 : .unwrap();
2744 446 :
2745 446 : let standby_horizon_gauge = STANDBY_HORIZON
2746 446 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2747 446 : .unwrap();
2748 446 : let resident_physical_size_gauge = RESIDENT_PHYSICAL_SIZE
2749 446 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2750 446 : .unwrap();
2751 446 : let visible_physical_size_gauge = VISIBLE_PHYSICAL_SIZE
2752 446 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2753 446 : .unwrap();
2754 446 : // TODO: we shouldn't expose this metric
2755 446 : let current_logical_size_gauge = CURRENT_LOGICAL_SIZE
2756 446 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2757 446 : .unwrap();
2758 446 : let aux_file_size_gauge = AUX_FILE_SIZE
2759 446 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2760 446 : .unwrap();
2761 446 : // TODO use impl Trait syntax here once we have ability to use it: https://github.com/rust-lang/rust/issues/63065
2762 446 : let directory_entries_count_gauge_closure = {
2763 446 : let tenant_shard_id = *tenant_shard_id;
2764 446 : let timeline_id_raw = *timeline_id_raw;
2765 0 : move || {
2766 0 : let tenant_id = tenant_shard_id.tenant_id.to_string();
2767 0 : let shard_id = format!("{}", tenant_shard_id.shard_slug());
2768 0 : let timeline_id = timeline_id_raw.to_string();
2769 0 : let gauge: UIntGauge = DIRECTORY_ENTRIES_COUNT
2770 0 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2771 0 : .unwrap();
2772 0 : gauge
2773 0 : }
2774 : };
2775 446 : let directory_entries_count_gauge: Lazy<UIntGauge, Box<dyn Send + Fn() -> UIntGauge>> =
2776 446 : Lazy::new(Box::new(directory_entries_count_gauge_closure));
2777 446 : let evictions = EVICTIONS
2778 446 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2779 446 : .unwrap();
2780 446 : let evictions_with_low_residence_duration = evictions_with_low_residence_duration_builder
2781 446 : .build(&tenant_id, &shard_id, &timeline_id);
2782 446 :
2783 446 : let valid_lsn_lease_count_gauge = VALID_LSN_LEASE_COUNT
2784 446 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2785 446 : .unwrap();
2786 446 :
2787 446 : let wal_records_received = PAGESERVER_TIMELINE_WAL_RECORDS_RECEIVED
2788 446 : .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
2789 446 : .unwrap();
2790 446 :
2791 446 : TimelineMetrics {
2792 446 : tenant_id,
2793 446 : shard_id,
2794 446 : timeline_id,
2795 446 : flush_time_histo,
2796 446 : flush_wait_upload_time_gauge,
2797 446 : compact_time_histo,
2798 446 : create_images_time_histo,
2799 446 : logical_size_histo,
2800 446 : imitate_logical_size_histo,
2801 446 : garbage_collect_histo,
2802 446 : find_gc_cutoffs_histo,
2803 446 : load_layer_map_histo,
2804 446 : last_record_lsn_gauge,
2805 446 : disk_consistent_lsn_gauge,
2806 446 : pitr_history_size,
2807 446 : archival_size,
2808 446 : layer_size_image,
2809 446 : layer_count_image,
2810 446 : layer_size_delta,
2811 446 : layer_count_delta,
2812 446 : standby_horizon_gauge,
2813 446 : resident_physical_size_gauge,
2814 446 : visible_physical_size_gauge,
2815 446 : current_logical_size_gauge,
2816 446 : aux_file_size_gauge,
2817 446 : directory_entries_count_gauge,
2818 446 : evictions,
2819 446 : evictions_with_low_residence_duration: std::sync::RwLock::new(
2820 446 : evictions_with_low_residence_duration,
2821 446 : ),
2822 446 : valid_lsn_lease_count_gauge,
2823 446 : wal_records_received,
2824 446 : shutdown: std::sync::atomic::AtomicBool::default(),
2825 446 : }
2826 446 : }
2827 :
2828 1572 : pub(crate) fn record_new_file_metrics(&self, sz: u64) {
2829 1572 : self.resident_physical_size_add(sz);
2830 1572 : }
2831 :
2832 526 : pub(crate) fn resident_physical_size_sub(&self, sz: u64) {
2833 526 : self.resident_physical_size_gauge.sub(sz);
2834 526 : crate::metrics::RESIDENT_PHYSICAL_SIZE_GLOBAL.sub(sz);
2835 526 : }
2836 :
2837 1700 : pub(crate) fn resident_physical_size_add(&self, sz: u64) {
2838 1700 : self.resident_physical_size_gauge.add(sz);
2839 1700 : crate::metrics::RESIDENT_PHYSICAL_SIZE_GLOBAL.add(sz);
2840 1700 : }
2841 :
2842 10 : pub(crate) fn resident_physical_size_get(&self) -> u64 {
2843 10 : self.resident_physical_size_gauge.get()
2844 10 : }
2845 :
2846 1172 : pub(crate) fn flush_wait_upload_time_gauge_add(&self, duration: f64) {
2847 1172 : self.flush_wait_upload_time_gauge.add(duration);
2848 1172 : crate::metrics::FLUSH_WAIT_UPLOAD_TIME
2849 1172 : .get_metric_with_label_values(&[&self.tenant_id, &self.shard_id, &self.timeline_id])
2850 1172 : .unwrap()
2851 1172 : .add(duration);
2852 1172 : }
2853 :
2854 10 : pub(crate) fn shutdown(&self) {
2855 10 : let was_shutdown = self
2856 10 : .shutdown
2857 10 : .swap(true, std::sync::atomic::Ordering::Relaxed);
2858 10 :
2859 10 : if was_shutdown {
2860 : // this happens on tenant deletion because tenant first shuts down timelines, then
2861 : // invokes timeline deletion which first shuts down the timeline again.
2862 : // TODO: this can be removed once https://github.com/neondatabase/neon/issues/5080
2863 0 : return;
2864 10 : }
2865 10 :
2866 10 : let tenant_id = &self.tenant_id;
2867 10 : let timeline_id = &self.timeline_id;
2868 10 : let shard_id = &self.shard_id;
2869 10 : let _ = LAST_RECORD_LSN.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2870 10 : let _ = DISK_CONSISTENT_LSN.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2871 10 : let _ = FLUSH_WAIT_UPLOAD_TIME.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2872 10 : let _ = STANDBY_HORIZON.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2873 10 : {
2874 10 : RESIDENT_PHYSICAL_SIZE_GLOBAL.sub(self.resident_physical_size_get());
2875 10 : let _ = RESIDENT_PHYSICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2876 10 : }
2877 10 : let _ = VISIBLE_PHYSICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2878 10 : let _ = CURRENT_LOGICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2879 10 : if let Some(metric) = Lazy::get(&DIRECTORY_ENTRIES_COUNT) {
2880 0 : let _ = metric.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2881 10 : }
2882 :
2883 10 : let _ = TIMELINE_ARCHIVE_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2884 10 : let _ = PITR_HISTORY_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2885 10 :
2886 10 : let _ = TIMELINE_LAYER_SIZE.remove_label_values(&[
2887 10 : tenant_id,
2888 10 : shard_id,
2889 10 : timeline_id,
2890 10 : MetricLayerKind::Image.into(),
2891 10 : ]);
2892 10 : let _ = TIMELINE_LAYER_COUNT.remove_label_values(&[
2893 10 : tenant_id,
2894 10 : shard_id,
2895 10 : timeline_id,
2896 10 : MetricLayerKind::Image.into(),
2897 10 : ]);
2898 10 : let _ = TIMELINE_LAYER_SIZE.remove_label_values(&[
2899 10 : tenant_id,
2900 10 : shard_id,
2901 10 : timeline_id,
2902 10 : MetricLayerKind::Delta.into(),
2903 10 : ]);
2904 10 : let _ = TIMELINE_LAYER_COUNT.remove_label_values(&[
2905 10 : tenant_id,
2906 10 : shard_id,
2907 10 : timeline_id,
2908 10 : MetricLayerKind::Delta.into(),
2909 10 : ]);
2910 10 :
2911 10 : let _ = EVICTIONS.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2912 10 : let _ = AUX_FILE_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2913 10 : let _ = VALID_LSN_LEASE_COUNT.remove_label_values(&[tenant_id, shard_id, timeline_id]);
2914 10 :
2915 10 : self.evictions_with_low_residence_duration
2916 10 : .write()
2917 10 : .unwrap()
2918 10 : .remove(tenant_id, shard_id, timeline_id);
2919 :
2920 : // The following metrics are born outside of the TimelineMetrics lifecycle but still
2921 : // removed at the end of it. The idea is to have the metrics outlive the
2922 : // entity during which they're observed, e.g., the smgr metrics shall
2923 : // outlive an individual smgr connection, but not the timeline.
2924 :
2925 90 : for op in StorageTimeOperation::VARIANTS {
2926 80 : let _ = STORAGE_TIME_SUM_PER_TIMELINE.remove_label_values(&[
2927 80 : op,
2928 80 : tenant_id,
2929 80 : shard_id,
2930 80 : timeline_id,
2931 80 : ]);
2932 80 : let _ = STORAGE_TIME_COUNT_PER_TIMELINE.remove_label_values(&[
2933 80 : op,
2934 80 : tenant_id,
2935 80 : shard_id,
2936 80 : timeline_id,
2937 80 : ]);
2938 80 : }
2939 :
2940 30 : for op in STORAGE_IO_SIZE_OPERATIONS {
2941 20 : let _ = STORAGE_IO_SIZE.remove_label_values(&[op, tenant_id, shard_id, timeline_id]);
2942 20 : }
2943 :
2944 10 : let _ = SMGR_QUERY_STARTED_PER_TENANT_TIMELINE.remove_label_values(&[
2945 10 : SmgrQueryType::GetPageAtLsn.into(),
2946 10 : tenant_id,
2947 10 : shard_id,
2948 10 : timeline_id,
2949 10 : ]);
2950 10 : let _ = SMGR_QUERY_TIME_PER_TENANT_TIMELINE.remove_label_values(&[
2951 10 : SmgrQueryType::GetPageAtLsn.into(),
2952 10 : tenant_id,
2953 10 : shard_id,
2954 10 : timeline_id,
2955 10 : ]);
2956 10 : let _ = PAGE_SERVICE_BATCH_SIZE_PER_TENANT_TIMELINE.remove_label_values(&[
2957 10 : tenant_id,
2958 10 : shard_id,
2959 10 : timeline_id,
2960 10 : ]);
2961 10 : let _ = PAGESERVER_TIMELINE_WAL_RECORDS_RECEIVED.remove_label_values(&[
2962 10 : tenant_id,
2963 10 : shard_id,
2964 10 : timeline_id,
2965 10 : ]);
2966 10 : let _ = PAGE_SERVICE_SMGR_FLUSH_INPROGRESS_MICROS.remove_label_values(&[
2967 10 : tenant_id,
2968 10 : shard_id,
2969 10 : timeline_id,
2970 10 : ]);
2971 10 : let _ = PAGE_SERVICE_SMGR_BATCH_WAIT_TIME.remove_label_values(&[
2972 10 : tenant_id,
2973 10 : shard_id,
2974 10 : timeline_id,
2975 10 : ]);
2976 10 : }
2977 : }
2978 :
2979 6 : pub(crate) fn remove_tenant_metrics(tenant_shard_id: &TenantShardId) {
2980 6 : // Only shard zero deals in synthetic sizes
2981 6 : if tenant_shard_id.is_shard_zero() {
2982 6 : let tid = tenant_shard_id.tenant_id.to_string();
2983 6 : let _ = TENANT_SYNTHETIC_SIZE_METRIC.remove_label_values(&[&tid]);
2984 6 : }
2985 :
2986 6 : tenant_throttling::remove_tenant_metrics(tenant_shard_id);
2987 6 :
2988 6 : // we leave the BROKEN_TENANTS_SET entry if any
2989 6 : }
2990 :
2991 : use futures::Future;
2992 : use pin_project_lite::pin_project;
2993 : use std::collections::HashMap;
2994 : use std::num::NonZeroUsize;
2995 : use std::pin::Pin;
2996 : use std::sync::atomic::AtomicU64;
2997 : use std::sync::{Arc, Mutex};
2998 : use std::task::{Context, Poll};
2999 : use std::time::{Duration, Instant};
3000 :
3001 : use crate::config::PageServerConf;
3002 : use crate::context::{PageContentKind, RequestContext};
3003 : use crate::task_mgr::TaskKind;
3004 : use crate::tenant::mgr::TenantSlot;
3005 : use crate::tenant::tasks::BackgroundLoopKind;
3006 : use crate::tenant::throttle::ThrottleResult;
3007 : use crate::tenant::Timeline;
3008 :
3009 : /// Maintain a per timeline gauge in addition to the global gauge.
3010 : pub(crate) struct PerTimelineRemotePhysicalSizeGauge {
3011 : last_set: AtomicU64,
3012 : gauge: UIntGauge,
3013 : }
3014 :
3015 : impl PerTimelineRemotePhysicalSizeGauge {
3016 456 : fn new(per_timeline_gauge: UIntGauge) -> Self {
3017 456 : Self {
3018 456 : last_set: AtomicU64::new(0),
3019 456 : gauge: per_timeline_gauge,
3020 456 : }
3021 456 : }
3022 1924 : pub(crate) fn set(&self, sz: u64) {
3023 1924 : self.gauge.set(sz);
3024 1924 : let prev = self.last_set.swap(sz, std::sync::atomic::Ordering::Relaxed);
3025 1924 : if sz < prev {
3026 22 : REMOTE_PHYSICAL_SIZE_GLOBAL.sub(prev - sz);
3027 1902 : } else {
3028 1902 : REMOTE_PHYSICAL_SIZE_GLOBAL.add(sz - prev);
3029 1902 : };
3030 1924 : }
3031 2 : pub(crate) fn get(&self) -> u64 {
3032 2 : self.gauge.get()
3033 2 : }
3034 : }
3035 :
3036 : impl Drop for PerTimelineRemotePhysicalSizeGauge {
3037 20 : fn drop(&mut self) {
3038 20 : REMOTE_PHYSICAL_SIZE_GLOBAL.sub(self.last_set.load(std::sync::atomic::Ordering::Relaxed));
3039 20 : }
3040 : }
3041 :
3042 : pub(crate) struct RemoteTimelineClientMetrics {
3043 : tenant_id: String,
3044 : shard_id: String,
3045 : timeline_id: String,
3046 : pub(crate) remote_physical_size_gauge: PerTimelineRemotePhysicalSizeGauge,
3047 : calls: Mutex<HashMap<(&'static str, &'static str), IntCounterPair>>,
3048 : bytes_started_counter: Mutex<HashMap<(&'static str, &'static str), IntCounter>>,
3049 : bytes_finished_counter: Mutex<HashMap<(&'static str, &'static str), IntCounter>>,
3050 : pub(crate) projected_remote_consistent_lsn_gauge: UIntGauge,
3051 : }
3052 :
3053 : impl RemoteTimelineClientMetrics {
3054 456 : pub fn new(tenant_shard_id: &TenantShardId, timeline_id: &TimelineId) -> Self {
3055 456 : let tenant_id_str = tenant_shard_id.tenant_id.to_string();
3056 456 : let shard_id_str = format!("{}", tenant_shard_id.shard_slug());
3057 456 : let timeline_id_str = timeline_id.to_string();
3058 456 :
3059 456 : let remote_physical_size_gauge = PerTimelineRemotePhysicalSizeGauge::new(
3060 456 : REMOTE_PHYSICAL_SIZE
3061 456 : .get_metric_with_label_values(&[&tenant_id_str, &shard_id_str, &timeline_id_str])
3062 456 : .unwrap(),
3063 456 : );
3064 456 :
3065 456 : let projected_remote_consistent_lsn_gauge = PROJECTED_REMOTE_CONSISTENT_LSN
3066 456 : .get_metric_with_label_values(&[&tenant_id_str, &shard_id_str, &timeline_id_str])
3067 456 : .unwrap();
3068 456 :
3069 456 : RemoteTimelineClientMetrics {
3070 456 : tenant_id: tenant_id_str,
3071 456 : shard_id: shard_id_str,
3072 456 : timeline_id: timeline_id_str,
3073 456 : calls: Mutex::new(HashMap::default()),
3074 456 : bytes_started_counter: Mutex::new(HashMap::default()),
3075 456 : bytes_finished_counter: Mutex::new(HashMap::default()),
3076 456 : remote_physical_size_gauge,
3077 456 : projected_remote_consistent_lsn_gauge,
3078 456 : }
3079 456 : }
3080 :
3081 2909 : pub fn remote_operation_time(
3082 2909 : &self,
3083 2909 : file_kind: &RemoteOpFileKind,
3084 2909 : op_kind: &RemoteOpKind,
3085 2909 : status: &'static str,
3086 2909 : ) -> Histogram {
3087 2909 : let key = (file_kind.as_str(), op_kind.as_str(), status);
3088 2909 : REMOTE_OPERATION_TIME
3089 2909 : .get_metric_with_label_values(&[key.0, key.1, key.2])
3090 2909 : .unwrap()
3091 2909 : }
3092 :
3093 6872 : fn calls_counter_pair(
3094 6872 : &self,
3095 6872 : file_kind: &RemoteOpFileKind,
3096 6872 : op_kind: &RemoteOpKind,
3097 6872 : ) -> IntCounterPair {
3098 6872 : let mut guard = self.calls.lock().unwrap();
3099 6872 : let key = (file_kind.as_str(), op_kind.as_str());
3100 6872 : let metric = guard.entry(key).or_insert_with(move || {
3101 814 : REMOTE_TIMELINE_CLIENT_CALLS
3102 814 : .get_metric_with_label_values(&[
3103 814 : &self.tenant_id,
3104 814 : &self.shard_id,
3105 814 : &self.timeline_id,
3106 814 : key.0,
3107 814 : key.1,
3108 814 : ])
3109 814 : .unwrap()
3110 6872 : });
3111 6872 : metric.clone()
3112 6872 : }
3113 :
3114 1580 : fn bytes_started_counter(
3115 1580 : &self,
3116 1580 : file_kind: &RemoteOpFileKind,
3117 1580 : op_kind: &RemoteOpKind,
3118 1580 : ) -> IntCounter {
3119 1580 : let mut guard = self.bytes_started_counter.lock().unwrap();
3120 1580 : let key = (file_kind.as_str(), op_kind.as_str());
3121 1580 : let metric = guard.entry(key).or_insert_with(move || {
3122 318 : REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER
3123 318 : .get_metric_with_label_values(&[
3124 318 : &self.tenant_id,
3125 318 : &self.shard_id,
3126 318 : &self.timeline_id,
3127 318 : key.0,
3128 318 : key.1,
3129 318 : ])
3130 318 : .unwrap()
3131 1580 : });
3132 1580 : metric.clone()
3133 1580 : }
3134 :
3135 2985 : fn bytes_finished_counter(
3136 2985 : &self,
3137 2985 : file_kind: &RemoteOpFileKind,
3138 2985 : op_kind: &RemoteOpKind,
3139 2985 : ) -> IntCounter {
3140 2985 : let mut guard = self.bytes_finished_counter.lock().unwrap();
3141 2985 : let key = (file_kind.as_str(), op_kind.as_str());
3142 2985 : let metric = guard.entry(key).or_insert_with(move || {
3143 318 : REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER
3144 318 : .get_metric_with_label_values(&[
3145 318 : &self.tenant_id,
3146 318 : &self.shard_id,
3147 318 : &self.timeline_id,
3148 318 : key.0,
3149 318 : key.1,
3150 318 : ])
3151 318 : .unwrap()
3152 2985 : });
3153 2985 : metric.clone()
3154 2985 : }
3155 : }
3156 :
3157 : #[cfg(test)]
3158 : impl RemoteTimelineClientMetrics {
3159 6 : pub fn get_bytes_started_counter_value(
3160 6 : &self,
3161 6 : file_kind: &RemoteOpFileKind,
3162 6 : op_kind: &RemoteOpKind,
3163 6 : ) -> Option<u64> {
3164 6 : let guard = self.bytes_started_counter.lock().unwrap();
3165 6 : let key = (file_kind.as_str(), op_kind.as_str());
3166 6 : guard.get(&key).map(|counter| counter.get())
3167 6 : }
3168 :
3169 6 : pub fn get_bytes_finished_counter_value(
3170 6 : &self,
3171 6 : file_kind: &RemoteOpFileKind,
3172 6 : op_kind: &RemoteOpKind,
3173 6 : ) -> Option<u64> {
3174 6 : let guard = self.bytes_finished_counter.lock().unwrap();
3175 6 : let key = (file_kind.as_str(), op_kind.as_str());
3176 6 : guard.get(&key).map(|counter| counter.get())
3177 6 : }
3178 : }
3179 :
3180 : /// See [`RemoteTimelineClientMetrics::call_begin`].
3181 : #[must_use]
3182 : pub(crate) struct RemoteTimelineClientCallMetricGuard {
3183 : /// Decremented on drop.
3184 : calls_counter_pair: Option<IntCounterPair>,
3185 : /// If Some(), this references the bytes_finished metric, and we increment it by the given `u64` on drop.
3186 : bytes_finished: Option<(IntCounter, u64)>,
3187 : }
3188 :
3189 : impl RemoteTimelineClientCallMetricGuard {
3190 : /// Consume this guard object without performing the metric updates it would do on `drop()`.
3191 : /// The caller vouches to do the metric updates manually.
3192 3628 : pub fn will_decrement_manually(mut self) {
3193 3628 : let RemoteTimelineClientCallMetricGuard {
3194 3628 : calls_counter_pair,
3195 3628 : bytes_finished,
3196 3628 : } = &mut self;
3197 3628 : calls_counter_pair.take();
3198 3628 : bytes_finished.take();
3199 3628 : }
3200 : }
3201 :
3202 : impl Drop for RemoteTimelineClientCallMetricGuard {
3203 3654 : fn drop(&mut self) {
3204 3654 : let RemoteTimelineClientCallMetricGuard {
3205 3654 : calls_counter_pair,
3206 3654 : bytes_finished,
3207 3654 : } = self;
3208 3654 : if let Some(guard) = calls_counter_pair.take() {
3209 26 : guard.dec();
3210 3628 : }
3211 3654 : if let Some((bytes_finished_metric, value)) = bytes_finished {
3212 0 : bytes_finished_metric.inc_by(*value);
3213 3654 : }
3214 3654 : }
3215 : }
3216 :
3217 : /// The enum variants communicate to the [`RemoteTimelineClientMetrics`] whether to
3218 : /// track the byte size of this call in applicable metric(s).
3219 : pub(crate) enum RemoteTimelineClientMetricsCallTrackSize {
3220 : /// Do not account for this call's byte size in any metrics.
3221 : /// The `reason` field is there to make the call sites self-documenting
3222 : /// about why they don't need the metric.
3223 : DontTrackSize { reason: &'static str },
3224 : /// Track the byte size of the call in applicable metric(s).
3225 : Bytes(u64),
3226 : }
3227 :
3228 : impl RemoteTimelineClientMetrics {
3229 : /// Update the metrics that change when a call to the remote timeline client instance starts.
3230 : ///
3231 : /// Drop the returned guard object once the operation is finished to updates corresponding metrics that track completions.
3232 : /// Or, use [`RemoteTimelineClientCallMetricGuard::will_decrement_manually`] and [`call_end`](Self::call_end) if that
3233 : /// is more suitable.
3234 : /// Never do both.
3235 3654 : pub(crate) fn call_begin(
3236 3654 : &self,
3237 3654 : file_kind: &RemoteOpFileKind,
3238 3654 : op_kind: &RemoteOpKind,
3239 3654 : size: RemoteTimelineClientMetricsCallTrackSize,
3240 3654 : ) -> RemoteTimelineClientCallMetricGuard {
3241 3654 : let calls_counter_pair = self.calls_counter_pair(file_kind, op_kind);
3242 3654 : calls_counter_pair.inc();
3243 :
3244 3654 : let bytes_finished = match size {
3245 2074 : RemoteTimelineClientMetricsCallTrackSize::DontTrackSize { reason: _reason } => {
3246 2074 : // nothing to do
3247 2074 : None
3248 : }
3249 1580 : RemoteTimelineClientMetricsCallTrackSize::Bytes(size) => {
3250 1580 : self.bytes_started_counter(file_kind, op_kind).inc_by(size);
3251 1580 : let finished_counter = self.bytes_finished_counter(file_kind, op_kind);
3252 1580 : Some((finished_counter, size))
3253 : }
3254 : };
3255 3654 : RemoteTimelineClientCallMetricGuard {
3256 3654 : calls_counter_pair: Some(calls_counter_pair),
3257 3654 : bytes_finished,
3258 3654 : }
3259 3654 : }
3260 :
3261 : /// Manually udpate the metrics that track completions, instead of using the guard object.
3262 : /// Using the guard object is generally preferable.
3263 : /// See [`call_begin`](Self::call_begin) for more context.
3264 3218 : pub(crate) fn call_end(
3265 3218 : &self,
3266 3218 : file_kind: &RemoteOpFileKind,
3267 3218 : op_kind: &RemoteOpKind,
3268 3218 : size: RemoteTimelineClientMetricsCallTrackSize,
3269 3218 : ) {
3270 3218 : let calls_counter_pair = self.calls_counter_pair(file_kind, op_kind);
3271 3218 : calls_counter_pair.dec();
3272 3218 : match size {
3273 1813 : RemoteTimelineClientMetricsCallTrackSize::DontTrackSize { reason: _reason } => {}
3274 1405 : RemoteTimelineClientMetricsCallTrackSize::Bytes(size) => {
3275 1405 : self.bytes_finished_counter(file_kind, op_kind).inc_by(size);
3276 1405 : }
3277 : }
3278 3218 : }
3279 : }
3280 :
3281 : impl Drop for RemoteTimelineClientMetrics {
3282 20 : fn drop(&mut self) {
3283 20 : let RemoteTimelineClientMetrics {
3284 20 : tenant_id,
3285 20 : shard_id,
3286 20 : timeline_id,
3287 20 : remote_physical_size_gauge,
3288 20 : calls,
3289 20 : bytes_started_counter,
3290 20 : bytes_finished_counter,
3291 20 : projected_remote_consistent_lsn_gauge,
3292 20 : } = self;
3293 24 : for ((a, b), _) in calls.get_mut().unwrap().drain() {
3294 24 : let mut res = [Ok(()), Ok(())];
3295 24 : REMOTE_TIMELINE_CLIENT_CALLS
3296 24 : .remove_label_values(&mut res, &[tenant_id, shard_id, timeline_id, a, b]);
3297 24 : // don't care about results
3298 24 : }
3299 20 : for ((a, b), _) in bytes_started_counter.get_mut().unwrap().drain() {
3300 6 : let _ = REMOTE_TIMELINE_CLIENT_BYTES_STARTED_COUNTER.remove_label_values(&[
3301 6 : tenant_id,
3302 6 : shard_id,
3303 6 : timeline_id,
3304 6 : a,
3305 6 : b,
3306 6 : ]);
3307 6 : }
3308 20 : for ((a, b), _) in bytes_finished_counter.get_mut().unwrap().drain() {
3309 6 : let _ = REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER.remove_label_values(&[
3310 6 : tenant_id,
3311 6 : shard_id,
3312 6 : timeline_id,
3313 6 : a,
3314 6 : b,
3315 6 : ]);
3316 6 : }
3317 20 : {
3318 20 : let _ = remote_physical_size_gauge; // use to avoid 'unused' warning in desctructuring above
3319 20 : let _ = REMOTE_PHYSICAL_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
3320 20 : }
3321 20 : {
3322 20 : let _ = projected_remote_consistent_lsn_gauge;
3323 20 : let _ = PROJECTED_REMOTE_CONSISTENT_LSN.remove_label_values(&[
3324 20 : tenant_id,
3325 20 : shard_id,
3326 20 : timeline_id,
3327 20 : ]);
3328 20 : }
3329 20 : }
3330 : }
3331 :
3332 : /// Wrapper future that measures the time spent by a remote storage operation,
3333 : /// and records the time and success/failure as a prometheus metric.
3334 : pub(crate) trait MeasureRemoteOp: Sized {
3335 2994 : fn measure_remote_op(
3336 2994 : self,
3337 2994 : file_kind: RemoteOpFileKind,
3338 2994 : op: RemoteOpKind,
3339 2994 : metrics: Arc<RemoteTimelineClientMetrics>,
3340 2994 : ) -> MeasuredRemoteOp<Self> {
3341 2994 : let start = Instant::now();
3342 2994 : MeasuredRemoteOp {
3343 2994 : inner: self,
3344 2994 : file_kind,
3345 2994 : op,
3346 2994 : start,
3347 2994 : metrics,
3348 2994 : }
3349 2994 : }
3350 : }
3351 :
3352 : impl<T: Sized> MeasureRemoteOp for T {}
3353 :
3354 : pin_project! {
3355 : pub(crate) struct MeasuredRemoteOp<F>
3356 : {
3357 : #[pin]
3358 : inner: F,
3359 : file_kind: RemoteOpFileKind,
3360 : op: RemoteOpKind,
3361 : start: Instant,
3362 : metrics: Arc<RemoteTimelineClientMetrics>,
3363 : }
3364 : }
3365 :
3366 : impl<F: Future<Output = Result<O, E>>, O, E> Future for MeasuredRemoteOp<F> {
3367 : type Output = Result<O, E>;
3368 :
3369 47129 : fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
3370 47129 : let this = self.project();
3371 47129 : let poll_result = this.inner.poll(cx);
3372 47129 : if let Poll::Ready(ref res) = poll_result {
3373 2909 : let duration = this.start.elapsed();
3374 2909 : let status = if res.is_ok() { &"success" } else { &"failure" };
3375 2909 : this.metrics
3376 2909 : .remote_operation_time(this.file_kind, this.op, status)
3377 2909 : .observe(duration.as_secs_f64());
3378 44220 : }
3379 47129 : poll_result
3380 47129 : }
3381 : }
3382 :
3383 : pub mod tokio_epoll_uring {
3384 : use std::{
3385 : collections::HashMap,
3386 : sync::{Arc, Mutex},
3387 : };
3388 :
3389 : use metrics::{register_histogram, register_int_counter, Histogram, LocalHistogram, UIntGauge};
3390 : use once_cell::sync::Lazy;
3391 :
3392 : /// Shared storage for tokio-epoll-uring thread local metrics.
3393 : pub(crate) static THREAD_LOCAL_METRICS_STORAGE: Lazy<ThreadLocalMetricsStorage> =
3394 116 : Lazy::new(|| {
3395 116 : let slots_submission_queue_depth = register_histogram!(
3396 116 : "pageserver_tokio_epoll_uring_slots_submission_queue_depth",
3397 116 : "The slots waiters queue depth of each tokio_epoll_uring system",
3398 116 : vec![1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0],
3399 116 : )
3400 116 : .expect("failed to define a metric");
3401 116 : ThreadLocalMetricsStorage {
3402 116 : observers: Mutex::new(HashMap::new()),
3403 116 : slots_submission_queue_depth,
3404 116 : }
3405 116 : });
3406 :
3407 : pub struct ThreadLocalMetricsStorage {
3408 : /// List of thread local metrics observers.
3409 : observers: Mutex<HashMap<u64, Arc<ThreadLocalMetrics>>>,
3410 : /// A histogram shared between all thread local systems
3411 : /// for collecting slots submission queue depth.
3412 : slots_submission_queue_depth: Histogram,
3413 : }
3414 :
3415 : /// Each thread-local [`tokio_epoll_uring::System`] gets one of these as its
3416 : /// [`tokio_epoll_uring::metrics::PerSystemMetrics`] generic.
3417 : ///
3418 : /// The System makes observations into [`Self`] and periodically, the collector
3419 : /// comes along and flushes [`Self`] into the shared storage [`THREAD_LOCAL_METRICS_STORAGE`].
3420 : ///
3421 : /// [`LocalHistogram`] is `!Send`, so, we need to put it behind a [`Mutex`].
3422 : /// But except for the periodic flush, the lock is uncontended so there's no waiting
3423 : /// for cache coherence protocol to get an exclusive cache line.
3424 : pub struct ThreadLocalMetrics {
3425 : /// Local observer of thread local tokio-epoll-uring system's slots waiters queue depth.
3426 : slots_submission_queue_depth: Mutex<LocalHistogram>,
3427 : }
3428 :
3429 : impl ThreadLocalMetricsStorage {
3430 : /// Registers a new thread local system. Returns a thread local metrics observer.
3431 527 : pub fn register_system(&self, id: u64) -> Arc<ThreadLocalMetrics> {
3432 527 : let per_system_metrics = Arc::new(ThreadLocalMetrics::new(
3433 527 : self.slots_submission_queue_depth.local(),
3434 527 : ));
3435 527 : let mut g = self.observers.lock().unwrap();
3436 527 : g.insert(id, Arc::clone(&per_system_metrics));
3437 527 : per_system_metrics
3438 527 : }
3439 :
3440 : /// Removes metrics observer for a thread local system.
3441 : /// This should be called before dropping a thread local system.
3442 116 : pub fn remove_system(&self, id: u64) {
3443 116 : let mut g = self.observers.lock().unwrap();
3444 116 : g.remove(&id);
3445 116 : }
3446 :
3447 : /// Flush all thread local metrics to the shared storage.
3448 0 : pub fn flush_thread_local_metrics(&self) {
3449 0 : let g = self.observers.lock().unwrap();
3450 0 : g.values().for_each(|local| {
3451 0 : local.flush();
3452 0 : });
3453 0 : }
3454 : }
3455 :
3456 : impl ThreadLocalMetrics {
3457 527 : pub fn new(slots_submission_queue_depth: LocalHistogram) -> Self {
3458 527 : ThreadLocalMetrics {
3459 527 : slots_submission_queue_depth: Mutex::new(slots_submission_queue_depth),
3460 527 : }
3461 527 : }
3462 :
3463 : /// Flushes the thread local metrics to shared aggregator.
3464 0 : pub fn flush(&self) {
3465 0 : let Self {
3466 0 : slots_submission_queue_depth,
3467 0 : } = self;
3468 0 : slots_submission_queue_depth.lock().unwrap().flush();
3469 0 : }
3470 : }
3471 :
3472 : impl tokio_epoll_uring::metrics::PerSystemMetrics for ThreadLocalMetrics {
3473 904529 : fn observe_slots_submission_queue_depth(&self, queue_depth: u64) {
3474 904529 : let Self {
3475 904529 : slots_submission_queue_depth,
3476 904529 : } = self;
3477 904529 : slots_submission_queue_depth
3478 904529 : .lock()
3479 904529 : .unwrap()
3480 904529 : .observe(queue_depth as f64);
3481 904529 : }
3482 : }
3483 :
3484 : pub struct Collector {
3485 : descs: Vec<metrics::core::Desc>,
3486 : systems_created: UIntGauge,
3487 : systems_destroyed: UIntGauge,
3488 : thread_local_metrics_storage: &'static ThreadLocalMetricsStorage,
3489 : }
3490 :
3491 : impl metrics::core::Collector for Collector {
3492 0 : fn desc(&self) -> Vec<&metrics::core::Desc> {
3493 0 : self.descs.iter().collect()
3494 0 : }
3495 :
3496 0 : fn collect(&self) -> Vec<metrics::proto::MetricFamily> {
3497 0 : let mut mfs = Vec::with_capacity(Self::NMETRICS);
3498 0 : let tokio_epoll_uring::metrics::GlobalMetrics {
3499 0 : systems_created,
3500 0 : systems_destroyed,
3501 0 : } = tokio_epoll_uring::metrics::global();
3502 0 : self.systems_created.set(systems_created);
3503 0 : mfs.extend(self.systems_created.collect());
3504 0 : self.systems_destroyed.set(systems_destroyed);
3505 0 : mfs.extend(self.systems_destroyed.collect());
3506 0 :
3507 0 : self.thread_local_metrics_storage
3508 0 : .flush_thread_local_metrics();
3509 0 :
3510 0 : mfs.extend(
3511 0 : self.thread_local_metrics_storage
3512 0 : .slots_submission_queue_depth
3513 0 : .collect(),
3514 0 : );
3515 0 : mfs
3516 0 : }
3517 : }
3518 :
3519 : impl Collector {
3520 : const NMETRICS: usize = 3;
3521 :
3522 : #[allow(clippy::new_without_default)]
3523 0 : pub fn new() -> Self {
3524 0 : let mut descs = Vec::new();
3525 0 :
3526 0 : let systems_created = UIntGauge::new(
3527 0 : "pageserver_tokio_epoll_uring_systems_created",
3528 0 : "counter of tokio-epoll-uring systems that were created",
3529 0 : )
3530 0 : .unwrap();
3531 0 : descs.extend(
3532 0 : metrics::core::Collector::desc(&systems_created)
3533 0 : .into_iter()
3534 0 : .cloned(),
3535 0 : );
3536 0 :
3537 0 : let systems_destroyed = UIntGauge::new(
3538 0 : "pageserver_tokio_epoll_uring_systems_destroyed",
3539 0 : "counter of tokio-epoll-uring systems that were destroyed",
3540 0 : )
3541 0 : .unwrap();
3542 0 : descs.extend(
3543 0 : metrics::core::Collector::desc(&systems_destroyed)
3544 0 : .into_iter()
3545 0 : .cloned(),
3546 0 : );
3547 0 :
3548 0 : Self {
3549 0 : descs,
3550 0 : systems_created,
3551 0 : systems_destroyed,
3552 0 : thread_local_metrics_storage: &THREAD_LOCAL_METRICS_STORAGE,
3553 0 : }
3554 0 : }
3555 : }
3556 :
3557 116 : pub(crate) static THREAD_LOCAL_LAUNCH_SUCCESSES: Lazy<metrics::IntCounter> = Lazy::new(|| {
3558 116 : register_int_counter!(
3559 116 : "pageserver_tokio_epoll_uring_pageserver_thread_local_launch_success_count",
3560 116 : "Number of times where thread_local_system creation spanned multiple executor threads",
3561 116 : )
3562 116 : .unwrap()
3563 116 : });
3564 :
3565 0 : pub(crate) static THREAD_LOCAL_LAUNCH_FAILURES: Lazy<metrics::IntCounter> = Lazy::new(|| {
3566 0 : register_int_counter!(
3567 0 : "pageserver_tokio_epoll_uring_pageserver_thread_local_launch_failures_count",
3568 0 : "Number of times thread_local_system creation failed and was retried after back-off.",
3569 0 : )
3570 0 : .unwrap()
3571 0 : });
3572 : }
3573 :
3574 : pub(crate) mod tenant_throttling {
3575 : use metrics::{register_int_counter_vec, IntCounter};
3576 : use once_cell::sync::Lazy;
3577 : use utils::shard::TenantShardId;
3578 :
3579 : pub(crate) struct GlobalAndPerTenantIntCounter {
3580 : global: IntCounter,
3581 : per_tenant: IntCounter,
3582 : }
3583 :
3584 : impl GlobalAndPerTenantIntCounter {
3585 : #[inline(always)]
3586 0 : pub(crate) fn inc(&self) {
3587 0 : self.inc_by(1)
3588 0 : }
3589 : #[inline(always)]
3590 0 : pub(crate) fn inc_by(&self, n: u64) {
3591 0 : self.global.inc_by(n);
3592 0 : self.per_tenant.inc_by(n);
3593 0 : }
3594 : }
3595 :
3596 : pub(crate) struct Metrics<const KIND: usize> {
3597 : pub(super) count_accounted_start: GlobalAndPerTenantIntCounter,
3598 : pub(super) count_accounted_finish: GlobalAndPerTenantIntCounter,
3599 : pub(super) wait_time: GlobalAndPerTenantIntCounter,
3600 : pub(super) count_throttled: GlobalAndPerTenantIntCounter,
3601 : }
3602 :
3603 202 : static COUNT_ACCOUNTED_START: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
3604 202 : register_int_counter_vec!(
3605 202 : "pageserver_tenant_throttling_count_accounted_start_global",
3606 202 : "Count of tenant throttling starts, by kind of throttle.",
3607 202 : &["kind"]
3608 202 : )
3609 202 : .unwrap()
3610 202 : });
3611 202 : static COUNT_ACCOUNTED_START_PER_TENANT: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
3612 202 : register_int_counter_vec!(
3613 202 : "pageserver_tenant_throttling_count_accounted_start",
3614 202 : "Count of tenant throttling starts, by kind of throttle.",
3615 202 : &["kind", "tenant_id", "shard_id"]
3616 202 : )
3617 202 : .unwrap()
3618 202 : });
3619 202 : static COUNT_ACCOUNTED_FINISH: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
3620 202 : register_int_counter_vec!(
3621 202 : "pageserver_tenant_throttling_count_accounted_finish_global",
3622 202 : "Count of tenant throttling finishes, by kind of throttle.",
3623 202 : &["kind"]
3624 202 : )
3625 202 : .unwrap()
3626 202 : });
3627 202 : static COUNT_ACCOUNTED_FINISH_PER_TENANT: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
3628 202 : register_int_counter_vec!(
3629 202 : "pageserver_tenant_throttling_count_accounted_finish",
3630 202 : "Count of tenant throttling finishes, by kind of throttle.",
3631 202 : &["kind", "tenant_id", "shard_id"]
3632 202 : )
3633 202 : .unwrap()
3634 202 : });
3635 202 : static WAIT_USECS: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
3636 202 : register_int_counter_vec!(
3637 202 : "pageserver_tenant_throttling_wait_usecs_sum_global",
3638 202 : "Sum of microseconds that spent waiting throttle by kind of throttle.",
3639 202 : &["kind"]
3640 202 : )
3641 202 : .unwrap()
3642 202 : });
3643 202 : static WAIT_USECS_PER_TENANT: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
3644 202 : register_int_counter_vec!(
3645 202 : "pageserver_tenant_throttling_wait_usecs_sum",
3646 202 : "Sum of microseconds that spent waiting throttle by kind of throttle.",
3647 202 : &["kind", "tenant_id", "shard_id"]
3648 202 : )
3649 202 : .unwrap()
3650 202 : });
3651 :
3652 202 : static WAIT_COUNT: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
3653 202 : register_int_counter_vec!(
3654 202 : "pageserver_tenant_throttling_count_global",
3655 202 : "Count of tenant throttlings, by kind of throttle.",
3656 202 : &["kind"]
3657 202 : )
3658 202 : .unwrap()
3659 202 : });
3660 202 : static WAIT_COUNT_PER_TENANT: Lazy<metrics::IntCounterVec> = Lazy::new(|| {
3661 202 : register_int_counter_vec!(
3662 202 : "pageserver_tenant_throttling_count",
3663 202 : "Count of tenant throttlings, by kind of throttle.",
3664 202 : &["kind", "tenant_id", "shard_id"]
3665 202 : )
3666 202 : .unwrap()
3667 202 : });
3668 :
3669 : const KINDS: &[&str] = &["pagestream"];
3670 : pub type Pagestream = Metrics<0>;
3671 :
3672 : impl<const KIND: usize> Metrics<KIND> {
3673 220 : pub(crate) fn new(tenant_shard_id: &TenantShardId) -> Self {
3674 220 : let per_tenant_label_values = &[
3675 220 : KINDS[KIND],
3676 220 : &tenant_shard_id.tenant_id.to_string(),
3677 220 : &tenant_shard_id.shard_slug().to_string(),
3678 220 : ];
3679 220 : Metrics {
3680 220 : count_accounted_start: {
3681 220 : GlobalAndPerTenantIntCounter {
3682 220 : global: COUNT_ACCOUNTED_START.with_label_values(&[KINDS[KIND]]),
3683 220 : per_tenant: COUNT_ACCOUNTED_START_PER_TENANT
3684 220 : .with_label_values(per_tenant_label_values),
3685 220 : }
3686 220 : },
3687 220 : count_accounted_finish: {
3688 220 : GlobalAndPerTenantIntCounter {
3689 220 : global: COUNT_ACCOUNTED_FINISH.with_label_values(&[KINDS[KIND]]),
3690 220 : per_tenant: COUNT_ACCOUNTED_FINISH_PER_TENANT
3691 220 : .with_label_values(per_tenant_label_values),
3692 220 : }
3693 220 : },
3694 220 : wait_time: {
3695 220 : GlobalAndPerTenantIntCounter {
3696 220 : global: WAIT_USECS.with_label_values(&[KINDS[KIND]]),
3697 220 : per_tenant: WAIT_USECS_PER_TENANT
3698 220 : .with_label_values(per_tenant_label_values),
3699 220 : }
3700 220 : },
3701 220 : count_throttled: {
3702 220 : GlobalAndPerTenantIntCounter {
3703 220 : global: WAIT_COUNT.with_label_values(&[KINDS[KIND]]),
3704 220 : per_tenant: WAIT_COUNT_PER_TENANT
3705 220 : .with_label_values(per_tenant_label_values),
3706 220 : }
3707 220 : },
3708 220 : }
3709 220 : }
3710 : }
3711 :
3712 0 : pub(crate) fn preinitialize_global_metrics() {
3713 0 : Lazy::force(&COUNT_ACCOUNTED_START);
3714 0 : Lazy::force(&COUNT_ACCOUNTED_FINISH);
3715 0 : Lazy::force(&WAIT_USECS);
3716 0 : Lazy::force(&WAIT_COUNT);
3717 0 : }
3718 :
3719 6 : pub(crate) fn remove_tenant_metrics(tenant_shard_id: &TenantShardId) {
3720 24 : for m in &[
3721 6 : &COUNT_ACCOUNTED_START_PER_TENANT,
3722 6 : &COUNT_ACCOUNTED_FINISH_PER_TENANT,
3723 6 : &WAIT_USECS_PER_TENANT,
3724 6 : &WAIT_COUNT_PER_TENANT,
3725 6 : ] {
3726 48 : for kind in KINDS {
3727 24 : let _ = m.remove_label_values(&[
3728 24 : kind,
3729 24 : &tenant_shard_id.tenant_id.to_string(),
3730 24 : &tenant_shard_id.shard_slug().to_string(),
3731 24 : ]);
3732 24 : }
3733 : }
3734 6 : }
3735 : }
3736 :
3737 : pub(crate) mod disk_usage_based_eviction {
3738 : use super::*;
3739 :
3740 : pub(crate) struct Metrics {
3741 : pub(crate) tenant_collection_time: Histogram,
3742 : pub(crate) tenant_layer_count: Histogram,
3743 : pub(crate) layers_collected: IntCounter,
3744 : pub(crate) layers_selected: IntCounter,
3745 : pub(crate) layers_evicted: IntCounter,
3746 : }
3747 :
3748 : impl Default for Metrics {
3749 0 : fn default() -> Self {
3750 0 : let tenant_collection_time = register_histogram!(
3751 0 : "pageserver_disk_usage_based_eviction_tenant_collection_seconds",
3752 0 : "Time spent collecting layers from a tenant -- not normalized by collected layer amount",
3753 0 : vec![0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0, 10.0]
3754 0 : )
3755 0 : .unwrap();
3756 0 :
3757 0 : let tenant_layer_count = register_histogram!(
3758 0 : "pageserver_disk_usage_based_eviction_tenant_collected_layers",
3759 0 : "Amount of layers gathered from a tenant",
3760 0 : vec![5.0, 50.0, 500.0, 5000.0, 50000.0]
3761 0 : )
3762 0 : .unwrap();
3763 0 :
3764 0 : let layers_collected = register_int_counter!(
3765 0 : "pageserver_disk_usage_based_eviction_collected_layers_total",
3766 0 : "Amount of layers collected"
3767 0 : )
3768 0 : .unwrap();
3769 0 :
3770 0 : let layers_selected = register_int_counter!(
3771 0 : "pageserver_disk_usage_based_eviction_select_layers_total",
3772 0 : "Amount of layers selected"
3773 0 : )
3774 0 : .unwrap();
3775 0 :
3776 0 : let layers_evicted = register_int_counter!(
3777 0 : "pageserver_disk_usage_based_eviction_evicted_layers_total",
3778 0 : "Amount of layers successfully evicted"
3779 0 : )
3780 0 : .unwrap();
3781 0 :
3782 0 : Self {
3783 0 : tenant_collection_time,
3784 0 : tenant_layer_count,
3785 0 : layers_collected,
3786 0 : layers_selected,
3787 0 : layers_evicted,
3788 0 : }
3789 0 : }
3790 : }
3791 :
3792 : pub(crate) static METRICS: Lazy<Metrics> = Lazy::new(Metrics::default);
3793 : }
3794 :
3795 196 : static TOKIO_EXECUTOR_THREAD_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
3796 196 : register_uint_gauge_vec!(
3797 196 : "pageserver_tokio_executor_thread_configured_count",
3798 196 : "Total number of configued tokio executor threads in the process.
3799 196 : The `setup` label denotes whether we're running with multiple runtimes or a single runtime.",
3800 196 : &["setup"],
3801 196 : )
3802 196 : .unwrap()
3803 196 : });
3804 :
3805 196 : pub(crate) fn set_tokio_runtime_setup(setup: &str, num_threads: NonZeroUsize) {
3806 : static SERIALIZE: std::sync::Mutex<()> = std::sync::Mutex::new(());
3807 196 : let _guard = SERIALIZE.lock().unwrap();
3808 196 : TOKIO_EXECUTOR_THREAD_COUNT.reset();
3809 196 : TOKIO_EXECUTOR_THREAD_COUNT
3810 196 : .get_metric_with_label_values(&[setup])
3811 196 : .unwrap()
3812 196 : .set(u64::try_from(num_threads.get()).unwrap());
3813 196 : }
3814 :
3815 0 : pub fn preinitialize_metrics(conf: &'static PageServerConf) {
3816 0 : set_page_service_config_max_batch_size(&conf.page_service_pipelining);
3817 0 :
3818 0 : // Python tests need these and on some we do alerting.
3819 0 : //
3820 0 : // FIXME(4813): make it so that we have no top level metrics as this fn will easily fall out of
3821 0 : // order:
3822 0 : // - global metrics reside in a Lazy<PageserverMetrics>
3823 0 : // - access via crate::metrics::PS_METRICS.some_metric.inc()
3824 0 : // - could move the statics into TimelineMetrics::new()?
3825 0 :
3826 0 : // counters
3827 0 : [
3828 0 : &UNEXPECTED_ONDEMAND_DOWNLOADS,
3829 0 : &WALRECEIVER_STARTED_CONNECTIONS,
3830 0 : &WALRECEIVER_BROKER_UPDATES,
3831 0 : &WALRECEIVER_CANDIDATES_ADDED,
3832 0 : &WALRECEIVER_CANDIDATES_REMOVED,
3833 0 : &tokio_epoll_uring::THREAD_LOCAL_LAUNCH_FAILURES,
3834 0 : &tokio_epoll_uring::THREAD_LOCAL_LAUNCH_SUCCESSES,
3835 0 : &REMOTE_ONDEMAND_DOWNLOADED_LAYERS,
3836 0 : &REMOTE_ONDEMAND_DOWNLOADED_BYTES,
3837 0 : &CIRCUIT_BREAKERS_BROKEN,
3838 0 : &CIRCUIT_BREAKERS_UNBROKEN,
3839 0 : &PAGE_SERVICE_SMGR_FLUSH_INPROGRESS_MICROS_GLOBAL,
3840 0 : ]
3841 0 : .into_iter()
3842 0 : .for_each(|c| {
3843 0 : Lazy::force(c);
3844 0 : });
3845 0 :
3846 0 : // Deletion queue stats
3847 0 : Lazy::force(&DELETION_QUEUE);
3848 0 :
3849 0 : // Tenant stats
3850 0 : Lazy::force(&TENANT);
3851 0 :
3852 0 : // Tenant manager stats
3853 0 : Lazy::force(&TENANT_MANAGER);
3854 0 :
3855 0 : Lazy::force(&crate::tenant::storage_layer::layer::LAYER_IMPL_METRICS);
3856 0 : Lazy::force(&disk_usage_based_eviction::METRICS);
3857 :
3858 0 : for state_name in pageserver_api::models::TenantState::VARIANTS {
3859 0 : // initialize the metric for all gauges, otherwise the time series might seemingly show
3860 0 : // values from last restart.
3861 0 : TENANT_STATE_METRIC.with_label_values(&[state_name]).set(0);
3862 0 : }
3863 :
3864 : // countervecs
3865 0 : [
3866 0 : &BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT,
3867 0 : &SMGR_QUERY_STARTED_GLOBAL,
3868 0 : ]
3869 0 : .into_iter()
3870 0 : .for_each(|c| {
3871 0 : Lazy::force(c);
3872 0 : });
3873 0 :
3874 0 : // gauges
3875 0 : WALRECEIVER_ACTIVE_MANAGERS.get();
3876 0 :
3877 0 : // histograms
3878 0 : [
3879 0 : &VEC_READ_NUM_LAYERS_VISITED,
3880 0 : &WAIT_LSN_TIME,
3881 0 : &WAL_REDO_TIME,
3882 0 : &WAL_REDO_RECORDS_HISTOGRAM,
3883 0 : &WAL_REDO_BYTES_HISTOGRAM,
3884 0 : &WAL_REDO_PROCESS_LAUNCH_DURATION_HISTOGRAM,
3885 0 : &PAGE_SERVICE_BATCH_SIZE_GLOBAL,
3886 0 : &PAGE_SERVICE_SMGR_BATCH_WAIT_TIME_GLOBAL,
3887 0 : ]
3888 0 : .into_iter()
3889 0 : .for_each(|h| {
3890 0 : Lazy::force(h);
3891 0 : });
3892 0 :
3893 0 : // Custom
3894 0 : Lazy::force(&RECONSTRUCT_TIME);
3895 0 : Lazy::force(&BASEBACKUP_QUERY_TIME);
3896 0 : Lazy::force(&COMPUTE_COMMANDS_COUNTERS);
3897 0 : Lazy::force(&tokio_epoll_uring::THREAD_LOCAL_METRICS_STORAGE);
3898 0 :
3899 0 : tenant_throttling::preinitialize_global_metrics();
3900 0 : }
|
