LCOV - code coverage report
Current view: top level - storage_controller/src - metrics.rs (source / functions) Coverage Total Hit
Test: 1b0a6a0c05cee5a7de360813c8034804e105ce1c.info Lines: 27.6 % 76 21
Test Date: 2025-03-12 00:01:28 Functions: 73.5 % 34 25

            Line data    Source code
       1              : //!
       2              : //! This module provides metric definitions for the storage controller.
       3              : //!
       4              : //! All metrics are grouped in [`StorageControllerMetricGroup`]. [`StorageControllerMetrics`] holds
       5              : //! the mentioned metrics and their encoder. It's globally available via the [`METRICS_REGISTRY`]
       6              : //! constant.
       7              : //!
       8              : //! The rest of the code defines label group types and deals with converting outer types to labels.
       9              : //!
      10              : use std::sync::Mutex;
      11              : 
      12              : use bytes::Bytes;
      13              : use measured::label::LabelValue;
      14              : use measured::metric::histogram;
      15              : use measured::{FixedCardinalityLabel, MetricGroup};
      16              : use metrics::NeonMetrics;
      17              : use once_cell::sync::Lazy;
      18              : use strum::IntoEnumIterator;
      19              : 
      20              : use crate::persistence::{DatabaseError, DatabaseOperation};
      21              : use crate::service::LeadershipStatus;
      22              : 
      23              : pub(crate) static METRICS_REGISTRY: Lazy<StorageControllerMetrics> =
      24              :     Lazy::new(StorageControllerMetrics::default);
      25              : 
      26            0 : pub fn preinitialize_metrics() {
      27            0 :     Lazy::force(&METRICS_REGISTRY);
      28            0 : }
      29              : 
      30              : pub(crate) struct StorageControllerMetrics {
      31              :     pub(crate) metrics_group: StorageControllerMetricGroup,
      32              :     encoder: Mutex<measured::text::BufferedTextEncoder>,
      33              : }
      34              : 
      35           17 : #[derive(measured::MetricGroup)]
      36              : #[metric(new())]
      37              : pub(crate) struct StorageControllerMetricGroup {
      38              :     /// Count of how many times we spawn a reconcile task
      39              :     pub(crate) storage_controller_reconcile_spawn: measured::Counter,
      40              : 
      41              :     /// Size of the in-memory map of tenant shards
      42              :     pub(crate) storage_controller_tenant_shards: measured::Gauge,
      43              : 
      44              :     /// Size of the in-memory map of pageserver_nodes
      45              :     pub(crate) storage_controller_pageserver_nodes: measured::Gauge,
      46              : 
      47              :     /// Reconciler tasks completed, broken down by success/failure/cancelled
      48              :     pub(crate) storage_controller_reconcile_complete:
      49              :         measured::CounterVec<ReconcileCompleteLabelGroupSet>,
      50              : 
      51              :     /// Count of how many times we make an optimization change to a tenant's scheduling
      52              :     pub(crate) storage_controller_schedule_optimization: measured::Counter,
      53              : 
      54              :     /// How many shards are not scheduled into their preferred AZ
      55              :     pub(crate) storage_controller_schedule_az_violation: measured::Gauge,
      56              : 
      57              :     /// How many shard locations (secondary or attached) on each node
      58              :     pub(crate) storage_controller_node_shards: measured::GaugeVec<NodeLabelGroupSet>,
      59              : 
      60              :     /// How many _attached_ shard locations on each node
      61              :     pub(crate) storage_controller_node_attached_shards: measured::GaugeVec<NodeLabelGroupSet>,
      62              : 
      63              :     /// How many _home_ shard locations on each node (i.e. the node's AZ matches the shard's
      64              :     /// preferred AZ)
      65              :     pub(crate) storage_controller_node_home_shards: measured::GaugeVec<NodeLabelGroupSet>,
      66              : 
      67              :     /// How many shards would like to reconcile but were blocked by concurrency limits
      68              :     pub(crate) storage_controller_pending_reconciles: measured::Gauge,
      69              : 
      70              :     /// HTTP request status counters for handled requests
      71              :     pub(crate) storage_controller_http_request_status:
      72              :         measured::CounterVec<HttpRequestStatusLabelGroupSet>,
      73              : 
      74              :     /// HTTP request handler latency across all status codes
      75              :     #[metric(metadata = histogram::Thresholds::exponential_buckets(0.1, 2.0))]
      76              :     pub(crate) storage_controller_http_request_latency:
      77              :         measured::HistogramVec<HttpRequestLatencyLabelGroupSet, 5>,
      78              : 
      79              :     /// HTTP rate limiting latency across all tenants and endpoints
      80              :     #[metric(metadata = histogram::Thresholds::exponential_buckets(0.1, 10.0))]
      81              :     pub(crate) storage_controller_http_request_rate_limited: measured::Histogram<10>,
      82              : 
      83              :     /// Count of HTTP requests to the pageserver that resulted in an error,
      84              :     /// broken down by the pageserver node id, request name and method
      85              :     pub(crate) storage_controller_pageserver_request_error:
      86              :         measured::CounterVec<PageserverRequestLabelGroupSet>,
      87              : 
      88              :     /// Count of HTTP requests to the safekeeper that resulted in an error,
      89              :     /// broken down by the safekeeper node id, request name and method
      90              :     pub(crate) storage_controller_safekeeper_request_error:
      91              :         measured::CounterVec<PageserverRequestLabelGroupSet>,
      92              : 
      93              :     /// Latency of HTTP requests to the pageserver, broken down by pageserver
      94              :     /// node id, request name and method. This include both successful and unsuccessful
      95              :     /// requests.
      96              :     #[metric(metadata = histogram::Thresholds::exponential_buckets(0.1, 2.0))]
      97              :     pub(crate) storage_controller_pageserver_request_latency:
      98              :         measured::HistogramVec<PageserverRequestLabelGroupSet, 5>,
      99              : 
     100              :     /// Latency of HTTP requests to the safekeeper, broken down by safekeeper
     101              :     /// node id, request name and method. This include both successful and unsuccessful
     102              :     /// requests.
     103              :     #[metric(metadata = histogram::Thresholds::exponential_buckets(0.1, 2.0))]
     104              :     pub(crate) storage_controller_safekeeper_request_latency:
     105              :         measured::HistogramVec<PageserverRequestLabelGroupSet, 5>,
     106              : 
     107              :     /// Count of pass-through HTTP requests to the pageserver that resulted in an error,
     108              :     /// broken down by the pageserver node id, request name and method
     109              :     pub(crate) storage_controller_passthrough_request_error:
     110              :         measured::CounterVec<PageserverRequestLabelGroupSet>,
     111              : 
     112              :     /// Latency of pass-through HTTP requests to the pageserver, broken down by pageserver
     113              :     /// node id, request name and method. This include both successful and unsuccessful
     114              :     /// requests.
     115              :     #[metric(metadata = histogram::Thresholds::exponential_buckets(0.1, 2.0))]
     116              :     pub(crate) storage_controller_passthrough_request_latency:
     117              :         measured::HistogramVec<PageserverRequestLabelGroupSet, 5>,
     118              : 
     119              :     /// Count of errors in database queries, broken down by error type and operation.
     120              :     pub(crate) storage_controller_database_query_error:
     121              :         measured::CounterVec<DatabaseQueryErrorLabelGroupSet>,
     122              : 
     123              :     /// Latency of database queries, broken down by operation.
     124              :     #[metric(metadata = histogram::Thresholds::exponential_buckets(0.1, 2.0))]
     125              :     pub(crate) storage_controller_database_query_latency:
     126              :         measured::HistogramVec<DatabaseQueryLatencyLabelGroupSet, 5>,
     127              : 
     128              :     pub(crate) storage_controller_leadership_status: measured::GaugeVec<LeadershipStatusGroupSet>,
     129              : 
     130              :     /// HTTP request status counters for handled requests
     131              :     pub(crate) storage_controller_reconcile_long_running:
     132              :         measured::CounterVec<ReconcileLongRunningLabelGroupSet>,
     133              : }
     134              : 
     135              : impl StorageControllerMetrics {
     136            0 :     pub(crate) fn encode(&self, neon_metrics: &NeonMetrics) -> Bytes {
     137            0 :         let mut encoder = self.encoder.lock().unwrap();
     138            0 :         neon_metrics
     139            0 :             .collect_group_into(&mut *encoder)
     140            0 :             .unwrap_or_else(|infallible| match infallible {});
     141            0 :         self.metrics_group
     142            0 :             .collect_group_into(&mut *encoder)
     143            0 :             .unwrap_or_else(|infallible| match infallible {});
     144            0 :         encoder.finish()
     145            0 :     }
     146              : }
     147              : 
     148              : impl Default for StorageControllerMetrics {
     149           17 :     fn default() -> Self {
     150           17 :         let mut metrics_group = StorageControllerMetricGroup::new();
     151           17 :         metrics_group
     152           17 :             .storage_controller_reconcile_complete
     153           17 :             .init_all_dense();
     154           17 : 
     155           17 :         Self {
     156           17 :             metrics_group,
     157           17 :             encoder: Mutex::new(measured::text::BufferedTextEncoder::new()),
     158           17 :         }
     159           17 :     }
     160              : }
     161              : 
     162          102 : #[derive(measured::LabelGroup, Clone)]
     163              : #[label(set = NodeLabelGroupSet)]
     164              : pub(crate) struct NodeLabelGroup<'a> {
     165              :     #[label(dynamic_with = lasso::ThreadedRodeo, default)]
     166              :     pub(crate) az: &'a str,
     167              :     #[label(dynamic_with = lasso::ThreadedRodeo, default)]
     168              :     pub(crate) node_id: &'a str,
     169              : }
     170              : 
     171           51 : #[derive(measured::LabelGroup)]
     172              : #[label(set = ReconcileCompleteLabelGroupSet)]
     173              : pub(crate) struct ReconcileCompleteLabelGroup {
     174              :     pub(crate) status: ReconcileOutcome,
     175              : }
     176              : 
     177           34 : #[derive(measured::LabelGroup)]
     178              : #[label(set = HttpRequestStatusLabelGroupSet)]
     179              : pub(crate) struct HttpRequestStatusLabelGroup<'a> {
     180              :     #[label(dynamic_with = lasso::ThreadedRodeo, default)]
     181              :     pub(crate) path: &'a str,
     182              :     pub(crate) method: Method,
     183              :     pub(crate) status: StatusCode,
     184              : }
     185              : 
     186           34 : #[derive(measured::LabelGroup)]
     187              : #[label(set = HttpRequestLatencyLabelGroupSet)]
     188              : pub(crate) struct HttpRequestLatencyLabelGroup<'a> {
     189              :     #[label(dynamic_with = lasso::ThreadedRodeo, default)]
     190              :     pub(crate) path: &'a str,
     191              :     pub(crate) method: Method,
     192              : }
     193              : 
     194          204 : #[derive(measured::LabelGroup, Clone)]
     195              : #[label(set = PageserverRequestLabelGroupSet)]
     196              : pub(crate) struct PageserverRequestLabelGroup<'a> {
     197              :     #[label(dynamic_with = lasso::ThreadedRodeo, default)]
     198              :     pub(crate) pageserver_id: &'a str,
     199              :     #[label(dynamic_with = lasso::ThreadedRodeo, default)]
     200              :     pub(crate) path: &'a str,
     201              :     pub(crate) method: Method,
     202              : }
     203              : 
     204           68 : #[derive(measured::LabelGroup)]
     205              : #[label(set = DatabaseQueryErrorLabelGroupSet)]
     206              : pub(crate) struct DatabaseQueryErrorLabelGroup {
     207              :     pub(crate) error_type: DatabaseErrorLabel,
     208              :     pub(crate) operation: DatabaseOperation,
     209              : }
     210              : 
     211           51 : #[derive(measured::LabelGroup)]
     212              : #[label(set = DatabaseQueryLatencyLabelGroupSet)]
     213              : pub(crate) struct DatabaseQueryLatencyLabelGroup {
     214              :     pub(crate) operation: DatabaseOperation,
     215              : }
     216              : 
     217           51 : #[derive(measured::LabelGroup)]
     218              : #[label(set = LeadershipStatusGroupSet)]
     219              : pub(crate) struct LeadershipStatusGroup {
     220              :     pub(crate) status: LeadershipStatus,
     221              : }
     222              : 
     223           34 : #[derive(measured::LabelGroup, Clone)]
     224              : #[label(set = ReconcileLongRunningLabelGroupSet)]
     225              : pub(crate) struct ReconcileLongRunningLabelGroup<'a> {
     226              :     #[label(dynamic_with = lasso::ThreadedRodeo, default)]
     227              :     pub(crate) tenant_id: &'a str,
     228              :     #[label(dynamic_with = lasso::ThreadedRodeo, default)]
     229              :     pub(crate) shard_number: &'a str,
     230              :     #[label(dynamic_with = lasso::ThreadedRodeo, default)]
     231              :     pub(crate) sequence: &'a str,
     232              : }
     233              : 
     234              : #[derive(FixedCardinalityLabel, Clone, Copy)]
     235              : pub(crate) enum ReconcileOutcome {
     236              :     #[label(rename = "ok")]
     237              :     Success,
     238              :     Error,
     239              :     Cancel,
     240              : }
     241              : 
     242              : #[derive(FixedCardinalityLabel, Copy, Clone)]
     243              : pub(crate) enum Method {
     244              :     Get,
     245              :     Put,
     246              :     Post,
     247              :     Delete,
     248              :     Other,
     249              : }
     250              : 
     251              : impl From<hyper::Method> for Method {
     252            0 :     fn from(value: hyper::Method) -> Self {
     253            0 :         if value == hyper::Method::GET {
     254            0 :             Method::Get
     255            0 :         } else if value == hyper::Method::PUT {
     256            0 :             Method::Put
     257            0 :         } else if value == hyper::Method::POST {
     258            0 :             Method::Post
     259            0 :         } else if value == hyper::Method::DELETE {
     260            0 :             Method::Delete
     261              :         } else {
     262            0 :             Method::Other
     263              :         }
     264            0 :     }
     265              : }
     266              : 
     267              : #[derive(Clone, Copy)]
     268              : pub(crate) struct StatusCode(pub(crate) hyper::http::StatusCode);
     269              : 
     270              : impl LabelValue for StatusCode {
     271            0 :     fn visit<V: measured::label::LabelVisitor>(&self, v: V) -> V::Output {
     272            0 :         v.write_int(self.0.as_u16() as i64)
     273            0 :     }
     274              : }
     275              : 
     276              : impl FixedCardinalityLabel for StatusCode {
     277            0 :     fn cardinality() -> usize {
     278            0 :         (100..1000).len()
     279            0 :     }
     280              : 
     281            0 :     fn encode(&self) -> usize {
     282            0 :         self.0.as_u16() as usize
     283            0 :     }
     284              : 
     285            0 :     fn decode(value: usize) -> Self {
     286            0 :         Self(hyper::http::StatusCode::from_u16(u16::try_from(value).unwrap()).unwrap())
     287            0 :     }
     288              : }
     289              : 
     290              : #[derive(FixedCardinalityLabel, Clone, Copy)]
     291              : pub(crate) enum DatabaseErrorLabel {
     292              :     Query,
     293              :     Connection,
     294              :     ConnectionPool,
     295              :     Logical,
     296              :     Migration,
     297              : }
     298              : 
     299              : impl DatabaseError {
     300            0 :     pub(crate) fn error_label(&self) -> DatabaseErrorLabel {
     301            0 :         match self {
     302            0 :             Self::Query(_) => DatabaseErrorLabel::Query,
     303            0 :             Self::Connection(_) => DatabaseErrorLabel::Connection,
     304            0 :             Self::ConnectionPool(_) => DatabaseErrorLabel::ConnectionPool,
     305            0 :             Self::Logical(_) => DatabaseErrorLabel::Logical,
     306            0 :             Self::Migration(_) => DatabaseErrorLabel::Migration,
     307              :         }
     308            0 :     }
     309              : }
     310              : 
     311              : /// Update the leadership status metric gauges to reflect the requested status
     312            0 : pub(crate) fn update_leadership_status(status: LeadershipStatus) {
     313            0 :     let status_metric = &METRICS_REGISTRY
     314            0 :         .metrics_group
     315            0 :         .storage_controller_leadership_status;
     316              : 
     317            0 :     for s in LeadershipStatus::iter() {
     318            0 :         if s == status {
     319            0 :             status_metric.set(LeadershipStatusGroup { status: s }, 1);
     320            0 :         } else {
     321            0 :             status_metric.set(LeadershipStatusGroup { status: s }, 0);
     322            0 :         }
     323              :     }
     324            0 : }
        

Generated by: LCOV version 2.1-beta