Line data Source code
1 : //!
2 : //! This module provides metric definitions for the storage controller.
3 : //!
4 : //! All metrics are grouped in [`StorageControllerMetricGroup`]. [`StorageControllerMetrics`] holds
5 : //! the mentioned metrics and their encoder. It's globally available via the [`METRICS_REGISTRY`]
6 : //! constant.
7 : //!
8 : //! The rest of the code defines label group types and deals with converting outer types to labels.
9 : //!
10 : use bytes::Bytes;
11 : use measured::{label::LabelValue, metric::histogram, FixedCardinalityLabel, MetricGroup};
12 : use metrics::NeonMetrics;
13 : use once_cell::sync::Lazy;
14 : use std::sync::Mutex;
15 : use strum::IntoEnumIterator;
16 :
17 : use crate::{
18 : persistence::{DatabaseError, DatabaseOperation},
19 : service::LeadershipStatus,
20 : };
21 :
22 : pub(crate) static METRICS_REGISTRY: Lazy<StorageControllerMetrics> =
23 : Lazy::new(StorageControllerMetrics::default);
24 :
25 0 : pub fn preinitialize_metrics() {
26 0 : Lazy::force(&METRICS_REGISTRY);
27 0 : }
28 :
29 : pub(crate) struct StorageControllerMetrics {
30 : pub(crate) metrics_group: StorageControllerMetricGroup,
31 : encoder: Mutex<measured::text::BufferedTextEncoder>,
32 : }
33 :
34 15 : #[derive(measured::MetricGroup)]
35 : #[metric(new())]
36 : pub(crate) struct StorageControllerMetricGroup {
37 : /// Count of how many times we spawn a reconcile task
38 : pub(crate) storage_controller_reconcile_spawn: measured::Counter,
39 :
40 : /// Size of the in-memory map of tenant shards
41 : pub(crate) storage_controller_tenant_shards: measured::Gauge,
42 :
43 : /// Size of the in-memory map of pageserver_nodes
44 : pub(crate) storage_controller_pageserver_nodes: measured::Gauge,
45 :
46 : /// Reconciler tasks completed, broken down by success/failure/cancelled
47 : pub(crate) storage_controller_reconcile_complete:
48 : measured::CounterVec<ReconcileCompleteLabelGroupSet>,
49 :
50 : /// Count of how many times we make an optimization change to a tenant's scheduling
51 : pub(crate) storage_controller_schedule_optimization: measured::Counter,
52 :
53 : /// How many shards are not scheduled into their preferred AZ
54 : pub(crate) storage_controller_schedule_az_violation: measured::Gauge,
55 :
56 : /// How many shard locations (secondary or attached) on each node
57 : pub(crate) storage_controller_node_shards: measured::GaugeVec<NodeLabelGroupSet>,
58 :
59 : /// How many _attached_ shard locations on each node
60 : pub(crate) storage_controller_node_attached_shards: measured::GaugeVec<NodeLabelGroupSet>,
61 :
62 : /// How many _home_ shard locations on each node (i.e. the node's AZ matches the shard's
63 : /// preferred AZ)
64 : pub(crate) storage_controller_node_home_shards: measured::GaugeVec<NodeLabelGroupSet>,
65 :
66 : /// How many shards would like to reconcile but were blocked by concurrency limits
67 : pub(crate) storage_controller_pending_reconciles: measured::Gauge,
68 :
69 : /// HTTP request status counters for handled requests
70 : pub(crate) storage_controller_http_request_status:
71 : measured::CounterVec<HttpRequestStatusLabelGroupSet>,
72 :
73 : /// HTTP request handler latency across all status codes
74 : #[metric(metadata = histogram::Thresholds::exponential_buckets(0.1, 2.0))]
75 : pub(crate) storage_controller_http_request_latency:
76 : measured::HistogramVec<HttpRequestLatencyLabelGroupSet, 5>,
77 :
78 : /// Count of HTTP requests to the pageserver that resulted in an error,
79 : /// broken down by the pageserver node id, request name and method
80 : pub(crate) storage_controller_pageserver_request_error:
81 : measured::CounterVec<PageserverRequestLabelGroupSet>,
82 :
83 : /// Count of HTTP requests to the safekeeper that resulted in an error,
84 : /// broken down by the safekeeper node id, request name and method
85 : pub(crate) storage_controller_safekeeper_request_error:
86 : measured::CounterVec<PageserverRequestLabelGroupSet>,
87 :
88 : /// Latency of HTTP requests to the pageserver, broken down by pageserver
89 : /// node id, request name and method. This include both successful and unsuccessful
90 : /// requests.
91 : #[metric(metadata = histogram::Thresholds::exponential_buckets(0.1, 2.0))]
92 : pub(crate) storage_controller_pageserver_request_latency:
93 : measured::HistogramVec<PageserverRequestLabelGroupSet, 5>,
94 :
95 : /// Latency of HTTP requests to the safekeeper, broken down by safekeeper
96 : /// node id, request name and method. This include both successful and unsuccessful
97 : /// requests.
98 : #[metric(metadata = histogram::Thresholds::exponential_buckets(0.1, 2.0))]
99 : pub(crate) storage_controller_safekeeper_request_latency:
100 : measured::HistogramVec<PageserverRequestLabelGroupSet, 5>,
101 :
102 : /// Count of pass-through HTTP requests to the pageserver that resulted in an error,
103 : /// broken down by the pageserver node id, request name and method
104 : pub(crate) storage_controller_passthrough_request_error:
105 : measured::CounterVec<PageserverRequestLabelGroupSet>,
106 :
107 : /// Latency of pass-through HTTP requests to the pageserver, broken down by pageserver
108 : /// node id, request name and method. This include both successful and unsuccessful
109 : /// requests.
110 : #[metric(metadata = histogram::Thresholds::exponential_buckets(0.1, 2.0))]
111 : pub(crate) storage_controller_passthrough_request_latency:
112 : measured::HistogramVec<PageserverRequestLabelGroupSet, 5>,
113 :
114 : /// Count of errors in database queries, broken down by error type and operation.
115 : pub(crate) storage_controller_database_query_error:
116 : measured::CounterVec<DatabaseQueryErrorLabelGroupSet>,
117 :
118 : /// Latency of database queries, broken down by operation.
119 : #[metric(metadata = histogram::Thresholds::exponential_buckets(0.1, 2.0))]
120 : pub(crate) storage_controller_database_query_latency:
121 : measured::HistogramVec<DatabaseQueryLatencyLabelGroupSet, 5>,
122 :
123 : pub(crate) storage_controller_leadership_status: measured::GaugeVec<LeadershipStatusGroupSet>,
124 :
125 : /// HTTP request status counters for handled requests
126 : pub(crate) storage_controller_reconcile_long_running:
127 : measured::CounterVec<ReconcileLongRunningLabelGroupSet>,
128 : }
129 :
130 : impl StorageControllerMetrics {
131 0 : pub(crate) fn encode(&self, neon_metrics: &NeonMetrics) -> Bytes {
132 0 : let mut encoder = self.encoder.lock().unwrap();
133 0 : neon_metrics
134 0 : .collect_group_into(&mut *encoder)
135 0 : .unwrap_or_else(|infallible| match infallible {});
136 0 : self.metrics_group
137 0 : .collect_group_into(&mut *encoder)
138 0 : .unwrap_or_else(|infallible| match infallible {});
139 0 : encoder.finish()
140 0 : }
141 : }
142 :
143 : impl Default for StorageControllerMetrics {
144 15 : fn default() -> Self {
145 15 : let mut metrics_group = StorageControllerMetricGroup::new();
146 15 : metrics_group
147 15 : .storage_controller_reconcile_complete
148 15 : .init_all_dense();
149 15 :
150 15 : Self {
151 15 : metrics_group,
152 15 : encoder: Mutex::new(measured::text::BufferedTextEncoder::new()),
153 15 : }
154 15 : }
155 : }
156 :
157 90 : #[derive(measured::LabelGroup, Clone)]
158 : #[label(set = NodeLabelGroupSet)]
159 : pub(crate) struct NodeLabelGroup<'a> {
160 : #[label(dynamic_with = lasso::ThreadedRodeo, default)]
161 : pub(crate) az: &'a str,
162 : #[label(dynamic_with = lasso::ThreadedRodeo, default)]
163 : pub(crate) node_id: &'a str,
164 : }
165 :
166 45 : #[derive(measured::LabelGroup)]
167 : #[label(set = ReconcileCompleteLabelGroupSet)]
168 : pub(crate) struct ReconcileCompleteLabelGroup {
169 : pub(crate) status: ReconcileOutcome,
170 : }
171 :
172 30 : #[derive(measured::LabelGroup)]
173 : #[label(set = HttpRequestStatusLabelGroupSet)]
174 : pub(crate) struct HttpRequestStatusLabelGroup<'a> {
175 : #[label(dynamic_with = lasso::ThreadedRodeo, default)]
176 : pub(crate) path: &'a str,
177 : pub(crate) method: Method,
178 : pub(crate) status: StatusCode,
179 : }
180 :
181 30 : #[derive(measured::LabelGroup)]
182 : #[label(set = HttpRequestLatencyLabelGroupSet)]
183 : pub(crate) struct HttpRequestLatencyLabelGroup<'a> {
184 : #[label(dynamic_with = lasso::ThreadedRodeo, default)]
185 : pub(crate) path: &'a str,
186 : pub(crate) method: Method,
187 : }
188 :
189 180 : #[derive(measured::LabelGroup, Clone)]
190 : #[label(set = PageserverRequestLabelGroupSet)]
191 : pub(crate) struct PageserverRequestLabelGroup<'a> {
192 : #[label(dynamic_with = lasso::ThreadedRodeo, default)]
193 : pub(crate) pageserver_id: &'a str,
194 : #[label(dynamic_with = lasso::ThreadedRodeo, default)]
195 : pub(crate) path: &'a str,
196 : pub(crate) method: Method,
197 : }
198 :
199 60 : #[derive(measured::LabelGroup)]
200 : #[label(set = DatabaseQueryErrorLabelGroupSet)]
201 : pub(crate) struct DatabaseQueryErrorLabelGroup {
202 : pub(crate) error_type: DatabaseErrorLabel,
203 : pub(crate) operation: DatabaseOperation,
204 : }
205 :
206 45 : #[derive(measured::LabelGroup)]
207 : #[label(set = DatabaseQueryLatencyLabelGroupSet)]
208 : pub(crate) struct DatabaseQueryLatencyLabelGroup {
209 : pub(crate) operation: DatabaseOperation,
210 : }
211 :
212 45 : #[derive(measured::LabelGroup)]
213 : #[label(set = LeadershipStatusGroupSet)]
214 : pub(crate) struct LeadershipStatusGroup {
215 : pub(crate) status: LeadershipStatus,
216 : }
217 :
218 30 : #[derive(measured::LabelGroup, Clone)]
219 : #[label(set = ReconcileLongRunningLabelGroupSet)]
220 : pub(crate) struct ReconcileLongRunningLabelGroup<'a> {
221 : #[label(dynamic_with = lasso::ThreadedRodeo, default)]
222 : pub(crate) tenant_id: &'a str,
223 : #[label(dynamic_with = lasso::ThreadedRodeo, default)]
224 : pub(crate) shard_number: &'a str,
225 : #[label(dynamic_with = lasso::ThreadedRodeo, default)]
226 : pub(crate) sequence: &'a str,
227 : }
228 :
229 : #[derive(FixedCardinalityLabel, Clone, Copy)]
230 : pub(crate) enum ReconcileOutcome {
231 : #[label(rename = "ok")]
232 : Success,
233 : Error,
234 : Cancel,
235 : }
236 :
237 : #[derive(FixedCardinalityLabel, Copy, Clone)]
238 : pub(crate) enum Method {
239 : Get,
240 : Put,
241 : Post,
242 : Delete,
243 : Other,
244 : }
245 :
246 : impl From<hyper::Method> for Method {
247 0 : fn from(value: hyper::Method) -> Self {
248 0 : if value == hyper::Method::GET {
249 0 : Method::Get
250 0 : } else if value == hyper::Method::PUT {
251 0 : Method::Put
252 0 : } else if value == hyper::Method::POST {
253 0 : Method::Post
254 0 : } else if value == hyper::Method::DELETE {
255 0 : Method::Delete
256 : } else {
257 0 : Method::Other
258 : }
259 0 : }
260 : }
261 :
262 : #[derive(Clone, Copy)]
263 : pub(crate) struct StatusCode(pub(crate) hyper::http::StatusCode);
264 :
265 : impl LabelValue for StatusCode {
266 0 : fn visit<V: measured::label::LabelVisitor>(&self, v: V) -> V::Output {
267 0 : v.write_int(self.0.as_u16() as i64)
268 0 : }
269 : }
270 :
271 : impl FixedCardinalityLabel for StatusCode {
272 0 : fn cardinality() -> usize {
273 0 : (100..1000).len()
274 0 : }
275 :
276 0 : fn encode(&self) -> usize {
277 0 : self.0.as_u16() as usize
278 0 : }
279 :
280 0 : fn decode(value: usize) -> Self {
281 0 : Self(hyper::http::StatusCode::from_u16(u16::try_from(value).unwrap()).unwrap())
282 0 : }
283 : }
284 :
285 : #[derive(FixedCardinalityLabel, Clone, Copy)]
286 : pub(crate) enum DatabaseErrorLabel {
287 : Query,
288 : Connection,
289 : ConnectionPool,
290 : Logical,
291 : Migration,
292 : }
293 :
294 : impl DatabaseError {
295 0 : pub(crate) fn error_label(&self) -> DatabaseErrorLabel {
296 0 : match self {
297 0 : Self::Query(_) => DatabaseErrorLabel::Query,
298 0 : Self::Connection(_) => DatabaseErrorLabel::Connection,
299 0 : Self::ConnectionPool(_) => DatabaseErrorLabel::ConnectionPool,
300 0 : Self::Logical(_) => DatabaseErrorLabel::Logical,
301 0 : Self::Migration(_) => DatabaseErrorLabel::Migration,
302 : }
303 0 : }
304 : }
305 :
306 : /// Update the leadership status metric gauges to reflect the requested status
307 0 : pub(crate) fn update_leadership_status(status: LeadershipStatus) {
308 0 : let status_metric = &METRICS_REGISTRY
309 0 : .metrics_group
310 0 : .storage_controller_leadership_status;
311 :
312 0 : for s in LeadershipStatus::iter() {
313 0 : if s == status {
314 0 : status_metric.set(LeadershipStatusGroup { status: s }, 1);
315 0 : } else {
316 0 : status_metric.set(LeadershipStatusGroup { status: s }, 0);
317 0 : }
318 : }
319 0 : }
|
