Line data Source code
1 : use std::sync::{Arc, OnceLock};
2 :
3 : use lasso::ThreadedRodeo;
4 : use measured::{
5 : label::{FixedCardinalitySet, LabelName, LabelSet, LabelValue, StaticLabelSet},
6 : metric::{histogram::Thresholds, name::MetricName},
7 : Counter, CounterVec, FixedCardinalityLabel, Gauge, GaugeVec, Histogram, HistogramVec,
8 : LabelGroup, MetricGroup,
9 : };
10 : use metrics::{CounterPairAssoc, CounterPairVec, HyperLogLog, HyperLogLogVec};
11 :
12 : use tokio::time::{self, Instant};
13 :
14 : use crate::console::messages::ColdStartInfo;
15 :
16 72 : #[derive(MetricGroup)]
17 : #[metric(new(thread_pool: Arc<ThreadPoolMetrics>))]
18 : pub struct Metrics {
19 : #[metric(namespace = "proxy")]
20 : #[metric(init = ProxyMetrics::new(thread_pool))]
21 : pub proxy: ProxyMetrics,
22 :
23 : #[metric(namespace = "wake_compute_lock")]
24 : pub wake_compute_lock: ApiLockMetrics,
25 : }
26 :
27 : static SELF: OnceLock<Metrics> = OnceLock::new();
28 : impl Metrics {
29 0 : pub fn install(thread_pool: Arc<ThreadPoolMetrics>) {
30 0 : SELF.set(Metrics::new(thread_pool))
31 0 : .ok()
32 0 : .expect("proxy metrics must not be installed more than once");
33 0 : }
34 :
35 178 : pub fn get() -> &'static Self {
36 178 : #[cfg(test)]
37 178 : return SELF.get_or_init(|| Metrics::new(Arc::new(ThreadPoolMetrics::new(0))));
38 178 :
39 178 : #[cfg(not(test))]
40 178 : SELF.get()
41 178 : .expect("proxy metrics must be installed by the main() function")
42 178 : }
43 : }
44 :
45 72 : #[derive(MetricGroup)]
46 : #[metric(new(thread_pool: Arc<ThreadPoolMetrics>))]
47 : pub struct ProxyMetrics {
48 : #[metric(flatten)]
49 : pub db_connections: CounterPairVec<NumDbConnectionsGauge>,
50 : #[metric(flatten)]
51 : pub client_connections: CounterPairVec<NumClientConnectionsGauge>,
52 : #[metric(flatten)]
53 : pub connection_requests: CounterPairVec<NumConnectionRequestsGauge>,
54 : #[metric(flatten)]
55 : pub http_endpoint_pools: HttpEndpointPools,
56 :
57 : /// Time it took for proxy to establish a connection to the compute endpoint.
58 : // largest bucket = 2^16 * 0.5ms = 32s
59 : #[metric(metadata = Thresholds::exponential_buckets(0.0005, 2.0))]
60 : pub compute_connection_latency_seconds: HistogramVec<ComputeConnectionLatencySet, 16>,
61 :
62 : /// Time it took for proxy to receive a response from control plane.
63 : #[metric(
64 : // largest bucket = 2^16 * 0.2ms = 13s
65 : metadata = Thresholds::exponential_buckets(0.0002, 2.0),
66 : )]
67 : pub console_request_latency: HistogramVec<ConsoleRequestSet, 16>,
68 :
69 : /// Time it takes to acquire a token to call console plane.
70 : // largest bucket = 3^16 * 0.05ms = 2.15s
71 : #[metric(metadata = Thresholds::exponential_buckets(0.00005, 3.0))]
72 : pub control_plane_token_acquire_seconds: Histogram<16>,
73 :
74 : /// Size of the HTTP request body lengths.
75 : // smallest bucket = 16 bytes
76 : // largest bucket = 4^12 * 16 bytes = 256MB
77 : #[metric(metadata = Thresholds::exponential_buckets(16.0, 4.0))]
78 : pub http_conn_content_length_bytes: HistogramVec<StaticLabelSet<HttpDirection>, 12>,
79 :
80 : /// Time it takes to reclaim unused connection pools.
81 : #[metric(metadata = Thresholds::exponential_buckets(1e-6, 2.0))]
82 : pub http_pool_reclaimation_lag_seconds: Histogram<16>,
83 :
84 : /// Number of opened connections to a database.
85 : pub http_pool_opened_connections: Gauge,
86 :
87 : /// Number of cache hits/misses for allowed ips.
88 : pub allowed_ips_cache_misses: CounterVec<StaticLabelSet<CacheOutcome>>,
89 :
90 : /// Number of allowed ips
91 : #[metric(metadata = Thresholds::with_buckets([0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 10.0, 20.0, 50.0, 100.0]))]
92 : pub allowed_ips_number: Histogram<10>,
93 :
94 : /// Number of connections (per sni).
95 : pub accepted_connections_by_sni: CounterVec<StaticLabelSet<SniKind>>,
96 :
97 : /// Number of connection failures (per kind).
98 : pub connection_failures_total: CounterVec<StaticLabelSet<ConnectionFailureKind>>,
99 :
100 : /// Number of wake-up failures (per kind).
101 : pub connection_failures_breakdown: CounterVec<ConnectionFailuresBreakdownSet>,
102 :
103 : /// Number of bytes sent/received between all clients and backends.
104 : pub io_bytes: CounterVec<StaticLabelSet<Direction>>,
105 :
106 : /// Number of errors by a given classification.
107 : pub errors_total: CounterVec<StaticLabelSet<crate::error::ErrorKind>>,
108 :
109 : /// Number of cancellation requests (per found/not_found).
110 : pub cancellation_requests_total: CounterVec<CancellationRequestSet>,
111 :
112 : /// Number of errors by a given classification
113 : pub redis_errors_total: CounterVec<RedisErrorsSet>,
114 :
115 : /// Number of TLS handshake failures
116 : pub tls_handshake_failures: Counter,
117 :
118 : /// Number of connection requests affected by authentication rate limits
119 : pub requests_auth_rate_limits_total: Counter,
120 :
121 : /// HLL approximate cardinality of endpoints that are connecting
122 : pub connecting_endpoints: HyperLogLogVec<StaticLabelSet<Protocol>, 32>,
123 :
124 : /// Number of endpoints affected by errors of a given classification
125 : pub endpoints_affected_by_errors: HyperLogLogVec<StaticLabelSet<crate::error::ErrorKind>, 32>,
126 :
127 : /// Number of endpoints affected by authentication rate limits
128 : pub endpoints_auth_rate_limits: HyperLogLog<32>,
129 :
130 : /// Number of invalid endpoints (per protocol, per rejected).
131 : pub invalid_endpoints_total: CounterVec<InvalidEndpointsSet>,
132 :
133 : /// Number of retries (per outcome, per retry_type).
134 : #[metric(metadata = Thresholds::with_buckets([0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]))]
135 : pub retries_metric: HistogramVec<RetriesMetricSet, 9>,
136 :
137 : /// Number of events consumed from redis (per event type).
138 : pub redis_events_count: CounterVec<StaticLabelSet<RedisEventsCount>>,
139 :
140 : #[metric(namespace = "connect_compute_lock")]
141 : pub connect_compute_lock: ApiLockMetrics,
142 :
143 : #[metric(namespace = "scram_pool")]
144 : #[metric(init = thread_pool)]
145 : pub scram_pool: Arc<ThreadPoolMetrics>,
146 : }
147 :
148 144 : #[derive(MetricGroup)]
149 : #[metric(new())]
150 : pub struct ApiLockMetrics {
151 : /// Number of semaphores registered in this api lock
152 : pub semaphores_registered: Counter,
153 : /// Number of semaphores unregistered in this api lock
154 : pub semaphores_unregistered: Counter,
155 : /// Time it takes to reclaim unused semaphores in the api lock
156 : #[metric(metadata = Thresholds::exponential_buckets(1e-6, 2.0))]
157 : pub reclamation_lag_seconds: Histogram<16>,
158 : /// Time it takes to acquire a semaphore lock
159 : #[metric(metadata = Thresholds::exponential_buckets(1e-4, 2.0))]
160 : pub semaphore_acquire_seconds: Histogram<16>,
161 : }
162 :
163 : impl Default for ApiLockMetrics {
164 144 : fn default() -> Self {
165 144 : Self::new()
166 144 : }
167 : }
168 :
169 0 : #[derive(FixedCardinalityLabel, Copy, Clone)]
170 : #[label(singleton = "direction")]
171 : pub enum HttpDirection {
172 : Request,
173 : Response,
174 : }
175 :
176 0 : #[derive(FixedCardinalityLabel, Copy, Clone)]
177 : #[label(singleton = "direction")]
178 : pub enum Direction {
179 : Tx,
180 : Rx,
181 : }
182 :
183 0 : #[derive(FixedCardinalityLabel, Clone, Copy, Debug)]
184 : #[label(singleton = "protocol")]
185 : pub enum Protocol {
186 : Http,
187 : Ws,
188 : Tcp,
189 : SniRouter,
190 : }
191 :
192 : impl Protocol {
193 0 : pub fn as_str(&self) -> &'static str {
194 0 : match self {
195 0 : Protocol::Http => "http",
196 0 : Protocol::Ws => "ws",
197 0 : Protocol::Tcp => "tcp",
198 0 : Protocol::SniRouter => "sni_router",
199 : }
200 0 : }
201 : }
202 :
203 : impl std::fmt::Display for Protocol {
204 0 : fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
205 0 : f.write_str(self.as_str())
206 0 : }
207 : }
208 :
209 : #[derive(FixedCardinalityLabel, Copy, Clone)]
210 : pub enum Bool {
211 : True,
212 : False,
213 : }
214 :
215 0 : #[derive(FixedCardinalityLabel, Copy, Clone)]
216 : #[label(singleton = "outcome")]
217 : pub enum Outcome {
218 : Success,
219 : Failed,
220 : }
221 :
222 0 : #[derive(FixedCardinalityLabel, Copy, Clone)]
223 : #[label(singleton = "outcome")]
224 : pub enum CacheOutcome {
225 : Hit,
226 : Miss,
227 : }
228 :
229 144 : #[derive(LabelGroup)]
230 : #[label(set = ConsoleRequestSet)]
231 : pub struct ConsoleRequest<'a> {
232 : #[label(dynamic_with = ThreadedRodeo, default)]
233 : pub request: &'a str,
234 : }
235 :
236 : #[derive(MetricGroup, Default)]
237 : pub struct HttpEndpointPools {
238 : /// Number of endpoints we have registered pools for
239 : pub http_pool_endpoints_registered_total: Counter,
240 : /// Number of endpoints we have unregistered pools for
241 : pub http_pool_endpoints_unregistered_total: Counter,
242 : }
243 :
244 : pub struct HttpEndpointPoolsGuard<'a> {
245 : dec: &'a Counter,
246 : }
247 :
248 : impl Drop for HttpEndpointPoolsGuard<'_> {
249 4 : fn drop(&mut self) {
250 4 : self.dec.inc();
251 4 : }
252 : }
253 :
254 : impl HttpEndpointPools {
255 4 : pub fn guard(&self) -> HttpEndpointPoolsGuard {
256 4 : self.http_pool_endpoints_registered_total.inc();
257 4 : HttpEndpointPoolsGuard {
258 4 : dec: &self.http_pool_endpoints_unregistered_total,
259 4 : }
260 4 : }
261 : }
262 : pub struct NumDbConnectionsGauge;
263 : impl CounterPairAssoc for NumDbConnectionsGauge {
264 : const INC_NAME: &'static MetricName = MetricName::from_str("opened_db_connections_total");
265 : const DEC_NAME: &'static MetricName = MetricName::from_str("closed_db_connections_total");
266 : const INC_HELP: &'static str = "Number of opened connections to a database.";
267 : const DEC_HELP: &'static str = "Number of closed connections to a database.";
268 : type LabelGroupSet = StaticLabelSet<Protocol>;
269 : }
270 : pub type NumDbConnectionsGuard<'a> = metrics::MeasuredCounterPairGuard<'a, NumDbConnectionsGauge>;
271 :
272 : pub struct NumClientConnectionsGauge;
273 : impl CounterPairAssoc for NumClientConnectionsGauge {
274 : const INC_NAME: &'static MetricName = MetricName::from_str("opened_client_connections_total");
275 : const DEC_NAME: &'static MetricName = MetricName::from_str("closed_client_connections_total");
276 : const INC_HELP: &'static str = "Number of opened connections from a client.";
277 : const DEC_HELP: &'static str = "Number of closed connections from a client.";
278 : type LabelGroupSet = StaticLabelSet<Protocol>;
279 : }
280 : pub type NumClientConnectionsGuard<'a> =
281 : metrics::MeasuredCounterPairGuard<'a, NumClientConnectionsGauge>;
282 :
283 : pub struct NumConnectionRequestsGauge;
284 : impl CounterPairAssoc for NumConnectionRequestsGauge {
285 : const INC_NAME: &'static MetricName = MetricName::from_str("accepted_connections_total");
286 : const DEC_NAME: &'static MetricName = MetricName::from_str("closed_connections_total");
287 : const INC_HELP: &'static str = "Number of client connections accepted.";
288 : const DEC_HELP: &'static str = "Number of client connections closed.";
289 : type LabelGroupSet = StaticLabelSet<Protocol>;
290 : }
291 : pub type NumConnectionRequestsGuard<'a> =
292 : metrics::MeasuredCounterPairGuard<'a, NumConnectionRequestsGauge>;
293 :
294 432 : #[derive(LabelGroup)]
295 : #[label(set = ComputeConnectionLatencySet)]
296 : pub struct ComputeConnectionLatencyGroup {
297 : protocol: Protocol,
298 : cold_start_info: ColdStartInfo,
299 : outcome: ConnectOutcome,
300 : excluded: LatencyExclusions,
301 : }
302 :
303 : #[derive(FixedCardinalityLabel, Copy, Clone)]
304 : pub enum LatencyExclusions {
305 : Client,
306 : ClientAndCplane,
307 : ClientCplaneCompute,
308 : ClientCplaneComputeRetry,
309 : }
310 :
311 0 : #[derive(FixedCardinalityLabel, Copy, Clone)]
312 : #[label(singleton = "kind")]
313 : pub enum SniKind {
314 : Sni,
315 : NoSni,
316 : PasswordHack,
317 : }
318 :
319 0 : #[derive(FixedCardinalityLabel, Copy, Clone)]
320 : #[label(singleton = "kind")]
321 : pub enum ConnectionFailureKind {
322 : ComputeCached,
323 : ComputeUncached,
324 : }
325 :
326 0 : #[derive(FixedCardinalityLabel, Copy, Clone)]
327 : #[label(singleton = "kind")]
328 : pub enum WakeupFailureKind {
329 : BadComputeAddress,
330 : ApiTransportError,
331 : QuotaExceeded,
332 : ApiConsoleLocked,
333 : ApiConsoleBadRequest,
334 : ApiConsoleOtherServerError,
335 : ApiConsoleOtherError,
336 : TimeoutError,
337 : }
338 :
339 288 : #[derive(LabelGroup)]
340 : #[label(set = ConnectionFailuresBreakdownSet)]
341 : pub struct ConnectionFailuresBreakdownGroup {
342 : pub kind: WakeupFailureKind,
343 : pub retry: Bool,
344 : }
345 :
346 144 : #[derive(LabelGroup, Copy, Clone)]
347 : #[label(set = RedisErrorsSet)]
348 : pub struct RedisErrors<'a> {
349 : #[label(dynamic_with = ThreadedRodeo, default)]
350 : pub channel: &'a str,
351 : }
352 :
353 : #[derive(FixedCardinalityLabel, Copy, Clone)]
354 : pub enum CancellationSource {
355 : FromClient,
356 : FromRedis,
357 : Local,
358 : }
359 :
360 : #[derive(FixedCardinalityLabel, Copy, Clone)]
361 : pub enum CancellationOutcome {
362 : NotFound,
363 : Found,
364 : }
365 :
366 288 : #[derive(LabelGroup)]
367 : #[label(set = CancellationRequestSet)]
368 : pub struct CancellationRequest {
369 : pub source: CancellationSource,
370 : pub kind: CancellationOutcome,
371 : }
372 :
373 : pub enum Waiting {
374 : Cplane,
375 : Client,
376 : Compute,
377 : RetryTimeout,
378 : }
379 :
380 : #[derive(Default)]
381 : struct Accumulated {
382 : cplane: time::Duration,
383 : client: time::Duration,
384 : compute: time::Duration,
385 : retry: time::Duration,
386 : }
387 :
388 : pub struct LatencyTimer {
389 : // time since the stopwatch was started
390 : start: time::Instant,
391 : // time since the stopwatch was stopped
392 : stop: Option<time::Instant>,
393 : // accumulated time on the stopwatch
394 : accumulated: Accumulated,
395 : // label data
396 : protocol: Protocol,
397 : cold_start_info: ColdStartInfo,
398 : outcome: ConnectOutcome,
399 : }
400 :
401 : pub struct LatencyTimerPause<'a> {
402 : timer: &'a mut LatencyTimer,
403 : start: time::Instant,
404 : waiting_for: Waiting,
405 : }
406 :
407 : impl LatencyTimer {
408 70 : pub fn new(protocol: Protocol) -> Self {
409 70 : Self {
410 70 : start: time::Instant::now(),
411 70 : stop: None,
412 70 : accumulated: Accumulated::default(),
413 70 : protocol,
414 70 : cold_start_info: ColdStartInfo::Unknown,
415 70 : // assume failed unless otherwise specified
416 70 : outcome: ConnectOutcome::Failed,
417 70 : }
418 70 : }
419 :
420 42 : pub fn pause(&mut self, waiting_for: Waiting) -> LatencyTimerPause<'_> {
421 42 : LatencyTimerPause {
422 42 : timer: self,
423 42 : start: Instant::now(),
424 42 : waiting_for,
425 42 : }
426 42 : }
427 :
428 0 : pub fn cold_start_info(&mut self, cold_start_info: ColdStartInfo) {
429 0 : self.cold_start_info = cold_start_info;
430 0 : }
431 :
432 8 : pub fn success(&mut self) {
433 8 : // stop the stopwatch and record the time that we have accumulated
434 8 : self.stop = Some(time::Instant::now());
435 8 :
436 8 : // success
437 8 : self.outcome = ConnectOutcome::Success;
438 8 : }
439 : }
440 :
441 : impl Drop for LatencyTimerPause<'_> {
442 42 : fn drop(&mut self) {
443 42 : let dur = self.start.elapsed();
444 42 : match self.waiting_for {
445 0 : Waiting::Cplane => self.timer.accumulated.cplane += dur,
446 30 : Waiting::Client => self.timer.accumulated.client += dur,
447 0 : Waiting::Compute => self.timer.accumulated.compute += dur,
448 12 : Waiting::RetryTimeout => self.timer.accumulated.retry += dur,
449 : }
450 42 : }
451 : }
452 :
453 : #[derive(FixedCardinalityLabel, Clone, Copy, Debug)]
454 : pub enum ConnectOutcome {
455 : Success,
456 : Failed,
457 : }
458 :
459 : impl Drop for LatencyTimer {
460 70 : fn drop(&mut self) {
461 70 : let duration = self
462 70 : .stop
463 70 : .unwrap_or_else(time::Instant::now)
464 70 : .duration_since(self.start);
465 70 :
466 70 : let metric = &Metrics::get().proxy.compute_connection_latency_seconds;
467 70 :
468 70 : // Excluding client communication from the accumulated time.
469 70 : metric.observe(
470 70 : ComputeConnectionLatencyGroup {
471 70 : protocol: self.protocol,
472 70 : cold_start_info: self.cold_start_info,
473 70 : outcome: self.outcome,
474 70 : excluded: LatencyExclusions::Client,
475 70 : },
476 70 : duration
477 70 : .saturating_sub(self.accumulated.client)
478 70 : .as_secs_f64(),
479 70 : );
480 70 :
481 70 : // Exclude client and cplane communication from the accumulated time.
482 70 : let accumulated_total = self.accumulated.client + self.accumulated.cplane;
483 70 : metric.observe(
484 70 : ComputeConnectionLatencyGroup {
485 70 : protocol: self.protocol,
486 70 : cold_start_info: self.cold_start_info,
487 70 : outcome: self.outcome,
488 70 : excluded: LatencyExclusions::ClientAndCplane,
489 70 : },
490 70 : duration.saturating_sub(accumulated_total).as_secs_f64(),
491 70 : );
492 70 :
493 70 : // Exclude client cplane, compue communication from the accumulated time.
494 70 : let accumulated_total =
495 70 : self.accumulated.client + self.accumulated.cplane + self.accumulated.compute;
496 70 : metric.observe(
497 70 : ComputeConnectionLatencyGroup {
498 70 : protocol: self.protocol,
499 70 : cold_start_info: self.cold_start_info,
500 70 : outcome: self.outcome,
501 70 : excluded: LatencyExclusions::ClientCplaneCompute,
502 70 : },
503 70 : duration.saturating_sub(accumulated_total).as_secs_f64(),
504 70 : );
505 70 :
506 70 : // Exclude client cplane, compue, retry communication from the accumulated time.
507 70 : let accumulated_total = self.accumulated.client
508 70 : + self.accumulated.cplane
509 70 : + self.accumulated.compute
510 70 : + self.accumulated.retry;
511 70 : metric.observe(
512 70 : ComputeConnectionLatencyGroup {
513 70 : protocol: self.protocol,
514 70 : cold_start_info: self.cold_start_info,
515 70 : outcome: self.outcome,
516 70 : excluded: LatencyExclusions::ClientCplaneComputeRetry,
517 70 : },
518 70 : duration.saturating_sub(accumulated_total).as_secs_f64(),
519 70 : );
520 70 : }
521 : }
522 :
523 : impl From<bool> for Bool {
524 6 : fn from(value: bool) -> Self {
525 6 : if value {
526 4 : Bool::True
527 : } else {
528 2 : Bool::False
529 : }
530 6 : }
531 : }
532 :
533 360 : #[derive(LabelGroup)]
534 : #[label(set = InvalidEndpointsSet)]
535 : pub struct InvalidEndpointsGroup {
536 : pub protocol: Protocol,
537 : pub rejected: Bool,
538 : pub outcome: ConnectOutcome,
539 : }
540 :
541 288 : #[derive(LabelGroup)]
542 : #[label(set = RetriesMetricSet)]
543 : pub struct RetriesMetricGroup {
544 : pub outcome: ConnectOutcome,
545 : pub retry_type: RetryType,
546 : }
547 :
548 : #[derive(FixedCardinalityLabel, Clone, Copy, Debug)]
549 : pub enum RetryType {
550 : WakeCompute,
551 : ConnectToCompute,
552 : }
553 :
554 0 : #[derive(FixedCardinalityLabel, Clone, Copy, Debug)]
555 : #[label(singleton = "event")]
556 : pub enum RedisEventsCount {
557 : EndpointCreated,
558 : BranchCreated,
559 : ProjectCreated,
560 : CancelSession,
561 : PasswordUpdate,
562 : AllowedIpsUpdate,
563 : }
564 :
565 : pub struct ThreadPoolWorkers(usize);
566 : pub struct ThreadPoolWorkerId(pub usize);
567 :
568 : impl LabelValue for ThreadPoolWorkerId {
569 0 : fn visit<V: measured::label::LabelVisitor>(&self, v: V) -> V::Output {
570 0 : v.write_int(self.0 as i64)
571 0 : }
572 : }
573 :
574 : impl LabelGroup for ThreadPoolWorkerId {
575 0 : fn visit_values(&self, v: &mut impl measured::label::LabelGroupVisitor) {
576 0 : v.write_value(LabelName::from_str("worker"), self);
577 0 : }
578 : }
579 :
580 : impl LabelSet for ThreadPoolWorkers {
581 : type Value<'a> = ThreadPoolWorkerId;
582 :
583 252 : fn dynamic_cardinality(&self) -> Option<usize> {
584 252 : Some(self.0)
585 252 : }
586 :
587 42 : fn encode(&self, value: Self::Value<'_>) -> Option<usize> {
588 42 : (value.0 < self.0).then_some(value.0)
589 42 : }
590 :
591 0 : fn decode(&self, value: usize) -> Self::Value<'_> {
592 0 : ThreadPoolWorkerId(value)
593 0 : }
594 : }
595 :
596 : impl FixedCardinalitySet for ThreadPoolWorkers {
597 0 : fn cardinality(&self) -> usize {
598 0 : self.0
599 0 : }
600 : }
601 :
602 84 : #[derive(MetricGroup)]
603 : #[metric(new(workers: usize))]
604 : pub struct ThreadPoolMetrics {
605 : pub injector_queue_depth: Gauge,
606 : #[metric(init = GaugeVec::with_label_set(ThreadPoolWorkers(workers)))]
607 : pub worker_queue_depth: GaugeVec<ThreadPoolWorkers>,
608 : #[metric(init = CounterVec::with_label_set(ThreadPoolWorkers(workers)))]
609 : pub worker_task_turns_total: CounterVec<ThreadPoolWorkers>,
610 : #[metric(init = CounterVec::with_label_set(ThreadPoolWorkers(workers)))]
611 : pub worker_task_skips_total: CounterVec<ThreadPoolWorkers>,
612 : }
|
