TLA Line data Source code
1 : use metrics::{
2 : register_histogram_vec, register_int_counter, register_int_counter_vec, Histogram, IntCounter,
3 : };
4 : use once_cell::sync::Lazy;
5 :
6 : pub(super) static BUCKET_METRICS: Lazy<BucketMetrics> = Lazy::new(Default::default);
7 :
8 UBC 0 : #[derive(Clone, Copy, Debug)]
9 : pub(crate) enum RequestKind {
10 : Get = 0,
11 : Put = 1,
12 : Delete = 2,
13 : List = 3,
14 : Copy = 4,
15 : }
16 :
17 : use RequestKind::*;
18 :
19 : impl RequestKind {
20 CBC 5400 : const fn as_str(&self) -> &'static str {
21 5400 : match self {
22 1080 : Get => "get_object",
23 1080 : Put => "put_object",
24 1080 : Delete => "delete_object",
25 1080 : List => "list_objects",
26 1080 : Copy => "copy_object",
27 : }
28 5400 : }
29 62387 : const fn as_index(&self) -> usize {
30 62387 : *self as usize
31 62387 : }
32 : }
33 :
34 : pub(super) struct RequestTyped<C>([C; 5]);
35 :
36 : impl<C> RequestTyped<C> {
37 56987 : pub(super) fn get(&self, kind: RequestKind) -> &C {
38 56987 : &self.0[kind.as_index()]
39 56987 : }
40 :
41 1080 : fn build_with(mut f: impl FnMut(RequestKind) -> C) -> Self {
42 1080 : use RequestKind::*;
43 1080 : let mut it = [Get, Put, Delete, List, Copy].into_iter();
44 5400 : let arr = std::array::from_fn::<C, 5, _>(|index| {
45 5400 : let next = it.next().unwrap();
46 5400 : assert_eq!(index, next.as_index());
47 5400 : f(next)
48 5400 : });
49 :
50 1080 : if let Some(next) = it.next() {
51 UBC 0 : panic!("unexpected {next:?}");
52 CBC 1080 : }
53 1080 :
54 1080 : RequestTyped(arr)
55 1080 : }
56 : }
57 :
58 : impl RequestTyped<Histogram> {
59 28494 : pub(super) fn observe_elapsed(&self, kind: RequestKind, started_at: std::time::Instant) {
60 28494 : self.get(kind).observe(started_at.elapsed().as_secs_f64())
61 28494 : }
62 : }
63 :
64 : pub(super) struct PassFailCancelledRequestTyped<C> {
65 : success: RequestTyped<C>,
66 : fail: RequestTyped<C>,
67 : cancelled: RequestTyped<C>,
68 : }
69 :
70 UBC 0 : #[derive(Debug, Clone, Copy)]
71 : pub(super) enum AttemptOutcome {
72 : Ok,
73 : Err,
74 : Cancelled,
75 : }
76 :
77 : impl<T, E> From<&Result<T, E>> for AttemptOutcome {
78 CBC 17943 : fn from(value: &Result<T, E>) -> Self {
79 17943 : match value {
80 17943 : Ok(_) => AttemptOutcome::Ok,
81 LBC (1) : Err(_) => AttemptOutcome::Err,
82 : }
83 CBC 17943 : }
84 : }
85 :
86 : impl AttemptOutcome {
87 3240 : pub(super) fn as_str(&self) -> &'static str {
88 3240 : match self {
89 1080 : AttemptOutcome::Ok => "ok",
90 1080 : AttemptOutcome::Err => "err",
91 1080 : AttemptOutcome::Cancelled => "cancelled",
92 : }
93 3240 : }
94 : }
95 :
96 : impl<C> PassFailCancelledRequestTyped<C> {
97 28493 : pub(super) fn get(&self, kind: RequestKind, outcome: AttemptOutcome) -> &C {
98 28493 : let target = match outcome {
99 28452 : AttemptOutcome::Ok => &self.success,
100 LBC (1) : AttemptOutcome::Err => &self.fail,
101 CBC 41 : AttemptOutcome::Cancelled => &self.cancelled,
102 : };
103 28493 : target.get(kind)
104 28493 : }
105 :
106 216 : fn build_with(mut f: impl FnMut(RequestKind, AttemptOutcome) -> C) -> Self {
107 1080 : let success = RequestTyped::build_with(|kind| f(kind, AttemptOutcome::Ok));
108 1080 : let fail = RequestTyped::build_with(|kind| f(kind, AttemptOutcome::Err));
109 1080 : let cancelled = RequestTyped::build_with(|kind| f(kind, AttemptOutcome::Cancelled));
110 216 :
111 216 : PassFailCancelledRequestTyped {
112 216 : success,
113 216 : fail,
114 216 : cancelled,
115 216 : }
116 216 : }
117 : }
118 :
119 : impl PassFailCancelledRequestTyped<Histogram> {
120 28401 : pub(super) fn observe_elapsed(
121 28401 : &self,
122 28401 : kind: RequestKind,
123 28401 : outcome: impl Into<AttemptOutcome>,
124 28401 : started_at: std::time::Instant,
125 28401 : ) {
126 28401 : self.get(kind, outcome.into())
127 28401 : .observe(started_at.elapsed().as_secs_f64())
128 28401 : }
129 : }
130 :
131 : pub(super) struct BucketMetrics {
132 : /// Full request duration until successful completion, error or cancellation.
133 : pub(super) req_seconds: PassFailCancelledRequestTyped<Histogram>,
134 : /// Total amount of seconds waited on queue.
135 : pub(super) wait_seconds: RequestTyped<Histogram>,
136 :
137 : /// Track how many semaphore awaits were cancelled per request type.
138 : ///
139 : /// This is in case cancellations are happening more than expected.
140 : pub(super) cancelled_waits: RequestTyped<IntCounter>,
141 :
142 : /// Total amount of deleted objects in batches or single requests.
143 : pub(super) deleted_objects_total: IntCounter,
144 : }
145 :
146 : impl Default for BucketMetrics {
147 216 : fn default() -> Self {
148 216 : let buckets = [0.01, 0.10, 0.5, 1.0, 5.0, 10.0, 50.0, 100.0];
149 216 :
150 216 : let req_seconds = register_histogram_vec!(
151 216 : "remote_storage_s3_request_seconds",
152 216 : "Seconds to complete a request",
153 216 : &["request_type", "result"],
154 216 : buckets.to_vec(),
155 216 : )
156 216 : .unwrap();
157 3240 : let req_seconds = PassFailCancelledRequestTyped::build_with(|kind, outcome| {
158 3240 : req_seconds.with_label_values(&[kind.as_str(), outcome.as_str()])
159 3240 : });
160 216 :
161 216 : let wait_seconds = register_histogram_vec!(
162 216 : "remote_storage_s3_wait_seconds",
163 216 : "Seconds rate limited",
164 216 : &["request_type"],
165 216 : buckets.to_vec(),
166 216 : )
167 216 : .unwrap();
168 216 : let wait_seconds =
169 1080 : RequestTyped::build_with(|kind| wait_seconds.with_label_values(&[kind.as_str()]));
170 216 :
171 216 : let cancelled_waits = register_int_counter_vec!(
172 216 : "remote_storage_s3_cancelled_waits_total",
173 216 : "Times a semaphore wait has been cancelled per request type",
174 216 : &["request_type"],
175 216 : )
176 216 : .unwrap();
177 216 : let cancelled_waits =
178 1080 : RequestTyped::build_with(|kind| cancelled_waits.with_label_values(&[kind.as_str()]));
179 216 :
180 216 : let deleted_objects_total = register_int_counter!(
181 216 : "remote_storage_s3_deleted_objects_total",
182 216 : "Amount of deleted objects in total",
183 216 : )
184 216 : .unwrap();
185 216 :
186 216 : Self {
187 216 : req_seconds,
188 216 : wait_seconds,
189 216 : cancelled_waits,
190 216 : deleted_objects_total,
191 216 : }
192 216 : }
193 : }
|
