TLA Line data Source code
1 : use metrics::{
2 : register_histogram_vec, register_int_counter, register_int_counter_vec, Histogram, IntCounter,
3 : };
4 : use once_cell::sync::Lazy;
5 :
6 : pub(super) static BUCKET_METRICS: Lazy<BucketMetrics> = Lazy::new(Default::default);
7 :
8 UBC 0 : #[derive(Clone, Copy, Debug)]
9 : pub(crate) enum RequestKind {
10 : Get = 0,
11 : Put = 1,
12 : Delete = 2,
13 : List = 3,
14 : }
15 :
16 : use RequestKind::*;
17 :
18 : impl RequestKind {
19 CBC 5000 : const fn as_str(&self) -> &'static str {
20 5000 : match self {
21 1250 : Get => "get_object",
22 1250 : Put => "put_object",
23 1250 : Delete => "delete_object",
24 1250 : List => "list_objects",
25 : }
26 5000 : }
27 41209 : const fn as_index(&self) -> usize {
28 41209 : *self as usize
29 41209 : }
30 : }
31 :
32 : pub(super) struct RequestTyped<C>([C; 4]);
33 :
34 : impl<C> RequestTyped<C> {
35 36209 : pub(super) fn get(&self, kind: RequestKind) -> &C {
36 36209 : &self.0[kind.as_index()]
37 36209 : }
38 :
39 1250 : fn build_with(mut f: impl FnMut(RequestKind) -> C) -> Self {
40 1250 : use RequestKind::*;
41 1250 : let mut it = [Get, Put, Delete, List].into_iter();
42 5000 : let arr = std::array::from_fn::<C, 4, _>(|index| {
43 5000 : let next = it.next().unwrap();
44 5000 : assert_eq!(index, next.as_index());
45 5000 : f(next)
46 5000 : });
47 :
48 1250 : if let Some(next) = it.next() {
49 UBC 0 : panic!("unexpected {next:?}");
50 CBC 1250 : }
51 1250 :
52 1250 : RequestTyped(arr)
53 1250 : }
54 : }
55 :
56 : impl RequestTyped<Histogram> {
57 18141 : pub(super) fn observe_elapsed(&self, kind: RequestKind, started_at: std::time::Instant) {
58 18141 : self.get(kind).observe(started_at.elapsed().as_secs_f64())
59 18141 : }
60 : }
61 :
62 : pub(super) struct PassFailCancelledRequestTyped<C> {
63 : success: RequestTyped<C>,
64 : fail: RequestTyped<C>,
65 : cancelled: RequestTyped<C>,
66 : }
67 :
68 UBC 0 : #[derive(Debug, Clone, Copy)]
69 : pub(super) enum AttemptOutcome {
70 : Ok,
71 : Err,
72 : Cancelled,
73 : }
74 :
75 : impl<T, E> From<&Result<T, E>> for AttemptOutcome {
76 CBC 16872 : fn from(value: &Result<T, E>) -> Self {
77 16872 : match value {
78 16869 : Ok(_) => AttemptOutcome::Ok,
79 3 : Err(_) => AttemptOutcome::Err,
80 : }
81 16872 : }
82 : }
83 :
84 : impl AttemptOutcome {
85 3000 : pub(super) fn as_str(&self) -> &'static str {
86 3000 : match self {
87 1000 : AttemptOutcome::Ok => "ok",
88 1000 : AttemptOutcome::Err => "err",
89 1000 : AttemptOutcome::Cancelled => "cancelled",
90 : }
91 3000 : }
92 : }
93 :
94 : impl<C> PassFailCancelledRequestTyped<C> {
95 18068 : pub(super) fn get(&self, kind: RequestKind, outcome: AttemptOutcome) -> &C {
96 18068 : let target = match outcome {
97 17771 : AttemptOutcome::Ok => &self.success,
98 289 : AttemptOutcome::Err => &self.fail,
99 8 : AttemptOutcome::Cancelled => &self.cancelled,
100 : };
101 18068 : target.get(kind)
102 18068 : }
103 :
104 250 : fn build_with(mut f: impl FnMut(RequestKind, AttemptOutcome) -> C) -> Self {
105 1000 : let success = RequestTyped::build_with(|kind| f(kind, AttemptOutcome::Ok));
106 1000 : let fail = RequestTyped::build_with(|kind| f(kind, AttemptOutcome::Err));
107 1000 : let cancelled = RequestTyped::build_with(|kind| f(kind, AttemptOutcome::Cancelled));
108 250 :
109 250 : PassFailCancelledRequestTyped {
110 250 : success,
111 250 : fail,
112 250 : cancelled,
113 250 : }
114 250 : }
115 : }
116 :
117 : impl PassFailCancelledRequestTyped<Histogram> {
118 18068 : pub(super) fn observe_elapsed(
119 18068 : &self,
120 18068 : kind: RequestKind,
121 18068 : outcome: impl Into<AttemptOutcome>,
122 18068 : started_at: std::time::Instant,
123 18068 : ) {
124 18068 : self.get(kind, outcome.into())
125 18068 : .observe(started_at.elapsed().as_secs_f64())
126 18068 : }
127 : }
128 :
129 : pub(super) struct BucketMetrics {
130 : /// Full request duration until successful completion, error or cancellation.
131 : pub(super) req_seconds: PassFailCancelledRequestTyped<Histogram>,
132 : /// Total amount of seconds waited on queue.
133 : pub(super) wait_seconds: RequestTyped<Histogram>,
134 :
135 : /// Track how many semaphore awaits were cancelled per request type.
136 : ///
137 : /// This is in case cancellations are happening more than expected.
138 : pub(super) cancelled_waits: RequestTyped<IntCounter>,
139 :
140 : /// Total amount of deleted objects in batches or single requests.
141 : pub(super) deleted_objects_total: IntCounter,
142 : }
143 :
144 : impl Default for BucketMetrics {
145 250 : fn default() -> Self {
146 250 : let buckets = [0.01, 0.10, 0.5, 1.0, 5.0, 10.0, 50.0, 100.0];
147 250 :
148 250 : let req_seconds = register_histogram_vec!(
149 250 : "remote_storage_s3_request_seconds",
150 250 : "Seconds to complete a request",
151 250 : &["request_type", "result"],
152 250 : buckets.to_vec(),
153 250 : )
154 250 : .unwrap();
155 3000 : let req_seconds = PassFailCancelledRequestTyped::build_with(|kind, outcome| {
156 3000 : req_seconds.with_label_values(&[kind.as_str(), outcome.as_str()])
157 3000 : });
158 250 :
159 250 : let wait_seconds = register_histogram_vec!(
160 250 : "remote_storage_s3_wait_seconds",
161 250 : "Seconds rate limited",
162 250 : &["request_type"],
163 250 : buckets.to_vec(),
164 250 : )
165 250 : .unwrap();
166 250 : let wait_seconds =
167 1000 : RequestTyped::build_with(|kind| wait_seconds.with_label_values(&[kind.as_str()]));
168 250 :
169 250 : let cancelled_waits = register_int_counter_vec!(
170 250 : "remote_storage_s3_cancelled_waits_total",
171 250 : "Times a semaphore wait has been cancelled per request type",
172 250 : &["request_type"],
173 250 : )
174 250 : .unwrap();
175 250 : let cancelled_waits =
176 1000 : RequestTyped::build_with(|kind| cancelled_waits.with_label_values(&[kind.as_str()]));
177 250 :
178 250 : let deleted_objects_total = register_int_counter!(
179 250 : "remote_storage_s3_deleted_objects_total",
180 250 : "Amount of deleted objects in total",
181 250 : )
182 250 : .unwrap();
183 250 :
184 250 : Self {
185 250 : req_seconds,
186 250 : wait_seconds,
187 250 : cancelled_waits,
188 250 : deleted_objects_total,
189 250 : }
190 250 : }
191 : }
|
