Line data Source code
1 : use metrics::{
2 : register_histogram_vec, register_int_counter, register_int_counter_vec, Histogram, IntCounter,
3 : };
4 : use once_cell::sync::Lazy;
5 :
6 : pub(super) static BUCKET_METRICS: Lazy<BucketMetrics> = Lazy::new(Default::default);
7 :
8 0 : #[derive(Clone, Copy, Debug)]
9 : pub(super) enum RequestKind {
10 : Get = 0,
11 : Put = 1,
12 : Delete = 2,
13 : List = 3,
14 : }
15 :
16 : use RequestKind::*;
17 :
18 : impl RequestKind {
19 4440 : const fn as_str(&self) -> &'static str {
20 4440 : match self {
21 1110 : Get => "get_object",
22 1110 : Put => "put_object",
23 1110 : Delete => "delete_object",
24 1110 : List => "list_objects",
25 : }
26 4440 : }
27 40647 : const fn as_index(&self) -> usize {
28 40647 : *self as usize
29 40647 : }
30 : }
31 :
32 : pub(super) struct RequestTyped<C>([C; 4]);
33 :
34 : impl<C> RequestTyped<C> {
35 36207 : pub(super) fn get(&self, kind: RequestKind) -> &C {
36 36207 : &self.0[kind.as_index()]
37 36207 : }
38 :
39 1110 : fn build_with(mut f: impl FnMut(RequestKind) -> C) -> Self {
40 1110 : use RequestKind::*;
41 1110 : let mut it = [Get, Put, Delete, List].into_iter();
42 4440 : let arr = std::array::from_fn::<C, 4, _>(|index| {
43 4440 : let next = it.next().unwrap();
44 4440 : assert_eq!(index, next.as_index());
45 4440 : f(next)
46 4440 : });
47 :
48 1110 : if let Some(next) = it.next() {
49 0 : panic!("unexpected {next:?}");
50 1110 : }
51 1110 :
52 1110 : RequestTyped(arr)
53 1110 : }
54 : }
55 :
56 : impl RequestTyped<Histogram> {
57 18107 : pub(super) fn observe_elapsed(&self, kind: RequestKind, started_at: std::time::Instant) {
58 18107 : self.get(kind).observe(started_at.elapsed().as_secs_f64())
59 18107 : }
60 : }
61 :
62 : pub(super) struct PassFailCancelledRequestTyped<C> {
63 : success: RequestTyped<C>,
64 : fail: RequestTyped<C>,
65 : cancelled: RequestTyped<C>,
66 : }
67 :
68 0 : #[derive(Debug, Clone, Copy)]
69 : pub(super) enum AttemptOutcome {
70 : Ok,
71 : Err,
72 : Cancelled,
73 : }
74 :
75 : impl<T, E> From<&Result<T, E>> for AttemptOutcome {
76 17338 : fn from(value: &Result<T, E>) -> Self {
77 17338 : match value {
78 17337 : Ok(_) => AttemptOutcome::Ok,
79 1 : Err(_) => AttemptOutcome::Err,
80 : }
81 17338 : }
82 : }
83 :
84 : impl AttemptOutcome {
85 2664 : pub(super) fn as_str(&self) -> &'static str {
86 2664 : match self {
87 888 : AttemptOutcome::Ok => "ok",
88 888 : AttemptOutcome::Err => "err",
89 888 : AttemptOutcome::Cancelled => "cancelled",
90 : }
91 2664 : }
92 : }
93 :
94 : impl<C> PassFailCancelledRequestTyped<C> {
95 18100 : pub(super) fn get(&self, kind: RequestKind, outcome: AttemptOutcome) -> &C {
96 18100 : let target = match outcome {
97 17849 : AttemptOutcome::Ok => &self.success,
98 240 : AttemptOutcome::Err => &self.fail,
99 11 : AttemptOutcome::Cancelled => &self.cancelled,
100 : };
101 18100 : target.get(kind)
102 18100 : }
103 :
104 222 : fn build_with(mut f: impl FnMut(RequestKind, AttemptOutcome) -> C) -> Self {
105 888 : let success = RequestTyped::build_with(|kind| f(kind, AttemptOutcome::Ok));
106 888 : let fail = RequestTyped::build_with(|kind| f(kind, AttemptOutcome::Err));
107 888 : let cancelled = RequestTyped::build_with(|kind| f(kind, AttemptOutcome::Cancelled));
108 222 :
109 222 : PassFailCancelledRequestTyped {
110 222 : success,
111 222 : fail,
112 222 : cancelled,
113 222 : }
114 222 : }
115 : }
116 :
117 : impl PassFailCancelledRequestTyped<Histogram> {
118 18100 : pub(super) fn observe_elapsed(
119 18100 : &self,
120 18100 : kind: RequestKind,
121 18100 : outcome: impl Into<AttemptOutcome>,
122 18100 : started_at: std::time::Instant,
123 18100 : ) {
124 18100 : self.get(kind, outcome.into())
125 18100 : .observe(started_at.elapsed().as_secs_f64())
126 18100 : }
127 : }
128 :
129 : pub(super) struct BucketMetrics {
130 : /// Full request duration until successful completion, error or cancellation.
131 : pub(super) req_seconds: PassFailCancelledRequestTyped<Histogram>,
132 : /// Total amount of seconds waited on queue.
133 : pub(super) wait_seconds: RequestTyped<Histogram>,
134 :
135 : /// Track how many semaphore awaits were cancelled per request type.
136 : ///
137 : /// This is in case cancellations are happening more than expected.
138 : pub(super) cancelled_waits: RequestTyped<IntCounter>,
139 :
140 : /// Total amount of deleted objects in batches or single requests.
141 : pub(super) deleted_objects_total: IntCounter,
142 : }
143 :
144 : impl Default for BucketMetrics {
145 222 : fn default() -> Self {
146 222 : let buckets = [0.01, 0.10, 0.5, 1.0, 5.0, 10.0, 50.0, 100.0];
147 222 :
148 222 : let req_seconds = register_histogram_vec!(
149 222 : "remote_storage_s3_request_seconds",
150 222 : "Seconds to complete a request",
151 222 : &["request_type", "result"],
152 222 : buckets.to_vec(),
153 222 : )
154 222 : .unwrap();
155 2664 : let req_seconds = PassFailCancelledRequestTyped::build_with(|kind, outcome| {
156 2664 : req_seconds.with_label_values(&[kind.as_str(), outcome.as_str()])
157 2664 : });
158 222 :
159 222 : let wait_seconds = register_histogram_vec!(
160 222 : "remote_storage_s3_wait_seconds",
161 222 : "Seconds rate limited",
162 222 : &["request_type"],
163 222 : buckets.to_vec(),
164 222 : )
165 222 : .unwrap();
166 222 : let wait_seconds =
167 888 : RequestTyped::build_with(|kind| wait_seconds.with_label_values(&[kind.as_str()]));
168 222 :
169 222 : let cancelled_waits = register_int_counter_vec!(
170 222 : "remote_storage_s3_cancelled_waits_total",
171 222 : "Times a semaphore wait has been cancelled per request type",
172 222 : &["request_type"],
173 222 : )
174 222 : .unwrap();
175 222 : let cancelled_waits =
176 888 : RequestTyped::build_with(|kind| cancelled_waits.with_label_values(&[kind.as_str()]));
177 222 :
178 222 : let deleted_objects_total = register_int_counter!(
179 222 : "remote_storage_s3_deleted_objects_total",
180 222 : "Amount of deleted objects in total",
181 222 : )
182 222 : .unwrap();
183 222 :
184 222 : Self {
185 222 : req_seconds,
186 222 : wait_seconds,
187 222 : cancelled_waits,
188 222 : deleted_objects_total,
189 222 : }
190 222 : }
191 : }
|
