Line data Source code
1 : use metrics::{
2 : register_histogram_vec, register_int_counter, register_int_counter_vec, Histogram, IntCounter,
3 : };
4 : use once_cell::sync::Lazy;
5 :
6 : pub(super) static BUCKET_METRICS: Lazy<BucketMetrics> = Lazy::new(Default::default);
7 :
8 0 : #[derive(Clone, Copy, Debug)]
9 : pub(crate) enum RequestKind {
10 : Get = 0,
11 : Put = 1,
12 : Delete = 2,
13 : List = 3,
14 : Copy = 4,
15 : TimeTravel = 5,
16 : }
17 :
18 : use RequestKind::*;
19 :
20 : impl RequestKind {
21 7320 : const fn as_str(&self) -> &'static str {
22 7320 : match self {
23 1220 : Get => "get_object",
24 1220 : Put => "put_object",
25 1220 : Delete => "delete_object",
26 1220 : List => "list_objects",
27 1220 : Copy => "copy_object",
28 1220 : TimeTravel => "time_travel_recover",
29 : }
30 7320 : }
31 66940 : const fn as_index(&self) -> usize {
32 66940 : *self as usize
33 66940 : }
34 : }
35 :
36 : pub(super) struct RequestTyped<C>([C; 6]);
37 :
38 : impl<C> RequestTyped<C> {
39 59620 : pub(super) fn get(&self, kind: RequestKind) -> &C {
40 59620 : &self.0[kind.as_index()]
41 59620 : }
42 :
43 1220 : fn build_with(mut f: impl FnMut(RequestKind) -> C) -> Self {
44 1220 : use RequestKind::*;
45 1220 : let mut it = [Get, Put, Delete, List, Copy, TimeTravel].into_iter();
46 7320 : let arr = std::array::from_fn::<C, 6, _>(|index| {
47 7320 : let next = it.next().unwrap();
48 7320 : assert_eq!(index, next.as_index());
49 7320 : f(next)
50 7320 : });
51 :
52 1220 : if let Some(next) = it.next() {
53 0 : panic!("unexpected {next:?}");
54 1220 : }
55 1220 :
56 1220 : RequestTyped(arr)
57 1220 : }
58 : }
59 :
60 : impl RequestTyped<Histogram> {
61 29813 : pub(super) fn observe_elapsed(&self, kind: RequestKind, started_at: std::time::Instant) {
62 29813 : self.get(kind).observe(started_at.elapsed().as_secs_f64())
63 29813 : }
64 : }
65 :
66 : pub(super) struct PassFailCancelledRequestTyped<C> {
67 : success: RequestTyped<C>,
68 : fail: RequestTyped<C>,
69 : cancelled: RequestTyped<C>,
70 : }
71 :
72 0 : #[derive(Debug, Clone, Copy)]
73 : pub(super) enum AttemptOutcome {
74 : Ok,
75 : Err,
76 : Cancelled,
77 : }
78 :
79 : impl<T, E> From<&Result<T, E>> for AttemptOutcome {
80 18806 : fn from(value: &Result<T, E>) -> Self {
81 18806 : match value {
82 18806 : Ok(_) => AttemptOutcome::Ok,
83 0 : Err(_) => AttemptOutcome::Err,
84 : }
85 18806 : }
86 : }
87 :
88 : impl AttemptOutcome {
89 4392 : pub(super) fn as_str(&self) -> &'static str {
90 4392 : match self {
91 1464 : AttemptOutcome::Ok => "ok",
92 1464 : AttemptOutcome::Err => "err",
93 1464 : AttemptOutcome::Cancelled => "cancelled",
94 : }
95 4392 : }
96 : }
97 :
98 : impl<C> PassFailCancelledRequestTyped<C> {
99 29550 : pub(super) fn get(&self, kind: RequestKind, outcome: AttemptOutcome) -> &C {
100 29550 : let target = match outcome {
101 29508 : AttemptOutcome::Ok => &self.success,
102 0 : AttemptOutcome::Err => &self.fail,
103 42 : AttemptOutcome::Cancelled => &self.cancelled,
104 : };
105 29550 : target.get(kind)
106 29550 : }
107 :
108 244 : fn build_with(mut f: impl FnMut(RequestKind, AttemptOutcome) -> C) -> Self {
109 1464 : let success = RequestTyped::build_with(|kind| f(kind, AttemptOutcome::Ok));
110 1464 : let fail = RequestTyped::build_with(|kind| f(kind, AttemptOutcome::Err));
111 1464 : let cancelled = RequestTyped::build_with(|kind| f(kind, AttemptOutcome::Cancelled));
112 244 :
113 244 : PassFailCancelledRequestTyped {
114 244 : success,
115 244 : fail,
116 244 : cancelled,
117 244 : }
118 244 : }
119 : }
120 :
121 : impl PassFailCancelledRequestTyped<Histogram> {
122 29550 : pub(super) fn observe_elapsed(
123 29550 : &self,
124 29550 : kind: RequestKind,
125 29550 : outcome: impl Into<AttemptOutcome>,
126 29550 : started_at: std::time::Instant,
127 29550 : ) {
128 29550 : self.get(kind, outcome.into())
129 29550 : .observe(started_at.elapsed().as_secs_f64())
130 29550 : }
131 : }
132 :
133 : pub(super) struct BucketMetrics {
134 : /// Full request duration until successful completion, error or cancellation.
135 : pub(super) req_seconds: PassFailCancelledRequestTyped<Histogram>,
136 : /// Total amount of seconds waited on queue.
137 : pub(super) wait_seconds: RequestTyped<Histogram>,
138 :
139 : /// Track how many semaphore awaits were cancelled per request type.
140 : ///
141 : /// This is in case cancellations are happening more than expected.
142 : pub(super) cancelled_waits: RequestTyped<IntCounter>,
143 :
144 : /// Total amount of deleted objects in batches or single requests.
145 : pub(super) deleted_objects_total: IntCounter,
146 : }
147 :
148 : impl Default for BucketMetrics {
149 244 : fn default() -> Self {
150 244 : let buckets = [0.01, 0.10, 0.5, 1.0, 5.0, 10.0, 50.0, 100.0];
151 244 :
152 244 : let req_seconds = register_histogram_vec!(
153 244 : "remote_storage_s3_request_seconds",
154 244 : "Seconds to complete a request",
155 244 : &["request_type", "result"],
156 244 : buckets.to_vec(),
157 244 : )
158 244 : .unwrap();
159 4392 : let req_seconds = PassFailCancelledRequestTyped::build_with(|kind, outcome| {
160 4392 : req_seconds.with_label_values(&[kind.as_str(), outcome.as_str()])
161 4392 : });
162 244 :
163 244 : let wait_seconds = register_histogram_vec!(
164 244 : "remote_storage_s3_wait_seconds",
165 244 : "Seconds rate limited",
166 244 : &["request_type"],
167 244 : buckets.to_vec(),
168 244 : )
169 244 : .unwrap();
170 244 : let wait_seconds =
171 1464 : RequestTyped::build_with(|kind| wait_seconds.with_label_values(&[kind.as_str()]));
172 244 :
173 244 : let cancelled_waits = register_int_counter_vec!(
174 244 : "remote_storage_s3_cancelled_waits_total",
175 244 : "Times a semaphore wait has been cancelled per request type",
176 244 : &["request_type"],
177 244 : )
178 244 : .unwrap();
179 244 : let cancelled_waits =
180 1464 : RequestTyped::build_with(|kind| cancelled_waits.with_label_values(&[kind.as_str()]));
181 244 :
182 244 : let deleted_objects_total = register_int_counter!(
183 244 : "remote_storage_s3_deleted_objects_total",
184 244 : "Amount of deleted objects in total",
185 244 : )
186 244 : .unwrap();
187 244 :
188 244 : Self {
189 244 : req_seconds,
190 244 : wait_seconds,
191 244 : cancelled_waits,
192 244 : deleted_objects_total,
193 244 : }
194 244 : }
195 : }
|
