Line data Source code
1 : use metrics::{
2 : register_histogram_vec, register_int_counter, register_int_counter_vec, Histogram, IntCounter,
3 : };
4 : use once_cell::sync::Lazy;
5 :
6 : pub(super) static BUCKET_METRICS: Lazy<BucketMetrics> = Lazy::new(Default::default);
7 :
8 : #[derive(Clone, Copy, Debug)]
9 : pub(crate) enum RequestKind {
10 : Get = 0,
11 : Put = 1,
12 : Delete = 2,
13 : List = 3,
14 : Copy = 4,
15 : TimeTravel = 5,
16 : }
17 :
18 : use RequestKind::*;
19 :
20 : impl RequestKind {
21 540 : const fn as_str(&self) -> &'static str {
22 540 : match self {
23 90 : Get => "get_object",
24 90 : Put => "put_object",
25 90 : Delete => "delete_object",
26 90 : List => "list_objects",
27 90 : Copy => "copy_object",
28 90 : TimeTravel => "time_travel_recover",
29 : }
30 540 : }
31 1076 : const fn as_index(&self) -> usize {
32 1076 : *self as usize
33 1076 : }
34 : }
35 :
36 : pub(super) struct RequestTyped<C>([C; 6]);
37 :
38 : impl<C> RequestTyped<C> {
39 536 : pub(super) fn get(&self, kind: RequestKind) -> &C {
40 536 : &self.0[kind.as_index()]
41 536 : }
42 :
43 90 : fn build_with(mut f: impl FnMut(RequestKind) -> C) -> Self {
44 90 : use RequestKind::*;
45 90 : let mut it = [Get, Put, Delete, List, Copy, TimeTravel].into_iter();
46 540 : let arr = std::array::from_fn::<C, 6, _>(|index| {
47 540 : let next = it.next().unwrap();
48 540 : assert_eq!(index, next.as_index());
49 540 : f(next)
50 540 : });
51 :
52 90 : if let Some(next) = it.next() {
53 0 : panic!("unexpected {next:?}");
54 90 : }
55 90 :
56 90 : RequestTyped(arr)
57 90 : }
58 : }
59 :
60 : impl RequestTyped<Histogram> {
61 265 : pub(super) fn observe_elapsed(&self, kind: RequestKind, started_at: std::time::Instant) {
62 265 : self.get(kind).observe(started_at.elapsed().as_secs_f64())
63 265 : }
64 : }
65 :
66 : pub(super) struct PassFailCancelledRequestTyped<C> {
67 : success: RequestTyped<C>,
68 : fail: RequestTyped<C>,
69 : cancelled: RequestTyped<C>,
70 : }
71 :
72 : #[derive(Debug, Clone, Copy)]
73 : pub(super) enum AttemptOutcome {
74 : Ok,
75 : Err,
76 : Cancelled,
77 : }
78 :
79 : impl<T, E> From<&Result<T, E>> for AttemptOutcome {
80 0 : fn from(value: &Result<T, E>) -> Self {
81 0 : match value {
82 0 : Ok(_) => AttemptOutcome::Ok,
83 0 : Err(_) => AttemptOutcome::Err,
84 : }
85 0 : }
86 : }
87 :
88 : impl AttemptOutcome {
89 324 : pub(super) fn as_str(&self) -> &'static str {
90 324 : match self {
91 108 : AttemptOutcome::Ok => "ok",
92 108 : AttemptOutcome::Err => "err",
93 108 : AttemptOutcome::Cancelled => "cancelled",
94 : }
95 324 : }
96 : }
97 :
98 : impl<C> PassFailCancelledRequestTyped<C> {
99 0 : pub(super) fn get(&self, kind: RequestKind, outcome: AttemptOutcome) -> &C {
100 0 : let target = match outcome {
101 0 : AttemptOutcome::Ok => &self.success,
102 0 : AttemptOutcome::Err => &self.fail,
103 0 : AttemptOutcome::Cancelled => &self.cancelled,
104 : };
105 0 : target.get(kind)
106 0 : }
107 :
108 18 : fn build_with(mut f: impl FnMut(RequestKind, AttemptOutcome) -> C) -> Self {
109 108 : let success = RequestTyped::build_with(|kind| f(kind, AttemptOutcome::Ok));
110 108 : let fail = RequestTyped::build_with(|kind| f(kind, AttemptOutcome::Err));
111 108 : let cancelled = RequestTyped::build_with(|kind| f(kind, AttemptOutcome::Cancelled));
112 18 :
113 18 : PassFailCancelledRequestTyped {
114 18 : success,
115 18 : fail,
116 18 : cancelled,
117 18 : }
118 18 : }
119 : }
120 :
121 : impl PassFailCancelledRequestTyped<Histogram> {
122 0 : pub(super) fn observe_elapsed(
123 0 : &self,
124 0 : kind: RequestKind,
125 0 : outcome: impl Into<AttemptOutcome>,
126 0 : started_at: std::time::Instant,
127 0 : ) {
128 0 : self.get(kind, outcome.into())
129 0 : .observe(started_at.elapsed().as_secs_f64())
130 0 : }
131 : }
132 :
133 : pub(super) struct BucketMetrics {
134 : /// Full request duration until successful completion, error or cancellation.
135 : pub(super) req_seconds: PassFailCancelledRequestTyped<Histogram>,
136 : /// Total amount of seconds waited on queue.
137 : pub(super) wait_seconds: RequestTyped<Histogram>,
138 :
139 : /// Track how many semaphore awaits were cancelled per request type.
140 : ///
141 : /// This is in case cancellations are happening more than expected.
142 : pub(super) cancelled_waits: RequestTyped<IntCounter>,
143 :
144 : /// Total amount of deleted objects in batches or single requests.
145 : pub(super) deleted_objects_total: IntCounter,
146 : }
147 :
148 : impl Default for BucketMetrics {
149 18 : fn default() -> Self {
150 18 : let buckets = [0.01, 0.10, 0.5, 1.0, 5.0, 10.0, 50.0, 100.0];
151 18 :
152 18 : let req_seconds = register_histogram_vec!(
153 : "remote_storage_s3_request_seconds",
154 : "Seconds to complete a request",
155 : &["request_type", "result"],
156 : buckets.to_vec(),
157 : )
158 18 : .unwrap();
159 324 : let req_seconds = PassFailCancelledRequestTyped::build_with(|kind, outcome| {
160 324 : req_seconds.with_label_values(&[kind.as_str(), outcome.as_str()])
161 324 : });
162 18 :
163 18 : let wait_seconds = register_histogram_vec!(
164 : "remote_storage_s3_wait_seconds",
165 : "Seconds rate limited",
166 : &["request_type"],
167 : buckets.to_vec(),
168 : )
169 18 : .unwrap();
170 18 : let wait_seconds =
171 108 : RequestTyped::build_with(|kind| wait_seconds.with_label_values(&[kind.as_str()]));
172 18 :
173 18 : let cancelled_waits = register_int_counter_vec!(
174 : "remote_storage_s3_cancelled_waits_total",
175 : "Times a semaphore wait has been cancelled per request type",
176 : &["request_type"],
177 : )
178 18 : .unwrap();
179 18 : let cancelled_waits =
180 108 : RequestTyped::build_with(|kind| cancelled_waits.with_label_values(&[kind.as_str()]));
181 18 :
182 18 : let deleted_objects_total = register_int_counter!(
183 : "remote_storage_s3_deleted_objects_total",
184 : "Amount of deleted objects in total",
185 : )
186 18 : .unwrap();
187 18 :
188 18 : Self {
189 18 : req_seconds,
190 18 : wait_seconds,
191 18 : cancelled_waits,
192 18 : deleted_objects_total,
193 18 : }
194 18 : }
195 : }
|
