Line data Source code
1 : use metrics::{
2 : register_histogram_vec, register_int_counter, register_int_counter_vec, Histogram, IntCounter,
3 : };
4 : use once_cell::sync::Lazy;
5 :
6 : pub(super) static BUCKET_METRICS: Lazy<BucketMetrics> = Lazy::new(Default::default);
7 :
8 0 : #[derive(Clone, Copy, Debug)]
9 : pub(crate) enum RequestKind {
10 : Get = 0,
11 : Put = 1,
12 : Delete = 2,
13 : List = 3,
14 : Copy = 4,
15 : TimeTravel = 5,
16 : }
17 :
18 : use RequestKind::*;
19 :
20 : impl RequestKind {
21 7830 : const fn as_str(&self) -> &'static str {
22 7830 : match self {
23 1305 : Get => "get_object",
24 1305 : Put => "put_object",
25 1305 : Delete => "delete_object",
26 1305 : List => "list_objects",
27 1305 : Copy => "copy_object",
28 1305 : TimeTravel => "time_travel_recover",
29 : }
30 7830 : }
31 67292 : const fn as_index(&self) -> usize {
32 67292 : *self as usize
33 67292 : }
34 : }
35 :
36 : pub(super) struct RequestTyped<C>([C; 6]);
37 :
38 : impl<C> RequestTyped<C> {
39 59462 : pub(super) fn get(&self, kind: RequestKind) -> &C {
40 59462 : &self.0[kind.as_index()]
41 59462 : }
42 :
43 1305 : fn build_with(mut f: impl FnMut(RequestKind) -> C) -> Self {
44 1305 : use RequestKind::*;
45 1305 : let mut it = [Get, Put, Delete, List, Copy, TimeTravel].into_iter();
46 7830 : let arr = std::array::from_fn::<C, 6, _>(|index| {
47 7830 : let next = it.next().unwrap();
48 7830 : assert_eq!(index, next.as_index());
49 7830 : f(next)
50 7830 : });
51 :
52 1305 : if let Some(next) = it.next() {
53 0 : panic!("unexpected {next:?}");
54 1305 : }
55 1305 :
56 1305 : RequestTyped(arr)
57 1305 : }
58 : }
59 :
60 : impl RequestTyped<Histogram> {
61 29733 : pub(super) fn observe_elapsed(&self, kind: RequestKind, started_at: std::time::Instant) {
62 29733 : self.get(kind).observe(started_at.elapsed().as_secs_f64())
63 29733 : }
64 : }
65 :
66 : pub(super) struct PassFailCancelledRequestTyped<C> {
67 : success: RequestTyped<C>,
68 : fail: RequestTyped<C>,
69 : cancelled: RequestTyped<C>,
70 : }
71 :
72 0 : #[derive(Debug, Clone, Copy)]
73 : pub(super) enum AttemptOutcome {
74 : Ok,
75 : Err,
76 : Cancelled,
77 : }
78 :
79 : impl<T, E> From<&Result<T, E>> for AttemptOutcome {
80 19004 : fn from(value: &Result<T, E>) -> Self {
81 19004 : match value {
82 19004 : Ok(_) => AttemptOutcome::Ok,
83 0 : Err(_) => AttemptOutcome::Err,
84 : }
85 19004 : }
86 : }
87 :
88 : impl AttemptOutcome {
89 4698 : pub(super) fn as_str(&self) -> &'static str {
90 4698 : match self {
91 1566 : AttemptOutcome::Ok => "ok",
92 1566 : AttemptOutcome::Err => "err",
93 1566 : AttemptOutcome::Cancelled => "cancelled",
94 : }
95 4698 : }
96 : }
97 :
98 : impl<C> PassFailCancelledRequestTyped<C> {
99 29467 : pub(super) fn get(&self, kind: RequestKind, outcome: AttemptOutcome) -> &C {
100 29467 : let target = match outcome {
101 29428 : AttemptOutcome::Ok => &self.success,
102 0 : AttemptOutcome::Err => &self.fail,
103 39 : AttemptOutcome::Cancelled => &self.cancelled,
104 : };
105 29467 : target.get(kind)
106 29467 : }
107 :
108 261 : fn build_with(mut f: impl FnMut(RequestKind, AttemptOutcome) -> C) -> Self {
109 1566 : let success = RequestTyped::build_with(|kind| f(kind, AttemptOutcome::Ok));
110 1566 : let fail = RequestTyped::build_with(|kind| f(kind, AttemptOutcome::Err));
111 1566 : let cancelled = RequestTyped::build_with(|kind| f(kind, AttemptOutcome::Cancelled));
112 261 :
113 261 : PassFailCancelledRequestTyped {
114 261 : success,
115 261 : fail,
116 261 : cancelled,
117 261 : }
118 261 : }
119 : }
120 :
121 : impl PassFailCancelledRequestTyped<Histogram> {
122 29467 : pub(super) fn observe_elapsed(
123 29467 : &self,
124 29467 : kind: RequestKind,
125 29467 : outcome: impl Into<AttemptOutcome>,
126 29467 : started_at: std::time::Instant,
127 29467 : ) {
128 29467 : self.get(kind, outcome.into())
129 29467 : .observe(started_at.elapsed().as_secs_f64())
130 29467 : }
131 : }
132 :
133 : pub(super) struct BucketMetrics {
134 : /// Full request duration until successful completion, error or cancellation.
135 : pub(super) req_seconds: PassFailCancelledRequestTyped<Histogram>,
136 : /// Total amount of seconds waited on queue.
137 : pub(super) wait_seconds: RequestTyped<Histogram>,
138 :
139 : /// Track how many semaphore awaits were cancelled per request type.
140 : ///
141 : /// This is in case cancellations are happening more than expected.
142 : pub(super) cancelled_waits: RequestTyped<IntCounter>,
143 :
144 : /// Total amount of deleted objects in batches or single requests.
145 : pub(super) deleted_objects_total: IntCounter,
146 : }
147 :
148 : impl Default for BucketMetrics {
149 261 : fn default() -> Self {
150 261 : let buckets = [0.01, 0.10, 0.5, 1.0, 5.0, 10.0, 50.0, 100.0];
151 261 :
152 261 : let req_seconds = register_histogram_vec!(
153 261 : "remote_storage_s3_request_seconds",
154 261 : "Seconds to complete a request",
155 261 : &["request_type", "result"],
156 261 : buckets.to_vec(),
157 261 : )
158 261 : .unwrap();
159 4698 : let req_seconds = PassFailCancelledRequestTyped::build_with(|kind, outcome| {
160 4698 : req_seconds.with_label_values(&[kind.as_str(), outcome.as_str()])
161 4698 : });
162 261 :
163 261 : let wait_seconds = register_histogram_vec!(
164 261 : "remote_storage_s3_wait_seconds",
165 261 : "Seconds rate limited",
166 261 : &["request_type"],
167 261 : buckets.to_vec(),
168 261 : )
169 261 : .unwrap();
170 261 : let wait_seconds =
171 1566 : RequestTyped::build_with(|kind| wait_seconds.with_label_values(&[kind.as_str()]));
172 261 :
173 261 : let cancelled_waits = register_int_counter_vec!(
174 261 : "remote_storage_s3_cancelled_waits_total",
175 261 : "Times a semaphore wait has been cancelled per request type",
176 261 : &["request_type"],
177 261 : )
178 261 : .unwrap();
179 261 : let cancelled_waits =
180 1566 : RequestTyped::build_with(|kind| cancelled_waits.with_label_values(&[kind.as_str()]));
181 261 :
182 261 : let deleted_objects_total = register_int_counter!(
183 261 : "remote_storage_s3_deleted_objects_total",
184 261 : "Amount of deleted objects in total",
185 261 : )
186 261 : .unwrap();
187 261 :
188 261 : Self {
189 261 : req_seconds,
190 261 : wait_seconds,
191 261 : cancelled_waits,
192 261 : deleted_objects_total,
193 261 : }
194 261 : }
195 : }
|
