Line data Source code
1 : use ::metrics::{
2 : exponential_buckets, register_histogram, register_histogram_vec, register_hll_vec,
3 : register_int_counter_pair_vec, register_int_counter_vec, register_int_gauge,
4 : register_int_gauge_vec, Histogram, HistogramVec, HyperLogLogVec, IntCounterPairVec,
5 : IntCounterVec, IntGauge, IntGaugeVec,
6 : };
7 : use metrics::{register_int_counter_pair, IntCounterPair};
8 :
9 : use once_cell::sync::Lazy;
10 : use tokio::time;
11 :
12 23 : pub static NUM_DB_CONNECTIONS_GAUGE: Lazy<IntCounterPairVec> = Lazy::new(|| {
13 46 : register_int_counter_pair_vec!(
14 46 : "proxy_opened_db_connections_total",
15 46 : "Number of opened connections to a database.",
16 46 : "proxy_closed_db_connections_total",
17 46 : "Number of closed connections to a database.",
18 46 : &["protocol"],
19 46 : )
20 23 : .unwrap()
21 23 : });
22 :
23 24 : pub static NUM_CLIENT_CONNECTION_GAUGE: Lazy<IntCounterPairVec> = Lazy::new(|| {
24 48 : register_int_counter_pair_vec!(
25 48 : "proxy_opened_client_connections_total",
26 48 : "Number of opened connections from a client.",
27 48 : "proxy_closed_client_connections_total",
28 48 : "Number of closed connections from a client.",
29 48 : &["protocol"],
30 48 : )
31 24 : .unwrap()
32 24 : });
33 :
34 24 : pub static NUM_CONNECTION_REQUESTS_GAUGE: Lazy<IntCounterPairVec> = Lazy::new(|| {
35 48 : register_int_counter_pair_vec!(
36 48 : "proxy_accepted_connections_total",
37 48 : "Number of client connections accepted.",
38 48 : "proxy_closed_connections_total",
39 48 : "Number of client connections closed.",
40 48 : &["protocol"],
41 48 : )
42 24 : .unwrap()
43 24 : });
44 :
45 91 : pub static COMPUTE_CONNECTION_LATENCY: Lazy<HistogramVec> = Lazy::new(|| {
46 91 : register_histogram_vec!(
47 91 : "proxy_compute_connection_latency_seconds",
48 91 : "Time it took for proxy to establish a connection to the compute endpoint",
49 91 : // http/ws/tcp, true/false, true/false, success/failure
50 91 : // 3 * 2 * 2 * 2 = 24 counters
51 91 : &["protocol", "cache_miss", "pool_miss", "outcome"],
52 91 : // largest bucket = 2^16 * 0.5ms = 32s
53 91 : exponential_buckets(0.0005, 2.0, 16).unwrap(),
54 91 : )
55 91 : .unwrap()
56 91 : });
57 :
58 1 : pub static CONSOLE_REQUEST_LATENCY: Lazy<HistogramVec> = Lazy::new(|| {
59 1 : register_histogram_vec!(
60 1 : "proxy_console_request_latency",
61 1 : "Time it took for proxy to establish a connection to the compute endpoint",
62 1 : // proxy_wake_compute/proxy_get_role_info
63 1 : &["request"],
64 1 : // largest bucket = 2^16 * 0.2ms = 13s
65 1 : exponential_buckets(0.0002, 2.0, 16).unwrap(),
66 1 : )
67 1 : .unwrap()
68 1 : });
69 :
70 1 : pub static ALLOWED_IPS_BY_CACHE_OUTCOME: Lazy<IntCounterVec> = Lazy::new(|| {
71 1 : register_int_counter_vec!(
72 1 : "proxy_allowed_ips_cache_misses",
73 1 : "Number of cache hits/misses for allowed ips",
74 1 : // hit/miss
75 1 : &["outcome"],
76 1 : )
77 1 : .unwrap()
78 1 : });
79 :
80 3 : pub static RATE_LIMITER_ACQUIRE_LATENCY: Lazy<Histogram> = Lazy::new(|| {
81 3 : register_histogram!(
82 3 : "proxy_control_plane_token_acquire_seconds",
83 3 : "Time it took for proxy to establish a connection to the compute endpoint",
84 3 : // largest bucket = 3^16 * 0.05ms = 2.15s
85 3 : exponential_buckets(0.00005, 3.0, 16).unwrap(),
86 3 : )
87 3 : .unwrap()
88 3 : });
89 :
90 17 : pub static RATE_LIMITER_LIMIT: Lazy<IntGaugeVec> = Lazy::new(|| {
91 17 : register_int_gauge_vec!(
92 17 : "semaphore_control_plane_limit",
93 17 : "Current limit of the semaphore control plane",
94 17 : &["limit"], // 2 counters
95 17 : )
96 17 : .unwrap()
97 17 : });
98 :
99 42 : pub static NUM_CONNECTION_ACCEPTED_BY_SNI: Lazy<IntCounterVec> = Lazy::new(|| {
100 42 : register_int_counter_vec!(
101 42 : "proxy_accepted_connections_by_sni",
102 42 : "Number of connections (per sni).",
103 42 : &["kind"],
104 42 : )
105 42 : .unwrap()
106 42 : });
107 :
108 0 : pub static ALLOWED_IPS_NUMBER: Lazy<Histogram> = Lazy::new(|| {
109 0 : register_histogram!(
110 0 : "proxy_allowed_ips_number",
111 0 : "Number of allowed ips",
112 0 : vec![0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 10.0, 20.0, 50.0, 100.0],
113 0 : )
114 0 : .unwrap()
115 0 : });
116 :
117 13 : pub static HTTP_CONTENT_LENGTH: Lazy<Histogram> = Lazy::new(|| {
118 13 : register_histogram!(
119 13 : "proxy_http_conn_content_length_bytes",
120 13 : "Time it took for proxy to establish a connection to the compute endpoint",
121 13 : // largest bucket = 3^16 * 0.05ms = 2.15s
122 13 : exponential_buckets(8.0, 2.0, 20).unwrap()
123 13 : )
124 13 : .unwrap()
125 13 : });
126 :
127 27 : pub static GC_LATENCY: Lazy<Histogram> = Lazy::new(|| {
128 27 : register_histogram!(
129 27 : "proxy_http_pool_reclaimation_lag_seconds",
130 27 : "Time it takes to reclaim unused connection pools",
131 27 : // 1us -> 65ms
132 27 : exponential_buckets(1e-6, 2.0, 16).unwrap(),
133 27 : )
134 27 : .unwrap()
135 27 : });
136 :
137 15 : pub static ENDPOINT_POOLS: Lazy<IntCounterPair> = Lazy::new(|| {
138 30 : register_int_counter_pair!(
139 30 : "proxy_http_pool_endpoints_registered_total",
140 30 : "Number of endpoints we have registered pools for",
141 30 : "proxy_http_pool_endpoints_unregistered_total",
142 30 : "Number of endpoints we have unregistered pools for",
143 30 : )
144 15 : .unwrap()
145 15 : });
146 :
147 14 : pub static NUM_OPEN_CLIENTS_IN_HTTP_POOL: Lazy<IntGauge> = Lazy::new(|| {
148 14 : register_int_gauge!(
149 14 : "proxy_http_pool_opened_connections",
150 14 : "Number of opened connections to a database.",
151 14 : )
152 14 : .unwrap()
153 14 : });
154 :
155 0 : pub static NUM_CANCELLATION_REQUESTS: Lazy<IntCounterVec> = Lazy::new(|| {
156 0 : register_int_counter_vec!(
157 0 : "proxy_cancellation_requests_total",
158 0 : "Number of cancellation requests (per found/not_found).",
159 0 : &["source", "kind"],
160 0 : )
161 0 : .unwrap()
162 0 : });
163 :
164 0 : #[derive(Clone)]
165 : pub struct LatencyTimer {
166 : // time since the stopwatch was started
167 : start: Option<time::Instant>,
168 : // accumulated time on the stopwatch
169 : pub accumulated: std::time::Duration,
170 : // label data
171 : protocol: &'static str,
172 : cache_miss: bool,
173 : pool_miss: bool,
174 : outcome: &'static str,
175 : }
176 :
177 : pub struct LatencyTimerPause<'a> {
178 : timer: &'a mut LatencyTimer,
179 : }
180 :
181 : impl LatencyTimer {
182 180 : pub fn new(protocol: &'static str) -> Self {
183 180 : Self {
184 180 : start: Some(time::Instant::now()),
185 180 : accumulated: std::time::Duration::ZERO,
186 180 : protocol,
187 180 : cache_miss: false,
188 180 : // by default we don't do pooling
189 180 : pool_miss: true,
190 180 : // assume failed unless otherwise specified
191 180 : outcome: "failed",
192 180 : }
193 180 : }
194 :
195 129 : pub fn pause(&mut self) -> LatencyTimerPause<'_> {
196 129 : // stop the stopwatch and record the time that we have accumulated
197 129 : let start = self.start.take().expect("latency timer should be started");
198 129 : self.accumulated += start.elapsed();
199 129 : LatencyTimerPause { timer: self }
200 129 : }
201 :
202 8 : pub fn cache_miss(&mut self) {
203 8 : self.cache_miss = true;
204 8 : }
205 :
206 4 : pub fn pool_hit(&mut self) {
207 4 : self.pool_miss = false;
208 4 : }
209 :
210 139 : pub fn success(&mut self) {
211 : // stop the stopwatch and record the time that we have accumulated
212 139 : if let Some(start) = self.start.take() {
213 95 : self.accumulated += start.elapsed();
214 95 : }
215 :
216 : // success
217 139 : self.outcome = "success";
218 139 : }
219 : }
220 :
221 : impl Drop for LatencyTimerPause<'_> {
222 129 : fn drop(&mut self) {
223 129 : // start the stopwatch again
224 129 : self.timer.start = Some(time::Instant::now());
225 129 : }
226 : }
227 :
228 : impl Drop for LatencyTimer {
229 180 : fn drop(&mut self) {
230 180 : let duration =
231 180 : self.start.map(|start| start.elapsed()).unwrap_or_default() + self.accumulated;
232 180 : COMPUTE_CONNECTION_LATENCY
233 180 : .with_label_values(&[
234 180 : self.protocol,
235 180 : bool_to_str(self.cache_miss),
236 180 : bool_to_str(self.pool_miss),
237 180 : self.outcome,
238 180 : ])
239 180 : .observe(duration.as_secs_f64())
240 180 : }
241 : }
242 :
243 8 : pub static NUM_CONNECTION_FAILURES: Lazy<IntCounterVec> = Lazy::new(|| {
244 8 : register_int_counter_vec!(
245 8 : "proxy_connection_failures_total",
246 8 : "Number of connection failures (per kind).",
247 8 : &["kind"],
248 8 : )
249 8 : .unwrap()
250 8 : });
251 :
252 4 : pub static NUM_WAKEUP_FAILURES: Lazy<IntCounterVec> = Lazy::new(|| {
253 4 : register_int_counter_vec!(
254 4 : "proxy_connection_failures_breakdown",
255 4 : "Number of wake-up failures (per kind).",
256 4 : &["retry", "kind"],
257 4 : )
258 4 : .unwrap()
259 4 : });
260 :
261 24 : pub static NUM_BYTES_PROXIED_COUNTER: Lazy<IntCounterVec> = Lazy::new(|| {
262 24 : register_int_counter_vec!(
263 24 : "proxy_io_bytes",
264 24 : "Number of bytes sent/received between all clients and backends.",
265 24 : &["direction"],
266 24 : )
267 24 : .unwrap()
268 24 : });
269 :
270 366 : pub const fn bool_to_str(x: bool) -> &'static str {
271 366 : if x {
272 188 : "true"
273 : } else {
274 178 : "false"
275 : }
276 366 : }
277 :
278 34 : pub static CONNECTING_ENDPOINTS: Lazy<HyperLogLogVec<32>> = Lazy::new(|| {
279 34 : register_hll_vec!(
280 34 : 32,
281 34 : "proxy_connecting_endpoints",
282 34 : "HLL approximate cardinality of endpoints that are connecting",
283 34 : &["protocol"],
284 34 : )
285 34 : .unwrap()
286 34 : });
287 :
288 4 : pub static ERROR_BY_KIND: Lazy<IntCounterVec> = Lazy::new(|| {
289 4 : register_int_counter_vec!(
290 4 : "proxy_errors_total",
291 4 : "Number of errors by a given classification",
292 4 : &["type"],
293 4 : )
294 4 : .unwrap()
295 4 : });
296 :
297 3 : pub static ENDPOINT_ERRORS_BY_KIND: Lazy<HyperLogLogVec<32>> = Lazy::new(|| {
298 3 : register_hll_vec!(
299 3 : 32,
300 3 : "proxy_endpoints_affected_by_errors",
301 3 : "Number of endpoints affected by errors of a given classification",
302 3 : &["type"],
303 3 : )
304 3 : .unwrap()
305 3 : });
|
