Line data Source code
1 : use ::metrics::{
2 : exponential_buckets, register_histogram, register_histogram_vec, register_hll_vec,
3 : register_int_counter_pair_vec, register_int_counter_vec, register_int_gauge,
4 : register_int_gauge_vec, Histogram, HistogramVec, HyperLogLogVec, IntCounterPairVec,
5 : IntCounterVec, IntGauge, IntGaugeVec,
6 : };
7 : use metrics::{register_int_counter_pair, IntCounterPair};
8 :
9 : use once_cell::sync::Lazy;
10 : use tokio::time;
11 :
12 0 : pub static NUM_DB_CONNECTIONS_GAUGE: Lazy<IntCounterPairVec> = Lazy::new(|| {
13 0 : register_int_counter_pair_vec!(
14 0 : "proxy_opened_db_connections_total",
15 0 : "Number of opened connections to a database.",
16 0 : "proxy_closed_db_connections_total",
17 0 : "Number of closed connections to a database.",
18 0 : &["protocol"],
19 0 : )
20 0 : .unwrap()
21 0 : });
22 :
23 0 : pub static NUM_CLIENT_CONNECTION_GAUGE: Lazy<IntCounterPairVec> = Lazy::new(|| {
24 0 : register_int_counter_pair_vec!(
25 0 : "proxy_opened_client_connections_total",
26 0 : "Number of opened connections from a client.",
27 0 : "proxy_closed_client_connections_total",
28 0 : "Number of closed connections from a client.",
29 0 : &["protocol"],
30 0 : )
31 0 : .unwrap()
32 0 : });
33 :
34 0 : pub static NUM_CONNECTION_REQUESTS_GAUGE: Lazy<IntCounterPairVec> = Lazy::new(|| {
35 0 : register_int_counter_pair_vec!(
36 0 : "proxy_accepted_connections_total",
37 0 : "Number of client connections accepted.",
38 0 : "proxy_closed_connections_total",
39 0 : "Number of client connections closed.",
40 0 : &["protocol"],
41 0 : )
42 0 : .unwrap()
43 0 : });
44 :
45 62 : pub static COMPUTE_CONNECTION_LATENCY: Lazy<HistogramVec> = Lazy::new(|| {
46 62 : register_histogram_vec!(
47 62 : "proxy_compute_connection_latency_seconds",
48 62 : "Time it took for proxy to establish a connection to the compute endpoint",
49 62 : // http/ws/tcp, true/false, true/false, success/failure
50 62 : // 3 * 2 * 2 * 2 = 24 counters
51 62 : &["protocol", "cache_miss", "pool_miss", "outcome"],
52 62 : // largest bucket = 2^16 * 0.5ms = 32s
53 62 : exponential_buckets(0.0005, 2.0, 16).unwrap(),
54 62 : )
55 62 : .unwrap()
56 62 : });
57 :
58 0 : pub static CONSOLE_REQUEST_LATENCY: Lazy<HistogramVec> = Lazy::new(|| {
59 0 : register_histogram_vec!(
60 0 : "proxy_console_request_latency",
61 0 : "Time it took for proxy to establish a connection to the compute endpoint",
62 0 : // proxy_wake_compute/proxy_get_role_info
63 0 : &["request"],
64 0 : // largest bucket = 2^16 * 0.2ms = 13s
65 0 : exponential_buckets(0.0002, 2.0, 16).unwrap(),
66 0 : )
67 0 : .unwrap()
68 0 : });
69 :
70 0 : pub static ALLOWED_IPS_BY_CACHE_OUTCOME: Lazy<IntCounterVec> = Lazy::new(|| {
71 0 : register_int_counter_vec!(
72 0 : "proxy_allowed_ips_cache_misses",
73 0 : "Number of cache hits/misses for allowed ips",
74 0 : // hit/miss
75 0 : &["outcome"],
76 0 : )
77 0 : .unwrap()
78 0 : });
79 :
80 2 : pub static RATE_LIMITER_ACQUIRE_LATENCY: Lazy<Histogram> = Lazy::new(|| {
81 2 : register_histogram!(
82 2 : "proxy_control_plane_token_acquire_seconds",
83 2 : "Time it took for proxy to establish a connection to the compute endpoint",
84 2 : // largest bucket = 3^16 * 0.05ms = 2.15s
85 2 : exponential_buckets(0.00005, 3.0, 16).unwrap(),
86 2 : )
87 2 : .unwrap()
88 2 : });
89 :
90 16 : pub static RATE_LIMITER_LIMIT: Lazy<IntGaugeVec> = Lazy::new(|| {
91 16 : register_int_gauge_vec!(
92 16 : "semaphore_control_plane_limit",
93 16 : "Current limit of the semaphore control plane",
94 16 : &["limit"], // 2 counters
95 16 : )
96 16 : .unwrap()
97 16 : });
98 :
99 20 : pub static NUM_CONNECTION_ACCEPTED_BY_SNI: Lazy<IntCounterVec> = Lazy::new(|| {
100 20 : register_int_counter_vec!(
101 20 : "proxy_accepted_connections_by_sni",
102 20 : "Number of connections (per sni).",
103 20 : &["kind"],
104 20 : )
105 20 : .unwrap()
106 20 : });
107 :
108 0 : pub static ALLOWED_IPS_NUMBER: Lazy<Histogram> = Lazy::new(|| {
109 0 : register_histogram!(
110 0 : "proxy_allowed_ips_number",
111 0 : "Number of allowed ips",
112 0 : vec![0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 10.0, 20.0, 50.0, 100.0],
113 0 : )
114 0 : .unwrap()
115 0 : });
116 :
117 0 : pub static HTTP_CONTENT_LENGTH: Lazy<Histogram> = Lazy::new(|| {
118 0 : register_histogram!(
119 0 : "proxy_http_conn_content_length_bytes",
120 0 : "Time it took for proxy to establish a connection to the compute endpoint",
121 0 : // largest bucket = 3^16 * 0.05ms = 2.15s
122 0 : exponential_buckets(8.0, 2.0, 20).unwrap()
123 0 : )
124 0 : .unwrap()
125 0 : });
126 :
127 2 : pub static GC_LATENCY: Lazy<Histogram> = Lazy::new(|| {
128 2 : register_histogram!(
129 2 : "proxy_http_pool_reclaimation_lag_seconds",
130 2 : "Time it takes to reclaim unused connection pools",
131 2 : // 1us -> 65ms
132 2 : exponential_buckets(1e-6, 2.0, 16).unwrap(),
133 2 : )
134 2 : .unwrap()
135 2 : });
136 :
137 2 : pub static ENDPOINT_POOLS: Lazy<IntCounterPair> = Lazy::new(|| {
138 4 : register_int_counter_pair!(
139 4 : "proxy_http_pool_endpoints_registered_total",
140 4 : "Number of endpoints we have registered pools for",
141 4 : "proxy_http_pool_endpoints_unregistered_total",
142 4 : "Number of endpoints we have unregistered pools for",
143 4 : )
144 2 : .unwrap()
145 2 : });
146 :
147 2 : pub static NUM_OPEN_CLIENTS_IN_HTTP_POOL: Lazy<IntGauge> = Lazy::new(|| {
148 2 : register_int_gauge!(
149 2 : "proxy_http_pool_opened_connections",
150 2 : "Number of opened connections to a database.",
151 2 : )
152 2 : .unwrap()
153 2 : });
154 :
155 0 : pub static NUM_CANCELLATION_REQUESTS: Lazy<IntCounterVec> = Lazy::new(|| {
156 0 : register_int_counter_vec!(
157 0 : "proxy_cancellation_requests_total",
158 0 : "Number of cancellation requests (per found/not_found).",
159 0 : &["source", "kind"],
160 0 : )
161 0 : .unwrap()
162 0 : });
163 :
164 0 : #[derive(Clone)]
165 : pub struct LatencyTimer {
166 : // time since the stopwatch was started
167 : start: Option<time::Instant>,
168 : // accumulated time on the stopwatch
169 : pub accumulated: std::time::Duration,
170 : // label data
171 : protocol: &'static str,
172 : cache_miss: bool,
173 : pool_miss: bool,
174 : outcome: &'static str,
175 : }
176 :
177 : pub struct LatencyTimerPause<'a> {
178 : timer: &'a mut LatencyTimer,
179 : }
180 :
181 : impl LatencyTimer {
182 64 : pub fn new(protocol: &'static str) -> Self {
183 64 : Self {
184 64 : start: Some(time::Instant::now()),
185 64 : accumulated: std::time::Duration::ZERO,
186 64 : protocol,
187 64 : cache_miss: false,
188 64 : // by default we don't do pooling
189 64 : pool_miss: true,
190 64 : // assume failed unless otherwise specified
191 64 : outcome: "failed",
192 64 : }
193 64 : }
194 :
195 24 : pub fn pause(&mut self) -> LatencyTimerPause<'_> {
196 24 : // stop the stopwatch and record the time that we have accumulated
197 24 : let start = self.start.take().expect("latency timer should be started");
198 24 : self.accumulated += start.elapsed();
199 24 : LatencyTimerPause { timer: self }
200 24 : }
201 :
202 8 : pub fn cache_miss(&mut self) {
203 8 : self.cache_miss = true;
204 8 : }
205 :
206 0 : pub fn pool_hit(&mut self) {
207 0 : self.pool_miss = false;
208 0 : }
209 :
210 8 : pub fn success(&mut self) {
211 : // stop the stopwatch and record the time that we have accumulated
212 8 : if let Some(start) = self.start.take() {
213 8 : self.accumulated += start.elapsed();
214 8 : }
215 :
216 : // success
217 8 : self.outcome = "success";
218 8 : }
219 : }
220 :
221 : impl Drop for LatencyTimerPause<'_> {
222 24 : fn drop(&mut self) {
223 24 : // start the stopwatch again
224 24 : self.timer.start = Some(time::Instant::now());
225 24 : }
226 : }
227 :
228 : impl Drop for LatencyTimer {
229 64 : fn drop(&mut self) {
230 64 : let duration =
231 64 : self.start.map(|start| start.elapsed()).unwrap_or_default() + self.accumulated;
232 64 : COMPUTE_CONNECTION_LATENCY
233 64 : .with_label_values(&[
234 64 : self.protocol,
235 64 : bool_to_str(self.cache_miss),
236 64 : bool_to_str(self.pool_miss),
237 64 : self.outcome,
238 64 : ])
239 64 : .observe(duration.as_secs_f64())
240 64 : }
241 : }
242 :
243 8 : pub static NUM_CONNECTION_FAILURES: Lazy<IntCounterVec> = Lazy::new(|| {
244 8 : register_int_counter_vec!(
245 8 : "proxy_connection_failures_total",
246 8 : "Number of connection failures (per kind).",
247 8 : &["kind"],
248 8 : )
249 8 : .unwrap()
250 8 : });
251 :
252 4 : pub static NUM_WAKEUP_FAILURES: Lazy<IntCounterVec> = Lazy::new(|| {
253 4 : register_int_counter_vec!(
254 4 : "proxy_connection_failures_breakdown",
255 4 : "Number of wake-up failures (per kind).",
256 4 : &["retry", "kind"],
257 4 : )
258 4 : .unwrap()
259 4 : });
260 :
261 0 : pub static NUM_BYTES_PROXIED_COUNTER: Lazy<IntCounterVec> = Lazy::new(|| {
262 0 : register_int_counter_vec!(
263 0 : "proxy_io_bytes",
264 0 : "Number of bytes sent/received between all clients and backends.",
265 0 : &["direction"],
266 0 : )
267 0 : .unwrap()
268 0 : });
269 :
270 134 : pub const fn bool_to_str(x: bool) -> &'static str {
271 134 : if x {
272 76 : "true"
273 : } else {
274 58 : "false"
275 : }
276 134 : }
277 :
278 12 : pub static CONNECTING_ENDPOINTS: Lazy<HyperLogLogVec<32>> = Lazy::new(|| {
279 12 : register_hll_vec!(
280 12 : 32,
281 12 : "proxy_connecting_endpoints",
282 12 : "HLL approximate cardinality of endpoints that are connecting",
283 12 : &["protocol"],
284 12 : )
285 12 : .unwrap()
286 12 : });
287 :
288 0 : pub static ERROR_BY_KIND: Lazy<IntCounterVec> = Lazy::new(|| {
289 0 : register_int_counter_vec!(
290 0 : "proxy_errors_total",
291 0 : "Number of errors by a given classification",
292 0 : &["type"],
293 0 : )
294 0 : .unwrap()
295 0 : });
296 :
297 0 : pub static ENDPOINT_ERRORS_BY_KIND: Lazy<HyperLogLogVec<32>> = Lazy::new(|| {
298 0 : register_hll_vec!(
299 0 : 32,
300 0 : "proxy_endpoints_affected_by_errors",
301 0 : "Number of endpoints affected by errors of a given classification",
302 0 : &["type"],
303 0 : )
304 0 : .unwrap()
305 0 : });
|
