Line data Source code
1 : use super::*;
2 : use std::collections::HashMap;
3 : use std::time::SystemTime;
4 : use utils::lsn::Lsn;
5 :
6 2 : #[test]
7 2 : fn startup_collected_timeline_metrics_before_advancing() {
8 2 : let tenant_id = TenantId::generate();
9 2 : let timeline_id = TimelineId::generate();
10 2 :
11 2 : let mut metrics = Vec::new();
12 2 : let cache = HashMap::new();
13 2 :
14 2 : let initdb_lsn = Lsn(0x10000);
15 2 : let disk_consistent_lsn = Lsn(initdb_lsn.0 * 2);
16 2 :
17 2 : let snap = TimelineSnapshot {
18 2 : loaded_at: (disk_consistent_lsn, SystemTime::now()),
19 2 : last_record_lsn: disk_consistent_lsn,
20 2 : current_exact_logical_size: Some(0x42000),
21 2 : };
22 2 :
23 2 : let now = DateTime::<Utc>::from(SystemTime::now());
24 2 :
25 2 : snap.to_metrics(tenant_id, timeline_id, now, &mut metrics, &cache);
26 2 :
27 2 : assert_eq!(
28 2 : metrics,
29 2 : &[
30 2 : MetricsKey::written_size_delta(tenant_id, timeline_id).from_until(
31 2 : snap.loaded_at.1.into(),
32 2 : now,
33 2 : 0
34 2 : ),
35 2 : MetricsKey::written_size(tenant_id, timeline_id).at(now, disk_consistent_lsn.0),
36 2 : MetricsKey::timeline_logical_size(tenant_id, timeline_id).at(now, 0x42000)
37 2 : ]
38 2 : );
39 2 : }
40 :
41 2 : #[test]
42 2 : fn startup_collected_timeline_metrics_second_round() {
43 2 : let tenant_id = TenantId::generate();
44 2 : let timeline_id = TimelineId::generate();
45 2 :
46 2 : let [now, before, init] = time_backwards();
47 2 :
48 2 : let now = DateTime::<Utc>::from(now);
49 2 : let before = DateTime::<Utc>::from(before);
50 2 :
51 2 : let initdb_lsn = Lsn(0x10000);
52 2 : let disk_consistent_lsn = Lsn(initdb_lsn.0 * 2);
53 2 :
54 2 : let mut metrics = Vec::new();
55 2 : let cache = HashMap::from([
56 2 : MetricsKey::written_size(tenant_id, timeline_id).at(before, disk_consistent_lsn.0)
57 2 : ]);
58 2 :
59 2 : let snap = TimelineSnapshot {
60 2 : loaded_at: (disk_consistent_lsn, init),
61 2 : last_record_lsn: disk_consistent_lsn,
62 2 : current_exact_logical_size: Some(0x42000),
63 2 : };
64 2 :
65 2 : snap.to_metrics(tenant_id, timeline_id, now, &mut metrics, &cache);
66 2 :
67 2 : assert_eq!(
68 2 : metrics,
69 2 : &[
70 2 : MetricsKey::written_size_delta(tenant_id, timeline_id).from_until(before, now, 0),
71 2 : MetricsKey::written_size(tenant_id, timeline_id).at(now, disk_consistent_lsn.0),
72 2 : MetricsKey::timeline_logical_size(tenant_id, timeline_id).at(now, 0x42000)
73 2 : ]
74 2 : );
75 2 : }
76 :
77 2 : #[test]
78 2 : fn startup_collected_timeline_metrics_nth_round_at_same_lsn() {
79 2 : let tenant_id = TenantId::generate();
80 2 : let timeline_id = TimelineId::generate();
81 2 :
82 2 : let [now, just_before, before, init] = time_backwards();
83 2 :
84 2 : let now = DateTime::<Utc>::from(now);
85 2 : let just_before = DateTime::<Utc>::from(just_before);
86 2 : let before = DateTime::<Utc>::from(before);
87 2 :
88 2 : let initdb_lsn = Lsn(0x10000);
89 2 : let disk_consistent_lsn = Lsn(initdb_lsn.0 * 2);
90 2 :
91 2 : let mut metrics = Vec::new();
92 2 : let cache = HashMap::from([
93 2 : // at t=before was the last time the last_record_lsn changed
94 2 : MetricsKey::written_size(tenant_id, timeline_id).at(before, disk_consistent_lsn.0),
95 2 : // end time of this event is used for the next ones
96 2 : MetricsKey::written_size_delta(tenant_id, timeline_id).from_until(before, just_before, 0),
97 2 : ]);
98 2 :
99 2 : let snap = TimelineSnapshot {
100 2 : loaded_at: (disk_consistent_lsn, init),
101 2 : last_record_lsn: disk_consistent_lsn,
102 2 : current_exact_logical_size: Some(0x42000),
103 2 : };
104 2 :
105 2 : snap.to_metrics(tenant_id, timeline_id, now, &mut metrics, &cache);
106 2 :
107 2 : assert_eq!(
108 2 : metrics,
109 2 : &[
110 2 : MetricsKey::written_size_delta(tenant_id, timeline_id).from_until(just_before, now, 0),
111 2 : MetricsKey::written_size(tenant_id, timeline_id).at(now, disk_consistent_lsn.0),
112 2 : MetricsKey::timeline_logical_size(tenant_id, timeline_id).at(now, 0x42000)
113 2 : ]
114 2 : );
115 2 : }
116 :
117 2 : #[test]
118 2 : fn post_restart_written_sizes_with_rolled_back_last_record_lsn() {
119 2 : // it can happen that we lose the inmemorylayer but have previously sent metrics and we
120 2 : // should never go backwards
121 2 :
122 2 : let tenant_id = TenantId::generate();
123 2 : let timeline_id = TimelineId::generate();
124 2 :
125 2 : let [later, now, at_restart] = time_backwards();
126 2 :
127 2 : // FIXME: tests would be so much easier if we did not need to juggle back and forth
128 2 : // SystemTime and DateTime::<Utc> ... Could do the conversion only at upload time?
129 2 : let now = DateTime::<Utc>::from(now);
130 2 : let later = DateTime::<Utc>::from(later);
131 2 : let before_restart = at_restart - std::time::Duration::from_secs(5 * 60);
132 2 : let way_before = before_restart - std::time::Duration::from_secs(10 * 60);
133 2 : let before_restart = DateTime::<Utc>::from(before_restart);
134 2 : let way_before = DateTime::<Utc>::from(way_before);
135 2 :
136 2 : let snap = TimelineSnapshot {
137 2 : loaded_at: (Lsn(50), at_restart),
138 2 : last_record_lsn: Lsn(50),
139 2 : current_exact_logical_size: None,
140 2 : };
141 2 :
142 2 : let mut cache = HashMap::from([
143 2 : MetricsKey::written_size(tenant_id, timeline_id).at(before_restart, 100),
144 2 : MetricsKey::written_size_delta(tenant_id, timeline_id).from_until(
145 2 : way_before,
146 2 : before_restart,
147 2 : // not taken into account, but the timestamps are important
148 2 : 999_999_999,
149 2 : ),
150 2 : ]);
151 2 :
152 2 : let mut metrics = Vec::new();
153 2 : snap.to_metrics(tenant_id, timeline_id, now, &mut metrics, &cache);
154 2 :
155 2 : assert_eq!(
156 2 : metrics,
157 2 : &[
158 2 : MetricsKey::written_size_delta(tenant_id, timeline_id).from_until(
159 2 : before_restart,
160 2 : now,
161 2 : 0
162 2 : ),
163 2 : MetricsKey::written_size(tenant_id, timeline_id).at(now, 100),
164 2 : ]
165 2 : );
166 :
167 : // now if we cache these metrics, and re-run while "still in recovery"
168 2 : cache.extend(metrics.drain(..));
169 2 :
170 2 : // "still in recovery", because our snapshot did not change
171 2 : snap.to_metrics(tenant_id, timeline_id, later, &mut metrics, &cache);
172 2 :
173 2 : assert_eq!(
174 2 : metrics,
175 2 : &[
176 2 : MetricsKey::written_size_delta(tenant_id, timeline_id).from_until(now, later, 0),
177 2 : MetricsKey::written_size(tenant_id, timeline_id).at(later, 100),
178 2 : ]
179 2 : );
180 2 : }
181 :
182 2 : #[test]
183 2 : fn post_restart_current_exact_logical_size_uses_cached() {
184 2 : let tenant_id = TenantId::generate();
185 2 : let timeline_id = TimelineId::generate();
186 2 :
187 2 : let [now, at_restart] = time_backwards();
188 2 :
189 2 : let now = DateTime::<Utc>::from(now);
190 2 : let before_restart = at_restart - std::time::Duration::from_secs(5 * 60);
191 2 : let before_restart = DateTime::<Utc>::from(before_restart);
192 2 :
193 2 : let snap = TimelineSnapshot {
194 2 : loaded_at: (Lsn(50), at_restart),
195 2 : last_record_lsn: Lsn(50),
196 2 : current_exact_logical_size: None,
197 2 : };
198 2 :
199 2 : let cache = HashMap::from([
200 2 : MetricsKey::timeline_logical_size(tenant_id, timeline_id).at(before_restart, 100)
201 2 : ]);
202 2 :
203 2 : let mut metrics = Vec::new();
204 2 : snap.to_metrics(tenant_id, timeline_id, now, &mut metrics, &cache);
205 2 :
206 6 : metrics.retain(|(key, _)| key.metric == Name::LogicalSize);
207 2 :
208 2 : assert_eq!(
209 2 : metrics,
210 2 : &[MetricsKey::timeline_logical_size(tenant_id, timeline_id).at(now, 100)]
211 2 : );
212 2 : }
213 :
214 2 : #[test]
215 2 : fn post_restart_synthetic_size_uses_cached_if_available() {
216 2 : let tenant_id = TenantId::generate();
217 2 :
218 2 : let ts = TenantSnapshot {
219 2 : resident_size: 1000,
220 2 : remote_size: 1000,
221 2 : // not yet calculated
222 2 : synthetic_size: 0,
223 2 : };
224 2 :
225 2 : let now = SystemTime::now();
226 2 : let before_restart = DateTime::<Utc>::from(now - std::time::Duration::from_secs(5 * 60));
227 2 : let now = DateTime::<Utc>::from(now);
228 2 :
229 2 : let cached = HashMap::from([MetricsKey::synthetic_size(tenant_id).at(before_restart, 1000)]);
230 2 :
231 2 : let mut metrics = Vec::new();
232 2 : ts.to_metrics(tenant_id, now, &cached, &mut metrics);
233 2 :
234 2 : assert_eq!(
235 2 : metrics,
236 2 : &[
237 2 : MetricsKey::remote_storage_size(tenant_id).at(now, 1000),
238 2 : MetricsKey::resident_size(tenant_id).at(now, 1000),
239 2 : MetricsKey::synthetic_size(tenant_id).at(now, 1000),
240 2 : ]
241 2 : );
242 2 : }
243 :
244 2 : #[test]
245 2 : fn post_restart_synthetic_size_is_not_sent_when_not_cached() {
246 2 : let tenant_id = TenantId::generate();
247 2 :
248 2 : let ts = TenantSnapshot {
249 2 : resident_size: 1000,
250 2 : remote_size: 1000,
251 2 : // not yet calculated
252 2 : synthetic_size: 0,
253 2 : };
254 2 :
255 2 : let now = SystemTime::now();
256 2 : let now = DateTime::<Utc>::from(now);
257 2 :
258 2 : let cached = HashMap::new();
259 2 :
260 2 : let mut metrics = Vec::new();
261 2 : ts.to_metrics(tenant_id, now, &cached, &mut metrics);
262 2 :
263 2 : assert_eq!(
264 2 : metrics,
265 2 : &[
266 2 : MetricsKey::remote_storage_size(tenant_id).at(now, 1000),
267 2 : MetricsKey::resident_size(tenant_id).at(now, 1000),
268 2 : // no synthetic size here
269 2 : ]
270 2 : );
271 2 : }
272 :
273 8 : fn time_backwards<const N: usize>() -> [std::time::SystemTime; N] {
274 8 : let mut times = [std::time::SystemTime::UNIX_EPOCH; N];
275 8 : times[0] = std::time::SystemTime::now();
276 24 : for behind in 1..N {
277 16 : times[behind] = times[0] - std::time::Duration::from_secs(behind as u64);
278 16 : }
279 :
280 8 : times
281 8 : }
282 :
283 4 : pub(crate) const fn metric_examples(
284 4 : tenant_id: TenantId,
285 4 : timeline_id: TimelineId,
286 4 : now: DateTime<Utc>,
287 4 : before: DateTime<Utc>,
288 4 : ) -> [RawMetric; 6] {
289 4 : [
290 4 : MetricsKey::written_size(tenant_id, timeline_id).at(now, 0),
291 4 : MetricsKey::written_size_delta(tenant_id, timeline_id).from_until(before, now, 0),
292 4 : MetricsKey::timeline_logical_size(tenant_id, timeline_id).at(now, 0),
293 4 : MetricsKey::remote_storage_size(tenant_id).at(now, 0),
294 4 : MetricsKey::resident_size(tenant_id).at(now, 0),
295 4 : MetricsKey::synthetic_size(tenant_id).at(now, 1),
296 4 : ]
297 4 : }
|
