TLA Line data Source code
1 : use super::*;
2 : use std::collections::HashMap;
3 : use std::time::SystemTime;
4 : use utils::lsn::Lsn;
5 :
6 CBC 1 : #[test]
7 1 : fn startup_collected_timeline_metrics_before_advancing() {
8 1 : let tenant_id = TenantId::generate();
9 1 : let timeline_id = TimelineId::generate();
10 1 :
11 1 : let mut metrics = Vec::new();
12 1 : let cache = HashMap::new();
13 1 :
14 1 : let initdb_lsn = Lsn(0x10000);
15 1 : let disk_consistent_lsn = Lsn(initdb_lsn.0 * 2);
16 1 :
17 1 : let snap = TimelineSnapshot {
18 1 : loaded_at: (disk_consistent_lsn, SystemTime::now()),
19 1 : last_record_lsn: disk_consistent_lsn,
20 1 : current_exact_logical_size: Some(0x42000),
21 1 : };
22 1 :
23 1 : let now = DateTime::<Utc>::from(SystemTime::now());
24 1 :
25 1 : snap.to_metrics(tenant_id, timeline_id, now, &mut metrics, &cache);
26 1 :
27 1 : assert_eq!(
28 1 : metrics,
29 1 : &[
30 1 : MetricsKey::written_size_delta(tenant_id, timeline_id).from_until(
31 1 : snap.loaded_at.1.into(),
32 1 : now,
33 1 : 0
34 1 : ),
35 1 : MetricsKey::written_size(tenant_id, timeline_id).at(now, disk_consistent_lsn.0),
36 1 : MetricsKey::timeline_logical_size(tenant_id, timeline_id).at(now, 0x42000)
37 1 : ]
38 1 : );
39 1 : }
40 :
41 1 : #[test]
42 1 : fn startup_collected_timeline_metrics_second_round() {
43 1 : let tenant_id = TenantId::generate();
44 1 : let timeline_id = TimelineId::generate();
45 1 :
46 1 : let [now, before, init] = time_backwards();
47 1 :
48 1 : let now = DateTime::<Utc>::from(now);
49 1 : let before = DateTime::<Utc>::from(before);
50 1 :
51 1 : let initdb_lsn = Lsn(0x10000);
52 1 : let disk_consistent_lsn = Lsn(initdb_lsn.0 * 2);
53 1 :
54 1 : let mut metrics = Vec::new();
55 1 : let cache = HashMap::from([
56 1 : MetricsKey::written_size(tenant_id, timeline_id).at(before, disk_consistent_lsn.0)
57 1 : ]);
58 1 :
59 1 : let snap = TimelineSnapshot {
60 1 : loaded_at: (disk_consistent_lsn, init),
61 1 : last_record_lsn: disk_consistent_lsn,
62 1 : current_exact_logical_size: Some(0x42000),
63 1 : };
64 1 :
65 1 : snap.to_metrics(tenant_id, timeline_id, now, &mut metrics, &cache);
66 1 :
67 1 : assert_eq!(
68 1 : metrics,
69 1 : &[
70 1 : MetricsKey::written_size_delta(tenant_id, timeline_id).from_until(before, now, 0),
71 1 : MetricsKey::written_size(tenant_id, timeline_id).at(now, disk_consistent_lsn.0),
72 1 : MetricsKey::timeline_logical_size(tenant_id, timeline_id).at(now, 0x42000)
73 1 : ]
74 1 : );
75 1 : }
76 :
77 1 : #[test]
78 1 : fn startup_collected_timeline_metrics_nth_round_at_same_lsn() {
79 1 : let tenant_id = TenantId::generate();
80 1 : let timeline_id = TimelineId::generate();
81 1 :
82 1 : let [now, just_before, before, init] = time_backwards();
83 1 :
84 1 : let now = DateTime::<Utc>::from(now);
85 1 : let just_before = DateTime::<Utc>::from(just_before);
86 1 : let before = DateTime::<Utc>::from(before);
87 1 :
88 1 : let initdb_lsn = Lsn(0x10000);
89 1 : let disk_consistent_lsn = Lsn(initdb_lsn.0 * 2);
90 1 :
91 1 : let mut metrics = Vec::new();
92 1 : let cache = HashMap::from([
93 1 : // at t=before was the last time the last_record_lsn changed
94 1 : MetricsKey::written_size(tenant_id, timeline_id).at(before, disk_consistent_lsn.0),
95 1 : // end time of this event is used for the next ones
96 1 : MetricsKey::written_size_delta(tenant_id, timeline_id).from_until(before, just_before, 0),
97 1 : ]);
98 1 :
99 1 : let snap = TimelineSnapshot {
100 1 : loaded_at: (disk_consistent_lsn, init),
101 1 : last_record_lsn: disk_consistent_lsn,
102 1 : current_exact_logical_size: Some(0x42000),
103 1 : };
104 1 :
105 1 : snap.to_metrics(tenant_id, timeline_id, now, &mut metrics, &cache);
106 1 :
107 1 : assert_eq!(
108 1 : metrics,
109 1 : &[
110 1 : MetricsKey::written_size_delta(tenant_id, timeline_id).from_until(just_before, now, 0),
111 1 : MetricsKey::written_size(tenant_id, timeline_id).at(now, disk_consistent_lsn.0),
112 1 : MetricsKey::timeline_logical_size(tenant_id, timeline_id).at(now, 0x42000)
113 1 : ]
114 1 : );
115 1 : }
116 :
117 1 : #[test]
118 1 : fn post_restart_written_sizes_with_rolled_back_last_record_lsn() {
119 1 : // it can happen that we lose the inmemorylayer but have previously sent metrics and we
120 1 : // should never go backwards
121 1 :
122 1 : let tenant_id = TenantId::generate();
123 1 : let timeline_id = TimelineId::generate();
124 1 :
125 1 : let [later, now, at_restart] = time_backwards();
126 1 :
127 1 : // FIXME: tests would be so much easier if we did not need to juggle back and forth
128 1 : // SystemTime and DateTime::<Utc> ... Could do the conversion only at upload time?
129 1 : let now = DateTime::<Utc>::from(now);
130 1 : let later = DateTime::<Utc>::from(later);
131 1 : let before_restart = at_restart - std::time::Duration::from_secs(5 * 60);
132 1 : let way_before = before_restart - std::time::Duration::from_secs(10 * 60);
133 1 : let before_restart = DateTime::<Utc>::from(before_restart);
134 1 : let way_before = DateTime::<Utc>::from(way_before);
135 1 :
136 1 : let snap = TimelineSnapshot {
137 1 : loaded_at: (Lsn(50), at_restart),
138 1 : last_record_lsn: Lsn(50),
139 1 : current_exact_logical_size: None,
140 1 : };
141 1 :
142 1 : let mut cache = HashMap::from([
143 1 : MetricsKey::written_size(tenant_id, timeline_id).at(before_restart, 100),
144 1 : MetricsKey::written_size_delta(tenant_id, timeline_id).from_until(
145 1 : way_before,
146 1 : before_restart,
147 1 : // not taken into account, but the timestamps are important
148 1 : 999_999_999,
149 1 : ),
150 1 : ]);
151 1 :
152 1 : let mut metrics = Vec::new();
153 1 : snap.to_metrics(tenant_id, timeline_id, now, &mut metrics, &cache);
154 1 :
155 1 : assert_eq!(
156 1 : metrics,
157 1 : &[
158 1 : MetricsKey::written_size_delta(tenant_id, timeline_id).from_until(
159 1 : before_restart,
160 1 : now,
161 1 : 0
162 1 : ),
163 1 : MetricsKey::written_size(tenant_id, timeline_id).at(now, 100),
164 1 : ]
165 1 : );
166 :
167 : // now if we cache these metrics, and re-run while "still in recovery"
168 1 : cache.extend(metrics.drain(..));
169 1 :
170 1 : // "still in recovery", because our snapshot did not change
171 1 : snap.to_metrics(tenant_id, timeline_id, later, &mut metrics, &cache);
172 1 :
173 1 : assert_eq!(
174 1 : metrics,
175 1 : &[
176 1 : MetricsKey::written_size_delta(tenant_id, timeline_id).from_until(now, later, 0),
177 1 : MetricsKey::written_size(tenant_id, timeline_id).at(later, 100),
178 1 : ]
179 1 : );
180 1 : }
181 :
182 1 : #[test]
183 1 : fn post_restart_current_exact_logical_size_uses_cached() {
184 1 : let tenant_id = TenantId::generate();
185 1 : let timeline_id = TimelineId::generate();
186 1 :
187 1 : let [now, at_restart] = time_backwards();
188 1 :
189 1 : let now = DateTime::<Utc>::from(now);
190 1 : let before_restart = at_restart - std::time::Duration::from_secs(5 * 60);
191 1 : let before_restart = DateTime::<Utc>::from(before_restart);
192 1 :
193 1 : let snap = TimelineSnapshot {
194 1 : loaded_at: (Lsn(50), at_restart),
195 1 : last_record_lsn: Lsn(50),
196 1 : current_exact_logical_size: None,
197 1 : };
198 1 :
199 1 : let cache = HashMap::from([
200 1 : MetricsKey::timeline_logical_size(tenant_id, timeline_id).at(before_restart, 100)
201 1 : ]);
202 1 :
203 1 : let mut metrics = Vec::new();
204 1 : snap.to_metrics(tenant_id, timeline_id, now, &mut metrics, &cache);
205 1 :
206 3 : metrics.retain(|(key, _)| key.metric == Name::LogicalSize);
207 1 :
208 1 : assert_eq!(
209 1 : metrics,
210 1 : &[MetricsKey::timeline_logical_size(tenant_id, timeline_id).at(now, 100)]
211 1 : );
212 1 : }
213 :
214 1 : #[test]
215 1 : fn post_restart_synthetic_size_uses_cached_if_available() {
216 1 : let tenant_id = TenantId::generate();
217 1 :
218 1 : let ts = TenantSnapshot {
219 1 : resident_size: 1000,
220 1 : remote_size: 1000,
221 1 : // not yet calculated
222 1 : synthetic_size: 0,
223 1 : };
224 1 :
225 1 : let now = SystemTime::now();
226 1 : let before_restart = DateTime::<Utc>::from(now - std::time::Duration::from_secs(5 * 60));
227 1 : let now = DateTime::<Utc>::from(now);
228 1 :
229 1 : let cached = HashMap::from([MetricsKey::synthetic_size(tenant_id).at(before_restart, 1000)]);
230 1 :
231 1 : let mut metrics = Vec::new();
232 1 : ts.to_metrics(tenant_id, now, &cached, &mut metrics);
233 1 :
234 1 : assert_eq!(
235 1 : metrics,
236 1 : &[
237 1 : MetricsKey::remote_storage_size(tenant_id).at(now, 1000),
238 1 : MetricsKey::resident_size(tenant_id).at(now, 1000),
239 1 : MetricsKey::synthetic_size(tenant_id).at(now, 1000),
240 1 : ]
241 1 : );
242 1 : }
243 :
244 1 : #[test]
245 1 : fn post_restart_synthetic_size_is_not_sent_when_not_cached() {
246 1 : let tenant_id = TenantId::generate();
247 1 :
248 1 : let ts = TenantSnapshot {
249 1 : resident_size: 1000,
250 1 : remote_size: 1000,
251 1 : // not yet calculated
252 1 : synthetic_size: 0,
253 1 : };
254 1 :
255 1 : let now = SystemTime::now();
256 1 : let now = DateTime::<Utc>::from(now);
257 1 :
258 1 : let cached = HashMap::new();
259 1 :
260 1 : let mut metrics = Vec::new();
261 1 : ts.to_metrics(tenant_id, now, &cached, &mut metrics);
262 1 :
263 1 : assert_eq!(
264 1 : metrics,
265 1 : &[
266 1 : MetricsKey::remote_storage_size(tenant_id).at(now, 1000),
267 1 : MetricsKey::resident_size(tenant_id).at(now, 1000),
268 1 : // no synthetic size here
269 1 : ]
270 1 : );
271 1 : }
272 :
273 4 : fn time_backwards<const N: usize>() -> [std::time::SystemTime; N] {
274 4 : let mut times = [std::time::SystemTime::UNIX_EPOCH; N];
275 4 : times[0] = std::time::SystemTime::now();
276 12 : for behind in 1..N {
277 8 : times[behind] = times[0] - std::time::Duration::from_secs(behind as u64);
278 8 : }
279 :
280 4 : times
281 4 : }
282 :
283 2 : pub(crate) const fn metric_examples(
284 2 : tenant_id: TenantId,
285 2 : timeline_id: TimelineId,
286 2 : now: DateTime<Utc>,
287 2 : before: DateTime<Utc>,
288 2 : ) -> [RawMetric; 6] {
289 2 : [
290 2 : MetricsKey::written_size(tenant_id, timeline_id).at(now, 0),
291 2 : MetricsKey::written_size_delta(tenant_id, timeline_id).from_until(before, now, 0),
292 2 : MetricsKey::timeline_logical_size(tenant_id, timeline_id).at(now, 0),
293 2 : MetricsKey::remote_storage_size(tenant_id).at(now, 0),
294 2 : MetricsKey::resident_size(tenant_id).at(now, 0),
295 2 : MetricsKey::synthetic_size(tenant_id).at(now, 1),
296 2 : ]
297 2 : }
|
