Line data Source code
1 : use std::collections::BTreeMap;
2 : use std::ops::Range;
3 :
4 : use tracing::info;
5 :
6 : use crate::tenant::storage_layer::PersistentLayerDesc;
7 :
8 : use super::layer_coverage::LayerCoverageTuple;
9 :
10 : /// Layers in this module are identified and indexed by this data.
11 : ///
12 : /// This is a helper struct to enable sorting layers by lsn.start.
13 : ///
14 : /// These three values are enough to uniquely identify a layer, since
15 : /// a layer is obligated to contain all contents within range, so two
16 : /// deltas (or images) with the same range have identical content.
17 : #[derive(Debug, PartialEq, Eq, Clone)]
18 : pub struct LayerKey {
19 : // TODO I use i128 and u64 because it was easy for prototyping,
20 : // testing, and benchmarking. If we can use the Lsn and Key
21 : // types without overhead that would be preferable.
22 : pub key: Range<i128>,
23 : pub lsn: Range<u64>,
24 : pub is_image: bool,
25 : }
26 :
27 : impl PartialOrd for LayerKey {
28 0 : fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
29 0 : Some(self.cmp(other))
30 0 : }
31 : }
32 :
33 : impl Ord for LayerKey {
34 30441 : fn cmp(&self, other: &Self) -> std::cmp::Ordering {
35 30441 : // NOTE we really care about comparing by lsn.start first
36 30441 : self.lsn
37 30441 : .start
38 30441 : .cmp(&other.lsn.start)
39 30441 : .then(self.lsn.end.cmp(&other.lsn.end))
40 30441 : .then(self.key.start.cmp(&other.key.start))
41 30441 : .then(self.key.end.cmp(&other.key.end))
42 30441 : .then(self.is_image.cmp(&other.is_image))
43 30441 : }
44 : }
45 :
46 : impl From<&PersistentLayerDesc> for LayerKey {
47 1998 : fn from(layer: &PersistentLayerDesc) -> Self {
48 1998 : let kr = layer.get_key_range();
49 1998 : let lr = layer.get_lsn_range();
50 1998 : LayerKey {
51 1998 : key: kr.start.to_i128()..kr.end.to_i128(),
52 1998 : lsn: lr.start.0..lr.end.0,
53 1998 : is_image: !layer.is_incremental(),
54 1998 : }
55 1998 : }
56 : }
57 :
58 : /// Efficiently queryable layer coverage for each LSN.
59 : ///
60 : /// Allows answering layer map queries very efficiently,
61 : /// but doesn't allow retroactive insertion, which is
62 : /// sometimes necessary. See BufferedHistoricLayerCoverage.
63 : pub struct HistoricLayerCoverage<Value> {
64 : /// The latest state
65 : head: LayerCoverageTuple<Value>,
66 :
67 : /// All previous states
68 : historic: BTreeMap<u64, LayerCoverageTuple<Value>>,
69 : }
70 :
71 : impl<T: Clone> Default for HistoricLayerCoverage<T> {
72 0 : fn default() -> Self {
73 0 : Self::new()
74 0 : }
75 : }
76 :
77 : impl<Value: Clone> HistoricLayerCoverage<Value> {
78 399 : pub fn new() -> Self {
79 399 : Self {
80 399 : head: LayerCoverageTuple::default(),
81 399 : historic: BTreeMap::default(),
82 399 : }
83 399 : }
84 :
85 : /// Add a layer
86 : ///
87 : /// Panics if new layer has older lsn.start than an existing layer.
88 : /// See BufferedHistoricLayerCoverage for a more general insertion method.
89 1712 : pub fn insert(&mut self, layer_key: LayerKey, value: Value) {
90 : // It's only a persistent map, not a retroactive one
91 1712 : if let Some(last_entry) = self.historic.iter().next_back() {
92 1402 : let last_lsn = last_entry.0;
93 1402 : if layer_key.lsn.start < *last_lsn {
94 0 : panic!("unexpected retroactive insert");
95 1402 : }
96 310 : }
97 :
98 : // Insert into data structure
99 1712 : let target = if layer_key.is_image {
100 306 : &mut self.head.image_coverage
101 : } else {
102 1406 : &mut self.head.delta_coverage
103 : };
104 :
105 1712 : target.insert(layer_key.key, layer_key.lsn.clone(), value);
106 1712 :
107 1712 : // Remember history. Clone is O(1)
108 1712 : self.historic.insert(layer_key.lsn.start, self.head.clone());
109 1712 : }
110 :
111 : /// Query at a particular LSN, inclusive
112 370441 : pub fn get_version(&self, lsn: u64) -> Option<&LayerCoverageTuple<Value>> {
113 370441 : match self.historic.range(..=lsn).next_back() {
114 370283 : Some((_, v)) => Some(v),
115 158 : None => None,
116 : }
117 370441 : }
118 :
119 : /// Remove all entries after a certain LSN (inclusive)
120 1254 : pub fn trim(&mut self, begin: &u64) {
121 1254 : self.historic.split_off(begin);
122 1254 : self.head = self
123 1254 : .historic
124 1254 : .iter()
125 1254 : .next_back()
126 1254 : .map(|(_, v)| v.clone())
127 1254 : .unwrap_or_default();
128 1254 : }
129 : }
130 :
131 : /// This is the most basic test that demonstrates intended usage.
132 : /// All layers in this test have height 1.
133 : #[test]
134 2 : fn test_persistent_simple() {
135 2 : let mut map = HistoricLayerCoverage::<String>::new();
136 2 : map.insert(
137 2 : LayerKey {
138 2 : key: 0..5,
139 2 : lsn: 100..101,
140 2 : is_image: true,
141 2 : },
142 2 : "Layer 1".to_string(),
143 2 : );
144 2 : map.insert(
145 2 : LayerKey {
146 2 : key: 3..9,
147 2 : lsn: 110..111,
148 2 : is_image: true,
149 2 : },
150 2 : "Layer 2".to_string(),
151 2 : );
152 2 : map.insert(
153 2 : LayerKey {
154 2 : key: 5..6,
155 2 : lsn: 120..121,
156 2 : is_image: true,
157 2 : },
158 2 : "Layer 3".to_string(),
159 2 : );
160 2 :
161 2 : // After Layer 1 insertion
162 2 : let version = map.get_version(105).unwrap();
163 2 : assert_eq!(version.image_coverage.query(1), Some("Layer 1".to_string()));
164 2 : assert_eq!(version.image_coverage.query(4), Some("Layer 1".to_string()));
165 :
166 : // After Layer 2 insertion
167 2 : let version = map.get_version(115).unwrap();
168 2 : assert_eq!(version.image_coverage.query(4), Some("Layer 2".to_string()));
169 2 : assert_eq!(version.image_coverage.query(8), Some("Layer 2".to_string()));
170 2 : assert_eq!(version.image_coverage.query(11), None);
171 :
172 : // After Layer 3 insertion
173 2 : let version = map.get_version(125).unwrap();
174 2 : assert_eq!(version.image_coverage.query(4), Some("Layer 2".to_string()));
175 2 : assert_eq!(version.image_coverage.query(5), Some("Layer 3".to_string()));
176 2 : assert_eq!(version.image_coverage.query(7), Some("Layer 2".to_string()));
177 2 : }
178 :
179 : /// Cover simple off-by-one edge cases
180 : #[test]
181 2 : fn test_off_by_one() {
182 2 : let mut map = HistoricLayerCoverage::<String>::new();
183 2 : map.insert(
184 2 : LayerKey {
185 2 : key: 3..5,
186 2 : lsn: 100..110,
187 2 : is_image: true,
188 2 : },
189 2 : "Layer 1".to_string(),
190 2 : );
191 2 :
192 2 : // Check different LSNs
193 2 : let version = map.get_version(99);
194 2 : assert!(version.is_none());
195 2 : let version = map.get_version(100).unwrap();
196 2 : assert_eq!(version.image_coverage.query(4), Some("Layer 1".to_string()));
197 2 : let version = map.get_version(110).unwrap();
198 2 : assert_eq!(version.image_coverage.query(4), Some("Layer 1".to_string()));
199 :
200 : // Check different keys
201 2 : let version = map.get_version(105).unwrap();
202 2 : assert_eq!(version.image_coverage.query(2), None);
203 2 : assert_eq!(version.image_coverage.query(3), Some("Layer 1".to_string()));
204 2 : assert_eq!(version.image_coverage.query(4), Some("Layer 1".to_string()));
205 2 : assert_eq!(version.image_coverage.query(5), None);
206 2 : }
207 :
208 : /// White-box regression test, checking for incorrect removal of node at key.end
209 : #[test]
210 2 : fn test_regression() {
211 2 : let mut map = HistoricLayerCoverage::<String>::new();
212 2 : map.insert(
213 2 : LayerKey {
214 2 : key: 0..5,
215 2 : lsn: 0..5,
216 2 : is_image: false,
217 2 : },
218 2 : "Layer 1".to_string(),
219 2 : );
220 2 : map.insert(
221 2 : LayerKey {
222 2 : key: 0..5,
223 2 : lsn: 1..2,
224 2 : is_image: false,
225 2 : },
226 2 : "Layer 2".to_string(),
227 2 : );
228 2 :
229 2 : // If an insertion operation improperly deletes the endpoint of a previous layer
230 2 : // (which is more likely to happen with layers that collide on key.end), we will
231 2 : // end up with an infinite layer, covering the entire keyspace. Here we assert
232 2 : // that there's no layer at key 100 because we didn't insert any layer there.
233 2 : let version = map.get_version(100).unwrap();
234 2 : assert_eq!(version.delta_coverage.query(100), None);
235 2 : }
236 :
237 : /// Cover edge cases where layers begin or end on the same key
238 : #[test]
239 2 : fn test_key_collision() {
240 2 : let mut map = HistoricLayerCoverage::<String>::new();
241 2 :
242 2 : map.insert(
243 2 : LayerKey {
244 2 : key: 3..5,
245 2 : lsn: 100..110,
246 2 : is_image: true,
247 2 : },
248 2 : "Layer 10".to_string(),
249 2 : );
250 2 : map.insert(
251 2 : LayerKey {
252 2 : key: 5..8,
253 2 : lsn: 100..110,
254 2 : is_image: true,
255 2 : },
256 2 : "Layer 11".to_string(),
257 2 : );
258 2 : map.insert(
259 2 : LayerKey {
260 2 : key: 3..4,
261 2 : lsn: 200..210,
262 2 : is_image: true,
263 2 : },
264 2 : "Layer 20".to_string(),
265 2 : );
266 2 :
267 2 : // Check after layer 11
268 2 : let version = map.get_version(105).unwrap();
269 2 : assert_eq!(version.image_coverage.query(2), None);
270 2 : assert_eq!(
271 2 : version.image_coverage.query(3),
272 2 : Some("Layer 10".to_string())
273 2 : );
274 2 : assert_eq!(
275 2 : version.image_coverage.query(5),
276 2 : Some("Layer 11".to_string())
277 2 : );
278 2 : assert_eq!(
279 2 : version.image_coverage.query(7),
280 2 : Some("Layer 11".to_string())
281 2 : );
282 2 : assert_eq!(version.image_coverage.query(8), None);
283 :
284 : // Check after layer 20
285 2 : let version = map.get_version(205).unwrap();
286 2 : assert_eq!(version.image_coverage.query(2), None);
287 2 : assert_eq!(
288 2 : version.image_coverage.query(3),
289 2 : Some("Layer 20".to_string())
290 2 : );
291 2 : assert_eq!(
292 2 : version.image_coverage.query(5),
293 2 : Some("Layer 11".to_string())
294 2 : );
295 2 : assert_eq!(
296 2 : version.image_coverage.query(7),
297 2 : Some("Layer 11".to_string())
298 2 : );
299 2 : assert_eq!(version.image_coverage.query(8), None);
300 2 : }
301 :
302 : /// Test when rectangles have nontrivial height and possibly overlap
303 : #[test]
304 2 : fn test_persistent_overlapping() {
305 2 : let mut map = HistoricLayerCoverage::<String>::new();
306 2 :
307 2 : // Add 3 key-disjoint layers with varying LSN ranges
308 2 : map.insert(
309 2 : LayerKey {
310 2 : key: 1..2,
311 2 : lsn: 100..200,
312 2 : is_image: true,
313 2 : },
314 2 : "Layer 1".to_string(),
315 2 : );
316 2 : map.insert(
317 2 : LayerKey {
318 2 : key: 4..5,
319 2 : lsn: 110..200,
320 2 : is_image: true,
321 2 : },
322 2 : "Layer 2".to_string(),
323 2 : );
324 2 : map.insert(
325 2 : LayerKey {
326 2 : key: 7..8,
327 2 : lsn: 120..300,
328 2 : is_image: true,
329 2 : },
330 2 : "Layer 3".to_string(),
331 2 : );
332 2 :
333 2 : // Add wide and short layer
334 2 : map.insert(
335 2 : LayerKey {
336 2 : key: 0..9,
337 2 : lsn: 130..199,
338 2 : is_image: true,
339 2 : },
340 2 : "Layer 4".to_string(),
341 2 : );
342 2 :
343 2 : // Add wide layer taller than some
344 2 : map.insert(
345 2 : LayerKey {
346 2 : key: 0..9,
347 2 : lsn: 140..201,
348 2 : is_image: true,
349 2 : },
350 2 : "Layer 5".to_string(),
351 2 : );
352 2 :
353 2 : // Add wide layer taller than all
354 2 : map.insert(
355 2 : LayerKey {
356 2 : key: 0..9,
357 2 : lsn: 150..301,
358 2 : is_image: true,
359 2 : },
360 2 : "Layer 6".to_string(),
361 2 : );
362 2 :
363 2 : // After layer 4 insertion
364 2 : let version = map.get_version(135).unwrap();
365 2 : assert_eq!(version.image_coverage.query(0), Some("Layer 4".to_string()));
366 2 : assert_eq!(version.image_coverage.query(1), Some("Layer 1".to_string()));
367 2 : assert_eq!(version.image_coverage.query(2), Some("Layer 4".to_string()));
368 2 : assert_eq!(version.image_coverage.query(4), Some("Layer 2".to_string()));
369 2 : assert_eq!(version.image_coverage.query(5), Some("Layer 4".to_string()));
370 2 : assert_eq!(version.image_coverage.query(7), Some("Layer 3".to_string()));
371 2 : assert_eq!(version.image_coverage.query(8), Some("Layer 4".to_string()));
372 :
373 : // After layer 5 insertion
374 2 : let version = map.get_version(145).unwrap();
375 2 : assert_eq!(version.image_coverage.query(0), Some("Layer 5".to_string()));
376 2 : assert_eq!(version.image_coverage.query(1), Some("Layer 5".to_string()));
377 2 : assert_eq!(version.image_coverage.query(2), Some("Layer 5".to_string()));
378 2 : assert_eq!(version.image_coverage.query(4), Some("Layer 5".to_string()));
379 2 : assert_eq!(version.image_coverage.query(5), Some("Layer 5".to_string()));
380 2 : assert_eq!(version.image_coverage.query(7), Some("Layer 3".to_string()));
381 2 : assert_eq!(version.image_coverage.query(8), Some("Layer 5".to_string()));
382 :
383 : // After layer 6 insertion
384 2 : let version = map.get_version(155).unwrap();
385 2 : assert_eq!(version.image_coverage.query(0), Some("Layer 6".to_string()));
386 2 : assert_eq!(version.image_coverage.query(1), Some("Layer 6".to_string()));
387 2 : assert_eq!(version.image_coverage.query(2), Some("Layer 6".to_string()));
388 2 : assert_eq!(version.image_coverage.query(4), Some("Layer 6".to_string()));
389 2 : assert_eq!(version.image_coverage.query(5), Some("Layer 6".to_string()));
390 2 : assert_eq!(version.image_coverage.query(7), Some("Layer 6".to_string()));
391 2 : assert_eq!(version.image_coverage.query(8), Some("Layer 6".to_string()));
392 2 : }
393 :
394 : /// Wrapper for HistoricLayerCoverage that allows us to hack around the lack
395 : /// of support for retroactive insertion by rebuilding the map since the
396 : /// change.
397 : ///
398 : /// Why is this needed? We most often insert new layers with newer LSNs,
399 : /// but during compaction we create layers with non-latest LSN, and during
400 : /// GC we delete historic layers.
401 : ///
402 : /// Even though rebuilding is an expensive (N log N) solution to the problem,
403 : /// it's not critical since we do something equally expensive just to decide
404 : /// whether or not to create new image layers.
405 : /// TODO It's not expensive but it's not great to hold a layer map write lock
406 : /// for that long.
407 : ///
408 : /// If this becomes an actual bottleneck, one solution would be to build a
409 : /// segment tree that holds PersistentLayerMaps. Though this would mean that
410 : /// we take an additional log(N) performance hit for queries, which will probably
411 : /// still be more critical.
412 : ///
413 : /// See this for more on persistent and retroactive techniques:
414 : /// <https://www.youtube.com/watch?v=WqCWghETNDc&t=581s>
415 : pub struct BufferedHistoricLayerCoverage<Value> {
416 : /// A persistent layer map that we rebuild when we need to retroactively update
417 : historic_coverage: HistoricLayerCoverage<Value>,
418 :
419 : /// We buffer insertion into the PersistentLayerMap to decrease the number of rebuilds.
420 : buffer: BTreeMap<LayerKey, Option<Value>>,
421 :
422 : /// All current layers. This is not used for search. Only to make rebuilds easier.
423 : layers: BTreeMap<LayerKey, Value>,
424 : }
425 :
426 : impl<T: std::fmt::Debug> std::fmt::Debug for BufferedHistoricLayerCoverage<T> {
427 0 : fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
428 0 : f.debug_struct("RetroactiveLayerMap")
429 0 : .field("buffer", &self.buffer)
430 0 : .field("layers", &self.layers)
431 0 : .finish()
432 0 : }
433 : }
434 :
435 : impl<T: Clone> Default for BufferedHistoricLayerCoverage<T> {
436 385 : fn default() -> Self {
437 385 : Self::new()
438 385 : }
439 : }
440 :
441 : impl<Value: Clone> BufferedHistoricLayerCoverage<Value> {
442 389 : pub fn new() -> Self {
443 389 : Self {
444 389 : historic_coverage: HistoricLayerCoverage::<Value>::new(),
445 389 : buffer: BTreeMap::new(),
446 389 : layers: BTreeMap::new(),
447 389 : }
448 389 : }
449 :
450 1576 : pub fn insert(&mut self, layer_key: LayerKey, value: Value) {
451 1576 : self.buffer.insert(layer_key, Some(value));
452 1576 : }
453 :
454 436 : pub fn remove(&mut self, layer_key: LayerKey) {
455 436 : self.buffer.insert(layer_key, None);
456 436 : }
457 :
458 1604 : pub fn rebuild(&mut self) {
459 : // Find the first LSN that needs to be rebuilt
460 1604 : let rebuild_since: u64 = match self.buffer.iter().next() {
461 1254 : Some((LayerKey { lsn, .. }, _)) => lsn.start,
462 350 : None => return, // No need to rebuild if buffer is empty
463 : };
464 :
465 : // Apply buffered updates to self.layers
466 1254 : let num_updates = self.buffer.len();
467 2012 : self.buffer.retain(|layer_key, layer| {
468 2012 : match layer {
469 1576 : Some(l) => {
470 1576 : self.layers.insert(layer_key.clone(), l.clone());
471 1576 : }
472 436 : None => {
473 436 : self.layers.remove(layer_key);
474 436 : }
475 : };
476 2012 : false
477 2012 : });
478 1254 :
479 1254 : // Rebuild
480 1254 : let mut num_inserted = 0;
481 1254 : self.historic_coverage.trim(&rebuild_since);
482 1682 : for (layer_key, layer) in self.layers.range(
483 1254 : LayerKey {
484 1254 : lsn: rebuild_since..0,
485 1254 : key: 0..0,
486 1254 : is_image: false,
487 1254 : }..,
488 1682 : ) {
489 1682 : self.historic_coverage
490 1682 : .insert(layer_key.clone(), layer.clone());
491 1682 : num_inserted += 1;
492 1682 : }
493 :
494 : // TODO maybe only warn if ratio is at least 10
495 1254 : info!(
496 0 : "Rebuilt layer map. Did {} insertions to process a batch of {} updates.",
497 : num_inserted, num_updates,
498 : )
499 1604 : }
500 :
501 : /// Iterate all the layers
502 772 : pub fn iter(&self) -> impl '_ + Iterator<Item = Value> {
503 772 : // NOTE we can actually perform this without rebuilding,
504 772 : // but it's not necessary for now.
505 772 : if !self.buffer.is_empty() {
506 0 : panic!("rebuild pls")
507 772 : }
508 772 :
509 772 : self.layers.values().cloned()
510 772 : }
511 :
512 : /// Return a reference to a queryable map, assuming all updates
513 : /// have already been processed using self.rebuild()
514 370407 : pub fn get(&self) -> anyhow::Result<&HistoricLayerCoverage<Value>> {
515 370407 : // NOTE we error here instead of implicitly rebuilding because
516 370407 : // rebuilding is somewhat expensive.
517 370407 : // TODO maybe implicitly rebuild and log/sentry an error?
518 370407 : if !self.buffer.is_empty() {
519 0 : anyhow::bail!("rebuild required")
520 370407 : }
521 370407 :
522 370407 : Ok(&self.historic_coverage)
523 370407 : }
524 : }
525 :
526 : #[test]
527 2 : fn test_retroactive_regression_1() {
528 2 : let mut map = BufferedHistoricLayerCoverage::new();
529 2 :
530 2 : map.insert(
531 2 : LayerKey {
532 2 : key: 0..21267647932558653966460912964485513215,
533 2 : lsn: 23761336..23761457,
534 2 : is_image: false,
535 2 : },
536 2 : "sdfsdfs".to_string(),
537 2 : );
538 2 :
539 2 : map.rebuild();
540 2 :
541 2 : let version = map.get().unwrap().get_version(23761457).unwrap();
542 2 : assert_eq!(
543 2 : version.delta_coverage.query(100),
544 2 : Some("sdfsdfs".to_string())
545 2 : );
546 2 : }
547 :
548 : #[test]
549 2 : fn test_retroactive_simple() {
550 2 : let mut map = BufferedHistoricLayerCoverage::new();
551 2 :
552 2 : // Append some images in increasing LSN order
553 2 : map.insert(
554 2 : LayerKey {
555 2 : key: 0..5,
556 2 : lsn: 100..101,
557 2 : is_image: true,
558 2 : },
559 2 : "Image 1".to_string(),
560 2 : );
561 2 : map.insert(
562 2 : LayerKey {
563 2 : key: 3..9,
564 2 : lsn: 110..111,
565 2 : is_image: true,
566 2 : },
567 2 : "Image 2".to_string(),
568 2 : );
569 2 : map.insert(
570 2 : LayerKey {
571 2 : key: 4..6,
572 2 : lsn: 120..121,
573 2 : is_image: true,
574 2 : },
575 2 : "Image 3".to_string(),
576 2 : );
577 2 : map.insert(
578 2 : LayerKey {
579 2 : key: 8..9,
580 2 : lsn: 120..121,
581 2 : is_image: true,
582 2 : },
583 2 : "Image 4".to_string(),
584 2 : );
585 2 :
586 2 : // Add a delta layer out of order
587 2 : map.insert(
588 2 : LayerKey {
589 2 : key: 2..5,
590 2 : lsn: 105..106,
591 2 : is_image: false,
592 2 : },
593 2 : "Delta 1".to_string(),
594 2 : );
595 2 :
596 2 : // Rebuild so we can start querying
597 2 : map.rebuild();
598 2 :
599 2 : {
600 2 : let map = map.get().expect("rebuilt");
601 2 :
602 2 : let version = map.get_version(90);
603 2 : assert!(version.is_none());
604 2 : let version = map.get_version(102).unwrap();
605 2 : assert_eq!(version.image_coverage.query(4), Some("Image 1".to_string()));
606 :
607 2 : let version = map.get_version(107).unwrap();
608 2 : assert_eq!(version.image_coverage.query(4), Some("Image 1".to_string()));
609 2 : assert_eq!(version.delta_coverage.query(4), Some("Delta 1".to_string()));
610 :
611 2 : let version = map.get_version(115).unwrap();
612 2 : assert_eq!(version.image_coverage.query(4), Some("Image 2".to_string()));
613 :
614 2 : let version = map.get_version(125).unwrap();
615 2 : assert_eq!(version.image_coverage.query(4), Some("Image 3".to_string()));
616 : }
617 :
618 : // Remove Image 3
619 2 : map.remove(LayerKey {
620 2 : key: 4..6,
621 2 : lsn: 120..121,
622 2 : is_image: true,
623 2 : });
624 2 : map.rebuild();
625 2 :
626 2 : {
627 2 : // Check deletion worked
628 2 : let map = map.get().expect("rebuilt");
629 2 : let version = map.get_version(125).unwrap();
630 2 : assert_eq!(version.image_coverage.query(4), Some("Image 2".to_string()));
631 2 : assert_eq!(version.image_coverage.query(8), Some("Image 4".to_string()));
632 : }
633 2 : }
|
