Line data Source code
1 : use std::collections::BTreeMap;
2 : use std::ops::Range;
3 :
4 : use tracing::info;
5 :
6 : use crate::tenant::storage_layer::PersistentLayerDesc;
7 :
8 : use super::layer_coverage::LayerCoverageTuple;
9 :
10 : /// Layers in this module are identified and indexed by this data.
11 : ///
12 : /// This is a helper struct to enable sorting layers by lsn.start.
13 : ///
14 : /// These three values are enough to uniquely identify a layer, since
15 : /// a layer is obligated to contain all contents within range, so two
16 : /// deltas (or images) with the same range have identical content.
17 : #[derive(Debug, PartialEq, Eq, Clone)]
18 : pub struct LayerKey {
19 : // TODO I use i128 and u64 because it was easy for prototyping,
20 : // testing, and benchmarking. If we can use the Lsn and Key
21 : // types without overhead that would be preferable.
22 : pub key: Range<i128>,
23 : pub lsn: Range<u64>,
24 : pub is_image: bool,
25 : }
26 :
27 : impl PartialOrd for LayerKey {
28 0 : fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
29 0 : Some(self.cmp(other))
30 0 : }
31 : }
32 :
33 : impl Ord for LayerKey {
34 444910 : fn cmp(&self, other: &Self) -> std::cmp::Ordering {
35 444910 : // NOTE we really care about comparing by lsn.start first
36 444910 : self.lsn
37 444910 : .start
38 444910 : .cmp(&other.lsn.start)
39 444910 : .then(self.lsn.end.cmp(&other.lsn.end))
40 444910 : .then(self.key.start.cmp(&other.key.start))
41 444910 : .then(self.key.end.cmp(&other.key.end))
42 444910 : .then(self.is_image.cmp(&other.is_image))
43 444910 : }
44 : }
45 :
46 : impl From<&PersistentLayerDesc> for LayerKey {
47 15798 : fn from(layer: &PersistentLayerDesc) -> Self {
48 15798 : let kr = layer.get_key_range();
49 15798 : let lr = layer.get_lsn_range();
50 15798 : LayerKey {
51 15798 : key: kr.start.to_i128()..kr.end.to_i128(),
52 15798 : lsn: lr.start.0..lr.end.0,
53 15798 : is_image: !layer.is_incremental(),
54 15798 : }
55 15798 : }
56 : }
57 :
58 : /// Efficiently queryable layer coverage for each LSN.
59 : ///
60 : /// Allows answering layer map queries very efficiently,
61 : /// but doesn't allow retroactive insertion, which is
62 : /// sometimes necessary. See BufferedHistoricLayerCoverage.
63 : pub struct HistoricLayerCoverage<Value> {
64 : /// The latest state
65 : head: LayerCoverageTuple<Value>,
66 :
67 : /// All previous states
68 : historic: BTreeMap<u64, LayerCoverageTuple<Value>>,
69 : }
70 :
71 : impl<T: Clone> Default for HistoricLayerCoverage<T> {
72 0 : fn default() -> Self {
73 0 : Self::new()
74 0 : }
75 : }
76 :
77 : impl<Value: Clone> HistoricLayerCoverage<Value> {
78 1308 : pub fn new() -> Self {
79 1308 : Self {
80 1308 : head: LayerCoverageTuple::default(),
81 1308 : historic: BTreeMap::default(),
82 1308 : }
83 1308 : }
84 :
85 : /// Add a layer
86 : ///
87 : /// Panics if new layer has older lsn.start than an existing layer.
88 : /// See BufferedHistoricLayerCoverage for a more general insertion method.
89 14994 : pub fn insert(&mut self, layer_key: LayerKey, value: Value) {
90 : // It's only a persistent map, not a retroactive one
91 14994 : if let Some(last_entry) = self.historic.iter().next_back() {
92 13848 : let last_lsn = last_entry.0;
93 13848 : if layer_key.lsn.start < *last_lsn {
94 0 : panic!("unexpected retroactive insert");
95 13848 : }
96 1146 : }
97 :
98 : // Insert into data structure
99 14994 : let target = if layer_key.is_image {
100 5982 : &mut self.head.image_coverage
101 : } else {
102 9012 : &mut self.head.delta_coverage
103 : };
104 :
105 14994 : target.insert(layer_key.key, layer_key.lsn.clone(), value);
106 14994 :
107 14994 : // Remember history. Clone is O(1)
108 14994 : self.historic.insert(layer_key.lsn.start, self.head.clone());
109 14994 : }
110 :
111 : /// Query at a particular LSN, inclusive
112 1790969 : pub fn get_version(&self, lsn: u64) -> Option<&LayerCoverageTuple<Value>> {
113 1790969 : match self.historic.range(..=lsn).next_back() {
114 1113925 : Some((_, v)) => Some(v),
115 677044 : None => None,
116 : }
117 1790969 : }
118 :
119 : /// Remove all entries after a certain LSN (inclusive)
120 4014 : pub fn trim(&mut self, begin: &u64) {
121 4014 : self.historic.split_off(begin);
122 4014 : self.head = self
123 4014 : .historic
124 4014 : .iter()
125 4014 : .next_back()
126 4014 : .map(|(_, v)| v.clone())
127 4014 : .unwrap_or_default();
128 4014 : }
129 : }
130 :
131 : /// This is the most basic test that demonstrates intended usage.
132 : /// All layers in this test have height 1.
133 : #[test]
134 6 : fn test_persistent_simple() {
135 6 : let mut map = HistoricLayerCoverage::<String>::new();
136 6 : map.insert(
137 6 : LayerKey {
138 6 : key: 0..5,
139 6 : lsn: 100..101,
140 6 : is_image: true,
141 6 : },
142 6 : "Layer 1".to_string(),
143 6 : );
144 6 : map.insert(
145 6 : LayerKey {
146 6 : key: 3..9,
147 6 : lsn: 110..111,
148 6 : is_image: true,
149 6 : },
150 6 : "Layer 2".to_string(),
151 6 : );
152 6 : map.insert(
153 6 : LayerKey {
154 6 : key: 5..6,
155 6 : lsn: 120..121,
156 6 : is_image: true,
157 6 : },
158 6 : "Layer 3".to_string(),
159 6 : );
160 6 :
161 6 : // After Layer 1 insertion
162 6 : let version = map.get_version(105).unwrap();
163 6 : assert_eq!(version.image_coverage.query(1), Some("Layer 1".to_string()));
164 6 : assert_eq!(version.image_coverage.query(4), Some("Layer 1".to_string()));
165 :
166 : // After Layer 2 insertion
167 6 : let version = map.get_version(115).unwrap();
168 6 : assert_eq!(version.image_coverage.query(4), Some("Layer 2".to_string()));
169 6 : assert_eq!(version.image_coverage.query(8), Some("Layer 2".to_string()));
170 6 : assert_eq!(version.image_coverage.query(11), None);
171 :
172 : // After Layer 3 insertion
173 6 : let version = map.get_version(125).unwrap();
174 6 : assert_eq!(version.image_coverage.query(4), Some("Layer 2".to_string()));
175 6 : assert_eq!(version.image_coverage.query(5), Some("Layer 3".to_string()));
176 6 : assert_eq!(version.image_coverage.query(7), Some("Layer 2".to_string()));
177 6 : }
178 :
179 : /// Cover simple off-by-one edge cases
180 : #[test]
181 6 : fn test_off_by_one() {
182 6 : let mut map = HistoricLayerCoverage::<String>::new();
183 6 : map.insert(
184 6 : LayerKey {
185 6 : key: 3..5,
186 6 : lsn: 100..110,
187 6 : is_image: true,
188 6 : },
189 6 : "Layer 1".to_string(),
190 6 : );
191 6 :
192 6 : // Check different LSNs
193 6 : let version = map.get_version(99);
194 6 : assert!(version.is_none());
195 6 : let version = map.get_version(100).unwrap();
196 6 : assert_eq!(version.image_coverage.query(4), Some("Layer 1".to_string()));
197 6 : let version = map.get_version(110).unwrap();
198 6 : assert_eq!(version.image_coverage.query(4), Some("Layer 1".to_string()));
199 :
200 : // Check different keys
201 6 : let version = map.get_version(105).unwrap();
202 6 : assert_eq!(version.image_coverage.query(2), None);
203 6 : assert_eq!(version.image_coverage.query(3), Some("Layer 1".to_string()));
204 6 : assert_eq!(version.image_coverage.query(4), Some("Layer 1".to_string()));
205 6 : assert_eq!(version.image_coverage.query(5), None);
206 6 : }
207 :
208 : /// White-box regression test, checking for incorrect removal of node at key.end
209 : #[test]
210 6 : fn test_regression() {
211 6 : let mut map = HistoricLayerCoverage::<String>::new();
212 6 : map.insert(
213 6 : LayerKey {
214 6 : key: 0..5,
215 6 : lsn: 0..5,
216 6 : is_image: false,
217 6 : },
218 6 : "Layer 1".to_string(),
219 6 : );
220 6 : map.insert(
221 6 : LayerKey {
222 6 : key: 0..5,
223 6 : lsn: 1..2,
224 6 : is_image: false,
225 6 : },
226 6 : "Layer 2".to_string(),
227 6 : );
228 6 :
229 6 : // If an insertion operation improperly deletes the endpoint of a previous layer
230 6 : // (which is more likely to happen with layers that collide on key.end), we will
231 6 : // end up with an infinite layer, covering the entire keyspace. Here we assert
232 6 : // that there's no layer at key 100 because we didn't insert any layer there.
233 6 : let version = map.get_version(100).unwrap();
234 6 : assert_eq!(version.delta_coverage.query(100), None);
235 6 : }
236 :
237 : /// Cover edge cases where layers begin or end on the same key
238 : #[test]
239 6 : fn test_key_collision() {
240 6 : let mut map = HistoricLayerCoverage::<String>::new();
241 6 :
242 6 : map.insert(
243 6 : LayerKey {
244 6 : key: 3..5,
245 6 : lsn: 100..110,
246 6 : is_image: true,
247 6 : },
248 6 : "Layer 10".to_string(),
249 6 : );
250 6 : map.insert(
251 6 : LayerKey {
252 6 : key: 5..8,
253 6 : lsn: 100..110,
254 6 : is_image: true,
255 6 : },
256 6 : "Layer 11".to_string(),
257 6 : );
258 6 : map.insert(
259 6 : LayerKey {
260 6 : key: 3..4,
261 6 : lsn: 200..210,
262 6 : is_image: true,
263 6 : },
264 6 : "Layer 20".to_string(),
265 6 : );
266 6 :
267 6 : // Check after layer 11
268 6 : let version = map.get_version(105).unwrap();
269 6 : assert_eq!(version.image_coverage.query(2), None);
270 6 : assert_eq!(
271 6 : version.image_coverage.query(3),
272 6 : Some("Layer 10".to_string())
273 6 : );
274 6 : assert_eq!(
275 6 : version.image_coverage.query(5),
276 6 : Some("Layer 11".to_string())
277 6 : );
278 6 : assert_eq!(
279 6 : version.image_coverage.query(7),
280 6 : Some("Layer 11".to_string())
281 6 : );
282 6 : assert_eq!(version.image_coverage.query(8), None);
283 :
284 : // Check after layer 20
285 6 : let version = map.get_version(205).unwrap();
286 6 : assert_eq!(version.image_coverage.query(2), None);
287 6 : assert_eq!(
288 6 : version.image_coverage.query(3),
289 6 : Some("Layer 20".to_string())
290 6 : );
291 6 : assert_eq!(
292 6 : version.image_coverage.query(5),
293 6 : Some("Layer 11".to_string())
294 6 : );
295 6 : assert_eq!(
296 6 : version.image_coverage.query(7),
297 6 : Some("Layer 11".to_string())
298 6 : );
299 6 : assert_eq!(version.image_coverage.query(8), None);
300 6 : }
301 :
302 : /// Test when rectangles have nontrivial height and possibly overlap
303 : #[test]
304 6 : fn test_persistent_overlapping() {
305 6 : let mut map = HistoricLayerCoverage::<String>::new();
306 6 :
307 6 : // Add 3 key-disjoint layers with varying LSN ranges
308 6 : map.insert(
309 6 : LayerKey {
310 6 : key: 1..2,
311 6 : lsn: 100..200,
312 6 : is_image: true,
313 6 : },
314 6 : "Layer 1".to_string(),
315 6 : );
316 6 : map.insert(
317 6 : LayerKey {
318 6 : key: 4..5,
319 6 : lsn: 110..200,
320 6 : is_image: true,
321 6 : },
322 6 : "Layer 2".to_string(),
323 6 : );
324 6 : map.insert(
325 6 : LayerKey {
326 6 : key: 7..8,
327 6 : lsn: 120..300,
328 6 : is_image: true,
329 6 : },
330 6 : "Layer 3".to_string(),
331 6 : );
332 6 :
333 6 : // Add wide and short layer
334 6 : map.insert(
335 6 : LayerKey {
336 6 : key: 0..9,
337 6 : lsn: 130..199,
338 6 : is_image: true,
339 6 : },
340 6 : "Layer 4".to_string(),
341 6 : );
342 6 :
343 6 : // Add wide layer taller than some
344 6 : map.insert(
345 6 : LayerKey {
346 6 : key: 0..9,
347 6 : lsn: 140..201,
348 6 : is_image: true,
349 6 : },
350 6 : "Layer 5".to_string(),
351 6 : );
352 6 :
353 6 : // Add wide layer taller than all
354 6 : map.insert(
355 6 : LayerKey {
356 6 : key: 0..9,
357 6 : lsn: 150..301,
358 6 : is_image: true,
359 6 : },
360 6 : "Layer 6".to_string(),
361 6 : );
362 6 :
363 6 : // After layer 4 insertion
364 6 : let version = map.get_version(135).unwrap();
365 6 : assert_eq!(version.image_coverage.query(0), Some("Layer 4".to_string()));
366 6 : assert_eq!(version.image_coverage.query(1), Some("Layer 1".to_string()));
367 6 : assert_eq!(version.image_coverage.query(2), Some("Layer 4".to_string()));
368 6 : assert_eq!(version.image_coverage.query(4), Some("Layer 2".to_string()));
369 6 : assert_eq!(version.image_coverage.query(5), Some("Layer 4".to_string()));
370 6 : assert_eq!(version.image_coverage.query(7), Some("Layer 3".to_string()));
371 6 : assert_eq!(version.image_coverage.query(8), Some("Layer 4".to_string()));
372 :
373 : // After layer 5 insertion
374 6 : let version = map.get_version(145).unwrap();
375 6 : assert_eq!(version.image_coverage.query(0), Some("Layer 5".to_string()));
376 6 : assert_eq!(version.image_coverage.query(1), Some("Layer 5".to_string()));
377 6 : assert_eq!(version.image_coverage.query(2), Some("Layer 5".to_string()));
378 6 : assert_eq!(version.image_coverage.query(4), Some("Layer 5".to_string()));
379 6 : assert_eq!(version.image_coverage.query(5), Some("Layer 5".to_string()));
380 6 : assert_eq!(version.image_coverage.query(7), Some("Layer 3".to_string()));
381 6 : assert_eq!(version.image_coverage.query(8), Some("Layer 5".to_string()));
382 :
383 : // After layer 6 insertion
384 6 : let version = map.get_version(155).unwrap();
385 6 : assert_eq!(version.image_coverage.query(0), Some("Layer 6".to_string()));
386 6 : assert_eq!(version.image_coverage.query(1), Some("Layer 6".to_string()));
387 6 : assert_eq!(version.image_coverage.query(2), Some("Layer 6".to_string()));
388 6 : assert_eq!(version.image_coverage.query(4), Some("Layer 6".to_string()));
389 6 : assert_eq!(version.image_coverage.query(5), Some("Layer 6".to_string()));
390 6 : assert_eq!(version.image_coverage.query(7), Some("Layer 6".to_string()));
391 6 : assert_eq!(version.image_coverage.query(8), Some("Layer 6".to_string()));
392 6 : }
393 :
394 : /// Wrapper for HistoricLayerCoverage that allows us to hack around the lack
395 : /// of support for retroactive insertion by rebuilding the map since the
396 : /// change.
397 : ///
398 : /// Why is this needed? We most often insert new layers with newer LSNs,
399 : /// but during compaction we create layers with non-latest LSN, and during
400 : /// GC we delete historic layers.
401 : ///
402 : /// Even though rebuilding is an expensive (N log N) solution to the problem,
403 : /// it's not critical since we do something equally expensive just to decide
404 : /// whether or not to create new image layers.
405 : /// TODO It's not expensive but it's not great to hold a layer map write lock
406 : /// for that long.
407 : ///
408 : /// If this becomes an actual bottleneck, one solution would be to build a
409 : /// segment tree that holds PersistentLayerMaps. Though this would mean that
410 : /// we take an additional log(N) performance hit for queries, which will probably
411 : /// still be more critical.
412 : ///
413 : /// See this for more on persistent and retroactive techniques:
414 : /// <https://www.youtube.com/watch?v=WqCWghETNDc&t=581s>
415 : pub struct BufferedHistoricLayerCoverage<Value> {
416 : /// A persistent layer map that we rebuild when we need to retroactively update
417 : historic_coverage: HistoricLayerCoverage<Value>,
418 :
419 : /// We buffer insertion into the PersistentLayerMap to decrease the number of rebuilds.
420 : buffer: BTreeMap<LayerKey, Option<Value>>,
421 :
422 : /// All current layers. This is not used for search. Only to make rebuilds easier.
423 : layers: BTreeMap<LayerKey, Value>,
424 : }
425 :
426 : impl<T: std::fmt::Debug> std::fmt::Debug for BufferedHistoricLayerCoverage<T> {
427 0 : fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
428 0 : f.debug_struct("RetroactiveLayerMap")
429 0 : .field("buffer", &self.buffer)
430 0 : .field("layers", &self.layers)
431 0 : .finish()
432 0 : }
433 : }
434 :
435 : impl<T: Clone> Default for BufferedHistoricLayerCoverage<T> {
436 1266 : fn default() -> Self {
437 1266 : Self::new()
438 1266 : }
439 : }
440 :
441 : impl<Value: Clone> BufferedHistoricLayerCoverage<Value> {
442 1278 : pub fn new() -> Self {
443 1278 : Self {
444 1278 : historic_coverage: HistoricLayerCoverage::<Value>::new(),
445 1278 : buffer: BTreeMap::new(),
446 1278 : layers: BTreeMap::new(),
447 1278 : }
448 1278 : }
449 :
450 14436 : pub fn insert(&mut self, layer_key: LayerKey, value: Value) {
451 14436 : self.buffer.insert(layer_key, Some(value));
452 14436 : }
453 :
454 1404 : pub fn remove(&mut self, layer_key: LayerKey) {
455 1404 : self.buffer.insert(layer_key, None);
456 1404 : }
457 :
458 5610 : pub fn rebuild(&mut self) {
459 : // Find the first LSN that needs to be rebuilt
460 5610 : let rebuild_since: u64 = match self.buffer.iter().next() {
461 4014 : Some((LayerKey { lsn, .. }, _)) => lsn.start,
462 1596 : None => return, // No need to rebuild if buffer is empty
463 : };
464 :
465 : // Apply buffered updates to self.layers
466 4014 : let num_updates = self.buffer.len();
467 15840 : self.buffer.retain(|layer_key, layer| {
468 15840 : match layer {
469 14436 : Some(l) => {
470 14436 : self.layers.insert(layer_key.clone(), l.clone());
471 14436 : }
472 1404 : None => {
473 1404 : self.layers.remove(layer_key);
474 1404 : }
475 : };
476 15840 : false
477 15840 : });
478 4014 :
479 4014 : // Rebuild
480 4014 : let mut num_inserted = 0;
481 4014 : self.historic_coverage.trim(&rebuild_since);
482 14904 : for (layer_key, layer) in self.layers.range(
483 4014 : LayerKey {
484 4014 : lsn: rebuild_since..0,
485 4014 : key: 0..0,
486 4014 : is_image: false,
487 4014 : }..,
488 14904 : ) {
489 14904 : self.historic_coverage
490 14904 : .insert(layer_key.clone(), layer.clone());
491 14904 : num_inserted += 1;
492 14904 : }
493 :
494 : // TODO maybe only warn if ratio is at least 10
495 4014 : info!(
496 0 : "Rebuilt layer map. Did {} insertions to process a batch of {} updates.",
497 : num_inserted, num_updates,
498 : )
499 5610 : }
500 :
501 : /// Iterate all the layers
502 12489 : pub fn iter(&self) -> impl '_ + Iterator<Item = Value> {
503 12489 : // NOTE we can actually perform this without rebuilding,
504 12489 : // but it's not necessary for now.
505 12489 : if !self.buffer.is_empty() {
506 0 : panic!("rebuild pls")
507 12489 : }
508 12489 :
509 12489 : self.layers.values().cloned()
510 12489 : }
511 :
512 : /// Return a reference to a queryable map, assuming all updates
513 : /// have already been processed using self.rebuild()
514 1790867 : pub fn get(&self) -> anyhow::Result<&HistoricLayerCoverage<Value>> {
515 1790867 : // NOTE we error here instead of implicitly rebuilding because
516 1790867 : // rebuilding is somewhat expensive.
517 1790867 : // TODO maybe implicitly rebuild and log/sentry an error?
518 1790867 : if !self.buffer.is_empty() {
519 0 : anyhow::bail!("rebuild required")
520 1790867 : }
521 1790867 :
522 1790867 : Ok(&self.historic_coverage)
523 1790867 : }
524 :
525 1200 : pub(crate) fn len(&self) -> usize {
526 1200 : self.layers.len()
527 1200 : }
528 : }
529 :
530 : #[test]
531 6 : fn test_retroactive_regression_1() {
532 6 : let mut map = BufferedHistoricLayerCoverage::new();
533 6 :
534 6 : map.insert(
535 6 : LayerKey {
536 6 : key: 0..21267647932558653966460912964485513215,
537 6 : lsn: 23761336..23761457,
538 6 : is_image: false,
539 6 : },
540 6 : "sdfsdfs".to_string(),
541 6 : );
542 6 :
543 6 : map.rebuild();
544 6 :
545 6 : let version = map.get().unwrap().get_version(23761457).unwrap();
546 6 : assert_eq!(
547 6 : version.delta_coverage.query(100),
548 6 : Some("sdfsdfs".to_string())
549 6 : );
550 6 : }
551 :
552 : #[test]
553 6 : fn test_retroactive_simple() {
554 6 : let mut map = BufferedHistoricLayerCoverage::new();
555 6 :
556 6 : // Append some images in increasing LSN order
557 6 : map.insert(
558 6 : LayerKey {
559 6 : key: 0..5,
560 6 : lsn: 100..101,
561 6 : is_image: true,
562 6 : },
563 6 : "Image 1".to_string(),
564 6 : );
565 6 : map.insert(
566 6 : LayerKey {
567 6 : key: 3..9,
568 6 : lsn: 110..111,
569 6 : is_image: true,
570 6 : },
571 6 : "Image 2".to_string(),
572 6 : );
573 6 : map.insert(
574 6 : LayerKey {
575 6 : key: 4..6,
576 6 : lsn: 120..121,
577 6 : is_image: true,
578 6 : },
579 6 : "Image 3".to_string(),
580 6 : );
581 6 : map.insert(
582 6 : LayerKey {
583 6 : key: 8..9,
584 6 : lsn: 120..121,
585 6 : is_image: true,
586 6 : },
587 6 : "Image 4".to_string(),
588 6 : );
589 6 :
590 6 : // Add a delta layer out of order
591 6 : map.insert(
592 6 : LayerKey {
593 6 : key: 2..5,
594 6 : lsn: 105..106,
595 6 : is_image: false,
596 6 : },
597 6 : "Delta 1".to_string(),
598 6 : );
599 6 :
600 6 : // Rebuild so we can start querying
601 6 : map.rebuild();
602 6 :
603 6 : {
604 6 : let map = map.get().expect("rebuilt");
605 6 :
606 6 : let version = map.get_version(90);
607 6 : assert!(version.is_none());
608 6 : let version = map.get_version(102).unwrap();
609 6 : assert_eq!(version.image_coverage.query(4), Some("Image 1".to_string()));
610 :
611 6 : let version = map.get_version(107).unwrap();
612 6 : assert_eq!(version.image_coverage.query(4), Some("Image 1".to_string()));
613 6 : assert_eq!(version.delta_coverage.query(4), Some("Delta 1".to_string()));
614 :
615 6 : let version = map.get_version(115).unwrap();
616 6 : assert_eq!(version.image_coverage.query(4), Some("Image 2".to_string()));
617 :
618 6 : let version = map.get_version(125).unwrap();
619 6 : assert_eq!(version.image_coverage.query(4), Some("Image 3".to_string()));
620 : }
621 :
622 : // Remove Image 3
623 6 : map.remove(LayerKey {
624 6 : key: 4..6,
625 6 : lsn: 120..121,
626 6 : is_image: true,
627 6 : });
628 6 : map.rebuild();
629 6 :
630 6 : {
631 6 : // Check deletion worked
632 6 : let map = map.get().expect("rebuilt");
633 6 : let version = map.get_version(125).unwrap();
634 6 : assert_eq!(version.image_coverage.query(4), Some("Image 2".to_string()));
635 6 : assert_eq!(version.image_coverage.query(8), Some("Image 4".to_string()));
636 : }
637 6 : }
|
