Line data Source code
1 : //! This is what the compaction implementation needs to know about
2 : //! layers, keyspace etc.
3 : //!
4 : //! All the heavy lifting is done by the create_image and create_delta
5 : //! functions that the implementor provides.
6 : use futures::Future;
7 : use pageserver_api::{key::Key, keyspace::key_range_size};
8 : use std::ops::Range;
9 : use utils::lsn::Lsn;
10 :
11 : /// Public interface. This is the main thing that the implementor needs to provide
12 : pub trait CompactionJobExecutor {
13 : // Type system.
14 : //
15 : // We assume that there are two kinds of layers, deltas and images. The
16 : // compaction doesn't distinguish whether they are stored locally or
17 : // remotely.
18 : //
19 : // The keyspace is defined by the CompactionKey trait.
20 : type Key: CompactionKey;
21 :
22 : type Layer: CompactionLayer<Self::Key> + Clone;
23 : type DeltaLayer: CompactionDeltaLayer<Self> + Clone;
24 : type ImageLayer: CompactionImageLayer<Self> + Clone;
25 :
26 : // This is passed through to all the interface functions. The compaction
27 : // implementation doesn't do anything with it, but it might be useful for
28 : // the interface implementation.
29 : type RequestContext: CompactionRequestContext;
30 :
31 : // ----
32 : // Functions that the planner uses to support its decisions
33 : // ----
34 :
35 : /// Return all layers that overlap the given bounding box.
36 : fn get_layers(
37 : &mut self,
38 : key_range: &Range<Self::Key>,
39 : lsn_range: &Range<Lsn>,
40 : ctx: &Self::RequestContext,
41 : ) -> impl Future<Output = anyhow::Result<Vec<Self::Layer>>> + Send;
42 :
43 : fn get_keyspace(
44 : &mut self,
45 : key_range: &Range<Self::Key>,
46 : lsn: Lsn,
47 : ctx: &Self::RequestContext,
48 : ) -> impl Future<Output = anyhow::Result<CompactionKeySpace<Self::Key>>> + Send;
49 :
50 : /// NB: This is a pretty expensive operation. In the real pageserver
51 : /// implementation, it downloads the layer, and keeps it resident
52 : /// until the DeltaLayer is dropped.
53 : fn downcast_delta_layer(
54 : &self,
55 : layer: &Self::Layer,
56 : ) -> impl Future<Output = anyhow::Result<Option<Self::DeltaLayer>>> + Send;
57 :
58 : // ----
59 : // Functions to execute the plan
60 : // ----
61 :
62 : /// Create a new image layer, materializing all the values in the key range,
63 : /// at given 'lsn'.
64 : fn create_image(
65 : &mut self,
66 : lsn: Lsn,
67 : key_range: &Range<Self::Key>,
68 : ctx: &Self::RequestContext,
69 : ) -> impl Future<Output = anyhow::Result<()>> + Send;
70 :
71 : /// Create a new delta layer, containing all the values from 'input_layers'
72 : /// in the given key and LSN range.
73 : fn create_delta(
74 : &mut self,
75 : lsn_range: &Range<Lsn>,
76 : key_range: &Range<Self::Key>,
77 : input_layers: &[Self::DeltaLayer],
78 : ctx: &Self::RequestContext,
79 : ) -> impl Future<Output = anyhow::Result<()>> + Send;
80 :
81 : /// Delete a layer. The compaction implementation will call this only after
82 : /// all the create_image() or create_delta() calls that deletion of this
83 : /// layer depends on have finished. But if the implementor has extra lazy
84 : /// background tasks, like uploading the index json file to remote storage.
85 : /// it is the implementation's responsibility to track those.
86 : fn delete_layer(
87 : &mut self,
88 : layer: &Self::Layer,
89 : ctx: &Self::RequestContext,
90 : ) -> impl Future<Output = anyhow::Result<()>> + Send;
91 : }
92 :
93 : pub trait CompactionKey: std::cmp::Ord + Clone + Copy + std::fmt::Display {
94 : const MIN: Self;
95 : const MAX: Self;
96 :
97 : /// Calculate distance between key_range.start and key_range.end.
98 : ///
99 : /// This returns u32, for compatibility with Repository::key. If the
100 : /// distance is larger, return u32::MAX.
101 : fn key_range_size(key_range: &Range<Self>) -> u32;
102 :
103 : // return "self + 1"
104 : fn next(&self) -> Self;
105 :
106 : // return "self + <some decent amount to skip>". The amount to skip
107 : // is left to the implementation.
108 : // FIXME: why not just "add(u32)" ? This is hard to use
109 : fn skip_some(&self) -> Self;
110 : }
111 :
112 : impl CompactionKey for Key {
113 : const MIN: Self = Self::MIN;
114 : const MAX: Self = Self::MAX;
115 :
116 0 : fn key_range_size(r: &std::ops::Range<Self>) -> u32 {
117 0 : key_range_size(r)
118 0 : }
119 0 : fn next(&self) -> Key {
120 0 : (self as &Key).next()
121 0 : }
122 0 : fn skip_some(&self) -> Key {
123 0 : self.add(128)
124 0 : }
125 : }
126 :
127 : /// Contiguous ranges of keys that belong to the key space. In key order, and
128 : /// with no overlap.
129 : pub type CompactionKeySpace<K> = Vec<Range<K>>;
130 :
131 : /// Functions needed from all layers.
132 : pub trait CompactionLayer<K: CompactionKey + ?Sized> {
133 : fn key_range(&self) -> &Range<K>;
134 : fn lsn_range(&self) -> &Range<Lsn>;
135 :
136 : fn file_size(&self) -> u64;
137 :
138 : /// For debugging, short human-readable representation of the layer. E.g. filename.
139 : fn short_id(&self) -> String;
140 :
141 : fn is_delta(&self) -> bool;
142 : }
143 : pub trait CompactionDeltaLayer<E: CompactionJobExecutor + ?Sized>: CompactionLayer<E::Key> {
144 : type DeltaEntry<'a>: CompactionDeltaEntry<'a, E::Key>
145 : where
146 : Self: 'a;
147 :
148 : /// Return all keys in this delta layer.
149 : fn load_keys<'a>(
150 : &self,
151 : ctx: &E::RequestContext,
152 : ) -> impl Future<Output = anyhow::Result<Vec<Self::DeltaEntry<'_>>>> + Send;
153 : }
154 :
155 : pub trait CompactionImageLayer<E: CompactionJobExecutor + ?Sized>: CompactionLayer<E::Key> {}
156 :
157 : pub trait CompactionDeltaEntry<'a, K> {
158 : fn key(&self) -> K;
159 : fn lsn(&self) -> Lsn;
160 : fn size(&self) -> u64;
161 : }
162 :
163 : pub trait CompactionRequestContext {}
|
