Line data Source code
1 : //! This is what the compaction implementation needs to know about
2 : //! layers, keyspace etc.
3 : //!
4 : //! All the heavy lifting is done by the create_image and create_delta
5 : //! functions that the implementor provides.
6 : use std::ops::Range;
7 :
8 : use futures::Future;
9 : use pageserver_api::key::Key;
10 : use pageserver_api::keyspace::ShardedRange;
11 : use pageserver_api::shard::ShardIdentity;
12 : use utils::lsn::Lsn;
13 :
14 : /// Public interface. This is the main thing that the implementor needs to provide
15 : pub trait CompactionJobExecutor {
16 : // Type system.
17 : //
18 : // We assume that there are two kinds of layers, deltas and images. The
19 : // compaction doesn't distinguish whether they are stored locally or
20 : // remotely.
21 : //
22 : // The keyspace is defined by the CompactionKey trait.
23 : type Key: CompactionKey;
24 :
25 : type Layer: CompactionLayer<Self::Key> + Clone;
26 : type DeltaLayer: CompactionDeltaLayer<Self> + Clone;
27 : type ImageLayer: CompactionImageLayer<Self> + Clone;
28 :
29 : // This is passed through to all the interface functions. The compaction
30 : // implementation doesn't do anything with it, but it might be useful for
31 : // the interface implementation.
32 : type RequestContext: CompactionRequestContext;
33 :
34 : // ----
35 : // Functions that the planner uses to support its decisions
36 : // ----
37 :
38 : fn get_shard_identity(&self) -> &ShardIdentity;
39 :
40 : /// Return all layers that overlap the given bounding box.
41 : fn get_layers(
42 : &mut self,
43 : key_range: &Range<Self::Key>,
44 : lsn_range: &Range<Lsn>,
45 : ctx: &Self::RequestContext,
46 : ) -> impl Future<Output = anyhow::Result<Vec<Self::Layer>>> + Send;
47 :
48 : fn get_keyspace(
49 : &mut self,
50 : key_range: &Range<Self::Key>,
51 : lsn: Lsn,
52 : ctx: &Self::RequestContext,
53 : ) -> impl Future<Output = anyhow::Result<CompactionKeySpace<Self::Key>>> + Send;
54 :
55 : /// NB: This is a pretty expensive operation. In the real pageserver
56 : /// implementation, it downloads the layer, and keeps it resident
57 : /// until the DeltaLayer is dropped.
58 : fn downcast_delta_layer(
59 : &self,
60 : layer: &Self::Layer,
61 : ctx: &Self::RequestContext,
62 : ) -> impl Future<Output = anyhow::Result<Option<Self::DeltaLayer>>> + Send;
63 :
64 : // ----
65 : // Functions to execute the plan
66 : // ----
67 :
68 : /// Create a new image layer, materializing all the values in the key range,
69 : /// at given 'lsn'.
70 : fn create_image(
71 : &mut self,
72 : lsn: Lsn,
73 : key_range: &Range<Self::Key>,
74 : ctx: &Self::RequestContext,
75 : ) -> impl Future<Output = anyhow::Result<()>> + Send;
76 :
77 : /// Create a new delta layer, containing all the values from 'input_layers'
78 : /// in the given key and LSN range.
79 : fn create_delta(
80 : &mut self,
81 : lsn_range: &Range<Lsn>,
82 : key_range: &Range<Self::Key>,
83 : input_layers: &[Self::DeltaLayer],
84 : ctx: &Self::RequestContext,
85 : ) -> impl Future<Output = anyhow::Result<()>> + Send;
86 :
87 : /// Delete a layer. The compaction implementation will call this only after
88 : /// all the create_image() or create_delta() calls that deletion of this
89 : /// layer depends on have finished. But if the implementor has extra lazy
90 : /// background tasks, like uploading the index json file to remote storage.
91 : /// it is the implementation's responsibility to track those.
92 : fn delete_layer(
93 : &mut self,
94 : layer: &Self::Layer,
95 : ctx: &Self::RequestContext,
96 : ) -> impl Future<Output = anyhow::Result<()>> + Send;
97 : }
98 :
99 : pub trait CompactionKey: std::cmp::Ord + Clone + Copy + std::fmt::Display {
100 : const MIN: Self;
101 : const MAX: Self;
102 :
103 : /// Calculate distance between key_range.start and key_range.end.
104 : ///
105 : /// This returns u32, for compatibility with Repository::key. If the
106 : /// distance is larger, return u32::MAX.
107 : fn key_range_size(key_range: &Range<Self>, shard_identity: &ShardIdentity) -> u32;
108 :
109 : // return "self + 1"
110 : fn next(&self) -> Self;
111 :
112 : // return "self + <some decent amount to skip>". The amount to skip
113 : // is left to the implementation.
114 : // FIXME: why not just "add(u32)" ? This is hard to use
115 : fn skip_some(&self) -> Self;
116 : }
117 :
118 : impl CompactionKey for Key {
119 : const MIN: Self = Self::MIN;
120 : const MAX: Self = Self::MAX;
121 :
122 0 : fn key_range_size(r: &std::ops::Range<Self>, shard_identity: &ShardIdentity) -> u32 {
123 0 : ShardedRange::new(r.clone(), shard_identity).page_count()
124 0 : }
125 0 : fn next(&self) -> Key {
126 0 : (self as &Key).next()
127 0 : }
128 0 : fn skip_some(&self) -> Key {
129 0 : self.add(128)
130 0 : }
131 : }
132 :
133 : /// Contiguous ranges of keys that belong to the key space. In key order, and
134 : /// with no overlap.
135 : pub type CompactionKeySpace<K> = Vec<Range<K>>;
136 :
137 : /// Functions needed from all layers.
138 : pub trait CompactionLayer<K: CompactionKey> {
139 : fn key_range(&self) -> &Range<K>;
140 : fn lsn_range(&self) -> &Range<Lsn>;
141 :
142 : fn file_size(&self) -> u64;
143 :
144 : /// For debugging, short human-readable representation of the layer. E.g. filename.
145 : fn short_id(&self) -> String;
146 :
147 : fn is_delta(&self) -> bool;
148 : }
149 : pub trait CompactionDeltaLayer<E: CompactionJobExecutor + ?Sized>: CompactionLayer<E::Key> {
150 : type DeltaEntry<'a>: CompactionDeltaEntry<'a, E::Key>
151 : where
152 : Self: 'a;
153 :
154 : /// Return all keys in this delta layer.
155 : fn load_keys(
156 : &self,
157 : ctx: &E::RequestContext,
158 : ) -> impl Future<Output = anyhow::Result<Vec<Self::DeltaEntry<'_>>>> + Send;
159 : }
160 :
161 : pub trait CompactionImageLayer<E: CompactionJobExecutor + ?Sized>: CompactionLayer<E::Key> {}
162 :
163 : pub trait CompactionDeltaEntry<'a, K> {
164 : fn key(&self) -> K;
165 : fn lsn(&self) -> Lsn;
166 : fn size(&self) -> u64;
167 : }
168 :
169 : pub trait CompactionRequestContext {}
|
