Line data Source code
1 : //! This is what the compaction implementation needs to know about
2 : //! layers, keyspace etc.
3 : //!
4 : //! All the heavy lifting is done by the create_image and create_delta
5 : //! functions that the implementor provides.
6 : use async_trait::async_trait;
7 : use futures::Future;
8 : use pageserver_api::{key::Key, keyspace::key_range_size};
9 : use std::ops::Range;
10 : use utils::lsn::Lsn;
11 :
12 : /// Public interface. This is the main thing that the implementor needs to provide
13 : pub trait CompactionJobExecutor {
14 : // Type system.
15 : //
16 : // We assume that there are two kinds of layers, deltas and images. The
17 : // compaction doesn't distinguish whether they are stored locally or
18 : // remotely.
19 : //
20 : // The keyspace is defined by the CompactionKey trait.
21 : type Key: CompactionKey;
22 :
23 : type Layer: CompactionLayer<Self::Key> + Clone;
24 : type DeltaLayer: CompactionDeltaLayer<Self> + Clone;
25 : type ImageLayer: CompactionImageLayer<Self> + Clone;
26 :
27 : // This is passed through to all the interface functions. The compaction
28 : // implementation doesn't do anything with it, but it might be useful for
29 : // the interface implementation.
30 : type RequestContext: CompactionRequestContext;
31 :
32 : // ----
33 : // Functions that the planner uses to support its decisions
34 : // ----
35 :
36 : /// Return all layers that overlap the given bounding box.
37 : fn get_layers(
38 : &mut self,
39 : key_range: &Range<Self::Key>,
40 : lsn_range: &Range<Lsn>,
41 : ctx: &Self::RequestContext,
42 : ) -> impl Future<Output = anyhow::Result<Vec<Self::Layer>>> + Send;
43 :
44 : fn get_keyspace(
45 : &mut self,
46 : key_range: &Range<Self::Key>,
47 : lsn: Lsn,
48 : ctx: &Self::RequestContext,
49 : ) -> impl Future<Output = anyhow::Result<CompactionKeySpace<Self::Key>>> + Send;
50 :
51 : /// NB: This is a pretty expensive operation. In the real pageserver
52 : /// implementation, it downloads the layer, and keeps it resident
53 : /// until the DeltaLayer is dropped.
54 : fn downcast_delta_layer(
55 : &self,
56 : layer: &Self::Layer,
57 : ) -> impl Future<Output = anyhow::Result<Option<Self::DeltaLayer>>> + Send;
58 :
59 : // ----
60 : // Functions to execute the plan
61 : // ----
62 :
63 : /// Create a new image layer, materializing all the values in the key range,
64 : /// at given 'lsn'.
65 : fn create_image(
66 : &mut self,
67 : lsn: Lsn,
68 : key_range: &Range<Self::Key>,
69 : ctx: &Self::RequestContext,
70 : ) -> impl Future<Output = anyhow::Result<()>> + Send;
71 :
72 : /// Create a new delta layer, containing all the values from 'input_layers'
73 : /// in the given key and LSN range.
74 : fn create_delta(
75 : &mut self,
76 : lsn_range: &Range<Lsn>,
77 : key_range: &Range<Self::Key>,
78 : input_layers: &[Self::DeltaLayer],
79 : ctx: &Self::RequestContext,
80 : ) -> impl Future<Output = anyhow::Result<()>> + Send;
81 :
82 : /// Delete a layer. The compaction implementation will call this only after
83 : /// all the create_image() or create_delta() calls that deletion of this
84 : /// layer depends on have finished. But if the implementor has extra lazy
85 : /// background tasks, like uploading the index json file to remote storage.
86 : /// it is the implementation's responsibility to track those.
87 : fn delete_layer(
88 : &mut self,
89 : layer: &Self::Layer,
90 : ctx: &Self::RequestContext,
91 : ) -> impl Future<Output = anyhow::Result<()>> + Send;
92 : }
93 :
94 : pub trait CompactionKey: std::cmp::Ord + Clone + Copy + std::fmt::Display {
95 : const MIN: Self;
96 : const MAX: Self;
97 :
98 : /// Calculate distance between key_range.start and key_range.end.
99 : ///
100 : /// This returns u32, for compatibility with Repository::key. If the
101 : /// distance is larger, return u32::MAX.
102 : fn key_range_size(key_range: &Range<Self>) -> u32;
103 :
104 : // return "self + 1"
105 : fn next(&self) -> Self;
106 :
107 : // return "self + <some decent amount to skip>". The amount to skip
108 : // is left to the implementation.
109 : // FIXME: why not just "add(u32)" ? This is hard to use
110 : fn skip_some(&self) -> Self;
111 : }
112 :
113 : impl CompactionKey for Key {
114 : const MIN: Self = Self::MIN;
115 : const MAX: Self = Self::MAX;
116 :
117 0 : fn key_range_size(r: &std::ops::Range<Self>) -> u32 {
118 0 : key_range_size(r)
119 0 : }
120 0 : fn next(&self) -> Key {
121 0 : (self as &Key).next()
122 0 : }
123 0 : fn skip_some(&self) -> Key {
124 0 : self.add(128)
125 0 : }
126 : }
127 :
128 : /// Contiguous ranges of keys that belong to the key space. In key order, and
129 : /// with no overlap.
130 : pub type CompactionKeySpace<K> = Vec<Range<K>>;
131 :
132 : /// Functions needed from all layers.
133 : pub trait CompactionLayer<K: CompactionKey + ?Sized> {
134 : fn key_range(&self) -> &Range<K>;
135 : fn lsn_range(&self) -> &Range<Lsn>;
136 :
137 : fn file_size(&self) -> u64;
138 :
139 : /// For debugging, short human-readable representation of the layer. E.g. filename.
140 : fn short_id(&self) -> String;
141 :
142 : fn is_delta(&self) -> bool;
143 : }
144 :
145 : #[async_trait]
146 : pub trait CompactionDeltaLayer<E: CompactionJobExecutor + ?Sized>: CompactionLayer<E::Key> {
147 : type DeltaEntry<'a>: CompactionDeltaEntry<'a, E::Key>
148 : where
149 : Self: 'a;
150 :
151 : /// Return all keys in this delta layer.
152 : async fn load_keys<'a>(
153 : &self,
154 : ctx: &E::RequestContext,
155 : ) -> anyhow::Result<Vec<Self::DeltaEntry<'_>>>;
156 : }
157 :
158 : pub trait CompactionImageLayer<E: CompactionJobExecutor + ?Sized>: CompactionLayer<E::Key> {}
159 :
160 : pub trait CompactionDeltaEntry<'a, K> {
161 : fn key(&self) -> K;
162 : fn lsn(&self) -> Lsn;
163 : fn size(&self) -> u64;
164 : }
165 :
166 : pub trait CompactionRequestContext {}
|
