Line data Source code
1 : //! This is what the compaction implementation needs to know about
2 : //! layers, keyspace etc.
3 : //!
4 : //! All the heavy lifting is done by the create_image and create_delta
5 : //! functions that the implementor provides.
6 : use async_trait::async_trait;
7 : use pageserver_api::{key::Key, keyspace::key_range_size};
8 : use std::ops::Range;
9 : use utils::lsn::Lsn;
10 :
11 : /// Public interface. This is the main thing that the implementor needs to provide
12 : #[async_trait]
13 : pub trait CompactionJobExecutor {
14 : // Type system.
15 : //
16 : // We assume that there are two kinds of layers, deltas and images. The
17 : // compaction doesn't distinguish whether they are stored locally or
18 : // remotely.
19 : //
20 : // The keyspace is defined by CompactionKey trait.
21 : //
22 : type Key: CompactionKey;
23 :
24 : type Layer: CompactionLayer<Self::Key> + Clone;
25 : type DeltaLayer: CompactionDeltaLayer<Self> + Clone;
26 : type ImageLayer: CompactionImageLayer<Self> + Clone;
27 :
28 : // This is passed through to all the interface functions. The compaction
29 : // implementation doesn't do anything with it, but it might be useful for
30 : // the interface implementation.
31 : type RequestContext: CompactionRequestContext;
32 :
33 : // ----
34 : // Functions that the planner uses to support its decisions
35 : // ----
36 :
37 : /// Return all layers that overlap the given bounding box.
38 : async fn get_layers(
39 : &mut self,
40 : key_range: &Range<Self::Key>,
41 : lsn_range: &Range<Lsn>,
42 : ctx: &Self::RequestContext,
43 : ) -> anyhow::Result<Vec<Self::Layer>>;
44 :
45 : async fn get_keyspace(
46 : &mut self,
47 : key_range: &Range<Self::Key>,
48 : lsn: Lsn,
49 : ctx: &Self::RequestContext,
50 : ) -> anyhow::Result<CompactionKeySpace<Self::Key>>;
51 :
52 : /// NB: This is a pretty expensive operation. In the real pageserver
53 : /// implementation, it downloads the layer, and keeps it resident
54 : /// until the DeltaLayer is dropped.
55 : async fn downcast_delta_layer(
56 : &self,
57 : layer: &Self::Layer,
58 : ) -> anyhow::Result<Option<Self::DeltaLayer>>;
59 :
60 : // ----
61 : // Functions to execute the plan
62 : // ----
63 :
64 : /// Create a new image layer, materializing all the values in the key range,
65 : /// at given 'lsn'.
66 : async fn create_image(
67 : &mut self,
68 : lsn: Lsn,
69 : key_range: &Range<Self::Key>,
70 : ctx: &Self::RequestContext,
71 : ) -> anyhow::Result<()>;
72 :
73 : /// Create a new delta layer, containing all the values from 'input_layers'
74 : /// in the given key and LSN range.
75 : async fn create_delta(
76 : &mut self,
77 : lsn_range: &Range<Lsn>,
78 : key_range: &Range<Self::Key>,
79 : input_layers: &[Self::DeltaLayer],
80 : ctx: &Self::RequestContext,
81 : ) -> anyhow::Result<()>;
82 :
83 : /// Delete a layer. The compaction implementation will call this only after
84 : /// all the create_image() or create_delta() calls that deletion of this
85 : /// layer depends on have finished. But if the implementor has extra lazy
86 : /// background tasks, like uploading the index json file to remote storage,
87 : /// it is the implementation's responsibility to track those.
88 : async fn delete_layer(
89 : &mut self,
90 : layer: &Self::Layer,
91 : ctx: &Self::RequestContext,
92 : ) -> anyhow::Result<()>;
93 : }
94 :
95 : pub trait CompactionKey: std::cmp::Ord + Clone + Copy + std::fmt::Display {
96 : const MIN: Self;
97 : const MAX: Self;
98 :
99 : /// Calculate distance between key_range.start and key_range.end.
100 : ///
101 : /// This returns u32, for compatibility with Repository::key. If the
102 : /// distance is larger, return u32::MAX.
103 : fn key_range_size(key_range: &Range<Self>) -> u32;
104 :
105 : // return "self + 1"
106 : fn next(&self) -> Self;
107 :
108 : // return "self + <some decent amount to skip>". The amount to skip
109 : // is left to the implementation.
110 : // FIXME: why not just "add(u32)" ? This is hard to use
111 : fn skip_some(&self) -> Self;
112 : }
113 :
114 : impl CompactionKey for Key {
115 : const MIN: Self = Self::MIN;
116 : const MAX: Self = Self::MAX;
117 :
118 0 : fn key_range_size(r: &std::ops::Range<Self>) -> u32 {
119 0 : key_range_size(r)
120 0 : }
121 0 : fn next(&self) -> Key {
122 0 : (self as &Key).next()
123 0 : }
124 0 : fn skip_some(&self) -> Key {
125 0 : self.add(128)
126 0 : }
127 : }
128 :
129 : /// Contiguous ranges of keys that belong to the key space. In key order, and
130 : /// with no overlap.
131 : pub type CompactionKeySpace<K> = Vec<Range<K>>;
132 :
133 : /// Functions needed from all layers.
134 : pub trait CompactionLayer<K: CompactionKey + ?Sized> {
135 : fn key_range(&self) -> &Range<K>;
136 : fn lsn_range(&self) -> &Range<Lsn>;
137 :
138 : fn file_size(&self) -> u64;
139 :
140 : /// For debugging, short human-readable representation of the layer. E.g. filename.
141 : fn short_id(&self) -> String;
142 :
143 : fn is_delta(&self) -> bool;
144 : }
145 :
146 : #[async_trait]
147 : pub trait CompactionDeltaLayer<E: CompactionJobExecutor + ?Sized>: CompactionLayer<E::Key> {
148 : type DeltaEntry<'a>: CompactionDeltaEntry<'a, E::Key>
149 : where
150 : Self: 'a;
151 :
152 : /// Return all keys in this delta layer.
153 : async fn load_keys<'a>(
154 : &self,
155 : ctx: &E::RequestContext,
156 : ) -> anyhow::Result<Vec<Self::DeltaEntry<'_>>>;
157 : }
158 :
159 : pub trait CompactionImageLayer<E: CompactionJobExecutor + ?Sized>: CompactionLayer<E::Key> {}
160 :
161 : pub trait CompactionDeltaEntry<'a, K> {
162 : fn key(&self) -> K;
163 : fn lsn(&self) -> Lsn;
164 : fn size(&self) -> u64;
165 : }
166 :
167 : pub trait CompactionRequestContext {}
|
