LCOV - code coverage report
Current view: top level - pageserver/compaction/src - interface.rs (source / functions) Coverage Total Hit
Test: 1b0a6a0c05cee5a7de360813c8034804e105ce1c.info Lines: 0.0 % 9 0
Test Date: 2025-03-12 00:01:28 Functions: 0.0 % 3 0

            Line data    Source code
       1              : //! This is what the compaction implementation needs to know about
       2              : //! layers, keyspace etc.
       3              : //!
       4              : //! All the heavy lifting is done by the create_image and create_delta
       5              : //! functions that the implementor provides.
       6              : use std::ops::Range;
       7              : 
       8              : use futures::Future;
       9              : use pageserver_api::key::Key;
      10              : use pageserver_api::keyspace::ShardedRange;
      11              : use pageserver_api::shard::ShardIdentity;
      12              : use utils::lsn::Lsn;
      13              : 
      14              : /// Public interface. This is the main thing that the implementor needs to provide
      15              : pub trait CompactionJobExecutor {
      16              :     // Type system.
      17              :     //
      18              :     // We assume that there are two kinds of layers, deltas and images. The
      19              :     // compaction doesn't distinguish whether they are stored locally or
      20              :     // remotely.
      21              :     //
      22              :     // The keyspace is defined by the CompactionKey trait.
      23              :     type Key: CompactionKey;
      24              : 
      25              :     type Layer: CompactionLayer<Self::Key> + Clone;
      26              :     type DeltaLayer: CompactionDeltaLayer<Self> + Clone;
      27              :     type ImageLayer: CompactionImageLayer<Self> + Clone;
      28              : 
      29              :     // This is passed through to all the interface functions. The compaction
      30              :     // implementation doesn't do anything with it, but it might be useful for
      31              :     // the interface implementation.
      32              :     type RequestContext: CompactionRequestContext;
      33              : 
      34              :     // ----
      35              :     // Functions that the planner uses to support its decisions
      36              :     // ----
      37              : 
      38              :     fn get_shard_identity(&self) -> &ShardIdentity;
      39              : 
      40              :     /// Return all layers that overlap the given bounding box.
      41              :     fn get_layers(
      42              :         &mut self,
      43              :         key_range: &Range<Self::Key>,
      44              :         lsn_range: &Range<Lsn>,
      45              :         ctx: &Self::RequestContext,
      46              :     ) -> impl Future<Output = anyhow::Result<Vec<Self::Layer>>> + Send;
      47              : 
      48              :     fn get_keyspace(
      49              :         &mut self,
      50              :         key_range: &Range<Self::Key>,
      51              :         lsn: Lsn,
      52              :         ctx: &Self::RequestContext,
      53              :     ) -> impl Future<Output = anyhow::Result<CompactionKeySpace<Self::Key>>> + Send;
      54              : 
      55              :     /// NB: This is a pretty expensive operation. In the real pageserver
      56              :     /// implementation, it downloads the layer, and keeps it resident
      57              :     /// until the DeltaLayer is dropped.
      58              :     fn downcast_delta_layer(
      59              :         &self,
      60              :         layer: &Self::Layer,
      61              :         ctx: &Self::RequestContext,
      62              :     ) -> impl Future<Output = anyhow::Result<Option<Self::DeltaLayer>>> + Send;
      63              : 
      64              :     // ----
      65              :     // Functions to execute the plan
      66              :     // ----
      67              : 
      68              :     /// Create a new image layer, materializing all the values in the key range,
      69              :     /// at given 'lsn'.
      70              :     fn create_image(
      71              :         &mut self,
      72              :         lsn: Lsn,
      73              :         key_range: &Range<Self::Key>,
      74              :         ctx: &Self::RequestContext,
      75              :     ) -> impl Future<Output = anyhow::Result<()>> + Send;
      76              : 
      77              :     /// Create a new delta layer, containing all the values from 'input_layers'
      78              :     /// in the given key and LSN range.
      79              :     fn create_delta(
      80              :         &mut self,
      81              :         lsn_range: &Range<Lsn>,
      82              :         key_range: &Range<Self::Key>,
      83              :         input_layers: &[Self::DeltaLayer],
      84              :         ctx: &Self::RequestContext,
      85              :     ) -> impl Future<Output = anyhow::Result<()>> + Send;
      86              : 
      87              :     /// Delete a layer. The compaction implementation will call this only after
      88              :     /// all the create_image() or create_delta() calls that deletion of this
      89              :     /// layer depends on have finished. But if the implementor has extra lazy
      90              :     /// background tasks, like uploading the index json file to remote storage.
      91              :     /// it is the implementation's responsibility to track those.
      92              :     fn delete_layer(
      93              :         &mut self,
      94              :         layer: &Self::Layer,
      95              :         ctx: &Self::RequestContext,
      96              :     ) -> impl Future<Output = anyhow::Result<()>> + Send;
      97              : }
      98              : 
      99              : pub trait CompactionKey: std::cmp::Ord + Clone + Copy + std::fmt::Display {
     100              :     const MIN: Self;
     101              :     const MAX: Self;
     102              : 
     103              :     /// Calculate distance between key_range.start and key_range.end.
     104              :     ///
     105              :     /// This returns u32, for compatibility with Repository::key. If the
     106              :     /// distance is larger, return u32::MAX.
     107              :     fn key_range_size(key_range: &Range<Self>, shard_identity: &ShardIdentity) -> u32;
     108              : 
     109              :     // return "self + 1"
     110              :     fn next(&self) -> Self;
     111              : 
     112              :     // return "self + <some decent amount to skip>". The amount to skip
     113              :     // is left to the implementation.
     114              :     // FIXME: why not just "add(u32)" ?  This is hard to use
     115              :     fn skip_some(&self) -> Self;
     116              : }
     117              : 
     118              : impl CompactionKey for Key {
     119              :     const MIN: Self = Self::MIN;
     120              :     const MAX: Self = Self::MAX;
     121              : 
     122            0 :     fn key_range_size(r: &std::ops::Range<Self>, shard_identity: &ShardIdentity) -> u32 {
     123            0 :         ShardedRange::new(r.clone(), shard_identity).page_count()
     124            0 :     }
     125            0 :     fn next(&self) -> Key {
     126            0 :         (self as &Key).next()
     127            0 :     }
     128            0 :     fn skip_some(&self) -> Key {
     129            0 :         self.add(128)
     130            0 :     }
     131              : }
     132              : 
     133              : /// Contiguous ranges of keys that belong to the key space. In key order, and
     134              : /// with no overlap.
     135              : pub type CompactionKeySpace<K> = Vec<Range<K>>;
     136              : 
     137              : /// Functions needed from all layers.
     138              : pub trait CompactionLayer<K: CompactionKey> {
     139              :     fn key_range(&self) -> &Range<K>;
     140              :     fn lsn_range(&self) -> &Range<Lsn>;
     141              : 
     142              :     fn file_size(&self) -> u64;
     143              : 
     144              :     /// For debugging, short human-readable representation of the layer. E.g. filename.
     145              :     fn short_id(&self) -> String;
     146              : 
     147              :     fn is_delta(&self) -> bool;
     148              : }
     149              : pub trait CompactionDeltaLayer<E: CompactionJobExecutor + ?Sized>: CompactionLayer<E::Key> {
     150              :     type DeltaEntry<'a>: CompactionDeltaEntry<'a, E::Key>
     151              :     where
     152              :         Self: 'a;
     153              : 
     154              :     /// Return all keys in this delta layer.
     155              :     fn load_keys(
     156              :         &self,
     157              :         ctx: &E::RequestContext,
     158              :     ) -> impl Future<Output = anyhow::Result<Vec<Self::DeltaEntry<'_>>>> + Send;
     159              : }
     160              : 
     161              : pub trait CompactionImageLayer<E: CompactionJobExecutor + ?Sized>: CompactionLayer<E::Key> {}
     162              : 
     163              : pub trait CompactionDeltaEntry<'a, K> {
     164              :     fn key(&self) -> K;
     165              :     fn lsn(&self) -> Lsn;
     166              :     fn size(&self) -> u64;
     167              : }
     168              : 
     169              : pub trait CompactionRequestContext {}
        

Generated by: LCOV version 2.1-beta