LCOV - code coverage report
Current view: top level - pageserver/src - tenant.rs (source / functions) Coverage Total Hit
Test: 4f58e98c51285c7fa348e0b410c88a10caf68ad2.info Lines: 75.7 % 8350 6319
Test Date: 2025-01-07 20:58:07 Functions: 60.0 % 438 263

            Line data    Source code
       1              : //! Timeline repository implementation that keeps old data in layer files, and
       2              : //! the recent changes in ephemeral files.
       3              : //!
       4              : //! See tenant/*_layer.rs files. The functions here are responsible for locating
       5              : //! the correct layer for the get/put call, walking back the timeline branching
       6              : //! history as needed.
       7              : //!
       8              : //! The files are stored in the .neon/tenants/<tenant_id>/timelines/<timeline_id>
       9              : //! directory. See docs/pageserver-storage.md for how the files are managed.
      10              : //! In addition to the layer files, there is a metadata file in the same
      11              : //! directory that contains information about the timeline, in particular its
      12              : //! parent timeline, and the last LSN that has been written to disk.
      13              : //!
      14              : 
      15              : use anyhow::{bail, Context};
      16              : use arc_swap::ArcSwap;
      17              : use camino::Utf8Path;
      18              : use camino::Utf8PathBuf;
      19              : use chrono::NaiveDateTime;
      20              : use enumset::EnumSet;
      21              : use futures::stream::FuturesUnordered;
      22              : use futures::StreamExt;
      23              : use pageserver_api::models;
      24              : use pageserver_api::models::LsnLease;
      25              : use pageserver_api::models::TimelineArchivalState;
      26              : use pageserver_api::models::TimelineState;
      27              : use pageserver_api::models::TopTenantShardItem;
      28              : use pageserver_api::models::WalRedoManagerStatus;
      29              : use pageserver_api::shard::ShardIdentity;
      30              : use pageserver_api::shard::ShardStripeSize;
      31              : use pageserver_api::shard::TenantShardId;
      32              : use remote_storage::DownloadError;
      33              : use remote_storage::GenericRemoteStorage;
      34              : use remote_storage::TimeoutOrCancel;
      35              : use remote_timeline_client::manifest::{
      36              :     OffloadedTimelineManifest, TenantManifest, LATEST_TENANT_MANIFEST_VERSION,
      37              : };
      38              : use remote_timeline_client::UploadQueueNotReadyError;
      39              : use std::collections::BTreeMap;
      40              : use std::collections::VecDeque;
      41              : use std::fmt;
      42              : use std::future::Future;
      43              : use std::sync::atomic::AtomicBool;
      44              : use std::sync::Weak;
      45              : use std::time::SystemTime;
      46              : use storage_broker::BrokerClientChannel;
      47              : use timeline::compaction::GcCompactJob;
      48              : use timeline::compaction::ScheduledCompactionTask;
      49              : use timeline::import_pgdata;
      50              : use timeline::offload::offload_timeline;
      51              : use timeline::CompactFlags;
      52              : use timeline::CompactOptions;
      53              : use timeline::CompactionError;
      54              : use timeline::ShutdownMode;
      55              : use tokio::io::BufReader;
      56              : use tokio::sync::watch;
      57              : use tokio::task::JoinSet;
      58              : use tokio_util::sync::CancellationToken;
      59              : use tracing::*;
      60              : use upload_queue::NotInitialized;
      61              : use utils::backoff;
      62              : use utils::circuit_breaker::CircuitBreaker;
      63              : use utils::completion;
      64              : use utils::crashsafe::path_with_suffix_extension;
      65              : use utils::failpoint_support;
      66              : use utils::fs_ext;
      67              : use utils::pausable_failpoint;
      68              : use utils::sync::gate::Gate;
      69              : use utils::sync::gate::GateGuard;
      70              : use utils::timeout::timeout_cancellable;
      71              : use utils::timeout::TimeoutCancellableError;
      72              : use utils::try_rcu::ArcSwapExt;
      73              : use utils::zstd::create_zst_tarball;
      74              : use utils::zstd::extract_zst_tarball;
      75              : 
      76              : use self::config::AttachedLocationConfig;
      77              : use self::config::AttachmentMode;
      78              : use self::config::LocationConf;
      79              : use self::config::TenantConf;
      80              : use self::metadata::TimelineMetadata;
      81              : use self::mgr::GetActiveTenantError;
      82              : use self::mgr::GetTenantError;
      83              : use self::remote_timeline_client::upload::{upload_index_part, upload_tenant_manifest};
      84              : use self::remote_timeline_client::{RemoteTimelineClient, WaitCompletionError};
      85              : use self::timeline::uninit::TimelineCreateGuard;
      86              : use self::timeline::uninit::TimelineExclusionError;
      87              : use self::timeline::uninit::UninitializedTimeline;
      88              : use self::timeline::EvictionTaskTenantState;
      89              : use self::timeline::GcCutoffs;
      90              : use self::timeline::TimelineDeleteProgress;
      91              : use self::timeline::TimelineResources;
      92              : use self::timeline::WaitLsnError;
      93              : use crate::config::PageServerConf;
      94              : use crate::context::{DownloadBehavior, RequestContext};
      95              : use crate::deletion_queue::DeletionQueueClient;
      96              : use crate::deletion_queue::DeletionQueueError;
      97              : use crate::import_datadir;
      98              : use crate::is_uninit_mark;
      99              : use crate::l0_flush::L0FlushGlobalState;
     100              : use crate::metrics::TENANT;
     101              : use crate::metrics::{
     102              :     remove_tenant_metrics, BROKEN_TENANTS_SET, CIRCUIT_BREAKERS_BROKEN, CIRCUIT_BREAKERS_UNBROKEN,
     103              :     TENANT_STATE_METRIC, TENANT_SYNTHETIC_SIZE_METRIC,
     104              : };
     105              : use crate::task_mgr;
     106              : use crate::task_mgr::TaskKind;
     107              : use crate::tenant::config::LocationMode;
     108              : use crate::tenant::config::TenantConfOpt;
     109              : use crate::tenant::gc_result::GcResult;
     110              : pub use crate::tenant::remote_timeline_client::index::IndexPart;
     111              : use crate::tenant::remote_timeline_client::remote_initdb_archive_path;
     112              : use crate::tenant::remote_timeline_client::MaybeDeletedIndexPart;
     113              : use crate::tenant::remote_timeline_client::INITDB_PATH;
     114              : use crate::tenant::storage_layer::DeltaLayer;
     115              : use crate::tenant::storage_layer::ImageLayer;
     116              : use crate::walingest::WalLagCooldown;
     117              : use crate::walredo;
     118              : use crate::InitializationOrder;
     119              : use std::collections::hash_map::Entry;
     120              : use std::collections::HashMap;
     121              : use std::collections::HashSet;
     122              : use std::fmt::Debug;
     123              : use std::fmt::Display;
     124              : use std::fs;
     125              : use std::fs::File;
     126              : use std::sync::atomic::{AtomicU64, Ordering};
     127              : use std::sync::Arc;
     128              : use std::sync::Mutex;
     129              : use std::time::{Duration, Instant};
     130              : 
     131              : use crate::span;
     132              : use crate::tenant::timeline::delete::DeleteTimelineFlow;
     133              : use crate::tenant::timeline::uninit::cleanup_timeline_directory;
     134              : use crate::virtual_file::VirtualFile;
     135              : use crate::walredo::PostgresRedoManager;
     136              : use crate::TEMP_FILE_SUFFIX;
     137              : use once_cell::sync::Lazy;
     138              : pub use pageserver_api::models::TenantState;
     139              : use tokio::sync::Semaphore;
     140              : 
     141            0 : static INIT_DB_SEMAPHORE: Lazy<Semaphore> = Lazy::new(|| Semaphore::new(8));
     142              : use utils::{
     143              :     crashsafe,
     144              :     generation::Generation,
     145              :     id::TimelineId,
     146              :     lsn::{Lsn, RecordLsn},
     147              : };
     148              : 
     149              : pub mod blob_io;
     150              : pub mod block_io;
     151              : pub mod vectored_blob_io;
     152              : 
     153              : pub mod disk_btree;
     154              : pub(crate) mod ephemeral_file;
     155              : pub mod layer_map;
     156              : 
     157              : pub mod metadata;
     158              : pub mod remote_timeline_client;
     159              : pub mod storage_layer;
     160              : 
     161              : pub mod checks;
     162              : pub mod config;
     163              : pub mod mgr;
     164              : pub mod secondary;
     165              : pub mod tasks;
     166              : pub mod upload_queue;
     167              : 
     168              : pub(crate) mod timeline;
     169              : 
     170              : pub mod size;
     171              : 
     172              : mod gc_block;
     173              : mod gc_result;
     174              : pub(crate) mod throttle;
     175              : 
     176              : pub(crate) use crate::span::debug_assert_current_span_has_tenant_and_timeline_id;
     177              : pub(crate) use timeline::{LogicalSizeCalculationCause, PageReconstructError, Timeline};
     178              : 
     179              : // re-export for use in walreceiver
     180              : pub use crate::tenant::timeline::WalReceiverInfo;
     181              : 
     182              : /// The "tenants" part of `tenants/<tenant>/timelines...`
     183              : pub const TENANTS_SEGMENT_NAME: &str = "tenants";
     184              : 
     185              : /// Parts of the `.neon/tenants/<tenant_id>/timelines/<timeline_id>` directory prefix.
     186              : pub const TIMELINES_SEGMENT_NAME: &str = "timelines";
     187              : 
     188              : /// References to shared objects that are passed into each tenant, such
     189              : /// as the shared remote storage client and process initialization state.
     190              : #[derive(Clone)]
     191              : pub struct TenantSharedResources {
     192              :     pub broker_client: storage_broker::BrokerClientChannel,
     193              :     pub remote_storage: GenericRemoteStorage,
     194              :     pub deletion_queue_client: DeletionQueueClient,
     195              :     pub l0_flush_global_state: L0FlushGlobalState,
     196              : }
     197              : 
     198              : /// A [`Tenant`] is really an _attached_ tenant.  The configuration
     199              : /// for an attached tenant is a subset of the [`LocationConf`], represented
     200              : /// in this struct.
     201              : #[derive(Clone)]
     202              : pub(super) struct AttachedTenantConf {
     203              :     tenant_conf: TenantConfOpt,
     204              :     location: AttachedLocationConfig,
     205              :     /// The deadline before which we are blocked from GC so that
     206              :     /// leases have a chance to be renewed.
     207              :     lsn_lease_deadline: Option<tokio::time::Instant>,
     208              : }
     209              : 
     210              : impl AttachedTenantConf {
     211          196 :     fn new(tenant_conf: TenantConfOpt, location: AttachedLocationConfig) -> Self {
     212              :         // Sets a deadline before which we cannot proceed to GC due to lsn lease.
     213              :         //
     214              :         // We do this as the leases mapping are not persisted to disk. By delaying GC by lease
     215              :         // length, we guarantee that all the leases we granted before will have a chance to renew
     216              :         // when we run GC for the first time after restart / transition from AttachedMulti to AttachedSingle.
     217          196 :         let lsn_lease_deadline = if location.attach_mode == AttachmentMode::Single {
     218          196 :             Some(
     219          196 :                 tokio::time::Instant::now()
     220          196 :                     + tenant_conf
     221          196 :                         .lsn_lease_length
     222          196 :                         .unwrap_or(LsnLease::DEFAULT_LENGTH),
     223          196 :             )
     224              :         } else {
     225              :             // We don't use `lsn_lease_deadline` to delay GC in AttachedMulti and AttachedStale
     226              :             // because we don't do GC in these modes.
     227            0 :             None
     228              :         };
     229              : 
     230          196 :         Self {
     231          196 :             tenant_conf,
     232          196 :             location,
     233          196 :             lsn_lease_deadline,
     234          196 :         }
     235          196 :     }
     236              : 
     237          196 :     fn try_from(location_conf: LocationConf) -> anyhow::Result<Self> {
     238          196 :         match &location_conf.mode {
     239          196 :             LocationMode::Attached(attach_conf) => {
     240          196 :                 Ok(Self::new(location_conf.tenant_conf, *attach_conf))
     241              :             }
     242              :             LocationMode::Secondary(_) => {
     243            0 :                 anyhow::bail!("Attempted to construct AttachedTenantConf from a LocationConf in secondary mode")
     244              :             }
     245              :         }
     246          196 :     }
     247              : 
     248          762 :     fn is_gc_blocked_by_lsn_lease_deadline(&self) -> bool {
     249          762 :         self.lsn_lease_deadline
     250          762 :             .map(|d| tokio::time::Instant::now() < d)
     251          762 :             .unwrap_or(false)
     252          762 :     }
     253              : }
     254              : struct TimelinePreload {
     255              :     timeline_id: TimelineId,
     256              :     client: RemoteTimelineClient,
     257              :     index_part: Result<MaybeDeletedIndexPart, DownloadError>,
     258              : }
     259              : 
     260              : pub(crate) struct TenantPreload {
     261              :     tenant_manifest: TenantManifest,
     262              :     /// Map from timeline ID to a possible timeline preload. It is None iff the timeline is offloaded according to the manifest.
     263              :     timelines: HashMap<TimelineId, Option<TimelinePreload>>,
     264              : }
     265              : 
     266              : /// When we spawn a tenant, there is a special mode for tenant creation that
     267              : /// avoids trying to read anything from remote storage.
     268              : pub(crate) enum SpawnMode {
     269              :     /// Activate as soon as possible
     270              :     Eager,
     271              :     /// Lazy activation in the background, with the option to skip the queue if the need comes up
     272              :     Lazy,
     273              : }
     274              : 
     275              : ///
     276              : /// Tenant consists of multiple timelines. Keep them in a hash table.
     277              : ///
     278              : pub struct Tenant {
     279              :     // Global pageserver config parameters
     280              :     pub conf: &'static PageServerConf,
     281              : 
     282              :     /// The value creation timestamp, used to measure activation delay, see:
     283              :     /// <https://github.com/neondatabase/neon/issues/4025>
     284              :     constructed_at: Instant,
     285              : 
     286              :     state: watch::Sender<TenantState>,
     287              : 
     288              :     // Overridden tenant-specific config parameters.
     289              :     // We keep TenantConfOpt sturct here to preserve the information
     290              :     // about parameters that are not set.
     291              :     // This is necessary to allow global config updates.
     292              :     tenant_conf: Arc<ArcSwap<AttachedTenantConf>>,
     293              : 
     294              :     tenant_shard_id: TenantShardId,
     295              : 
     296              :     // The detailed sharding information, beyond the number/count in tenant_shard_id
     297              :     shard_identity: ShardIdentity,
     298              : 
     299              :     /// The remote storage generation, used to protect S3 objects from split-brain.
     300              :     /// Does not change over the lifetime of the [`Tenant`] object.
     301              :     ///
     302              :     /// This duplicates the generation stored in LocationConf, but that structure is mutable:
     303              :     /// this copy enforces the invariant that generatio doesn't change during a Tenant's lifetime.
     304              :     generation: Generation,
     305              : 
     306              :     timelines: Mutex<HashMap<TimelineId, Arc<Timeline>>>,
     307              : 
     308              :     /// During timeline creation, we first insert the TimelineId to the
     309              :     /// creating map, then `timelines`, then remove it from the creating map.
     310              :     /// **Lock order**: if acquiring all (or a subset), acquire them in order `timelines`, `timelines_offloaded`, `timelines_creating`
     311              :     timelines_creating: std::sync::Mutex<HashSet<TimelineId>>,
     312              : 
     313              :     /// Possibly offloaded and archived timelines
     314              :     /// **Lock order**: if acquiring all (or a subset), acquire them in order `timelines`, `timelines_offloaded`, `timelines_creating`
     315              :     timelines_offloaded: Mutex<HashMap<TimelineId, Arc<OffloadedTimeline>>>,
     316              : 
     317              :     /// Serialize writes of the tenant manifest to remote storage.  If there are concurrent operations
     318              :     /// affecting the manifest, such as timeline deletion and timeline offload, they must wait for
     319              :     /// each other (this could be optimized to coalesce writes if necessary).
     320              :     ///
     321              :     /// The contents of the Mutex are the last manifest we successfully uploaded
     322              :     tenant_manifest_upload: tokio::sync::Mutex<Option<TenantManifest>>,
     323              : 
     324              :     // This mutex prevents creation of new timelines during GC.
     325              :     // Adding yet another mutex (in addition to `timelines`) is needed because holding
     326              :     // `timelines` mutex during all GC iteration
     327              :     // may block for a long time `get_timeline`, `get_timelines_state`,... and other operations
     328              :     // with timelines, which in turn may cause dropping replication connection, expiration of wait_for_lsn
     329              :     // timeout...
     330              :     gc_cs: tokio::sync::Mutex<()>,
     331              :     walredo_mgr: Option<Arc<WalRedoManager>>,
     332              : 
     333              :     // provides access to timeline data sitting in the remote storage
     334              :     pub(crate) remote_storage: GenericRemoteStorage,
     335              : 
     336              :     // Access to global deletion queue for when this tenant wants to schedule a deletion
     337              :     deletion_queue_client: DeletionQueueClient,
     338              : 
     339              :     /// Cached logical sizes updated updated on each [`Tenant::gather_size_inputs`].
     340              :     cached_logical_sizes: tokio::sync::Mutex<HashMap<(TimelineId, Lsn), u64>>,
     341              :     cached_synthetic_tenant_size: Arc<AtomicU64>,
     342              : 
     343              :     eviction_task_tenant_state: tokio::sync::Mutex<EvictionTaskTenantState>,
     344              : 
     345              :     /// Track repeated failures to compact, so that we can back off.
     346              :     /// Overhead of mutex is acceptable because compaction is done with a multi-second period.
     347              :     compaction_circuit_breaker: std::sync::Mutex<CircuitBreaker>,
     348              : 
     349              :     /// Scheduled compaction tasks. Currently, this can only be populated by triggering
     350              :     /// a manual gc-compaction from the manual compaction API.
     351              :     scheduled_compaction_tasks:
     352              :         std::sync::Mutex<HashMap<TimelineId, VecDeque<ScheduledCompactionTask>>>,
     353              : 
     354              :     /// If the tenant is in Activating state, notify this to encourage it
     355              :     /// to proceed to Active as soon as possible, rather than waiting for lazy
     356              :     /// background warmup.
     357              :     pub(crate) activate_now_sem: tokio::sync::Semaphore,
     358              : 
     359              :     /// Time it took for the tenant to activate. Zero if not active yet.
     360              :     attach_wal_lag_cooldown: Arc<std::sync::OnceLock<WalLagCooldown>>,
     361              : 
     362              :     // Cancellation token fires when we have entered shutdown().  This is a parent of
     363              :     // Timelines' cancellation token.
     364              :     pub(crate) cancel: CancellationToken,
     365              : 
     366              :     // Users of the Tenant such as the page service must take this Gate to avoid
     367              :     // trying to use a Tenant which is shutting down.
     368              :     pub(crate) gate: Gate,
     369              : 
     370              :     /// Throttle applied at the top of [`Timeline::get`].
     371              :     /// All [`Tenant::timelines`] of a given [`Tenant`] instance share the same [`throttle::Throttle`] instance.
     372              :     pub(crate) pagestream_throttle:
     373              :         Arc<throttle::Throttle<crate::metrics::tenant_throttling::Pagestream>>,
     374              : 
     375              :     /// An ongoing timeline detach concurrency limiter.
     376              :     ///
     377              :     /// As a tenant will likely be restarted as part of timeline detach ancestor it makes no sense
     378              :     /// to have two running at the same time. A different one can be started if an earlier one
     379              :     /// has failed for whatever reason.
     380              :     ongoing_timeline_detach: std::sync::Mutex<Option<(TimelineId, utils::completion::Barrier)>>,
     381              : 
     382              :     /// `index_part.json` based gc blocking reason tracking.
     383              :     ///
     384              :     /// New gc iterations must start a new iteration by acquiring `GcBlock::start` before
     385              :     /// proceeding.
     386              :     pub(crate) gc_block: gc_block::GcBlock,
     387              : 
     388              :     l0_flush_global_state: L0FlushGlobalState,
     389              : }
     390              : impl std::fmt::Debug for Tenant {
     391            0 :     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
     392            0 :         write!(f, "{} ({})", self.tenant_shard_id, self.current_state())
     393            0 :     }
     394              : }
     395              : 
     396              : pub(crate) enum WalRedoManager {
     397              :     Prod(WalredoManagerId, PostgresRedoManager),
     398              :     #[cfg(test)]
     399              :     Test(harness::TestRedoManager),
     400              : }
     401              : 
     402              : #[derive(thiserror::Error, Debug)]
     403              : #[error("pageserver is shutting down")]
     404              : pub(crate) struct GlobalShutDown;
     405              : 
     406              : impl WalRedoManager {
     407            0 :     pub(crate) fn new(mgr: PostgresRedoManager) -> Result<Arc<Self>, GlobalShutDown> {
     408            0 :         let id = WalredoManagerId::next();
     409            0 :         let arc = Arc::new(Self::Prod(id, mgr));
     410            0 :         let mut guard = WALREDO_MANAGERS.lock().unwrap();
     411            0 :         match &mut *guard {
     412            0 :             Some(map) => {
     413            0 :                 map.insert(id, Arc::downgrade(&arc));
     414            0 :                 Ok(arc)
     415              :             }
     416            0 :             None => Err(GlobalShutDown),
     417              :         }
     418            0 :     }
     419              : }
     420              : 
     421              : impl Drop for WalRedoManager {
     422           10 :     fn drop(&mut self) {
     423           10 :         match self {
     424            0 :             Self::Prod(id, _) => {
     425            0 :                 let mut guard = WALREDO_MANAGERS.lock().unwrap();
     426            0 :                 if let Some(map) = &mut *guard {
     427            0 :                     map.remove(id).expect("new() registers, drop() unregisters");
     428            0 :                 }
     429              :             }
     430              :             #[cfg(test)]
     431           10 :             Self::Test(_) => {
     432           10 :                 // Not applicable to test redo manager
     433           10 :             }
     434              :         }
     435           10 :     }
     436              : }
     437              : 
     438              : /// Global registry of all walredo managers so that [`crate::shutdown_pageserver`] can shut down
     439              : /// the walredo processes outside of the regular order.
     440              : ///
     441              : /// This is necessary to work around a systemd bug where it freezes if there are
     442              : /// walredo processes left => <https://github.com/neondatabase/cloud/issues/11387>
     443              : #[allow(clippy::type_complexity)]
     444              : pub(crate) static WALREDO_MANAGERS: once_cell::sync::Lazy<
     445              :     Mutex<Option<HashMap<WalredoManagerId, Weak<WalRedoManager>>>>,
     446            0 : > = once_cell::sync::Lazy::new(|| Mutex::new(Some(HashMap::new())));
     447              : #[derive(PartialEq, Eq, Hash, Clone, Copy, Debug)]
     448              : pub(crate) struct WalredoManagerId(u64);
     449              : impl WalredoManagerId {
     450            0 :     pub fn next() -> Self {
     451              :         static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
     452            0 :         let id = NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
     453            0 :         if id == 0 {
     454            0 :             panic!("WalredoManagerId::new() returned 0, indicating wraparound, risking it's no longer unique");
     455            0 :         }
     456            0 :         Self(id)
     457            0 :     }
     458              : }
     459              : 
     460              : #[cfg(test)]
     461              : impl From<harness::TestRedoManager> for WalRedoManager {
     462          196 :     fn from(mgr: harness::TestRedoManager) -> Self {
     463          196 :         Self::Test(mgr)
     464          196 :     }
     465              : }
     466              : 
     467              : impl WalRedoManager {
     468            6 :     pub(crate) async fn shutdown(&self) -> bool {
     469            6 :         match self {
     470            0 :             Self::Prod(_, mgr) => mgr.shutdown().await,
     471              :             #[cfg(test)]
     472              :             Self::Test(_) => {
     473              :                 // Not applicable to test redo manager
     474            6 :                 true
     475              :             }
     476              :         }
     477            6 :     }
     478              : 
     479            0 :     pub(crate) fn maybe_quiesce(&self, idle_timeout: Duration) {
     480            0 :         match self {
     481            0 :             Self::Prod(_, mgr) => mgr.maybe_quiesce(idle_timeout),
     482            0 :             #[cfg(test)]
     483            0 :             Self::Test(_) => {
     484            0 :                 // Not applicable to test redo manager
     485            0 :             }
     486            0 :         }
     487            0 :     }
     488              : 
     489              :     /// # Cancel-Safety
     490              :     ///
     491              :     /// This method is cancellation-safe.
     492          520 :     pub async fn request_redo(
     493          520 :         &self,
     494          520 :         key: pageserver_api::key::Key,
     495          520 :         lsn: Lsn,
     496          520 :         base_img: Option<(Lsn, bytes::Bytes)>,
     497          520 :         records: Vec<(Lsn, pageserver_api::record::NeonWalRecord)>,
     498          520 :         pg_version: u32,
     499          520 :     ) -> Result<bytes::Bytes, walredo::Error> {
     500          520 :         match self {
     501            0 :             Self::Prod(_, mgr) => {
     502            0 :                 mgr.request_redo(key, lsn, base_img, records, pg_version)
     503            0 :                     .await
     504              :             }
     505              :             #[cfg(test)]
     506          520 :             Self::Test(mgr) => {
     507          520 :                 mgr.request_redo(key, lsn, base_img, records, pg_version)
     508          520 :                     .await
     509              :             }
     510              :         }
     511          520 :     }
     512              : 
     513            0 :     pub(crate) fn status(&self) -> Option<WalRedoManagerStatus> {
     514            0 :         match self {
     515            0 :             WalRedoManager::Prod(_, m) => Some(m.status()),
     516            0 :             #[cfg(test)]
     517            0 :             WalRedoManager::Test(_) => None,
     518            0 :         }
     519            0 :     }
     520              : }
     521              : 
     522              : /// A very lightweight memory representation of an offloaded timeline.
     523              : ///
     524              : /// We need to store the list of offloaded timelines so that we can perform operations on them,
     525              : /// like unoffloading them, or (at a later date), decide to perform flattening.
     526              : /// This type has a much smaller memory impact than [`Timeline`], and thus we can store many
     527              : /// more offloaded timelines than we can manage ones that aren't.
     528              : pub struct OffloadedTimeline {
     529              :     pub tenant_shard_id: TenantShardId,
     530              :     pub timeline_id: TimelineId,
     531              :     pub ancestor_timeline_id: Option<TimelineId>,
     532              :     /// Whether to retain the branch lsn at the ancestor or not
     533              :     pub ancestor_retain_lsn: Option<Lsn>,
     534              : 
     535              :     /// When the timeline was archived.
     536              :     ///
     537              :     /// Present for future flattening deliberations.
     538              :     pub archived_at: NaiveDateTime,
     539              : 
     540              :     /// Prevent two tasks from deleting the timeline at the same time. If held, the
     541              :     /// timeline is being deleted. If 'true', the timeline has already been deleted.
     542              :     pub delete_progress: TimelineDeleteProgress,
     543              : 
     544              :     /// Part of the `OffloadedTimeline` object's lifecycle: this needs to be set before we drop it
     545              :     pub deleted_from_ancestor: AtomicBool,
     546              : }
     547              : 
     548              : impl OffloadedTimeline {
     549              :     /// Obtains an offloaded timeline from a given timeline object.
     550              :     ///
     551              :     /// Returns `None` if the `archived_at` flag couldn't be obtained, i.e.
     552              :     /// the timeline is not in a stopped state.
     553              :     /// Panics if the timeline is not archived.
     554            2 :     fn from_timeline(timeline: &Timeline) -> Result<Self, UploadQueueNotReadyError> {
     555            2 :         let (ancestor_retain_lsn, ancestor_timeline_id) =
     556            2 :             if let Some(ancestor_timeline) = timeline.ancestor_timeline() {
     557            2 :                 let ancestor_lsn = timeline.get_ancestor_lsn();
     558            2 :                 let ancestor_timeline_id = ancestor_timeline.timeline_id;
     559            2 :                 let mut gc_info = ancestor_timeline.gc_info.write().unwrap();
     560            2 :                 gc_info.insert_child(timeline.timeline_id, ancestor_lsn, MaybeOffloaded::Yes);
     561            2 :                 (Some(ancestor_lsn), Some(ancestor_timeline_id))
     562              :             } else {
     563            0 :                 (None, None)
     564              :             };
     565            2 :         let archived_at = timeline
     566            2 :             .remote_client
     567            2 :             .archived_at_stopped_queue()?
     568            2 :             .expect("must be called on an archived timeline");
     569            2 :         Ok(Self {
     570            2 :             tenant_shard_id: timeline.tenant_shard_id,
     571            2 :             timeline_id: timeline.timeline_id,
     572            2 :             ancestor_timeline_id,
     573            2 :             ancestor_retain_lsn,
     574            2 :             archived_at,
     575            2 : 
     576            2 :             delete_progress: timeline.delete_progress.clone(),
     577            2 :             deleted_from_ancestor: AtomicBool::new(false),
     578            2 :         })
     579            2 :     }
     580            0 :     fn from_manifest(tenant_shard_id: TenantShardId, manifest: &OffloadedTimelineManifest) -> Self {
     581            0 :         // We expect to reach this case in tenant loading, where the `retain_lsn` is populated in the parent's `gc_info`
     582            0 :         // by the `initialize_gc_info` function.
     583            0 :         let OffloadedTimelineManifest {
     584            0 :             timeline_id,
     585            0 :             ancestor_timeline_id,
     586            0 :             ancestor_retain_lsn,
     587            0 :             archived_at,
     588            0 :         } = *manifest;
     589            0 :         Self {
     590            0 :             tenant_shard_id,
     591            0 :             timeline_id,
     592            0 :             ancestor_timeline_id,
     593            0 :             ancestor_retain_lsn,
     594            0 :             archived_at,
     595            0 :             delete_progress: TimelineDeleteProgress::default(),
     596            0 :             deleted_from_ancestor: AtomicBool::new(false),
     597            0 :         }
     598            0 :     }
     599            2 :     fn manifest(&self) -> OffloadedTimelineManifest {
     600            2 :         let Self {
     601            2 :             timeline_id,
     602            2 :             ancestor_timeline_id,
     603            2 :             ancestor_retain_lsn,
     604            2 :             archived_at,
     605            2 :             ..
     606            2 :         } = self;
     607            2 :         OffloadedTimelineManifest {
     608            2 :             timeline_id: *timeline_id,
     609            2 :             ancestor_timeline_id: *ancestor_timeline_id,
     610            2 :             ancestor_retain_lsn: *ancestor_retain_lsn,
     611            2 :             archived_at: *archived_at,
     612            2 :         }
     613            2 :     }
     614              :     /// Delete this timeline's retain_lsn from its ancestor, if present in the given tenant
     615            0 :     fn delete_from_ancestor_with_timelines(
     616            0 :         &self,
     617            0 :         timelines: &std::sync::MutexGuard<'_, HashMap<TimelineId, Arc<Timeline>>>,
     618            0 :     ) {
     619            0 :         if let (Some(_retain_lsn), Some(ancestor_timeline_id)) =
     620            0 :             (self.ancestor_retain_lsn, self.ancestor_timeline_id)
     621              :         {
     622            0 :             if let Some((_, ancestor_timeline)) = timelines
     623            0 :                 .iter()
     624            0 :                 .find(|(tid, _tl)| **tid == ancestor_timeline_id)
     625              :             {
     626            0 :                 let removal_happened = ancestor_timeline
     627            0 :                     .gc_info
     628            0 :                     .write()
     629            0 :                     .unwrap()
     630            0 :                     .remove_child_offloaded(self.timeline_id);
     631            0 :                 if !removal_happened {
     632            0 :                     tracing::error!(tenant_id = %self.tenant_shard_id.tenant_id, shard_id = %self.tenant_shard_id.shard_slug(), timeline_id = %self.timeline_id,
     633            0 :                         "Couldn't remove retain_lsn entry from offloaded timeline's parent: already removed");
     634            0 :                 }
     635            0 :             }
     636            0 :         }
     637            0 :         self.deleted_from_ancestor.store(true, Ordering::Release);
     638            0 :     }
     639              :     /// Call [`Self::delete_from_ancestor_with_timelines`] instead if possible.
     640              :     ///
     641              :     /// As the entire tenant is being dropped, don't bother deregistering the `retain_lsn` from the ancestor.
     642            2 :     fn defuse_for_tenant_drop(&self) {
     643            2 :         self.deleted_from_ancestor.store(true, Ordering::Release);
     644            2 :     }
     645              : }
     646              : 
     647              : impl fmt::Debug for OffloadedTimeline {
     648            0 :     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
     649            0 :         write!(f, "OffloadedTimeline<{}>", self.timeline_id)
     650            0 :     }
     651              : }
     652              : 
     653              : impl Drop for OffloadedTimeline {
     654            2 :     fn drop(&mut self) {
     655            2 :         if !self.deleted_from_ancestor.load(Ordering::Acquire) {
     656            0 :             tracing::warn!(
     657            0 :                 "offloaded timeline {} was dropped without having cleaned it up at the ancestor",
     658              :                 self.timeline_id
     659              :             );
     660            2 :         }
     661            2 :     }
     662              : }
     663              : 
     664              : #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)]
     665              : pub enum MaybeOffloaded {
     666              :     Yes,
     667              :     No,
     668              : }
     669              : 
     670              : #[derive(Clone, Debug)]
     671              : pub enum TimelineOrOffloaded {
     672              :     Timeline(Arc<Timeline>),
     673              :     Offloaded(Arc<OffloadedTimeline>),
     674              : }
     675              : 
     676              : impl TimelineOrOffloaded {
     677            0 :     pub fn arc_ref(&self) -> TimelineOrOffloadedArcRef<'_> {
     678            0 :         match self {
     679            0 :             TimelineOrOffloaded::Timeline(timeline) => {
     680            0 :                 TimelineOrOffloadedArcRef::Timeline(timeline)
     681              :             }
     682            0 :             TimelineOrOffloaded::Offloaded(offloaded) => {
     683            0 :                 TimelineOrOffloadedArcRef::Offloaded(offloaded)
     684              :             }
     685              :         }
     686            0 :     }
     687            0 :     pub fn tenant_shard_id(&self) -> TenantShardId {
     688            0 :         self.arc_ref().tenant_shard_id()
     689            0 :     }
     690            0 :     pub fn timeline_id(&self) -> TimelineId {
     691            0 :         self.arc_ref().timeline_id()
     692            0 :     }
     693            2 :     pub fn delete_progress(&self) -> &Arc<tokio::sync::Mutex<DeleteTimelineFlow>> {
     694            2 :         match self {
     695            2 :             TimelineOrOffloaded::Timeline(timeline) => &timeline.delete_progress,
     696            0 :             TimelineOrOffloaded::Offloaded(offloaded) => &offloaded.delete_progress,
     697              :         }
     698            2 :     }
     699            0 :     fn maybe_remote_client(&self) -> Option<Arc<RemoteTimelineClient>> {
     700            0 :         match self {
     701            0 :             TimelineOrOffloaded::Timeline(timeline) => Some(timeline.remote_client.clone()),
     702            0 :             TimelineOrOffloaded::Offloaded(_offloaded) => None,
     703              :         }
     704            0 :     }
     705              : }
     706              : 
     707              : pub enum TimelineOrOffloadedArcRef<'a> {
     708              :     Timeline(&'a Arc<Timeline>),
     709              :     Offloaded(&'a Arc<OffloadedTimeline>),
     710              : }
     711              : 
     712              : impl TimelineOrOffloadedArcRef<'_> {
     713            0 :     pub fn tenant_shard_id(&self) -> TenantShardId {
     714            0 :         match self {
     715            0 :             TimelineOrOffloadedArcRef::Timeline(timeline) => timeline.tenant_shard_id,
     716            0 :             TimelineOrOffloadedArcRef::Offloaded(offloaded) => offloaded.tenant_shard_id,
     717              :         }
     718            0 :     }
     719            0 :     pub fn timeline_id(&self) -> TimelineId {
     720            0 :         match self {
     721            0 :             TimelineOrOffloadedArcRef::Timeline(timeline) => timeline.timeline_id,
     722            0 :             TimelineOrOffloadedArcRef::Offloaded(offloaded) => offloaded.timeline_id,
     723              :         }
     724            0 :     }
     725              : }
     726              : 
     727              : impl<'a> From<&'a Arc<Timeline>> for TimelineOrOffloadedArcRef<'a> {
     728            0 :     fn from(timeline: &'a Arc<Timeline>) -> Self {
     729            0 :         Self::Timeline(timeline)
     730            0 :     }
     731              : }
     732              : 
     733              : impl<'a> From<&'a Arc<OffloadedTimeline>> for TimelineOrOffloadedArcRef<'a> {
     734            0 :     fn from(timeline: &'a Arc<OffloadedTimeline>) -> Self {
     735            0 :         Self::Offloaded(timeline)
     736            0 :     }
     737              : }
     738              : 
     739              : #[derive(Debug, thiserror::Error, PartialEq, Eq)]
     740              : pub enum GetTimelineError {
     741              :     #[error("Timeline is shutting down")]
     742              :     ShuttingDown,
     743              :     #[error("Timeline {tenant_id}/{timeline_id} is not active, state: {state:?}")]
     744              :     NotActive {
     745              :         tenant_id: TenantShardId,
     746              :         timeline_id: TimelineId,
     747              :         state: TimelineState,
     748              :     },
     749              :     #[error("Timeline {tenant_id}/{timeline_id} was not found")]
     750              :     NotFound {
     751              :         tenant_id: TenantShardId,
     752              :         timeline_id: TimelineId,
     753              :     },
     754              : }
     755              : 
     756              : #[derive(Debug, thiserror::Error)]
     757              : pub enum LoadLocalTimelineError {
     758              :     #[error("FailedToLoad")]
     759              :     Load(#[source] anyhow::Error),
     760              :     #[error("FailedToResumeDeletion")]
     761              :     ResumeDeletion(#[source] anyhow::Error),
     762              : }
     763              : 
     764              : #[derive(thiserror::Error)]
     765              : pub enum DeleteTimelineError {
     766              :     #[error("NotFound")]
     767              :     NotFound,
     768              : 
     769              :     #[error("HasChildren")]
     770              :     HasChildren(Vec<TimelineId>),
     771              : 
     772              :     #[error("Timeline deletion is already in progress")]
     773              :     AlreadyInProgress(Arc<tokio::sync::Mutex<DeleteTimelineFlow>>),
     774              : 
     775              :     #[error("Cancelled")]
     776              :     Cancelled,
     777              : 
     778              :     #[error(transparent)]
     779              :     Other(#[from] anyhow::Error),
     780              : }
     781              : 
     782              : impl Debug for DeleteTimelineError {
     783            0 :     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
     784            0 :         match self {
     785            0 :             Self::NotFound => write!(f, "NotFound"),
     786            0 :             Self::HasChildren(c) => f.debug_tuple("HasChildren").field(c).finish(),
     787            0 :             Self::AlreadyInProgress(_) => f.debug_tuple("AlreadyInProgress").finish(),
     788            0 :             Self::Cancelled => f.debug_tuple("Cancelled").finish(),
     789            0 :             Self::Other(e) => f.debug_tuple("Other").field(e).finish(),
     790              :         }
     791            0 :     }
     792              : }
     793              : 
     794              : #[derive(thiserror::Error)]
     795              : pub enum TimelineArchivalError {
     796              :     #[error("NotFound")]
     797              :     NotFound,
     798              : 
     799              :     #[error("Timeout")]
     800              :     Timeout,
     801              : 
     802              :     #[error("Cancelled")]
     803              :     Cancelled,
     804              : 
     805              :     #[error("ancestor is archived: {}", .0)]
     806              :     HasArchivedParent(TimelineId),
     807              : 
     808              :     #[error("HasUnarchivedChildren")]
     809              :     HasUnarchivedChildren(Vec<TimelineId>),
     810              : 
     811              :     #[error("Timeline archival is already in progress")]
     812              :     AlreadyInProgress,
     813              : 
     814              :     #[error(transparent)]
     815              :     Other(anyhow::Error),
     816              : }
     817              : 
     818              : #[derive(thiserror::Error, Debug)]
     819              : pub(crate) enum TenantManifestError {
     820              :     #[error("Remote storage error: {0}")]
     821              :     RemoteStorage(anyhow::Error),
     822              : 
     823              :     #[error("Cancelled")]
     824              :     Cancelled,
     825              : }
     826              : 
     827              : impl From<TenantManifestError> for TimelineArchivalError {
     828            0 :     fn from(e: TenantManifestError) -> Self {
     829            0 :         match e {
     830            0 :             TenantManifestError::RemoteStorage(e) => Self::Other(e),
     831            0 :             TenantManifestError::Cancelled => Self::Cancelled,
     832              :         }
     833            0 :     }
     834              : }
     835              : 
     836              : impl Debug for TimelineArchivalError {
     837            0 :     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
     838            0 :         match self {
     839            0 :             Self::NotFound => write!(f, "NotFound"),
     840            0 :             Self::Timeout => write!(f, "Timeout"),
     841            0 :             Self::Cancelled => write!(f, "Cancelled"),
     842            0 :             Self::HasArchivedParent(p) => f.debug_tuple("HasArchivedParent").field(p).finish(),
     843            0 :             Self::HasUnarchivedChildren(c) => {
     844            0 :                 f.debug_tuple("HasUnarchivedChildren").field(c).finish()
     845              :             }
     846            0 :             Self::AlreadyInProgress => f.debug_tuple("AlreadyInProgress").finish(),
     847            0 :             Self::Other(e) => f.debug_tuple("Other").field(e).finish(),
     848              :         }
     849            0 :     }
     850              : }
     851              : 
     852              : pub enum SetStoppingError {
     853              :     AlreadyStopping(completion::Barrier),
     854              :     Broken,
     855              : }
     856              : 
     857              : impl Debug for SetStoppingError {
     858            0 :     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
     859            0 :         match self {
     860            0 :             Self::AlreadyStopping(_) => f.debug_tuple("AlreadyStopping").finish(),
     861            0 :             Self::Broken => write!(f, "Broken"),
     862              :         }
     863            0 :     }
     864              : }
     865              : 
     866              : /// Arguments to [`Tenant::create_timeline`].
     867              : ///
     868              : /// Not usable as an idempotency key for timeline creation because if [`CreateTimelineParamsBranch::ancestor_start_lsn`]
     869              : /// is `None`, the result of the timeline create call is not deterministic.
     870              : ///
     871              : /// See [`CreateTimelineIdempotency`] for an idempotency key.
     872              : #[derive(Debug)]
     873              : pub(crate) enum CreateTimelineParams {
     874              :     Bootstrap(CreateTimelineParamsBootstrap),
     875              :     Branch(CreateTimelineParamsBranch),
     876              :     ImportPgdata(CreateTimelineParamsImportPgdata),
     877              : }
     878              : 
     879              : #[derive(Debug)]
     880              : pub(crate) struct CreateTimelineParamsBootstrap {
     881              :     pub(crate) new_timeline_id: TimelineId,
     882              :     pub(crate) existing_initdb_timeline_id: Option<TimelineId>,
     883              :     pub(crate) pg_version: u32,
     884              : }
     885              : 
     886              : /// NB: See comment on [`CreateTimelineIdempotency::Branch`] for why there's no `pg_version` here.
     887              : #[derive(Debug)]
     888              : pub(crate) struct CreateTimelineParamsBranch {
     889              :     pub(crate) new_timeline_id: TimelineId,
     890              :     pub(crate) ancestor_timeline_id: TimelineId,
     891              :     pub(crate) ancestor_start_lsn: Option<Lsn>,
     892              : }
     893              : 
     894              : #[derive(Debug)]
     895              : pub(crate) struct CreateTimelineParamsImportPgdata {
     896              :     pub(crate) new_timeline_id: TimelineId,
     897              :     pub(crate) location: import_pgdata::index_part_format::Location,
     898              :     pub(crate) idempotency_key: import_pgdata::index_part_format::IdempotencyKey,
     899              : }
     900              : 
     901              : /// What is used to determine idempotency of a [`Tenant::create_timeline`] call in  [`Tenant::start_creating_timeline`] in  [`Tenant::start_creating_timeline`].
     902              : ///
     903              : /// Each [`Timeline`] object holds [`Self`] as an immutable property in [`Timeline::create_idempotency`].
     904              : ///
     905              : /// We lower timeline creation requests to [`Self`], and then use [`PartialEq::eq`] to compare [`Timeline::create_idempotency`] with the request.
     906              : /// If they are equal, we return a reference to the existing timeline, otherwise it's an idempotency conflict.
     907              : ///
     908              : /// There is special treatment for [`Self::FailWithConflict`] to always return an idempotency conflict.
     909              : /// It would be nice to have more advanced derive macros to make that special treatment declarative.
     910              : ///
     911              : /// Notes:
     912              : /// - Unlike [`CreateTimelineParams`], ancestor LSN is fixed, so, branching will be at a deterministic LSN.
     913              : /// - We make some trade-offs though, e.g., [`CreateTimelineParamsBootstrap::existing_initdb_timeline_id`]
     914              : ///   is not considered for idempotency. We can improve on this over time if we deem it necessary.
     915              : ///
     916              : #[derive(Debug, Clone, PartialEq, Eq)]
     917              : pub(crate) enum CreateTimelineIdempotency {
     918              :     /// NB: special treatment, see comment in [`Self`].
     919              :     FailWithConflict,
     920              :     Bootstrap {
     921              :         pg_version: u32,
     922              :     },
     923              :     /// NB: branches always have the same `pg_version` as their ancestor.
     924              :     /// While [`pageserver_api::models::TimelineCreateRequestMode::Branch::pg_version`]
     925              :     /// exists as a field, and is set by cplane, it has always been ignored by pageserver when
     926              :     /// determining the child branch pg_version.
     927              :     Branch {
     928              :         ancestor_timeline_id: TimelineId,
     929              :         ancestor_start_lsn: Lsn,
     930              :     },
     931              :     ImportPgdata(CreatingTimelineIdempotencyImportPgdata),
     932              : }
     933              : 
     934              : #[derive(Debug, Clone, PartialEq, Eq)]
     935              : pub(crate) struct CreatingTimelineIdempotencyImportPgdata {
     936              :     idempotency_key: import_pgdata::index_part_format::IdempotencyKey,
     937              : }
     938              : 
     939              : /// What is returned by [`Tenant::start_creating_timeline`].
     940              : #[must_use]
     941              : enum StartCreatingTimelineResult {
     942              :     CreateGuard(TimelineCreateGuard),
     943              :     Idempotent(Arc<Timeline>),
     944              : }
     945              : 
     946              : enum TimelineInitAndSyncResult {
     947              :     ReadyToActivate(Arc<Timeline>),
     948              :     NeedsSpawnImportPgdata(TimelineInitAndSyncNeedsSpawnImportPgdata),
     949              : }
     950              : 
     951              : impl TimelineInitAndSyncResult {
     952            0 :     fn ready_to_activate(self) -> Option<Arc<Timeline>> {
     953            0 :         match self {
     954            0 :             Self::ReadyToActivate(timeline) => Some(timeline),
     955            0 :             _ => None,
     956              :         }
     957            0 :     }
     958              : }
     959              : 
     960              : #[must_use]
     961              : struct TimelineInitAndSyncNeedsSpawnImportPgdata {
     962              :     timeline: Arc<Timeline>,
     963              :     import_pgdata: import_pgdata::index_part_format::Root,
     964              :     guard: TimelineCreateGuard,
     965              : }
     966              : 
     967              : /// What is returned by [`Tenant::create_timeline`].
     968              : enum CreateTimelineResult {
     969              :     Created(Arc<Timeline>),
     970              :     Idempotent(Arc<Timeline>),
     971              :     /// IMPORTANT: This [`Arc<Timeline>`] object is not in [`Tenant::timelines`] when
     972              :     /// we return this result, nor will this concrete object ever be added there.
     973              :     /// Cf method comment on [`Tenant::create_timeline_import_pgdata`].
     974              :     ImportSpawned(Arc<Timeline>),
     975              : }
     976              : 
     977              : impl CreateTimelineResult {
     978            0 :     fn discriminant(&self) -> &'static str {
     979            0 :         match self {
     980            0 :             Self::Created(_) => "Created",
     981            0 :             Self::Idempotent(_) => "Idempotent",
     982            0 :             Self::ImportSpawned(_) => "ImportSpawned",
     983              :         }
     984            0 :     }
     985            0 :     fn timeline(&self) -> &Arc<Timeline> {
     986            0 :         match self {
     987            0 :             Self::Created(t) | Self::Idempotent(t) | Self::ImportSpawned(t) => t,
     988            0 :         }
     989            0 :     }
     990              :     /// Unit test timelines aren't activated, test has to do it if it needs to.
     991              :     #[cfg(test)]
     992          230 :     fn into_timeline_for_test(self) -> Arc<Timeline> {
     993          230 :         match self {
     994          230 :             Self::Created(t) | Self::Idempotent(t) | Self::ImportSpawned(t) => t,
     995          230 :         }
     996          230 :     }
     997              : }
     998              : 
     999              : #[derive(thiserror::Error, Debug)]
    1000              : pub enum CreateTimelineError {
    1001              :     #[error("creation of timeline with the given ID is in progress")]
    1002              :     AlreadyCreating,
    1003              :     #[error("timeline already exists with different parameters")]
    1004              :     Conflict,
    1005              :     #[error(transparent)]
    1006              :     AncestorLsn(anyhow::Error),
    1007              :     #[error("ancestor timeline is not active")]
    1008              :     AncestorNotActive,
    1009              :     #[error("ancestor timeline is archived")]
    1010              :     AncestorArchived,
    1011              :     #[error("tenant shutting down")]
    1012              :     ShuttingDown,
    1013              :     #[error(transparent)]
    1014              :     Other(#[from] anyhow::Error),
    1015              : }
    1016              : 
    1017              : #[derive(thiserror::Error, Debug)]
    1018              : pub enum InitdbError {
    1019              :     #[error("Operation was cancelled")]
    1020              :     Cancelled,
    1021              :     #[error(transparent)]
    1022              :     Other(anyhow::Error),
    1023              :     #[error(transparent)]
    1024              :     Inner(postgres_initdb::Error),
    1025              : }
    1026              : 
    1027              : enum CreateTimelineCause {
    1028              :     Load,
    1029              :     Delete,
    1030              : }
    1031              : 
    1032              : enum LoadTimelineCause {
    1033              :     Attach,
    1034              :     Unoffload,
    1035              :     ImportPgdata {
    1036              :         create_guard: TimelineCreateGuard,
    1037              :         activate: ActivateTimelineArgs,
    1038              :     },
    1039              : }
    1040              : 
    1041              : #[derive(thiserror::Error, Debug)]
    1042              : pub(crate) enum GcError {
    1043              :     // The tenant is shutting down
    1044              :     #[error("tenant shutting down")]
    1045              :     TenantCancelled,
    1046              : 
    1047              :     // The tenant is shutting down
    1048              :     #[error("timeline shutting down")]
    1049              :     TimelineCancelled,
    1050              : 
    1051              :     // The tenant is in a state inelegible to run GC
    1052              :     #[error("not active")]
    1053              :     NotActive,
    1054              : 
    1055              :     // A requested GC cutoff LSN was invalid, for example it tried to move backwards
    1056              :     #[error("not active")]
    1057              :     BadLsn { why: String },
    1058              : 
    1059              :     // A remote storage error while scheduling updates after compaction
    1060              :     #[error(transparent)]
    1061              :     Remote(anyhow::Error),
    1062              : 
    1063              :     // An error reading while calculating GC cutoffs
    1064              :     #[error(transparent)]
    1065              :     GcCutoffs(PageReconstructError),
    1066              : 
    1067              :     // If GC was invoked for a particular timeline, this error means it didn't exist
    1068              :     #[error("timeline not found")]
    1069              :     TimelineNotFound,
    1070              : }
    1071              : 
    1072              : impl From<PageReconstructError> for GcError {
    1073            0 :     fn from(value: PageReconstructError) -> Self {
    1074            0 :         match value {
    1075            0 :             PageReconstructError::Cancelled => Self::TimelineCancelled,
    1076            0 :             other => Self::GcCutoffs(other),
    1077              :         }
    1078            0 :     }
    1079              : }
    1080              : 
    1081              : impl From<NotInitialized> for GcError {
    1082            0 :     fn from(value: NotInitialized) -> Self {
    1083            0 :         match value {
    1084            0 :             NotInitialized::Uninitialized => GcError::Remote(value.into()),
    1085            0 :             NotInitialized::Stopped | NotInitialized::ShuttingDown => GcError::TimelineCancelled,
    1086              :         }
    1087            0 :     }
    1088              : }
    1089              : 
    1090              : impl From<timeline::layer_manager::Shutdown> for GcError {
    1091            0 :     fn from(_: timeline::layer_manager::Shutdown) -> Self {
    1092            0 :         GcError::TimelineCancelled
    1093            0 :     }
    1094              : }
    1095              : 
    1096              : #[derive(thiserror::Error, Debug)]
    1097              : pub(crate) enum LoadConfigError {
    1098              :     #[error("TOML deserialization error: '{0}'")]
    1099              :     DeserializeToml(#[from] toml_edit::de::Error),
    1100              : 
    1101              :     #[error("Config not found at {0}")]
    1102              :     NotFound(Utf8PathBuf),
    1103              : }
    1104              : 
    1105              : impl Tenant {
    1106              :     /// Yet another helper for timeline initialization.
    1107              :     ///
    1108              :     /// - Initializes the Timeline struct and inserts it into the tenant's hash map
    1109              :     /// - Scans the local timeline directory for layer files and builds the layer map
    1110              :     /// - Downloads remote index file and adds remote files to the layer map
    1111              :     /// - Schedules remote upload tasks for any files that are present locally but missing from remote storage.
    1112              :     ///
    1113              :     /// If the operation fails, the timeline is left in the tenant's hash map in Broken state. On success,
    1114              :     /// it is marked as Active.
    1115              :     #[allow(clippy::too_many_arguments)]
    1116            6 :     async fn timeline_init_and_sync(
    1117            6 :         self: &Arc<Self>,
    1118            6 :         timeline_id: TimelineId,
    1119            6 :         resources: TimelineResources,
    1120            6 :         mut index_part: IndexPart,
    1121            6 :         metadata: TimelineMetadata,
    1122            6 :         ancestor: Option<Arc<Timeline>>,
    1123            6 :         cause: LoadTimelineCause,
    1124            6 :         ctx: &RequestContext,
    1125            6 :     ) -> anyhow::Result<TimelineInitAndSyncResult> {
    1126            6 :         let tenant_id = self.tenant_shard_id;
    1127            6 : 
    1128            6 :         let import_pgdata = index_part.import_pgdata.take();
    1129            6 :         let idempotency = match &import_pgdata {
    1130            0 :             Some(import_pgdata) => {
    1131            0 :                 CreateTimelineIdempotency::ImportPgdata(CreatingTimelineIdempotencyImportPgdata {
    1132            0 :                     idempotency_key: import_pgdata.idempotency_key().clone(),
    1133            0 :                 })
    1134              :             }
    1135              :             None => {
    1136            6 :                 if metadata.ancestor_timeline().is_none() {
    1137            4 :                     CreateTimelineIdempotency::Bootstrap {
    1138            4 :                         pg_version: metadata.pg_version(),
    1139            4 :                     }
    1140              :                 } else {
    1141            2 :                     CreateTimelineIdempotency::Branch {
    1142            2 :                         ancestor_timeline_id: metadata.ancestor_timeline().unwrap(),
    1143            2 :                         ancestor_start_lsn: metadata.ancestor_lsn(),
    1144            2 :                     }
    1145              :                 }
    1146              :             }
    1147              :         };
    1148              : 
    1149            6 :         let timeline = self.create_timeline_struct(
    1150            6 :             timeline_id,
    1151            6 :             &metadata,
    1152            6 :             ancestor.clone(),
    1153            6 :             resources,
    1154            6 :             CreateTimelineCause::Load,
    1155            6 :             idempotency.clone(),
    1156            6 :         )?;
    1157            6 :         let disk_consistent_lsn = timeline.get_disk_consistent_lsn();
    1158            6 :         anyhow::ensure!(
    1159            6 :             disk_consistent_lsn.is_valid(),
    1160            0 :             "Timeline {tenant_id}/{timeline_id} has invalid disk_consistent_lsn"
    1161              :         );
    1162            6 :         assert_eq!(
    1163            6 :             disk_consistent_lsn,
    1164            6 :             metadata.disk_consistent_lsn(),
    1165            0 :             "these are used interchangeably"
    1166              :         );
    1167              : 
    1168            6 :         timeline.remote_client.init_upload_queue(&index_part)?;
    1169              : 
    1170            6 :         timeline
    1171            6 :             .load_layer_map(disk_consistent_lsn, index_part)
    1172            6 :             .await
    1173            6 :             .with_context(|| {
    1174            0 :                 format!("Failed to load layermap for timeline {tenant_id}/{timeline_id}")
    1175            6 :             })?;
    1176              : 
    1177            0 :         match import_pgdata {
    1178            0 :             Some(import_pgdata) if !import_pgdata.is_done() => {
    1179            0 :                 match cause {
    1180            0 :                     LoadTimelineCause::Attach | LoadTimelineCause::Unoffload => (),
    1181              :                     LoadTimelineCause::ImportPgdata { .. } => {
    1182            0 :                         unreachable!("ImportPgdata should not be reloading timeline import is done and persisted as such in s3")
    1183              :                     }
    1184              :                 }
    1185            0 :                 let mut guard = self.timelines_creating.lock().unwrap();
    1186            0 :                 if !guard.insert(timeline_id) {
    1187              :                     // We should never try and load the same timeline twice during startup
    1188            0 :                     unreachable!("Timeline {tenant_id}/{timeline_id} is already being created")
    1189            0 :                 }
    1190            0 :                 let timeline_create_guard = TimelineCreateGuard {
    1191            0 :                     _tenant_gate_guard: self.gate.enter()?,
    1192            0 :                     owning_tenant: self.clone(),
    1193            0 :                     timeline_id,
    1194            0 :                     idempotency,
    1195            0 :                     // The users of this specific return value don't need the timline_path in there.
    1196            0 :                     timeline_path: timeline
    1197            0 :                         .conf
    1198            0 :                         .timeline_path(&timeline.tenant_shard_id, &timeline.timeline_id),
    1199            0 :                 };
    1200            0 :                 Ok(TimelineInitAndSyncResult::NeedsSpawnImportPgdata(
    1201            0 :                     TimelineInitAndSyncNeedsSpawnImportPgdata {
    1202            0 :                         timeline,
    1203            0 :                         import_pgdata,
    1204            0 :                         guard: timeline_create_guard,
    1205            0 :                     },
    1206            0 :                 ))
    1207              :             }
    1208              :             Some(_) | None => {
    1209              :                 {
    1210            6 :                     let mut timelines_accessor = self.timelines.lock().unwrap();
    1211            6 :                     match timelines_accessor.entry(timeline_id) {
    1212              :                         // We should never try and load the same timeline twice during startup
    1213              :                         Entry::Occupied(_) => {
    1214            0 :                             unreachable!(
    1215            0 :                             "Timeline {tenant_id}/{timeline_id} already exists in the tenant map"
    1216            0 :                         );
    1217              :                         }
    1218            6 :                         Entry::Vacant(v) => {
    1219            6 :                             v.insert(Arc::clone(&timeline));
    1220            6 :                             timeline.maybe_spawn_flush_loop();
    1221            6 :                         }
    1222              :                     }
    1223              :                 }
    1224              : 
    1225              :                 // Sanity check: a timeline should have some content.
    1226            6 :                 anyhow::ensure!(
    1227            6 :                     ancestor.is_some()
    1228            4 :                         || timeline
    1229            4 :                             .layers
    1230            4 :                             .read()
    1231            4 :                             .await
    1232            4 :                             .layer_map()
    1233            4 :                             .expect("currently loading, layer manager cannot be shutdown already")
    1234            4 :                             .iter_historic_layers()
    1235            4 :                             .next()
    1236            4 :                             .is_some(),
    1237            0 :                     "Timeline has no ancestor and no layer files"
    1238              :                 );
    1239              : 
    1240            6 :                 match cause {
    1241            6 :                     LoadTimelineCause::Attach | LoadTimelineCause::Unoffload => (),
    1242              :                     LoadTimelineCause::ImportPgdata {
    1243            0 :                         create_guard,
    1244            0 :                         activate,
    1245            0 :                     } => {
    1246            0 :                         // TODO: see the comment in the task code above how I'm not so certain
    1247            0 :                         // it is safe to activate here because of concurrent shutdowns.
    1248            0 :                         match activate {
    1249            0 :                             ActivateTimelineArgs::Yes { broker_client } => {
    1250            0 :                                 info!("activating timeline after reload from pgdata import task");
    1251            0 :                                 timeline.activate(self.clone(), broker_client, None, ctx);
    1252              :                             }
    1253            0 :                             ActivateTimelineArgs::No => (),
    1254              :                         }
    1255            0 :                         drop(create_guard);
    1256              :                     }
    1257              :                 }
    1258              : 
    1259            6 :                 Ok(TimelineInitAndSyncResult::ReadyToActivate(timeline))
    1260              :             }
    1261              :         }
    1262            6 :     }
    1263              : 
    1264              :     /// Attach a tenant that's available in cloud storage.
    1265              :     ///
    1266              :     /// This returns quickly, after just creating the in-memory object
    1267              :     /// Tenant struct and launching a background task to download
    1268              :     /// the remote index files.  On return, the tenant is most likely still in
    1269              :     /// Attaching state, and it will become Active once the background task
    1270              :     /// finishes. You can use wait_until_active() to wait for the task to
    1271              :     /// complete.
    1272              :     ///
    1273              :     #[allow(clippy::too_many_arguments)]
    1274            0 :     pub(crate) fn spawn(
    1275            0 :         conf: &'static PageServerConf,
    1276            0 :         tenant_shard_id: TenantShardId,
    1277            0 :         resources: TenantSharedResources,
    1278            0 :         attached_conf: AttachedTenantConf,
    1279            0 :         shard_identity: ShardIdentity,
    1280            0 :         init_order: Option<InitializationOrder>,
    1281            0 :         mode: SpawnMode,
    1282            0 :         ctx: &RequestContext,
    1283            0 :     ) -> Result<Arc<Tenant>, GlobalShutDown> {
    1284            0 :         let wal_redo_manager =
    1285            0 :             WalRedoManager::new(PostgresRedoManager::new(conf, tenant_shard_id))?;
    1286              : 
    1287              :         let TenantSharedResources {
    1288            0 :             broker_client,
    1289            0 :             remote_storage,
    1290            0 :             deletion_queue_client,
    1291            0 :             l0_flush_global_state,
    1292            0 :         } = resources;
    1293            0 : 
    1294            0 :         let attach_mode = attached_conf.location.attach_mode;
    1295            0 :         let generation = attached_conf.location.generation;
    1296            0 : 
    1297            0 :         let tenant = Arc::new(Tenant::new(
    1298            0 :             TenantState::Attaching,
    1299            0 :             conf,
    1300            0 :             attached_conf,
    1301            0 :             shard_identity,
    1302            0 :             Some(wal_redo_manager),
    1303            0 :             tenant_shard_id,
    1304            0 :             remote_storage.clone(),
    1305            0 :             deletion_queue_client,
    1306            0 :             l0_flush_global_state,
    1307            0 :         ));
    1308            0 : 
    1309            0 :         // The attach task will carry a GateGuard, so that shutdown() reliably waits for it to drop out if
    1310            0 :         // we shut down while attaching.
    1311            0 :         let attach_gate_guard = tenant
    1312            0 :             .gate
    1313            0 :             .enter()
    1314            0 :             .expect("We just created the Tenant: nothing else can have shut it down yet");
    1315            0 : 
    1316            0 :         // Do all the hard work in the background
    1317            0 :         let tenant_clone = Arc::clone(&tenant);
    1318            0 :         let ctx = ctx.detached_child(TaskKind::Attach, DownloadBehavior::Warn);
    1319            0 :         task_mgr::spawn(
    1320            0 :             &tokio::runtime::Handle::current(),
    1321            0 :             TaskKind::Attach,
    1322            0 :             tenant_shard_id,
    1323            0 :             None,
    1324            0 :             "attach tenant",
    1325            0 :             async move {
    1326            0 : 
    1327            0 :                 info!(
    1328              :                     ?attach_mode,
    1329            0 :                     "Attaching tenant"
    1330              :                 );
    1331              : 
    1332            0 :                 let _gate_guard = attach_gate_guard;
    1333            0 : 
    1334            0 :                 // Is this tenant being spawned as part of process startup?
    1335            0 :                 let starting_up = init_order.is_some();
    1336            0 :                 scopeguard::defer! {
    1337            0 :                     if starting_up {
    1338            0 :                         TENANT.startup_complete.inc();
    1339            0 :                     }
    1340            0 :                 }
    1341              : 
    1342              :                 // Ideally we should use Tenant::set_broken_no_wait, but it is not supposed to be used when tenant is in loading state.
    1343              :                 enum BrokenVerbosity {
    1344              :                     Error,
    1345              :                     Info
    1346              :                 }
    1347            0 :                 let make_broken =
    1348            0 :                     |t: &Tenant, err: anyhow::Error, verbosity: BrokenVerbosity| {
    1349            0 :                         match verbosity {
    1350              :                             BrokenVerbosity::Info => {
    1351            0 :                                 info!("attach cancelled, setting tenant state to Broken: {err}");
    1352              :                             },
    1353              :                             BrokenVerbosity::Error => {
    1354            0 :                                 error!("attach failed, setting tenant state to Broken: {err:?}");
    1355              :                             }
    1356              :                         }
    1357            0 :                         t.state.send_modify(|state| {
    1358            0 :                             // The Stopping case is for when we have passed control on to DeleteTenantFlow:
    1359            0 :                             // if it errors, we will call make_broken when tenant is already in Stopping.
    1360            0 :                             assert!(
    1361            0 :                                 matches!(*state, TenantState::Attaching | TenantState::Stopping { .. }),
    1362            0 :                                 "the attach task owns the tenant state until activation is complete"
    1363              :                             );
    1364              : 
    1365            0 :                             *state = TenantState::broken_from_reason(err.to_string());
    1366            0 :                         });
    1367            0 :                     };
    1368              : 
    1369              :                 // TODO: should also be rejecting tenant conf changes that violate this check.
    1370            0 :                 if let Err(e) = crate::tenant::storage_layer::inmemory_layer::IndexEntry::validate_checkpoint_distance(tenant_clone.get_checkpoint_distance()) {
    1371            0 :                     make_broken(&tenant_clone, anyhow::anyhow!(e), BrokenVerbosity::Error);
    1372            0 :                     return Ok(());
    1373            0 :                 }
    1374            0 : 
    1375            0 :                 let mut init_order = init_order;
    1376            0 :                 // take the completion because initial tenant loading will complete when all of
    1377            0 :                 // these tasks complete.
    1378            0 :                 let _completion = init_order
    1379            0 :                     .as_mut()
    1380            0 :                     .and_then(|x| x.initial_tenant_load.take());
    1381            0 :                 let remote_load_completion = init_order
    1382            0 :                     .as_mut()
    1383            0 :                     .and_then(|x| x.initial_tenant_load_remote.take());
    1384              : 
    1385              :                 enum AttachType<'a> {
    1386              :                     /// We are attaching this tenant lazily in the background.
    1387              :                     Warmup {
    1388              :                         _permit: tokio::sync::SemaphorePermit<'a>,
    1389              :                         during_startup: bool
    1390              :                     },
    1391              :                     /// We are attaching this tenant as soon as we can, because for example an
    1392              :                     /// endpoint tried to access it.
    1393              :                     OnDemand,
    1394              :                     /// During normal operations after startup, we are attaching a tenant, and
    1395              :                     /// eager attach was requested.
    1396              :                     Normal,
    1397              :                 }
    1398              : 
    1399            0 :                 let attach_type = if matches!(mode, SpawnMode::Lazy) {
    1400              :                     // Before doing any I/O, wait for at least one of:
    1401              :                     // - A client attempting to access to this tenant (on-demand loading)
    1402              :                     // - A permit becoming available in the warmup semaphore (background warmup)
    1403              : 
    1404            0 :                     tokio::select!(
    1405            0 :                         permit = tenant_clone.activate_now_sem.acquire() => {
    1406            0 :                             let _ = permit.expect("activate_now_sem is never closed");
    1407            0 :                             tracing::info!("Activating tenant (on-demand)");
    1408            0 :                             AttachType::OnDemand
    1409              :                         },
    1410            0 :                         permit = conf.concurrent_tenant_warmup.inner().acquire() => {
    1411            0 :                             let _permit = permit.expect("concurrent_tenant_warmup semaphore is never closed");
    1412            0 :                             tracing::info!("Activating tenant (warmup)");
    1413            0 :                             AttachType::Warmup {
    1414            0 :                                 _permit,
    1415            0 :                                 during_startup: init_order.is_some()
    1416            0 :                             }
    1417              :                         }
    1418            0 :                         _ = tenant_clone.cancel.cancelled() => {
    1419              :                             // This is safe, but should be pretty rare: it is interesting if a tenant
    1420              :                             // stayed in Activating for such a long time that shutdown found it in
    1421              :                             // that state.
    1422            0 :                             tracing::info!(state=%tenant_clone.current_state(), "Tenant shut down before activation");
    1423              :                             // Make the tenant broken so that set_stopping will not hang waiting for it to leave
    1424              :                             // the Attaching state.  This is an over-reaction (nothing really broke, the tenant is
    1425              :                             // just shutting down), but ensures progress.
    1426            0 :                             make_broken(&tenant_clone, anyhow::anyhow!("Shut down while Attaching"), BrokenVerbosity::Info);
    1427            0 :                             return Ok(());
    1428              :                         },
    1429              :                     )
    1430              :                 } else {
    1431              :                     // SpawnMode::{Create,Eager} always cause jumping ahead of the
    1432              :                     // concurrent_tenant_warmup queue
    1433            0 :                     AttachType::Normal
    1434              :                 };
    1435              : 
    1436            0 :                 let preload = match &mode {
    1437              :                     SpawnMode::Eager | SpawnMode::Lazy => {
    1438            0 :                         let _preload_timer = TENANT.preload.start_timer();
    1439            0 :                         let res = tenant_clone
    1440            0 :                             .preload(&remote_storage, task_mgr::shutdown_token())
    1441            0 :                             .await;
    1442            0 :                         match res {
    1443            0 :                             Ok(p) => Some(p),
    1444            0 :                             Err(e) => {
    1445            0 :                                 make_broken(&tenant_clone, anyhow::anyhow!(e), BrokenVerbosity::Error);
    1446            0 :                                 return Ok(());
    1447              :                             }
    1448              :                         }
    1449              :                     }
    1450              : 
    1451              :                 };
    1452              : 
    1453              :                 // Remote preload is complete.
    1454            0 :                 drop(remote_load_completion);
    1455            0 : 
    1456            0 : 
    1457            0 :                 // We will time the duration of the attach phase unless this is a creation (attach will do no work)
    1458            0 :                 let attach_start = std::time::Instant::now();
    1459            0 :                 let attached = {
    1460            0 :                     let _attach_timer = Some(TENANT.attach.start_timer());
    1461            0 :                     tenant_clone.attach(preload, &ctx).await
    1462              :                 };
    1463            0 :                 let attach_duration = attach_start.elapsed();
    1464            0 :                 _ = tenant_clone.attach_wal_lag_cooldown.set(WalLagCooldown::new(attach_start, attach_duration));
    1465            0 : 
    1466            0 :                 match attached {
    1467              :                     Ok(()) => {
    1468            0 :                         info!("attach finished, activating");
    1469            0 :                         tenant_clone.activate(broker_client, None, &ctx);
    1470              :                     }
    1471            0 :                     Err(e) => {
    1472            0 :                         make_broken(&tenant_clone, anyhow::anyhow!(e), BrokenVerbosity::Error);
    1473            0 :                     }
    1474              :                 }
    1475              : 
    1476              :                 // If we are doing an opportunistic warmup attachment at startup, initialize
    1477              :                 // logical size at the same time.  This is better than starting a bunch of idle tenants
    1478              :                 // with cold caches and then coming back later to initialize their logical sizes.
    1479              :                 //
    1480              :                 // It also prevents the warmup proccess competing with the concurrency limit on
    1481              :                 // logical size calculations: if logical size calculation semaphore is saturated,
    1482              :                 // then warmup will wait for that before proceeding to the next tenant.
    1483            0 :                 if matches!(attach_type, AttachType::Warmup { during_startup: true, .. }) {
    1484            0 :                     let mut futs: FuturesUnordered<_> = tenant_clone.timelines.lock().unwrap().values().cloned().map(|t| t.await_initial_logical_size()).collect();
    1485            0 :                     tracing::info!("Waiting for initial logical sizes while warming up...");
    1486            0 :                     while futs.next().await.is_some() {}
    1487            0 :                     tracing::info!("Warm-up complete");
    1488            0 :                 }
    1489              : 
    1490            0 :                 Ok(())
    1491            0 :             }
    1492            0 :             .instrument(tracing::info_span!(parent: None, "attach", tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug(), gen=?generation)),
    1493              :         );
    1494            0 :         Ok(tenant)
    1495            0 :     }
    1496              : 
    1497          196 :     #[instrument(skip_all)]
    1498              :     pub(crate) async fn preload(
    1499              :         self: &Arc<Self>,
    1500              :         remote_storage: &GenericRemoteStorage,
    1501              :         cancel: CancellationToken,
    1502              :     ) -> anyhow::Result<TenantPreload> {
    1503              :         span::debug_assert_current_span_has_tenant_id();
    1504              :         // Get list of remote timelines
    1505              :         // download index files for every tenant timeline
    1506              :         info!("listing remote timelines");
    1507              :         let (mut remote_timeline_ids, other_keys) = remote_timeline_client::list_remote_timelines(
    1508              :             remote_storage,
    1509              :             self.tenant_shard_id,
    1510              :             cancel.clone(),
    1511              :         )
    1512              :         .await?;
    1513              :         let (offloaded_add, tenant_manifest) =
    1514              :             match remote_timeline_client::download_tenant_manifest(
    1515              :                 remote_storage,
    1516              :                 &self.tenant_shard_id,
    1517              :                 self.generation,
    1518              :                 &cancel,
    1519              :             )
    1520              :             .await
    1521              :             {
    1522              :                 Ok((tenant_manifest, _generation, _manifest_mtime)) => (
    1523              :                     format!("{} offloaded", tenant_manifest.offloaded_timelines.len()),
    1524              :                     tenant_manifest,
    1525              :                 ),
    1526              :                 Err(DownloadError::NotFound) => {
    1527              :                     ("no manifest".to_string(), TenantManifest::empty())
    1528              :                 }
    1529              :                 Err(e) => Err(e)?,
    1530              :             };
    1531              : 
    1532              :         info!(
    1533              :             "found {} timelines, and {offloaded_add}",
    1534              :             remote_timeline_ids.len()
    1535              :         );
    1536              : 
    1537              :         for k in other_keys {
    1538              :             warn!("Unexpected non timeline key {k}");
    1539              :         }
    1540              : 
    1541              :         // Avoid downloading IndexPart of offloaded timelines.
    1542              :         let mut offloaded_with_prefix = HashSet::new();
    1543              :         for offloaded in tenant_manifest.offloaded_timelines.iter() {
    1544              :             if remote_timeline_ids.remove(&offloaded.timeline_id) {
    1545              :                 offloaded_with_prefix.insert(offloaded.timeline_id);
    1546              :             } else {
    1547              :                 // We'll take care later of timelines in the manifest without a prefix
    1548              :             }
    1549              :         }
    1550              : 
    1551              :         let timelines = self
    1552              :             .load_timelines_metadata(remote_timeline_ids, remote_storage, cancel)
    1553              :             .await?;
    1554              : 
    1555              :         Ok(TenantPreload {
    1556              :             tenant_manifest,
    1557              :             timelines: timelines
    1558              :                 .into_iter()
    1559            6 :                 .map(|(id, tl)| (id, Some(tl)))
    1560            0 :                 .chain(offloaded_with_prefix.into_iter().map(|id| (id, None)))
    1561              :                 .collect(),
    1562              :         })
    1563              :     }
    1564              : 
    1565              :     ///
    1566              :     /// Background task that downloads all data for a tenant and brings it to Active state.
    1567              :     ///
    1568              :     /// No background tasks are started as part of this routine.
    1569              :     ///
    1570          196 :     async fn attach(
    1571          196 :         self: &Arc<Tenant>,
    1572          196 :         preload: Option<TenantPreload>,
    1573          196 :         ctx: &RequestContext,
    1574          196 :     ) -> anyhow::Result<()> {
    1575          196 :         span::debug_assert_current_span_has_tenant_id();
    1576          196 : 
    1577          196 :         failpoint_support::sleep_millis_async!("before-attaching-tenant");
    1578              : 
    1579          196 :         let Some(preload) = preload else {
    1580            0 :             anyhow::bail!("local-only deployment is no longer supported, https://github.com/neondatabase/neon/issues/5624");
    1581              :         };
    1582              : 
    1583          196 :         let mut offloaded_timeline_ids = HashSet::new();
    1584          196 :         let mut offloaded_timelines_list = Vec::new();
    1585          196 :         for timeline_manifest in preload.tenant_manifest.offloaded_timelines.iter() {
    1586            0 :             let timeline_id = timeline_manifest.timeline_id;
    1587            0 :             let offloaded_timeline =
    1588            0 :                 OffloadedTimeline::from_manifest(self.tenant_shard_id, timeline_manifest);
    1589            0 :             offloaded_timelines_list.push((timeline_id, Arc::new(offloaded_timeline)));
    1590            0 :             offloaded_timeline_ids.insert(timeline_id);
    1591            0 :         }
    1592              :         // Complete deletions for offloaded timeline id's from manifest.
    1593              :         // The manifest will be uploaded later in this function.
    1594          196 :         offloaded_timelines_list
    1595          196 :             .retain(|(offloaded_id, offloaded)| {
    1596            0 :                 // Existence of a timeline is finally determined by the existence of an index-part.json in remote storage.
    1597            0 :                 // If there is dangling references in another location, they need to be cleaned up.
    1598            0 :                 let delete = !preload.timelines.contains_key(offloaded_id);
    1599            0 :                 if delete {
    1600            0 :                     tracing::info!("Removing offloaded timeline {offloaded_id} from manifest as no remote prefix was found");
    1601            0 :                     offloaded.defuse_for_tenant_drop();
    1602            0 :                 }
    1603            0 :                 !delete
    1604          196 :         });
    1605          196 : 
    1606          196 :         let mut timelines_to_resume_deletions = vec![];
    1607          196 : 
    1608          196 :         let mut remote_index_and_client = HashMap::new();
    1609          196 :         let mut timeline_ancestors = HashMap::new();
    1610          196 :         let mut existent_timelines = HashSet::new();
    1611          202 :         for (timeline_id, preload) in preload.timelines {
    1612            6 :             let Some(preload) = preload else { continue };
    1613              :             // This is an invariant of the `preload` function's API
    1614            6 :             assert!(!offloaded_timeline_ids.contains(&timeline_id));
    1615            6 :             let index_part = match preload.index_part {
    1616            6 :                 Ok(i) => {
    1617            6 :                     debug!("remote index part exists for timeline {timeline_id}");
    1618              :                     // We found index_part on the remote, this is the standard case.
    1619            6 :                     existent_timelines.insert(timeline_id);
    1620            6 :                     i
    1621              :                 }
    1622              :                 Err(DownloadError::NotFound) => {
    1623              :                     // There is no index_part on the remote. We only get here
    1624              :                     // if there is some prefix for the timeline in the remote storage.
    1625              :                     // This can e.g. be the initdb.tar.zst archive, maybe a
    1626              :                     // remnant from a prior incomplete creation or deletion attempt.
    1627              :                     // Delete the local directory as the deciding criterion for a
    1628              :                     // timeline's existence is presence of index_part.
    1629            0 :                     info!(%timeline_id, "index_part not found on remote");
    1630            0 :                     continue;
    1631              :                 }
    1632            0 :                 Err(DownloadError::Fatal(why)) => {
    1633            0 :                     // If, while loading one remote timeline, we saw an indication that our generation
    1634            0 :                     // number is likely invalid, then we should not load the whole tenant.
    1635            0 :                     error!(%timeline_id, "Fatal error loading timeline: {why}");
    1636            0 :                     anyhow::bail!(why.to_string());
    1637              :                 }
    1638            0 :                 Err(e) => {
    1639            0 :                     // Some (possibly ephemeral) error happened during index_part download.
    1640            0 :                     // Pretend the timeline exists to not delete the timeline directory,
    1641            0 :                     // as it might be a temporary issue and we don't want to re-download
    1642            0 :                     // everything after it resolves.
    1643            0 :                     warn!(%timeline_id, "Failed to load index_part from remote storage, failed creation? ({e})");
    1644              : 
    1645            0 :                     existent_timelines.insert(timeline_id);
    1646            0 :                     continue;
    1647              :                 }
    1648              :             };
    1649            6 :             match index_part {
    1650            6 :                 MaybeDeletedIndexPart::IndexPart(index_part) => {
    1651            6 :                     timeline_ancestors.insert(timeline_id, index_part.metadata.clone());
    1652            6 :                     remote_index_and_client.insert(timeline_id, (index_part, preload.client));
    1653            6 :                 }
    1654            0 :                 MaybeDeletedIndexPart::Deleted(index_part) => {
    1655            0 :                     info!(
    1656            0 :                         "timeline {} is deleted, picking to resume deletion",
    1657              :                         timeline_id
    1658              :                     );
    1659            0 :                     timelines_to_resume_deletions.push((timeline_id, index_part, preload.client));
    1660              :                 }
    1661              :             }
    1662              :         }
    1663              : 
    1664          196 :         let mut gc_blocks = HashMap::new();
    1665              : 
    1666              :         // For every timeline, download the metadata file, scan the local directory,
    1667              :         // and build a layer map that contains an entry for each remote and local
    1668              :         // layer file.
    1669          196 :         let sorted_timelines = tree_sort_timelines(timeline_ancestors, |m| m.ancestor_timeline())?;
    1670          202 :         for (timeline_id, remote_metadata) in sorted_timelines {
    1671            6 :             let (index_part, remote_client) = remote_index_and_client
    1672            6 :                 .remove(&timeline_id)
    1673            6 :                 .expect("just put it in above");
    1674              : 
    1675            6 :             if let Some(blocking) = index_part.gc_blocking.as_ref() {
    1676              :                 // could just filter these away, but it helps while testing
    1677            0 :                 anyhow::ensure!(
    1678            0 :                     !blocking.reasons.is_empty(),
    1679            0 :                     "index_part for {timeline_id} is malformed: it should not have gc blocking with zero reasons"
    1680              :                 );
    1681            0 :                 let prev = gc_blocks.insert(timeline_id, blocking.reasons);
    1682            0 :                 assert!(prev.is_none());
    1683            6 :             }
    1684              : 
    1685              :             // TODO again handle early failure
    1686            6 :             let effect = self
    1687            6 :                 .load_remote_timeline(
    1688            6 :                     timeline_id,
    1689            6 :                     index_part,
    1690            6 :                     remote_metadata,
    1691            6 :                     TimelineResources {
    1692            6 :                         remote_client,
    1693            6 :                         pagestream_throttle: self.pagestream_throttle.clone(),
    1694            6 :                         l0_flush_global_state: self.l0_flush_global_state.clone(),
    1695            6 :                     },
    1696            6 :                     LoadTimelineCause::Attach,
    1697            6 :                     ctx,
    1698            6 :                 )
    1699            6 :                 .await
    1700            6 :                 .with_context(|| {
    1701            0 :                     format!(
    1702            0 :                         "failed to load remote timeline {} for tenant {}",
    1703            0 :                         timeline_id, self.tenant_shard_id
    1704            0 :                     )
    1705            6 :                 })?;
    1706              : 
    1707            6 :             match effect {
    1708            6 :                 TimelineInitAndSyncResult::ReadyToActivate(_) => {
    1709            6 :                     // activation happens later, on Tenant::activate
    1710            6 :                 }
    1711              :                 TimelineInitAndSyncResult::NeedsSpawnImportPgdata(
    1712              :                     TimelineInitAndSyncNeedsSpawnImportPgdata {
    1713            0 :                         timeline,
    1714            0 :                         import_pgdata,
    1715            0 :                         guard,
    1716            0 :                     },
    1717            0 :                 ) => {
    1718            0 :                     tokio::task::spawn(self.clone().create_timeline_import_pgdata_task(
    1719            0 :                         timeline,
    1720            0 :                         import_pgdata,
    1721            0 :                         ActivateTimelineArgs::No,
    1722            0 :                         guard,
    1723            0 :                     ));
    1724            0 :                 }
    1725              :             }
    1726              :         }
    1727              : 
    1728              :         // Walk through deleted timelines, resume deletion
    1729          196 :         for (timeline_id, index_part, remote_timeline_client) in timelines_to_resume_deletions {
    1730            0 :             remote_timeline_client
    1731            0 :                 .init_upload_queue_stopped_to_continue_deletion(&index_part)
    1732            0 :                 .context("init queue stopped")
    1733            0 :                 .map_err(LoadLocalTimelineError::ResumeDeletion)?;
    1734              : 
    1735            0 :             DeleteTimelineFlow::resume_deletion(
    1736            0 :                 Arc::clone(self),
    1737            0 :                 timeline_id,
    1738            0 :                 &index_part.metadata,
    1739            0 :                 remote_timeline_client,
    1740            0 :             )
    1741            0 :             .instrument(tracing::info_span!("timeline_delete", %timeline_id))
    1742            0 :             .await
    1743            0 :             .context("resume_deletion")
    1744            0 :             .map_err(LoadLocalTimelineError::ResumeDeletion)?;
    1745              :         }
    1746          196 :         let needs_manifest_upload =
    1747          196 :             offloaded_timelines_list.len() != preload.tenant_manifest.offloaded_timelines.len();
    1748          196 :         {
    1749          196 :             let mut offloaded_timelines_accessor = self.timelines_offloaded.lock().unwrap();
    1750          196 :             offloaded_timelines_accessor.extend(offloaded_timelines_list.into_iter());
    1751          196 :         }
    1752          196 :         if needs_manifest_upload {
    1753            0 :             self.store_tenant_manifest().await?;
    1754          196 :         }
    1755              : 
    1756              :         // The local filesystem contents are a cache of what's in the remote IndexPart;
    1757              :         // IndexPart is the source of truth.
    1758          196 :         self.clean_up_timelines(&existent_timelines)?;
    1759              : 
    1760          196 :         self.gc_block.set_scanned(gc_blocks);
    1761          196 : 
    1762          196 :         fail::fail_point!("attach-before-activate", |_| {
    1763            0 :             anyhow::bail!("attach-before-activate");
    1764          196 :         });
    1765          196 :         failpoint_support::sleep_millis_async!("attach-before-activate-sleep", &self.cancel);
    1766              : 
    1767          196 :         info!("Done");
    1768              : 
    1769          196 :         Ok(())
    1770          196 :     }
    1771              : 
    1772              :     /// Check for any local timeline directories that are temporary, or do not correspond to a
    1773              :     /// timeline that still exists: this can happen if we crashed during a deletion/creation, or
    1774              :     /// if a timeline was deleted while the tenant was attached to a different pageserver.
    1775          196 :     fn clean_up_timelines(&self, existent_timelines: &HashSet<TimelineId>) -> anyhow::Result<()> {
    1776          196 :         let timelines_dir = self.conf.timelines_path(&self.tenant_shard_id);
    1777              : 
    1778          196 :         let entries = match timelines_dir.read_dir_utf8() {
    1779          196 :             Ok(d) => d,
    1780            0 :             Err(e) => {
    1781            0 :                 if e.kind() == std::io::ErrorKind::NotFound {
    1782            0 :                     return Ok(());
    1783              :                 } else {
    1784            0 :                     return Err(e).context("list timelines directory for tenant");
    1785              :                 }
    1786              :             }
    1787              :         };
    1788              : 
    1789          204 :         for entry in entries {
    1790            8 :             let entry = entry.context("read timeline dir entry")?;
    1791            8 :             let entry_path = entry.path();
    1792              : 
    1793            8 :             let purge = if crate::is_temporary(entry_path)
    1794              :                 // TODO: remove uninit mark code (https://github.com/neondatabase/neon/issues/5718)
    1795            8 :                 || is_uninit_mark(entry_path)
    1796            8 :                 || crate::is_delete_mark(entry_path)
    1797              :             {
    1798            0 :                 true
    1799              :             } else {
    1800            8 :                 match TimelineId::try_from(entry_path.file_name()) {
    1801            8 :                     Ok(i) => {
    1802            8 :                         // Purge if the timeline ID does not exist in remote storage: remote storage is the authority.
    1803            8 :                         !existent_timelines.contains(&i)
    1804              :                     }
    1805            0 :                     Err(e) => {
    1806            0 :                         tracing::warn!(
    1807            0 :                             "Unparseable directory in timelines directory: {entry_path}, ignoring ({e})"
    1808              :                         );
    1809              :                         // Do not purge junk: if we don't recognize it, be cautious and leave it for a human.
    1810            0 :                         false
    1811              :                     }
    1812              :                 }
    1813              :             };
    1814              : 
    1815            8 :             if purge {
    1816            2 :                 tracing::info!("Purging stale timeline dentry {entry_path}");
    1817            2 :                 if let Err(e) = match entry.file_type() {
    1818            2 :                     Ok(t) => if t.is_dir() {
    1819            2 :                         std::fs::remove_dir_all(entry_path)
    1820              :                     } else {
    1821            0 :                         std::fs::remove_file(entry_path)
    1822              :                     }
    1823            2 :                     .or_else(fs_ext::ignore_not_found),
    1824            0 :                     Err(e) => Err(e),
    1825              :                 } {
    1826            0 :                     tracing::warn!("Failed to purge stale timeline dentry {entry_path}: {e}");
    1827            2 :                 }
    1828            6 :             }
    1829              :         }
    1830              : 
    1831          196 :         Ok(())
    1832          196 :     }
    1833              : 
    1834              :     /// Get sum of all remote timelines sizes
    1835              :     ///
    1836              :     /// This function relies on the index_part instead of listing the remote storage
    1837            0 :     pub fn remote_size(&self) -> u64 {
    1838            0 :         let mut size = 0;
    1839              : 
    1840            0 :         for timeline in self.list_timelines() {
    1841            0 :             size += timeline.remote_client.get_remote_physical_size();
    1842            0 :         }
    1843              : 
    1844            0 :         size
    1845            0 :     }
    1846              : 
    1847            6 :     #[instrument(skip_all, fields(timeline_id=%timeline_id))]
    1848              :     async fn load_remote_timeline(
    1849              :         self: &Arc<Self>,
    1850              :         timeline_id: TimelineId,
    1851              :         index_part: IndexPart,
    1852              :         remote_metadata: TimelineMetadata,
    1853              :         resources: TimelineResources,
    1854              :         cause: LoadTimelineCause,
    1855              :         ctx: &RequestContext,
    1856              :     ) -> anyhow::Result<TimelineInitAndSyncResult> {
    1857              :         span::debug_assert_current_span_has_tenant_id();
    1858              : 
    1859              :         info!("downloading index file for timeline {}", timeline_id);
    1860              :         tokio::fs::create_dir_all(self.conf.timeline_path(&self.tenant_shard_id, &timeline_id))
    1861              :             .await
    1862              :             .context("Failed to create new timeline directory")?;
    1863              : 
    1864              :         let ancestor = if let Some(ancestor_id) = remote_metadata.ancestor_timeline() {
    1865              :             let timelines = self.timelines.lock().unwrap();
    1866              :             Some(Arc::clone(timelines.get(&ancestor_id).ok_or_else(
    1867            0 :                 || {
    1868            0 :                     anyhow::anyhow!(
    1869            0 :                         "cannot find ancestor timeline {ancestor_id} for timeline {timeline_id}"
    1870            0 :                     )
    1871            0 :                 },
    1872              :             )?))
    1873              :         } else {
    1874              :             None
    1875              :         };
    1876              : 
    1877              :         self.timeline_init_and_sync(
    1878              :             timeline_id,
    1879              :             resources,
    1880              :             index_part,
    1881              :             remote_metadata,
    1882              :             ancestor,
    1883              :             cause,
    1884              :             ctx,
    1885              :         )
    1886              :         .await
    1887              :     }
    1888              : 
    1889          196 :     async fn load_timelines_metadata(
    1890          196 :         self: &Arc<Tenant>,
    1891          196 :         timeline_ids: HashSet<TimelineId>,
    1892          196 :         remote_storage: &GenericRemoteStorage,
    1893          196 :         cancel: CancellationToken,
    1894          196 :     ) -> anyhow::Result<HashMap<TimelineId, TimelinePreload>> {
    1895          196 :         let mut part_downloads = JoinSet::new();
    1896          202 :         for timeline_id in timeline_ids {
    1897            6 :             let cancel_clone = cancel.clone();
    1898            6 :             part_downloads.spawn(
    1899            6 :                 self.load_timeline_metadata(timeline_id, remote_storage.clone(), cancel_clone)
    1900            6 :                     .instrument(info_span!("download_index_part", %timeline_id)),
    1901              :             );
    1902              :         }
    1903              : 
    1904          196 :         let mut timeline_preloads: HashMap<TimelineId, TimelinePreload> = HashMap::new();
    1905              : 
    1906              :         loop {
    1907          202 :             tokio::select!(
    1908          202 :                 next = part_downloads.join_next() => {
    1909          202 :                     match next {
    1910            6 :                         Some(result) => {
    1911            6 :                             let preload = result.context("join preload task")?;
    1912            6 :                             timeline_preloads.insert(preload.timeline_id, preload);
    1913              :                         },
    1914              :                         None => {
    1915          196 :                             break;
    1916              :                         }
    1917              :                     }
    1918              :                 },
    1919          202 :                 _ = cancel.cancelled() => {
    1920            0 :                     anyhow::bail!("Cancelled while waiting for remote index download")
    1921              :                 }
    1922              :             )
    1923              :         }
    1924              : 
    1925          196 :         Ok(timeline_preloads)
    1926          196 :     }
    1927              : 
    1928            6 :     fn build_timeline_client(
    1929            6 :         &self,
    1930            6 :         timeline_id: TimelineId,
    1931            6 :         remote_storage: GenericRemoteStorage,
    1932            6 :     ) -> RemoteTimelineClient {
    1933            6 :         RemoteTimelineClient::new(
    1934            6 :             remote_storage.clone(),
    1935            6 :             self.deletion_queue_client.clone(),
    1936            6 :             self.conf,
    1937            6 :             self.tenant_shard_id,
    1938            6 :             timeline_id,
    1939            6 :             self.generation,
    1940            6 :             &self.tenant_conf.load().location,
    1941            6 :         )
    1942            6 :     }
    1943              : 
    1944            6 :     fn load_timeline_metadata(
    1945            6 :         self: &Arc<Tenant>,
    1946            6 :         timeline_id: TimelineId,
    1947            6 :         remote_storage: GenericRemoteStorage,
    1948            6 :         cancel: CancellationToken,
    1949            6 :     ) -> impl Future<Output = TimelinePreload> {
    1950            6 :         let client = self.build_timeline_client(timeline_id, remote_storage);
    1951            6 :         async move {
    1952            6 :             debug_assert_current_span_has_tenant_and_timeline_id();
    1953            6 :             debug!("starting index part download");
    1954              : 
    1955            6 :             let index_part = client.download_index_file(&cancel).await;
    1956              : 
    1957            6 :             debug!("finished index part download");
    1958              : 
    1959            6 :             TimelinePreload {
    1960            6 :                 client,
    1961            6 :                 timeline_id,
    1962            6 :                 index_part,
    1963            6 :             }
    1964            6 :         }
    1965            6 :     }
    1966              : 
    1967            0 :     fn check_to_be_archived_has_no_unarchived_children(
    1968            0 :         timeline_id: TimelineId,
    1969            0 :         timelines: &std::sync::MutexGuard<'_, HashMap<TimelineId, Arc<Timeline>>>,
    1970            0 :     ) -> Result<(), TimelineArchivalError> {
    1971            0 :         let children: Vec<TimelineId> = timelines
    1972            0 :             .iter()
    1973            0 :             .filter_map(|(id, entry)| {
    1974            0 :                 if entry.get_ancestor_timeline_id() != Some(timeline_id) {
    1975            0 :                     return None;
    1976            0 :                 }
    1977            0 :                 if entry.is_archived() == Some(true) {
    1978            0 :                     return None;
    1979            0 :                 }
    1980            0 :                 Some(*id)
    1981            0 :             })
    1982            0 :             .collect();
    1983            0 : 
    1984            0 :         if !children.is_empty() {
    1985            0 :             return Err(TimelineArchivalError::HasUnarchivedChildren(children));
    1986            0 :         }
    1987            0 :         Ok(())
    1988            0 :     }
    1989              : 
    1990            0 :     fn check_ancestor_of_to_be_unarchived_is_not_archived(
    1991            0 :         ancestor_timeline_id: TimelineId,
    1992            0 :         timelines: &std::sync::MutexGuard<'_, HashMap<TimelineId, Arc<Timeline>>>,
    1993            0 :         offloaded_timelines: &std::sync::MutexGuard<
    1994            0 :             '_,
    1995            0 :             HashMap<TimelineId, Arc<OffloadedTimeline>>,
    1996            0 :         >,
    1997            0 :     ) -> Result<(), TimelineArchivalError> {
    1998            0 :         let has_archived_parent =
    1999            0 :             if let Some(ancestor_timeline) = timelines.get(&ancestor_timeline_id) {
    2000            0 :                 ancestor_timeline.is_archived() == Some(true)
    2001            0 :             } else if offloaded_timelines.contains_key(&ancestor_timeline_id) {
    2002            0 :                 true
    2003              :             } else {
    2004            0 :                 error!("ancestor timeline {ancestor_timeline_id} not found");
    2005            0 :                 if cfg!(debug_assertions) {
    2006            0 :                     panic!("ancestor timeline {ancestor_timeline_id} not found");
    2007            0 :                 }
    2008            0 :                 return Err(TimelineArchivalError::NotFound);
    2009              :             };
    2010            0 :         if has_archived_parent {
    2011            0 :             return Err(TimelineArchivalError::HasArchivedParent(
    2012            0 :                 ancestor_timeline_id,
    2013            0 :             ));
    2014            0 :         }
    2015            0 :         Ok(())
    2016            0 :     }
    2017              : 
    2018            0 :     fn check_to_be_unarchived_timeline_has_no_archived_parent(
    2019            0 :         timeline: &Arc<Timeline>,
    2020            0 :     ) -> Result<(), TimelineArchivalError> {
    2021            0 :         if let Some(ancestor_timeline) = timeline.ancestor_timeline() {
    2022            0 :             if ancestor_timeline.is_archived() == Some(true) {
    2023            0 :                 return Err(TimelineArchivalError::HasArchivedParent(
    2024            0 :                     ancestor_timeline.timeline_id,
    2025            0 :                 ));
    2026            0 :             }
    2027            0 :         }
    2028            0 :         Ok(())
    2029            0 :     }
    2030              : 
    2031              :     /// Loads the specified (offloaded) timeline from S3 and attaches it as a loaded timeline
    2032              :     ///
    2033              :     /// Counterpart to [`offload_timeline`].
    2034            0 :     async fn unoffload_timeline(
    2035            0 :         self: &Arc<Self>,
    2036            0 :         timeline_id: TimelineId,
    2037            0 :         broker_client: storage_broker::BrokerClientChannel,
    2038            0 :         ctx: RequestContext,
    2039            0 :     ) -> Result<Arc<Timeline>, TimelineArchivalError> {
    2040            0 :         info!("unoffloading timeline");
    2041              : 
    2042              :         // We activate the timeline below manually, so this must be called on an active timeline.
    2043              :         // We expect callers of this function to ensure this.
    2044            0 :         match self.current_state() {
    2045              :             TenantState::Activating { .. }
    2046              :             | TenantState::Attaching
    2047              :             | TenantState::Broken { .. } => {
    2048            0 :                 panic!("Timeline expected to be active")
    2049              :             }
    2050            0 :             TenantState::Stopping { .. } => return Err(TimelineArchivalError::Cancelled),
    2051            0 :             TenantState::Active => {}
    2052            0 :         }
    2053            0 :         let cancel = self.cancel.clone();
    2054            0 : 
    2055            0 :         // Protect against concurrent attempts to use this TimelineId
    2056            0 :         // We don't care much about idempotency, as it's ensured a layer above.
    2057            0 :         let allow_offloaded = true;
    2058            0 :         let _create_guard = self
    2059            0 :             .create_timeline_create_guard(
    2060            0 :                 timeline_id,
    2061            0 :                 CreateTimelineIdempotency::FailWithConflict,
    2062            0 :                 allow_offloaded,
    2063            0 :             )
    2064            0 :             .map_err(|err| match err {
    2065            0 :                 TimelineExclusionError::AlreadyCreating => TimelineArchivalError::AlreadyInProgress,
    2066              :                 TimelineExclusionError::AlreadyExists { .. } => {
    2067            0 :                     TimelineArchivalError::Other(anyhow::anyhow!("Timeline already exists"))
    2068              :                 }
    2069            0 :                 TimelineExclusionError::Other(e) => TimelineArchivalError::Other(e),
    2070            0 :                 TimelineExclusionError::ShuttingDown => TimelineArchivalError::Cancelled,
    2071            0 :             })?;
    2072              : 
    2073            0 :         let timeline_preload = self
    2074            0 :             .load_timeline_metadata(timeline_id, self.remote_storage.clone(), cancel.clone())
    2075            0 :             .await;
    2076              : 
    2077            0 :         let index_part = match timeline_preload.index_part {
    2078            0 :             Ok(index_part) => {
    2079            0 :                 debug!("remote index part exists for timeline {timeline_id}");
    2080            0 :                 index_part
    2081              :             }
    2082              :             Err(DownloadError::NotFound) => {
    2083            0 :                 error!(%timeline_id, "index_part not found on remote");
    2084            0 :                 return Err(TimelineArchivalError::NotFound);
    2085              :             }
    2086            0 :             Err(DownloadError::Cancelled) => return Err(TimelineArchivalError::Cancelled),
    2087            0 :             Err(e) => {
    2088            0 :                 // Some (possibly ephemeral) error happened during index_part download.
    2089            0 :                 warn!(%timeline_id, "Failed to load index_part from remote storage, failed creation? ({e})");
    2090            0 :                 return Err(TimelineArchivalError::Other(
    2091            0 :                     anyhow::Error::new(e).context("downloading index_part from remote storage"),
    2092            0 :                 ));
    2093              :             }
    2094              :         };
    2095            0 :         let index_part = match index_part {
    2096            0 :             MaybeDeletedIndexPart::IndexPart(index_part) => index_part,
    2097            0 :             MaybeDeletedIndexPart::Deleted(_index_part) => {
    2098            0 :                 info!("timeline is deleted according to index_part.json");
    2099            0 :                 return Err(TimelineArchivalError::NotFound);
    2100              :             }
    2101              :         };
    2102            0 :         let remote_metadata = index_part.metadata.clone();
    2103            0 :         let timeline_resources = self.build_timeline_resources(timeline_id);
    2104            0 :         self.load_remote_timeline(
    2105            0 :             timeline_id,
    2106            0 :             index_part,
    2107            0 :             remote_metadata,
    2108            0 :             timeline_resources,
    2109            0 :             LoadTimelineCause::Unoffload,
    2110            0 :             &ctx,
    2111            0 :         )
    2112            0 :         .await
    2113            0 :         .with_context(|| {
    2114            0 :             format!(
    2115            0 :                 "failed to load remote timeline {} for tenant {}",
    2116            0 :                 timeline_id, self.tenant_shard_id
    2117            0 :             )
    2118            0 :         })
    2119            0 :         .map_err(TimelineArchivalError::Other)?;
    2120              : 
    2121            0 :         let timeline = {
    2122            0 :             let timelines = self.timelines.lock().unwrap();
    2123            0 :             let Some(timeline) = timelines.get(&timeline_id) else {
    2124            0 :                 warn!("timeline not available directly after attach");
    2125              :                 // This is not a panic because no locks are held between `load_remote_timeline`
    2126              :                 // which puts the timeline into timelines, and our look into the timeline map.
    2127            0 :                 return Err(TimelineArchivalError::Other(anyhow::anyhow!(
    2128            0 :                     "timeline not available directly after attach"
    2129            0 :                 )));
    2130              :             };
    2131            0 :             let mut offloaded_timelines = self.timelines_offloaded.lock().unwrap();
    2132            0 :             match offloaded_timelines.remove(&timeline_id) {
    2133            0 :                 Some(offloaded) => {
    2134            0 :                     offloaded.delete_from_ancestor_with_timelines(&timelines);
    2135            0 :                 }
    2136            0 :                 None => warn!("timeline already removed from offloaded timelines"),
    2137              :             }
    2138              : 
    2139            0 :             self.initialize_gc_info(&timelines, &offloaded_timelines, Some(timeline_id));
    2140            0 : 
    2141            0 :             Arc::clone(timeline)
    2142            0 :         };
    2143            0 : 
    2144            0 :         // Upload new list of offloaded timelines to S3
    2145            0 :         self.store_tenant_manifest().await?;
    2146              : 
    2147              :         // Activate the timeline (if it makes sense)
    2148            0 :         if !(timeline.is_broken() || timeline.is_stopping()) {
    2149            0 :             let background_jobs_can_start = None;
    2150            0 :             timeline.activate(
    2151            0 :                 self.clone(),
    2152            0 :                 broker_client.clone(),
    2153            0 :                 background_jobs_can_start,
    2154            0 :                 &ctx,
    2155            0 :             );
    2156            0 :         }
    2157              : 
    2158            0 :         info!("timeline unoffloading complete");
    2159            0 :         Ok(timeline)
    2160            0 :     }
    2161              : 
    2162            0 :     pub(crate) async fn apply_timeline_archival_config(
    2163            0 :         self: &Arc<Self>,
    2164            0 :         timeline_id: TimelineId,
    2165            0 :         new_state: TimelineArchivalState,
    2166            0 :         broker_client: storage_broker::BrokerClientChannel,
    2167            0 :         ctx: RequestContext,
    2168            0 :     ) -> Result<(), TimelineArchivalError> {
    2169            0 :         info!("setting timeline archival config");
    2170              :         // First part: figure out what is needed to do, and do validation
    2171            0 :         let timeline_or_unarchive_offloaded = 'outer: {
    2172            0 :             let timelines = self.timelines.lock().unwrap();
    2173              : 
    2174            0 :             let Some(timeline) = timelines.get(&timeline_id) else {
    2175            0 :                 let offloaded_timelines = self.timelines_offloaded.lock().unwrap();
    2176            0 :                 let Some(offloaded) = offloaded_timelines.get(&timeline_id) else {
    2177            0 :                     return Err(TimelineArchivalError::NotFound);
    2178              :                 };
    2179            0 :                 if new_state == TimelineArchivalState::Archived {
    2180              :                     // It's offloaded already, so nothing to do
    2181            0 :                     return Ok(());
    2182            0 :                 }
    2183            0 :                 if let Some(ancestor_timeline_id) = offloaded.ancestor_timeline_id {
    2184            0 :                     Self::check_ancestor_of_to_be_unarchived_is_not_archived(
    2185            0 :                         ancestor_timeline_id,
    2186            0 :                         &timelines,
    2187            0 :                         &offloaded_timelines,
    2188            0 :                     )?;
    2189            0 :                 }
    2190            0 :                 break 'outer None;
    2191              :             };
    2192              : 
    2193              :             // Do some validation. We release the timelines lock below, so there is potential
    2194              :             // for race conditions: these checks are more present to prevent misunderstandings of
    2195              :             // the API's capabilities, instead of serving as the sole way to defend their invariants.
    2196            0 :             match new_state {
    2197              :                 TimelineArchivalState::Unarchived => {
    2198            0 :                     Self::check_to_be_unarchived_timeline_has_no_archived_parent(timeline)?
    2199              :                 }
    2200              :                 TimelineArchivalState::Archived => {
    2201            0 :                     Self::check_to_be_archived_has_no_unarchived_children(timeline_id, &timelines)?
    2202              :                 }
    2203              :             }
    2204            0 :             Some(Arc::clone(timeline))
    2205              :         };
    2206              : 
    2207              :         // Second part: unoffload timeline (if needed)
    2208            0 :         let timeline = if let Some(timeline) = timeline_or_unarchive_offloaded {
    2209            0 :             timeline
    2210              :         } else {
    2211              :             // Turn offloaded timeline into a non-offloaded one
    2212            0 :             self.unoffload_timeline(timeline_id, broker_client, ctx)
    2213            0 :                 .await?
    2214              :         };
    2215              : 
    2216              :         // Third part: upload new timeline archival state and block until it is present in S3
    2217            0 :         let upload_needed = match timeline
    2218            0 :             .remote_client
    2219            0 :             .schedule_index_upload_for_timeline_archival_state(new_state)
    2220              :         {
    2221            0 :             Ok(upload_needed) => upload_needed,
    2222            0 :             Err(e) => {
    2223            0 :                 if timeline.cancel.is_cancelled() {
    2224            0 :                     return Err(TimelineArchivalError::Cancelled);
    2225              :                 } else {
    2226            0 :                     return Err(TimelineArchivalError::Other(e));
    2227              :                 }
    2228              :             }
    2229              :         };
    2230              : 
    2231            0 :         if upload_needed {
    2232            0 :             info!("Uploading new state");
    2233              :             const MAX_WAIT: Duration = Duration::from_secs(10);
    2234            0 :             let Ok(v) =
    2235            0 :                 tokio::time::timeout(MAX_WAIT, timeline.remote_client.wait_completion()).await
    2236              :             else {
    2237            0 :                 tracing::warn!("reached timeout for waiting on upload queue");
    2238            0 :                 return Err(TimelineArchivalError::Timeout);
    2239              :             };
    2240            0 :             v.map_err(|e| match e {
    2241            0 :                 WaitCompletionError::NotInitialized(e) => {
    2242            0 :                     TimelineArchivalError::Other(anyhow::anyhow!(e))
    2243              :                 }
    2244              :                 WaitCompletionError::UploadQueueShutDownOrStopped => {
    2245            0 :                     TimelineArchivalError::Cancelled
    2246              :                 }
    2247            0 :             })?;
    2248            0 :         }
    2249            0 :         Ok(())
    2250            0 :     }
    2251              : 
    2252            2 :     pub fn get_offloaded_timeline(
    2253            2 :         &self,
    2254            2 :         timeline_id: TimelineId,
    2255            2 :     ) -> Result<Arc<OffloadedTimeline>, GetTimelineError> {
    2256            2 :         self.timelines_offloaded
    2257            2 :             .lock()
    2258            2 :             .unwrap()
    2259            2 :             .get(&timeline_id)
    2260            2 :             .map(Arc::clone)
    2261            2 :             .ok_or(GetTimelineError::NotFound {
    2262            2 :                 tenant_id: self.tenant_shard_id,
    2263            2 :                 timeline_id,
    2264            2 :             })
    2265            2 :     }
    2266              : 
    2267            4 :     pub(crate) fn tenant_shard_id(&self) -> TenantShardId {
    2268            4 :         self.tenant_shard_id
    2269            4 :     }
    2270              : 
    2271              :     /// Get Timeline handle for given Neon timeline ID.
    2272              :     /// This function is idempotent. It doesn't change internal state in any way.
    2273          222 :     pub fn get_timeline(
    2274          222 :         &self,
    2275          222 :         timeline_id: TimelineId,
    2276          222 :         active_only: bool,
    2277          222 :     ) -> Result<Arc<Timeline>, GetTimelineError> {
    2278          222 :         let timelines_accessor = self.timelines.lock().unwrap();
    2279          222 :         let timeline = timelines_accessor
    2280          222 :             .get(&timeline_id)
    2281          222 :             .ok_or(GetTimelineError::NotFound {
    2282          222 :                 tenant_id: self.tenant_shard_id,
    2283          222 :                 timeline_id,
    2284          222 :             })?;
    2285              : 
    2286          220 :         if active_only && !timeline.is_active() {
    2287            0 :             Err(GetTimelineError::NotActive {
    2288            0 :                 tenant_id: self.tenant_shard_id,
    2289            0 :                 timeline_id,
    2290            0 :                 state: timeline.current_state(),
    2291            0 :             })
    2292              :         } else {
    2293          220 :             Ok(Arc::clone(timeline))
    2294              :         }
    2295          222 :     }
    2296              : 
    2297              :     /// Lists timelines the tenant contains.
    2298              :     /// It's up to callers to omit certain timelines that are not considered ready for use.
    2299            0 :     pub fn list_timelines(&self) -> Vec<Arc<Timeline>> {
    2300            0 :         self.timelines
    2301            0 :             .lock()
    2302            0 :             .unwrap()
    2303            0 :             .values()
    2304            0 :             .map(Arc::clone)
    2305            0 :             .collect()
    2306            0 :     }
    2307              : 
    2308              :     /// Lists timelines the tenant manages, including offloaded ones.
    2309              :     ///
    2310              :     /// It's up to callers to omit certain timelines that are not considered ready for use.
    2311            0 :     pub fn list_timelines_and_offloaded(
    2312            0 :         &self,
    2313            0 :     ) -> (Vec<Arc<Timeline>>, Vec<Arc<OffloadedTimeline>>) {
    2314            0 :         let timelines = self
    2315            0 :             .timelines
    2316            0 :             .lock()
    2317            0 :             .unwrap()
    2318            0 :             .values()
    2319            0 :             .map(Arc::clone)
    2320            0 :             .collect();
    2321            0 :         let offloaded = self
    2322            0 :             .timelines_offloaded
    2323            0 :             .lock()
    2324            0 :             .unwrap()
    2325            0 :             .values()
    2326            0 :             .map(Arc::clone)
    2327            0 :             .collect();
    2328            0 :         (timelines, offloaded)
    2329            0 :     }
    2330              : 
    2331            0 :     pub fn list_timeline_ids(&self) -> Vec<TimelineId> {
    2332            0 :         self.timelines.lock().unwrap().keys().cloned().collect()
    2333            0 :     }
    2334              : 
    2335              :     /// This is used by tests & import-from-basebackup.
    2336              :     ///
    2337              :     /// The returned [`UninitializedTimeline`] contains no data nor metadata and it is in
    2338              :     /// a state that will fail [`Tenant::load_remote_timeline`] because `disk_consistent_lsn=Lsn(0)`.
    2339              :     ///
    2340              :     /// The caller is responsible for getting the timeline into a state that will be accepted
    2341              :     /// by [`Tenant::load_remote_timeline`] / [`Tenant::attach`].
    2342              :     /// Then they may call [`UninitializedTimeline::finish_creation`] to add the timeline
    2343              :     /// to the [`Tenant::timelines`].
    2344              :     ///
    2345              :     /// Tests should use `Tenant::create_test_timeline` to set up the minimum required metadata keys.
    2346          188 :     pub(crate) async fn create_empty_timeline(
    2347          188 :         self: &Arc<Self>,
    2348          188 :         new_timeline_id: TimelineId,
    2349          188 :         initdb_lsn: Lsn,
    2350          188 :         pg_version: u32,
    2351          188 :         _ctx: &RequestContext,
    2352          188 :     ) -> anyhow::Result<UninitializedTimeline> {
    2353          188 :         anyhow::ensure!(
    2354          188 :             self.is_active(),
    2355            0 :             "Cannot create empty timelines on inactive tenant"
    2356              :         );
    2357              : 
    2358              :         // Protect against concurrent attempts to use this TimelineId
    2359          188 :         let create_guard = match self
    2360          188 :             .start_creating_timeline(new_timeline_id, CreateTimelineIdempotency::FailWithConflict)
    2361          188 :             .await?
    2362              :         {
    2363          186 :             StartCreatingTimelineResult::CreateGuard(guard) => guard,
    2364              :             StartCreatingTimelineResult::Idempotent(_) => {
    2365            0 :                 unreachable!("FailWithConflict implies we get an error instead")
    2366              :             }
    2367              :         };
    2368              : 
    2369          186 :         let new_metadata = TimelineMetadata::new(
    2370          186 :             // Initialize disk_consistent LSN to 0, The caller must import some data to
    2371          186 :             // make it valid, before calling finish_creation()
    2372          186 :             Lsn(0),
    2373          186 :             None,
    2374          186 :             None,
    2375          186 :             Lsn(0),
    2376          186 :             initdb_lsn,
    2377          186 :             initdb_lsn,
    2378          186 :             pg_version,
    2379          186 :         );
    2380          186 :         self.prepare_new_timeline(
    2381          186 :             new_timeline_id,
    2382          186 :             &new_metadata,
    2383          186 :             create_guard,
    2384          186 :             initdb_lsn,
    2385          186 :             None,
    2386          186 :         )
    2387          186 :         .await
    2388          188 :     }
    2389              : 
    2390              :     /// Helper for unit tests to create an empty timeline.
    2391              :     ///
    2392              :     /// The timeline is has state value `Active` but its background loops are not running.
    2393              :     // This makes the various functions which anyhow::ensure! for Active state work in tests.
    2394              :     // Our current tests don't need the background loops.
    2395              :     #[cfg(test)]
    2396          178 :     pub async fn create_test_timeline(
    2397          178 :         self: &Arc<Self>,
    2398          178 :         new_timeline_id: TimelineId,
    2399          178 :         initdb_lsn: Lsn,
    2400          178 :         pg_version: u32,
    2401          178 :         ctx: &RequestContext,
    2402          178 :     ) -> anyhow::Result<Arc<Timeline>> {
    2403          178 :         let uninit_tl = self
    2404          178 :             .create_empty_timeline(new_timeline_id, initdb_lsn, pg_version, ctx)
    2405          178 :             .await?;
    2406          178 :         let tline = uninit_tl.raw_timeline().expect("we just created it");
    2407          178 :         assert_eq!(tline.get_last_record_lsn(), Lsn(0));
    2408              : 
    2409              :         // Setup minimum keys required for the timeline to be usable.
    2410          178 :         let mut modification = tline.begin_modification(initdb_lsn);
    2411          178 :         modification
    2412          178 :             .init_empty_test_timeline()
    2413          178 :             .context("init_empty_test_timeline")?;
    2414          178 :         modification
    2415          178 :             .commit(ctx)
    2416          178 :             .await
    2417          178 :             .context("commit init_empty_test_timeline modification")?;
    2418              : 
    2419              :         // Flush to disk so that uninit_tl's check for valid disk_consistent_lsn passes.
    2420          178 :         tline.maybe_spawn_flush_loop();
    2421          178 :         tline.freeze_and_flush().await.context("freeze_and_flush")?;
    2422              : 
    2423              :         // Make sure the freeze_and_flush reaches remote storage.
    2424          178 :         tline.remote_client.wait_completion().await.unwrap();
    2425              : 
    2426          178 :         let tl = uninit_tl.finish_creation()?;
    2427              :         // The non-test code would call tl.activate() here.
    2428          178 :         tl.set_state(TimelineState::Active);
    2429          178 :         Ok(tl)
    2430          178 :     }
    2431              : 
    2432              :     /// Helper for unit tests to create a timeline with some pre-loaded states.
    2433              :     #[cfg(test)]
    2434              :     #[allow(clippy::too_many_arguments)]
    2435           36 :     pub async fn create_test_timeline_with_layers(
    2436           36 :         self: &Arc<Self>,
    2437           36 :         new_timeline_id: TimelineId,
    2438           36 :         initdb_lsn: Lsn,
    2439           36 :         pg_version: u32,
    2440           36 :         ctx: &RequestContext,
    2441           36 :         delta_layer_desc: Vec<timeline::DeltaLayerTestDesc>,
    2442           36 :         image_layer_desc: Vec<(Lsn, Vec<(pageserver_api::key::Key, bytes::Bytes)>)>,
    2443           36 :         end_lsn: Lsn,
    2444           36 :     ) -> anyhow::Result<Arc<Timeline>> {
    2445              :         use checks::check_valid_layermap;
    2446              :         use itertools::Itertools;
    2447              : 
    2448           36 :         let tline = self
    2449           36 :             .create_test_timeline(new_timeline_id, initdb_lsn, pg_version, ctx)
    2450           36 :             .await?;
    2451           36 :         tline.force_advance_lsn(end_lsn);
    2452          120 :         for deltas in delta_layer_desc {
    2453           84 :             tline
    2454           84 :                 .force_create_delta_layer(deltas, Some(initdb_lsn), ctx)
    2455           84 :                 .await?;
    2456              :         }
    2457           88 :         for (lsn, images) in image_layer_desc {
    2458           52 :             tline
    2459           52 :                 .force_create_image_layer(lsn, images, Some(initdb_lsn), ctx)
    2460           52 :                 .await?;
    2461              :         }
    2462           36 :         let layer_names = tline
    2463           36 :             .layers
    2464           36 :             .read()
    2465           36 :             .await
    2466           36 :             .layer_map()
    2467           36 :             .unwrap()
    2468           36 :             .iter_historic_layers()
    2469          172 :             .map(|layer| layer.layer_name())
    2470           36 :             .collect_vec();
    2471           36 :         if let Some(err) = check_valid_layermap(&layer_names) {
    2472            0 :             bail!("invalid layermap: {err}");
    2473           36 :         }
    2474           36 :         Ok(tline)
    2475           36 :     }
    2476              : 
    2477              :     /// Create a new timeline.
    2478              :     ///
    2479              :     /// Returns the new timeline ID and reference to its Timeline object.
    2480              :     ///
    2481              :     /// If the caller specified the timeline ID to use (`new_timeline_id`), and timeline with
    2482              :     /// the same timeline ID already exists, returns CreateTimelineError::AlreadyExists.
    2483              :     #[allow(clippy::too_many_arguments)]
    2484            0 :     pub(crate) async fn create_timeline(
    2485            0 :         self: &Arc<Tenant>,
    2486            0 :         params: CreateTimelineParams,
    2487            0 :         broker_client: storage_broker::BrokerClientChannel,
    2488            0 :         ctx: &RequestContext,
    2489            0 :     ) -> Result<Arc<Timeline>, CreateTimelineError> {
    2490            0 :         if !self.is_active() {
    2491            0 :             if matches!(self.current_state(), TenantState::Stopping { .. }) {
    2492            0 :                 return Err(CreateTimelineError::ShuttingDown);
    2493              :             } else {
    2494            0 :                 return Err(CreateTimelineError::Other(anyhow::anyhow!(
    2495            0 :                     "Cannot create timelines on inactive tenant"
    2496            0 :                 )));
    2497              :             }
    2498            0 :         }
    2499              : 
    2500            0 :         let _gate = self
    2501            0 :             .gate
    2502            0 :             .enter()
    2503            0 :             .map_err(|_| CreateTimelineError::ShuttingDown)?;
    2504              : 
    2505            0 :         let result: CreateTimelineResult = match params {
    2506              :             CreateTimelineParams::Bootstrap(CreateTimelineParamsBootstrap {
    2507            0 :                 new_timeline_id,
    2508            0 :                 existing_initdb_timeline_id,
    2509            0 :                 pg_version,
    2510            0 :             }) => {
    2511            0 :                 self.bootstrap_timeline(
    2512            0 :                     new_timeline_id,
    2513            0 :                     pg_version,
    2514            0 :                     existing_initdb_timeline_id,
    2515            0 :                     ctx,
    2516            0 :                 )
    2517            0 :                 .await?
    2518              :             }
    2519              :             CreateTimelineParams::Branch(CreateTimelineParamsBranch {
    2520            0 :                 new_timeline_id,
    2521            0 :                 ancestor_timeline_id,
    2522            0 :                 mut ancestor_start_lsn,
    2523              :             }) => {
    2524            0 :                 let ancestor_timeline = self
    2525            0 :                     .get_timeline(ancestor_timeline_id, false)
    2526            0 :                     .context("Cannot branch off the timeline that's not present in pageserver")?;
    2527              : 
    2528              :                 // instead of waiting around, just deny the request because ancestor is not yet
    2529              :                 // ready for other purposes either.
    2530            0 :                 if !ancestor_timeline.is_active() {
    2531            0 :                     return Err(CreateTimelineError::AncestorNotActive);
    2532            0 :                 }
    2533            0 : 
    2534            0 :                 if ancestor_timeline.is_archived() == Some(true) {
    2535            0 :                     info!("tried to branch archived timeline");
    2536            0 :                     return Err(CreateTimelineError::AncestorArchived);
    2537            0 :                 }
    2538              : 
    2539            0 :                 if let Some(lsn) = ancestor_start_lsn.as_mut() {
    2540            0 :                     *lsn = lsn.align();
    2541            0 : 
    2542            0 :                     let ancestor_ancestor_lsn = ancestor_timeline.get_ancestor_lsn();
    2543            0 :                     if ancestor_ancestor_lsn > *lsn {
    2544              :                         // can we safely just branch from the ancestor instead?
    2545            0 :                         return Err(CreateTimelineError::AncestorLsn(anyhow::anyhow!(
    2546            0 :                             "invalid start lsn {} for ancestor timeline {}: less than timeline ancestor lsn {}",
    2547            0 :                             lsn,
    2548            0 :                             ancestor_timeline_id,
    2549            0 :                             ancestor_ancestor_lsn,
    2550            0 :                         )));
    2551            0 :                     }
    2552            0 : 
    2553            0 :                     // Wait for the WAL to arrive and be processed on the parent branch up
    2554            0 :                     // to the requested branch point. The repository code itself doesn't
    2555            0 :                     // require it, but if we start to receive WAL on the new timeline,
    2556            0 :                     // decoding the new WAL might need to look up previous pages, relation
    2557            0 :                     // sizes etc. and that would get confused if the previous page versions
    2558            0 :                     // are not in the repository yet.
    2559            0 :                     ancestor_timeline
    2560            0 :                         .wait_lsn(*lsn, timeline::WaitLsnWaiter::Tenant, ctx)
    2561            0 :                         .await
    2562            0 :                         .map_err(|e| match e {
    2563            0 :                             e @ (WaitLsnError::Timeout(_) | WaitLsnError::BadState { .. }) => {
    2564            0 :                                 CreateTimelineError::AncestorLsn(anyhow::anyhow!(e))
    2565              :                             }
    2566            0 :                             WaitLsnError::Shutdown => CreateTimelineError::ShuttingDown,
    2567            0 :                         })?;
    2568            0 :                 }
    2569              : 
    2570            0 :                 self.branch_timeline(&ancestor_timeline, new_timeline_id, ancestor_start_lsn, ctx)
    2571            0 :                     .await?
    2572              :             }
    2573            0 :             CreateTimelineParams::ImportPgdata(params) => {
    2574            0 :                 self.create_timeline_import_pgdata(
    2575            0 :                     params,
    2576            0 :                     ActivateTimelineArgs::Yes {
    2577            0 :                         broker_client: broker_client.clone(),
    2578            0 :                     },
    2579            0 :                     ctx,
    2580            0 :                 )
    2581            0 :                 .await?
    2582              :             }
    2583              :         };
    2584              : 
    2585              :         // At this point we have dropped our guard on [`Self::timelines_creating`], and
    2586              :         // the timeline is visible in [`Self::timelines`], but it is _not_ durable yet.  We must
    2587              :         // not send a success to the caller until it is.  The same applies to idempotent retries.
    2588              :         //
    2589              :         // TODO: the timeline is already visible in [`Self::timelines`]; a caller could incorrectly
    2590              :         // assume that, because they can see the timeline via API, that the creation is done and
    2591              :         // that it is durable. Ideally, we would keep the timeline hidden (in [`Self::timelines_creating`])
    2592              :         // until it is durable, e.g., by extending the time we hold the creation guard. This also
    2593              :         // interacts with UninitializedTimeline and is generally a bit tricky.
    2594              :         //
    2595              :         // To re-emphasize: the only correct way to create a timeline is to repeat calling the
    2596              :         // creation API until it returns success. Only then is durability guaranteed.
    2597            0 :         info!(creation_result=%result.discriminant(), "waiting for timeline to be durable");
    2598            0 :         result
    2599            0 :             .timeline()
    2600            0 :             .remote_client
    2601            0 :             .wait_completion()
    2602            0 :             .await
    2603            0 :             .map_err(|e| match e {
    2604              :                 WaitCompletionError::NotInitialized(
    2605            0 :                     e, // If the queue is already stopped, it's a shutdown error.
    2606            0 :                 ) if e.is_stopping() => CreateTimelineError::ShuttingDown,
    2607              :                 WaitCompletionError::NotInitialized(_) => {
    2608              :                     // This is a bug: we should never try to wait for uploads before initializing the timeline
    2609            0 :                     debug_assert!(false);
    2610            0 :                     CreateTimelineError::Other(anyhow::anyhow!("timeline not initialized"))
    2611              :                 }
    2612              :                 WaitCompletionError::UploadQueueShutDownOrStopped => {
    2613            0 :                     CreateTimelineError::ShuttingDown
    2614              :                 }
    2615            0 :             })?;
    2616              : 
    2617              :         // The creating task is responsible for activating the timeline.
    2618              :         // We do this after `wait_completion()` so that we don't spin up tasks that start
    2619              :         // doing stuff before the IndexPart is durable in S3, which is done by the previous section.
    2620            0 :         let activated_timeline = match result {
    2621            0 :             CreateTimelineResult::Created(timeline) => {
    2622            0 :                 timeline.activate(self.clone(), broker_client, None, ctx);
    2623            0 :                 timeline
    2624              :             }
    2625            0 :             CreateTimelineResult::Idempotent(timeline) => {
    2626            0 :                 info!(
    2627            0 :                     "request was deemed idempotent, activation will be done by the creating task"
    2628              :                 );
    2629            0 :                 timeline
    2630              :             }
    2631            0 :             CreateTimelineResult::ImportSpawned(timeline) => {
    2632            0 :                 info!("import task spawned, timeline will become visible and activated once the import is done");
    2633            0 :                 timeline
    2634              :             }
    2635              :         };
    2636              : 
    2637            0 :         Ok(activated_timeline)
    2638            0 :     }
    2639              : 
    2640              :     /// The returned [`Arc<Timeline>`] is NOT in the [`Tenant::timelines`] map until the import
    2641              :     /// completes in the background. A DIFFERENT [`Arc<Timeline>`] will be inserted into the
    2642              :     /// [`Tenant::timelines`] map when the import completes.
    2643              :     /// We only return an [`Arc<Timeline>`] here so the API handler can create a [`pageserver_api::models::TimelineInfo`]
    2644              :     /// for the response.
    2645            0 :     async fn create_timeline_import_pgdata(
    2646            0 :         self: &Arc<Tenant>,
    2647            0 :         params: CreateTimelineParamsImportPgdata,
    2648            0 :         activate: ActivateTimelineArgs,
    2649            0 :         ctx: &RequestContext,
    2650            0 :     ) -> Result<CreateTimelineResult, CreateTimelineError> {
    2651            0 :         let CreateTimelineParamsImportPgdata {
    2652            0 :             new_timeline_id,
    2653            0 :             location,
    2654            0 :             idempotency_key,
    2655            0 :         } = params;
    2656            0 : 
    2657            0 :         let started_at = chrono::Utc::now().naive_utc();
    2658              : 
    2659              :         //
    2660              :         // There's probably a simpler way to upload an index part, but, remote_timeline_client
    2661              :         // is the canonical way we do it.
    2662              :         // - create an empty timeline in-memory
    2663              :         // - use its remote_timeline_client to do the upload
    2664              :         // - dispose of the uninit timeline
    2665              :         // - keep the creation guard alive
    2666              : 
    2667            0 :         let timeline_create_guard = match self
    2668            0 :             .start_creating_timeline(
    2669            0 :                 new_timeline_id,
    2670            0 :                 CreateTimelineIdempotency::ImportPgdata(CreatingTimelineIdempotencyImportPgdata {
    2671            0 :                     idempotency_key: idempotency_key.clone(),
    2672            0 :                 }),
    2673            0 :             )
    2674            0 :             .await?
    2675              :         {
    2676            0 :             StartCreatingTimelineResult::CreateGuard(guard) => guard,
    2677            0 :             StartCreatingTimelineResult::Idempotent(timeline) => {
    2678            0 :                 return Ok(CreateTimelineResult::Idempotent(timeline))
    2679              :             }
    2680              :         };
    2681              : 
    2682            0 :         let mut uninit_timeline = {
    2683            0 :             let this = &self;
    2684            0 :             let initdb_lsn = Lsn(0);
    2685            0 :             let _ctx = ctx;
    2686            0 :             async move {
    2687            0 :                 let new_metadata = TimelineMetadata::new(
    2688            0 :                     // Initialize disk_consistent LSN to 0, The caller must import some data to
    2689            0 :                     // make it valid, before calling finish_creation()
    2690            0 :                     Lsn(0),
    2691            0 :                     None,
    2692            0 :                     None,
    2693            0 :                     Lsn(0),
    2694            0 :                     initdb_lsn,
    2695            0 :                     initdb_lsn,
    2696            0 :                     15,
    2697            0 :                 );
    2698            0 :                 this.prepare_new_timeline(
    2699            0 :                     new_timeline_id,
    2700            0 :                     &new_metadata,
    2701            0 :                     timeline_create_guard,
    2702            0 :                     initdb_lsn,
    2703            0 :                     None,
    2704            0 :                 )
    2705            0 :                 .await
    2706            0 :             }
    2707            0 :         }
    2708            0 :         .await?;
    2709              : 
    2710            0 :         let in_progress = import_pgdata::index_part_format::InProgress {
    2711            0 :             idempotency_key,
    2712            0 :             location,
    2713            0 :             started_at,
    2714            0 :         };
    2715            0 :         let index_part = import_pgdata::index_part_format::Root::V1(
    2716            0 :             import_pgdata::index_part_format::V1::InProgress(in_progress),
    2717            0 :         );
    2718            0 :         uninit_timeline
    2719            0 :             .raw_timeline()
    2720            0 :             .unwrap()
    2721            0 :             .remote_client
    2722            0 :             .schedule_index_upload_for_import_pgdata_state_update(Some(index_part.clone()))?;
    2723              : 
    2724              :         // wait_completion happens in caller
    2725              : 
    2726            0 :         let (timeline, timeline_create_guard) = uninit_timeline.finish_creation_myself();
    2727            0 : 
    2728            0 :         tokio::spawn(self.clone().create_timeline_import_pgdata_task(
    2729            0 :             timeline.clone(),
    2730            0 :             index_part,
    2731            0 :             activate,
    2732            0 :             timeline_create_guard,
    2733            0 :         ));
    2734            0 : 
    2735            0 :         // NB: the timeline doesn't exist in self.timelines at this point
    2736            0 :         Ok(CreateTimelineResult::ImportSpawned(timeline))
    2737            0 :     }
    2738              : 
    2739            0 :     #[instrument(skip_all, fields(tenant_id=%self.tenant_shard_id.tenant_id, shard_id=%self.tenant_shard_id.shard_slug(), timeline_id=%timeline.timeline_id))]
    2740              :     async fn create_timeline_import_pgdata_task(
    2741              :         self: Arc<Tenant>,
    2742              :         timeline: Arc<Timeline>,
    2743              :         index_part: import_pgdata::index_part_format::Root,
    2744              :         activate: ActivateTimelineArgs,
    2745              :         timeline_create_guard: TimelineCreateGuard,
    2746              :     ) {
    2747              :         debug_assert_current_span_has_tenant_and_timeline_id();
    2748              :         info!("starting");
    2749              :         scopeguard::defer! {info!("exiting")};
    2750              : 
    2751              :         let res = self
    2752              :             .create_timeline_import_pgdata_task_impl(
    2753              :                 timeline,
    2754              :                 index_part,
    2755              :                 activate,
    2756              :                 timeline_create_guard,
    2757              :             )
    2758              :             .await;
    2759              :         if let Err(err) = &res {
    2760              :             error!(?err, "task failed");
    2761              :             // TODO sleep & retry, sensitive to tenant shutdown
    2762              :             // TODO: allow timeline deletion requests => should cancel the task
    2763              :         }
    2764              :     }
    2765              : 
    2766            0 :     async fn create_timeline_import_pgdata_task_impl(
    2767            0 :         self: Arc<Tenant>,
    2768            0 :         timeline: Arc<Timeline>,
    2769            0 :         index_part: import_pgdata::index_part_format::Root,
    2770            0 :         activate: ActivateTimelineArgs,
    2771            0 :         timeline_create_guard: TimelineCreateGuard,
    2772            0 :     ) -> Result<(), anyhow::Error> {
    2773            0 :         let ctx = RequestContext::new(TaskKind::ImportPgdata, DownloadBehavior::Warn);
    2774            0 : 
    2775            0 :         info!("importing pgdata");
    2776            0 :         import_pgdata::doit(&timeline, index_part, &ctx, self.cancel.clone())
    2777            0 :             .await
    2778            0 :             .context("import")?;
    2779            0 :         info!("import done");
    2780              : 
    2781              :         //
    2782              :         // Reload timeline from remote.
    2783              :         // This proves that the remote state is attachable, and it reuses the code.
    2784              :         //
    2785              :         // TODO: think about whether this is safe to do with concurrent Tenant::shutdown.
    2786              :         // timeline_create_guard hols the tenant gate open, so, shutdown cannot _complete_ until we exit.
    2787              :         // But our activate() call might launch new background tasks after Tenant::shutdown
    2788              :         // already went past shutting down the Tenant::timelines, which this timeline here is no part of.
    2789              :         // I think the same problem exists with the bootstrap & branch mgmt API tasks (tenant shutting
    2790              :         // down while bootstrapping/branching + activating), but, the race condition is much more likely
    2791              :         // to manifest because of the long runtime of this import task.
    2792              : 
    2793              :         //        in theory this shouldn't even .await anything except for coop yield
    2794            0 :         info!("shutting down timeline");
    2795            0 :         timeline.shutdown(ShutdownMode::Hard).await;
    2796            0 :         info!("timeline shut down, reloading from remote");
    2797              :         // TODO: we can't do the following check because create_timeline_import_pgdata must return an Arc<Timeline>
    2798              :         // let Some(timeline) = Arc::into_inner(timeline) else {
    2799              :         //     anyhow::bail!("implementation error: timeline that we shut down was still referenced from somewhere");
    2800              :         // };
    2801            0 :         let timeline_id = timeline.timeline_id;
    2802            0 : 
    2803            0 :         // load from object storage like Tenant::attach does
    2804            0 :         let resources = self.build_timeline_resources(timeline_id);
    2805            0 :         let index_part = resources
    2806            0 :             .remote_client
    2807            0 :             .download_index_file(&self.cancel)
    2808            0 :             .await?;
    2809            0 :         let index_part = match index_part {
    2810              :             MaybeDeletedIndexPart::Deleted(_) => {
    2811              :                 // likely concurrent delete call, cplane should prevent this
    2812            0 :                 anyhow::bail!("index part says deleted but we are not done creating yet, this should not happen but")
    2813              :             }
    2814            0 :             MaybeDeletedIndexPart::IndexPart(p) => p,
    2815            0 :         };
    2816            0 :         let metadata = index_part.metadata.clone();
    2817            0 :         self
    2818            0 :             .load_remote_timeline(timeline_id, index_part, metadata, resources, LoadTimelineCause::ImportPgdata{
    2819            0 :                 create_guard: timeline_create_guard, activate, }, &ctx)
    2820            0 :             .await?
    2821            0 :             .ready_to_activate()
    2822            0 :             .context("implementation error: reloaded timeline still needs import after import reported success")?;
    2823              : 
    2824            0 :         anyhow::Ok(())
    2825            0 :     }
    2826              : 
    2827            0 :     pub(crate) async fn delete_timeline(
    2828            0 :         self: Arc<Self>,
    2829            0 :         timeline_id: TimelineId,
    2830            0 :     ) -> Result<(), DeleteTimelineError> {
    2831            0 :         DeleteTimelineFlow::run(&self, timeline_id).await?;
    2832              : 
    2833            0 :         Ok(())
    2834            0 :     }
    2835              : 
    2836              :     /// perform one garbage collection iteration, removing old data files from disk.
    2837              :     /// this function is periodically called by gc task.
    2838              :     /// also it can be explicitly requested through page server api 'do_gc' command.
    2839              :     ///
    2840              :     /// `target_timeline_id` specifies the timeline to GC, or None for all.
    2841              :     ///
    2842              :     /// The `horizon` an `pitr` parameters determine how much WAL history needs to be retained.
    2843              :     /// Also known as the retention period, or the GC cutoff point. `horizon` specifies
    2844              :     /// the amount of history, as LSN difference from current latest LSN on each timeline.
    2845              :     /// `pitr` specifies the same as a time difference from the current time. The effective
    2846              :     /// GC cutoff point is determined conservatively by either `horizon` and `pitr`, whichever
    2847              :     /// requires more history to be retained.
    2848              :     //
    2849          754 :     pub(crate) async fn gc_iteration(
    2850          754 :         &self,
    2851          754 :         target_timeline_id: Option<TimelineId>,
    2852          754 :         horizon: u64,
    2853          754 :         pitr: Duration,
    2854          754 :         cancel: &CancellationToken,
    2855          754 :         ctx: &RequestContext,
    2856          754 :     ) -> Result<GcResult, GcError> {
    2857          754 :         // Don't start doing work during shutdown
    2858          754 :         if let TenantState::Stopping { .. } = self.current_state() {
    2859            0 :             return Ok(GcResult::default());
    2860          754 :         }
    2861          754 : 
    2862          754 :         // there is a global allowed_error for this
    2863          754 :         if !self.is_active() {
    2864            0 :             return Err(GcError::NotActive);
    2865          754 :         }
    2866          754 : 
    2867          754 :         {
    2868          754 :             let conf = self.tenant_conf.load();
    2869          754 : 
    2870          754 :             // If we may not delete layers, then simply skip GC.  Even though a tenant
    2871          754 :             // in AttachedMulti state could do GC and just enqueue the blocked deletions,
    2872          754 :             // the only advantage to doing it is to perhaps shrink the LayerMap metadata
    2873          754 :             // a bit sooner than we would achieve by waiting for AttachedSingle status.
    2874          754 :             if !conf.location.may_delete_layers_hint() {
    2875            0 :                 info!("Skipping GC in location state {:?}", conf.location);
    2876            0 :                 return Ok(GcResult::default());
    2877          754 :             }
    2878          754 : 
    2879          754 :             if conf.is_gc_blocked_by_lsn_lease_deadline() {
    2880          750 :                 info!("Skipping GC because lsn lease deadline is not reached");
    2881          750 :                 return Ok(GcResult::default());
    2882            4 :             }
    2883              :         }
    2884              : 
    2885            4 :         let _guard = match self.gc_block.start().await {
    2886            4 :             Ok(guard) => guard,
    2887            0 :             Err(reasons) => {
    2888            0 :                 info!("Skipping GC: {reasons}");
    2889            0 :                 return Ok(GcResult::default());
    2890              :             }
    2891              :         };
    2892              : 
    2893            4 :         self.gc_iteration_internal(target_timeline_id, horizon, pitr, cancel, ctx)
    2894            4 :             .await
    2895          754 :     }
    2896              : 
    2897              :     /// Perform one compaction iteration.
    2898              :     /// This function is periodically called by compactor task.
    2899              :     /// Also it can be explicitly requested per timeline through page server
    2900              :     /// api's 'compact' command.
    2901              :     ///
    2902              :     /// Returns whether we have pending compaction task.
    2903            0 :     async fn compaction_iteration(
    2904            0 :         self: &Arc<Self>,
    2905            0 :         cancel: &CancellationToken,
    2906            0 :         ctx: &RequestContext,
    2907            0 :     ) -> Result<bool, timeline::CompactionError> {
    2908            0 :         // Don't start doing work during shutdown, or when broken, we do not need those in the logs
    2909            0 :         if !self.is_active() {
    2910            0 :             return Ok(false);
    2911            0 :         }
    2912            0 : 
    2913            0 :         {
    2914            0 :             let conf = self.tenant_conf.load();
    2915            0 : 
    2916            0 :             // Note that compaction usually requires deletions, but we don't respect
    2917            0 :             // may_delete_layers_hint here: that is because tenants in AttachedMulti
    2918            0 :             // should proceed with compaction even if they can't do deletion, to avoid
    2919            0 :             // accumulating dangerously deep stacks of L0 layers.  Deletions will be
    2920            0 :             // enqueued inside RemoteTimelineClient, and executed layer if/when we transition
    2921            0 :             // to AttachedSingle state.
    2922            0 :             if !conf.location.may_upload_layers_hint() {
    2923            0 :                 info!("Skipping compaction in location state {:?}", conf.location);
    2924            0 :                 return Ok(false);
    2925            0 :             }
    2926            0 :         }
    2927            0 : 
    2928            0 :         // Scan through the hashmap and collect a list of all the timelines,
    2929            0 :         // while holding the lock. Then drop the lock and actually perform the
    2930            0 :         // compactions.  We don't want to block everything else while the
    2931            0 :         // compaction runs.
    2932            0 :         let timelines_to_compact_or_offload;
    2933            0 :         {
    2934            0 :             let timelines = self.timelines.lock().unwrap();
    2935            0 :             timelines_to_compact_or_offload = timelines
    2936            0 :                 .iter()
    2937            0 :                 .filter_map(|(timeline_id, timeline)| {
    2938            0 :                     let (is_active, (can_offload, _)) =
    2939            0 :                         (timeline.is_active(), timeline.can_offload());
    2940            0 :                     let has_no_unoffloaded_children = {
    2941            0 :                         !timelines
    2942            0 :                             .iter()
    2943            0 :                             .any(|(_id, tl)| tl.get_ancestor_timeline_id() == Some(*timeline_id))
    2944              :                     };
    2945            0 :                     let config_allows_offload = self.conf.timeline_offloading
    2946            0 :                         || self
    2947            0 :                             .tenant_conf
    2948            0 :                             .load()
    2949            0 :                             .tenant_conf
    2950            0 :                             .timeline_offloading
    2951            0 :                             .unwrap_or_default();
    2952            0 :                     let can_offload =
    2953            0 :                         can_offload && has_no_unoffloaded_children && config_allows_offload;
    2954            0 :                     if (is_active, can_offload) == (false, false) {
    2955            0 :                         None
    2956              :                     } else {
    2957            0 :                         Some((*timeline_id, timeline.clone(), (is_active, can_offload)))
    2958              :                     }
    2959            0 :                 })
    2960            0 :                 .collect::<Vec<_>>();
    2961            0 :             drop(timelines);
    2962            0 :         }
    2963            0 : 
    2964            0 :         // Before doing any I/O work, check our circuit breaker
    2965            0 :         if self.compaction_circuit_breaker.lock().unwrap().is_broken() {
    2966            0 :             info!("Skipping compaction due to previous failures");
    2967            0 :             return Ok(false);
    2968            0 :         }
    2969            0 : 
    2970            0 :         let mut has_pending_task = false;
    2971              : 
    2972            0 :         for (timeline_id, timeline, (can_compact, can_offload)) in &timelines_to_compact_or_offload
    2973              :         {
    2974              :             // pending_task_left == None: cannot compact, maybe still pending tasks
    2975              :             // pending_task_left == Some(true): compaction task left
    2976              :             // pending_task_left == Some(false): no compaction task left
    2977            0 :             let pending_task_left = if *can_compact {
    2978            0 :                 let has_pending_l0_compaction_task = timeline
    2979            0 :                     .compact(cancel, EnumSet::empty(), ctx)
    2980            0 :                     .instrument(info_span!("compact_timeline", %timeline_id))
    2981            0 :                     .await
    2982            0 :                     .inspect_err(|e| match e {
    2983            0 :                         timeline::CompactionError::ShuttingDown => (),
    2984            0 :                         timeline::CompactionError::Offload(_) => {
    2985            0 :                             // Failures to offload timelines do not trip the circuit breaker, because
    2986            0 :                             // they do not do lots of writes the way compaction itself does: it is cheap
    2987            0 :                             // to retry, and it would be bad to stop all compaction because of an issue with offloading.
    2988            0 :                         }
    2989            0 :                         timeline::CompactionError::Other(e) => {
    2990            0 :                             self.compaction_circuit_breaker
    2991            0 :                                 .lock()
    2992            0 :                                 .unwrap()
    2993            0 :                                 .fail(&CIRCUIT_BREAKERS_BROKEN, e);
    2994            0 :                         }
    2995            0 :                     })?;
    2996            0 :                 if has_pending_l0_compaction_task {
    2997            0 :                     Some(true)
    2998              :                 } else {
    2999              :                     let mut has_pending_scheduled_compaction_task;
    3000            0 :                     let next_scheduled_compaction_task = {
    3001            0 :                         let mut guard = self.scheduled_compaction_tasks.lock().unwrap();
    3002            0 :                         if let Some(tline_pending_tasks) = guard.get_mut(timeline_id) {
    3003            0 :                             if !tline_pending_tasks.is_empty() {
    3004            0 :                                 info!(
    3005            0 :                                     "{} tasks left in the compaction schedule queue",
    3006            0 :                                     tline_pending_tasks.len()
    3007              :                                 );
    3008            0 :                             }
    3009            0 :                             let next_task = tline_pending_tasks.pop_front();
    3010            0 :                             has_pending_scheduled_compaction_task = !tline_pending_tasks.is_empty();
    3011            0 :                             next_task
    3012              :                         } else {
    3013            0 :                             has_pending_scheduled_compaction_task = false;
    3014            0 :                             None
    3015              :                         }
    3016              :                     };
    3017            0 :                     if let Some(mut next_scheduled_compaction_task) = next_scheduled_compaction_task
    3018              :                     {
    3019            0 :                         if !next_scheduled_compaction_task
    3020            0 :                             .options
    3021            0 :                             .flags
    3022            0 :                             .contains(CompactFlags::EnhancedGcBottomMostCompaction)
    3023              :                         {
    3024            0 :                             warn!("ignoring scheduled compaction task: scheduled task must be gc compaction: {:?}", next_scheduled_compaction_task.options);
    3025            0 :                         } else if next_scheduled_compaction_task.options.sub_compaction {
    3026            0 :                             info!("running scheduled enhanced gc bottom-most compaction with sub-compaction, splitting compaction jobs");
    3027            0 :                             let jobs: Vec<GcCompactJob> = timeline
    3028            0 :                                 .gc_compaction_split_jobs(
    3029            0 :                                     GcCompactJob::from_compact_options(
    3030            0 :                                         next_scheduled_compaction_task.options.clone(),
    3031            0 :                                     ),
    3032            0 :                                     next_scheduled_compaction_task
    3033            0 :                                         .options
    3034            0 :                                         .sub_compaction_max_job_size_mb,
    3035            0 :                                 )
    3036            0 :                                 .await
    3037            0 :                                 .map_err(CompactionError::Other)?;
    3038            0 :                             if jobs.is_empty() {
    3039            0 :                                 info!("no jobs to run, skipping scheduled compaction task");
    3040              :                             } else {
    3041            0 :                                 has_pending_scheduled_compaction_task = true;
    3042            0 :                                 let jobs_len = jobs.len();
    3043            0 :                                 let mut guard = self.scheduled_compaction_tasks.lock().unwrap();
    3044            0 :                                 let tline_pending_tasks = guard.entry(*timeline_id).or_default();
    3045            0 :                                 for (idx, job) in jobs.into_iter().enumerate() {
    3046              :                                     // Unfortunately we need to convert the `GcCompactJob` back to `CompactionOptions`
    3047              :                                     // until we do further refactors to allow directly call `compact_with_gc`.
    3048            0 :                                     let mut flags: EnumSet<CompactFlags> = EnumSet::default();
    3049            0 :                                     flags |= CompactFlags::EnhancedGcBottomMostCompaction;
    3050            0 :                                     if job.dry_run {
    3051            0 :                                         flags |= CompactFlags::DryRun;
    3052            0 :                                     }
    3053            0 :                                     let options = CompactOptions {
    3054            0 :                                         flags,
    3055            0 :                                         sub_compaction: false,
    3056            0 :                                         compact_key_range: Some(job.compact_key_range.into()),
    3057            0 :                                         compact_lsn_range: Some(job.compact_lsn_range.into()),
    3058            0 :                                         sub_compaction_max_job_size_mb: None,
    3059            0 :                                     };
    3060            0 :                                     tline_pending_tasks.push_back(if idx == jobs_len - 1 {
    3061            0 :                                         ScheduledCompactionTask {
    3062            0 :                                             options,
    3063            0 :                                             // The last job in the queue sends the signal and releases the gc guard
    3064            0 :                                             result_tx: next_scheduled_compaction_task
    3065            0 :                                                 .result_tx
    3066            0 :                                                 .take(),
    3067            0 :                                             gc_block: next_scheduled_compaction_task
    3068            0 :                                                 .gc_block
    3069            0 :                                                 .take(),
    3070            0 :                                         }
    3071              :                                     } else {
    3072            0 :                                         ScheduledCompactionTask {
    3073            0 :                                             options,
    3074            0 :                                             result_tx: None,
    3075            0 :                                             gc_block: None,
    3076            0 :                                         }
    3077              :                                     });
    3078              :                                 }
    3079            0 :                                 info!("scheduled enhanced gc bottom-most compaction with sub-compaction, split into {} jobs", jobs_len);
    3080              :                             }
    3081              :                         } else {
    3082            0 :                             let _ = timeline
    3083            0 :                                 .compact_with_options(
    3084            0 :                                     cancel,
    3085            0 :                                     next_scheduled_compaction_task.options,
    3086            0 :                                     ctx,
    3087            0 :                                 )
    3088            0 :                                 .instrument(info_span!("scheduled_compact_timeline", %timeline_id))
    3089            0 :                                 .await?;
    3090            0 :                             if let Some(tx) = next_scheduled_compaction_task.result_tx.take() {
    3091            0 :                                 // TODO: we can send compaction statistics in the future
    3092            0 :                                 tx.send(()).ok();
    3093            0 :                             }
    3094              :                         }
    3095            0 :                     }
    3096            0 :                     Some(has_pending_scheduled_compaction_task)
    3097              :                 }
    3098              :             } else {
    3099            0 :                 None
    3100              :             };
    3101            0 :             has_pending_task |= pending_task_left.unwrap_or(false);
    3102            0 :             if pending_task_left == Some(false) && *can_offload {
    3103            0 :                 offload_timeline(self, timeline)
    3104            0 :                     .instrument(info_span!("offload_timeline", %timeline_id))
    3105            0 :                     .await?;
    3106            0 :             }
    3107              :         }
    3108              : 
    3109            0 :         self.compaction_circuit_breaker
    3110            0 :             .lock()
    3111            0 :             .unwrap()
    3112            0 :             .success(&CIRCUIT_BREAKERS_UNBROKEN);
    3113            0 : 
    3114            0 :         Ok(has_pending_task)
    3115            0 :     }
    3116              : 
    3117              :     /// Cancel scheduled compaction tasks
    3118            0 :     pub(crate) fn cancel_scheduled_compaction(
    3119            0 :         &self,
    3120            0 :         timeline_id: TimelineId,
    3121            0 :     ) -> Vec<ScheduledCompactionTask> {
    3122            0 :         let mut guard = self.scheduled_compaction_tasks.lock().unwrap();
    3123            0 :         if let Some(tline_pending_tasks) = guard.get_mut(&timeline_id) {
    3124            0 :             let current_tline_pending_tasks = std::mem::take(tline_pending_tasks);
    3125            0 :             current_tline_pending_tasks.into_iter().collect()
    3126              :         } else {
    3127            0 :             Vec::new()
    3128              :         }
    3129            0 :     }
    3130              : 
    3131            0 :     pub(crate) fn get_scheduled_compaction_tasks(
    3132            0 :         &self,
    3133            0 :         timeline_id: TimelineId,
    3134            0 :     ) -> Vec<CompactOptions> {
    3135              :         use itertools::Itertools;
    3136            0 :         let guard = self.scheduled_compaction_tasks.lock().unwrap();
    3137            0 :         guard
    3138            0 :             .get(&timeline_id)
    3139            0 :             .map(|tline_pending_tasks| {
    3140            0 :                 tline_pending_tasks
    3141            0 :                     .iter()
    3142            0 :                     .map(|x| x.options.clone())
    3143            0 :                     .collect_vec()
    3144            0 :             })
    3145            0 :             .unwrap_or_default()
    3146            0 :     }
    3147              : 
    3148              :     /// Schedule a compaction task for a timeline.
    3149            0 :     pub(crate) async fn schedule_compaction(
    3150            0 :         &self,
    3151            0 :         timeline_id: TimelineId,
    3152            0 :         options: CompactOptions,
    3153            0 :     ) -> anyhow::Result<tokio::sync::oneshot::Receiver<()>> {
    3154            0 :         let gc_guard = match self.gc_block.start().await {
    3155            0 :             Ok(guard) => guard,
    3156            0 :             Err(e) => {
    3157            0 :                 bail!("cannot run gc-compaction because gc is blocked: {}", e);
    3158              :             }
    3159              :         };
    3160            0 :         let (tx, rx) = tokio::sync::oneshot::channel();
    3161            0 :         let mut guard = self.scheduled_compaction_tasks.lock().unwrap();
    3162            0 :         let tline_pending_tasks = guard.entry(timeline_id).or_default();
    3163            0 :         tline_pending_tasks.push_back(ScheduledCompactionTask {
    3164            0 :             options,
    3165            0 :             result_tx: Some(tx),
    3166            0 :             gc_block: Some(gc_guard),
    3167            0 :         });
    3168            0 :         Ok(rx)
    3169            0 :     }
    3170              : 
    3171              :     // Call through to all timelines to freeze ephemeral layers if needed.  Usually
    3172              :     // this happens during ingest: this background housekeeping is for freezing layers
    3173              :     // that are open but haven't been written to for some time.
    3174            0 :     async fn ingest_housekeeping(&self) {
    3175            0 :         // Scan through the hashmap and collect a list of all the timelines,
    3176            0 :         // while holding the lock. Then drop the lock and actually perform the
    3177            0 :         // compactions.  We don't want to block everything else while the
    3178            0 :         // compaction runs.
    3179            0 :         let timelines = {
    3180            0 :             self.timelines
    3181            0 :                 .lock()
    3182            0 :                 .unwrap()
    3183            0 :                 .values()
    3184            0 :                 .filter_map(|timeline| {
    3185            0 :                     if timeline.is_active() {
    3186            0 :                         Some(timeline.clone())
    3187              :                     } else {
    3188            0 :                         None
    3189              :                     }
    3190            0 :                 })
    3191            0 :                 .collect::<Vec<_>>()
    3192              :         };
    3193              : 
    3194            0 :         for timeline in &timelines {
    3195            0 :             timeline.maybe_freeze_ephemeral_layer().await;
    3196              :         }
    3197            0 :     }
    3198              : 
    3199            0 :     pub fn timeline_has_no_attached_children(&self, timeline_id: TimelineId) -> bool {
    3200            0 :         let timelines = self.timelines.lock().unwrap();
    3201            0 :         !timelines
    3202            0 :             .iter()
    3203            0 :             .any(|(_id, tl)| tl.get_ancestor_timeline_id() == Some(timeline_id))
    3204            0 :     }
    3205              : 
    3206         1708 :     pub fn current_state(&self) -> TenantState {
    3207         1708 :         self.state.borrow().clone()
    3208         1708 :     }
    3209              : 
    3210          946 :     pub fn is_active(&self) -> bool {
    3211          946 :         self.current_state() == TenantState::Active
    3212          946 :     }
    3213              : 
    3214            0 :     pub fn generation(&self) -> Generation {
    3215            0 :         self.generation
    3216            0 :     }
    3217              : 
    3218            0 :     pub(crate) fn wal_redo_manager_status(&self) -> Option<WalRedoManagerStatus> {
    3219            0 :         self.walredo_mgr.as_ref().and_then(|mgr| mgr.status())
    3220            0 :     }
    3221              : 
    3222              :     /// Changes tenant status to active, unless shutdown was already requested.
    3223              :     ///
    3224              :     /// `background_jobs_can_start` is an optional barrier set to a value during pageserver startup
    3225              :     /// to delay background jobs. Background jobs can be started right away when None is given.
    3226            0 :     fn activate(
    3227            0 :         self: &Arc<Self>,
    3228            0 :         broker_client: BrokerClientChannel,
    3229            0 :         background_jobs_can_start: Option<&completion::Barrier>,
    3230            0 :         ctx: &RequestContext,
    3231            0 :     ) {
    3232            0 :         span::debug_assert_current_span_has_tenant_id();
    3233            0 : 
    3234            0 :         let mut activating = false;
    3235            0 :         self.state.send_modify(|current_state| {
    3236              :             use pageserver_api::models::ActivatingFrom;
    3237            0 :             match &*current_state {
    3238              :                 TenantState::Activating(_) | TenantState::Active | TenantState::Broken { .. } | TenantState::Stopping { .. } => {
    3239            0 :                     panic!("caller is responsible for calling activate() only on Loading / Attaching tenants, got {state:?}", state = current_state);
    3240              :                 }
    3241            0 :                 TenantState::Attaching => {
    3242            0 :                     *current_state = TenantState::Activating(ActivatingFrom::Attaching);
    3243            0 :                 }
    3244            0 :             }
    3245            0 :             debug!(tenant_id = %self.tenant_shard_id.tenant_id, shard_id = %self.tenant_shard_id.shard_slug(), "Activating tenant");
    3246            0 :             activating = true;
    3247            0 :             // Continue outside the closure. We need to grab timelines.lock()
    3248            0 :             // and we plan to turn it into a tokio::sync::Mutex in a future patch.
    3249            0 :         });
    3250            0 : 
    3251            0 :         if activating {
    3252            0 :             let timelines_accessor = self.timelines.lock().unwrap();
    3253            0 :             let timelines_offloaded_accessor = self.timelines_offloaded.lock().unwrap();
    3254            0 :             let timelines_to_activate = timelines_accessor
    3255            0 :                 .values()
    3256            0 :                 .filter(|timeline| !(timeline.is_broken() || timeline.is_stopping()));
    3257            0 : 
    3258            0 :             // Before activation, populate each Timeline's GcInfo with information about its children
    3259            0 :             self.initialize_gc_info(&timelines_accessor, &timelines_offloaded_accessor, None);
    3260            0 : 
    3261            0 :             // Spawn gc and compaction loops. The loops will shut themselves
    3262            0 :             // down when they notice that the tenant is inactive.
    3263            0 :             tasks::start_background_loops(self, background_jobs_can_start);
    3264            0 : 
    3265            0 :             let mut activated_timelines = 0;
    3266              : 
    3267            0 :             for timeline in timelines_to_activate {
    3268            0 :                 timeline.activate(
    3269            0 :                     self.clone(),
    3270            0 :                     broker_client.clone(),
    3271            0 :                     background_jobs_can_start,
    3272            0 :                     ctx,
    3273            0 :                 );
    3274            0 :                 activated_timelines += 1;
    3275            0 :             }
    3276              : 
    3277            0 :             self.state.send_modify(move |current_state| {
    3278            0 :                 assert!(
    3279            0 :                     matches!(current_state, TenantState::Activating(_)),
    3280            0 :                     "set_stopping and set_broken wait for us to leave Activating state",
    3281              :                 );
    3282            0 :                 *current_state = TenantState::Active;
    3283            0 : 
    3284            0 :                 let elapsed = self.constructed_at.elapsed();
    3285            0 :                 let total_timelines = timelines_accessor.len();
    3286            0 : 
    3287            0 :                 // log a lot of stuff, because some tenants sometimes suffer from user-visible
    3288            0 :                 // times to activate. see https://github.com/neondatabase/neon/issues/4025
    3289            0 :                 info!(
    3290            0 :                     since_creation_millis = elapsed.as_millis(),
    3291            0 :                     tenant_id = %self.tenant_shard_id.tenant_id,
    3292            0 :                     shard_id = %self.tenant_shard_id.shard_slug(),
    3293            0 :                     activated_timelines,
    3294            0 :                     total_timelines,
    3295            0 :                     post_state = <&'static str>::from(&*current_state),
    3296            0 :                     "activation attempt finished"
    3297              :                 );
    3298              : 
    3299            0 :                 TENANT.activation.observe(elapsed.as_secs_f64());
    3300            0 :             });
    3301            0 :         }
    3302            0 :     }
    3303              : 
    3304              :     /// Shutdown the tenant and join all of the spawned tasks.
    3305              :     ///
    3306              :     /// The method caters for all use-cases:
    3307              :     /// - pageserver shutdown (freeze_and_flush == true)
    3308              :     /// - detach + ignore (freeze_and_flush == false)
    3309              :     ///
    3310              :     /// This will attempt to shutdown even if tenant is broken.
    3311              :     ///
    3312              :     /// `shutdown_progress` is a [`completion::Barrier`] for the shutdown initiated by this call.
    3313              :     /// If the tenant is already shutting down, we return a clone of the first shutdown call's
    3314              :     /// `Barrier` as an `Err`. This not-first caller can use the returned barrier to join with
    3315              :     /// the ongoing shutdown.
    3316            6 :     async fn shutdown(
    3317            6 :         &self,
    3318            6 :         shutdown_progress: completion::Barrier,
    3319            6 :         shutdown_mode: timeline::ShutdownMode,
    3320            6 :     ) -> Result<(), completion::Barrier> {
    3321            6 :         span::debug_assert_current_span_has_tenant_id();
    3322              : 
    3323              :         // Set tenant (and its timlines) to Stoppping state.
    3324              :         //
    3325              :         // Since we can only transition into Stopping state after activation is complete,
    3326              :         // run it in a JoinSet so all tenants have a chance to stop before we get SIGKILLed.
    3327              :         //
    3328              :         // Transitioning tenants to Stopping state has a couple of non-obvious side effects:
    3329              :         // 1. Lock out any new requests to the tenants.
    3330              :         // 2. Signal cancellation to WAL receivers (we wait on it below).
    3331              :         // 3. Signal cancellation for other tenant background loops.
    3332              :         // 4. ???
    3333              :         //
    3334              :         // The waiting for the cancellation is not done uniformly.
    3335              :         // We certainly wait for WAL receivers to shut down.
    3336              :         // That is necessary so that no new data comes in before the freeze_and_flush.
    3337              :         // But the tenant background loops are joined-on in our caller.
    3338              :         // It's mesed up.
    3339              :         // we just ignore the failure to stop
    3340              : 
    3341              :         // If we're still attaching, fire the cancellation token early to drop out: this
    3342              :         // will prevent us flushing, but ensures timely shutdown if some I/O during attach
    3343              :         // is very slow.
    3344            6 :         let shutdown_mode = if matches!(self.current_state(), TenantState::Attaching) {
    3345            0 :             self.cancel.cancel();
    3346            0 : 
    3347            0 :             // Having fired our cancellation token, do not try and flush timelines: their cancellation tokens
    3348            0 :             // are children of ours, so their flush loops will have shut down already
    3349            0 :             timeline::ShutdownMode::Hard
    3350              :         } else {
    3351            6 :             shutdown_mode
    3352              :         };
    3353              : 
    3354            6 :         match self.set_stopping(shutdown_progress, false, false).await {
    3355            6 :             Ok(()) => {}
    3356            0 :             Err(SetStoppingError::Broken) => {
    3357            0 :                 // assume that this is acceptable
    3358            0 :             }
    3359            0 :             Err(SetStoppingError::AlreadyStopping(other)) => {
    3360            0 :                 // give caller the option to wait for this this shutdown
    3361            0 :                 info!("Tenant::shutdown: AlreadyStopping");
    3362            0 :                 return Err(other);
    3363              :             }
    3364              :         };
    3365              : 
    3366            6 :         let mut js = tokio::task::JoinSet::new();
    3367            6 :         {
    3368            6 :             let timelines = self.timelines.lock().unwrap();
    3369            6 :             timelines.values().for_each(|timeline| {
    3370            6 :                 let timeline = Arc::clone(timeline);
    3371            6 :                 let timeline_id = timeline.timeline_id;
    3372            6 :                 let span = tracing::info_span!("timeline_shutdown", %timeline_id, ?shutdown_mode);
    3373            6 :                 js.spawn(async move { timeline.shutdown(shutdown_mode).instrument(span).await });
    3374            6 :             });
    3375            6 :         }
    3376            6 :         {
    3377            6 :             let timelines_offloaded = self.timelines_offloaded.lock().unwrap();
    3378            6 :             timelines_offloaded.values().for_each(|timeline| {
    3379            0 :                 timeline.defuse_for_tenant_drop();
    3380            6 :             });
    3381            6 :         }
    3382            6 :         // test_long_timeline_create_then_tenant_delete is leaning on this message
    3383            6 :         tracing::info!("Waiting for timelines...");
    3384           12 :         while let Some(res) = js.join_next().await {
    3385            0 :             match res {
    3386            6 :                 Ok(()) => {}
    3387            0 :                 Err(je) if je.is_cancelled() => unreachable!("no cancelling used"),
    3388            0 :                 Err(je) if je.is_panic() => { /* logged already */ }
    3389            0 :                 Err(je) => warn!("unexpected JoinError: {je:?}"),
    3390              :             }
    3391              :         }
    3392              : 
    3393            6 :         if let ShutdownMode::Reload = shutdown_mode {
    3394            0 :             tracing::info!("Flushing deletion queue");
    3395            0 :             if let Err(e) = self.deletion_queue_client.flush().await {
    3396            0 :                 match e {
    3397            0 :                     DeletionQueueError::ShuttingDown => {
    3398            0 :                         // This is the only error we expect for now. In the future, if more error
    3399            0 :                         // variants are added, we should handle them here.
    3400            0 :                     }
    3401              :                 }
    3402            0 :             }
    3403            6 :         }
    3404              : 
    3405              :         // We cancel the Tenant's cancellation token _after_ the timelines have all shut down.  This permits
    3406              :         // them to continue to do work during their shutdown methods, e.g. flushing data.
    3407            6 :         tracing::debug!("Cancelling CancellationToken");
    3408            6 :         self.cancel.cancel();
    3409            6 : 
    3410            6 :         // shutdown all tenant and timeline tasks: gc, compaction, page service
    3411            6 :         // No new tasks will be started for this tenant because it's in `Stopping` state.
    3412            6 :         //
    3413            6 :         // this will additionally shutdown and await all timeline tasks.
    3414            6 :         tracing::debug!("Waiting for tasks...");
    3415            6 :         task_mgr::shutdown_tasks(None, Some(self.tenant_shard_id), None).await;
    3416              : 
    3417            6 :         if let Some(walredo_mgr) = self.walredo_mgr.as_ref() {
    3418            6 :             walredo_mgr.shutdown().await;
    3419            0 :         }
    3420              : 
    3421              :         // Wait for any in-flight operations to complete
    3422            6 :         self.gate.close().await;
    3423              : 
    3424            6 :         remove_tenant_metrics(&self.tenant_shard_id);
    3425            6 : 
    3426            6 :         Ok(())
    3427            6 :     }
    3428              : 
    3429              :     /// Change tenant status to Stopping, to mark that it is being shut down.
    3430              :     ///
    3431              :     /// This function waits for the tenant to become active if it isn't already, before transitioning it into Stopping state.
    3432              :     ///
    3433              :     /// This function is not cancel-safe!
    3434              :     ///
    3435              :     /// `allow_transition_from_loading` is needed for the special case of loading task deleting the tenant.
    3436              :     /// `allow_transition_from_attaching` is needed for the special case of attaching deleted tenant.
    3437            6 :     async fn set_stopping(
    3438            6 :         &self,
    3439            6 :         progress: completion::Barrier,
    3440            6 :         _allow_transition_from_loading: bool,
    3441            6 :         allow_transition_from_attaching: bool,
    3442            6 :     ) -> Result<(), SetStoppingError> {
    3443            6 :         let mut rx = self.state.subscribe();
    3444            6 : 
    3445            6 :         // cannot stop before we're done activating, so wait out until we're done activating
    3446            6 :         rx.wait_for(|state| match state {
    3447            0 :             TenantState::Attaching if allow_transition_from_attaching => true,
    3448              :             TenantState::Activating(_) | TenantState::Attaching => {
    3449            0 :                 info!(
    3450            0 :                     "waiting for {} to turn Active|Broken|Stopping",
    3451            0 :                     <&'static str>::from(state)
    3452              :                 );
    3453            0 :                 false
    3454              :             }
    3455            6 :             TenantState::Active | TenantState::Broken { .. } | TenantState::Stopping { .. } => true,
    3456            6 :         })
    3457            6 :         .await
    3458            6 :         .expect("cannot drop self.state while on a &self method");
    3459            6 : 
    3460            6 :         // we now know we're done activating, let's see whether this task is the winner to transition into Stopping
    3461            6 :         let mut err = None;
    3462            6 :         let stopping = self.state.send_if_modified(|current_state| match current_state {
    3463              :             TenantState::Activating(_) => {
    3464            0 :                 unreachable!("1we ensured above that we're done with activation, and, there is no re-activation")
    3465              :             }
    3466              :             TenantState::Attaching => {
    3467            0 :                 if !allow_transition_from_attaching {
    3468            0 :                     unreachable!("2we ensured above that we're done with activation, and, there is no re-activation")
    3469            0 :                 };
    3470            0 :                 *current_state = TenantState::Stopping { progress };
    3471            0 :                 true
    3472              :             }
    3473              :             TenantState::Active => {
    3474              :                 // FIXME: due to time-of-check vs time-of-use issues, it can happen that new timelines
    3475              :                 // are created after the transition to Stopping. That's harmless, as the Timelines
    3476              :                 // won't be accessible to anyone afterwards, because the Tenant is in Stopping state.
    3477            6 :                 *current_state = TenantState::Stopping { progress };
    3478            6 :                 // Continue stopping outside the closure. We need to grab timelines.lock()
    3479            6 :                 // and we plan to turn it into a tokio::sync::Mutex in a future patch.
    3480            6 :                 true
    3481              :             }
    3482            0 :             TenantState::Broken { reason, .. } => {
    3483            0 :                 info!(
    3484            0 :                     "Cannot set tenant to Stopping state, it is in Broken state due to: {reason}"
    3485              :                 );
    3486            0 :                 err = Some(SetStoppingError::Broken);
    3487            0 :                 false
    3488              :             }
    3489            0 :             TenantState::Stopping { progress } => {
    3490            0 :                 info!("Tenant is already in Stopping state");
    3491            0 :                 err = Some(SetStoppingError::AlreadyStopping(progress.clone()));
    3492            0 :                 false
    3493              :             }
    3494            6 :         });
    3495            6 :         match (stopping, err) {
    3496            6 :             (true, None) => {} // continue
    3497            0 :             (false, Some(err)) => return Err(err),
    3498            0 :             (true, Some(_)) => unreachable!(
    3499            0 :                 "send_if_modified closure must error out if not transitioning to Stopping"
    3500            0 :             ),
    3501            0 :             (false, None) => unreachable!(
    3502            0 :                 "send_if_modified closure must return true if transitioning to Stopping"
    3503            0 :             ),
    3504              :         }
    3505              : 
    3506            6 :         let timelines_accessor = self.timelines.lock().unwrap();
    3507            6 :         let not_broken_timelines = timelines_accessor
    3508            6 :             .values()
    3509            6 :             .filter(|timeline| !timeline.is_broken());
    3510           12 :         for timeline in not_broken_timelines {
    3511            6 :             timeline.set_state(TimelineState::Stopping);
    3512            6 :         }
    3513            6 :         Ok(())
    3514            6 :     }
    3515              : 
    3516              :     /// Method for tenant::mgr to transition us into Broken state in case of a late failure in
    3517              :     /// `remove_tenant_from_memory`
    3518              :     ///
    3519              :     /// This function waits for the tenant to become active if it isn't already, before transitioning it into Stopping state.
    3520              :     ///
    3521              :     /// In tests, we also use this to set tenants to Broken state on purpose.
    3522            0 :     pub(crate) async fn set_broken(&self, reason: String) {
    3523            0 :         let mut rx = self.state.subscribe();
    3524            0 : 
    3525            0 :         // The load & attach routines own the tenant state until it has reached `Active`.
    3526            0 :         // So, wait until it's done.
    3527            0 :         rx.wait_for(|state| match state {
    3528              :             TenantState::Activating(_) | TenantState::Attaching => {
    3529            0 :                 info!(
    3530            0 :                     "waiting for {} to turn Active|Broken|Stopping",
    3531            0 :                     <&'static str>::from(state)
    3532              :                 );
    3533            0 :                 false
    3534              :             }
    3535            0 :             TenantState::Active | TenantState::Broken { .. } | TenantState::Stopping { .. } => true,
    3536            0 :         })
    3537            0 :         .await
    3538            0 :         .expect("cannot drop self.state while on a &self method");
    3539            0 : 
    3540            0 :         // we now know we're done activating, let's see whether this task is the winner to transition into Broken
    3541            0 :         self.set_broken_no_wait(reason)
    3542            0 :     }
    3543              : 
    3544            0 :     pub(crate) fn set_broken_no_wait(&self, reason: impl Display) {
    3545            0 :         let reason = reason.to_string();
    3546            0 :         self.state.send_modify(|current_state| {
    3547            0 :             match *current_state {
    3548              :                 TenantState::Activating(_) | TenantState::Attaching => {
    3549            0 :                     unreachable!("we ensured above that we're done with activation, and, there is no re-activation")
    3550              :                 }
    3551              :                 TenantState::Active => {
    3552            0 :                     if cfg!(feature = "testing") {
    3553            0 :                         warn!("Changing Active tenant to Broken state, reason: {}", reason);
    3554            0 :                         *current_state = TenantState::broken_from_reason(reason);
    3555              :                     } else {
    3556            0 :                         unreachable!("not allowed to call set_broken on Active tenants in non-testing builds")
    3557              :                     }
    3558              :                 }
    3559              :                 TenantState::Broken { .. } => {
    3560            0 :                     warn!("Tenant is already in Broken state");
    3561              :                 }
    3562              :                 // This is the only "expected" path, any other path is a bug.
    3563              :                 TenantState::Stopping { .. } => {
    3564            0 :                     warn!(
    3565            0 :                         "Marking Stopping tenant as Broken state, reason: {}",
    3566              :                         reason
    3567              :                     );
    3568            0 :                     *current_state = TenantState::broken_from_reason(reason);
    3569              :                 }
    3570              :            }
    3571            0 :         });
    3572            0 :     }
    3573              : 
    3574            0 :     pub fn subscribe_for_state_updates(&self) -> watch::Receiver<TenantState> {
    3575            0 :         self.state.subscribe()
    3576            0 :     }
    3577              : 
    3578              :     /// The activate_now semaphore is initialized with zero units.  As soon as
    3579              :     /// we add a unit, waiters will be able to acquire a unit and proceed.
    3580            0 :     pub(crate) fn activate_now(&self) {
    3581            0 :         self.activate_now_sem.add_permits(1);
    3582            0 :     }
    3583              : 
    3584            0 :     pub(crate) async fn wait_to_become_active(
    3585            0 :         &self,
    3586            0 :         timeout: Duration,
    3587            0 :     ) -> Result<(), GetActiveTenantError> {
    3588            0 :         let mut receiver = self.state.subscribe();
    3589              :         loop {
    3590            0 :             let current_state = receiver.borrow_and_update().clone();
    3591            0 :             match current_state {
    3592              :                 TenantState::Attaching | TenantState::Activating(_) => {
    3593              :                     // in these states, there's a chance that we can reach ::Active
    3594            0 :                     self.activate_now();
    3595            0 :                     match timeout_cancellable(timeout, &self.cancel, receiver.changed()).await {
    3596            0 :                         Ok(r) => {
    3597            0 :                             r.map_err(
    3598            0 :                             |_e: tokio::sync::watch::error::RecvError|
    3599              :                                 // Tenant existed but was dropped: report it as non-existent
    3600            0 :                                 GetActiveTenantError::NotFound(GetTenantError::ShardNotFound(self.tenant_shard_id))
    3601            0 :                         )?
    3602              :                         }
    3603              :                         Err(TimeoutCancellableError::Cancelled) => {
    3604            0 :                             return Err(GetActiveTenantError::Cancelled);
    3605              :                         }
    3606              :                         Err(TimeoutCancellableError::Timeout) => {
    3607            0 :                             return Err(GetActiveTenantError::WaitForActiveTimeout {
    3608            0 :                                 latest_state: Some(self.current_state()),
    3609            0 :                                 wait_time: timeout,
    3610            0 :                             });
    3611              :                         }
    3612              :                     }
    3613              :                 }
    3614              :                 TenantState::Active { .. } => {
    3615            0 :                     return Ok(());
    3616              :                 }
    3617            0 :                 TenantState::Broken { reason, .. } => {
    3618            0 :                     // This is fatal, and reported distinctly from the general case of "will never be active" because
    3619            0 :                     // it's logically a 500 to external API users (broken is always a bug).
    3620            0 :                     return Err(GetActiveTenantError::Broken(reason));
    3621              :                 }
    3622              :                 TenantState::Stopping { .. } => {
    3623              :                     // There's no chance the tenant can transition back into ::Active
    3624            0 :                     return Err(GetActiveTenantError::WillNotBecomeActive(current_state));
    3625              :                 }
    3626              :             }
    3627              :         }
    3628            0 :     }
    3629              : 
    3630            0 :     pub(crate) fn get_attach_mode(&self) -> AttachmentMode {
    3631            0 :         self.tenant_conf.load().location.attach_mode
    3632            0 :     }
    3633              : 
    3634              :     /// For API access: generate a LocationConfig equivalent to the one that would be used to
    3635              :     /// create a Tenant in the same state.  Do not use this in hot paths: it's for relatively
    3636              :     /// rare external API calls, like a reconciliation at startup.
    3637            0 :     pub(crate) fn get_location_conf(&self) -> models::LocationConfig {
    3638            0 :         let conf = self.tenant_conf.load();
    3639              : 
    3640            0 :         let location_config_mode = match conf.location.attach_mode {
    3641            0 :             AttachmentMode::Single => models::LocationConfigMode::AttachedSingle,
    3642            0 :             AttachmentMode::Multi => models::LocationConfigMode::AttachedMulti,
    3643            0 :             AttachmentMode::Stale => models::LocationConfigMode::AttachedStale,
    3644              :         };
    3645              : 
    3646              :         // We have a pageserver TenantConf, we need the API-facing TenantConfig.
    3647            0 :         let tenant_config: models::TenantConfig = conf.tenant_conf.clone().into();
    3648            0 : 
    3649            0 :         models::LocationConfig {
    3650            0 :             mode: location_config_mode,
    3651            0 :             generation: self.generation.into(),
    3652            0 :             secondary_conf: None,
    3653            0 :             shard_number: self.shard_identity.number.0,
    3654            0 :             shard_count: self.shard_identity.count.literal(),
    3655            0 :             shard_stripe_size: self.shard_identity.stripe_size.0,
    3656            0 :             tenant_conf: tenant_config,
    3657            0 :         }
    3658            0 :     }
    3659              : 
    3660            0 :     pub(crate) fn get_tenant_shard_id(&self) -> &TenantShardId {
    3661            0 :         &self.tenant_shard_id
    3662            0 :     }
    3663              : 
    3664            0 :     pub(crate) fn get_shard_stripe_size(&self) -> ShardStripeSize {
    3665            0 :         self.shard_identity.stripe_size
    3666            0 :     }
    3667              : 
    3668            0 :     pub(crate) fn get_generation(&self) -> Generation {
    3669            0 :         self.generation
    3670            0 :     }
    3671              : 
    3672              :     /// This function partially shuts down the tenant (it shuts down the Timelines) and is fallible,
    3673              :     /// and can leave the tenant in a bad state if it fails.  The caller is responsible for
    3674              :     /// resetting this tenant to a valid state if we fail.
    3675            0 :     pub(crate) async fn split_prepare(
    3676            0 :         &self,
    3677            0 :         child_shards: &Vec<TenantShardId>,
    3678            0 :     ) -> anyhow::Result<()> {
    3679            0 :         let (timelines, offloaded) = {
    3680            0 :             let timelines = self.timelines.lock().unwrap();
    3681            0 :             let offloaded = self.timelines_offloaded.lock().unwrap();
    3682            0 :             (timelines.clone(), offloaded.clone())
    3683            0 :         };
    3684            0 :         let timelines_iter = timelines
    3685            0 :             .values()
    3686            0 :             .map(TimelineOrOffloadedArcRef::<'_>::from)
    3687            0 :             .chain(
    3688            0 :                 offloaded
    3689            0 :                     .values()
    3690            0 :                     .map(TimelineOrOffloadedArcRef::<'_>::from),
    3691            0 :             );
    3692            0 :         for timeline in timelines_iter {
    3693              :             // We do not block timeline creation/deletion during splits inside the pageserver: it is up to higher levels
    3694              :             // to ensure that they do not start a split if currently in the process of doing these.
    3695              : 
    3696            0 :             let timeline_id = timeline.timeline_id();
    3697              : 
    3698            0 :             if let TimelineOrOffloadedArcRef::Timeline(timeline) = timeline {
    3699              :                 // Upload an index from the parent: this is partly to provide freshness for the
    3700              :                 // child tenants that will copy it, and partly for general ease-of-debugging: there will
    3701              :                 // always be a parent shard index in the same generation as we wrote the child shard index.
    3702            0 :                 tracing::info!(%timeline_id, "Uploading index");
    3703            0 :                 timeline
    3704            0 :                     .remote_client
    3705            0 :                     .schedule_index_upload_for_file_changes()?;
    3706            0 :                 timeline.remote_client.wait_completion().await?;
    3707            0 :             }
    3708              : 
    3709            0 :             let remote_client = match timeline {
    3710            0 :                 TimelineOrOffloadedArcRef::Timeline(timeline) => timeline.remote_client.clone(),
    3711            0 :                 TimelineOrOffloadedArcRef::Offloaded(offloaded) => {
    3712            0 :                     let remote_client = self
    3713            0 :                         .build_timeline_client(offloaded.timeline_id, self.remote_storage.clone());
    3714            0 :                     Arc::new(remote_client)
    3715              :                 }
    3716              :             };
    3717              : 
    3718              :             // Shut down the timeline's remote client: this means that the indices we write
    3719              :             // for child shards will not be invalidated by the parent shard deleting layers.
    3720            0 :             tracing::info!(%timeline_id, "Shutting down remote storage client");
    3721            0 :             remote_client.shutdown().await;
    3722              : 
    3723              :             // Download methods can still be used after shutdown, as they don't flow through the remote client's
    3724              :             // queue.  In principal the RemoteTimelineClient could provide this without downloading it, but this
    3725              :             // operation is rare, so it's simpler to just download it (and robustly guarantees that the index
    3726              :             // we use here really is the remotely persistent one).
    3727            0 :             tracing::info!(%timeline_id, "Downloading index_part from parent");
    3728            0 :             let result = remote_client
    3729            0 :                 .download_index_file(&self.cancel)
    3730            0 :                 .instrument(info_span!("download_index_file", tenant_id=%self.tenant_shard_id.tenant_id, shard_id=%self.tenant_shard_id.shard_slug(), %timeline_id))
    3731            0 :                 .await?;
    3732            0 :             let index_part = match result {
    3733              :                 MaybeDeletedIndexPart::Deleted(_) => {
    3734            0 :                     anyhow::bail!("Timeline deletion happened concurrently with split")
    3735              :                 }
    3736            0 :                 MaybeDeletedIndexPart::IndexPart(p) => p,
    3737              :             };
    3738              : 
    3739            0 :             for child_shard in child_shards {
    3740            0 :                 tracing::info!(%timeline_id, "Uploading index_part for child {}", child_shard.to_index());
    3741            0 :                 upload_index_part(
    3742            0 :                     &self.remote_storage,
    3743            0 :                     child_shard,
    3744            0 :                     &timeline_id,
    3745            0 :                     self.generation,
    3746            0 :                     &index_part,
    3747            0 :                     &self.cancel,
    3748            0 :                 )
    3749            0 :                 .await?;
    3750              :             }
    3751              :         }
    3752              : 
    3753            0 :         let tenant_manifest = self.build_tenant_manifest();
    3754            0 :         for child_shard in child_shards {
    3755            0 :             tracing::info!(
    3756            0 :                 "Uploading tenant manifest for child {}",
    3757            0 :                 child_shard.to_index()
    3758              :             );
    3759            0 :             upload_tenant_manifest(
    3760            0 :                 &self.remote_storage,
    3761            0 :                 child_shard,
    3762            0 :                 self.generation,
    3763            0 :                 &tenant_manifest,
    3764            0 :                 &self.cancel,
    3765            0 :             )
    3766            0 :             .await?;
    3767              :         }
    3768              : 
    3769            0 :         Ok(())
    3770            0 :     }
    3771              : 
    3772            0 :     pub(crate) fn get_sizes(&self) -> TopTenantShardItem {
    3773            0 :         let mut result = TopTenantShardItem {
    3774            0 :             id: self.tenant_shard_id,
    3775            0 :             resident_size: 0,
    3776            0 :             physical_size: 0,
    3777            0 :             max_logical_size: 0,
    3778            0 :         };
    3779              : 
    3780            0 :         for timeline in self.timelines.lock().unwrap().values() {
    3781            0 :             result.resident_size += timeline.metrics.resident_physical_size_gauge.get();
    3782            0 : 
    3783            0 :             result.physical_size += timeline
    3784            0 :                 .remote_client
    3785            0 :                 .metrics
    3786            0 :                 .remote_physical_size_gauge
    3787            0 :                 .get();
    3788            0 :             result.max_logical_size = std::cmp::max(
    3789            0 :                 result.max_logical_size,
    3790            0 :                 timeline.metrics.current_logical_size_gauge.get(),
    3791            0 :             );
    3792            0 :         }
    3793              : 
    3794            0 :         result
    3795            0 :     }
    3796              : }
    3797              : 
    3798              : /// Given a Vec of timelines and their ancestors (timeline_id, ancestor_id),
    3799              : /// perform a topological sort, so that the parent of each timeline comes
    3800              : /// before the children.
    3801              : /// E extracts the ancestor from T
    3802              : /// This allows for T to be different. It can be TimelineMetadata, can be Timeline itself, etc.
    3803          196 : fn tree_sort_timelines<T, E>(
    3804          196 :     timelines: HashMap<TimelineId, T>,
    3805          196 :     extractor: E,
    3806          196 : ) -> anyhow::Result<Vec<(TimelineId, T)>>
    3807          196 : where
    3808          196 :     E: Fn(&T) -> Option<TimelineId>,
    3809          196 : {
    3810          196 :     let mut result = Vec::with_capacity(timelines.len());
    3811          196 : 
    3812          196 :     let mut now = Vec::with_capacity(timelines.len());
    3813          196 :     // (ancestor, children)
    3814          196 :     let mut later: HashMap<TimelineId, Vec<(TimelineId, T)>> =
    3815          196 :         HashMap::with_capacity(timelines.len());
    3816              : 
    3817          202 :     for (timeline_id, value) in timelines {
    3818            6 :         if let Some(ancestor_id) = extractor(&value) {
    3819            2 :             let children = later.entry(ancestor_id).or_default();
    3820            2 :             children.push((timeline_id, value));
    3821            4 :         } else {
    3822            4 :             now.push((timeline_id, value));
    3823            4 :         }
    3824              :     }
    3825              : 
    3826          202 :     while let Some((timeline_id, metadata)) = now.pop() {
    3827            6 :         result.push((timeline_id, metadata));
    3828              :         // All children of this can be loaded now
    3829            6 :         if let Some(mut children) = later.remove(&timeline_id) {
    3830            2 :             now.append(&mut children);
    3831            4 :         }
    3832              :     }
    3833              : 
    3834              :     // All timelines should be visited now. Unless there were timelines with missing ancestors.
    3835          196 :     if !later.is_empty() {
    3836            0 :         for (missing_id, orphan_ids) in later {
    3837            0 :             for (orphan_id, _) in orphan_ids {
    3838            0 :                 error!("could not load timeline {orphan_id} because its ancestor timeline {missing_id} could not be loaded");
    3839              :             }
    3840              :         }
    3841            0 :         bail!("could not load tenant because some timelines are missing ancestors");
    3842          196 :     }
    3843          196 : 
    3844          196 :     Ok(result)
    3845          196 : }
    3846              : 
    3847              : enum ActivateTimelineArgs {
    3848              :     Yes {
    3849              :         broker_client: storage_broker::BrokerClientChannel,
    3850              :     },
    3851              :     No,
    3852              : }
    3853              : 
    3854              : impl Tenant {
    3855            0 :     pub fn tenant_specific_overrides(&self) -> TenantConfOpt {
    3856            0 :         self.tenant_conf.load().tenant_conf.clone()
    3857            0 :     }
    3858              : 
    3859            0 :     pub fn effective_config(&self) -> TenantConf {
    3860            0 :         self.tenant_specific_overrides()
    3861            0 :             .merge(self.conf.default_tenant_conf.clone())
    3862            0 :     }
    3863              : 
    3864            0 :     pub fn get_checkpoint_distance(&self) -> u64 {
    3865            0 :         let tenant_conf = self.tenant_conf.load().tenant_conf.clone();
    3866            0 :         tenant_conf
    3867            0 :             .checkpoint_distance
    3868            0 :             .unwrap_or(self.conf.default_tenant_conf.checkpoint_distance)
    3869            0 :     }
    3870              : 
    3871            0 :     pub fn get_checkpoint_timeout(&self) -> Duration {
    3872            0 :         let tenant_conf = self.tenant_conf.load().tenant_conf.clone();
    3873            0 :         tenant_conf
    3874            0 :             .checkpoint_timeout
    3875            0 :             .unwrap_or(self.conf.default_tenant_conf.checkpoint_timeout)
    3876            0 :     }
    3877              : 
    3878            0 :     pub fn get_compaction_target_size(&self) -> u64 {
    3879            0 :         let tenant_conf = self.tenant_conf.load().tenant_conf.clone();
    3880            0 :         tenant_conf
    3881            0 :             .compaction_target_size
    3882            0 :             .unwrap_or(self.conf.default_tenant_conf.compaction_target_size)
    3883            0 :     }
    3884              : 
    3885            0 :     pub fn get_compaction_period(&self) -> Duration {
    3886            0 :         let tenant_conf = self.tenant_conf.load().tenant_conf.clone();
    3887            0 :         tenant_conf
    3888            0 :             .compaction_period
    3889            0 :             .unwrap_or(self.conf.default_tenant_conf.compaction_period)
    3890            0 :     }
    3891              : 
    3892            0 :     pub fn get_compaction_threshold(&self) -> usize {
    3893            0 :         let tenant_conf = self.tenant_conf.load().tenant_conf.clone();
    3894            0 :         tenant_conf
    3895            0 :             .compaction_threshold
    3896            0 :             .unwrap_or(self.conf.default_tenant_conf.compaction_threshold)
    3897            0 :     }
    3898              : 
    3899            0 :     pub fn get_gc_horizon(&self) -> u64 {
    3900            0 :         let tenant_conf = self.tenant_conf.load().tenant_conf.clone();
    3901            0 :         tenant_conf
    3902            0 :             .gc_horizon
    3903            0 :             .unwrap_or(self.conf.default_tenant_conf.gc_horizon)
    3904            0 :     }
    3905              : 
    3906            0 :     pub fn get_gc_period(&self) -> Duration {
    3907            0 :         let tenant_conf = self.tenant_conf.load().tenant_conf.clone();
    3908            0 :         tenant_conf
    3909            0 :             .gc_period
    3910            0 :             .unwrap_or(self.conf.default_tenant_conf.gc_period)
    3911            0 :     }
    3912              : 
    3913            0 :     pub fn get_image_creation_threshold(&self) -> usize {
    3914            0 :         let tenant_conf = self.tenant_conf.load().tenant_conf.clone();
    3915            0 :         tenant_conf
    3916            0 :             .image_creation_threshold
    3917            0 :             .unwrap_or(self.conf.default_tenant_conf.image_creation_threshold)
    3918            0 :     }
    3919              : 
    3920            0 :     pub fn get_pitr_interval(&self) -> Duration {
    3921            0 :         let tenant_conf = self.tenant_conf.load().tenant_conf.clone();
    3922            0 :         tenant_conf
    3923            0 :             .pitr_interval
    3924            0 :             .unwrap_or(self.conf.default_tenant_conf.pitr_interval)
    3925            0 :     }
    3926              : 
    3927            0 :     pub fn get_min_resident_size_override(&self) -> Option<u64> {
    3928            0 :         let tenant_conf = self.tenant_conf.load().tenant_conf.clone();
    3929            0 :         tenant_conf
    3930            0 :             .min_resident_size_override
    3931            0 :             .or(self.conf.default_tenant_conf.min_resident_size_override)
    3932            0 :     }
    3933              : 
    3934            0 :     pub fn get_heatmap_period(&self) -> Option<Duration> {
    3935            0 :         let tenant_conf = self.tenant_conf.load().tenant_conf.clone();
    3936            0 :         let heatmap_period = tenant_conf
    3937            0 :             .heatmap_period
    3938            0 :             .unwrap_or(self.conf.default_tenant_conf.heatmap_period);
    3939            0 :         if heatmap_period.is_zero() {
    3940            0 :             None
    3941              :         } else {
    3942            0 :             Some(heatmap_period)
    3943              :         }
    3944            0 :     }
    3945              : 
    3946            4 :     pub fn get_lsn_lease_length(&self) -> Duration {
    3947            4 :         let tenant_conf = self.tenant_conf.load().tenant_conf.clone();
    3948            4 :         tenant_conf
    3949            4 :             .lsn_lease_length
    3950            4 :             .unwrap_or(self.conf.default_tenant_conf.lsn_lease_length)
    3951            4 :     }
    3952              : 
    3953              :     /// Generate an up-to-date TenantManifest based on the state of this Tenant.
    3954            2 :     fn build_tenant_manifest(&self) -> TenantManifest {
    3955            2 :         let timelines_offloaded = self.timelines_offloaded.lock().unwrap();
    3956            2 : 
    3957            2 :         let mut timeline_manifests = timelines_offloaded
    3958            2 :             .iter()
    3959            2 :             .map(|(_timeline_id, offloaded)| offloaded.manifest())
    3960            2 :             .collect::<Vec<_>>();
    3961            2 :         // Sort the manifests so that our output is deterministic
    3962            2 :         timeline_manifests.sort_by_key(|timeline_manifest| timeline_manifest.timeline_id);
    3963            2 : 
    3964            2 :         TenantManifest {
    3965            2 :             version: LATEST_TENANT_MANIFEST_VERSION,
    3966            2 :             offloaded_timelines: timeline_manifests,
    3967            2 :         }
    3968            2 :     }
    3969              : 
    3970            0 :     pub fn update_tenant_config<F: Fn(TenantConfOpt) -> anyhow::Result<TenantConfOpt>>(
    3971            0 :         &self,
    3972            0 :         update: F,
    3973            0 :     ) -> anyhow::Result<TenantConfOpt> {
    3974            0 :         // Use read-copy-update in order to avoid overwriting the location config
    3975            0 :         // state if this races with [`Tenant::set_new_location_config`]. Note that
    3976            0 :         // this race is not possible if both request types come from the storage
    3977            0 :         // controller (as they should!) because an exclusive op lock is required
    3978            0 :         // on the storage controller side.
    3979            0 : 
    3980            0 :         self.tenant_conf
    3981            0 :             .try_rcu(|attached_conf| -> Result<_, anyhow::Error> {
    3982            0 :                 Ok(Arc::new(AttachedTenantConf {
    3983            0 :                     tenant_conf: update(attached_conf.tenant_conf.clone())?,
    3984            0 :                     location: attached_conf.location,
    3985            0 :                     lsn_lease_deadline: attached_conf.lsn_lease_deadline,
    3986              :                 }))
    3987            0 :             })?;
    3988              : 
    3989            0 :         let updated = self.tenant_conf.load();
    3990            0 : 
    3991            0 :         self.tenant_conf_updated(&updated.tenant_conf);
    3992            0 :         // Don't hold self.timelines.lock() during the notifies.
    3993            0 :         // There's no risk of deadlock right now, but there could be if we consolidate
    3994            0 :         // mutexes in struct Timeline in the future.
    3995            0 :         let timelines = self.list_timelines();
    3996            0 :         for timeline in timelines {
    3997            0 :             timeline.tenant_conf_updated(&updated);
    3998            0 :         }
    3999              : 
    4000            0 :         Ok(updated.tenant_conf.clone())
    4001            0 :     }
    4002              : 
    4003            0 :     pub(crate) fn set_new_location_config(&self, new_conf: AttachedTenantConf) {
    4004            0 :         let new_tenant_conf = new_conf.tenant_conf.clone();
    4005            0 : 
    4006            0 :         self.tenant_conf.store(Arc::new(new_conf.clone()));
    4007            0 : 
    4008            0 :         self.tenant_conf_updated(&new_tenant_conf);
    4009            0 :         // Don't hold self.timelines.lock() during the notifies.
    4010            0 :         // There's no risk of deadlock right now, but there could be if we consolidate
    4011            0 :         // mutexes in struct Timeline in the future.
    4012            0 :         let timelines = self.list_timelines();
    4013            0 :         for timeline in timelines {
    4014            0 :             timeline.tenant_conf_updated(&new_conf);
    4015            0 :         }
    4016            0 :     }
    4017              : 
    4018          196 :     fn get_pagestream_throttle_config(
    4019          196 :         psconf: &'static PageServerConf,
    4020          196 :         overrides: &TenantConfOpt,
    4021          196 :     ) -> throttle::Config {
    4022          196 :         overrides
    4023          196 :             .timeline_get_throttle
    4024          196 :             .clone()
    4025          196 :             .unwrap_or(psconf.default_tenant_conf.timeline_get_throttle.clone())
    4026          196 :     }
    4027              : 
    4028            0 :     pub(crate) fn tenant_conf_updated(&self, new_conf: &TenantConfOpt) {
    4029            0 :         let conf = Self::get_pagestream_throttle_config(self.conf, new_conf);
    4030            0 :         self.pagestream_throttle.reconfigure(conf)
    4031            0 :     }
    4032              : 
    4033              :     /// Helper function to create a new Timeline struct.
    4034              :     ///
    4035              :     /// The returned Timeline is in Loading state. The caller is responsible for
    4036              :     /// initializing any on-disk state, and for inserting the Timeline to the 'timelines'
    4037              :     /// map.
    4038              :     ///
    4039              :     /// `validate_ancestor == false` is used when a timeline is created for deletion
    4040              :     /// and we might not have the ancestor present anymore which is fine for to be
    4041              :     /// deleted timelines.
    4042              :     #[allow(clippy::too_many_arguments)]
    4043          422 :     fn create_timeline_struct(
    4044          422 :         &self,
    4045          422 :         new_timeline_id: TimelineId,
    4046          422 :         new_metadata: &TimelineMetadata,
    4047          422 :         ancestor: Option<Arc<Timeline>>,
    4048          422 :         resources: TimelineResources,
    4049          422 :         cause: CreateTimelineCause,
    4050          422 :         create_idempotency: CreateTimelineIdempotency,
    4051          422 :     ) -> anyhow::Result<Arc<Timeline>> {
    4052          422 :         let state = match cause {
    4053              :             CreateTimelineCause::Load => {
    4054          422 :                 let ancestor_id = new_metadata.ancestor_timeline();
    4055          422 :                 anyhow::ensure!(
    4056          422 :                     ancestor_id == ancestor.as_ref().map(|t| t.timeline_id),
    4057            0 :                     "Timeline's {new_timeline_id} ancestor {ancestor_id:?} was not found"
    4058              :                 );
    4059          422 :                 TimelineState::Loading
    4060              :             }
    4061            0 :             CreateTimelineCause::Delete => TimelineState::Stopping,
    4062              :         };
    4063              : 
    4064          422 :         let pg_version = new_metadata.pg_version();
    4065          422 : 
    4066          422 :         let timeline = Timeline::new(
    4067          422 :             self.conf,
    4068          422 :             Arc::clone(&self.tenant_conf),
    4069          422 :             new_metadata,
    4070          422 :             ancestor,
    4071          422 :             new_timeline_id,
    4072          422 :             self.tenant_shard_id,
    4073          422 :             self.generation,
    4074          422 :             self.shard_identity,
    4075          422 :             self.walredo_mgr.clone(),
    4076          422 :             resources,
    4077          422 :             pg_version,
    4078          422 :             state,
    4079          422 :             self.attach_wal_lag_cooldown.clone(),
    4080          422 :             create_idempotency,
    4081          422 :             self.cancel.child_token(),
    4082          422 :         );
    4083          422 : 
    4084          422 :         Ok(timeline)
    4085          422 :     }
    4086              : 
    4087              :     // Allow too_many_arguments because a constructor's argument list naturally grows with the
    4088              :     // number of attributes in the struct: breaking these out into a builder wouldn't be helpful.
    4089              :     #[allow(clippy::too_many_arguments)]
    4090          196 :     fn new(
    4091          196 :         state: TenantState,
    4092          196 :         conf: &'static PageServerConf,
    4093          196 :         attached_conf: AttachedTenantConf,
    4094          196 :         shard_identity: ShardIdentity,
    4095          196 :         walredo_mgr: Option<Arc<WalRedoManager>>,
    4096          196 :         tenant_shard_id: TenantShardId,
    4097          196 :         remote_storage: GenericRemoteStorage,
    4098          196 :         deletion_queue_client: DeletionQueueClient,
    4099          196 :         l0_flush_global_state: L0FlushGlobalState,
    4100          196 :     ) -> Tenant {
    4101          196 :         debug_assert!(
    4102          196 :             !attached_conf.location.generation.is_none() || conf.control_plane_api.is_none()
    4103              :         );
    4104              : 
    4105          196 :         let (state, mut rx) = watch::channel(state);
    4106          196 : 
    4107          196 :         tokio::spawn(async move {
    4108          196 :             // reflect tenant state in metrics:
    4109          196 :             // - global per tenant state: TENANT_STATE_METRIC
    4110          196 :             // - "set" of broken tenants: BROKEN_TENANTS_SET
    4111          196 :             //
    4112          196 :             // set of broken tenants should not have zero counts so that it remains accessible for
    4113          196 :             // alerting.
    4114          196 : 
    4115          196 :             let tid = tenant_shard_id.to_string();
    4116          196 :             let shard_id = tenant_shard_id.shard_slug().to_string();
    4117          196 :             let set_key = &[tid.as_str(), shard_id.as_str()][..];
    4118              : 
    4119          392 :             fn inspect_state(state: &TenantState) -> ([&'static str; 1], bool) {
    4120          392 :                 ([state.into()], matches!(state, TenantState::Broken { .. }))
    4121          392 :             }
    4122              : 
    4123          196 :             let mut tuple = inspect_state(&rx.borrow_and_update());
    4124          196 : 
    4125          196 :             let is_broken = tuple.1;
    4126          196 :             let mut counted_broken = if is_broken {
    4127              :                 // add the id to the set right away, there should not be any updates on the channel
    4128              :                 // after before tenant is removed, if ever
    4129            0 :                 BROKEN_TENANTS_SET.with_label_values(set_key).set(1);
    4130            0 :                 true
    4131              :             } else {
    4132          196 :                 false
    4133              :             };
    4134              : 
    4135              :             loop {
    4136          392 :                 let labels = &tuple.0;
    4137          392 :                 let current = TENANT_STATE_METRIC.with_label_values(labels);
    4138          392 :                 current.inc();
    4139          392 : 
    4140          392 :                 if rx.changed().await.is_err() {
    4141              :                     // tenant has been dropped
    4142           14 :                     current.dec();
    4143           14 :                     drop(BROKEN_TENANTS_SET.remove_label_values(set_key));
    4144           14 :                     break;
    4145          196 :                 }
    4146          196 : 
    4147          196 :                 current.dec();
    4148          196 :                 tuple = inspect_state(&rx.borrow_and_update());
    4149          196 : 
    4150          196 :                 let is_broken = tuple.1;
    4151          196 :                 if is_broken && !counted_broken {
    4152            0 :                     counted_broken = true;
    4153            0 :                     // insert the tenant_id (back) into the set while avoiding needless counter
    4154            0 :                     // access
    4155            0 :                     BROKEN_TENANTS_SET.with_label_values(set_key).set(1);
    4156          196 :                 }
    4157              :             }
    4158          196 :         });
    4159          196 : 
    4160          196 :         Tenant {
    4161          196 :             tenant_shard_id,
    4162          196 :             shard_identity,
    4163          196 :             generation: attached_conf.location.generation,
    4164          196 :             conf,
    4165          196 :             // using now here is good enough approximation to catch tenants with really long
    4166          196 :             // activation times.
    4167          196 :             constructed_at: Instant::now(),
    4168          196 :             timelines: Mutex::new(HashMap::new()),
    4169          196 :             timelines_creating: Mutex::new(HashSet::new()),
    4170          196 :             timelines_offloaded: Mutex::new(HashMap::new()),
    4171          196 :             tenant_manifest_upload: Default::default(),
    4172          196 :             gc_cs: tokio::sync::Mutex::new(()),
    4173          196 :             walredo_mgr,
    4174          196 :             remote_storage,
    4175          196 :             deletion_queue_client,
    4176          196 :             state,
    4177          196 :             cached_logical_sizes: tokio::sync::Mutex::new(HashMap::new()),
    4178          196 :             cached_synthetic_tenant_size: Arc::new(AtomicU64::new(0)),
    4179          196 :             eviction_task_tenant_state: tokio::sync::Mutex::new(EvictionTaskTenantState::default()),
    4180          196 :             compaction_circuit_breaker: std::sync::Mutex::new(CircuitBreaker::new(
    4181          196 :                 format!("compaction-{tenant_shard_id}"),
    4182          196 :                 5,
    4183          196 :                 // Compaction can be a very expensive operation, and might leak disk space.  It also ought
    4184          196 :                 // to be infallible, as long as remote storage is available.  So if it repeatedly fails,
    4185          196 :                 // use an extremely long backoff.
    4186          196 :                 Some(Duration::from_secs(3600 * 24)),
    4187          196 :             )),
    4188          196 :             scheduled_compaction_tasks: Mutex::new(Default::default()),
    4189          196 :             activate_now_sem: tokio::sync::Semaphore::new(0),
    4190          196 :             attach_wal_lag_cooldown: Arc::new(std::sync::OnceLock::new()),
    4191          196 :             cancel: CancellationToken::default(),
    4192          196 :             gate: Gate::default(),
    4193          196 :             pagestream_throttle: Arc::new(throttle::Throttle::new(
    4194          196 :                 Tenant::get_pagestream_throttle_config(conf, &attached_conf.tenant_conf),
    4195          196 :                 crate::metrics::tenant_throttling::Metrics::new(&tenant_shard_id),
    4196          196 :             )),
    4197          196 :             tenant_conf: Arc::new(ArcSwap::from_pointee(attached_conf)),
    4198          196 :             ongoing_timeline_detach: std::sync::Mutex::default(),
    4199          196 :             gc_block: Default::default(),
    4200          196 :             l0_flush_global_state,
    4201          196 :         }
    4202          196 :     }
    4203              : 
    4204              :     /// Locate and load config
    4205            0 :     pub(super) fn load_tenant_config(
    4206            0 :         conf: &'static PageServerConf,
    4207            0 :         tenant_shard_id: &TenantShardId,
    4208            0 :     ) -> Result<LocationConf, LoadConfigError> {
    4209            0 :         let config_path = conf.tenant_location_config_path(tenant_shard_id);
    4210            0 : 
    4211            0 :         info!("loading tenant configuration from {config_path}");
    4212              : 
    4213              :         // load and parse file
    4214            0 :         let config = fs::read_to_string(&config_path).map_err(|e| {
    4215            0 :             match e.kind() {
    4216              :                 std::io::ErrorKind::NotFound => {
    4217              :                     // The config should almost always exist for a tenant directory:
    4218              :                     //  - When attaching a tenant, the config is the first thing we write
    4219              :                     //  - When detaching a tenant, we atomically move the directory to a tmp location
    4220              :                     //    before deleting contents.
    4221              :                     //
    4222              :                     // The very rare edge case that can result in a missing config is if we crash during attach
    4223              :                     // between creating directory and writing config.  Callers should handle that as if the
    4224              :                     // directory didn't exist.
    4225              : 
    4226            0 :                     LoadConfigError::NotFound(config_path)
    4227              :                 }
    4228              :                 _ => {
    4229              :                     // No IO errors except NotFound are acceptable here: other kinds of error indicate local storage or permissions issues
    4230              :                     // that we cannot cleanly recover
    4231            0 :                     crate::virtual_file::on_fatal_io_error(&e, "Reading tenant config file")
    4232              :                 }
    4233              :             }
    4234            0 :         })?;
    4235              : 
    4236            0 :         Ok(toml_edit::de::from_str::<LocationConf>(&config)?)
    4237            0 :     }
    4238              : 
    4239            0 :     #[tracing::instrument(skip_all, fields(tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug()))]
    4240              :     pub(super) async fn persist_tenant_config(
    4241              :         conf: &'static PageServerConf,
    4242              :         tenant_shard_id: &TenantShardId,
    4243              :         location_conf: &LocationConf,
    4244              :     ) -> std::io::Result<()> {
    4245              :         let config_path = conf.tenant_location_config_path(tenant_shard_id);
    4246              : 
    4247              :         Self::persist_tenant_config_at(tenant_shard_id, &config_path, location_conf).await
    4248              :     }
    4249              : 
    4250            0 :     #[tracing::instrument(skip_all, fields(tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug()))]
    4251              :     pub(super) async fn persist_tenant_config_at(
    4252              :         tenant_shard_id: &TenantShardId,
    4253              :         config_path: &Utf8Path,
    4254              :         location_conf: &LocationConf,
    4255              :     ) -> std::io::Result<()> {
    4256              :         debug!("persisting tenantconf to {config_path}");
    4257              : 
    4258              :         let mut conf_content = r#"# This file contains a specific per-tenant's config.
    4259              : #  It is read in case of pageserver restart.
    4260              : "#
    4261              :         .to_string();
    4262              : 
    4263            0 :         fail::fail_point!("tenant-config-before-write", |_| {
    4264            0 :             Err(std::io::Error::new(
    4265            0 :                 std::io::ErrorKind::Other,
    4266            0 :                 "tenant-config-before-write",
    4267            0 :             ))
    4268            0 :         });
    4269              : 
    4270              :         // Convert the config to a toml file.
    4271              :         conf_content +=
    4272              :             &toml_edit::ser::to_string_pretty(&location_conf).expect("Config serialization failed");
    4273              : 
    4274              :         let temp_path = path_with_suffix_extension(config_path, TEMP_FILE_SUFFIX);
    4275              : 
    4276              :         let conf_content = conf_content.into_bytes();
    4277              :         VirtualFile::crashsafe_overwrite(config_path.to_owned(), temp_path, conf_content).await
    4278              :     }
    4279              : 
    4280              :     //
    4281              :     // How garbage collection works:
    4282              :     //
    4283              :     //                    +--bar------------->
    4284              :     //                   /
    4285              :     //             +----+-----foo---------------->
    4286              :     //            /
    4287              :     // ----main--+-------------------------->
    4288              :     //                \
    4289              :     //                 +-----baz-------->
    4290              :     //
    4291              :     //
    4292              :     // 1. Grab 'gc_cs' mutex to prevent new timelines from being created while Timeline's
    4293              :     //    `gc_infos` are being refreshed
    4294              :     // 2. Scan collected timelines, and on each timeline, make note of the
    4295              :     //    all the points where other timelines have been branched off.
    4296              :     //    We will refrain from removing page versions at those LSNs.
    4297              :     // 3. For each timeline, scan all layer files on the timeline.
    4298              :     //    Remove all files for which a newer file exists and which
    4299              :     //    don't cover any branch point LSNs.
    4300              :     //
    4301              :     // TODO:
    4302              :     // - if a relation has a non-incremental persistent layer on a child branch, then we
    4303              :     //   don't need to keep that in the parent anymore. But currently
    4304              :     //   we do.
    4305            4 :     async fn gc_iteration_internal(
    4306            4 :         &self,
    4307            4 :         target_timeline_id: Option<TimelineId>,
    4308            4 :         horizon: u64,
    4309            4 :         pitr: Duration,
    4310            4 :         cancel: &CancellationToken,
    4311            4 :         ctx: &RequestContext,
    4312            4 :     ) -> Result<GcResult, GcError> {
    4313            4 :         let mut totals: GcResult = Default::default();
    4314            4 :         let now = Instant::now();
    4315              : 
    4316            4 :         let gc_timelines = self
    4317            4 :             .refresh_gc_info_internal(target_timeline_id, horizon, pitr, cancel, ctx)
    4318            4 :             .await?;
    4319              : 
    4320            4 :         failpoint_support::sleep_millis_async!("gc_iteration_internal_after_getting_gc_timelines");
    4321              : 
    4322              :         // If there is nothing to GC, we don't want any messages in the INFO log.
    4323            4 :         if !gc_timelines.is_empty() {
    4324            4 :             info!("{} timelines need GC", gc_timelines.len());
    4325              :         } else {
    4326            0 :             debug!("{} timelines need GC", gc_timelines.len());
    4327              :         }
    4328              : 
    4329              :         // Perform GC for each timeline.
    4330              :         //
    4331              :         // Note that we don't hold the `Tenant::gc_cs` lock here because we don't want to delay the
    4332              :         // branch creation task, which requires the GC lock. A GC iteration can run concurrently
    4333              :         // with branch creation.
    4334              :         //
    4335              :         // See comments in [`Tenant::branch_timeline`] for more information about why branch
    4336              :         // creation task can run concurrently with timeline's GC iteration.
    4337            8 :         for timeline in gc_timelines {
    4338            4 :             if cancel.is_cancelled() {
    4339              :                 // We were requested to shut down. Stop and return with the progress we
    4340              :                 // made.
    4341            0 :                 break;
    4342            4 :             }
    4343            4 :             let result = match timeline.gc().await {
    4344              :                 Err(GcError::TimelineCancelled) => {
    4345            0 :                     if target_timeline_id.is_some() {
    4346              :                         // If we were targetting this specific timeline, surface cancellation to caller
    4347            0 :                         return Err(GcError::TimelineCancelled);
    4348              :                     } else {
    4349              :                         // A timeline may be shutting down independently of the tenant's lifecycle: we should
    4350              :                         // skip past this and proceed to try GC on other timelines.
    4351            0 :                         continue;
    4352              :                     }
    4353              :                 }
    4354            4 :                 r => r?,
    4355              :             };
    4356            4 :             totals += result;
    4357              :         }
    4358              : 
    4359            4 :         totals.elapsed = now.elapsed();
    4360            4 :         Ok(totals)
    4361            4 :     }
    4362              : 
    4363              :     /// Refreshes the Timeline::gc_info for all timelines, returning the
    4364              :     /// vector of timelines which have [`Timeline::get_last_record_lsn`] past
    4365              :     /// [`Tenant::get_gc_horizon`].
    4366              :     ///
    4367              :     /// This is usually executed as part of periodic gc, but can now be triggered more often.
    4368            0 :     pub(crate) async fn refresh_gc_info(
    4369            0 :         &self,
    4370            0 :         cancel: &CancellationToken,
    4371            0 :         ctx: &RequestContext,
    4372            0 :     ) -> Result<Vec<Arc<Timeline>>, GcError> {
    4373            0 :         // since this method can now be called at different rates than the configured gc loop, it
    4374            0 :         // might be that these configuration values get applied faster than what it was previously,
    4375            0 :         // since these were only read from the gc task.
    4376            0 :         let horizon = self.get_gc_horizon();
    4377            0 :         let pitr = self.get_pitr_interval();
    4378            0 : 
    4379            0 :         // refresh all timelines
    4380            0 :         let target_timeline_id = None;
    4381            0 : 
    4382            0 :         self.refresh_gc_info_internal(target_timeline_id, horizon, pitr, cancel, ctx)
    4383            0 :             .await
    4384            0 :     }
    4385              : 
    4386              :     /// Populate all Timelines' `GcInfo` with information about their children.  We do not set the
    4387              :     /// PITR cutoffs here, because that requires I/O: this is done later, before GC, by [`Self::refresh_gc_info_internal`]
    4388              :     ///
    4389              :     /// Subsequently, parent-child relationships are updated incrementally inside [`Timeline::new`] and [`Timeline::drop`].
    4390            0 :     fn initialize_gc_info(
    4391            0 :         &self,
    4392            0 :         timelines: &std::sync::MutexGuard<HashMap<TimelineId, Arc<Timeline>>>,
    4393            0 :         timelines_offloaded: &std::sync::MutexGuard<HashMap<TimelineId, Arc<OffloadedTimeline>>>,
    4394            0 :         restrict_to_timeline: Option<TimelineId>,
    4395            0 :     ) {
    4396            0 :         if restrict_to_timeline.is_none() {
    4397              :             // This function must be called before activation: after activation timeline create/delete operations
    4398              :             // might happen, and this function is not safe to run concurrently with those.
    4399            0 :             assert!(!self.is_active());
    4400            0 :         }
    4401              : 
    4402              :         // Scan all timelines. For each timeline, remember the timeline ID and
    4403              :         // the branch point where it was created.
    4404            0 :         let mut all_branchpoints: BTreeMap<TimelineId, Vec<(Lsn, TimelineId, MaybeOffloaded)>> =
    4405            0 :             BTreeMap::new();
    4406            0 :         timelines.iter().for_each(|(timeline_id, timeline_entry)| {
    4407            0 :             if let Some(ancestor_timeline_id) = &timeline_entry.get_ancestor_timeline_id() {
    4408            0 :                 let ancestor_children = all_branchpoints.entry(*ancestor_timeline_id).or_default();
    4409            0 :                 ancestor_children.push((
    4410            0 :                     timeline_entry.get_ancestor_lsn(),
    4411            0 :                     *timeline_id,
    4412            0 :                     MaybeOffloaded::No,
    4413            0 :                 ));
    4414            0 :             }
    4415            0 :         });
    4416            0 :         timelines_offloaded
    4417            0 :             .iter()
    4418            0 :             .for_each(|(timeline_id, timeline_entry)| {
    4419            0 :                 let Some(ancestor_timeline_id) = &timeline_entry.ancestor_timeline_id else {
    4420            0 :                     return;
    4421              :                 };
    4422            0 :                 let Some(retain_lsn) = timeline_entry.ancestor_retain_lsn else {
    4423            0 :                     return;
    4424              :                 };
    4425            0 :                 let ancestor_children = all_branchpoints.entry(*ancestor_timeline_id).or_default();
    4426            0 :                 ancestor_children.push((retain_lsn, *timeline_id, MaybeOffloaded::Yes));
    4427            0 :             });
    4428            0 : 
    4429            0 :         // The number of bytes we always keep, irrespective of PITR: this is a constant across timelines
    4430            0 :         let horizon = self.get_gc_horizon();
    4431              : 
    4432              :         // Populate each timeline's GcInfo with information about its child branches
    4433            0 :         let timelines_to_write = if let Some(timeline_id) = restrict_to_timeline {
    4434            0 :             itertools::Either::Left(timelines.get(&timeline_id).into_iter())
    4435              :         } else {
    4436            0 :             itertools::Either::Right(timelines.values())
    4437              :         };
    4438            0 :         for timeline in timelines_to_write {
    4439            0 :             let mut branchpoints: Vec<(Lsn, TimelineId, MaybeOffloaded)> = all_branchpoints
    4440            0 :                 .remove(&timeline.timeline_id)
    4441            0 :                 .unwrap_or_default();
    4442            0 : 
    4443            0 :             branchpoints.sort_by_key(|b| b.0);
    4444            0 : 
    4445            0 :             let mut target = timeline.gc_info.write().unwrap();
    4446            0 : 
    4447            0 :             target.retain_lsns = branchpoints;
    4448            0 : 
    4449            0 :             let space_cutoff = timeline
    4450            0 :                 .get_last_record_lsn()
    4451            0 :                 .checked_sub(horizon)
    4452            0 :                 .unwrap_or(Lsn(0));
    4453            0 : 
    4454            0 :             target.cutoffs = GcCutoffs {
    4455            0 :                 space: space_cutoff,
    4456            0 :                 time: Lsn::INVALID,
    4457            0 :             };
    4458            0 :         }
    4459            0 :     }
    4460              : 
    4461            4 :     async fn refresh_gc_info_internal(
    4462            4 :         &self,
    4463            4 :         target_timeline_id: Option<TimelineId>,
    4464            4 :         horizon: u64,
    4465            4 :         pitr: Duration,
    4466            4 :         cancel: &CancellationToken,
    4467            4 :         ctx: &RequestContext,
    4468            4 :     ) -> Result<Vec<Arc<Timeline>>, GcError> {
    4469            4 :         // before taking the gc_cs lock, do the heavier weight finding of gc_cutoff points for
    4470            4 :         // currently visible timelines.
    4471            4 :         let timelines = self
    4472            4 :             .timelines
    4473            4 :             .lock()
    4474            4 :             .unwrap()
    4475            4 :             .values()
    4476            4 :             .filter(|tl| match target_timeline_id.as_ref() {
    4477            4 :                 Some(target) => &tl.timeline_id == target,
    4478            0 :                 None => true,
    4479            4 :             })
    4480            4 :             .cloned()
    4481            4 :             .collect::<Vec<_>>();
    4482            4 : 
    4483            4 :         if target_timeline_id.is_some() && timelines.is_empty() {
    4484              :             // We were to act on a particular timeline and it wasn't found
    4485            0 :             return Err(GcError::TimelineNotFound);
    4486            4 :         }
    4487            4 : 
    4488            4 :         let mut gc_cutoffs: HashMap<TimelineId, GcCutoffs> =
    4489            4 :             HashMap::with_capacity(timelines.len());
    4490            4 : 
    4491            4 :         // Ensures all timelines use the same start time when computing the time cutoff.
    4492            4 :         let now_ts_for_pitr_calc = SystemTime::now();
    4493            4 :         for timeline in timelines.iter() {
    4494            4 :             let cutoff = timeline
    4495            4 :                 .get_last_record_lsn()
    4496            4 :                 .checked_sub(horizon)
    4497            4 :                 .unwrap_or(Lsn(0));
    4498              : 
    4499            4 :             let cutoffs = timeline
    4500            4 :                 .find_gc_cutoffs(now_ts_for_pitr_calc, cutoff, pitr, cancel, ctx)
    4501            4 :                 .await?;
    4502            4 :             let old = gc_cutoffs.insert(timeline.timeline_id, cutoffs);
    4503            4 :             assert!(old.is_none());
    4504              :         }
    4505              : 
    4506            4 :         if !self.is_active() || self.cancel.is_cancelled() {
    4507            0 :             return Err(GcError::TenantCancelled);
    4508            4 :         }
    4509              : 
    4510              :         // grab mutex to prevent new timelines from being created here; avoid doing long operations
    4511              :         // because that will stall branch creation.
    4512            4 :         let gc_cs = self.gc_cs.lock().await;
    4513              : 
    4514              :         // Ok, we now know all the branch points.
    4515              :         // Update the GC information for each timeline.
    4516            4 :         let mut gc_timelines = Vec::with_capacity(timelines.len());
    4517            8 :         for timeline in timelines {
    4518              :             // We filtered the timeline list above
    4519            4 :             if let Some(target_timeline_id) = target_timeline_id {
    4520            4 :                 assert_eq!(target_timeline_id, timeline.timeline_id);
    4521            0 :             }
    4522              : 
    4523              :             {
    4524            4 :                 let mut target = timeline.gc_info.write().unwrap();
    4525            4 : 
    4526            4 :                 // Cull any expired leases
    4527            4 :                 let now = SystemTime::now();
    4528            6 :                 target.leases.retain(|_, lease| !lease.is_expired(&now));
    4529            4 : 
    4530            4 :                 timeline
    4531            4 :                     .metrics
    4532            4 :                     .valid_lsn_lease_count_gauge
    4533            4 :                     .set(target.leases.len() as u64);
    4534              : 
    4535              :                 // Look up parent's PITR cutoff to update the child's knowledge of whether it is within parent's PITR
    4536            4 :                 if let Some(ancestor_id) = timeline.get_ancestor_timeline_id() {
    4537            0 :                     if let Some(ancestor_gc_cutoffs) = gc_cutoffs.get(&ancestor_id) {
    4538            0 :                         target.within_ancestor_pitr =
    4539            0 :                             timeline.get_ancestor_lsn() >= ancestor_gc_cutoffs.time;
    4540            0 :                     }
    4541            4 :                 }
    4542              : 
    4543              :                 // Update metrics that depend on GC state
    4544            4 :                 timeline
    4545            4 :                     .metrics
    4546            4 :                     .archival_size
    4547            4 :                     .set(if target.within_ancestor_pitr {
    4548            0 :                         timeline.metrics.current_logical_size_gauge.get()
    4549              :                     } else {
    4550            4 :                         0
    4551              :                     });
    4552            4 :                 timeline.metrics.pitr_history_size.set(
    4553            4 :                     timeline
    4554            4 :                         .get_last_record_lsn()
    4555            4 :                         .checked_sub(target.cutoffs.time)
    4556            4 :                         .unwrap_or(Lsn(0))
    4557            4 :                         .0,
    4558            4 :                 );
    4559              : 
    4560              :                 // Apply the cutoffs we found to the Timeline's GcInfo.  Why might we _not_ have cutoffs for a timeline?
    4561              :                 // - this timeline was created while we were finding cutoffs
    4562              :                 // - lsn for timestamp search fails for this timeline repeatedly
    4563            4 :                 if let Some(cutoffs) = gc_cutoffs.get(&timeline.timeline_id) {
    4564            4 :                     let original_cutoffs = target.cutoffs.clone();
    4565            4 :                     // GC cutoffs should never go back
    4566            4 :                     target.cutoffs = GcCutoffs {
    4567            4 :                         space: Lsn(cutoffs.space.0.max(original_cutoffs.space.0)),
    4568            4 :                         time: Lsn(cutoffs.time.0.max(original_cutoffs.time.0)),
    4569            4 :                     }
    4570            0 :                 }
    4571              :             }
    4572              : 
    4573            4 :             gc_timelines.push(timeline);
    4574              :         }
    4575            4 :         drop(gc_cs);
    4576            4 :         Ok(gc_timelines)
    4577            4 :     }
    4578              : 
    4579              :     /// A substitute for `branch_timeline` for use in unit tests.
    4580              :     /// The returned timeline will have state value `Active` to make various `anyhow::ensure!()`
    4581              :     /// calls pass, but, we do not actually call `.activate()` under the hood. So, none of the
    4582              :     /// timeline background tasks are launched, except the flush loop.
    4583              :     #[cfg(test)]
    4584          232 :     async fn branch_timeline_test(
    4585          232 :         self: &Arc<Self>,
    4586          232 :         src_timeline: &Arc<Timeline>,
    4587          232 :         dst_id: TimelineId,
    4588          232 :         ancestor_lsn: Option<Lsn>,
    4589          232 :         ctx: &RequestContext,
    4590          232 :     ) -> Result<Arc<Timeline>, CreateTimelineError> {
    4591          232 :         let tl = self
    4592          232 :             .branch_timeline_impl(src_timeline, dst_id, ancestor_lsn, ctx)
    4593          232 :             .await?
    4594          228 :             .into_timeline_for_test();
    4595          228 :         tl.set_state(TimelineState::Active);
    4596          228 :         Ok(tl)
    4597          232 :     }
    4598              : 
    4599              :     /// Helper for unit tests to branch a timeline with some pre-loaded states.
    4600              :     #[cfg(test)]
    4601              :     #[allow(clippy::too_many_arguments)]
    4602            6 :     pub async fn branch_timeline_test_with_layers(
    4603            6 :         self: &Arc<Self>,
    4604            6 :         src_timeline: &Arc<Timeline>,
    4605            6 :         dst_id: TimelineId,
    4606            6 :         ancestor_lsn: Option<Lsn>,
    4607            6 :         ctx: &RequestContext,
    4608            6 :         delta_layer_desc: Vec<timeline::DeltaLayerTestDesc>,
    4609            6 :         image_layer_desc: Vec<(Lsn, Vec<(pageserver_api::key::Key, bytes::Bytes)>)>,
    4610            6 :         end_lsn: Lsn,
    4611            6 :     ) -> anyhow::Result<Arc<Timeline>> {
    4612              :         use checks::check_valid_layermap;
    4613              :         use itertools::Itertools;
    4614              : 
    4615            6 :         let tline = self
    4616            6 :             .branch_timeline_test(src_timeline, dst_id, ancestor_lsn, ctx)
    4617            6 :             .await?;
    4618            6 :         let ancestor_lsn = if let Some(ancestor_lsn) = ancestor_lsn {
    4619            6 :             ancestor_lsn
    4620              :         } else {
    4621            0 :             tline.get_last_record_lsn()
    4622              :         };
    4623            6 :         assert!(end_lsn >= ancestor_lsn);
    4624            6 :         tline.force_advance_lsn(end_lsn);
    4625           12 :         for deltas in delta_layer_desc {
    4626            6 :             tline
    4627            6 :                 .force_create_delta_layer(deltas, Some(ancestor_lsn), ctx)
    4628            6 :                 .await?;
    4629              :         }
    4630           10 :         for (lsn, images) in image_layer_desc {
    4631            4 :             tline
    4632            4 :                 .force_create_image_layer(lsn, images, Some(ancestor_lsn), ctx)
    4633            4 :                 .await?;
    4634              :         }
    4635            6 :         let layer_names = tline
    4636            6 :             .layers
    4637            6 :             .read()
    4638            6 :             .await
    4639            6 :             .layer_map()
    4640            6 :             .unwrap()
    4641            6 :             .iter_historic_layers()
    4642           10 :             .map(|layer| layer.layer_name())
    4643            6 :             .collect_vec();
    4644            6 :         if let Some(err) = check_valid_layermap(&layer_names) {
    4645            0 :             bail!("invalid layermap: {err}");
    4646            6 :         }
    4647            6 :         Ok(tline)
    4648            6 :     }
    4649              : 
    4650              :     /// Branch an existing timeline.
    4651            0 :     async fn branch_timeline(
    4652            0 :         self: &Arc<Self>,
    4653            0 :         src_timeline: &Arc<Timeline>,
    4654            0 :         dst_id: TimelineId,
    4655            0 :         start_lsn: Option<Lsn>,
    4656            0 :         ctx: &RequestContext,
    4657            0 :     ) -> Result<CreateTimelineResult, CreateTimelineError> {
    4658            0 :         self.branch_timeline_impl(src_timeline, dst_id, start_lsn, ctx)
    4659            0 :             .await
    4660            0 :     }
    4661              : 
    4662          232 :     async fn branch_timeline_impl(
    4663          232 :         self: &Arc<Self>,
    4664          232 :         src_timeline: &Arc<Timeline>,
    4665          232 :         dst_id: TimelineId,
    4666          232 :         start_lsn: Option<Lsn>,
    4667          232 :         _ctx: &RequestContext,
    4668          232 :     ) -> Result<CreateTimelineResult, CreateTimelineError> {
    4669          232 :         let src_id = src_timeline.timeline_id;
    4670              : 
    4671              :         // We will validate our ancestor LSN in this function.  Acquire the GC lock so that
    4672              :         // this check cannot race with GC, and the ancestor LSN is guaranteed to remain
    4673              :         // valid while we are creating the branch.
    4674          232 :         let _gc_cs = self.gc_cs.lock().await;
    4675              : 
    4676              :         // If no start LSN is specified, we branch the new timeline from the source timeline's last record LSN
    4677          232 :         let start_lsn = start_lsn.unwrap_or_else(|| {
    4678            2 :             let lsn = src_timeline.get_last_record_lsn();
    4679            2 :             info!("branching timeline {dst_id} from timeline {src_id} at last record LSN: {lsn}");
    4680            2 :             lsn
    4681          232 :         });
    4682              : 
    4683              :         // we finally have determined the ancestor_start_lsn, so we can get claim exclusivity now
    4684          232 :         let timeline_create_guard = match self
    4685          232 :             .start_creating_timeline(
    4686          232 :                 dst_id,
    4687          232 :                 CreateTimelineIdempotency::Branch {
    4688          232 :                     ancestor_timeline_id: src_timeline.timeline_id,
    4689          232 :                     ancestor_start_lsn: start_lsn,
    4690          232 :                 },
    4691          232 :             )
    4692          232 :             .await?
    4693              :         {
    4694          232 :             StartCreatingTimelineResult::CreateGuard(guard) => guard,
    4695            0 :             StartCreatingTimelineResult::Idempotent(timeline) => {
    4696            0 :                 return Ok(CreateTimelineResult::Idempotent(timeline));
    4697              :             }
    4698              :         };
    4699              : 
    4700              :         // Ensure that `start_lsn` is valid, i.e. the LSN is within the PITR
    4701              :         // horizon on the source timeline
    4702              :         //
    4703              :         // We check it against both the planned GC cutoff stored in 'gc_info',
    4704              :         // and the 'latest_gc_cutoff' of the last GC that was performed.  The
    4705              :         // planned GC cutoff in 'gc_info' is normally larger than
    4706              :         // 'latest_gc_cutoff_lsn', but beware of corner cases like if you just
    4707              :         // changed the GC settings for the tenant to make the PITR window
    4708              :         // larger, but some of the data was already removed by an earlier GC
    4709              :         // iteration.
    4710              : 
    4711              :         // check against last actual 'latest_gc_cutoff' first
    4712          232 :         let latest_gc_cutoff_lsn = src_timeline.get_latest_gc_cutoff_lsn();
    4713          232 :         src_timeline
    4714          232 :             .check_lsn_is_in_scope(start_lsn, &latest_gc_cutoff_lsn)
    4715          232 :             .context(format!(
    4716          232 :                 "invalid branch start lsn: less than latest GC cutoff {}",
    4717          232 :                 *latest_gc_cutoff_lsn,
    4718          232 :             ))
    4719          232 :             .map_err(CreateTimelineError::AncestorLsn)?;
    4720              : 
    4721              :         // and then the planned GC cutoff
    4722              :         {
    4723          228 :             let gc_info = src_timeline.gc_info.read().unwrap();
    4724          228 :             let cutoff = gc_info.min_cutoff();
    4725          228 :             if start_lsn < cutoff {
    4726            0 :                 return Err(CreateTimelineError::AncestorLsn(anyhow::anyhow!(
    4727            0 :                     "invalid branch start lsn: less than planned GC cutoff {cutoff}"
    4728            0 :                 )));
    4729          228 :             }
    4730          228 :         }
    4731          228 : 
    4732          228 :         //
    4733          228 :         // The branch point is valid, and we are still holding the 'gc_cs' lock
    4734          228 :         // so that GC cannot advance the GC cutoff until we are finished.
    4735          228 :         // Proceed with the branch creation.
    4736          228 :         //
    4737          228 : 
    4738          228 :         // Determine prev-LSN for the new timeline. We can only determine it if
    4739          228 :         // the timeline was branched at the current end of the source timeline.
    4740          228 :         let RecordLsn {
    4741          228 :             last: src_last,
    4742          228 :             prev: src_prev,
    4743          228 :         } = src_timeline.get_last_record_rlsn();
    4744          228 :         let dst_prev = if src_last == start_lsn {
    4745          216 :             Some(src_prev)
    4746              :         } else {
    4747           12 :             None
    4748              :         };
    4749              : 
    4750              :         // Create the metadata file, noting the ancestor of the new timeline.
    4751              :         // There is initially no data in it, but all the read-calls know to look
    4752              :         // into the ancestor.
    4753          228 :         let metadata = TimelineMetadata::new(
    4754          228 :             start_lsn,
    4755          228 :             dst_prev,
    4756          228 :             Some(src_id),
    4757          228 :             start_lsn,
    4758          228 :             *src_timeline.latest_gc_cutoff_lsn.read(), // FIXME: should we hold onto this guard longer?
    4759          228 :             src_timeline.initdb_lsn,
    4760          228 :             src_timeline.pg_version,
    4761          228 :         );
    4762              : 
    4763          228 :         let uninitialized_timeline = self
    4764          228 :             .prepare_new_timeline(
    4765          228 :                 dst_id,
    4766          228 :                 &metadata,
    4767          228 :                 timeline_create_guard,
    4768          228 :                 start_lsn + 1,
    4769          228 :                 Some(Arc::clone(src_timeline)),
    4770          228 :             )
    4771          228 :             .await?;
    4772              : 
    4773          228 :         let new_timeline = uninitialized_timeline.finish_creation()?;
    4774              : 
    4775              :         // Root timeline gets its layers during creation and uploads them along with the metadata.
    4776              :         // A branch timeline though, when created, can get no writes for some time, hence won't get any layers created.
    4777              :         // We still need to upload its metadata eagerly: if other nodes `attach` the tenant and miss this timeline, their GC
    4778              :         // could get incorrect information and remove more layers, than needed.
    4779              :         // See also https://github.com/neondatabase/neon/issues/3865
    4780          228 :         new_timeline
    4781          228 :             .remote_client
    4782          228 :             .schedule_index_upload_for_full_metadata_update(&metadata)
    4783          228 :             .context("branch initial metadata upload")?;
    4784              : 
    4785              :         // Callers are responsible to wait for uploads to complete and for activating the timeline.
    4786              : 
    4787          228 :         Ok(CreateTimelineResult::Created(new_timeline))
    4788          232 :     }
    4789              : 
    4790              :     /// For unit tests, make this visible so that other modules can directly create timelines
    4791              :     #[cfg(test)]
    4792            2 :     #[tracing::instrument(skip_all, fields(tenant_id=%self.tenant_shard_id.tenant_id, shard_id=%self.tenant_shard_id.shard_slug(), %timeline_id))]
    4793              :     pub(crate) async fn bootstrap_timeline_test(
    4794              :         self: &Arc<Self>,
    4795              :         timeline_id: TimelineId,
    4796              :         pg_version: u32,
    4797              :         load_existing_initdb: Option<TimelineId>,
    4798              :         ctx: &RequestContext,
    4799              :     ) -> anyhow::Result<Arc<Timeline>> {
    4800              :         self.bootstrap_timeline(timeline_id, pg_version, load_existing_initdb, ctx)
    4801              :             .await
    4802              :             .map_err(anyhow::Error::new)
    4803            2 :             .map(|r| r.into_timeline_for_test())
    4804              :     }
    4805              : 
    4806              :     /// Get exclusive access to the timeline ID for creation.
    4807              :     ///
    4808              :     /// Timeline-creating code paths must use this function before making changes
    4809              :     /// to in-memory or persistent state.
    4810              :     ///
    4811              :     /// The `state` parameter is a description of the timeline creation operation
    4812              :     /// we intend to perform.
    4813              :     /// If the timeline was already created in the meantime, we check whether this
    4814              :     /// request conflicts or is idempotent , based on `state`.
    4815          422 :     async fn start_creating_timeline(
    4816          422 :         self: &Arc<Self>,
    4817          422 :         new_timeline_id: TimelineId,
    4818          422 :         idempotency: CreateTimelineIdempotency,
    4819          422 :     ) -> Result<StartCreatingTimelineResult, CreateTimelineError> {
    4820          422 :         let allow_offloaded = false;
    4821          422 :         match self.create_timeline_create_guard(new_timeline_id, idempotency, allow_offloaded) {
    4822          420 :             Ok(create_guard) => {
    4823          420 :                 pausable_failpoint!("timeline-creation-after-uninit");
    4824          420 :                 Ok(StartCreatingTimelineResult::CreateGuard(create_guard))
    4825              :             }
    4826            0 :             Err(TimelineExclusionError::ShuttingDown) => Err(CreateTimelineError::ShuttingDown),
    4827              :             Err(TimelineExclusionError::AlreadyCreating) => {
    4828              :                 // Creation is in progress, we cannot create it again, and we cannot
    4829              :                 // check if this request matches the existing one, so caller must try
    4830              :                 // again later.
    4831            0 :                 Err(CreateTimelineError::AlreadyCreating)
    4832              :             }
    4833            0 :             Err(TimelineExclusionError::Other(e)) => Err(CreateTimelineError::Other(e)),
    4834              :             Err(TimelineExclusionError::AlreadyExists {
    4835            0 :                 existing: TimelineOrOffloaded::Offloaded(_existing),
    4836            0 :                 ..
    4837            0 :             }) => {
    4838            0 :                 info!("timeline already exists but is offloaded");
    4839            0 :                 Err(CreateTimelineError::Conflict)
    4840              :             }
    4841              :             Err(TimelineExclusionError::AlreadyExists {
    4842            2 :                 existing: TimelineOrOffloaded::Timeline(existing),
    4843            2 :                 arg,
    4844            2 :             }) => {
    4845            2 :                 {
    4846            2 :                     let existing = &existing.create_idempotency;
    4847            2 :                     let _span = info_span!("idempotency_check", ?existing, ?arg).entered();
    4848            2 :                     debug!("timeline already exists");
    4849              : 
    4850            2 :                     match (existing, &arg) {
    4851              :                         // FailWithConflict => no idempotency check
    4852              :                         (CreateTimelineIdempotency::FailWithConflict, _)
    4853              :                         | (_, CreateTimelineIdempotency::FailWithConflict) => {
    4854            2 :                             warn!("timeline already exists, failing request");
    4855            2 :                             return Err(CreateTimelineError::Conflict);
    4856              :                         }
    4857              :                         // Idempotent <=> CreateTimelineIdempotency is identical
    4858            0 :                         (x, y) if x == y => {
    4859            0 :                             info!("timeline already exists and idempotency matches, succeeding request");
    4860              :                             // fallthrough
    4861              :                         }
    4862              :                         (_, _) => {
    4863            0 :                             warn!("idempotency conflict, failing request");
    4864            0 :                             return Err(CreateTimelineError::Conflict);
    4865              :                         }
    4866              :                     }
    4867              :                 }
    4868              : 
    4869            0 :                 Ok(StartCreatingTimelineResult::Idempotent(existing))
    4870              :             }
    4871              :         }
    4872          422 :     }
    4873              : 
    4874            0 :     async fn upload_initdb(
    4875            0 :         &self,
    4876            0 :         timelines_path: &Utf8PathBuf,
    4877            0 :         pgdata_path: &Utf8PathBuf,
    4878            0 :         timeline_id: &TimelineId,
    4879            0 :     ) -> anyhow::Result<()> {
    4880            0 :         let temp_path = timelines_path.join(format!(
    4881            0 :             "{INITDB_PATH}.upload-{timeline_id}.{TEMP_FILE_SUFFIX}"
    4882            0 :         ));
    4883            0 : 
    4884            0 :         scopeguard::defer! {
    4885            0 :             if let Err(e) = fs::remove_file(&temp_path) {
    4886            0 :                 error!("Failed to remove temporary initdb archive '{temp_path}': {e}");
    4887            0 :             }
    4888            0 :         }
    4889              : 
    4890            0 :         let (pgdata_zstd, tar_zst_size) = create_zst_tarball(pgdata_path, &temp_path).await?;
    4891              :         const INITDB_TAR_ZST_WARN_LIMIT: u64 = 2 * 1024 * 1024;
    4892            0 :         if tar_zst_size > INITDB_TAR_ZST_WARN_LIMIT {
    4893            0 :             warn!(
    4894            0 :                 "compressed {temp_path} size of {tar_zst_size} is above limit {INITDB_TAR_ZST_WARN_LIMIT}."
    4895              :             );
    4896            0 :         }
    4897              : 
    4898            0 :         pausable_failpoint!("before-initdb-upload");
    4899              : 
    4900            0 :         backoff::retry(
    4901            0 :             || async {
    4902            0 :                 self::remote_timeline_client::upload_initdb_dir(
    4903            0 :                     &self.remote_storage,
    4904            0 :                     &self.tenant_shard_id.tenant_id,
    4905            0 :                     timeline_id,
    4906            0 :                     pgdata_zstd.try_clone().await?,
    4907            0 :                     tar_zst_size,
    4908            0 :                     &self.cancel,
    4909            0 :                 )
    4910            0 :                 .await
    4911            0 :             },
    4912            0 :             |_| false,
    4913            0 :             3,
    4914            0 :             u32::MAX,
    4915            0 :             "persist_initdb_tar_zst",
    4916            0 :             &self.cancel,
    4917            0 :         )
    4918            0 :         .await
    4919            0 :         .ok_or_else(|| anyhow::Error::new(TimeoutOrCancel::Cancel))
    4920            0 :         .and_then(|x| x)
    4921            0 :     }
    4922              : 
    4923              :     /// - run initdb to init temporary instance and get bootstrap data
    4924              :     /// - after initialization completes, tar up the temp dir and upload it to S3.
    4925            2 :     async fn bootstrap_timeline(
    4926            2 :         self: &Arc<Self>,
    4927            2 :         timeline_id: TimelineId,
    4928            2 :         pg_version: u32,
    4929            2 :         load_existing_initdb: Option<TimelineId>,
    4930            2 :         ctx: &RequestContext,
    4931            2 :     ) -> Result<CreateTimelineResult, CreateTimelineError> {
    4932            2 :         let timeline_create_guard = match self
    4933            2 :             .start_creating_timeline(
    4934            2 :                 timeline_id,
    4935            2 :                 CreateTimelineIdempotency::Bootstrap { pg_version },
    4936            2 :             )
    4937            2 :             .await?
    4938              :         {
    4939            2 :             StartCreatingTimelineResult::CreateGuard(guard) => guard,
    4940            0 :             StartCreatingTimelineResult::Idempotent(timeline) => {
    4941            0 :                 return Ok(CreateTimelineResult::Idempotent(timeline))
    4942              :             }
    4943              :         };
    4944              : 
    4945              :         // create a `tenant/{tenant_id}/timelines/basebackup-{timeline_id}.{TEMP_FILE_SUFFIX}/`
    4946              :         // temporary directory for basebackup files for the given timeline.
    4947              : 
    4948            2 :         let timelines_path = self.conf.timelines_path(&self.tenant_shard_id);
    4949            2 :         let pgdata_path = path_with_suffix_extension(
    4950            2 :             timelines_path.join(format!("basebackup-{timeline_id}")),
    4951            2 :             TEMP_FILE_SUFFIX,
    4952            2 :         );
    4953            2 : 
    4954            2 :         // Remove whatever was left from the previous runs: safe because TimelineCreateGuard guarantees
    4955            2 :         // we won't race with other creations or existent timelines with the same path.
    4956            2 :         if pgdata_path.exists() {
    4957            0 :             fs::remove_dir_all(&pgdata_path).with_context(|| {
    4958            0 :                 format!("Failed to remove already existing initdb directory: {pgdata_path}")
    4959            0 :             })?;
    4960            2 :         }
    4961              : 
    4962              :         // this new directory is very temporary, set to remove it immediately after bootstrap, we don't need it
    4963            2 :         scopeguard::defer! {
    4964            2 :             if let Err(e) = fs::remove_dir_all(&pgdata_path) {
    4965            2 :                 // this is unlikely, but we will remove the directory on pageserver restart or another bootstrap call
    4966            2 :                 error!("Failed to remove temporary initdb directory '{pgdata_path}': {e}");
    4967            2 :             }
    4968            2 :         }
    4969            2 :         if let Some(existing_initdb_timeline_id) = load_existing_initdb {
    4970            2 :             if existing_initdb_timeline_id != timeline_id {
    4971            0 :                 let source_path = &remote_initdb_archive_path(
    4972            0 :                     &self.tenant_shard_id.tenant_id,
    4973            0 :                     &existing_initdb_timeline_id,
    4974            0 :                 );
    4975            0 :                 let dest_path =
    4976            0 :                     &remote_initdb_archive_path(&self.tenant_shard_id.tenant_id, &timeline_id);
    4977            0 : 
    4978            0 :                 // if this fails, it will get retried by retried control plane requests
    4979            0 :                 self.remote_storage
    4980            0 :                     .copy_object(source_path, dest_path, &self.cancel)
    4981            0 :                     .await
    4982            0 :                     .context("copy initdb tar")?;
    4983            2 :             }
    4984            2 :             let (initdb_tar_zst_path, initdb_tar_zst) =
    4985            2 :                 self::remote_timeline_client::download_initdb_tar_zst(
    4986            2 :                     self.conf,
    4987            2 :                     &self.remote_storage,
    4988            2 :                     &self.tenant_shard_id,
    4989            2 :                     &existing_initdb_timeline_id,
    4990            2 :                     &self.cancel,
    4991            2 :                 )
    4992            2 :                 .await
    4993            2 :                 .context("download initdb tar")?;
    4994              : 
    4995            2 :             scopeguard::defer! {
    4996            2 :                 if let Err(e) = fs::remove_file(&initdb_tar_zst_path) {
    4997            2 :                     error!("Failed to remove temporary initdb archive '{initdb_tar_zst_path}': {e}");
    4998            2 :                 }
    4999            2 :             }
    5000            2 : 
    5001            2 :             let buf_read =
    5002            2 :                 BufReader::with_capacity(remote_timeline_client::BUFFER_SIZE, initdb_tar_zst);
    5003            2 :             extract_zst_tarball(&pgdata_path, buf_read)
    5004            2 :                 .await
    5005            2 :                 .context("extract initdb tar")?;
    5006              :         } else {
    5007              :             // Init temporarily repo to get bootstrap data, this creates a directory in the `pgdata_path` path
    5008            0 :             run_initdb(self.conf, &pgdata_path, pg_version, &self.cancel)
    5009            0 :                 .await
    5010            0 :                 .context("run initdb")?;
    5011              : 
    5012              :             // Upload the created data dir to S3
    5013            0 :             if self.tenant_shard_id().is_shard_zero() {
    5014            0 :                 self.upload_initdb(&timelines_path, &pgdata_path, &timeline_id)
    5015            0 :                     .await?;
    5016            0 :             }
    5017              :         }
    5018            2 :         let pgdata_lsn = import_datadir::get_lsn_from_controlfile(&pgdata_path)?.align();
    5019            2 : 
    5020            2 :         // Import the contents of the data directory at the initial checkpoint
    5021            2 :         // LSN, and any WAL after that.
    5022            2 :         // Initdb lsn will be equal to last_record_lsn which will be set after import.
    5023            2 :         // Because we know it upfront avoid having an option or dummy zero value by passing it to the metadata.
    5024            2 :         let new_metadata = TimelineMetadata::new(
    5025            2 :             Lsn(0),
    5026            2 :             None,
    5027            2 :             None,
    5028            2 :             Lsn(0),
    5029            2 :             pgdata_lsn,
    5030            2 :             pgdata_lsn,
    5031            2 :             pg_version,
    5032            2 :         );
    5033            2 :         let raw_timeline = self
    5034            2 :             .prepare_new_timeline(
    5035            2 :                 timeline_id,
    5036            2 :                 &new_metadata,
    5037            2 :                 timeline_create_guard,
    5038            2 :                 pgdata_lsn,
    5039            2 :                 None,
    5040            2 :             )
    5041            2 :             .await?;
    5042              : 
    5043            2 :         let tenant_shard_id = raw_timeline.owning_tenant.tenant_shard_id;
    5044            2 :         let unfinished_timeline = raw_timeline.raw_timeline()?;
    5045              : 
    5046              :         // Flush the new layer files to disk, before we make the timeline as available to
    5047              :         // the outside world.
    5048              :         //
    5049              :         // Flush loop needs to be spawned in order to be able to flush.
    5050            2 :         unfinished_timeline.maybe_spawn_flush_loop();
    5051            2 : 
    5052            2 :         import_datadir::import_timeline_from_postgres_datadir(
    5053            2 :             unfinished_timeline,
    5054            2 :             &pgdata_path,
    5055            2 :             pgdata_lsn,
    5056            2 :             ctx,
    5057            2 :         )
    5058            2 :         .await
    5059            2 :         .with_context(|| {
    5060            0 :             format!("Failed to import pgdatadir for timeline {tenant_shard_id}/{timeline_id}")
    5061            2 :         })?;
    5062              : 
    5063            2 :         fail::fail_point!("before-checkpoint-new-timeline", |_| {
    5064            0 :             Err(CreateTimelineError::Other(anyhow::anyhow!(
    5065            0 :                 "failpoint before-checkpoint-new-timeline"
    5066            0 :             )))
    5067            2 :         });
    5068              : 
    5069            2 :         unfinished_timeline
    5070            2 :             .freeze_and_flush()
    5071            2 :             .await
    5072            2 :             .with_context(|| {
    5073            0 :                 format!(
    5074            0 :                     "Failed to flush after pgdatadir import for timeline {tenant_shard_id}/{timeline_id}"
    5075            0 :                 )
    5076            2 :             })?;
    5077              : 
    5078              :         // All done!
    5079            2 :         let timeline = raw_timeline.finish_creation()?;
    5080              : 
    5081              :         // Callers are responsible to wait for uploads to complete and for activating the timeline.
    5082              : 
    5083            2 :         Ok(CreateTimelineResult::Created(timeline))
    5084            2 :     }
    5085              : 
    5086          416 :     fn build_timeline_remote_client(&self, timeline_id: TimelineId) -> RemoteTimelineClient {
    5087          416 :         RemoteTimelineClient::new(
    5088          416 :             self.remote_storage.clone(),
    5089          416 :             self.deletion_queue_client.clone(),
    5090          416 :             self.conf,
    5091          416 :             self.tenant_shard_id,
    5092          416 :             timeline_id,
    5093          416 :             self.generation,
    5094          416 :             &self.tenant_conf.load().location,
    5095          416 :         )
    5096          416 :     }
    5097              : 
    5098              :     /// Call this before constructing a timeline, to build its required structures
    5099          416 :     fn build_timeline_resources(&self, timeline_id: TimelineId) -> TimelineResources {
    5100          416 :         TimelineResources {
    5101          416 :             remote_client: self.build_timeline_remote_client(timeline_id),
    5102          416 :             pagestream_throttle: self.pagestream_throttle.clone(),
    5103          416 :             l0_flush_global_state: self.l0_flush_global_state.clone(),
    5104          416 :         }
    5105          416 :     }
    5106              : 
    5107              :     /// Creates intermediate timeline structure and its files.
    5108              :     ///
    5109              :     /// An empty layer map is initialized, and new data and WAL can be imported starting
    5110              :     /// at 'disk_consistent_lsn'. After any initial data has been imported, call
    5111              :     /// `finish_creation` to insert the Timeline into the timelines map.
    5112          416 :     async fn prepare_new_timeline<'a>(
    5113          416 :         &'a self,
    5114          416 :         new_timeline_id: TimelineId,
    5115          416 :         new_metadata: &TimelineMetadata,
    5116          416 :         create_guard: TimelineCreateGuard,
    5117          416 :         start_lsn: Lsn,
    5118          416 :         ancestor: Option<Arc<Timeline>>,
    5119          416 :     ) -> anyhow::Result<UninitializedTimeline<'a>> {
    5120          416 :         let tenant_shard_id = self.tenant_shard_id;
    5121          416 : 
    5122          416 :         let resources = self.build_timeline_resources(new_timeline_id);
    5123          416 :         resources
    5124          416 :             .remote_client
    5125          416 :             .init_upload_queue_for_empty_remote(new_metadata)?;
    5126              : 
    5127          416 :         let timeline_struct = self
    5128          416 :             .create_timeline_struct(
    5129          416 :                 new_timeline_id,
    5130          416 :                 new_metadata,
    5131          416 :                 ancestor,
    5132          416 :                 resources,
    5133          416 :                 CreateTimelineCause::Load,
    5134          416 :                 create_guard.idempotency.clone(),
    5135          416 :             )
    5136          416 :             .context("Failed to create timeline data structure")?;
    5137              : 
    5138          416 :         timeline_struct.init_empty_layer_map(start_lsn);
    5139              : 
    5140          416 :         if let Err(e) = self
    5141          416 :             .create_timeline_files(&create_guard.timeline_path)
    5142          416 :             .await
    5143              :         {
    5144            0 :             error!("Failed to create initial files for timeline {tenant_shard_id}/{new_timeline_id}, cleaning up: {e:?}");
    5145            0 :             cleanup_timeline_directory(create_guard);
    5146            0 :             return Err(e);
    5147          416 :         }
    5148          416 : 
    5149          416 :         debug!(
    5150            0 :             "Successfully created initial files for timeline {tenant_shard_id}/{new_timeline_id}"
    5151              :         );
    5152              : 
    5153          416 :         Ok(UninitializedTimeline::new(
    5154          416 :             self,
    5155          416 :             new_timeline_id,
    5156          416 :             Some((timeline_struct, create_guard)),
    5157          416 :         ))
    5158          416 :     }
    5159              : 
    5160          416 :     async fn create_timeline_files(&self, timeline_path: &Utf8Path) -> anyhow::Result<()> {
    5161          416 :         crashsafe::create_dir(timeline_path).context("Failed to create timeline directory")?;
    5162              : 
    5163          416 :         fail::fail_point!("after-timeline-dir-creation", |_| {
    5164            0 :             anyhow::bail!("failpoint after-timeline-dir-creation");
    5165          416 :         });
    5166              : 
    5167          416 :         Ok(())
    5168          416 :     }
    5169              : 
    5170              :     /// Get a guard that provides exclusive access to the timeline directory, preventing
    5171              :     /// concurrent attempts to create the same timeline.
    5172              :     ///
    5173              :     /// The `allow_offloaded` parameter controls whether to tolerate the existence of
    5174              :     /// offloaded timelines or not.
    5175          422 :     fn create_timeline_create_guard(
    5176          422 :         self: &Arc<Self>,
    5177          422 :         timeline_id: TimelineId,
    5178          422 :         idempotency: CreateTimelineIdempotency,
    5179          422 :         allow_offloaded: bool,
    5180          422 :     ) -> Result<TimelineCreateGuard, TimelineExclusionError> {
    5181          422 :         let tenant_shard_id = self.tenant_shard_id;
    5182          422 : 
    5183          422 :         let timeline_path = self.conf.timeline_path(&tenant_shard_id, &timeline_id);
    5184              : 
    5185          422 :         let create_guard = TimelineCreateGuard::new(
    5186          422 :             self,
    5187          422 :             timeline_id,
    5188          422 :             timeline_path.clone(),
    5189          422 :             idempotency,
    5190          422 :             allow_offloaded,
    5191          422 :         )?;
    5192              : 
    5193              :         // At this stage, we have got exclusive access to in-memory state for this timeline ID
    5194              :         // for creation.
    5195              :         // A timeline directory should never exist on disk already:
    5196              :         // - a previous failed creation would have cleaned up after itself
    5197              :         // - a pageserver restart would clean up timeline directories that don't have valid remote state
    5198              :         //
    5199              :         // Therefore it is an unexpected internal error to encounter a timeline directory already existing here,
    5200              :         // this error may indicate a bug in cleanup on failed creations.
    5201          420 :         if timeline_path.exists() {
    5202            0 :             return Err(TimelineExclusionError::Other(anyhow::anyhow!(
    5203            0 :                 "Timeline directory already exists! This is a bug."
    5204            0 :             )));
    5205          420 :         }
    5206          420 : 
    5207          420 :         Ok(create_guard)
    5208          422 :     }
    5209              : 
    5210              :     /// Gathers inputs from all of the timelines to produce a sizing model input.
    5211              :     ///
    5212              :     /// Future is cancellation safe. Only one calculation can be running at once per tenant.
    5213            0 :     #[instrument(skip_all, fields(tenant_id=%self.tenant_shard_id.tenant_id, shard_id=%self.tenant_shard_id.shard_slug()))]
    5214              :     pub async fn gather_size_inputs(
    5215              :         &self,
    5216              :         // `max_retention_period` overrides the cutoff that is used to calculate the size
    5217              :         // (only if it is shorter than the real cutoff).
    5218              :         max_retention_period: Option<u64>,
    5219              :         cause: LogicalSizeCalculationCause,
    5220              :         cancel: &CancellationToken,
    5221              :         ctx: &RequestContext,
    5222              :     ) -> Result<size::ModelInputs, size::CalculateSyntheticSizeError> {
    5223              :         let logical_sizes_at_once = self
    5224              :             .conf
    5225              :             .concurrent_tenant_size_logical_size_queries
    5226              :             .inner();
    5227              : 
    5228              :         // TODO: Having a single mutex block concurrent reads is not great for performance.
    5229              :         //
    5230              :         // But the only case where we need to run multiple of these at once is when we
    5231              :         // request a size for a tenant manually via API, while another background calculation
    5232              :         // is in progress (which is not a common case).
    5233              :         //
    5234              :         // See more for on the issue #2748 condenced out of the initial PR review.
    5235              :         let mut shared_cache = tokio::select! {
    5236              :             locked = self.cached_logical_sizes.lock() => locked,
    5237              :             _ = cancel.cancelled() => return Err(size::CalculateSyntheticSizeError::Cancelled),
    5238              :             _ = self.cancel.cancelled() => return Err(size::CalculateSyntheticSizeError::Cancelled),
    5239              :         };
    5240              : 
    5241              :         size::gather_inputs(
    5242              :             self,
    5243              :             logical_sizes_at_once,
    5244              :             max_retention_period,
    5245              :             &mut shared_cache,
    5246              :             cause,
    5247              :             cancel,
    5248              :             ctx,
    5249              :         )
    5250              :         .await
    5251              :     }
    5252              : 
    5253              :     /// Calculate synthetic tenant size and cache the result.
    5254              :     /// This is periodically called by background worker.
    5255              :     /// result is cached in tenant struct
    5256            0 :     #[instrument(skip_all, fields(tenant_id=%self.tenant_shard_id.tenant_id, shard_id=%self.tenant_shard_id.shard_slug()))]
    5257              :     pub async fn calculate_synthetic_size(
    5258              :         &self,
    5259              :         cause: LogicalSizeCalculationCause,
    5260              :         cancel: &CancellationToken,
    5261              :         ctx: &RequestContext,
    5262              :     ) -> Result<u64, size::CalculateSyntheticSizeError> {
    5263              :         let inputs = self.gather_size_inputs(None, cause, cancel, ctx).await?;
    5264              : 
    5265              :         let size = inputs.calculate();
    5266              : 
    5267              :         self.set_cached_synthetic_size(size);
    5268              : 
    5269              :         Ok(size)
    5270              :     }
    5271              : 
    5272              :     /// Cache given synthetic size and update the metric value
    5273            0 :     pub fn set_cached_synthetic_size(&self, size: u64) {
    5274            0 :         self.cached_synthetic_tenant_size
    5275            0 :             .store(size, Ordering::Relaxed);
    5276            0 : 
    5277            0 :         // Only shard zero should be calculating synthetic sizes
    5278            0 :         debug_assert!(self.shard_identity.is_shard_zero());
    5279              : 
    5280            0 :         TENANT_SYNTHETIC_SIZE_METRIC
    5281            0 :             .get_metric_with_label_values(&[&self.tenant_shard_id.tenant_id.to_string()])
    5282            0 :             .unwrap()
    5283            0 :             .set(size);
    5284            0 :     }
    5285              : 
    5286            0 :     pub fn cached_synthetic_size(&self) -> u64 {
    5287            0 :         self.cached_synthetic_tenant_size.load(Ordering::Relaxed)
    5288            0 :     }
    5289              : 
    5290              :     /// Flush any in-progress layers, schedule uploads, and wait for uploads to complete.
    5291              :     ///
    5292              :     /// This function can take a long time: callers should wrap it in a timeout if calling
    5293              :     /// from an external API handler.
    5294              :     ///
    5295              :     /// Cancel-safety: cancelling this function may leave I/O running, but such I/O is
    5296              :     /// still bounded by tenant/timeline shutdown.
    5297            0 :     #[tracing::instrument(skip_all)]
    5298              :     pub(crate) async fn flush_remote(&self) -> anyhow::Result<()> {
    5299              :         let timelines = self.timelines.lock().unwrap().clone();
    5300              : 
    5301            0 :         async fn flush_timeline(_gate: GateGuard, timeline: Arc<Timeline>) -> anyhow::Result<()> {
    5302            0 :             tracing::info!(timeline_id=%timeline.timeline_id, "Flushing...");
    5303            0 :             timeline.freeze_and_flush().await?;
    5304            0 :             tracing::info!(timeline_id=%timeline.timeline_id, "Waiting for uploads...");
    5305            0 :             timeline.remote_client.wait_completion().await?;
    5306              : 
    5307            0 :             Ok(())
    5308            0 :         }
    5309              : 
    5310              :         // We do not use a JoinSet for these tasks, because we don't want them to be
    5311              :         // aborted when this function's future is cancelled: they should stay alive
    5312              :         // holding their GateGuard until they complete, to ensure their I/Os complete
    5313              :         // before Timeline shutdown completes.
    5314              :         let mut results = FuturesUnordered::new();
    5315              : 
    5316              :         for (_timeline_id, timeline) in timelines {
    5317              :             // Run each timeline's flush in a task holding the timeline's gate: this
    5318              :             // means that if this function's future is cancelled, the Timeline shutdown
    5319              :             // will still wait for any I/O in here to complete.
    5320              :             let Ok(gate) = timeline.gate.enter() else {
    5321              :                 continue;
    5322              :             };
    5323            0 :             let jh = tokio::task::spawn(async move { flush_timeline(gate, timeline).await });
    5324              :             results.push(jh);
    5325              :         }
    5326              : 
    5327              :         while let Some(r) = results.next().await {
    5328              :             if let Err(e) = r {
    5329              :                 if !e.is_cancelled() && !e.is_panic() {
    5330              :                     tracing::error!("unexpected join error: {e:?}");
    5331              :                 }
    5332              :             }
    5333              :         }
    5334              : 
    5335              :         // The flushes we did above were just writes, but the Tenant might have had
    5336              :         // pending deletions as well from recent compaction/gc: we want to flush those
    5337              :         // as well.  This requires flushing the global delete queue.  This is cheap
    5338              :         // because it's typically a no-op.
    5339              :         match self.deletion_queue_client.flush_execute().await {
    5340              :             Ok(_) => {}
    5341              :             Err(DeletionQueueError::ShuttingDown) => {}
    5342              :         }
    5343              : 
    5344              :         Ok(())
    5345              :     }
    5346              : 
    5347            0 :     pub(crate) fn get_tenant_conf(&self) -> TenantConfOpt {
    5348            0 :         self.tenant_conf.load().tenant_conf.clone()
    5349            0 :     }
    5350              : 
    5351              :     /// How much local storage would this tenant like to have?  It can cope with
    5352              :     /// less than this (via eviction and on-demand downloads), but this function enables
    5353              :     /// the Tenant to advertise how much storage it would prefer to have to provide fast I/O
    5354              :     /// by keeping important things on local disk.
    5355              :     ///
    5356              :     /// This is a heuristic, not a guarantee: tenants that are long-idle will actually use less
    5357              :     /// than they report here, due to layer eviction.  Tenants with many active branches may
    5358              :     /// actually use more than they report here.
    5359            0 :     pub(crate) fn local_storage_wanted(&self) -> u64 {
    5360            0 :         let timelines = self.timelines.lock().unwrap();
    5361            0 : 
    5362            0 :         // Heuristic: we use the max() of the timelines' visible sizes, rather than the sum.  This
    5363            0 :         // reflects the observation that on tenants with multiple large branches, typically only one
    5364            0 :         // of them is used actively enough to occupy space on disk.
    5365            0 :         timelines
    5366            0 :             .values()
    5367            0 :             .map(|t| t.metrics.visible_physical_size_gauge.get())
    5368            0 :             .max()
    5369            0 :             .unwrap_or(0)
    5370            0 :     }
    5371              : 
    5372              :     /// Serialize and write the latest TenantManifest to remote storage.
    5373            2 :     pub(crate) async fn store_tenant_manifest(&self) -> Result<(), TenantManifestError> {
    5374              :         // Only one manifest write may be done at at time, and the contents of the manifest
    5375              :         // must be loaded while holding this lock. This makes it safe to call this function
    5376              :         // from anywhere without worrying about colliding updates.
    5377            2 :         let mut guard = tokio::select! {
    5378            2 :             g = self.tenant_manifest_upload.lock() => {
    5379            2 :                 g
    5380              :             },
    5381            2 :             _ = self.cancel.cancelled() => {
    5382            0 :                 return Err(TenantManifestError::Cancelled);
    5383              :             }
    5384              :         };
    5385              : 
    5386            2 :         let manifest = self.build_tenant_manifest();
    5387            2 :         if Some(&manifest) == (*guard).as_ref() {
    5388              :             // Optimisation: skip uploads that don't change anything.
    5389            0 :             return Ok(());
    5390            2 :         }
    5391            2 : 
    5392            2 :         upload_tenant_manifest(
    5393            2 :             &self.remote_storage,
    5394            2 :             &self.tenant_shard_id,
    5395            2 :             self.generation,
    5396            2 :             &manifest,
    5397            2 :             &self.cancel,
    5398            2 :         )
    5399            2 :         .await
    5400            2 :         .map_err(|e| {
    5401            0 :             if self.cancel.is_cancelled() {
    5402            0 :                 TenantManifestError::Cancelled
    5403              :             } else {
    5404            0 :                 TenantManifestError::RemoteStorage(e)
    5405              :             }
    5406            2 :         })?;
    5407              : 
    5408              :         // Store the successfully uploaded manifest, so that future callers can avoid
    5409              :         // re-uploading the same thing.
    5410            2 :         *guard = Some(manifest);
    5411            2 : 
    5412            2 :         Ok(())
    5413            2 :     }
    5414              : }
    5415              : 
    5416              : /// Create the cluster temporarily in 'initdbpath' directory inside the repository
    5417              : /// to get bootstrap data for timeline initialization.
    5418            0 : async fn run_initdb(
    5419            0 :     conf: &'static PageServerConf,
    5420            0 :     initdb_target_dir: &Utf8Path,
    5421            0 :     pg_version: u32,
    5422            0 :     cancel: &CancellationToken,
    5423            0 : ) -> Result<(), InitdbError> {
    5424            0 :     let initdb_bin_path = conf
    5425            0 :         .pg_bin_dir(pg_version)
    5426            0 :         .map_err(InitdbError::Other)?
    5427            0 :         .join("initdb");
    5428            0 :     let initdb_lib_dir = conf.pg_lib_dir(pg_version).map_err(InitdbError::Other)?;
    5429            0 :     info!(
    5430            0 :         "running {} in {}, libdir: {}",
    5431              :         initdb_bin_path, initdb_target_dir, initdb_lib_dir,
    5432              :     );
    5433              : 
    5434            0 :     let _permit = INIT_DB_SEMAPHORE.acquire().await;
    5435              : 
    5436            0 :     let res = postgres_initdb::do_run_initdb(postgres_initdb::RunInitdbArgs {
    5437            0 :         superuser: &conf.superuser,
    5438            0 :         locale: &conf.locale,
    5439            0 :         initdb_bin: &initdb_bin_path,
    5440            0 :         pg_version,
    5441            0 :         library_search_path: &initdb_lib_dir,
    5442            0 :         pgdata: initdb_target_dir,
    5443            0 :     })
    5444            0 :     .await
    5445            0 :     .map_err(InitdbError::Inner);
    5446            0 : 
    5447            0 :     // This isn't true cancellation support, see above. Still return an error to
    5448            0 :     // excercise the cancellation code path.
    5449            0 :     if cancel.is_cancelled() {
    5450            0 :         return Err(InitdbError::Cancelled);
    5451            0 :     }
    5452            0 : 
    5453            0 :     res
    5454            0 : }
    5455              : 
    5456              : /// Dump contents of a layer file to stdout.
    5457            0 : pub async fn dump_layerfile_from_path(
    5458            0 :     path: &Utf8Path,
    5459            0 :     verbose: bool,
    5460            0 :     ctx: &RequestContext,
    5461            0 : ) -> anyhow::Result<()> {
    5462              :     use std::os::unix::fs::FileExt;
    5463              : 
    5464              :     // All layer files start with a two-byte "magic" value, to identify the kind of
    5465              :     // file.
    5466            0 :     let file = File::open(path)?;
    5467            0 :     let mut header_buf = [0u8; 2];
    5468            0 :     file.read_exact_at(&mut header_buf, 0)?;
    5469              : 
    5470            0 :     match u16::from_be_bytes(header_buf) {
    5471              :         crate::IMAGE_FILE_MAGIC => {
    5472            0 :             ImageLayer::new_for_path(path, file)?
    5473            0 :                 .dump(verbose, ctx)
    5474            0 :                 .await?
    5475              :         }
    5476              :         crate::DELTA_FILE_MAGIC => {
    5477            0 :             DeltaLayer::new_for_path(path, file)?
    5478            0 :                 .dump(verbose, ctx)
    5479            0 :                 .await?
    5480              :         }
    5481            0 :         magic => bail!("unrecognized magic identifier: {:?}", magic),
    5482              :     }
    5483              : 
    5484            0 :     Ok(())
    5485            0 : }
    5486              : 
    5487              : #[cfg(test)]
    5488              : pub(crate) mod harness {
    5489              :     use bytes::{Bytes, BytesMut};
    5490              :     use once_cell::sync::OnceCell;
    5491              :     use pageserver_api::models::ShardParameters;
    5492              :     use pageserver_api::shard::ShardIndex;
    5493              :     use utils::logging;
    5494              : 
    5495              :     use crate::deletion_queue::mock::MockDeletionQueue;
    5496              :     use crate::l0_flush::L0FlushConfig;
    5497              :     use crate::walredo::apply_neon;
    5498              :     use pageserver_api::key::Key;
    5499              :     use pageserver_api::record::NeonWalRecord;
    5500              : 
    5501              :     use super::*;
    5502              :     use hex_literal::hex;
    5503              :     use utils::id::TenantId;
    5504              : 
    5505              :     pub const TIMELINE_ID: TimelineId =
    5506              :         TimelineId::from_array(hex!("11223344556677881122334455667788"));
    5507              :     pub const NEW_TIMELINE_ID: TimelineId =
    5508              :         TimelineId::from_array(hex!("AA223344556677881122334455667788"));
    5509              : 
    5510              :     /// Convenience function to create a page image with given string as the only content
    5511      5028831 :     pub fn test_img(s: &str) -> Bytes {
    5512      5028831 :         let mut buf = BytesMut::new();
    5513      5028831 :         buf.extend_from_slice(s.as_bytes());
    5514      5028831 :         buf.resize(64, 0);
    5515      5028831 : 
    5516      5028831 :         buf.freeze()
    5517      5028831 :     }
    5518              : 
    5519              :     impl From<TenantConf> for TenantConfOpt {
    5520          196 :         fn from(tenant_conf: TenantConf) -> Self {
    5521          196 :             Self {
    5522          196 :                 checkpoint_distance: Some(tenant_conf.checkpoint_distance),
    5523          196 :                 checkpoint_timeout: Some(tenant_conf.checkpoint_timeout),
    5524          196 :                 compaction_target_size: Some(tenant_conf.compaction_target_size),
    5525          196 :                 compaction_period: Some(tenant_conf.compaction_period),
    5526          196 :                 compaction_threshold: Some(tenant_conf.compaction_threshold),
    5527          196 :                 compaction_algorithm: Some(tenant_conf.compaction_algorithm),
    5528          196 :                 gc_horizon: Some(tenant_conf.gc_horizon),
    5529          196 :                 gc_period: Some(tenant_conf.gc_period),
    5530          196 :                 image_creation_threshold: Some(tenant_conf.image_creation_threshold),
    5531          196 :                 pitr_interval: Some(tenant_conf.pitr_interval),
    5532          196 :                 walreceiver_connect_timeout: Some(tenant_conf.walreceiver_connect_timeout),
    5533          196 :                 lagging_wal_timeout: Some(tenant_conf.lagging_wal_timeout),
    5534          196 :                 max_lsn_wal_lag: Some(tenant_conf.max_lsn_wal_lag),
    5535          196 :                 eviction_policy: Some(tenant_conf.eviction_policy),
    5536          196 :                 min_resident_size_override: tenant_conf.min_resident_size_override,
    5537          196 :                 evictions_low_residence_duration_metric_threshold: Some(
    5538          196 :                     tenant_conf.evictions_low_residence_duration_metric_threshold,
    5539          196 :                 ),
    5540          196 :                 heatmap_period: Some(tenant_conf.heatmap_period),
    5541          196 :                 lazy_slru_download: Some(tenant_conf.lazy_slru_download),
    5542          196 :                 timeline_get_throttle: Some(tenant_conf.timeline_get_throttle),
    5543          196 :                 image_layer_creation_check_threshold: Some(
    5544          196 :                     tenant_conf.image_layer_creation_check_threshold,
    5545          196 :                 ),
    5546          196 :                 lsn_lease_length: Some(tenant_conf.lsn_lease_length),
    5547          196 :                 lsn_lease_length_for_ts: Some(tenant_conf.lsn_lease_length_for_ts),
    5548          196 :                 timeline_offloading: Some(tenant_conf.timeline_offloading),
    5549          196 :                 wal_receiver_protocol_override: tenant_conf.wal_receiver_protocol_override,
    5550          196 :             }
    5551          196 :         }
    5552              :     }
    5553              : 
    5554              :     pub struct TenantHarness {
    5555              :         pub conf: &'static PageServerConf,
    5556              :         pub tenant_conf: TenantConf,
    5557              :         pub tenant_shard_id: TenantShardId,
    5558              :         pub generation: Generation,
    5559              :         pub shard: ShardIndex,
    5560              :         pub remote_storage: GenericRemoteStorage,
    5561              :         pub remote_fs_dir: Utf8PathBuf,
    5562              :         pub deletion_queue: MockDeletionQueue,
    5563              :     }
    5564              : 
    5565              :     static LOG_HANDLE: OnceCell<()> = OnceCell::new();
    5566              : 
    5567          212 :     pub(crate) fn setup_logging() {
    5568          212 :         LOG_HANDLE.get_or_init(|| {
    5569          200 :             logging::init(
    5570          200 :                 logging::LogFormat::Test,
    5571          200 :                 // enable it in case the tests exercise code paths that use
    5572          200 :                 // debug_assert_current_span_has_tenant_and_timeline_id
    5573          200 :                 logging::TracingErrorLayerEnablement::EnableWithRustLogFilter,
    5574          200 :                 logging::Output::Stdout,
    5575          200 :             )
    5576          200 :             .expect("Failed to init test logging")
    5577          212 :         });
    5578          212 :     }
    5579              : 
    5580              :     impl TenantHarness {
    5581          196 :         pub async fn create_custom(
    5582          196 :             test_name: &'static str,
    5583          196 :             tenant_conf: TenantConf,
    5584          196 :             tenant_id: TenantId,
    5585          196 :             shard_identity: ShardIdentity,
    5586          196 :             generation: Generation,
    5587          196 :         ) -> anyhow::Result<Self> {
    5588          196 :             setup_logging();
    5589          196 : 
    5590          196 :             let repo_dir = PageServerConf::test_repo_dir(test_name);
    5591          196 :             let _ = fs::remove_dir_all(&repo_dir);
    5592          196 :             fs::create_dir_all(&repo_dir)?;
    5593              : 
    5594          196 :             let conf = PageServerConf::dummy_conf(repo_dir);
    5595          196 :             // Make a static copy of the config. This can never be free'd, but that's
    5596          196 :             // OK in a test.
    5597          196 :             let conf: &'static PageServerConf = Box::leak(Box::new(conf));
    5598          196 : 
    5599          196 :             let shard = shard_identity.shard_index();
    5600          196 :             let tenant_shard_id = TenantShardId {
    5601          196 :                 tenant_id,
    5602          196 :                 shard_number: shard.shard_number,
    5603          196 :                 shard_count: shard.shard_count,
    5604          196 :             };
    5605          196 :             fs::create_dir_all(conf.tenant_path(&tenant_shard_id))?;
    5606          196 :             fs::create_dir_all(conf.timelines_path(&tenant_shard_id))?;
    5607              : 
    5608              :             use remote_storage::{RemoteStorageConfig, RemoteStorageKind};
    5609          196 :             let remote_fs_dir = conf.workdir.join("localfs");
    5610          196 :             std::fs::create_dir_all(&remote_fs_dir).unwrap();
    5611          196 :             let config = RemoteStorageConfig {
    5612          196 :                 storage: RemoteStorageKind::LocalFs {
    5613          196 :                     local_path: remote_fs_dir.clone(),
    5614          196 :                 },
    5615          196 :                 timeout: RemoteStorageConfig::DEFAULT_TIMEOUT,
    5616          196 :                 small_timeout: RemoteStorageConfig::DEFAULT_SMALL_TIMEOUT,
    5617          196 :             };
    5618          196 :             let remote_storage = GenericRemoteStorage::from_config(&config).await.unwrap();
    5619          196 :             let deletion_queue = MockDeletionQueue::new(Some(remote_storage.clone()));
    5620          196 : 
    5621          196 :             Ok(Self {
    5622          196 :                 conf,
    5623          196 :                 tenant_conf,
    5624          196 :                 tenant_shard_id,
    5625          196 :                 generation,
    5626          196 :                 shard,
    5627          196 :                 remote_storage,
    5628          196 :                 remote_fs_dir,
    5629          196 :                 deletion_queue,
    5630          196 :             })
    5631          196 :         }
    5632              : 
    5633          184 :         pub async fn create(test_name: &'static str) -> anyhow::Result<Self> {
    5634          184 :             // Disable automatic GC and compaction to make the unit tests more deterministic.
    5635          184 :             // The tests perform them manually if needed.
    5636          184 :             let tenant_conf = TenantConf {
    5637          184 :                 gc_period: Duration::ZERO,
    5638          184 :                 compaction_period: Duration::ZERO,
    5639          184 :                 ..TenantConf::default()
    5640          184 :             };
    5641          184 :             let tenant_id = TenantId::generate();
    5642          184 :             let shard = ShardIdentity::unsharded();
    5643          184 :             Self::create_custom(
    5644          184 :                 test_name,
    5645          184 :                 tenant_conf,
    5646          184 :                 tenant_id,
    5647          184 :                 shard,
    5648          184 :                 Generation::new(0xdeadbeef),
    5649          184 :             )
    5650          184 :             .await
    5651          184 :         }
    5652              : 
    5653           20 :         pub fn span(&self) -> tracing::Span {
    5654           20 :             info_span!("TenantHarness", tenant_id=%self.tenant_shard_id.tenant_id, shard_id=%self.tenant_shard_id.shard_slug())
    5655           20 :         }
    5656              : 
    5657          196 :         pub(crate) async fn load(&self) -> (Arc<Tenant>, RequestContext) {
    5658          196 :             let ctx = RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Error);
    5659          196 :             (
    5660          196 :                 self.do_try_load(&ctx)
    5661          196 :                     .await
    5662          196 :                     .expect("failed to load test tenant"),
    5663          196 :                 ctx,
    5664          196 :             )
    5665          196 :         }
    5666              : 
    5667          196 :         #[instrument(skip_all, fields(tenant_id=%self.tenant_shard_id.tenant_id, shard_id=%self.tenant_shard_id.shard_slug()))]
    5668              :         pub(crate) async fn do_try_load(
    5669              :             &self,
    5670              :             ctx: &RequestContext,
    5671              :         ) -> anyhow::Result<Arc<Tenant>> {
    5672              :             let walredo_mgr = Arc::new(WalRedoManager::from(TestRedoManager));
    5673              : 
    5674              :             let tenant = Arc::new(Tenant::new(
    5675              :                 TenantState::Attaching,
    5676              :                 self.conf,
    5677              :                 AttachedTenantConf::try_from(LocationConf::attached_single(
    5678              :                     TenantConfOpt::from(self.tenant_conf.clone()),
    5679              :                     self.generation,
    5680              :                     &ShardParameters::default(),
    5681              :                 ))
    5682              :                 .unwrap(),
    5683              :                 // This is a legacy/test code path: sharding isn't supported here.
    5684              :                 ShardIdentity::unsharded(),
    5685              :                 Some(walredo_mgr),
    5686              :                 self.tenant_shard_id,
    5687              :                 self.remote_storage.clone(),
    5688              :                 self.deletion_queue.new_client(),
    5689              :                 // TODO: ideally we should run all unit tests with both configs
    5690              :                 L0FlushGlobalState::new(L0FlushConfig::default()),
    5691              :             ));
    5692              : 
    5693              :             let preload = tenant
    5694              :                 .preload(&self.remote_storage, CancellationToken::new())
    5695              :                 .await?;
    5696              :             tenant.attach(Some(preload), ctx).await?;
    5697              : 
    5698              :             tenant.state.send_replace(TenantState::Active);
    5699              :             for timeline in tenant.timelines.lock().unwrap().values() {
    5700              :                 timeline.set_state(TimelineState::Active);
    5701              :             }
    5702              :             Ok(tenant)
    5703              :         }
    5704              : 
    5705            2 :         pub fn timeline_path(&self, timeline_id: &TimelineId) -> Utf8PathBuf {
    5706            2 :             self.conf.timeline_path(&self.tenant_shard_id, timeline_id)
    5707            2 :         }
    5708              :     }
    5709              : 
    5710              :     // Mock WAL redo manager that doesn't do much
    5711              :     pub(crate) struct TestRedoManager;
    5712              : 
    5713              :     impl TestRedoManager {
    5714              :         /// # Cancel-Safety
    5715              :         ///
    5716              :         /// This method is cancellation-safe.
    5717          520 :         pub async fn request_redo(
    5718          520 :             &self,
    5719          520 :             key: Key,
    5720          520 :             lsn: Lsn,
    5721          520 :             base_img: Option<(Lsn, Bytes)>,
    5722          520 :             records: Vec<(Lsn, NeonWalRecord)>,
    5723          520 :             _pg_version: u32,
    5724          520 :         ) -> Result<Bytes, walredo::Error> {
    5725          770 :             let records_neon = records.iter().all(|r| apply_neon::can_apply_in_neon(&r.1));
    5726          520 :             if records_neon {
    5727              :                 // For Neon wal records, we can decode without spawning postgres, so do so.
    5728          520 :                 let mut page = match (base_img, records.first()) {
    5729          454 :                     (Some((_lsn, img)), _) => {
    5730          454 :                         let mut page = BytesMut::new();
    5731          454 :                         page.extend_from_slice(&img);
    5732          454 :                         page
    5733              :                     }
    5734           66 :                     (_, Some((_lsn, rec))) if rec.will_init() => BytesMut::new(),
    5735              :                     _ => {
    5736            0 :                         panic!("Neon WAL redo requires base image or will init record");
    5737              :                     }
    5738              :                 };
    5739              : 
    5740         1290 :                 for (record_lsn, record) in records {
    5741          770 :                     apply_neon::apply_in_neon(&record, record_lsn, key, &mut page)?;
    5742              :                 }
    5743          520 :                 Ok(page.freeze())
    5744              :             } else {
    5745              :                 // We never spawn a postgres walredo process in unit tests: just log what we might have done.
    5746            0 :                 let s = format!(
    5747            0 :                     "redo for {} to get to {}, with {} and {} records",
    5748            0 :                     key,
    5749            0 :                     lsn,
    5750            0 :                     if base_img.is_some() {
    5751            0 :                         "base image"
    5752              :                     } else {
    5753            0 :                         "no base image"
    5754              :                     },
    5755            0 :                     records.len()
    5756            0 :                 );
    5757            0 :                 println!("{s}");
    5758            0 : 
    5759            0 :                 Ok(test_img(&s))
    5760              :             }
    5761          520 :         }
    5762              :     }
    5763              : }
    5764              : 
    5765              : #[cfg(test)]
    5766              : mod tests {
    5767              :     use std::collections::{BTreeMap, BTreeSet};
    5768              : 
    5769              :     use super::*;
    5770              :     use crate::keyspace::KeySpaceAccum;
    5771              :     use crate::tenant::harness::*;
    5772              :     use crate::tenant::timeline::CompactFlags;
    5773              :     use crate::DEFAULT_PG_VERSION;
    5774              :     use bytes::{Bytes, BytesMut};
    5775              :     use hex_literal::hex;
    5776              :     use itertools::Itertools;
    5777              :     use pageserver_api::key::{Key, AUX_KEY_PREFIX, NON_INHERITED_RANGE};
    5778              :     use pageserver_api::keyspace::KeySpace;
    5779              :     use pageserver_api::models::{CompactionAlgorithm, CompactionAlgorithmSettings};
    5780              :     use pageserver_api::value::Value;
    5781              :     use pageserver_compaction::helpers::overlaps_with;
    5782              :     use rand::{thread_rng, Rng};
    5783              :     use storage_layer::PersistentLayerKey;
    5784              :     use tests::storage_layer::ValuesReconstructState;
    5785              :     use tests::timeline::{GetVectoredError, ShutdownMode};
    5786              :     use timeline::{CompactOptions, DeltaLayerTestDesc};
    5787              :     use utils::id::TenantId;
    5788              : 
    5789              :     #[cfg(feature = "testing")]
    5790              :     use models::CompactLsnRange;
    5791              :     #[cfg(feature = "testing")]
    5792              :     use pageserver_api::record::NeonWalRecord;
    5793              :     #[cfg(feature = "testing")]
    5794              :     use timeline::compaction::{KeyHistoryRetention, KeyLogAtLsn};
    5795              :     #[cfg(feature = "testing")]
    5796              :     use timeline::GcInfo;
    5797              : 
    5798              :     static TEST_KEY: Lazy<Key> =
    5799           18 :         Lazy::new(|| Key::from_slice(&hex!("010000000033333333444444445500000001")));
    5800              : 
    5801              :     #[tokio::test]
    5802            2 :     async fn test_basic() -> anyhow::Result<()> {
    5803            2 :         let (tenant, ctx) = TenantHarness::create("test_basic").await?.load().await;
    5804            2 :         let tline = tenant
    5805            2 :             .create_test_timeline(TIMELINE_ID, Lsn(0x08), DEFAULT_PG_VERSION, &ctx)
    5806            2 :             .await?;
    5807            2 : 
    5808            2 :         let mut writer = tline.writer().await;
    5809            2 :         writer
    5810            2 :             .put(
    5811            2 :                 *TEST_KEY,
    5812            2 :                 Lsn(0x10),
    5813            2 :                 &Value::Image(test_img("foo at 0x10")),
    5814            2 :                 &ctx,
    5815            2 :             )
    5816            2 :             .await?;
    5817            2 :         writer.finish_write(Lsn(0x10));
    5818            2 :         drop(writer);
    5819            2 : 
    5820            2 :         let mut writer = tline.writer().await;
    5821            2 :         writer
    5822            2 :             .put(
    5823            2 :                 *TEST_KEY,
    5824            2 :                 Lsn(0x20),
    5825            2 :                 &Value::Image(test_img("foo at 0x20")),
    5826            2 :                 &ctx,
    5827            2 :             )
    5828            2 :             .await?;
    5829            2 :         writer.finish_write(Lsn(0x20));
    5830            2 :         drop(writer);
    5831            2 : 
    5832            2 :         assert_eq!(
    5833            2 :             tline.get(*TEST_KEY, Lsn(0x10), &ctx).await?,
    5834            2 :             test_img("foo at 0x10")
    5835            2 :         );
    5836            2 :         assert_eq!(
    5837            2 :             tline.get(*TEST_KEY, Lsn(0x1f), &ctx).await?,
    5838            2 :             test_img("foo at 0x10")
    5839            2 :         );
    5840            2 :         assert_eq!(
    5841            2 :             tline.get(*TEST_KEY, Lsn(0x20), &ctx).await?,
    5842            2 :             test_img("foo at 0x20")
    5843            2 :         );
    5844            2 : 
    5845            2 :         Ok(())
    5846            2 :     }
    5847              : 
    5848              :     #[tokio::test]
    5849            2 :     async fn no_duplicate_timelines() -> anyhow::Result<()> {
    5850            2 :         let (tenant, ctx) = TenantHarness::create("no_duplicate_timelines")
    5851            2 :             .await?
    5852            2 :             .load()
    5853            2 :             .await;
    5854            2 :         let _ = tenant
    5855            2 :             .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx)
    5856            2 :             .await?;
    5857            2 : 
    5858            2 :         match tenant
    5859            2 :             .create_empty_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx)
    5860            2 :             .await
    5861            2 :         {
    5862            2 :             Ok(_) => panic!("duplicate timeline creation should fail"),
    5863            2 :             Err(e) => assert_eq!(
    5864            2 :                 e.to_string(),
    5865            2 :                 "timeline already exists with different parameters".to_string()
    5866            2 :             ),
    5867            2 :         }
    5868            2 : 
    5869            2 :         Ok(())
    5870            2 :     }
    5871              : 
    5872              :     /// Convenience function to create a page image with given string as the only content
    5873           10 :     pub fn test_value(s: &str) -> Value {
    5874           10 :         let mut buf = BytesMut::new();
    5875           10 :         buf.extend_from_slice(s.as_bytes());
    5876           10 :         Value::Image(buf.freeze())
    5877           10 :     }
    5878              : 
    5879              :     ///
    5880              :     /// Test branch creation
    5881              :     ///
    5882              :     #[tokio::test]
    5883            2 :     async fn test_branch() -> anyhow::Result<()> {
    5884            2 :         use std::str::from_utf8;
    5885            2 : 
    5886            2 :         let (tenant, ctx) = TenantHarness::create("test_branch").await?.load().await;
    5887            2 :         let tline = tenant
    5888            2 :             .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx)
    5889            2 :             .await?;
    5890            2 :         let mut writer = tline.writer().await;
    5891            2 : 
    5892            2 :         #[allow(non_snake_case)]
    5893            2 :         let TEST_KEY_A: Key = Key::from_hex("110000000033333333444444445500000001").unwrap();
    5894            2 :         #[allow(non_snake_case)]
    5895            2 :         let TEST_KEY_B: Key = Key::from_hex("110000000033333333444444445500000002").unwrap();
    5896            2 : 
    5897            2 :         // Insert a value on the timeline
    5898            2 :         writer
    5899            2 :             .put(TEST_KEY_A, Lsn(0x20), &test_value("foo at 0x20"), &ctx)
    5900            2 :             .await?;
    5901            2 :         writer
    5902            2 :             .put(TEST_KEY_B, Lsn(0x20), &test_value("foobar at 0x20"), &ctx)
    5903            2 :             .await?;
    5904            2 :         writer.finish_write(Lsn(0x20));
    5905            2 : 
    5906            2 :         writer
    5907            2 :             .put(TEST_KEY_A, Lsn(0x30), &test_value("foo at 0x30"), &ctx)
    5908            2 :             .await?;
    5909            2 :         writer.finish_write(Lsn(0x30));
    5910            2 :         writer
    5911            2 :             .put(TEST_KEY_A, Lsn(0x40), &test_value("foo at 0x40"), &ctx)
    5912            2 :             .await?;
    5913            2 :         writer.finish_write(Lsn(0x40));
    5914            2 : 
    5915            2 :         //assert_current_logical_size(&tline, Lsn(0x40));
    5916            2 : 
    5917            2 :         // Branch the history, modify relation differently on the new timeline
    5918            2 :         tenant
    5919            2 :             .branch_timeline_test(&tline, NEW_TIMELINE_ID, Some(Lsn(0x30)), &ctx)
    5920            2 :             .await?;
    5921            2 :         let newtline = tenant
    5922            2 :             .get_timeline(NEW_TIMELINE_ID, true)
    5923            2 :             .expect("Should have a local timeline");
    5924            2 :         let mut new_writer = newtline.writer().await;
    5925            2 :         new_writer
    5926            2 :             .put(TEST_KEY_A, Lsn(0x40), &test_value("bar at 0x40"), &ctx)
    5927            2 :             .await?;
    5928            2 :         new_writer.finish_write(Lsn(0x40));
    5929            2 : 
    5930            2 :         // Check page contents on both branches
    5931            2 :         assert_eq!(
    5932            2 :             from_utf8(&tline.get(TEST_KEY_A, Lsn(0x40), &ctx).await?)?,
    5933            2 :             "foo at 0x40"
    5934            2 :         );
    5935            2 :         assert_eq!(
    5936            2 :             from_utf8(&newtline.get(TEST_KEY_A, Lsn(0x40), &ctx).await?)?,
    5937            2 :             "bar at 0x40"
    5938            2 :         );
    5939            2 :         assert_eq!(
    5940            2 :             from_utf8(&newtline.get(TEST_KEY_B, Lsn(0x40), &ctx).await?)?,
    5941            2 :             "foobar at 0x20"
    5942            2 :         );
    5943            2 : 
    5944            2 :         //assert_current_logical_size(&tline, Lsn(0x40));
    5945            2 : 
    5946            2 :         Ok(())
    5947            2 :     }
    5948              : 
    5949           20 :     async fn make_some_layers(
    5950           20 :         tline: &Timeline,
    5951           20 :         start_lsn: Lsn,
    5952           20 :         ctx: &RequestContext,
    5953           20 :     ) -> anyhow::Result<()> {
    5954           20 :         let mut lsn = start_lsn;
    5955              :         {
    5956           20 :             let mut writer = tline.writer().await;
    5957              :             // Create a relation on the timeline
    5958           20 :             writer
    5959           20 :                 .put(
    5960           20 :                     *TEST_KEY,
    5961           20 :                     lsn,
    5962           20 :                     &Value::Image(test_img(&format!("foo at {}", lsn))),
    5963           20 :                     ctx,
    5964           20 :                 )
    5965           20 :                 .await?;
    5966           20 :             writer.finish_write(lsn);
    5967           20 :             lsn += 0x10;
    5968           20 :             writer
    5969           20 :                 .put(
    5970           20 :                     *TEST_KEY,
    5971           20 :                     lsn,
    5972           20 :                     &Value::Image(test_img(&format!("foo at {}", lsn))),
    5973           20 :                     ctx,
    5974           20 :                 )
    5975           20 :                 .await?;
    5976           20 :             writer.finish_write(lsn);
    5977           20 :             lsn += 0x10;
    5978           20 :         }
    5979           20 :         tline.freeze_and_flush().await?;
    5980              :         {
    5981           20 :             let mut writer = tline.writer().await;
    5982           20 :             writer
    5983           20 :                 .put(
    5984           20 :                     *TEST_KEY,
    5985           20 :                     lsn,
    5986           20 :                     &Value::Image(test_img(&format!("foo at {}", lsn))),
    5987           20 :                     ctx,
    5988           20 :                 )
    5989           20 :                 .await?;
    5990           20 :             writer.finish_write(lsn);
    5991           20 :             lsn += 0x10;
    5992           20 :             writer
    5993           20 :                 .put(
    5994           20 :                     *TEST_KEY,
    5995           20 :                     lsn,
    5996           20 :                     &Value::Image(test_img(&format!("foo at {}", lsn))),
    5997           20 :                     ctx,
    5998           20 :                 )
    5999           20 :                 .await?;
    6000           20 :             writer.finish_write(lsn);
    6001           20 :         }
    6002           20 :         tline.freeze_and_flush().await.map_err(|e| e.into())
    6003           20 :     }
    6004              : 
    6005              :     #[tokio::test(start_paused = true)]
    6006            2 :     async fn test_prohibit_branch_creation_on_garbage_collected_data() -> anyhow::Result<()> {
    6007            2 :         let (tenant, ctx) =
    6008            2 :             TenantHarness::create("test_prohibit_branch_creation_on_garbage_collected_data")
    6009            2 :                 .await?
    6010            2 :                 .load()
    6011            2 :                 .await;
    6012            2 :         // Advance to the lsn lease deadline so that GC is not blocked by
    6013            2 :         // initial transition into AttachedSingle.
    6014            2 :         tokio::time::advance(tenant.get_lsn_lease_length()).await;
    6015            2 :         tokio::time::resume();
    6016            2 :         let tline = tenant
    6017            2 :             .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx)
    6018            2 :             .await?;
    6019            2 :         make_some_layers(tline.as_ref(), Lsn(0x20), &ctx).await?;
    6020            2 : 
    6021            2 :         // this removes layers before lsn 40 (50 minus 10), so there are two remaining layers, image and delta for 31-50
    6022            2 :         // FIXME: this doesn't actually remove any layer currently, given how the flushing
    6023            2 :         // and compaction works. But it does set the 'cutoff' point so that the cross check
    6024            2 :         // below should fail.
    6025            2 :         tenant
    6026            2 :             .gc_iteration(
    6027            2 :                 Some(TIMELINE_ID),
    6028            2 :                 0x10,
    6029            2 :                 Duration::ZERO,
    6030            2 :                 &CancellationToken::new(),
    6031            2 :                 &ctx,
    6032            2 :             )
    6033            2 :             .await?;
    6034            2 : 
    6035            2 :         // try to branch at lsn 25, should fail because we already garbage collected the data
    6036            2 :         match tenant
    6037            2 :             .branch_timeline_test(&tline, NEW_TIMELINE_ID, Some(Lsn(0x25)), &ctx)
    6038            2 :             .await
    6039            2 :         {
    6040            2 :             Ok(_) => panic!("branching should have failed"),
    6041            2 :             Err(err) => {
    6042            2 :                 let CreateTimelineError::AncestorLsn(err) = err else {
    6043            2 :                     panic!("wrong error type")
    6044            2 :                 };
    6045            2 :                 assert!(err.to_string().contains("invalid branch start lsn"));
    6046            2 :                 assert!(err
    6047            2 :                     .source()
    6048            2 :                     .unwrap()
    6049            2 :                     .to_string()
    6050            2 :                     .contains("we might've already garbage collected needed data"))
    6051            2 :             }
    6052            2 :         }
    6053            2 : 
    6054            2 :         Ok(())
    6055            2 :     }
    6056              : 
    6057              :     #[tokio::test]
    6058            2 :     async fn test_prohibit_branch_creation_on_pre_initdb_lsn() -> anyhow::Result<()> {
    6059            2 :         let (tenant, ctx) =
    6060            2 :             TenantHarness::create("test_prohibit_branch_creation_on_pre_initdb_lsn")
    6061            2 :                 .await?
    6062            2 :                 .load()
    6063            2 :                 .await;
    6064            2 : 
    6065            2 :         let tline = tenant
    6066            2 :             .create_test_timeline(TIMELINE_ID, Lsn(0x50), DEFAULT_PG_VERSION, &ctx)
    6067            2 :             .await?;
    6068            2 :         // try to branch at lsn 0x25, should fail because initdb lsn is 0x50
    6069            2 :         match tenant
    6070            2 :             .branch_timeline_test(&tline, NEW_TIMELINE_ID, Some(Lsn(0x25)), &ctx)
    6071            2 :             .await
    6072            2 :         {
    6073            2 :             Ok(_) => panic!("branching should have failed"),
    6074            2 :             Err(err) => {
    6075            2 :                 let CreateTimelineError::AncestorLsn(err) = err else {
    6076            2 :                     panic!("wrong error type");
    6077            2 :                 };
    6078            2 :                 assert!(&err.to_string().contains("invalid branch start lsn"));
    6079            2 :                 assert!(&err
    6080            2 :                     .source()
    6081            2 :                     .unwrap()
    6082            2 :                     .to_string()
    6083            2 :                     .contains("is earlier than latest GC cutoff"));
    6084            2 :             }
    6085            2 :         }
    6086            2 : 
    6087            2 :         Ok(())
    6088            2 :     }
    6089              : 
    6090              :     /*
    6091              :     // FIXME: This currently fails to error out. Calling GC doesn't currently
    6092              :     // remove the old value, we'd need to work a little harder
    6093              :     #[tokio::test]
    6094              :     async fn test_prohibit_get_for_garbage_collected_data() -> anyhow::Result<()> {
    6095              :         let repo =
    6096              :             RepoHarness::create("test_prohibit_get_for_garbage_collected_data")?
    6097              :             .load();
    6098              : 
    6099              :         let tline = repo.create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)?;
    6100              :         make_some_layers(tline.as_ref(), Lsn(0x20), &ctx).await?;
    6101              : 
    6102              :         repo.gc_iteration(Some(TIMELINE_ID), 0x10, Duration::ZERO)?;
    6103              :         let latest_gc_cutoff_lsn = tline.get_latest_gc_cutoff_lsn();
    6104              :         assert!(*latest_gc_cutoff_lsn > Lsn(0x25));
    6105              :         match tline.get(*TEST_KEY, Lsn(0x25)) {
    6106              :             Ok(_) => panic!("request for page should have failed"),
    6107              :             Err(err) => assert!(err.to_string().contains("not found at")),
    6108              :         }
    6109              :         Ok(())
    6110              :     }
    6111              :      */
    6112              : 
    6113              :     #[tokio::test]
    6114            2 :     async fn test_get_branchpoints_from_an_inactive_timeline() -> anyhow::Result<()> {
    6115            2 :         let (tenant, ctx) =
    6116            2 :             TenantHarness::create("test_get_branchpoints_from_an_inactive_timeline")
    6117            2 :                 .await?
    6118            2 :                 .load()
    6119            2 :                 .await;
    6120            2 :         let tline = tenant
    6121            2 :             .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx)
    6122            2 :             .await?;
    6123            2 :         make_some_layers(tline.as_ref(), Lsn(0x20), &ctx).await?;
    6124            2 : 
    6125            2 :         tenant
    6126            2 :             .branch_timeline_test(&tline, NEW_TIMELINE_ID, Some(Lsn(0x40)), &ctx)
    6127            2 :             .await?;
    6128            2 :         let newtline = tenant
    6129            2 :             .get_timeline(NEW_TIMELINE_ID, true)
    6130            2 :             .expect("Should have a local timeline");
    6131            2 : 
    6132            2 :         make_some_layers(newtline.as_ref(), Lsn(0x60), &ctx).await?;
    6133            2 : 
    6134            2 :         tline.set_broken("test".to_owned());
    6135            2 : 
    6136            2 :         tenant
    6137            2 :             .gc_iteration(
    6138            2 :                 Some(TIMELINE_ID),
    6139            2 :                 0x10,
    6140            2 :                 Duration::ZERO,
    6141            2 :                 &CancellationToken::new(),
    6142            2 :                 &ctx,
    6143            2 :             )
    6144            2 :             .await?;
    6145            2 : 
    6146            2 :         // The branchpoints should contain all timelines, even ones marked
    6147            2 :         // as Broken.
    6148            2 :         {
    6149            2 :             let branchpoints = &tline.gc_info.read().unwrap().retain_lsns;
    6150            2 :             assert_eq!(branchpoints.len(), 1);
    6151            2 :             assert_eq!(
    6152            2 :                 branchpoints[0],
    6153            2 :                 (Lsn(0x40), NEW_TIMELINE_ID, MaybeOffloaded::No)
    6154            2 :             );
    6155            2 :         }
    6156            2 : 
    6157            2 :         // You can read the key from the child branch even though the parent is
    6158            2 :         // Broken, as long as you don't need to access data from the parent.
    6159            2 :         assert_eq!(
    6160            2 :             newtline.get(*TEST_KEY, Lsn(0x70), &ctx).await?,
    6161            2 :             test_img(&format!("foo at {}", Lsn(0x70)))
    6162            2 :         );
    6163            2 : 
    6164            2 :         // This needs to traverse to the parent, and fails.
    6165            2 :         let err = newtline.get(*TEST_KEY, Lsn(0x50), &ctx).await.unwrap_err();
    6166            2 :         assert!(
    6167            2 :             err.to_string().starts_with(&format!(
    6168            2 :                 "bad state on timeline {}: Broken",
    6169            2 :                 tline.timeline_id
    6170            2 :             )),
    6171            2 :             "{err}"
    6172            2 :         );
    6173            2 : 
    6174            2 :         Ok(())
    6175            2 :     }
    6176              : 
    6177              :     #[tokio::test]
    6178            2 :     async fn test_retain_data_in_parent_which_is_needed_for_child() -> anyhow::Result<()> {
    6179            2 :         let (tenant, ctx) =
    6180            2 :             TenantHarness::create("test_retain_data_in_parent_which_is_needed_for_child")
    6181            2 :                 .await?
    6182            2 :                 .load()
    6183            2 :                 .await;
    6184            2 :         let tline = tenant
    6185            2 :             .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx)
    6186            2 :             .await?;
    6187            2 :         make_some_layers(tline.as_ref(), Lsn(0x20), &ctx).await?;
    6188            2 : 
    6189            2 :         tenant
    6190            2 :             .branch_timeline_test(&tline, NEW_TIMELINE_ID, Some(Lsn(0x40)), &ctx)
    6191            2 :             .await?;
    6192            2 :         let newtline = tenant
    6193            2 :             .get_timeline(NEW_TIMELINE_ID, true)
    6194            2 :             .expect("Should have a local timeline");
    6195            2 :         // this removes layers before lsn 40 (50 minus 10), so there are two remaining layers, image and delta for 31-50
    6196            2 :         tenant
    6197            2 :             .gc_iteration(
    6198            2 :                 Some(TIMELINE_ID),
    6199            2 :                 0x10,
    6200            2 :                 Duration::ZERO,
    6201            2 :                 &CancellationToken::new(),
    6202            2 :                 &ctx,
    6203            2 :             )
    6204            2 :             .await?;
    6205            2 :         assert!(newtline.get(*TEST_KEY, Lsn(0x25), &ctx).await.is_ok());
    6206            2 : 
    6207            2 :         Ok(())
    6208            2 :     }
    6209              :     #[tokio::test]
    6210            2 :     async fn test_parent_keeps_data_forever_after_branching() -> anyhow::Result<()> {
    6211            2 :         let (tenant, ctx) = TenantHarness::create("test_parent_keeps_data_forever_after_branching")
    6212            2 :             .await?
    6213            2 :             .load()
    6214            2 :             .await;
    6215            2 :         let tline = tenant
    6216            2 :             .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx)
    6217            2 :             .await?;
    6218            2 :         make_some_layers(tline.as_ref(), Lsn(0x20), &ctx).await?;
    6219            2 : 
    6220            2 :         tenant
    6221            2 :             .branch_timeline_test(&tline, NEW_TIMELINE_ID, Some(Lsn(0x40)), &ctx)
    6222            2 :             .await?;
    6223            2 :         let newtline = tenant
    6224            2 :             .get_timeline(NEW_TIMELINE_ID, true)
    6225            2 :             .expect("Should have a local timeline");
    6226            2 : 
    6227            2 :         make_some_layers(newtline.as_ref(), Lsn(0x60), &ctx).await?;
    6228            2 : 
    6229            2 :         // run gc on parent
    6230            2 :         tenant
    6231            2 :             .gc_iteration(
    6232            2 :                 Some(TIMELINE_ID),
    6233            2 :                 0x10,
    6234            2 :                 Duration::ZERO,
    6235            2 :                 &CancellationToken::new(),
    6236            2 :                 &ctx,
    6237            2 :             )
    6238            2 :             .await?;
    6239            2 : 
    6240            2 :         // Check that the data is still accessible on the branch.
    6241            2 :         assert_eq!(
    6242            2 :             newtline.get(*TEST_KEY, Lsn(0x50), &ctx).await?,
    6243            2 :             test_img(&format!("foo at {}", Lsn(0x40)))
    6244            2 :         );
    6245            2 : 
    6246            2 :         Ok(())
    6247            2 :     }
    6248              : 
    6249              :     #[tokio::test]
    6250            2 :     async fn timeline_load() -> anyhow::Result<()> {
    6251            2 :         const TEST_NAME: &str = "timeline_load";
    6252            2 :         let harness = TenantHarness::create(TEST_NAME).await?;
    6253            2 :         {
    6254            2 :             let (tenant, ctx) = harness.load().await;
    6255            2 :             let tline = tenant
    6256            2 :                 .create_test_timeline(TIMELINE_ID, Lsn(0x7000), DEFAULT_PG_VERSION, &ctx)
    6257            2 :                 .await?;
    6258            2 :             make_some_layers(tline.as_ref(), Lsn(0x8000), &ctx).await?;
    6259            2 :             // so that all uploads finish & we can call harness.load() below again
    6260            2 :             tenant
    6261            2 :                 .shutdown(Default::default(), ShutdownMode::FreezeAndFlush)
    6262            2 :                 .instrument(harness.span())
    6263            2 :                 .await
    6264            2 :                 .ok()
    6265            2 :                 .unwrap();
    6266            2 :         }
    6267            2 : 
    6268            2 :         let (tenant, _ctx) = harness.load().await;
    6269            2 :         tenant
    6270            2 :             .get_timeline(TIMELINE_ID, true)
    6271            2 :             .expect("cannot load timeline");
    6272            2 : 
    6273            2 :         Ok(())
    6274            2 :     }
    6275              : 
    6276              :     #[tokio::test]
    6277            2 :     async fn timeline_load_with_ancestor() -> anyhow::Result<()> {
    6278            2 :         const TEST_NAME: &str = "timeline_load_with_ancestor";
    6279            2 :         let harness = TenantHarness::create(TEST_NAME).await?;
    6280            2 :         // create two timelines
    6281            2 :         {
    6282            2 :             let (tenant, ctx) = harness.load().await;
    6283            2 :             let tline = tenant
    6284            2 :                 .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx)
    6285            2 :                 .await?;
    6286            2 : 
    6287            2 :             make_some_layers(tline.as_ref(), Lsn(0x20), &ctx).await?;
    6288            2 : 
    6289            2 :             let child_tline = tenant
    6290            2 :                 .branch_timeline_test(&tline, NEW_TIMELINE_ID, Some(Lsn(0x40)), &ctx)
    6291            2 :                 .await?;
    6292            2 :             child_tline.set_state(TimelineState::Active);
    6293            2 : 
    6294            2 :             let newtline = tenant
    6295            2 :                 .get_timeline(NEW_TIMELINE_ID, true)
    6296            2 :                 .expect("Should have a local timeline");
    6297            2 : 
    6298            2 :             make_some_layers(newtline.as_ref(), Lsn(0x60), &ctx).await?;
    6299            2 : 
    6300            2 :             // so that all uploads finish & we can call harness.load() below again
    6301            2 :             tenant
    6302            2 :                 .shutdown(Default::default(), ShutdownMode::FreezeAndFlush)
    6303            2 :                 .instrument(harness.span())
    6304            2 :                 .await
    6305            2 :                 .ok()
    6306            2 :                 .unwrap();
    6307            2 :         }
    6308            2 : 
    6309            2 :         // check that both of them are initially unloaded
    6310            2 :         let (tenant, _ctx) = harness.load().await;
    6311            2 : 
    6312            2 :         // check that both, child and ancestor are loaded
    6313            2 :         let _child_tline = tenant
    6314            2 :             .get_timeline(NEW_TIMELINE_ID, true)
    6315            2 :             .expect("cannot get child timeline loaded");
    6316            2 : 
    6317            2 :         let _ancestor_tline = tenant
    6318            2 :             .get_timeline(TIMELINE_ID, true)
    6319            2 :             .expect("cannot get ancestor timeline loaded");
    6320            2 : 
    6321            2 :         Ok(())
    6322            2 :     }
    6323              : 
    6324              :     #[tokio::test]
    6325            2 :     async fn delta_layer_dumping() -> anyhow::Result<()> {
    6326            2 :         use storage_layer::AsLayerDesc;
    6327            2 :         let (tenant, ctx) = TenantHarness::create("test_layer_dumping")
    6328            2 :             .await?
    6329            2 :             .load()
    6330            2 :             .await;
    6331            2 :         let tline = tenant
    6332            2 :             .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx)
    6333            2 :             .await?;
    6334            2 :         make_some_layers(tline.as_ref(), Lsn(0x20), &ctx).await?;
    6335            2 : 
    6336            2 :         let layer_map = tline.layers.read().await;
    6337            2 :         let level0_deltas = layer_map
    6338            2 :             .layer_map()?
    6339            2 :             .level0_deltas()
    6340            2 :             .iter()
    6341            4 :             .map(|desc| layer_map.get_from_desc(desc))
    6342            2 :             .collect::<Vec<_>>();
    6343            2 : 
    6344            2 :         assert!(!level0_deltas.is_empty());
    6345            2 : 
    6346            6 :         for delta in level0_deltas {
    6347            2 :             // Ensure we are dumping a delta layer here
    6348            4 :             assert!(delta.layer_desc().is_delta);
    6349            4 :             delta.dump(true, &ctx).await.unwrap();
    6350            2 :         }
    6351            2 : 
    6352            2 :         Ok(())
    6353            2 :     }
    6354              : 
    6355              :     #[tokio::test]
    6356            2 :     async fn test_images() -> anyhow::Result<()> {
    6357            2 :         let (tenant, ctx) = TenantHarness::create("test_images").await?.load().await;
    6358            2 :         let tline = tenant
    6359            2 :             .create_test_timeline(TIMELINE_ID, Lsn(0x08), DEFAULT_PG_VERSION, &ctx)
    6360            2 :             .await?;
    6361            2 : 
    6362            2 :         let mut writer = tline.writer().await;
    6363            2 :         writer
    6364            2 :             .put(
    6365            2 :                 *TEST_KEY,
    6366            2 :                 Lsn(0x10),
    6367            2 :                 &Value::Image(test_img("foo at 0x10")),
    6368            2 :                 &ctx,
    6369            2 :             )
    6370            2 :             .await?;
    6371            2 :         writer.finish_write(Lsn(0x10));
    6372            2 :         drop(writer);
    6373            2 : 
    6374            2 :         tline.freeze_and_flush().await?;
    6375            2 :         tline
    6376            2 :             .compact(&CancellationToken::new(), EnumSet::empty(), &ctx)
    6377            2 :             .await?;
    6378            2 : 
    6379            2 :         let mut writer = tline.writer().await;
    6380            2 :         writer
    6381            2 :             .put(
    6382            2 :                 *TEST_KEY,
    6383            2 :                 Lsn(0x20),
    6384            2 :                 &Value::Image(test_img("foo at 0x20")),
    6385            2 :                 &ctx,
    6386            2 :             )
    6387            2 :             .await?;
    6388            2 :         writer.finish_write(Lsn(0x20));
    6389            2 :         drop(writer);
    6390            2 : 
    6391            2 :         tline.freeze_and_flush().await?;
    6392            2 :         tline
    6393            2 :             .compact(&CancellationToken::new(), EnumSet::empty(), &ctx)
    6394            2 :             .await?;
    6395            2 : 
    6396            2 :         let mut writer = tline.writer().await;
    6397            2 :         writer
    6398            2 :             .put(
    6399            2 :                 *TEST_KEY,
    6400            2 :                 Lsn(0x30),
    6401            2 :                 &Value::Image(test_img("foo at 0x30")),
    6402            2 :                 &ctx,
    6403            2 :             )
    6404            2 :             .await?;
    6405            2 :         writer.finish_write(Lsn(0x30));
    6406            2 :         drop(writer);
    6407            2 : 
    6408            2 :         tline.freeze_and_flush().await?;
    6409            2 :         tline
    6410            2 :             .compact(&CancellationToken::new(), EnumSet::empty(), &ctx)
    6411            2 :             .await?;
    6412            2 : 
    6413            2 :         let mut writer = tline.writer().await;
    6414            2 :         writer
    6415            2 :             .put(
    6416            2 :                 *TEST_KEY,
    6417            2 :                 Lsn(0x40),
    6418            2 :                 &Value::Image(test_img("foo at 0x40")),
    6419            2 :                 &ctx,
    6420            2 :             )
    6421            2 :             .await?;
    6422            2 :         writer.finish_write(Lsn(0x40));
    6423            2 :         drop(writer);
    6424            2 : 
    6425            2 :         tline.freeze_and_flush().await?;
    6426            2 :         tline
    6427            2 :             .compact(&CancellationToken::new(), EnumSet::empty(), &ctx)
    6428            2 :             .await?;
    6429            2 : 
    6430            2 :         assert_eq!(
    6431            2 :             tline.get(*TEST_KEY, Lsn(0x10), &ctx).await?,
    6432            2 :             test_img("foo at 0x10")
    6433            2 :         );
    6434            2 :         assert_eq!(
    6435            2 :             tline.get(*TEST_KEY, Lsn(0x1f), &ctx).await?,
    6436            2 :             test_img("foo at 0x10")
    6437            2 :         );
    6438            2 :         assert_eq!(
    6439            2 :             tline.get(*TEST_KEY, Lsn(0x20), &ctx).await?,
    6440            2 :             test_img("foo at 0x20")
    6441            2 :         );
    6442            2 :         assert_eq!(
    6443            2 :             tline.get(*TEST_KEY, Lsn(0x30), &ctx).await?,
    6444            2 :             test_img("foo at 0x30")
    6445            2 :         );
    6446            2 :         assert_eq!(
    6447            2 :             tline.get(*TEST_KEY, Lsn(0x40), &ctx).await?,
    6448            2 :             test_img("foo at 0x40")
    6449            2 :         );
    6450            2 : 
    6451            2 :         Ok(())
    6452            2 :     }
    6453              : 
    6454            4 :     async fn bulk_insert_compact_gc(
    6455            4 :         tenant: &Tenant,
    6456            4 :         timeline: &Arc<Timeline>,
    6457            4 :         ctx: &RequestContext,
    6458            4 :         lsn: Lsn,
    6459            4 :         repeat: usize,
    6460            4 :         key_count: usize,
    6461            4 :     ) -> anyhow::Result<HashMap<Key, BTreeSet<Lsn>>> {
    6462            4 :         let compact = true;
    6463            4 :         bulk_insert_maybe_compact_gc(tenant, timeline, ctx, lsn, repeat, key_count, compact).await
    6464            4 :     }
    6465              : 
    6466            8 :     async fn bulk_insert_maybe_compact_gc(
    6467            8 :         tenant: &Tenant,
    6468            8 :         timeline: &Arc<Timeline>,
    6469            8 :         ctx: &RequestContext,
    6470            8 :         mut lsn: Lsn,
    6471            8 :         repeat: usize,
    6472            8 :         key_count: usize,
    6473            8 :         compact: bool,
    6474            8 :     ) -> anyhow::Result<HashMap<Key, BTreeSet<Lsn>>> {
    6475            8 :         let mut inserted: HashMap<Key, BTreeSet<Lsn>> = Default::default();
    6476            8 : 
    6477            8 :         let mut test_key = Key::from_hex("010000000033333333444444445500000000").unwrap();
    6478            8 :         let mut blknum = 0;
    6479            8 : 
    6480            8 :         // Enforce that key range is monotonously increasing
    6481            8 :         let mut keyspace = KeySpaceAccum::new();
    6482            8 : 
    6483            8 :         let cancel = CancellationToken::new();
    6484            8 : 
    6485            8 :         for _ in 0..repeat {
    6486          400 :             for _ in 0..key_count {
    6487      4000000 :                 test_key.field6 = blknum;
    6488      4000000 :                 let mut writer = timeline.writer().await;
    6489      4000000 :                 writer
    6490      4000000 :                     .put(
    6491      4000000 :                         test_key,
    6492      4000000 :                         lsn,
    6493      4000000 :                         &Value::Image(test_img(&format!("{} at {}", blknum, lsn))),
    6494      4000000 :                         ctx,
    6495      4000000 :                     )
    6496      4000000 :                     .await?;
    6497      4000000 :                 inserted.entry(test_key).or_default().insert(lsn);
    6498      4000000 :                 writer.finish_write(lsn);
    6499      4000000 :                 drop(writer);
    6500      4000000 : 
    6501      4000000 :                 keyspace.add_key(test_key);
    6502      4000000 : 
    6503      4000000 :                 lsn = Lsn(lsn.0 + 0x10);
    6504      4000000 :                 blknum += 1;
    6505              :             }
    6506              : 
    6507          400 :             timeline.freeze_and_flush().await?;
    6508          400 :             if compact {
    6509              :                 // this requires timeline to be &Arc<Timeline>
    6510          200 :                 timeline.compact(&cancel, EnumSet::empty(), ctx).await?;
    6511          200 :             }
    6512              : 
    6513              :             // this doesn't really need to use the timeline_id target, but it is closer to what it
    6514              :             // originally was.
    6515          400 :             let res = tenant
    6516          400 :                 .gc_iteration(Some(timeline.timeline_id), 0, Duration::ZERO, &cancel, ctx)
    6517          400 :                 .await?;
    6518              : 
    6519          400 :             assert_eq!(res.layers_removed, 0, "this never removes anything");
    6520              :         }
    6521              : 
    6522            8 :         Ok(inserted)
    6523            8 :     }
    6524              : 
    6525              :     //
    6526              :     // Insert 1000 key-value pairs with increasing keys, flush, compact, GC.
    6527              :     // Repeat 50 times.
    6528              :     //
    6529              :     #[tokio::test]
    6530            2 :     async fn test_bulk_insert() -> anyhow::Result<()> {
    6531            2 :         let harness = TenantHarness::create("test_bulk_insert").await?;
    6532            2 :         let (tenant, ctx) = harness.load().await;
    6533            2 :         let tline = tenant
    6534            2 :             .create_test_timeline(TIMELINE_ID, Lsn(0x08), DEFAULT_PG_VERSION, &ctx)
    6535            2 :             .await?;
    6536            2 : 
    6537            2 :         let lsn = Lsn(0x10);
    6538            2 :         bulk_insert_compact_gc(&tenant, &tline, &ctx, lsn, 50, 10000).await?;
    6539            2 : 
    6540            2 :         Ok(())
    6541            2 :     }
    6542              : 
    6543              :     // Test the vectored get real implementation against a simple sequential implementation.
    6544              :     //
    6545              :     // The test generates a keyspace by repeatedly flushing the in-memory layer and compacting.
    6546              :     // Projected to 2D the key space looks like below. Lsn grows upwards on the Y axis and keys
    6547              :     // grow to the right on the X axis.
    6548              :     //                       [Delta]
    6549              :     //                 [Delta]
    6550              :     //           [Delta]
    6551              :     //    [Delta]
    6552              :     // ------------ Image ---------------
    6553              :     //
    6554              :     // After layer generation we pick the ranges to query as follows:
    6555              :     // 1. The beginning of each delta layer
    6556              :     // 2. At the seam between two adjacent delta layers
    6557              :     //
    6558              :     // There's one major downside to this test: delta layers only contains images,
    6559              :     // so the search can stop at the first delta layer and doesn't traverse any deeper.
    6560              :     #[tokio::test]
    6561            2 :     async fn test_get_vectored() -> anyhow::Result<()> {
    6562            2 :         let harness = TenantHarness::create("test_get_vectored").await?;
    6563            2 :         let (tenant, ctx) = harness.load().await;
    6564            2 :         let tline = tenant
    6565            2 :             .create_test_timeline(TIMELINE_ID, Lsn(0x08), DEFAULT_PG_VERSION, &ctx)
    6566            2 :             .await?;
    6567            2 : 
    6568            2 :         let lsn = Lsn(0x10);
    6569            2 :         let inserted = bulk_insert_compact_gc(&tenant, &tline, &ctx, lsn, 50, 10000).await?;
    6570            2 : 
    6571            2 :         let guard = tline.layers.read().await;
    6572            2 :         let lm = guard.layer_map()?;
    6573            2 : 
    6574            2 :         lm.dump(true, &ctx).await?;
    6575            2 : 
    6576            2 :         let mut reads = Vec::new();
    6577            2 :         let mut prev = None;
    6578           12 :         lm.iter_historic_layers().for_each(|desc| {
    6579           12 :             if !desc.is_delta() {
    6580            2 :                 prev = Some(desc.clone());
    6581            2 :                 return;
    6582           10 :             }
    6583           10 : 
    6584           10 :             let start = desc.key_range.start;
    6585           10 :             let end = desc
    6586           10 :                 .key_range
    6587           10 :                 .start
    6588           10 :                 .add(Timeline::MAX_GET_VECTORED_KEYS.try_into().unwrap());
    6589           10 :             reads.push(KeySpace {
    6590           10 :                 ranges: vec![start..end],
    6591           10 :             });
    6592            2 : 
    6593           10 :             if let Some(prev) = &prev {
    6594           10 :                 if !prev.is_delta() {
    6595           10 :                     return;
    6596            2 :                 }
    6597            0 : 
    6598            0 :                 let first_range = Key {
    6599            0 :                     field6: prev.key_range.end.field6 - 4,
    6600            0 :                     ..prev.key_range.end
    6601            0 :                 }..prev.key_range.end;
    6602            0 : 
    6603            0 :                 let second_range = desc.key_range.start..Key {
    6604            0 :                     field6: desc.key_range.start.field6 + 4,
    6605            0 :                     ..desc.key_range.start
    6606            0 :                 };
    6607            0 : 
    6608            0 :                 reads.push(KeySpace {
    6609            0 :                     ranges: vec![first_range, second_range],
    6610            0 :                 });
    6611            2 :             };
    6612            2 : 
    6613            2 :             prev = Some(desc.clone());
    6614           12 :         });
    6615            2 : 
    6616            2 :         drop(guard);
    6617            2 : 
    6618            2 :         // Pick a big LSN such that we query over all the changes.
    6619            2 :         let reads_lsn = Lsn(u64::MAX - 1);
    6620            2 : 
    6621           12 :         for read in reads {
    6622           10 :             info!("Doing vectored read on {:?}", read);
    6623            2 : 
    6624           10 :             let vectored_res = tline
    6625           10 :                 .get_vectored_impl(
    6626           10 :                     read.clone(),
    6627           10 :                     reads_lsn,
    6628           10 :                     &mut ValuesReconstructState::new(),
    6629           10 :                     &ctx,
    6630           10 :                 )
    6631           10 :                 .await;
    6632            2 : 
    6633           10 :             let mut expected_lsns: HashMap<Key, Lsn> = Default::default();
    6634           10 :             let mut expect_missing = false;
    6635           10 :             let mut key = read.start().unwrap();
    6636          330 :             while key != read.end().unwrap() {
    6637          320 :                 if let Some(lsns) = inserted.get(&key) {
    6638          320 :                     let expected_lsn = lsns.iter().rfind(|lsn| **lsn <= reads_lsn);
    6639          320 :                     match expected_lsn {
    6640          320 :                         Some(lsn) => {
    6641          320 :                             expected_lsns.insert(key, *lsn);
    6642          320 :                         }
    6643            2 :                         None => {
    6644            2 :                             expect_missing = true;
    6645            0 :                             break;
    6646            2 :                         }
    6647            2 :                     }
    6648            2 :                 } else {
    6649            2 :                     expect_missing = true;
    6650            0 :                     break;
    6651            2 :                 }
    6652            2 : 
    6653          320 :                 key = key.next();
    6654            2 :             }
    6655            2 : 
    6656           10 :             if expect_missing {
    6657            2 :                 assert!(matches!(vectored_res, Err(GetVectoredError::MissingKey(_))));
    6658            2 :             } else {
    6659          320 :                 for (key, image) in vectored_res? {
    6660          320 :                     let expected_lsn = expected_lsns.get(&key).expect("determined above");
    6661          320 :                     let expected_image = test_img(&format!("{} at {}", key.field6, expected_lsn));
    6662          320 :                     assert_eq!(image?, expected_image);
    6663            2 :                 }
    6664            2 :             }
    6665            2 :         }
    6666            2 : 
    6667            2 :         Ok(())
    6668            2 :     }
    6669              : 
    6670              :     #[tokio::test]
    6671            2 :     async fn test_get_vectored_aux_files() -> anyhow::Result<()> {
    6672            2 :         let harness = TenantHarness::create("test_get_vectored_aux_files").await?;
    6673            2 : 
    6674            2 :         let (tenant, ctx) = harness.load().await;
    6675            2 :         let tline = tenant
    6676            2 :             .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)
    6677            2 :             .await?;
    6678            2 :         let tline = tline.raw_timeline().unwrap();
    6679            2 : 
    6680            2 :         let mut modification = tline.begin_modification(Lsn(0x1000));
    6681            2 :         modification.put_file("foo/bar1", b"content1", &ctx).await?;
    6682            2 :         modification.set_lsn(Lsn(0x1008))?;
    6683            2 :         modification.put_file("foo/bar2", b"content2", &ctx).await?;
    6684            2 :         modification.commit(&ctx).await?;
    6685            2 : 
    6686            2 :         let child_timeline_id = TimelineId::generate();
    6687            2 :         tenant
    6688            2 :             .branch_timeline_test(
    6689            2 :                 tline,
    6690            2 :                 child_timeline_id,
    6691            2 :                 Some(tline.get_last_record_lsn()),
    6692            2 :                 &ctx,
    6693            2 :             )
    6694            2 :             .await?;
    6695            2 : 
    6696            2 :         let child_timeline = tenant
    6697            2 :             .get_timeline(child_timeline_id, true)
    6698            2 :             .expect("Should have the branched timeline");
    6699            2 : 
    6700            2 :         let aux_keyspace = KeySpace {
    6701            2 :             ranges: vec![NON_INHERITED_RANGE],
    6702            2 :         };
    6703            2 :         let read_lsn = child_timeline.get_last_record_lsn();
    6704            2 : 
    6705            2 :         let vectored_res = child_timeline
    6706            2 :             .get_vectored_impl(
    6707            2 :                 aux_keyspace.clone(),
    6708            2 :                 read_lsn,
    6709            2 :                 &mut ValuesReconstructState::new(),
    6710            2 :                 &ctx,
    6711            2 :             )
    6712            2 :             .await;
    6713            2 : 
    6714            2 :         let images = vectored_res?;
    6715            2 :         assert!(images.is_empty());
    6716            2 :         Ok(())
    6717            2 :     }
    6718              : 
    6719              :     // Test that vectored get handles layer gaps correctly
    6720              :     // by advancing into the next ancestor timeline if required.
    6721              :     //
    6722              :     // The test generates timelines that look like the diagram below.
    6723              :     // We leave a gap in one of the L1 layers at `gap_at_key` (`/` in the diagram).
    6724              :     // The reconstruct data for that key lies in the ancestor timeline (`X` in the diagram).
    6725              :     //
    6726              :     // ```
    6727              :     //-------------------------------+
    6728              :     //                          ...  |
    6729              :     //               [   L1   ]      |
    6730              :     //     [ / L1   ]                | Child Timeline
    6731              :     // ...                           |
    6732              :     // ------------------------------+
    6733              :     //     [ X L1   ]                | Parent Timeline
    6734              :     // ------------------------------+
    6735              :     // ```
    6736              :     #[tokio::test]
    6737            2 :     async fn test_get_vectored_key_gap() -> anyhow::Result<()> {
    6738            2 :         let tenant_conf = TenantConf {
    6739            2 :             // Make compaction deterministic
    6740            2 :             gc_period: Duration::ZERO,
    6741            2 :             compaction_period: Duration::ZERO,
    6742            2 :             // Encourage creation of L1 layers
    6743            2 :             checkpoint_distance: 16 * 1024,
    6744            2 :             compaction_target_size: 8 * 1024,
    6745            2 :             ..TenantConf::default()
    6746            2 :         };
    6747            2 : 
    6748            2 :         let harness = TenantHarness::create_custom(
    6749            2 :             "test_get_vectored_key_gap",
    6750            2 :             tenant_conf,
    6751            2 :             TenantId::generate(),
    6752            2 :             ShardIdentity::unsharded(),
    6753            2 :             Generation::new(0xdeadbeef),
    6754            2 :         )
    6755            2 :         .await?;
    6756            2 :         let (tenant, ctx) = harness.load().await;
    6757            2 : 
    6758            2 :         let mut current_key = Key::from_hex("010000000033333333444444445500000000").unwrap();
    6759            2 :         let gap_at_key = current_key.add(100);
    6760            2 :         let mut current_lsn = Lsn(0x10);
    6761            2 : 
    6762            2 :         const KEY_COUNT: usize = 10_000;
    6763            2 : 
    6764            2 :         let timeline_id = TimelineId::generate();
    6765            2 :         let current_timeline = tenant
    6766            2 :             .create_test_timeline(timeline_id, current_lsn, DEFAULT_PG_VERSION, &ctx)
    6767            2 :             .await?;
    6768            2 : 
    6769            2 :         current_lsn += 0x100;
    6770            2 : 
    6771            2 :         let mut writer = current_timeline.writer().await;
    6772            2 :         writer
    6773            2 :             .put(
    6774            2 :                 gap_at_key,
    6775            2 :                 current_lsn,
    6776            2 :                 &Value::Image(test_img(&format!("{} at {}", gap_at_key, current_lsn))),
    6777            2 :                 &ctx,
    6778            2 :             )
    6779            2 :             .await?;
    6780            2 :         writer.finish_write(current_lsn);
    6781            2 :         drop(writer);
    6782            2 : 
    6783            2 :         let mut latest_lsns = HashMap::new();
    6784            2 :         latest_lsns.insert(gap_at_key, current_lsn);
    6785            2 : 
    6786            2 :         current_timeline.freeze_and_flush().await?;
    6787            2 : 
    6788            2 :         let child_timeline_id = TimelineId::generate();
    6789            2 : 
    6790            2 :         tenant
    6791            2 :             .branch_timeline_test(
    6792            2 :                 &current_timeline,
    6793            2 :                 child_timeline_id,
    6794            2 :                 Some(current_lsn),
    6795            2 :                 &ctx,
    6796            2 :             )
    6797            2 :             .await?;
    6798            2 :         let child_timeline = tenant
    6799            2 :             .get_timeline(child_timeline_id, true)
    6800            2 :             .expect("Should have the branched timeline");
    6801            2 : 
    6802        20002 :         for i in 0..KEY_COUNT {
    6803        20000 :             if current_key == gap_at_key {
    6804            2 :                 current_key = current_key.next();
    6805            2 :                 continue;
    6806        19998 :             }
    6807        19998 : 
    6808        19998 :             current_lsn += 0x10;
    6809            2 : 
    6810        19998 :             let mut writer = child_timeline.writer().await;
    6811        19998 :             writer
    6812        19998 :                 .put(
    6813        19998 :                     current_key,
    6814        19998 :                     current_lsn,
    6815        19998 :                     &Value::Image(test_img(&format!("{} at {}", current_key, current_lsn))),
    6816        19998 :                     &ctx,
    6817        19998 :                 )
    6818        19998 :                 .await?;
    6819        19998 :             writer.finish_write(current_lsn);
    6820        19998 :             drop(writer);
    6821        19998 : 
    6822        19998 :             latest_lsns.insert(current_key, current_lsn);
    6823        19998 :             current_key = current_key.next();
    6824        19998 : 
    6825        19998 :             // Flush every now and then to encourage layer file creation.
    6826        19998 :             if i % 500 == 0 {
    6827           40 :                 child_timeline.freeze_and_flush().await?;
    6828        19958 :             }
    6829            2 :         }
    6830            2 : 
    6831            2 :         child_timeline.freeze_and_flush().await?;
    6832            2 :         let mut flags = EnumSet::new();
    6833            2 :         flags.insert(CompactFlags::ForceRepartition);
    6834            2 :         child_timeline
    6835            2 :             .compact(&CancellationToken::new(), flags, &ctx)
    6836            2 :             .await?;
    6837            2 : 
    6838            2 :         let key_near_end = {
    6839            2 :             let mut tmp = current_key;
    6840            2 :             tmp.field6 -= 10;
    6841            2 :             tmp
    6842            2 :         };
    6843            2 : 
    6844            2 :         let key_near_gap = {
    6845            2 :             let mut tmp = gap_at_key;
    6846            2 :             tmp.field6 -= 10;
    6847            2 :             tmp
    6848            2 :         };
    6849            2 : 
    6850            2 :         let read = KeySpace {
    6851            2 :             ranges: vec![key_near_gap..gap_at_key.next(), key_near_end..current_key],
    6852            2 :         };
    6853            2 :         let results = child_timeline
    6854            2 :             .get_vectored_impl(
    6855            2 :                 read.clone(),
    6856            2 :                 current_lsn,
    6857            2 :                 &mut ValuesReconstructState::new(),
    6858            2 :                 &ctx,
    6859            2 :             )
    6860            2 :             .await?;
    6861            2 : 
    6862           44 :         for (key, img_res) in results {
    6863           42 :             let expected = test_img(&format!("{} at {}", key, latest_lsns[&key]));
    6864           42 :             assert_eq!(img_res?, expected);
    6865            2 :         }
    6866            2 : 
    6867            2 :         Ok(())
    6868            2 :     }
    6869              : 
    6870              :     // Test that vectored get descends into ancestor timelines correctly and
    6871              :     // does not return an image that's newer than requested.
    6872              :     //
    6873              :     // The diagram below ilustrates an interesting case. We have a parent timeline
    6874              :     // (top of the Lsn range) and a child timeline. The request key cannot be reconstructed
    6875              :     // from the child timeline, so the parent timeline must be visited. When advacing into
    6876              :     // the child timeline, the read path needs to remember what the requested Lsn was in
    6877              :     // order to avoid returning an image that's too new. The test below constructs such
    6878              :     // a timeline setup and does a few queries around the Lsn of each page image.
    6879              :     // ```
    6880              :     //    LSN
    6881              :     //     ^
    6882              :     //     |
    6883              :     //     |
    6884              :     // 500 | --------------------------------------> branch point
    6885              :     // 400 |        X
    6886              :     // 300 |        X
    6887              :     // 200 | --------------------------------------> requested lsn
    6888              :     // 100 |        X
    6889              :     //     |---------------------------------------> Key
    6890              :     //              |
    6891              :     //              ------> requested key
    6892              :     //
    6893              :     // Legend:
    6894              :     // * X - page images
    6895              :     // ```
    6896              :     #[tokio::test]
    6897            2 :     async fn test_get_vectored_ancestor_descent() -> anyhow::Result<()> {
    6898            2 :         let harness = TenantHarness::create("test_get_vectored_on_lsn_axis").await?;
    6899            2 :         let (tenant, ctx) = harness.load().await;
    6900            2 : 
    6901            2 :         let start_key = Key::from_hex("010000000033333333444444445500000000").unwrap();
    6902            2 :         let end_key = start_key.add(1000);
    6903            2 :         let child_gap_at_key = start_key.add(500);
    6904            2 :         let mut parent_gap_lsns: BTreeMap<Lsn, String> = BTreeMap::new();
    6905            2 : 
    6906            2 :         let mut current_lsn = Lsn(0x10);
    6907            2 : 
    6908            2 :         let timeline_id = TimelineId::generate();
    6909            2 :         let parent_timeline = tenant
    6910            2 :             .create_test_timeline(timeline_id, current_lsn, DEFAULT_PG_VERSION, &ctx)
    6911            2 :             .await?;
    6912            2 : 
    6913            2 :         current_lsn += 0x100;
    6914            2 : 
    6915            8 :         for _ in 0..3 {
    6916            6 :             let mut key = start_key;
    6917         6006 :             while key < end_key {
    6918         6000 :                 current_lsn += 0x10;
    6919         6000 : 
    6920         6000 :                 let image_value = format!("{} at {}", child_gap_at_key, current_lsn);
    6921            2 : 
    6922         6000 :                 let mut writer = parent_timeline.writer().await;
    6923         6000 :                 writer
    6924         6000 :                     .put(
    6925         6000 :                         key,
    6926         6000 :                         current_lsn,
    6927         6000 :                         &Value::Image(test_img(&image_value)),
    6928         6000 :                         &ctx,
    6929         6000 :                     )
    6930         6000 :                     .await?;
    6931         6000 :                 writer.finish_write(current_lsn);
    6932         6000 : 
    6933         6000 :                 if key == child_gap_at_key {
    6934            6 :                     parent_gap_lsns.insert(current_lsn, image_value);
    6935         5994 :                 }
    6936            2 : 
    6937         6000 :                 key = key.next();
    6938            2 :             }
    6939            2 : 
    6940            6 :             parent_timeline.freeze_and_flush().await?;
    6941            2 :         }
    6942            2 : 
    6943            2 :         let child_timeline_id = TimelineId::generate();
    6944            2 : 
    6945            2 :         let child_timeline = tenant
    6946            2 :             .branch_timeline_test(&parent_timeline, child_timeline_id, Some(current_lsn), &ctx)
    6947            2 :             .await?;
    6948            2 : 
    6949            2 :         let mut key = start_key;
    6950         2002 :         while key < end_key {
    6951         2000 :             if key == child_gap_at_key {
    6952            2 :                 key = key.next();
    6953            2 :                 continue;
    6954         1998 :             }
    6955         1998 : 
    6956         1998 :             current_lsn += 0x10;
    6957            2 : 
    6958         1998 :             let mut writer = child_timeline.writer().await;
    6959         1998 :             writer
    6960         1998 :                 .put(
    6961         1998 :                     key,
    6962         1998 :                     current_lsn,
    6963         1998 :                     &Value::Image(test_img(&format!("{} at {}", key, current_lsn))),
    6964         1998 :                     &ctx,
    6965         1998 :                 )
    6966         1998 :                 .await?;
    6967         1998 :             writer.finish_write(current_lsn);
    6968         1998 : 
    6969         1998 :             key = key.next();
    6970            2 :         }
    6971            2 : 
    6972            2 :         child_timeline.freeze_and_flush().await?;
    6973            2 : 
    6974            2 :         let lsn_offsets: [i64; 5] = [-10, -1, 0, 1, 10];
    6975            2 :         let mut query_lsns = Vec::new();
    6976            6 :         for image_lsn in parent_gap_lsns.keys().rev() {
    6977           36 :             for offset in lsn_offsets {
    6978           30 :                 query_lsns.push(Lsn(image_lsn
    6979           30 :                     .0
    6980           30 :                     .checked_add_signed(offset)
    6981           30 :                     .expect("Shouldn't overflow")));
    6982           30 :             }
    6983            2 :         }
    6984            2 : 
    6985           32 :         for query_lsn in query_lsns {
    6986           30 :             let results = child_timeline
    6987           30 :                 .get_vectored_impl(
    6988           30 :                     KeySpace {
    6989           30 :                         ranges: vec![child_gap_at_key..child_gap_at_key.next()],
    6990           30 :                     },
    6991           30 :                     query_lsn,
    6992           30 :                     &mut ValuesReconstructState::new(),
    6993           30 :                     &ctx,
    6994           30 :                 )
    6995           30 :                 .await;
    6996            2 : 
    6997           30 :             let expected_item = parent_gap_lsns
    6998           30 :                 .iter()
    6999           30 :                 .rev()
    7000           68 :                 .find(|(lsn, _)| **lsn <= query_lsn);
    7001           30 : 
    7002           30 :             info!(
    7003            2 :                 "Doing vectored read at LSN {}. Expecting image to be: {:?}",
    7004            2 :                 query_lsn, expected_item
    7005            2 :             );
    7006            2 : 
    7007           30 :             match expected_item {
    7008           26 :                 Some((_, img_value)) => {
    7009           26 :                     let key_results = results.expect("No vectored get error expected");
    7010           26 :                     let key_result = &key_results[&child_gap_at_key];
    7011           26 :                     let returned_img = key_result
    7012           26 :                         .as_ref()
    7013           26 :                         .expect("No page reconstruct error expected");
    7014           26 : 
    7015           26 :                     info!(
    7016            2 :                         "Vectored read at LSN {} returned image {}",
    7017            0 :                         query_lsn,
    7018            0 :                         std::str::from_utf8(returned_img)?
    7019            2 :                     );
    7020           26 :                     assert_eq!(*returned_img, test_img(img_value));
    7021            2 :                 }
    7022            2 :                 None => {
    7023            4 :                     assert!(matches!(results, Err(GetVectoredError::MissingKey(_))));
    7024            2 :                 }
    7025            2 :             }
    7026            2 :         }
    7027            2 : 
    7028            2 :         Ok(())
    7029            2 :     }
    7030              : 
    7031              :     #[tokio::test]
    7032            2 :     async fn test_random_updates() -> anyhow::Result<()> {
    7033            2 :         let names_algorithms = [
    7034            2 :             ("test_random_updates_legacy", CompactionAlgorithm::Legacy),
    7035            2 :             ("test_random_updates_tiered", CompactionAlgorithm::Tiered),
    7036            2 :         ];
    7037            6 :         for (name, algorithm) in names_algorithms {
    7038            4 :             test_random_updates_algorithm(name, algorithm).await?;
    7039            2 :         }
    7040            2 :         Ok(())
    7041            2 :     }
    7042              : 
    7043            4 :     async fn test_random_updates_algorithm(
    7044            4 :         name: &'static str,
    7045            4 :         compaction_algorithm: CompactionAlgorithm,
    7046            4 :     ) -> anyhow::Result<()> {
    7047            4 :         let mut harness = TenantHarness::create(name).await?;
    7048            4 :         harness.tenant_conf.compaction_algorithm = CompactionAlgorithmSettings {
    7049            4 :             kind: compaction_algorithm,
    7050            4 :         };
    7051            4 :         let (tenant, ctx) = harness.load().await;
    7052            4 :         let tline = tenant
    7053            4 :             .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx)
    7054            4 :             .await?;
    7055              : 
    7056              :         const NUM_KEYS: usize = 1000;
    7057            4 :         let cancel = CancellationToken::new();
    7058            4 : 
    7059            4 :         let mut test_key = Key::from_hex("010000000033333333444444445500000000").unwrap();
    7060            4 :         let mut test_key_end = test_key;
    7061            4 :         test_key_end.field6 = NUM_KEYS as u32;
    7062            4 :         tline.add_extra_test_dense_keyspace(KeySpace::single(test_key..test_key_end));
    7063            4 : 
    7064            4 :         let mut keyspace = KeySpaceAccum::new();
    7065            4 : 
    7066            4 :         // Track when each page was last modified. Used to assert that
    7067            4 :         // a read sees the latest page version.
    7068            4 :         let mut updated = [Lsn(0); NUM_KEYS];
    7069            4 : 
    7070            4 :         let mut lsn = Lsn(0x10);
    7071              :         #[allow(clippy::needless_range_loop)]
    7072         4004 :         for blknum in 0..NUM_KEYS {
    7073         4000 :             lsn = Lsn(lsn.0 + 0x10);
    7074         4000 :             test_key.field6 = blknum as u32;
    7075         4000 :             let mut writer = tline.writer().await;
    7076         4000 :             writer
    7077         4000 :                 .put(
    7078         4000 :                     test_key,
    7079         4000 :                     lsn,
    7080         4000 :                     &Value::Image(test_img(&format!("{} at {}", blknum, lsn))),
    7081         4000 :                     &ctx,
    7082         4000 :                 )
    7083         4000 :                 .await?;
    7084         4000 :             writer.finish_write(lsn);
    7085         4000 :             updated[blknum] = lsn;
    7086         4000 :             drop(writer);
    7087         4000 : 
    7088         4000 :             keyspace.add_key(test_key);
    7089              :         }
    7090              : 
    7091          204 :         for _ in 0..50 {
    7092       200200 :             for _ in 0..NUM_KEYS {
    7093       200000 :                 lsn = Lsn(lsn.0 + 0x10);
    7094       200000 :                 let blknum = thread_rng().gen_range(0..NUM_KEYS);
    7095       200000 :                 test_key.field6 = blknum as u32;
    7096       200000 :                 let mut writer = tline.writer().await;
    7097       200000 :                 writer
    7098       200000 :                     .put(
    7099       200000 :                         test_key,
    7100       200000 :                         lsn,
    7101       200000 :                         &Value::Image(test_img(&format!("{} at {}", blknum, lsn))),
    7102       200000 :                         &ctx,
    7103       200000 :                     )
    7104       200000 :                     .await?;
    7105       200000 :                 writer.finish_write(lsn);
    7106       200000 :                 drop(writer);
    7107       200000 :                 updated[blknum] = lsn;
    7108              :             }
    7109              : 
    7110              :             // Read all the blocks
    7111       200000 :             for (blknum, last_lsn) in updated.iter().enumerate() {
    7112       200000 :                 test_key.field6 = blknum as u32;
    7113       200000 :                 assert_eq!(
    7114       200000 :                     tline.get(test_key, lsn, &ctx).await?,
    7115       200000 :                     test_img(&format!("{} at {}", blknum, last_lsn))
    7116              :                 );
    7117              :             }
    7118              : 
    7119              :             // Perform a cycle of flush, and GC
    7120          200 :             tline.freeze_and_flush().await?;
    7121          200 :             tenant
    7122          200 :                 .gc_iteration(Some(tline.timeline_id), 0, Duration::ZERO, &cancel, &ctx)
    7123          200 :                 .await?;
    7124              :         }
    7125              : 
    7126            4 :         Ok(())
    7127            4 :     }
    7128              : 
    7129              :     #[tokio::test]
    7130            2 :     async fn test_traverse_branches() -> anyhow::Result<()> {
    7131            2 :         let (tenant, ctx) = TenantHarness::create("test_traverse_branches")
    7132            2 :             .await?
    7133            2 :             .load()
    7134            2 :             .await;
    7135            2 :         let mut tline = tenant
    7136            2 :             .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx)
    7137            2 :             .await?;
    7138            2 : 
    7139            2 :         const NUM_KEYS: usize = 1000;
    7140            2 : 
    7141            2 :         let mut test_key = Key::from_hex("010000000033333333444444445500000000").unwrap();
    7142            2 : 
    7143            2 :         let mut keyspace = KeySpaceAccum::new();
    7144            2 : 
    7145            2 :         let cancel = CancellationToken::new();
    7146            2 : 
    7147            2 :         // Track when each page was last modified. Used to assert that
    7148            2 :         // a read sees the latest page version.
    7149            2 :         let mut updated = [Lsn(0); NUM_KEYS];
    7150            2 : 
    7151            2 :         let mut lsn = Lsn(0x10);
    7152            2 :         #[allow(clippy::needless_range_loop)]
    7153         2002 :         for blknum in 0..NUM_KEYS {
    7154         2000 :             lsn = Lsn(lsn.0 + 0x10);
    7155         2000 :             test_key.field6 = blknum as u32;
    7156         2000 :             let mut writer = tline.writer().await;
    7157         2000 :             writer
    7158         2000 :                 .put(
    7159         2000 :                     test_key,
    7160         2000 :                     lsn,
    7161         2000 :                     &Value::Image(test_img(&format!("{} at {}", blknum, lsn))),
    7162         2000 :                     &ctx,
    7163         2000 :                 )
    7164         2000 :                 .await?;
    7165         2000 :             writer.finish_write(lsn);
    7166         2000 :             updated[blknum] = lsn;
    7167         2000 :             drop(writer);
    7168         2000 : 
    7169         2000 :             keyspace.add_key(test_key);
    7170            2 :         }
    7171            2 : 
    7172          102 :         for _ in 0..50 {
    7173          100 :             let new_tline_id = TimelineId::generate();
    7174          100 :             tenant
    7175          100 :                 .branch_timeline_test(&tline, new_tline_id, Some(lsn), &ctx)
    7176          100 :                 .await?;
    7177          100 :             tline = tenant
    7178          100 :                 .get_timeline(new_tline_id, true)
    7179          100 :                 .expect("Should have the branched timeline");
    7180            2 : 
    7181       100100 :             for _ in 0..NUM_KEYS {
    7182       100000 :                 lsn = Lsn(lsn.0 + 0x10);
    7183       100000 :                 let blknum = thread_rng().gen_range(0..NUM_KEYS);
    7184       100000 :                 test_key.field6 = blknum as u32;
    7185       100000 :                 let mut writer = tline.writer().await;
    7186       100000 :                 writer
    7187       100000 :                     .put(
    7188       100000 :                         test_key,
    7189       100000 :                         lsn,
    7190       100000 :                         &Value::Image(test_img(&format!("{} at {}", blknum, lsn))),
    7191       100000 :                         &ctx,
    7192       100000 :                     )
    7193       100000 :                     .await?;
    7194       100000 :                 println!("updating {} at {}", blknum, lsn);
    7195       100000 :                 writer.finish_write(lsn);
    7196       100000 :                 drop(writer);
    7197       100000 :                 updated[blknum] = lsn;
    7198            2 :             }
    7199            2 : 
    7200            2 :             // Read all the blocks
    7201       100000 :             for (blknum, last_lsn) in updated.iter().enumerate() {
    7202       100000 :                 test_key.field6 = blknum as u32;
    7203       100000 :                 assert_eq!(
    7204       100000 :                     tline.get(test_key, lsn, &ctx).await?,
    7205       100000 :                     test_img(&format!("{} at {}", blknum, last_lsn))
    7206            2 :                 );
    7207            2 :             }
    7208            2 : 
    7209            2 :             // Perform a cycle of flush, compact, and GC
    7210          100 :             tline.freeze_and_flush().await?;
    7211          100 :             tline.compact(&cancel, EnumSet::empty(), &ctx).await?;
    7212          100 :             tenant
    7213          100 :                 .gc_iteration(Some(tline.timeline_id), 0, Duration::ZERO, &cancel, &ctx)
    7214          100 :                 .await?;
    7215            2 :         }
    7216            2 : 
    7217            2 :         Ok(())
    7218            2 :     }
    7219              : 
    7220              :     #[tokio::test]
    7221            2 :     async fn test_traverse_ancestors() -> anyhow::Result<()> {
    7222            2 :         let (tenant, ctx) = TenantHarness::create("test_traverse_ancestors")
    7223            2 :             .await?
    7224            2 :             .load()
    7225            2 :             .await;
    7226            2 :         let mut tline = tenant
    7227            2 :             .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx)
    7228            2 :             .await?;
    7229            2 : 
    7230            2 :         const NUM_KEYS: usize = 100;
    7231            2 :         const NUM_TLINES: usize = 50;
    7232            2 : 
    7233            2 :         let mut test_key = Key::from_hex("010000000033333333444444445500000000").unwrap();
    7234            2 :         // Track page mutation lsns across different timelines.
    7235            2 :         let mut updated = [[Lsn(0); NUM_KEYS]; NUM_TLINES];
    7236            2 : 
    7237            2 :         let mut lsn = Lsn(0x10);
    7238            2 : 
    7239            2 :         #[allow(clippy::needless_range_loop)]
    7240          102 :         for idx in 0..NUM_TLINES {
    7241          100 :             let new_tline_id = TimelineId::generate();
    7242          100 :             tenant
    7243          100 :                 .branch_timeline_test(&tline, new_tline_id, Some(lsn), &ctx)
    7244          100 :                 .await?;
    7245          100 :             tline = tenant
    7246          100 :                 .get_timeline(new_tline_id, true)
    7247          100 :                 .expect("Should have the branched timeline");
    7248            2 : 
    7249        10100 :             for _ in 0..NUM_KEYS {
    7250        10000 :                 lsn = Lsn(lsn.0 + 0x10);
    7251        10000 :                 let blknum = thread_rng().gen_range(0..NUM_KEYS);
    7252        10000 :                 test_key.field6 = blknum as u32;
    7253        10000 :                 let mut writer = tline.writer().await;
    7254        10000 :                 writer
    7255        10000 :                     .put(
    7256        10000 :                         test_key,
    7257        10000 :                         lsn,
    7258        10000 :                         &Value::Image(test_img(&format!("{} {} at {}", idx, blknum, lsn))),
    7259        10000 :                         &ctx,
    7260        10000 :                     )
    7261        10000 :                     .await?;
    7262        10000 :                 println!("updating [{}][{}] at {}", idx, blknum, lsn);
    7263        10000 :                 writer.finish_write(lsn);
    7264        10000 :                 drop(writer);
    7265        10000 :                 updated[idx][blknum] = lsn;
    7266            2 :             }
    7267            2 :         }
    7268            2 : 
    7269            2 :         // Read pages from leaf timeline across all ancestors.
    7270          100 :         for (idx, lsns) in updated.iter().enumerate() {
    7271        10000 :             for (blknum, lsn) in lsns.iter().enumerate() {
    7272            2 :                 // Skip empty mutations.
    7273        10000 :                 if lsn.0 == 0 {
    7274         3563 :                     continue;
    7275         6437 :                 }
    7276         6437 :                 println!("checking [{idx}][{blknum}] at {lsn}");
    7277         6437 :                 test_key.field6 = blknum as u32;
    7278         6437 :                 assert_eq!(
    7279         6437 :                     tline.get(test_key, *lsn, &ctx).await?,
    7280         6437 :                     test_img(&format!("{idx} {blknum} at {lsn}"))
    7281            2 :                 );
    7282            2 :             }
    7283            2 :         }
    7284            2 :         Ok(())
    7285            2 :     }
    7286              : 
    7287              :     #[tokio::test]
    7288            2 :     async fn test_write_at_initdb_lsn_takes_optimization_code_path() -> anyhow::Result<()> {
    7289            2 :         let (tenant, ctx) = TenantHarness::create("test_empty_test_timeline_is_usable")
    7290            2 :             .await?
    7291            2 :             .load()
    7292            2 :             .await;
    7293            2 : 
    7294            2 :         let initdb_lsn = Lsn(0x20);
    7295            2 :         let utline = tenant
    7296            2 :             .create_empty_timeline(TIMELINE_ID, initdb_lsn, DEFAULT_PG_VERSION, &ctx)
    7297            2 :             .await?;
    7298            2 :         let tline = utline.raw_timeline().unwrap();
    7299            2 : 
    7300            2 :         // Spawn flush loop now so that we can set the `expect_initdb_optimization`
    7301            2 :         tline.maybe_spawn_flush_loop();
    7302            2 : 
    7303            2 :         // Make sure the timeline has the minimum set of required keys for operation.
    7304            2 :         // The only operation you can always do on an empty timeline is to `put` new data.
    7305            2 :         // Except if you `put` at `initdb_lsn`.
    7306            2 :         // In that case, there's an optimization to directly create image layers instead of delta layers.
    7307            2 :         // It uses `repartition()`, which assumes some keys to be present.
    7308            2 :         // Let's make sure the test timeline can handle that case.
    7309            2 :         {
    7310            2 :             let mut state = tline.flush_loop_state.lock().unwrap();
    7311            2 :             assert_eq!(
    7312            2 :                 timeline::FlushLoopState::Running {
    7313            2 :                     expect_initdb_optimization: false,
    7314            2 :                     initdb_optimization_count: 0,
    7315            2 :                 },
    7316            2 :                 *state
    7317            2 :             );
    7318            2 :             *state = timeline::FlushLoopState::Running {
    7319            2 :                 expect_initdb_optimization: true,
    7320            2 :                 initdb_optimization_count: 0,
    7321            2 :             };
    7322            2 :         }
    7323            2 : 
    7324            2 :         // Make writes at the initdb_lsn. When we flush it below, it should be handled by the optimization.
    7325            2 :         // As explained above, the optimization requires some keys to be present.
    7326            2 :         // As per `create_empty_timeline` documentation, use init_empty to set them.
    7327            2 :         // This is what `create_test_timeline` does, by the way.
    7328            2 :         let mut modification = tline.begin_modification(initdb_lsn);
    7329            2 :         modification
    7330            2 :             .init_empty_test_timeline()
    7331            2 :             .context("init_empty_test_timeline")?;
    7332            2 :         modification
    7333            2 :             .commit(&ctx)
    7334            2 :             .await
    7335            2 :             .context("commit init_empty_test_timeline modification")?;
    7336            2 : 
    7337            2 :         // Do the flush. The flush code will check the expectations that we set above.
    7338            2 :         tline.freeze_and_flush().await?;
    7339            2 : 
    7340            2 :         // assert freeze_and_flush exercised the initdb optimization
    7341            2 :         {
    7342            2 :             let state = tline.flush_loop_state.lock().unwrap();
    7343            2 :             let timeline::FlushLoopState::Running {
    7344            2 :                 expect_initdb_optimization,
    7345            2 :                 initdb_optimization_count,
    7346            2 :             } = *state
    7347            2 :             else {
    7348            2 :                 panic!("unexpected state: {:?}", *state);
    7349            2 :             };
    7350            2 :             assert!(expect_initdb_optimization);
    7351            2 :             assert!(initdb_optimization_count > 0);
    7352            2 :         }
    7353            2 :         Ok(())
    7354            2 :     }
    7355              : 
    7356              :     #[tokio::test]
    7357            2 :     async fn test_create_guard_crash() -> anyhow::Result<()> {
    7358            2 :         let name = "test_create_guard_crash";
    7359            2 :         let harness = TenantHarness::create(name).await?;
    7360            2 :         {
    7361            2 :             let (tenant, ctx) = harness.load().await;
    7362            2 :             let tline = tenant
    7363            2 :                 .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)
    7364            2 :                 .await?;
    7365            2 :             // Leave the timeline ID in [`Tenant::timelines_creating`] to exclude attempting to create it again
    7366            2 :             let raw_tline = tline.raw_timeline().unwrap();
    7367            2 :             raw_tline
    7368            2 :                 .shutdown(super::timeline::ShutdownMode::Hard)
    7369            2 :                 .instrument(info_span!("test_shutdown", tenant_id=%raw_tline.tenant_shard_id, shard_id=%raw_tline.tenant_shard_id.shard_slug(), timeline_id=%TIMELINE_ID))
    7370            2 :                 .await;
    7371            2 :             std::mem::forget(tline);
    7372            2 :         }
    7373            2 : 
    7374            2 :         let (tenant, _) = harness.load().await;
    7375            2 :         match tenant.get_timeline(TIMELINE_ID, false) {
    7376            2 :             Ok(_) => panic!("timeline should've been removed during load"),
    7377            2 :             Err(e) => {
    7378            2 :                 assert_eq!(
    7379            2 :                     e,
    7380            2 :                     GetTimelineError::NotFound {
    7381            2 :                         tenant_id: tenant.tenant_shard_id,
    7382            2 :                         timeline_id: TIMELINE_ID,
    7383            2 :                     }
    7384            2 :                 )
    7385            2 :             }
    7386            2 :         }
    7387            2 : 
    7388            2 :         assert!(!harness
    7389            2 :             .conf
    7390            2 :             .timeline_path(&tenant.tenant_shard_id, &TIMELINE_ID)
    7391            2 :             .exists());
    7392            2 : 
    7393            2 :         Ok(())
    7394            2 :     }
    7395              : 
    7396              :     #[tokio::test]
    7397            2 :     async fn test_read_at_max_lsn() -> anyhow::Result<()> {
    7398            2 :         let names_algorithms = [
    7399            2 :             ("test_read_at_max_lsn_legacy", CompactionAlgorithm::Legacy),
    7400            2 :             ("test_read_at_max_lsn_tiered", CompactionAlgorithm::Tiered),
    7401            2 :         ];
    7402            6 :         for (name, algorithm) in names_algorithms {
    7403            4 :             test_read_at_max_lsn_algorithm(name, algorithm).await?;
    7404            2 :         }
    7405            2 :         Ok(())
    7406            2 :     }
    7407              : 
    7408            4 :     async fn test_read_at_max_lsn_algorithm(
    7409            4 :         name: &'static str,
    7410            4 :         compaction_algorithm: CompactionAlgorithm,
    7411            4 :     ) -> anyhow::Result<()> {
    7412            4 :         let mut harness = TenantHarness::create(name).await?;
    7413            4 :         harness.tenant_conf.compaction_algorithm = CompactionAlgorithmSettings {
    7414            4 :             kind: compaction_algorithm,
    7415            4 :         };
    7416            4 :         let (tenant, ctx) = harness.load().await;
    7417            4 :         let tline = tenant
    7418            4 :             .create_test_timeline(TIMELINE_ID, Lsn(0x08), DEFAULT_PG_VERSION, &ctx)
    7419            4 :             .await?;
    7420              : 
    7421            4 :         let lsn = Lsn(0x10);
    7422            4 :         let compact = false;
    7423            4 :         bulk_insert_maybe_compact_gc(&tenant, &tline, &ctx, lsn, 50, 10000, compact).await?;
    7424              : 
    7425            4 :         let test_key = Key::from_hex("010000000033333333444444445500000000").unwrap();
    7426            4 :         let read_lsn = Lsn(u64::MAX - 1);
    7427              : 
    7428            4 :         let result = tline.get(test_key, read_lsn, &ctx).await;
    7429            4 :         assert!(result.is_ok(), "result is not Ok: {}", result.unwrap_err());
    7430              : 
    7431            4 :         Ok(())
    7432            4 :     }
    7433              : 
    7434              :     #[tokio::test]
    7435            2 :     async fn test_metadata_scan() -> anyhow::Result<()> {
    7436            2 :         let harness = TenantHarness::create("test_metadata_scan").await?;
    7437            2 :         let (tenant, ctx) = harness.load().await;
    7438            2 :         let tline = tenant
    7439            2 :             .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx)
    7440            2 :             .await?;
    7441            2 : 
    7442            2 :         const NUM_KEYS: usize = 1000;
    7443            2 :         const STEP: usize = 10000; // random update + scan base_key + idx * STEP
    7444            2 : 
    7445            2 :         let cancel = CancellationToken::new();
    7446            2 : 
    7447            2 :         let mut base_key = Key::from_hex("000000000033333333444444445500000000").unwrap();
    7448            2 :         base_key.field1 = AUX_KEY_PREFIX;
    7449            2 :         let mut test_key = base_key;
    7450            2 : 
    7451            2 :         // Track when each page was last modified. Used to assert that
    7452            2 :         // a read sees the latest page version.
    7453            2 :         let mut updated = [Lsn(0); NUM_KEYS];
    7454            2 : 
    7455            2 :         let mut lsn = Lsn(0x10);
    7456            2 :         #[allow(clippy::needless_range_loop)]
    7457         2002 :         for blknum in 0..NUM_KEYS {
    7458         2000 :             lsn = Lsn(lsn.0 + 0x10);
    7459         2000 :             test_key.field6 = (blknum * STEP) as u32;
    7460         2000 :             let mut writer = tline.writer().await;
    7461         2000 :             writer
    7462         2000 :                 .put(
    7463         2000 :                     test_key,
    7464         2000 :                     lsn,
    7465         2000 :                     &Value::Image(test_img(&format!("{} at {}", blknum, lsn))),
    7466         2000 :                     &ctx,
    7467         2000 :                 )
    7468         2000 :                 .await?;
    7469         2000 :             writer.finish_write(lsn);
    7470         2000 :             updated[blknum] = lsn;
    7471         2000 :             drop(writer);
    7472            2 :         }
    7473            2 : 
    7474            2 :         let keyspace = KeySpace::single(base_key..base_key.add((NUM_KEYS * STEP) as u32));
    7475            2 : 
    7476           24 :         for iter in 0..=10 {
    7477            2 :             // Read all the blocks
    7478        22000 :             for (blknum, last_lsn) in updated.iter().enumerate() {
    7479        22000 :                 test_key.field6 = (blknum * STEP) as u32;
    7480        22000 :                 assert_eq!(
    7481        22000 :                     tline.get(test_key, lsn, &ctx).await?,
    7482        22000 :                     test_img(&format!("{} at {}", blknum, last_lsn))
    7483            2 :                 );
    7484            2 :             }
    7485            2 : 
    7486           22 :             let mut cnt = 0;
    7487        22000 :             for (key, value) in tline
    7488           22 :                 .get_vectored_impl(
    7489           22 :                     keyspace.clone(),
    7490           22 :                     lsn,
    7491           22 :                     &mut ValuesReconstructState::default(),
    7492           22 :                     &ctx,
    7493           22 :                 )
    7494           22 :                 .await?
    7495            2 :             {
    7496        22000 :                 let blknum = key.field6 as usize;
    7497        22000 :                 let value = value?;
    7498        22000 :                 assert!(blknum % STEP == 0);
    7499        22000 :                 let blknum = blknum / STEP;
    7500        22000 :                 assert_eq!(
    7501        22000 :                     value,
    7502        22000 :                     test_img(&format!("{} at {}", blknum, updated[blknum]))
    7503        22000 :                 );
    7504        22000 :                 cnt += 1;
    7505            2 :             }
    7506            2 : 
    7507           22 :             assert_eq!(cnt, NUM_KEYS);
    7508            2 : 
    7509        22022 :             for _ in 0..NUM_KEYS {
    7510        22000 :                 lsn = Lsn(lsn.0 + 0x10);
    7511        22000 :                 let blknum = thread_rng().gen_range(0..NUM_KEYS);
    7512        22000 :                 test_key.field6 = (blknum * STEP) as u32;
    7513        22000 :                 let mut writer = tline.writer().await;
    7514        22000 :                 writer
    7515        22000 :                     .put(
    7516        22000 :                         test_key,
    7517        22000 :                         lsn,
    7518        22000 :                         &Value::Image(test_img(&format!("{} at {}", blknum, lsn))),
    7519        22000 :                         &ctx,
    7520        22000 :                     )
    7521        22000 :                     .await?;
    7522        22000 :                 writer.finish_write(lsn);
    7523        22000 :                 drop(writer);
    7524        22000 :                 updated[blknum] = lsn;
    7525            2 :             }
    7526            2 : 
    7527            2 :             // Perform two cycles of flush, compact, and GC
    7528           66 :             for round in 0..2 {
    7529           44 :                 tline.freeze_and_flush().await?;
    7530           44 :                 tline
    7531           44 :                     .compact(
    7532           44 :                         &cancel,
    7533           44 :                         if iter % 5 == 0 && round == 0 {
    7534            6 :                             let mut flags = EnumSet::new();
    7535            6 :                             flags.insert(CompactFlags::ForceImageLayerCreation);
    7536            6 :                             flags.insert(CompactFlags::ForceRepartition);
    7537            6 :                             flags
    7538            2 :                         } else {
    7539           38 :                             EnumSet::empty()
    7540            2 :                         },
    7541           44 :                         &ctx,
    7542           44 :                     )
    7543           44 :                     .await?;
    7544           44 :                 tenant
    7545           44 :                     .gc_iteration(Some(tline.timeline_id), 0, Duration::ZERO, &cancel, &ctx)
    7546           44 :                     .await?;
    7547            2 :             }
    7548            2 :         }
    7549            2 : 
    7550            2 :         Ok(())
    7551            2 :     }
    7552              : 
    7553              :     #[tokio::test]
    7554            2 :     async fn test_metadata_compaction_trigger() -> anyhow::Result<()> {
    7555            2 :         let harness = TenantHarness::create("test_metadata_compaction_trigger").await?;
    7556            2 :         let (tenant, ctx) = harness.load().await;
    7557            2 :         let tline = tenant
    7558            2 :             .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx)
    7559            2 :             .await?;
    7560            2 : 
    7561            2 :         let cancel = CancellationToken::new();
    7562            2 : 
    7563            2 :         let mut base_key = Key::from_hex("000000000033333333444444445500000000").unwrap();
    7564            2 :         base_key.field1 = AUX_KEY_PREFIX;
    7565            2 :         let test_key = base_key;
    7566            2 :         let mut lsn = Lsn(0x10);
    7567            2 : 
    7568           42 :         for _ in 0..20 {
    7569           40 :             lsn = Lsn(lsn.0 + 0x10);
    7570           40 :             let mut writer = tline.writer().await;
    7571           40 :             writer
    7572           40 :                 .put(
    7573           40 :                     test_key,
    7574           40 :                     lsn,
    7575           40 :                     &Value::Image(test_img(&format!("{} at {}", 0, lsn))),
    7576           40 :                     &ctx,
    7577           40 :                 )
    7578           40 :                 .await?;
    7579           40 :             writer.finish_write(lsn);
    7580           40 :             drop(writer);
    7581           40 :             tline.freeze_and_flush().await?; // force create a delta layer
    7582            2 :         }
    7583            2 : 
    7584            2 :         let before_num_l0_delta_files =
    7585            2 :             tline.layers.read().await.layer_map()?.level0_deltas().len();
    7586            2 : 
    7587            2 :         tline.compact(&cancel, EnumSet::empty(), &ctx).await?;
    7588            2 : 
    7589            2 :         let after_num_l0_delta_files = tline.layers.read().await.layer_map()?.level0_deltas().len();
    7590            2 : 
    7591            2 :         assert!(after_num_l0_delta_files < before_num_l0_delta_files, "after_num_l0_delta_files={after_num_l0_delta_files}, before_num_l0_delta_files={before_num_l0_delta_files}");
    7592            2 : 
    7593            2 :         assert_eq!(
    7594            2 :             tline.get(test_key, lsn, &ctx).await?,
    7595            2 :             test_img(&format!("{} at {}", 0, lsn))
    7596            2 :         );
    7597            2 : 
    7598            2 :         Ok(())
    7599            2 :     }
    7600              : 
    7601              :     #[tokio::test]
    7602            2 :     async fn test_aux_file_e2e() {
    7603            2 :         let harness = TenantHarness::create("test_aux_file_e2e").await.unwrap();
    7604            2 : 
    7605            2 :         let (tenant, ctx) = harness.load().await;
    7606            2 : 
    7607            2 :         let mut lsn = Lsn(0x08);
    7608            2 : 
    7609            2 :         let tline: Arc<Timeline> = tenant
    7610            2 :             .create_test_timeline(TIMELINE_ID, lsn, DEFAULT_PG_VERSION, &ctx)
    7611            2 :             .await
    7612            2 :             .unwrap();
    7613            2 : 
    7614            2 :         {
    7615            2 :             lsn += 8;
    7616            2 :             let mut modification = tline.begin_modification(lsn);
    7617            2 :             modification
    7618            2 :                 .put_file("pg_logical/mappings/test1", b"first", &ctx)
    7619            2 :                 .await
    7620            2 :                 .unwrap();
    7621            2 :             modification.commit(&ctx).await.unwrap();
    7622            2 :         }
    7623            2 : 
    7624            2 :         // we can read everything from the storage
    7625            2 :         let files = tline.list_aux_files(lsn, &ctx).await.unwrap();
    7626            2 :         assert_eq!(
    7627            2 :             files.get("pg_logical/mappings/test1"),
    7628            2 :             Some(&bytes::Bytes::from_static(b"first"))
    7629            2 :         );
    7630            2 : 
    7631            2 :         {
    7632            2 :             lsn += 8;
    7633            2 :             let mut modification = tline.begin_modification(lsn);
    7634            2 :             modification
    7635            2 :                 .put_file("pg_logical/mappings/test2", b"second", &ctx)
    7636            2 :                 .await
    7637            2 :                 .unwrap();
    7638            2 :             modification.commit(&ctx).await.unwrap();
    7639            2 :         }
    7640            2 : 
    7641            2 :         let files = tline.list_aux_files(lsn, &ctx).await.unwrap();
    7642            2 :         assert_eq!(
    7643            2 :             files.get("pg_logical/mappings/test2"),
    7644            2 :             Some(&bytes::Bytes::from_static(b"second"))
    7645            2 :         );
    7646            2 : 
    7647            2 :         let child = tenant
    7648            2 :             .branch_timeline_test(&tline, NEW_TIMELINE_ID, Some(lsn), &ctx)
    7649            2 :             .await
    7650            2 :             .unwrap();
    7651            2 : 
    7652            2 :         let files = child.list_aux_files(lsn, &ctx).await.unwrap();
    7653            2 :         assert_eq!(files.get("pg_logical/mappings/test1"), None);
    7654            2 :         assert_eq!(files.get("pg_logical/mappings/test2"), None);
    7655            2 :     }
    7656              : 
    7657              :     #[tokio::test]
    7658            2 :     async fn test_metadata_image_creation() -> anyhow::Result<()> {
    7659            2 :         let harness = TenantHarness::create("test_metadata_image_creation").await?;
    7660            2 :         let (tenant, ctx) = harness.load().await;
    7661            2 :         let tline = tenant
    7662            2 :             .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx)
    7663            2 :             .await?;
    7664            2 : 
    7665            2 :         const NUM_KEYS: usize = 1000;
    7666            2 :         const STEP: usize = 10000; // random update + scan base_key + idx * STEP
    7667            2 : 
    7668            2 :         let cancel = CancellationToken::new();
    7669            2 : 
    7670            2 :         let base_key = Key::from_hex("620000000033333333444444445500000000").unwrap();
    7671            2 :         assert_eq!(base_key.field1, AUX_KEY_PREFIX); // in case someone accidentally changed the prefix...
    7672            2 :         let mut test_key = base_key;
    7673            2 :         let mut lsn = Lsn(0x10);
    7674            2 : 
    7675            8 :         async fn scan_with_statistics(
    7676            8 :             tline: &Timeline,
    7677            8 :             keyspace: &KeySpace,
    7678            8 :             lsn: Lsn,
    7679            8 :             ctx: &RequestContext,
    7680            8 :         ) -> anyhow::Result<(BTreeMap<Key, Result<Bytes, PageReconstructError>>, usize)> {
    7681            8 :             let mut reconstruct_state = ValuesReconstructState::default();
    7682            8 :             let res = tline
    7683            8 :                 .get_vectored_impl(keyspace.clone(), lsn, &mut reconstruct_state, ctx)
    7684            8 :                 .await?;
    7685            8 :             Ok((res, reconstruct_state.get_delta_layers_visited() as usize))
    7686            8 :         }
    7687            2 : 
    7688            2 :         #[allow(clippy::needless_range_loop)]
    7689         2002 :         for blknum in 0..NUM_KEYS {
    7690         2000 :             lsn = Lsn(lsn.0 + 0x10);
    7691         2000 :             test_key.field6 = (blknum * STEP) as u32;
    7692         2000 :             let mut writer = tline.writer().await;
    7693         2000 :             writer
    7694         2000 :                 .put(
    7695         2000 :                     test_key,
    7696         2000 :                     lsn,
    7697         2000 :                     &Value::Image(test_img(&format!("{} at {}", blknum, lsn))),
    7698         2000 :                     &ctx,
    7699         2000 :                 )
    7700         2000 :                 .await?;
    7701         2000 :             writer.finish_write(lsn);
    7702         2000 :             drop(writer);
    7703            2 :         }
    7704            2 : 
    7705            2 :         let keyspace = KeySpace::single(base_key..base_key.add((NUM_KEYS * STEP) as u32));
    7706            2 : 
    7707           22 :         for iter in 1..=10 {
    7708        20020 :             for _ in 0..NUM_KEYS {
    7709        20000 :                 lsn = Lsn(lsn.0 + 0x10);
    7710        20000 :                 let blknum = thread_rng().gen_range(0..NUM_KEYS);
    7711        20000 :                 test_key.field6 = (blknum * STEP) as u32;
    7712        20000 :                 let mut writer = tline.writer().await;
    7713        20000 :                 writer
    7714        20000 :                     .put(
    7715        20000 :                         test_key,
    7716        20000 :                         lsn,
    7717        20000 :                         &Value::Image(test_img(&format!("{} at {}", blknum, lsn))),
    7718        20000 :                         &ctx,
    7719        20000 :                     )
    7720        20000 :                     .await?;
    7721        20000 :                 writer.finish_write(lsn);
    7722        20000 :                 drop(writer);
    7723            2 :             }
    7724            2 : 
    7725           20 :             tline.freeze_and_flush().await?;
    7726            2 : 
    7727           20 :             if iter % 5 == 0 {
    7728            4 :                 let (_, before_delta_file_accessed) =
    7729            4 :                     scan_with_statistics(&tline, &keyspace, lsn, &ctx).await?;
    7730            4 :                 tline
    7731            4 :                     .compact(
    7732            4 :                         &cancel,
    7733            4 :                         {
    7734            4 :                             let mut flags = EnumSet::new();
    7735            4 :                             flags.insert(CompactFlags::ForceImageLayerCreation);
    7736            4 :                             flags.insert(CompactFlags::ForceRepartition);
    7737            4 :                             flags
    7738            4 :                         },
    7739            4 :                         &ctx,
    7740            4 :                     )
    7741            4 :                     .await?;
    7742            4 :                 let (_, after_delta_file_accessed) =
    7743            4 :                     scan_with_statistics(&tline, &keyspace, lsn, &ctx).await?;
    7744            4 :                 assert!(after_delta_file_accessed < before_delta_file_accessed, "after_delta_file_accessed={after_delta_file_accessed}, before_delta_file_accessed={before_delta_file_accessed}");
    7745            2 :                 // Given that we already produced an image layer, there should be no delta layer needed for the scan, but still setting a low threshold there for unforeseen circumstances.
    7746            4 :                 assert!(
    7747            4 :                     after_delta_file_accessed <= 2,
    7748            2 :                     "after_delta_file_accessed={after_delta_file_accessed}"
    7749            2 :                 );
    7750           16 :             }
    7751            2 :         }
    7752            2 : 
    7753            2 :         Ok(())
    7754            2 :     }
    7755              : 
    7756              :     #[tokio::test]
    7757            2 :     async fn test_vectored_missing_data_key_reads() -> anyhow::Result<()> {
    7758            2 :         let harness = TenantHarness::create("test_vectored_missing_data_key_reads").await?;
    7759            2 :         let (tenant, ctx) = harness.load().await;
    7760            2 : 
    7761            2 :         let base_key = Key::from_hex("000000000033333333444444445500000000").unwrap();
    7762            2 :         let base_key_child = Key::from_hex("000000000033333333444444445500000001").unwrap();
    7763            2 :         let base_key_nonexist = Key::from_hex("000000000033333333444444445500000002").unwrap();
    7764            2 : 
    7765            2 :         let tline = tenant
    7766            2 :             .create_test_timeline_with_layers(
    7767            2 :                 TIMELINE_ID,
    7768            2 :                 Lsn(0x10),
    7769            2 :                 DEFAULT_PG_VERSION,
    7770            2 :                 &ctx,
    7771            2 :                 Vec::new(), // delta layers
    7772            2 :                 vec![(Lsn(0x20), vec![(base_key, test_img("data key 1"))])], // image layers
    7773            2 :                 Lsn(0x20), // it's fine to not advance LSN to 0x30 while using 0x30 to get below because `get_vectored_impl` does not wait for LSN
    7774            2 :             )
    7775            2 :             .await?;
    7776            2 :         tline.add_extra_test_dense_keyspace(KeySpace::single(base_key..(base_key_nonexist.next())));
    7777            2 : 
    7778            2 :         let child = tenant
    7779            2 :             .branch_timeline_test_with_layers(
    7780            2 :                 &tline,
    7781            2 :                 NEW_TIMELINE_ID,
    7782            2 :                 Some(Lsn(0x20)),
    7783            2 :                 &ctx,
    7784            2 :                 Vec::new(), // delta layers
    7785            2 :                 vec![(Lsn(0x30), vec![(base_key_child, test_img("data key 2"))])], // image layers
    7786            2 :                 Lsn(0x30),
    7787            2 :             )
    7788            2 :             .await
    7789            2 :             .unwrap();
    7790            2 : 
    7791            2 :         let lsn = Lsn(0x30);
    7792            2 : 
    7793            2 :         // test vectored get on parent timeline
    7794            2 :         assert_eq!(
    7795            2 :             get_vectored_impl_wrapper(&tline, base_key, lsn, &ctx).await?,
    7796            2 :             Some(test_img("data key 1"))
    7797            2 :         );
    7798            2 :         assert!(get_vectored_impl_wrapper(&tline, base_key_child, lsn, &ctx)
    7799            2 :             .await
    7800            2 :             .unwrap_err()
    7801            2 :             .is_missing_key_error());
    7802            2 :         assert!(
    7803            2 :             get_vectored_impl_wrapper(&tline, base_key_nonexist, lsn, &ctx)
    7804            2 :                 .await
    7805            2 :                 .unwrap_err()
    7806            2 :                 .is_missing_key_error()
    7807            2 :         );
    7808            2 : 
    7809            2 :         // test vectored get on child timeline
    7810            2 :         assert_eq!(
    7811            2 :             get_vectored_impl_wrapper(&child, base_key, lsn, &ctx).await?,
    7812            2 :             Some(test_img("data key 1"))
    7813            2 :         );
    7814            2 :         assert_eq!(
    7815            2 :             get_vectored_impl_wrapper(&child, base_key_child, lsn, &ctx).await?,
    7816            2 :             Some(test_img("data key 2"))
    7817            2 :         );
    7818            2 :         assert!(
    7819            2 :             get_vectored_impl_wrapper(&child, base_key_nonexist, lsn, &ctx)
    7820            2 :                 .await
    7821            2 :                 .unwrap_err()
    7822            2 :                 .is_missing_key_error()
    7823            2 :         );
    7824            2 : 
    7825            2 :         Ok(())
    7826            2 :     }
    7827              : 
    7828              :     #[tokio::test]
    7829            2 :     async fn test_vectored_missing_metadata_key_reads() -> anyhow::Result<()> {
    7830            2 :         let harness = TenantHarness::create("test_vectored_missing_metadata_key_reads").await?;
    7831            2 :         let (tenant, ctx) = harness.load().await;
    7832            2 : 
    7833            2 :         let base_key = Key::from_hex("620000000033333333444444445500000000").unwrap();
    7834            2 :         let base_key_child = Key::from_hex("620000000033333333444444445500000001").unwrap();
    7835            2 :         let base_key_nonexist = Key::from_hex("620000000033333333444444445500000002").unwrap();
    7836            2 :         assert_eq!(base_key.field1, AUX_KEY_PREFIX); // in case someone accidentally changed the prefix...
    7837            2 : 
    7838            2 :         let tline = tenant
    7839            2 :             .create_test_timeline_with_layers(
    7840            2 :                 TIMELINE_ID,
    7841            2 :                 Lsn(0x10),
    7842            2 :                 DEFAULT_PG_VERSION,
    7843            2 :                 &ctx,
    7844            2 :                 Vec::new(), // delta layers
    7845            2 :                 vec![(Lsn(0x20), vec![(base_key, test_img("metadata key 1"))])], // image layers
    7846            2 :                 Lsn(0x20), // it's fine to not advance LSN to 0x30 while using 0x30 to get below because `get_vectored_impl` does not wait for LSN
    7847            2 :             )
    7848            2 :             .await?;
    7849            2 : 
    7850            2 :         let child = tenant
    7851            2 :             .branch_timeline_test_with_layers(
    7852            2 :                 &tline,
    7853            2 :                 NEW_TIMELINE_ID,
    7854            2 :                 Some(Lsn(0x20)),
    7855            2 :                 &ctx,
    7856            2 :                 Vec::new(), // delta layers
    7857            2 :                 vec![(
    7858            2 :                     Lsn(0x30),
    7859            2 :                     vec![(base_key_child, test_img("metadata key 2"))],
    7860            2 :                 )], // image layers
    7861            2 :                 Lsn(0x30),
    7862            2 :             )
    7863            2 :             .await
    7864            2 :             .unwrap();
    7865            2 : 
    7866            2 :         let lsn = Lsn(0x30);
    7867            2 : 
    7868            2 :         // test vectored get on parent timeline
    7869            2 :         assert_eq!(
    7870            2 :             get_vectored_impl_wrapper(&tline, base_key, lsn, &ctx).await?,
    7871            2 :             Some(test_img("metadata key 1"))
    7872            2 :         );
    7873            2 :         assert_eq!(
    7874            2 :             get_vectored_impl_wrapper(&tline, base_key_child, lsn, &ctx).await?,
    7875            2 :             None
    7876            2 :         );
    7877            2 :         assert_eq!(
    7878            2 :             get_vectored_impl_wrapper(&tline, base_key_nonexist, lsn, &ctx).await?,
    7879            2 :             None
    7880            2 :         );
    7881            2 : 
    7882            2 :         // test vectored get on child timeline
    7883            2 :         assert_eq!(
    7884            2 :             get_vectored_impl_wrapper(&child, base_key, lsn, &ctx).await?,
    7885            2 :             None
    7886            2 :         );
    7887            2 :         assert_eq!(
    7888            2 :             get_vectored_impl_wrapper(&child, base_key_child, lsn, &ctx).await?,
    7889            2 :             Some(test_img("metadata key 2"))
    7890            2 :         );
    7891            2 :         assert_eq!(
    7892            2 :             get_vectored_impl_wrapper(&child, base_key_nonexist, lsn, &ctx).await?,
    7893            2 :             None
    7894            2 :         );
    7895            2 : 
    7896            2 :         Ok(())
    7897            2 :     }
    7898              : 
    7899           36 :     async fn get_vectored_impl_wrapper(
    7900           36 :         tline: &Arc<Timeline>,
    7901           36 :         key: Key,
    7902           36 :         lsn: Lsn,
    7903           36 :         ctx: &RequestContext,
    7904           36 :     ) -> Result<Option<Bytes>, GetVectoredError> {
    7905           36 :         let mut reconstruct_state = ValuesReconstructState::new();
    7906           36 :         let mut res = tline
    7907           36 :             .get_vectored_impl(
    7908           36 :                 KeySpace::single(key..key.next()),
    7909           36 :                 lsn,
    7910           36 :                 &mut reconstruct_state,
    7911           36 :                 ctx,
    7912           36 :             )
    7913           36 :             .await?;
    7914           30 :         Ok(res.pop_last().map(|(k, v)| {
    7915           18 :             assert_eq!(k, key);
    7916           18 :             v.unwrap()
    7917           30 :         }))
    7918           36 :     }
    7919              : 
    7920              :     #[tokio::test]
    7921            2 :     async fn test_metadata_tombstone_reads() -> anyhow::Result<()> {
    7922            2 :         let harness = TenantHarness::create("test_metadata_tombstone_reads").await?;
    7923            2 :         let (tenant, ctx) = harness.load().await;
    7924            2 :         let key0 = Key::from_hex("620000000033333333444444445500000000").unwrap();
    7925            2 :         let key1 = Key::from_hex("620000000033333333444444445500000001").unwrap();
    7926            2 :         let key2 = Key::from_hex("620000000033333333444444445500000002").unwrap();
    7927            2 :         let key3 = Key::from_hex("620000000033333333444444445500000003").unwrap();
    7928            2 : 
    7929            2 :         // We emulate the situation that the compaction algorithm creates an image layer that removes the tombstones
    7930            2 :         // Lsn 0x30 key0, key3, no key1+key2
    7931            2 :         // Lsn 0x20 key1+key2 tomestones
    7932            2 :         // Lsn 0x10 key1 in image, key2 in delta
    7933            2 :         let tline = tenant
    7934            2 :             .create_test_timeline_with_layers(
    7935            2 :                 TIMELINE_ID,
    7936            2 :                 Lsn(0x10),
    7937            2 :                 DEFAULT_PG_VERSION,
    7938            2 :                 &ctx,
    7939            2 :                 // delta layers
    7940            2 :                 vec![
    7941            2 :                     DeltaLayerTestDesc::new_with_inferred_key_range(
    7942            2 :                         Lsn(0x10)..Lsn(0x20),
    7943            2 :                         vec![(key2, Lsn(0x10), Value::Image(test_img("metadata key 2")))],
    7944            2 :                     ),
    7945            2 :                     DeltaLayerTestDesc::new_with_inferred_key_range(
    7946            2 :                         Lsn(0x20)..Lsn(0x30),
    7947            2 :                         vec![(key1, Lsn(0x20), Value::Image(Bytes::new()))],
    7948            2 :                     ),
    7949            2 :                     DeltaLayerTestDesc::new_with_inferred_key_range(
    7950            2 :                         Lsn(0x20)..Lsn(0x30),
    7951            2 :                         vec![(key2, Lsn(0x20), Value::Image(Bytes::new()))],
    7952            2 :                     ),
    7953            2 :                 ],
    7954            2 :                 // image layers
    7955            2 :                 vec![
    7956            2 :                     (Lsn(0x10), vec![(key1, test_img("metadata key 1"))]),
    7957            2 :                     (
    7958            2 :                         Lsn(0x30),
    7959            2 :                         vec![
    7960            2 :                             (key0, test_img("metadata key 0")),
    7961            2 :                             (key3, test_img("metadata key 3")),
    7962            2 :                         ],
    7963            2 :                     ),
    7964            2 :                 ],
    7965            2 :                 Lsn(0x30),
    7966            2 :             )
    7967            2 :             .await?;
    7968            2 : 
    7969            2 :         let lsn = Lsn(0x30);
    7970            2 :         let old_lsn = Lsn(0x20);
    7971            2 : 
    7972            2 :         assert_eq!(
    7973            2 :             get_vectored_impl_wrapper(&tline, key0, lsn, &ctx).await?,
    7974            2 :             Some(test_img("metadata key 0"))
    7975            2 :         );
    7976            2 :         assert_eq!(
    7977            2 :             get_vectored_impl_wrapper(&tline, key1, lsn, &ctx).await?,
    7978            2 :             None,
    7979            2 :         );
    7980            2 :         assert_eq!(
    7981            2 :             get_vectored_impl_wrapper(&tline, key2, lsn, &ctx).await?,
    7982            2 :             None,
    7983            2 :         );
    7984            2 :         assert_eq!(
    7985            2 :             get_vectored_impl_wrapper(&tline, key1, old_lsn, &ctx).await?,
    7986            2 :             Some(Bytes::new()),
    7987            2 :         );
    7988            2 :         assert_eq!(
    7989            2 :             get_vectored_impl_wrapper(&tline, key2, old_lsn, &ctx).await?,
    7990            2 :             Some(Bytes::new()),
    7991            2 :         );
    7992            2 :         assert_eq!(
    7993            2 :             get_vectored_impl_wrapper(&tline, key3, lsn, &ctx).await?,
    7994            2 :             Some(test_img("metadata key 3"))
    7995            2 :         );
    7996            2 : 
    7997            2 :         Ok(())
    7998            2 :     }
    7999              : 
    8000              :     #[tokio::test]
    8001            2 :     async fn test_metadata_tombstone_image_creation() {
    8002            2 :         let harness = TenantHarness::create("test_metadata_tombstone_image_creation")
    8003            2 :             .await
    8004            2 :             .unwrap();
    8005            2 :         let (tenant, ctx) = harness.load().await;
    8006            2 : 
    8007            2 :         let key0 = Key::from_hex("620000000033333333444444445500000000").unwrap();
    8008            2 :         let key1 = Key::from_hex("620000000033333333444444445500000001").unwrap();
    8009            2 :         let key2 = Key::from_hex("620000000033333333444444445500000002").unwrap();
    8010            2 :         let key3 = Key::from_hex("620000000033333333444444445500000003").unwrap();
    8011            2 : 
    8012            2 :         let tline = tenant
    8013            2 :             .create_test_timeline_with_layers(
    8014            2 :                 TIMELINE_ID,
    8015            2 :                 Lsn(0x10),
    8016            2 :                 DEFAULT_PG_VERSION,
    8017            2 :                 &ctx,
    8018            2 :                 // delta layers
    8019            2 :                 vec![
    8020            2 :                     DeltaLayerTestDesc::new_with_inferred_key_range(
    8021            2 :                         Lsn(0x10)..Lsn(0x20),
    8022            2 :                         vec![(key2, Lsn(0x10), Value::Image(test_img("metadata key 2")))],
    8023            2 :                     ),
    8024            2 :                     DeltaLayerTestDesc::new_with_inferred_key_range(
    8025            2 :                         Lsn(0x20)..Lsn(0x30),
    8026            2 :                         vec![(key1, Lsn(0x20), Value::Image(Bytes::new()))],
    8027            2 :                     ),
    8028            2 :                     DeltaLayerTestDesc::new_with_inferred_key_range(
    8029            2 :                         Lsn(0x20)..Lsn(0x30),
    8030            2 :                         vec![(key2, Lsn(0x20), Value::Image(Bytes::new()))],
    8031            2 :                     ),
    8032            2 :                     DeltaLayerTestDesc::new_with_inferred_key_range(
    8033            2 :                         Lsn(0x30)..Lsn(0x40),
    8034            2 :                         vec![
    8035            2 :                             (key0, Lsn(0x30), Value::Image(test_img("metadata key 0"))),
    8036            2 :                             (key3, Lsn(0x30), Value::Image(test_img("metadata key 3"))),
    8037            2 :                         ],
    8038            2 :                     ),
    8039            2 :                 ],
    8040            2 :                 // image layers
    8041            2 :                 vec![(Lsn(0x10), vec![(key1, test_img("metadata key 1"))])],
    8042            2 :                 Lsn(0x40),
    8043            2 :             )
    8044            2 :             .await
    8045            2 :             .unwrap();
    8046            2 : 
    8047            2 :         let cancel = CancellationToken::new();
    8048            2 : 
    8049            2 :         tline
    8050            2 :             .compact(
    8051            2 :                 &cancel,
    8052            2 :                 {
    8053            2 :                     let mut flags = EnumSet::new();
    8054            2 :                     flags.insert(CompactFlags::ForceImageLayerCreation);
    8055            2 :                     flags.insert(CompactFlags::ForceRepartition);
    8056            2 :                     flags
    8057            2 :                 },
    8058            2 :                 &ctx,
    8059            2 :             )
    8060            2 :             .await
    8061            2 :             .unwrap();
    8062            2 : 
    8063            2 :         // Image layers are created at last_record_lsn
    8064            2 :         let images = tline
    8065            2 :             .inspect_image_layers(Lsn(0x40), &ctx)
    8066            2 :             .await
    8067            2 :             .unwrap()
    8068            2 :             .into_iter()
    8069           18 :             .filter(|(k, _)| k.is_metadata_key())
    8070            2 :             .collect::<Vec<_>>();
    8071            2 :         assert_eq!(images.len(), 2); // the image layer should only contain two existing keys, tombstones should be removed.
    8072            2 :     }
    8073              : 
    8074              :     #[tokio::test]
    8075            2 :     async fn test_metadata_tombstone_empty_image_creation() {
    8076            2 :         let harness = TenantHarness::create("test_metadata_tombstone_empty_image_creation")
    8077            2 :             .await
    8078            2 :             .unwrap();
    8079            2 :         let (tenant, ctx) = harness.load().await;
    8080            2 : 
    8081            2 :         let key1 = Key::from_hex("620000000033333333444444445500000001").unwrap();
    8082            2 :         let key2 = Key::from_hex("620000000033333333444444445500000002").unwrap();
    8083            2 : 
    8084            2 :         let tline = tenant
    8085            2 :             .create_test_timeline_with_layers(
    8086            2 :                 TIMELINE_ID,
    8087            2 :                 Lsn(0x10),
    8088            2 :                 DEFAULT_PG_VERSION,
    8089            2 :                 &ctx,
    8090            2 :                 // delta layers
    8091            2 :                 vec![
    8092            2 :                     DeltaLayerTestDesc::new_with_inferred_key_range(
    8093            2 :                         Lsn(0x10)..Lsn(0x20),
    8094            2 :                         vec![(key2, Lsn(0x10), Value::Image(test_img("metadata key 2")))],
    8095            2 :                     ),
    8096            2 :                     DeltaLayerTestDesc::new_with_inferred_key_range(
    8097            2 :                         Lsn(0x20)..Lsn(0x30),
    8098            2 :                         vec![(key1, Lsn(0x20), Value::Image(Bytes::new()))],
    8099            2 :                     ),
    8100            2 :                     DeltaLayerTestDesc::new_with_inferred_key_range(
    8101            2 :                         Lsn(0x20)..Lsn(0x30),
    8102            2 :                         vec![(key2, Lsn(0x20), Value::Image(Bytes::new()))],
    8103            2 :                     ),
    8104            2 :                 ],
    8105            2 :                 // image layers
    8106            2 :                 vec![(Lsn(0x10), vec![(key1, test_img("metadata key 1"))])],
    8107            2 :                 Lsn(0x30),
    8108            2 :             )
    8109            2 :             .await
    8110            2 :             .unwrap();
    8111            2 : 
    8112            2 :         let cancel = CancellationToken::new();
    8113            2 : 
    8114            2 :         tline
    8115            2 :             .compact(
    8116            2 :                 &cancel,
    8117            2 :                 {
    8118            2 :                     let mut flags = EnumSet::new();
    8119            2 :                     flags.insert(CompactFlags::ForceImageLayerCreation);
    8120            2 :                     flags.insert(CompactFlags::ForceRepartition);
    8121            2 :                     flags
    8122            2 :                 },
    8123            2 :                 &ctx,
    8124            2 :             )
    8125            2 :             .await
    8126            2 :             .unwrap();
    8127            2 : 
    8128            2 :         // Image layers are created at last_record_lsn
    8129            2 :         let images = tline
    8130            2 :             .inspect_image_layers(Lsn(0x30), &ctx)
    8131            2 :             .await
    8132            2 :             .unwrap()
    8133            2 :             .into_iter()
    8134           14 :             .filter(|(k, _)| k.is_metadata_key())
    8135            2 :             .collect::<Vec<_>>();
    8136            2 :         assert_eq!(images.len(), 0); // the image layer should not contain tombstones, or it is not created
    8137            2 :     }
    8138              : 
    8139              :     #[tokio::test]
    8140            2 :     async fn test_simple_bottom_most_compaction_images() -> anyhow::Result<()> {
    8141            2 :         let harness = TenantHarness::create("test_simple_bottom_most_compaction_images").await?;
    8142            2 :         let (tenant, ctx) = harness.load().await;
    8143            2 : 
    8144          102 :         fn get_key(id: u32) -> Key {
    8145          102 :             // using aux key here b/c they are guaranteed to be inside `collect_keyspace`.
    8146          102 :             let mut key = Key::from_hex("620000000033333333444444445500000000").unwrap();
    8147          102 :             key.field6 = id;
    8148          102 :             key
    8149          102 :         }
    8150            2 : 
    8151            2 :         // We create
    8152            2 :         // - one bottom-most image layer,
    8153            2 :         // - a delta layer D1 crossing the GC horizon with data below and above the horizon,
    8154            2 :         // - a delta layer D2 crossing the GC horizon with data only below the horizon,
    8155            2 :         // - a delta layer D3 above the horizon.
    8156            2 :         //
    8157            2 :         //                             | D3 |
    8158            2 :         //  | D1 |
    8159            2 :         // -|    |-- gc horizon -----------------
    8160            2 :         //  |    |                | D2 |
    8161            2 :         // --------- img layer ------------------
    8162            2 :         //
    8163            2 :         // What we should expact from this compaction is:
    8164            2 :         //                             | D3 |
    8165            2 :         //  | Part of D1 |
    8166            2 :         // --------- img layer with D1+D2 at GC horizon------------------
    8167            2 : 
    8168            2 :         // img layer at 0x10
    8169            2 :         let img_layer = (0..10)
    8170           20 :             .map(|id| (get_key(id), Bytes::from(format!("value {id}@0x10"))))
    8171            2 :             .collect_vec();
    8172            2 : 
    8173            2 :         let delta1 = vec![
    8174            2 :             (
    8175            2 :                 get_key(1),
    8176            2 :                 Lsn(0x20),
    8177            2 :                 Value::Image(Bytes::from("value 1@0x20")),
    8178            2 :             ),
    8179            2 :             (
    8180            2 :                 get_key(2),
    8181            2 :                 Lsn(0x30),
    8182            2 :                 Value::Image(Bytes::from("value 2@0x30")),
    8183            2 :             ),
    8184            2 :             (
    8185            2 :                 get_key(3),
    8186            2 :                 Lsn(0x40),
    8187            2 :                 Value::Image(Bytes::from("value 3@0x40")),
    8188            2 :             ),
    8189            2 :         ];
    8190            2 :         let delta2 = vec![
    8191            2 :             (
    8192            2 :                 get_key(5),
    8193            2 :                 Lsn(0x20),
    8194            2 :                 Value::Image(Bytes::from("value 5@0x20")),
    8195            2 :             ),
    8196            2 :             (
    8197            2 :                 get_key(6),
    8198            2 :                 Lsn(0x20),
    8199            2 :                 Value::Image(Bytes::from("value 6@0x20")),
    8200            2 :             ),
    8201            2 :         ];
    8202            2 :         let delta3 = vec![
    8203            2 :             (
    8204            2 :                 get_key(8),
    8205            2 :                 Lsn(0x48),
    8206            2 :                 Value::Image(Bytes::from("value 8@0x48")),
    8207            2 :             ),
    8208            2 :             (
    8209            2 :                 get_key(9),
    8210            2 :                 Lsn(0x48),
    8211            2 :                 Value::Image(Bytes::from("value 9@0x48")),
    8212            2 :             ),
    8213            2 :         ];
    8214            2 : 
    8215            2 :         let tline = tenant
    8216            2 :             .create_test_timeline_with_layers(
    8217            2 :                 TIMELINE_ID,
    8218            2 :                 Lsn(0x10),
    8219            2 :                 DEFAULT_PG_VERSION,
    8220            2 :                 &ctx,
    8221            2 :                 vec![
    8222            2 :                     DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x20)..Lsn(0x48), delta1),
    8223            2 :                     DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x20)..Lsn(0x48), delta2),
    8224            2 :                     DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x48)..Lsn(0x50), delta3),
    8225            2 :                 ], // delta layers
    8226            2 :                 vec![(Lsn(0x10), img_layer)], // image layers
    8227            2 :                 Lsn(0x50),
    8228            2 :             )
    8229            2 :             .await?;
    8230            2 :         {
    8231            2 :             tline
    8232            2 :                 .latest_gc_cutoff_lsn
    8233            2 :                 .lock_for_write()
    8234            2 :                 .store_and_unlock(Lsn(0x30))
    8235            2 :                 .wait()
    8236            2 :                 .await;
    8237            2 :             // Update GC info
    8238            2 :             let mut guard = tline.gc_info.write().unwrap();
    8239            2 :             guard.cutoffs.time = Lsn(0x30);
    8240            2 :             guard.cutoffs.space = Lsn(0x30);
    8241            2 :         }
    8242            2 : 
    8243            2 :         let expected_result = [
    8244            2 :             Bytes::from_static(b"value 0@0x10"),
    8245            2 :             Bytes::from_static(b"value 1@0x20"),
    8246            2 :             Bytes::from_static(b"value 2@0x30"),
    8247            2 :             Bytes::from_static(b"value 3@0x40"),
    8248            2 :             Bytes::from_static(b"value 4@0x10"),
    8249            2 :             Bytes::from_static(b"value 5@0x20"),
    8250            2 :             Bytes::from_static(b"value 6@0x20"),
    8251            2 :             Bytes::from_static(b"value 7@0x10"),
    8252            2 :             Bytes::from_static(b"value 8@0x48"),
    8253            2 :             Bytes::from_static(b"value 9@0x48"),
    8254            2 :         ];
    8255            2 : 
    8256           20 :         for (idx, expected) in expected_result.iter().enumerate() {
    8257           20 :             assert_eq!(
    8258           20 :                 tline
    8259           20 :                     .get(get_key(idx as u32), Lsn(0x50), &ctx)
    8260           20 :                     .await
    8261           20 :                     .unwrap(),
    8262            2 :                 expected
    8263            2 :             );
    8264            2 :         }
    8265            2 : 
    8266            2 :         let cancel = CancellationToken::new();
    8267            2 :         tline
    8268            2 :             .compact_with_gc(&cancel, CompactOptions::default(), &ctx)
    8269            2 :             .await
    8270            2 :             .unwrap();
    8271            2 : 
    8272           20 :         for (idx, expected) in expected_result.iter().enumerate() {
    8273           20 :             assert_eq!(
    8274           20 :                 tline
    8275           20 :                     .get(get_key(idx as u32), Lsn(0x50), &ctx)
    8276           20 :                     .await
    8277           20 :                     .unwrap(),
    8278            2 :                 expected
    8279            2 :             );
    8280            2 :         }
    8281            2 : 
    8282            2 :         // Check if the image layer at the GC horizon contains exactly what we want
    8283            2 :         let image_at_gc_horizon = tline
    8284            2 :             .inspect_image_layers(Lsn(0x30), &ctx)
    8285            2 :             .await
    8286            2 :             .unwrap()
    8287            2 :             .into_iter()
    8288           34 :             .filter(|(k, _)| k.is_metadata_key())
    8289            2 :             .collect::<Vec<_>>();
    8290            2 : 
    8291            2 :         assert_eq!(image_at_gc_horizon.len(), 10);
    8292            2 :         let expected_result = [
    8293            2 :             Bytes::from_static(b"value 0@0x10"),
    8294            2 :             Bytes::from_static(b"value 1@0x20"),
    8295            2 :             Bytes::from_static(b"value 2@0x30"),
    8296            2 :             Bytes::from_static(b"value 3@0x10"),
    8297            2 :             Bytes::from_static(b"value 4@0x10"),
    8298            2 :             Bytes::from_static(b"value 5@0x20"),
    8299            2 :             Bytes::from_static(b"value 6@0x20"),
    8300            2 :             Bytes::from_static(b"value 7@0x10"),
    8301            2 :             Bytes::from_static(b"value 8@0x10"),
    8302            2 :             Bytes::from_static(b"value 9@0x10"),
    8303            2 :         ];
    8304           22 :         for idx in 0..10 {
    8305           20 :             assert_eq!(
    8306           20 :                 image_at_gc_horizon[idx],
    8307           20 :                 (get_key(idx as u32), expected_result[idx].clone())
    8308           20 :             );
    8309            2 :         }
    8310            2 : 
    8311            2 :         // Check if old layers are removed / new layers have the expected LSN
    8312            2 :         let all_layers = inspect_and_sort(&tline, None).await;
    8313            2 :         assert_eq!(
    8314            2 :             all_layers,
    8315            2 :             vec![
    8316            2 :                 // Image layer at GC horizon
    8317            2 :                 PersistentLayerKey {
    8318            2 :                     key_range: Key::MIN..Key::MAX,
    8319            2 :                     lsn_range: Lsn(0x30)..Lsn(0x31),
    8320            2 :                     is_delta: false
    8321            2 :                 },
    8322            2 :                 // The delta layer below the horizon
    8323            2 :                 PersistentLayerKey {
    8324            2 :                     key_range: get_key(3)..get_key(4),
    8325            2 :                     lsn_range: Lsn(0x30)..Lsn(0x48),
    8326            2 :                     is_delta: true
    8327            2 :                 },
    8328            2 :                 // The delta3 layer that should not be picked for the compaction
    8329            2 :                 PersistentLayerKey {
    8330            2 :                     key_range: get_key(8)..get_key(10),
    8331            2 :                     lsn_range: Lsn(0x48)..Lsn(0x50),
    8332            2 :                     is_delta: true
    8333            2 :                 }
    8334            2 :             ]
    8335            2 :         );
    8336            2 : 
    8337            2 :         // increase GC horizon and compact again
    8338            2 :         {
    8339            2 :             tline
    8340            2 :                 .latest_gc_cutoff_lsn
    8341            2 :                 .lock_for_write()
    8342            2 :                 .store_and_unlock(Lsn(0x40))
    8343            2 :                 .wait()
    8344            2 :                 .await;
    8345            2 :             // Update GC info
    8346            2 :             let mut guard = tline.gc_info.write().unwrap();
    8347            2 :             guard.cutoffs.time = Lsn(0x40);
    8348            2 :             guard.cutoffs.space = Lsn(0x40);
    8349            2 :         }
    8350            2 :         tline
    8351            2 :             .compact_with_gc(&cancel, CompactOptions::default(), &ctx)
    8352            2 :             .await
    8353            2 :             .unwrap();
    8354            2 : 
    8355            2 :         Ok(())
    8356            2 :     }
    8357              : 
    8358              :     #[cfg(feature = "testing")]
    8359              :     #[tokio::test]
    8360            2 :     async fn test_neon_test_record() -> anyhow::Result<()> {
    8361            2 :         let harness = TenantHarness::create("test_neon_test_record").await?;
    8362            2 :         let (tenant, ctx) = harness.load().await;
    8363            2 : 
    8364           24 :         fn get_key(id: u32) -> Key {
    8365           24 :             // using aux key here b/c they are guaranteed to be inside `collect_keyspace`.
    8366           24 :             let mut key = Key::from_hex("620000000033333333444444445500000000").unwrap();
    8367           24 :             key.field6 = id;
    8368           24 :             key
    8369           24 :         }
    8370            2 : 
    8371            2 :         let delta1 = vec![
    8372            2 :             (
    8373            2 :                 get_key(1),
    8374            2 :                 Lsn(0x20),
    8375            2 :                 Value::WalRecord(NeonWalRecord::wal_append(",0x20")),
    8376            2 :             ),
    8377            2 :             (
    8378            2 :                 get_key(1),
    8379            2 :                 Lsn(0x30),
    8380            2 :                 Value::WalRecord(NeonWalRecord::wal_append(",0x30")),
    8381            2 :             ),
    8382            2 :             (get_key(2), Lsn(0x10), Value::Image("0x10".into())),
    8383            2 :             (
    8384            2 :                 get_key(2),
    8385            2 :                 Lsn(0x20),
    8386            2 :                 Value::WalRecord(NeonWalRecord::wal_append(",0x20")),
    8387            2 :             ),
    8388            2 :             (
    8389            2 :                 get_key(2),
    8390            2 :                 Lsn(0x30),
    8391            2 :                 Value::WalRecord(NeonWalRecord::wal_append(",0x30")),
    8392            2 :             ),
    8393            2 :             (get_key(3), Lsn(0x10), Value::Image("0x10".into())),
    8394            2 :             (
    8395            2 :                 get_key(3),
    8396            2 :                 Lsn(0x20),
    8397            2 :                 Value::WalRecord(NeonWalRecord::wal_clear("c")),
    8398            2 :             ),
    8399            2 :             (get_key(4), Lsn(0x10), Value::Image("0x10".into())),
    8400            2 :             (
    8401            2 :                 get_key(4),
    8402            2 :                 Lsn(0x20),
    8403            2 :                 Value::WalRecord(NeonWalRecord::wal_init("i")),
    8404            2 :             ),
    8405            2 :         ];
    8406            2 :         let image1 = vec![(get_key(1), "0x10".into())];
    8407            2 : 
    8408            2 :         let tline = tenant
    8409            2 :             .create_test_timeline_with_layers(
    8410            2 :                 TIMELINE_ID,
    8411            2 :                 Lsn(0x10),
    8412            2 :                 DEFAULT_PG_VERSION,
    8413            2 :                 &ctx,
    8414            2 :                 vec![DeltaLayerTestDesc::new_with_inferred_key_range(
    8415            2 :                     Lsn(0x10)..Lsn(0x40),
    8416            2 :                     delta1,
    8417            2 :                 )], // delta layers
    8418            2 :                 vec![(Lsn(0x10), image1)], // image layers
    8419            2 :                 Lsn(0x50),
    8420            2 :             )
    8421            2 :             .await?;
    8422            2 : 
    8423            2 :         assert_eq!(
    8424            2 :             tline.get(get_key(1), Lsn(0x50), &ctx).await?,
    8425            2 :             Bytes::from_static(b"0x10,0x20,0x30")
    8426            2 :         );
    8427            2 :         assert_eq!(
    8428            2 :             tline.get(get_key(2), Lsn(0x50), &ctx).await?,
    8429            2 :             Bytes::from_static(b"0x10,0x20,0x30")
    8430            2 :         );
    8431            2 : 
    8432            2 :         // Need to remove the limit of "Neon WAL redo requires base image".
    8433            2 : 
    8434            2 :         // assert_eq!(tline.get(get_key(3), Lsn(0x50), &ctx).await?, Bytes::new());
    8435            2 :         // assert_eq!(tline.get(get_key(4), Lsn(0x50), &ctx).await?, Bytes::new());
    8436            2 : 
    8437            2 :         Ok(())
    8438            2 :     }
    8439              : 
    8440              :     #[tokio::test(start_paused = true)]
    8441            2 :     async fn test_lsn_lease() -> anyhow::Result<()> {
    8442            2 :         let (tenant, ctx) = TenantHarness::create("test_lsn_lease")
    8443            2 :             .await
    8444            2 :             .unwrap()
    8445            2 :             .load()
    8446            2 :             .await;
    8447            2 :         // Advance to the lsn lease deadline so that GC is not blocked by
    8448            2 :         // initial transition into AttachedSingle.
    8449            2 :         tokio::time::advance(tenant.get_lsn_lease_length()).await;
    8450            2 :         tokio::time::resume();
    8451            2 :         let key = Key::from_hex("010000000033333333444444445500000000").unwrap();
    8452            2 : 
    8453            2 :         let end_lsn = Lsn(0x100);
    8454            2 :         let image_layers = (0x20..=0x90)
    8455            2 :             .step_by(0x10)
    8456           16 :             .map(|n| {
    8457           16 :                 (
    8458           16 :                     Lsn(n),
    8459           16 :                     vec![(key, test_img(&format!("data key at {:x}", n)))],
    8460           16 :                 )
    8461           16 :             })
    8462            2 :             .collect();
    8463            2 : 
    8464            2 :         let timeline = tenant
    8465            2 :             .create_test_timeline_with_layers(
    8466            2 :                 TIMELINE_ID,
    8467            2 :                 Lsn(0x10),
    8468            2 :                 DEFAULT_PG_VERSION,
    8469            2 :                 &ctx,
    8470            2 :                 Vec::new(),
    8471            2 :                 image_layers,
    8472            2 :                 end_lsn,
    8473            2 :             )
    8474            2 :             .await?;
    8475            2 : 
    8476            2 :         let leased_lsns = [0x30, 0x50, 0x70];
    8477            2 :         let mut leases = Vec::new();
    8478            6 :         leased_lsns.iter().for_each(|n| {
    8479            6 :             leases.push(
    8480            6 :                 timeline
    8481            6 :                     .init_lsn_lease(Lsn(*n), timeline.get_lsn_lease_length(), &ctx)
    8482            6 :                     .expect("lease request should succeed"),
    8483            6 :             );
    8484            6 :         });
    8485            2 : 
    8486            2 :         let updated_lease_0 = timeline
    8487            2 :             .renew_lsn_lease(Lsn(leased_lsns[0]), Duration::from_secs(0), &ctx)
    8488            2 :             .expect("lease renewal should succeed");
    8489            2 :         assert_eq!(
    8490            2 :             updated_lease_0.valid_until, leases[0].valid_until,
    8491            2 :             " Renewing with shorter lease should not change the lease."
    8492            2 :         );
    8493            2 : 
    8494            2 :         let updated_lease_1 = timeline
    8495            2 :             .renew_lsn_lease(
    8496            2 :                 Lsn(leased_lsns[1]),
    8497            2 :                 timeline.get_lsn_lease_length() * 2,
    8498            2 :                 &ctx,
    8499            2 :             )
    8500            2 :             .expect("lease renewal should succeed");
    8501            2 :         assert!(
    8502            2 :             updated_lease_1.valid_until > leases[1].valid_until,
    8503            2 :             "Renewing with a long lease should renew lease with later expiration time."
    8504            2 :         );
    8505            2 : 
    8506            2 :         // Force set disk consistent lsn so we can get the cutoff at `end_lsn`.
    8507            2 :         info!(
    8508            2 :             "latest_gc_cutoff_lsn: {}",
    8509            0 :             *timeline.get_latest_gc_cutoff_lsn()
    8510            2 :         );
    8511            2 :         timeline.force_set_disk_consistent_lsn(end_lsn);
    8512            2 : 
    8513            2 :         let res = tenant
    8514            2 :             .gc_iteration(
    8515            2 :                 Some(TIMELINE_ID),
    8516            2 :                 0,
    8517            2 :                 Duration::ZERO,
    8518            2 :                 &CancellationToken::new(),
    8519            2 :                 &ctx,
    8520            2 :             )
    8521            2 :             .await
    8522            2 :             .unwrap();
    8523            2 : 
    8524            2 :         // Keeping everything <= Lsn(0x80) b/c leases:
    8525            2 :         // 0/10: initdb layer
    8526            2 :         // (0/20..=0/70).step_by(0x10): image layers added when creating the timeline.
    8527            2 :         assert_eq!(res.layers_needed_by_leases, 7);
    8528            2 :         // Keeping 0/90 b/c it is the latest layer.
    8529            2 :         assert_eq!(res.layers_not_updated, 1);
    8530            2 :         // Removed 0/80.
    8531            2 :         assert_eq!(res.layers_removed, 1);
    8532            2 : 
    8533            2 :         // Make lease on a already GC-ed LSN.
    8534            2 :         // 0/80 does not have a valid lease + is below latest_gc_cutoff
    8535            2 :         assert!(Lsn(0x80) < *timeline.get_latest_gc_cutoff_lsn());
    8536            2 :         timeline
    8537            2 :             .init_lsn_lease(Lsn(0x80), timeline.get_lsn_lease_length(), &ctx)
    8538            2 :             .expect_err("lease request on GC-ed LSN should fail");
    8539            2 : 
    8540            2 :         // Should still be able to renew a currently valid lease
    8541            2 :         // Assumption: original lease to is still valid for 0/50.
    8542            2 :         // (use `Timeline::init_lsn_lease` for testing so it always does validation)
    8543            2 :         timeline
    8544            2 :             .init_lsn_lease(Lsn(leased_lsns[1]), timeline.get_lsn_lease_length(), &ctx)
    8545            2 :             .expect("lease renewal with validation should succeed");
    8546            2 : 
    8547            2 :         Ok(())
    8548            2 :     }
    8549              : 
    8550              :     #[cfg(feature = "testing")]
    8551              :     #[tokio::test]
    8552            2 :     async fn test_simple_bottom_most_compaction_deltas_1() -> anyhow::Result<()> {
    8553            2 :         test_simple_bottom_most_compaction_deltas_helper(
    8554            2 :             "test_simple_bottom_most_compaction_deltas_1",
    8555            2 :             false,
    8556            2 :         )
    8557            2 :         .await
    8558            2 :     }
    8559              : 
    8560              :     #[cfg(feature = "testing")]
    8561              :     #[tokio::test]
    8562            2 :     async fn test_simple_bottom_most_compaction_deltas_2() -> anyhow::Result<()> {
    8563            2 :         test_simple_bottom_most_compaction_deltas_helper(
    8564            2 :             "test_simple_bottom_most_compaction_deltas_2",
    8565            2 :             true,
    8566            2 :         )
    8567            2 :         .await
    8568            2 :     }
    8569              : 
    8570              :     #[cfg(feature = "testing")]
    8571            4 :     async fn test_simple_bottom_most_compaction_deltas_helper(
    8572            4 :         test_name: &'static str,
    8573            4 :         use_delta_bottom_layer: bool,
    8574            4 :     ) -> anyhow::Result<()> {
    8575            4 :         let harness = TenantHarness::create(test_name).await?;
    8576            4 :         let (tenant, ctx) = harness.load().await;
    8577              : 
    8578          276 :         fn get_key(id: u32) -> Key {
    8579          276 :             // using aux key here b/c they are guaranteed to be inside `collect_keyspace`.
    8580          276 :             let mut key = Key::from_hex("620000000033333333444444445500000000").unwrap();
    8581          276 :             key.field6 = id;
    8582          276 :             key
    8583          276 :         }
    8584              : 
    8585              :         // We create
    8586              :         // - one bottom-most image layer,
    8587              :         // - a delta layer D1 crossing the GC horizon with data below and above the horizon,
    8588              :         // - a delta layer D2 crossing the GC horizon with data only below the horizon,
    8589              :         // - a delta layer D3 above the horizon.
    8590              :         //
    8591              :         //                             | D3 |
    8592              :         //  | D1 |
    8593              :         // -|    |-- gc horizon -----------------
    8594              :         //  |    |                | D2 |
    8595              :         // --------- img layer ------------------
    8596              :         //
    8597              :         // What we should expact from this compaction is:
    8598              :         //                             | D3 |
    8599              :         //  | Part of D1 |
    8600              :         // --------- img layer with D1+D2 at GC horizon------------------
    8601              : 
    8602              :         // img layer at 0x10
    8603            4 :         let img_layer = (0..10)
    8604           40 :             .map(|id| (get_key(id), Bytes::from(format!("value {id}@0x10"))))
    8605            4 :             .collect_vec();
    8606            4 :         // or, delta layer at 0x10 if `use_delta_bottom_layer` is true
    8607            4 :         let delta4 = (0..10)
    8608           40 :             .map(|id| {
    8609           40 :                 (
    8610           40 :                     get_key(id),
    8611           40 :                     Lsn(0x08),
    8612           40 :                     Value::WalRecord(NeonWalRecord::wal_init(format!("value {id}@0x10"))),
    8613           40 :                 )
    8614           40 :             })
    8615            4 :             .collect_vec();
    8616            4 : 
    8617            4 :         let delta1 = vec![
    8618            4 :             (
    8619            4 :                 get_key(1),
    8620            4 :                 Lsn(0x20),
    8621            4 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x20")),
    8622            4 :             ),
    8623            4 :             (
    8624            4 :                 get_key(2),
    8625            4 :                 Lsn(0x30),
    8626            4 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x30")),
    8627            4 :             ),
    8628            4 :             (
    8629            4 :                 get_key(3),
    8630            4 :                 Lsn(0x28),
    8631            4 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x28")),
    8632            4 :             ),
    8633            4 :             (
    8634            4 :                 get_key(3),
    8635            4 :                 Lsn(0x30),
    8636            4 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x30")),
    8637            4 :             ),
    8638            4 :             (
    8639            4 :                 get_key(3),
    8640            4 :                 Lsn(0x40),
    8641            4 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x40")),
    8642            4 :             ),
    8643            4 :         ];
    8644            4 :         let delta2 = vec![
    8645            4 :             (
    8646            4 :                 get_key(5),
    8647            4 :                 Lsn(0x20),
    8648            4 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x20")),
    8649            4 :             ),
    8650            4 :             (
    8651            4 :                 get_key(6),
    8652            4 :                 Lsn(0x20),
    8653            4 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x20")),
    8654            4 :             ),
    8655            4 :         ];
    8656            4 :         let delta3 = vec![
    8657            4 :             (
    8658            4 :                 get_key(8),
    8659            4 :                 Lsn(0x48),
    8660            4 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x48")),
    8661            4 :             ),
    8662            4 :             (
    8663            4 :                 get_key(9),
    8664            4 :                 Lsn(0x48),
    8665            4 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x48")),
    8666            4 :             ),
    8667            4 :         ];
    8668              : 
    8669            4 :         let tline = if use_delta_bottom_layer {
    8670            2 :             tenant
    8671            2 :                 .create_test_timeline_with_layers(
    8672            2 :                     TIMELINE_ID,
    8673            2 :                     Lsn(0x08),
    8674            2 :                     DEFAULT_PG_VERSION,
    8675            2 :                     &ctx,
    8676            2 :                     vec![
    8677            2 :                         DeltaLayerTestDesc::new_with_inferred_key_range(
    8678            2 :                             Lsn(0x08)..Lsn(0x10),
    8679            2 :                             delta4,
    8680            2 :                         ),
    8681            2 :                         DeltaLayerTestDesc::new_with_inferred_key_range(
    8682            2 :                             Lsn(0x20)..Lsn(0x48),
    8683            2 :                             delta1,
    8684            2 :                         ),
    8685            2 :                         DeltaLayerTestDesc::new_with_inferred_key_range(
    8686            2 :                             Lsn(0x20)..Lsn(0x48),
    8687            2 :                             delta2,
    8688            2 :                         ),
    8689            2 :                         DeltaLayerTestDesc::new_with_inferred_key_range(
    8690            2 :                             Lsn(0x48)..Lsn(0x50),
    8691            2 :                             delta3,
    8692            2 :                         ),
    8693            2 :                     ], // delta layers
    8694            2 :                     vec![], // image layers
    8695            2 :                     Lsn(0x50),
    8696            2 :                 )
    8697            2 :                 .await?
    8698              :         } else {
    8699            2 :             tenant
    8700            2 :                 .create_test_timeline_with_layers(
    8701            2 :                     TIMELINE_ID,
    8702            2 :                     Lsn(0x10),
    8703            2 :                     DEFAULT_PG_VERSION,
    8704            2 :                     &ctx,
    8705            2 :                     vec![
    8706            2 :                         DeltaLayerTestDesc::new_with_inferred_key_range(
    8707            2 :                             Lsn(0x10)..Lsn(0x48),
    8708            2 :                             delta1,
    8709            2 :                         ),
    8710            2 :                         DeltaLayerTestDesc::new_with_inferred_key_range(
    8711            2 :                             Lsn(0x10)..Lsn(0x48),
    8712            2 :                             delta2,
    8713            2 :                         ),
    8714            2 :                         DeltaLayerTestDesc::new_with_inferred_key_range(
    8715            2 :                             Lsn(0x48)..Lsn(0x50),
    8716            2 :                             delta3,
    8717            2 :                         ),
    8718            2 :                     ], // delta layers
    8719            2 :                     vec![(Lsn(0x10), img_layer)], // image layers
    8720            2 :                     Lsn(0x50),
    8721            2 :                 )
    8722            2 :                 .await?
    8723              :         };
    8724              :         {
    8725            4 :             tline
    8726            4 :                 .latest_gc_cutoff_lsn
    8727            4 :                 .lock_for_write()
    8728            4 :                 .store_and_unlock(Lsn(0x30))
    8729            4 :                 .wait()
    8730            4 :                 .await;
    8731              :             // Update GC info
    8732            4 :             let mut guard = tline.gc_info.write().unwrap();
    8733            4 :             *guard = GcInfo {
    8734            4 :                 retain_lsns: vec![],
    8735            4 :                 cutoffs: GcCutoffs {
    8736            4 :                     time: Lsn(0x30),
    8737            4 :                     space: Lsn(0x30),
    8738            4 :                 },
    8739            4 :                 leases: Default::default(),
    8740            4 :                 within_ancestor_pitr: false,
    8741            4 :             };
    8742            4 :         }
    8743            4 : 
    8744            4 :         let expected_result = [
    8745            4 :             Bytes::from_static(b"value 0@0x10"),
    8746            4 :             Bytes::from_static(b"value 1@0x10@0x20"),
    8747            4 :             Bytes::from_static(b"value 2@0x10@0x30"),
    8748            4 :             Bytes::from_static(b"value 3@0x10@0x28@0x30@0x40"),
    8749            4 :             Bytes::from_static(b"value 4@0x10"),
    8750            4 :             Bytes::from_static(b"value 5@0x10@0x20"),
    8751            4 :             Bytes::from_static(b"value 6@0x10@0x20"),
    8752            4 :             Bytes::from_static(b"value 7@0x10"),
    8753            4 :             Bytes::from_static(b"value 8@0x10@0x48"),
    8754            4 :             Bytes::from_static(b"value 9@0x10@0x48"),
    8755            4 :         ];
    8756            4 : 
    8757            4 :         let expected_result_at_gc_horizon = [
    8758            4 :             Bytes::from_static(b"value 0@0x10"),
    8759            4 :             Bytes::from_static(b"value 1@0x10@0x20"),
    8760            4 :             Bytes::from_static(b"value 2@0x10@0x30"),
    8761            4 :             Bytes::from_static(b"value 3@0x10@0x28@0x30"),
    8762            4 :             Bytes::from_static(b"value 4@0x10"),
    8763            4 :             Bytes::from_static(b"value 5@0x10@0x20"),
    8764            4 :             Bytes::from_static(b"value 6@0x10@0x20"),
    8765            4 :             Bytes::from_static(b"value 7@0x10"),
    8766            4 :             Bytes::from_static(b"value 8@0x10"),
    8767            4 :             Bytes::from_static(b"value 9@0x10"),
    8768            4 :         ];
    8769              : 
    8770           44 :         for idx in 0..10 {
    8771           40 :             assert_eq!(
    8772           40 :                 tline
    8773           40 :                     .get(get_key(idx as u32), Lsn(0x50), &ctx)
    8774           40 :                     .await
    8775           40 :                     .unwrap(),
    8776           40 :                 &expected_result[idx]
    8777              :             );
    8778           40 :             assert_eq!(
    8779           40 :                 tline
    8780           40 :                     .get(get_key(idx as u32), Lsn(0x30), &ctx)
    8781           40 :                     .await
    8782           40 :                     .unwrap(),
    8783           40 :                 &expected_result_at_gc_horizon[idx]
    8784              :             );
    8785              :         }
    8786              : 
    8787            4 :         let cancel = CancellationToken::new();
    8788            4 :         tline
    8789            4 :             .compact_with_gc(&cancel, CompactOptions::default(), &ctx)
    8790            4 :             .await
    8791            4 :             .unwrap();
    8792              : 
    8793           44 :         for idx in 0..10 {
    8794           40 :             assert_eq!(
    8795           40 :                 tline
    8796           40 :                     .get(get_key(idx as u32), Lsn(0x50), &ctx)
    8797           40 :                     .await
    8798           40 :                     .unwrap(),
    8799           40 :                 &expected_result[idx]
    8800              :             );
    8801           40 :             assert_eq!(
    8802           40 :                 tline
    8803           40 :                     .get(get_key(idx as u32), Lsn(0x30), &ctx)
    8804           40 :                     .await
    8805           40 :                     .unwrap(),
    8806           40 :                 &expected_result_at_gc_horizon[idx]
    8807              :             );
    8808              :         }
    8809              : 
    8810              :         // increase GC horizon and compact again
    8811              :         {
    8812            4 :             tline
    8813            4 :                 .latest_gc_cutoff_lsn
    8814            4 :                 .lock_for_write()
    8815            4 :                 .store_and_unlock(Lsn(0x40))
    8816            4 :                 .wait()
    8817            4 :                 .await;
    8818              :             // Update GC info
    8819            4 :             let mut guard = tline.gc_info.write().unwrap();
    8820            4 :             guard.cutoffs.time = Lsn(0x40);
    8821            4 :             guard.cutoffs.space = Lsn(0x40);
    8822            4 :         }
    8823            4 :         tline
    8824            4 :             .compact_with_gc(&cancel, CompactOptions::default(), &ctx)
    8825            4 :             .await
    8826            4 :             .unwrap();
    8827            4 : 
    8828            4 :         Ok(())
    8829            4 :     }
    8830              : 
    8831              :     #[cfg(feature = "testing")]
    8832              :     #[tokio::test]
    8833            2 :     async fn test_generate_key_retention() -> anyhow::Result<()> {
    8834            2 :         let harness = TenantHarness::create("test_generate_key_retention").await?;
    8835            2 :         let (tenant, ctx) = harness.load().await;
    8836            2 :         let tline = tenant
    8837            2 :             .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx)
    8838            2 :             .await?;
    8839            2 :         tline.force_advance_lsn(Lsn(0x70));
    8840            2 :         let key = Key::from_hex("010000000033333333444444445500000000").unwrap();
    8841            2 :         let history = vec![
    8842            2 :             (
    8843            2 :                 key,
    8844            2 :                 Lsn(0x10),
    8845            2 :                 Value::WalRecord(NeonWalRecord::wal_init("0x10")),
    8846            2 :             ),
    8847            2 :             (
    8848            2 :                 key,
    8849            2 :                 Lsn(0x20),
    8850            2 :                 Value::WalRecord(NeonWalRecord::wal_append(";0x20")),
    8851            2 :             ),
    8852            2 :             (
    8853            2 :                 key,
    8854            2 :                 Lsn(0x30),
    8855            2 :                 Value::WalRecord(NeonWalRecord::wal_append(";0x30")),
    8856            2 :             ),
    8857            2 :             (
    8858            2 :                 key,
    8859            2 :                 Lsn(0x40),
    8860            2 :                 Value::WalRecord(NeonWalRecord::wal_append(";0x40")),
    8861            2 :             ),
    8862            2 :             (
    8863            2 :                 key,
    8864            2 :                 Lsn(0x50),
    8865            2 :                 Value::WalRecord(NeonWalRecord::wal_append(";0x50")),
    8866            2 :             ),
    8867            2 :             (
    8868            2 :                 key,
    8869            2 :                 Lsn(0x60),
    8870            2 :                 Value::WalRecord(NeonWalRecord::wal_append(";0x60")),
    8871            2 :             ),
    8872            2 :             (
    8873            2 :                 key,
    8874            2 :                 Lsn(0x70),
    8875            2 :                 Value::WalRecord(NeonWalRecord::wal_append(";0x70")),
    8876            2 :             ),
    8877            2 :             (
    8878            2 :                 key,
    8879            2 :                 Lsn(0x80),
    8880            2 :                 Value::Image(Bytes::copy_from_slice(
    8881            2 :                     b"0x10;0x20;0x30;0x40;0x50;0x60;0x70;0x80",
    8882            2 :                 )),
    8883            2 :             ),
    8884            2 :             (
    8885            2 :                 key,
    8886            2 :                 Lsn(0x90),
    8887            2 :                 Value::WalRecord(NeonWalRecord::wal_append(";0x90")),
    8888            2 :             ),
    8889            2 :         ];
    8890            2 :         let res = tline
    8891            2 :             .generate_key_retention(
    8892            2 :                 key,
    8893            2 :                 &history,
    8894            2 :                 Lsn(0x60),
    8895            2 :                 &[Lsn(0x20), Lsn(0x40), Lsn(0x50)],
    8896            2 :                 3,
    8897            2 :                 None,
    8898            2 :             )
    8899            2 :             .await
    8900            2 :             .unwrap();
    8901            2 :         let expected_res = KeyHistoryRetention {
    8902            2 :             below_horizon: vec![
    8903            2 :                 (
    8904            2 :                     Lsn(0x20),
    8905            2 :                     KeyLogAtLsn(vec![(
    8906            2 :                         Lsn(0x20),
    8907            2 :                         Value::Image(Bytes::from_static(b"0x10;0x20")),
    8908            2 :                     )]),
    8909            2 :                 ),
    8910            2 :                 (
    8911            2 :                     Lsn(0x40),
    8912            2 :                     KeyLogAtLsn(vec![
    8913            2 :                         (
    8914            2 :                             Lsn(0x30),
    8915            2 :                             Value::WalRecord(NeonWalRecord::wal_append(";0x30")),
    8916            2 :                         ),
    8917            2 :                         (
    8918            2 :                             Lsn(0x40),
    8919            2 :                             Value::WalRecord(NeonWalRecord::wal_append(";0x40")),
    8920            2 :                         ),
    8921            2 :                     ]),
    8922            2 :                 ),
    8923            2 :                 (
    8924            2 :                     Lsn(0x50),
    8925            2 :                     KeyLogAtLsn(vec![(
    8926            2 :                         Lsn(0x50),
    8927            2 :                         Value::Image(Bytes::copy_from_slice(b"0x10;0x20;0x30;0x40;0x50")),
    8928            2 :                     )]),
    8929            2 :                 ),
    8930            2 :                 (
    8931            2 :                     Lsn(0x60),
    8932            2 :                     KeyLogAtLsn(vec![(
    8933            2 :                         Lsn(0x60),
    8934            2 :                         Value::WalRecord(NeonWalRecord::wal_append(";0x60")),
    8935            2 :                     )]),
    8936            2 :                 ),
    8937            2 :             ],
    8938            2 :             above_horizon: KeyLogAtLsn(vec![
    8939            2 :                 (
    8940            2 :                     Lsn(0x70),
    8941            2 :                     Value::WalRecord(NeonWalRecord::wal_append(";0x70")),
    8942            2 :                 ),
    8943            2 :                 (
    8944            2 :                     Lsn(0x80),
    8945            2 :                     Value::Image(Bytes::copy_from_slice(
    8946            2 :                         b"0x10;0x20;0x30;0x40;0x50;0x60;0x70;0x80",
    8947            2 :                     )),
    8948            2 :                 ),
    8949            2 :                 (
    8950            2 :                     Lsn(0x90),
    8951            2 :                     Value::WalRecord(NeonWalRecord::wal_append(";0x90")),
    8952            2 :                 ),
    8953            2 :             ]),
    8954            2 :         };
    8955            2 :         assert_eq!(res, expected_res);
    8956            2 : 
    8957            2 :         // We expect GC-compaction to run with the original GC. This would create a situation that
    8958            2 :         // the original GC algorithm removes some delta layers b/c there are full image coverage,
    8959            2 :         // therefore causing some keys to have an incomplete history below the lowest retain LSN.
    8960            2 :         // For example, we have
    8961            2 :         // ```plain
    8962            2 :         // init delta @ 0x10, image @ 0x20, delta @ 0x30 (gc_horizon), image @ 0x40.
    8963            2 :         // ```
    8964            2 :         // Now the GC horizon moves up, and we have
    8965            2 :         // ```plain
    8966            2 :         // init delta @ 0x10, image @ 0x20, delta @ 0x30, image @ 0x40 (gc_horizon)
    8967            2 :         // ```
    8968            2 :         // The original GC algorithm kicks in, and removes delta @ 0x10, image @ 0x20.
    8969            2 :         // We will end up with
    8970            2 :         // ```plain
    8971            2 :         // delta @ 0x30, image @ 0x40 (gc_horizon)
    8972            2 :         // ```
    8973            2 :         // Now we run the GC-compaction, and this key does not have a full history.
    8974            2 :         // We should be able to handle this partial history and drop everything before the
    8975            2 :         // gc_horizon image.
    8976            2 : 
    8977            2 :         let history = vec![
    8978            2 :             (
    8979            2 :                 key,
    8980            2 :                 Lsn(0x20),
    8981            2 :                 Value::WalRecord(NeonWalRecord::wal_append(";0x20")),
    8982            2 :             ),
    8983            2 :             (
    8984            2 :                 key,
    8985            2 :                 Lsn(0x30),
    8986            2 :                 Value::WalRecord(NeonWalRecord::wal_append(";0x30")),
    8987            2 :             ),
    8988            2 :             (
    8989            2 :                 key,
    8990            2 :                 Lsn(0x40),
    8991            2 :                 Value::Image(Bytes::copy_from_slice(b"0x10;0x20;0x30;0x40")),
    8992            2 :             ),
    8993            2 :             (
    8994            2 :                 key,
    8995            2 :                 Lsn(0x50),
    8996            2 :                 Value::WalRecord(NeonWalRecord::wal_append(";0x50")),
    8997            2 :             ),
    8998            2 :             (
    8999            2 :                 key,
    9000            2 :                 Lsn(0x60),
    9001            2 :                 Value::WalRecord(NeonWalRecord::wal_append(";0x60")),
    9002            2 :             ),
    9003            2 :             (
    9004            2 :                 key,
    9005            2 :                 Lsn(0x70),
    9006            2 :                 Value::WalRecord(NeonWalRecord::wal_append(";0x70")),
    9007            2 :             ),
    9008            2 :             (
    9009            2 :                 key,
    9010            2 :                 Lsn(0x80),
    9011            2 :                 Value::Image(Bytes::copy_from_slice(
    9012            2 :                     b"0x10;0x20;0x30;0x40;0x50;0x60;0x70;0x80",
    9013            2 :                 )),
    9014            2 :             ),
    9015            2 :             (
    9016            2 :                 key,
    9017            2 :                 Lsn(0x90),
    9018            2 :                 Value::WalRecord(NeonWalRecord::wal_append(";0x90")),
    9019            2 :             ),
    9020            2 :         ];
    9021            2 :         let res = tline
    9022            2 :             .generate_key_retention(key, &history, Lsn(0x60), &[Lsn(0x40), Lsn(0x50)], 3, None)
    9023            2 :             .await
    9024            2 :             .unwrap();
    9025            2 :         let expected_res = KeyHistoryRetention {
    9026            2 :             below_horizon: vec![
    9027            2 :                 (
    9028            2 :                     Lsn(0x40),
    9029            2 :                     KeyLogAtLsn(vec![(
    9030            2 :                         Lsn(0x40),
    9031            2 :                         Value::Image(Bytes::copy_from_slice(b"0x10;0x20;0x30;0x40")),
    9032            2 :                     )]),
    9033            2 :                 ),
    9034            2 :                 (
    9035            2 :                     Lsn(0x50),
    9036            2 :                     KeyLogAtLsn(vec![(
    9037            2 :                         Lsn(0x50),
    9038            2 :                         Value::WalRecord(NeonWalRecord::wal_append(";0x50")),
    9039            2 :                     )]),
    9040            2 :                 ),
    9041            2 :                 (
    9042            2 :                     Lsn(0x60),
    9043            2 :                     KeyLogAtLsn(vec![(
    9044            2 :                         Lsn(0x60),
    9045            2 :                         Value::WalRecord(NeonWalRecord::wal_append(";0x60")),
    9046            2 :                     )]),
    9047            2 :                 ),
    9048            2 :             ],
    9049            2 :             above_horizon: KeyLogAtLsn(vec![
    9050            2 :                 (
    9051            2 :                     Lsn(0x70),
    9052            2 :                     Value::WalRecord(NeonWalRecord::wal_append(";0x70")),
    9053            2 :                 ),
    9054            2 :                 (
    9055            2 :                     Lsn(0x80),
    9056            2 :                     Value::Image(Bytes::copy_from_slice(
    9057            2 :                         b"0x10;0x20;0x30;0x40;0x50;0x60;0x70;0x80",
    9058            2 :                     )),
    9059            2 :                 ),
    9060            2 :                 (
    9061            2 :                     Lsn(0x90),
    9062            2 :                     Value::WalRecord(NeonWalRecord::wal_append(";0x90")),
    9063            2 :                 ),
    9064            2 :             ]),
    9065            2 :         };
    9066            2 :         assert_eq!(res, expected_res);
    9067            2 : 
    9068            2 :         // In case of branch compaction, the branch itself does not have the full history, and we need to provide
    9069            2 :         // the ancestor image in the test case.
    9070            2 : 
    9071            2 :         let history = vec![
    9072            2 :             (
    9073            2 :                 key,
    9074            2 :                 Lsn(0x20),
    9075            2 :                 Value::WalRecord(NeonWalRecord::wal_append(";0x20")),
    9076            2 :             ),
    9077            2 :             (
    9078            2 :                 key,
    9079            2 :                 Lsn(0x30),
    9080            2 :                 Value::WalRecord(NeonWalRecord::wal_append(";0x30")),
    9081            2 :             ),
    9082            2 :             (
    9083            2 :                 key,
    9084            2 :                 Lsn(0x40),
    9085            2 :                 Value::WalRecord(NeonWalRecord::wal_append(";0x40")),
    9086            2 :             ),
    9087            2 :             (
    9088            2 :                 key,
    9089            2 :                 Lsn(0x70),
    9090            2 :                 Value::WalRecord(NeonWalRecord::wal_append(";0x70")),
    9091            2 :             ),
    9092            2 :         ];
    9093            2 :         let res = tline
    9094            2 :             .generate_key_retention(
    9095            2 :                 key,
    9096            2 :                 &history,
    9097            2 :                 Lsn(0x60),
    9098            2 :                 &[],
    9099            2 :                 3,
    9100            2 :                 Some((key, Lsn(0x10), Bytes::copy_from_slice(b"0x10"))),
    9101            2 :             )
    9102            2 :             .await
    9103            2 :             .unwrap();
    9104            2 :         let expected_res = KeyHistoryRetention {
    9105            2 :             below_horizon: vec![(
    9106            2 :                 Lsn(0x60),
    9107            2 :                 KeyLogAtLsn(vec![(
    9108            2 :                     Lsn(0x60),
    9109            2 :                     Value::Image(Bytes::copy_from_slice(b"0x10;0x20;0x30;0x40")), // use the ancestor image to reconstruct the page
    9110            2 :                 )]),
    9111            2 :             )],
    9112            2 :             above_horizon: KeyLogAtLsn(vec![(
    9113            2 :                 Lsn(0x70),
    9114            2 :                 Value::WalRecord(NeonWalRecord::wal_append(";0x70")),
    9115            2 :             )]),
    9116            2 :         };
    9117            2 :         assert_eq!(res, expected_res);
    9118            2 : 
    9119            2 :         let history = vec![
    9120            2 :             (
    9121            2 :                 key,
    9122            2 :                 Lsn(0x20),
    9123            2 :                 Value::WalRecord(NeonWalRecord::wal_append(";0x20")),
    9124            2 :             ),
    9125            2 :             (
    9126            2 :                 key,
    9127            2 :                 Lsn(0x40),
    9128            2 :                 Value::WalRecord(NeonWalRecord::wal_append(";0x40")),
    9129            2 :             ),
    9130            2 :             (
    9131            2 :                 key,
    9132            2 :                 Lsn(0x60),
    9133            2 :                 Value::WalRecord(NeonWalRecord::wal_append(";0x60")),
    9134            2 :             ),
    9135            2 :             (
    9136            2 :                 key,
    9137            2 :                 Lsn(0x70),
    9138            2 :                 Value::WalRecord(NeonWalRecord::wal_append(";0x70")),
    9139            2 :             ),
    9140            2 :         ];
    9141            2 :         let res = tline
    9142            2 :             .generate_key_retention(
    9143            2 :                 key,
    9144            2 :                 &history,
    9145            2 :                 Lsn(0x60),
    9146            2 :                 &[Lsn(0x30)],
    9147            2 :                 3,
    9148            2 :                 Some((key, Lsn(0x10), Bytes::copy_from_slice(b"0x10"))),
    9149            2 :             )
    9150            2 :             .await
    9151            2 :             .unwrap();
    9152            2 :         let expected_res = KeyHistoryRetention {
    9153            2 :             below_horizon: vec![
    9154            2 :                 (
    9155            2 :                     Lsn(0x30),
    9156            2 :                     KeyLogAtLsn(vec![(
    9157            2 :                         Lsn(0x20),
    9158            2 :                         Value::WalRecord(NeonWalRecord::wal_append(";0x20")),
    9159            2 :                     )]),
    9160            2 :                 ),
    9161            2 :                 (
    9162            2 :                     Lsn(0x60),
    9163            2 :                     KeyLogAtLsn(vec![(
    9164            2 :                         Lsn(0x60),
    9165            2 :                         Value::Image(Bytes::copy_from_slice(b"0x10;0x20;0x40;0x60")),
    9166            2 :                     )]),
    9167            2 :                 ),
    9168            2 :             ],
    9169            2 :             above_horizon: KeyLogAtLsn(vec![(
    9170            2 :                 Lsn(0x70),
    9171            2 :                 Value::WalRecord(NeonWalRecord::wal_append(";0x70")),
    9172            2 :             )]),
    9173            2 :         };
    9174            2 :         assert_eq!(res, expected_res);
    9175            2 : 
    9176            2 :         Ok(())
    9177            2 :     }
    9178              : 
    9179              :     #[cfg(feature = "testing")]
    9180              :     #[tokio::test]
    9181            2 :     async fn test_simple_bottom_most_compaction_with_retain_lsns() -> anyhow::Result<()> {
    9182            2 :         let harness =
    9183            2 :             TenantHarness::create("test_simple_bottom_most_compaction_with_retain_lsns").await?;
    9184            2 :         let (tenant, ctx) = harness.load().await;
    9185            2 : 
    9186          518 :         fn get_key(id: u32) -> Key {
    9187          518 :             // using aux key here b/c they are guaranteed to be inside `collect_keyspace`.
    9188          518 :             let mut key = Key::from_hex("620000000033333333444444445500000000").unwrap();
    9189          518 :             key.field6 = id;
    9190          518 :             key
    9191          518 :         }
    9192            2 : 
    9193            2 :         let img_layer = (0..10)
    9194           20 :             .map(|id| (get_key(id), Bytes::from(format!("value {id}@0x10"))))
    9195            2 :             .collect_vec();
    9196            2 : 
    9197            2 :         let delta1 = vec![
    9198            2 :             (
    9199            2 :                 get_key(1),
    9200            2 :                 Lsn(0x20),
    9201            2 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x20")),
    9202            2 :             ),
    9203            2 :             (
    9204            2 :                 get_key(2),
    9205            2 :                 Lsn(0x30),
    9206            2 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x30")),
    9207            2 :             ),
    9208            2 :             (
    9209            2 :                 get_key(3),
    9210            2 :                 Lsn(0x28),
    9211            2 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x28")),
    9212            2 :             ),
    9213            2 :             (
    9214            2 :                 get_key(3),
    9215            2 :                 Lsn(0x30),
    9216            2 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x30")),
    9217            2 :             ),
    9218            2 :             (
    9219            2 :                 get_key(3),
    9220            2 :                 Lsn(0x40),
    9221            2 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x40")),
    9222            2 :             ),
    9223            2 :         ];
    9224            2 :         let delta2 = vec![
    9225            2 :             (
    9226            2 :                 get_key(5),
    9227            2 :                 Lsn(0x20),
    9228            2 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x20")),
    9229            2 :             ),
    9230            2 :             (
    9231            2 :                 get_key(6),
    9232            2 :                 Lsn(0x20),
    9233            2 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x20")),
    9234            2 :             ),
    9235            2 :         ];
    9236            2 :         let delta3 = vec![
    9237            2 :             (
    9238            2 :                 get_key(8),
    9239            2 :                 Lsn(0x48),
    9240            2 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x48")),
    9241            2 :             ),
    9242            2 :             (
    9243            2 :                 get_key(9),
    9244            2 :                 Lsn(0x48),
    9245            2 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x48")),
    9246            2 :             ),
    9247            2 :         ];
    9248            2 : 
    9249            2 :         let tline = tenant
    9250            2 :             .create_test_timeline_with_layers(
    9251            2 :                 TIMELINE_ID,
    9252            2 :                 Lsn(0x10),
    9253            2 :                 DEFAULT_PG_VERSION,
    9254            2 :                 &ctx,
    9255            2 :                 vec![
    9256            2 :                     DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x10)..Lsn(0x48), delta1),
    9257            2 :                     DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x10)..Lsn(0x48), delta2),
    9258            2 :                     DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x48)..Lsn(0x50), delta3),
    9259            2 :                 ], // delta layers
    9260            2 :                 vec![(Lsn(0x10), img_layer)], // image layers
    9261            2 :                 Lsn(0x50),
    9262            2 :             )
    9263            2 :             .await?;
    9264            2 :         {
    9265            2 :             tline
    9266            2 :                 .latest_gc_cutoff_lsn
    9267            2 :                 .lock_for_write()
    9268            2 :                 .store_and_unlock(Lsn(0x30))
    9269            2 :                 .wait()
    9270            2 :                 .await;
    9271            2 :             // Update GC info
    9272            2 :             let mut guard = tline.gc_info.write().unwrap();
    9273            2 :             *guard = GcInfo {
    9274            2 :                 retain_lsns: vec![
    9275            2 :                     (Lsn(0x10), tline.timeline_id, MaybeOffloaded::No),
    9276            2 :                     (Lsn(0x20), tline.timeline_id, MaybeOffloaded::No),
    9277            2 :                 ],
    9278            2 :                 cutoffs: GcCutoffs {
    9279            2 :                     time: Lsn(0x30),
    9280            2 :                     space: Lsn(0x30),
    9281            2 :                 },
    9282            2 :                 leases: Default::default(),
    9283            2 :                 within_ancestor_pitr: false,
    9284            2 :             };
    9285            2 :         }
    9286            2 : 
    9287            2 :         let expected_result = [
    9288            2 :             Bytes::from_static(b"value 0@0x10"),
    9289            2 :             Bytes::from_static(b"value 1@0x10@0x20"),
    9290            2 :             Bytes::from_static(b"value 2@0x10@0x30"),
    9291            2 :             Bytes::from_static(b"value 3@0x10@0x28@0x30@0x40"),
    9292            2 :             Bytes::from_static(b"value 4@0x10"),
    9293            2 :             Bytes::from_static(b"value 5@0x10@0x20"),
    9294            2 :             Bytes::from_static(b"value 6@0x10@0x20"),
    9295            2 :             Bytes::from_static(b"value 7@0x10"),
    9296            2 :             Bytes::from_static(b"value 8@0x10@0x48"),
    9297            2 :             Bytes::from_static(b"value 9@0x10@0x48"),
    9298            2 :         ];
    9299            2 : 
    9300            2 :         let expected_result_at_gc_horizon = [
    9301            2 :             Bytes::from_static(b"value 0@0x10"),
    9302            2 :             Bytes::from_static(b"value 1@0x10@0x20"),
    9303            2 :             Bytes::from_static(b"value 2@0x10@0x30"),
    9304            2 :             Bytes::from_static(b"value 3@0x10@0x28@0x30"),
    9305            2 :             Bytes::from_static(b"value 4@0x10"),
    9306            2 :             Bytes::from_static(b"value 5@0x10@0x20"),
    9307            2 :             Bytes::from_static(b"value 6@0x10@0x20"),
    9308            2 :             Bytes::from_static(b"value 7@0x10"),
    9309            2 :             Bytes::from_static(b"value 8@0x10"),
    9310            2 :             Bytes::from_static(b"value 9@0x10"),
    9311            2 :         ];
    9312            2 : 
    9313            2 :         let expected_result_at_lsn_20 = [
    9314            2 :             Bytes::from_static(b"value 0@0x10"),
    9315            2 :             Bytes::from_static(b"value 1@0x10@0x20"),
    9316            2 :             Bytes::from_static(b"value 2@0x10"),
    9317            2 :             Bytes::from_static(b"value 3@0x10"),
    9318            2 :             Bytes::from_static(b"value 4@0x10"),
    9319            2 :             Bytes::from_static(b"value 5@0x10@0x20"),
    9320            2 :             Bytes::from_static(b"value 6@0x10@0x20"),
    9321            2 :             Bytes::from_static(b"value 7@0x10"),
    9322            2 :             Bytes::from_static(b"value 8@0x10"),
    9323            2 :             Bytes::from_static(b"value 9@0x10"),
    9324            2 :         ];
    9325            2 : 
    9326            2 :         let expected_result_at_lsn_10 = [
    9327            2 :             Bytes::from_static(b"value 0@0x10"),
    9328            2 :             Bytes::from_static(b"value 1@0x10"),
    9329            2 :             Bytes::from_static(b"value 2@0x10"),
    9330            2 :             Bytes::from_static(b"value 3@0x10"),
    9331            2 :             Bytes::from_static(b"value 4@0x10"),
    9332            2 :             Bytes::from_static(b"value 5@0x10"),
    9333            2 :             Bytes::from_static(b"value 6@0x10"),
    9334            2 :             Bytes::from_static(b"value 7@0x10"),
    9335            2 :             Bytes::from_static(b"value 8@0x10"),
    9336            2 :             Bytes::from_static(b"value 9@0x10"),
    9337            2 :         ];
    9338            2 : 
    9339           12 :         let verify_result = || async {
    9340           12 :             let gc_horizon = {
    9341           12 :                 let gc_info = tline.gc_info.read().unwrap();
    9342           12 :                 gc_info.cutoffs.time
    9343            2 :             };
    9344          132 :             for idx in 0..10 {
    9345          120 :                 assert_eq!(
    9346          120 :                     tline
    9347          120 :                         .get(get_key(idx as u32), Lsn(0x50), &ctx)
    9348          120 :                         .await
    9349          120 :                         .unwrap(),
    9350          120 :                     &expected_result[idx]
    9351            2 :                 );
    9352          120 :                 assert_eq!(
    9353          120 :                     tline
    9354          120 :                         .get(get_key(idx as u32), gc_horizon, &ctx)
    9355          120 :                         .await
    9356          120 :                         .unwrap(),
    9357          120 :                     &expected_result_at_gc_horizon[idx]
    9358            2 :                 );
    9359          120 :                 assert_eq!(
    9360          120 :                     tline
    9361          120 :                         .get(get_key(idx as u32), Lsn(0x20), &ctx)
    9362          120 :                         .await
    9363          120 :                         .unwrap(),
    9364          120 :                     &expected_result_at_lsn_20[idx]
    9365            2 :                 );
    9366          120 :                 assert_eq!(
    9367          120 :                     tline
    9368          120 :                         .get(get_key(idx as u32), Lsn(0x10), &ctx)
    9369          120 :                         .await
    9370          120 :                         .unwrap(),
    9371          120 :                     &expected_result_at_lsn_10[idx]
    9372            2 :                 );
    9373            2 :             }
    9374           24 :         };
    9375            2 : 
    9376            2 :         verify_result().await;
    9377            2 : 
    9378            2 :         let cancel = CancellationToken::new();
    9379            2 :         let mut dryrun_flags = EnumSet::new();
    9380            2 :         dryrun_flags.insert(CompactFlags::DryRun);
    9381            2 : 
    9382            2 :         tline
    9383            2 :             .compact_with_gc(
    9384            2 :                 &cancel,
    9385            2 :                 CompactOptions {
    9386            2 :                     flags: dryrun_flags,
    9387            2 :                     ..Default::default()
    9388            2 :                 },
    9389            2 :                 &ctx,
    9390            2 :             )
    9391            2 :             .await
    9392            2 :             .unwrap();
    9393            2 :         // We expect layer map to be the same b/c the dry run flag, but we don't know whether there will be other background jobs
    9394            2 :         // cleaning things up, and therefore, we don't do sanity checks on the layer map during unit tests.
    9395            2 :         verify_result().await;
    9396            2 : 
    9397            2 :         tline
    9398            2 :             .compact_with_gc(&cancel, CompactOptions::default(), &ctx)
    9399            2 :             .await
    9400            2 :             .unwrap();
    9401            2 :         verify_result().await;
    9402            2 : 
    9403            2 :         // compact again
    9404            2 :         tline
    9405            2 :             .compact_with_gc(&cancel, CompactOptions::default(), &ctx)
    9406            2 :             .await
    9407            2 :             .unwrap();
    9408            2 :         verify_result().await;
    9409            2 : 
    9410            2 :         // increase GC horizon and compact again
    9411            2 :         {
    9412            2 :             tline
    9413            2 :                 .latest_gc_cutoff_lsn
    9414            2 :                 .lock_for_write()
    9415            2 :                 .store_and_unlock(Lsn(0x38))
    9416            2 :                 .wait()
    9417            2 :                 .await;
    9418            2 :             // Update GC info
    9419            2 :             let mut guard = tline.gc_info.write().unwrap();
    9420            2 :             guard.cutoffs.time = Lsn(0x38);
    9421            2 :             guard.cutoffs.space = Lsn(0x38);
    9422            2 :         }
    9423            2 :         tline
    9424            2 :             .compact_with_gc(&cancel, CompactOptions::default(), &ctx)
    9425            2 :             .await
    9426            2 :             .unwrap();
    9427            2 :         verify_result().await; // no wals between 0x30 and 0x38, so we should obtain the same result
    9428            2 : 
    9429            2 :         // not increasing the GC horizon and compact again
    9430            2 :         tline
    9431            2 :             .compact_with_gc(&cancel, CompactOptions::default(), &ctx)
    9432            2 :             .await
    9433            2 :             .unwrap();
    9434            2 :         verify_result().await;
    9435            2 : 
    9436            2 :         Ok(())
    9437            2 :     }
    9438              : 
    9439              :     #[cfg(feature = "testing")]
    9440              :     #[tokio::test]
    9441            2 :     async fn test_simple_bottom_most_compaction_with_retain_lsns_single_key() -> anyhow::Result<()>
    9442            2 :     {
    9443            2 :         let harness =
    9444            2 :             TenantHarness::create("test_simple_bottom_most_compaction_with_retain_lsns_single_key")
    9445            2 :                 .await?;
    9446            2 :         let (tenant, ctx) = harness.load().await;
    9447            2 : 
    9448          352 :         fn get_key(id: u32) -> Key {
    9449          352 :             // using aux key here b/c they are guaranteed to be inside `collect_keyspace`.
    9450          352 :             let mut key = Key::from_hex("620000000033333333444444445500000000").unwrap();
    9451          352 :             key.field6 = id;
    9452          352 :             key
    9453          352 :         }
    9454            2 : 
    9455            2 :         let img_layer = (0..10)
    9456           20 :             .map(|id| (get_key(id), Bytes::from(format!("value {id}@0x10"))))
    9457            2 :             .collect_vec();
    9458            2 : 
    9459            2 :         let delta1 = vec![
    9460            2 :             (
    9461            2 :                 get_key(1),
    9462            2 :                 Lsn(0x20),
    9463            2 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x20")),
    9464            2 :             ),
    9465            2 :             (
    9466            2 :                 get_key(1),
    9467            2 :                 Lsn(0x28),
    9468            2 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x28")),
    9469            2 :             ),
    9470            2 :         ];
    9471            2 :         let delta2 = vec![
    9472            2 :             (
    9473            2 :                 get_key(1),
    9474            2 :                 Lsn(0x30),
    9475            2 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x30")),
    9476            2 :             ),
    9477            2 :             (
    9478            2 :                 get_key(1),
    9479            2 :                 Lsn(0x38),
    9480            2 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x38")),
    9481            2 :             ),
    9482            2 :         ];
    9483            2 :         let delta3 = vec![
    9484            2 :             (
    9485            2 :                 get_key(8),
    9486            2 :                 Lsn(0x48),
    9487            2 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x48")),
    9488            2 :             ),
    9489            2 :             (
    9490            2 :                 get_key(9),
    9491            2 :                 Lsn(0x48),
    9492            2 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x48")),
    9493            2 :             ),
    9494            2 :         ];
    9495            2 : 
    9496            2 :         let tline = tenant
    9497            2 :             .create_test_timeline_with_layers(
    9498            2 :                 TIMELINE_ID,
    9499            2 :                 Lsn(0x10),
    9500            2 :                 DEFAULT_PG_VERSION,
    9501            2 :                 &ctx,
    9502            2 :                 vec![
    9503            2 :                     // delta1 and delta 2 only contain a single key but multiple updates
    9504            2 :                     DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x10)..Lsn(0x30), delta1),
    9505            2 :                     DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x30)..Lsn(0x50), delta2),
    9506            2 :                     DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x10)..Lsn(0x50), delta3),
    9507            2 :                 ], // delta layers
    9508            2 :                 vec![(Lsn(0x10), img_layer)], // image layers
    9509            2 :                 Lsn(0x50),
    9510            2 :             )
    9511            2 :             .await?;
    9512            2 :         {
    9513            2 :             tline
    9514            2 :                 .latest_gc_cutoff_lsn
    9515            2 :                 .lock_for_write()
    9516            2 :                 .store_and_unlock(Lsn(0x30))
    9517            2 :                 .wait()
    9518            2 :                 .await;
    9519            2 :             // Update GC info
    9520            2 :             let mut guard = tline.gc_info.write().unwrap();
    9521            2 :             *guard = GcInfo {
    9522            2 :                 retain_lsns: vec![
    9523            2 :                     (Lsn(0x10), tline.timeline_id, MaybeOffloaded::No),
    9524            2 :                     (Lsn(0x20), tline.timeline_id, MaybeOffloaded::No),
    9525            2 :                 ],
    9526            2 :                 cutoffs: GcCutoffs {
    9527            2 :                     time: Lsn(0x30),
    9528            2 :                     space: Lsn(0x30),
    9529            2 :                 },
    9530            2 :                 leases: Default::default(),
    9531            2 :                 within_ancestor_pitr: false,
    9532            2 :             };
    9533            2 :         }
    9534            2 : 
    9535            2 :         let expected_result = [
    9536            2 :             Bytes::from_static(b"value 0@0x10"),
    9537            2 :             Bytes::from_static(b"value 1@0x10@0x20@0x28@0x30@0x38"),
    9538            2 :             Bytes::from_static(b"value 2@0x10"),
    9539            2 :             Bytes::from_static(b"value 3@0x10"),
    9540            2 :             Bytes::from_static(b"value 4@0x10"),
    9541            2 :             Bytes::from_static(b"value 5@0x10"),
    9542            2 :             Bytes::from_static(b"value 6@0x10"),
    9543            2 :             Bytes::from_static(b"value 7@0x10"),
    9544            2 :             Bytes::from_static(b"value 8@0x10@0x48"),
    9545            2 :             Bytes::from_static(b"value 9@0x10@0x48"),
    9546            2 :         ];
    9547            2 : 
    9548            2 :         let expected_result_at_gc_horizon = [
    9549            2 :             Bytes::from_static(b"value 0@0x10"),
    9550            2 :             Bytes::from_static(b"value 1@0x10@0x20@0x28@0x30"),
    9551            2 :             Bytes::from_static(b"value 2@0x10"),
    9552            2 :             Bytes::from_static(b"value 3@0x10"),
    9553            2 :             Bytes::from_static(b"value 4@0x10"),
    9554            2 :             Bytes::from_static(b"value 5@0x10"),
    9555            2 :             Bytes::from_static(b"value 6@0x10"),
    9556            2 :             Bytes::from_static(b"value 7@0x10"),
    9557            2 :             Bytes::from_static(b"value 8@0x10"),
    9558            2 :             Bytes::from_static(b"value 9@0x10"),
    9559            2 :         ];
    9560            2 : 
    9561            2 :         let expected_result_at_lsn_20 = [
    9562            2 :             Bytes::from_static(b"value 0@0x10"),
    9563            2 :             Bytes::from_static(b"value 1@0x10@0x20"),
    9564            2 :             Bytes::from_static(b"value 2@0x10"),
    9565            2 :             Bytes::from_static(b"value 3@0x10"),
    9566            2 :             Bytes::from_static(b"value 4@0x10"),
    9567            2 :             Bytes::from_static(b"value 5@0x10"),
    9568            2 :             Bytes::from_static(b"value 6@0x10"),
    9569            2 :             Bytes::from_static(b"value 7@0x10"),
    9570            2 :             Bytes::from_static(b"value 8@0x10"),
    9571            2 :             Bytes::from_static(b"value 9@0x10"),
    9572            2 :         ];
    9573            2 : 
    9574            2 :         let expected_result_at_lsn_10 = [
    9575            2 :             Bytes::from_static(b"value 0@0x10"),
    9576            2 :             Bytes::from_static(b"value 1@0x10"),
    9577            2 :             Bytes::from_static(b"value 2@0x10"),
    9578            2 :             Bytes::from_static(b"value 3@0x10"),
    9579            2 :             Bytes::from_static(b"value 4@0x10"),
    9580            2 :             Bytes::from_static(b"value 5@0x10"),
    9581            2 :             Bytes::from_static(b"value 6@0x10"),
    9582            2 :             Bytes::from_static(b"value 7@0x10"),
    9583            2 :             Bytes::from_static(b"value 8@0x10"),
    9584            2 :             Bytes::from_static(b"value 9@0x10"),
    9585            2 :         ];
    9586            2 : 
    9587            8 :         let verify_result = || async {
    9588            8 :             let gc_horizon = {
    9589            8 :                 let gc_info = tline.gc_info.read().unwrap();
    9590            8 :                 gc_info.cutoffs.time
    9591            2 :             };
    9592           88 :             for idx in 0..10 {
    9593           80 :                 assert_eq!(
    9594           80 :                     tline
    9595           80 :                         .get(get_key(idx as u32), Lsn(0x50), &ctx)
    9596           80 :                         .await
    9597           80 :                         .unwrap(),
    9598           80 :                     &expected_result[idx]
    9599            2 :                 );
    9600           80 :                 assert_eq!(
    9601           80 :                     tline
    9602           80 :                         .get(get_key(idx as u32), gc_horizon, &ctx)
    9603           80 :                         .await
    9604           80 :                         .unwrap(),
    9605           80 :                     &expected_result_at_gc_horizon[idx]
    9606            2 :                 );
    9607           80 :                 assert_eq!(
    9608           80 :                     tline
    9609           80 :                         .get(get_key(idx as u32), Lsn(0x20), &ctx)
    9610           80 :                         .await
    9611           80 :                         .unwrap(),
    9612           80 :                     &expected_result_at_lsn_20[idx]
    9613            2 :                 );
    9614           80 :                 assert_eq!(
    9615           80 :                     tline
    9616           80 :                         .get(get_key(idx as u32), Lsn(0x10), &ctx)
    9617           80 :                         .await
    9618           80 :                         .unwrap(),
    9619           80 :                     &expected_result_at_lsn_10[idx]
    9620            2 :                 );
    9621            2 :             }
    9622           16 :         };
    9623            2 : 
    9624            2 :         verify_result().await;
    9625            2 : 
    9626            2 :         let cancel = CancellationToken::new();
    9627            2 :         let mut dryrun_flags = EnumSet::new();
    9628            2 :         dryrun_flags.insert(CompactFlags::DryRun);
    9629            2 : 
    9630            2 :         tline
    9631            2 :             .compact_with_gc(
    9632            2 :                 &cancel,
    9633            2 :                 CompactOptions {
    9634            2 :                     flags: dryrun_flags,
    9635            2 :                     ..Default::default()
    9636            2 :                 },
    9637            2 :                 &ctx,
    9638            2 :             )
    9639            2 :             .await
    9640            2 :             .unwrap();
    9641            2 :         // We expect layer map to be the same b/c the dry run flag, but we don't know whether there will be other background jobs
    9642            2 :         // cleaning things up, and therefore, we don't do sanity checks on the layer map during unit tests.
    9643            2 :         verify_result().await;
    9644            2 : 
    9645            2 :         tline
    9646            2 :             .compact_with_gc(&cancel, CompactOptions::default(), &ctx)
    9647            2 :             .await
    9648            2 :             .unwrap();
    9649            2 :         verify_result().await;
    9650            2 : 
    9651            2 :         // compact again
    9652            2 :         tline
    9653            2 :             .compact_with_gc(&cancel, CompactOptions::default(), &ctx)
    9654            2 :             .await
    9655            2 :             .unwrap();
    9656            2 :         verify_result().await;
    9657            2 : 
    9658            2 :         Ok(())
    9659            2 :     }
    9660              : 
    9661              :     #[cfg(feature = "testing")]
    9662              :     #[tokio::test]
    9663            2 :     async fn test_simple_bottom_most_compaction_on_branch() -> anyhow::Result<()> {
    9664            2 :         use models::CompactLsnRange;
    9665            2 : 
    9666            2 :         let harness = TenantHarness::create("test_simple_bottom_most_compaction_on_branch").await?;
    9667            2 :         let (tenant, ctx) = harness.load().await;
    9668            2 : 
    9669          166 :         fn get_key(id: u32) -> Key {
    9670          166 :             let mut key = Key::from_hex("000000000033333333444444445500000000").unwrap();
    9671          166 :             key.field6 = id;
    9672          166 :             key
    9673          166 :         }
    9674            2 : 
    9675            2 :         let img_layer = (0..10)
    9676           20 :             .map(|id| (get_key(id), Bytes::from(format!("value {id}@0x10"))))
    9677            2 :             .collect_vec();
    9678            2 : 
    9679            2 :         let delta1 = vec![
    9680            2 :             (
    9681            2 :                 get_key(1),
    9682            2 :                 Lsn(0x20),
    9683            2 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x20")),
    9684            2 :             ),
    9685            2 :             (
    9686            2 :                 get_key(2),
    9687            2 :                 Lsn(0x30),
    9688            2 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x30")),
    9689            2 :             ),
    9690            2 :             (
    9691            2 :                 get_key(3),
    9692            2 :                 Lsn(0x28),
    9693            2 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x28")),
    9694            2 :             ),
    9695            2 :             (
    9696            2 :                 get_key(3),
    9697            2 :                 Lsn(0x30),
    9698            2 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x30")),
    9699            2 :             ),
    9700            2 :             (
    9701            2 :                 get_key(3),
    9702            2 :                 Lsn(0x40),
    9703            2 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x40")),
    9704            2 :             ),
    9705            2 :         ];
    9706            2 :         let delta2 = vec![
    9707            2 :             (
    9708            2 :                 get_key(5),
    9709            2 :                 Lsn(0x20),
    9710            2 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x20")),
    9711            2 :             ),
    9712            2 :             (
    9713            2 :                 get_key(6),
    9714            2 :                 Lsn(0x20),
    9715            2 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x20")),
    9716            2 :             ),
    9717            2 :         ];
    9718            2 :         let delta3 = vec![
    9719            2 :             (
    9720            2 :                 get_key(8),
    9721            2 :                 Lsn(0x48),
    9722            2 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x48")),
    9723            2 :             ),
    9724            2 :             (
    9725            2 :                 get_key(9),
    9726            2 :                 Lsn(0x48),
    9727            2 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x48")),
    9728            2 :             ),
    9729            2 :         ];
    9730            2 : 
    9731            2 :         let parent_tline = tenant
    9732            2 :             .create_test_timeline_with_layers(
    9733            2 :                 TIMELINE_ID,
    9734            2 :                 Lsn(0x10),
    9735            2 :                 DEFAULT_PG_VERSION,
    9736            2 :                 &ctx,
    9737            2 :                 vec![],                       // delta layers
    9738            2 :                 vec![(Lsn(0x18), img_layer)], // image layers
    9739            2 :                 Lsn(0x18),
    9740            2 :             )
    9741            2 :             .await?;
    9742            2 : 
    9743            2 :         parent_tline.add_extra_test_dense_keyspace(KeySpace::single(get_key(0)..get_key(10)));
    9744            2 : 
    9745            2 :         let branch_tline = tenant
    9746            2 :             .branch_timeline_test_with_layers(
    9747            2 :                 &parent_tline,
    9748            2 :                 NEW_TIMELINE_ID,
    9749            2 :                 Some(Lsn(0x18)),
    9750            2 :                 &ctx,
    9751            2 :                 vec![
    9752            2 :                     DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x20)..Lsn(0x48), delta1),
    9753            2 :                     DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x20)..Lsn(0x48), delta2),
    9754            2 :                     DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x48)..Lsn(0x50), delta3),
    9755            2 :                 ], // delta layers
    9756            2 :                 vec![], // image layers
    9757            2 :                 Lsn(0x50),
    9758            2 :             )
    9759            2 :             .await?;
    9760            2 : 
    9761            2 :         branch_tline.add_extra_test_dense_keyspace(KeySpace::single(get_key(0)..get_key(10)));
    9762            2 : 
    9763            2 :         {
    9764            2 :             parent_tline
    9765            2 :                 .latest_gc_cutoff_lsn
    9766            2 :                 .lock_for_write()
    9767            2 :                 .store_and_unlock(Lsn(0x10))
    9768            2 :                 .wait()
    9769            2 :                 .await;
    9770            2 :             // Update GC info
    9771            2 :             let mut guard = parent_tline.gc_info.write().unwrap();
    9772            2 :             *guard = GcInfo {
    9773            2 :                 retain_lsns: vec![(Lsn(0x18), branch_tline.timeline_id, MaybeOffloaded::No)],
    9774            2 :                 cutoffs: GcCutoffs {
    9775            2 :                     time: Lsn(0x10),
    9776            2 :                     space: Lsn(0x10),
    9777            2 :                 },
    9778            2 :                 leases: Default::default(),
    9779            2 :                 within_ancestor_pitr: false,
    9780            2 :             };
    9781            2 :         }
    9782            2 : 
    9783            2 :         {
    9784            2 :             branch_tline
    9785            2 :                 .latest_gc_cutoff_lsn
    9786            2 :                 .lock_for_write()
    9787            2 :                 .store_and_unlock(Lsn(0x50))
    9788            2 :                 .wait()
    9789            2 :                 .await;
    9790            2 :             // Update GC info
    9791            2 :             let mut guard = branch_tline.gc_info.write().unwrap();
    9792            2 :             *guard = GcInfo {
    9793            2 :                 retain_lsns: vec![(Lsn(0x40), branch_tline.timeline_id, MaybeOffloaded::No)],
    9794            2 :                 cutoffs: GcCutoffs {
    9795            2 :                     time: Lsn(0x50),
    9796            2 :                     space: Lsn(0x50),
    9797            2 :                 },
    9798            2 :                 leases: Default::default(),
    9799            2 :                 within_ancestor_pitr: false,
    9800            2 :             };
    9801            2 :         }
    9802            2 : 
    9803            2 :         let expected_result_at_gc_horizon = [
    9804            2 :             Bytes::from_static(b"value 0@0x10"),
    9805            2 :             Bytes::from_static(b"value 1@0x10@0x20"),
    9806            2 :             Bytes::from_static(b"value 2@0x10@0x30"),
    9807            2 :             Bytes::from_static(b"value 3@0x10@0x28@0x30@0x40"),
    9808            2 :             Bytes::from_static(b"value 4@0x10"),
    9809            2 :             Bytes::from_static(b"value 5@0x10@0x20"),
    9810            2 :             Bytes::from_static(b"value 6@0x10@0x20"),
    9811            2 :             Bytes::from_static(b"value 7@0x10"),
    9812            2 :             Bytes::from_static(b"value 8@0x10@0x48"),
    9813            2 :             Bytes::from_static(b"value 9@0x10@0x48"),
    9814            2 :         ];
    9815            2 : 
    9816            2 :         let expected_result_at_lsn_40 = [
    9817            2 :             Bytes::from_static(b"value 0@0x10"),
    9818            2 :             Bytes::from_static(b"value 1@0x10@0x20"),
    9819            2 :             Bytes::from_static(b"value 2@0x10@0x30"),
    9820            2 :             Bytes::from_static(b"value 3@0x10@0x28@0x30@0x40"),
    9821            2 :             Bytes::from_static(b"value 4@0x10"),
    9822            2 :             Bytes::from_static(b"value 5@0x10@0x20"),
    9823            2 :             Bytes::from_static(b"value 6@0x10@0x20"),
    9824            2 :             Bytes::from_static(b"value 7@0x10"),
    9825            2 :             Bytes::from_static(b"value 8@0x10"),
    9826            2 :             Bytes::from_static(b"value 9@0x10"),
    9827            2 :         ];
    9828            2 : 
    9829            6 :         let verify_result = || async {
    9830           66 :             for idx in 0..10 {
    9831           60 :                 assert_eq!(
    9832           60 :                     branch_tline
    9833           60 :                         .get(get_key(idx as u32), Lsn(0x50), &ctx)
    9834           60 :                         .await
    9835           60 :                         .unwrap(),
    9836           60 :                     &expected_result_at_gc_horizon[idx]
    9837            2 :                 );
    9838           60 :                 assert_eq!(
    9839           60 :                     branch_tline
    9840           60 :                         .get(get_key(idx as u32), Lsn(0x40), &ctx)
    9841           60 :                         .await
    9842           60 :                         .unwrap(),
    9843           60 :                     &expected_result_at_lsn_40[idx]
    9844            2 :                 );
    9845            2 :             }
    9846           12 :         };
    9847            2 : 
    9848            2 :         verify_result().await;
    9849            2 : 
    9850            2 :         let cancel = CancellationToken::new();
    9851            2 :         branch_tline
    9852            2 :             .compact_with_gc(&cancel, CompactOptions::default(), &ctx)
    9853            2 :             .await
    9854            2 :             .unwrap();
    9855            2 : 
    9856            2 :         verify_result().await;
    9857            2 : 
    9858            2 :         // Piggyback a compaction with above_lsn. Ensure it works correctly when the specified LSN intersects with the layer files.
    9859            2 :         // Now we already have a single large delta layer, so the compaction min_layer_lsn should be the same as ancestor LSN (0x18).
    9860            2 :         branch_tline
    9861            2 :             .compact_with_gc(
    9862            2 :                 &cancel,
    9863            2 :                 CompactOptions {
    9864            2 :                     compact_lsn_range: Some(CompactLsnRange::above(Lsn(0x40))),
    9865            2 :                     ..Default::default()
    9866            2 :                 },
    9867            2 :                 &ctx,
    9868            2 :             )
    9869            2 :             .await
    9870            2 :             .unwrap();
    9871            2 : 
    9872            2 :         verify_result().await;
    9873            2 : 
    9874            2 :         Ok(())
    9875            2 :     }
    9876              : 
    9877              :     // Regression test for https://github.com/neondatabase/neon/issues/9012
    9878              :     // Create an image arrangement where we have to read at different LSN ranges
    9879              :     // from a delta layer. This is achieved by overlapping an image layer on top of
    9880              :     // a delta layer. Like so:
    9881              :     //
    9882              :     //     A      B
    9883              :     // +----------------+ -> delta_layer
    9884              :     // |                |                           ^ lsn
    9885              :     // |       =========|-> nested_image_layer      |
    9886              :     // |       C        |                           |
    9887              :     // +----------------+                           |
    9888              :     // ======== -> baseline_image_layer             +-------> key
    9889              :     //
    9890              :     //
    9891              :     // When querying the key range [A, B) we need to read at different LSN ranges
    9892              :     // for [A, C) and [C, B). This test checks that the described edge case is handled correctly.
    9893              :     #[cfg(feature = "testing")]
    9894              :     #[tokio::test]
    9895            2 :     async fn test_vectored_read_with_nested_image_layer() -> anyhow::Result<()> {
    9896            2 :         let harness = TenantHarness::create("test_vectored_read_with_nested_image_layer").await?;
    9897            2 :         let (tenant, ctx) = harness.load().await;
    9898            2 : 
    9899            2 :         let will_init_keys = [2, 6];
    9900           44 :         fn get_key(id: u32) -> Key {
    9901           44 :             let mut key = Key::from_hex("110000000033333333444444445500000000").unwrap();
    9902           44 :             key.field6 = id;
    9903           44 :             key
    9904           44 :         }
    9905            2 : 
    9906            2 :         let mut expected_key_values = HashMap::new();
    9907            2 : 
    9908            2 :         let baseline_image_layer_lsn = Lsn(0x10);
    9909            2 :         let mut baseline_img_layer = Vec::new();
    9910           12 :         for i in 0..5 {
    9911           10 :             let key = get_key(i);
    9912           10 :             let value = format!("value {i}@{baseline_image_layer_lsn}");
    9913           10 : 
    9914           10 :             let removed = expected_key_values.insert(key, value.clone());
    9915           10 :             assert!(removed.is_none());
    9916            2 : 
    9917           10 :             baseline_img_layer.push((key, Bytes::from(value)));
    9918            2 :         }
    9919            2 : 
    9920            2 :         let nested_image_layer_lsn = Lsn(0x50);
    9921            2 :         let mut nested_img_layer = Vec::new();
    9922           12 :         for i in 5..10 {
    9923           10 :             let key = get_key(i);
    9924           10 :             let value = format!("value {i}@{nested_image_layer_lsn}");
    9925           10 : 
    9926           10 :             let removed = expected_key_values.insert(key, value.clone());
    9927           10 :             assert!(removed.is_none());
    9928            2 : 
    9929           10 :             nested_img_layer.push((key, Bytes::from(value)));
    9930            2 :         }
    9931            2 : 
    9932            2 :         let mut delta_layer_spec = Vec::default();
    9933            2 :         let delta_layer_start_lsn = Lsn(0x20);
    9934            2 :         let mut delta_layer_end_lsn = delta_layer_start_lsn;
    9935            2 : 
    9936           22 :         for i in 0..10 {
    9937           20 :             let key = get_key(i);
    9938           20 :             let key_in_nested = nested_img_layer
    9939           20 :                 .iter()
    9940           80 :                 .any(|(key_with_img, _)| *key_with_img == key);
    9941           20 :             let lsn = {
    9942           20 :                 if key_in_nested {
    9943           10 :                     Lsn(nested_image_layer_lsn.0 + 0x10)
    9944            2 :                 } else {
    9945           10 :                     delta_layer_start_lsn
    9946            2 :                 }
    9947            2 :             };
    9948            2 : 
    9949           20 :             let will_init = will_init_keys.contains(&i);
    9950           20 :             if will_init {
    9951            4 :                 delta_layer_spec.push((key, lsn, Value::WalRecord(NeonWalRecord::wal_init(""))));
    9952            4 : 
    9953            4 :                 expected_key_values.insert(key, "".to_string());
    9954           16 :             } else {
    9955           16 :                 let delta = format!("@{lsn}");
    9956           16 :                 delta_layer_spec.push((
    9957           16 :                     key,
    9958           16 :                     lsn,
    9959           16 :                     Value::WalRecord(NeonWalRecord::wal_append(&delta)),
    9960           16 :                 ));
    9961           16 : 
    9962           16 :                 expected_key_values
    9963           16 :                     .get_mut(&key)
    9964           16 :                     .expect("An image exists for each key")
    9965           16 :                     .push_str(delta.as_str());
    9966           16 :             }
    9967           20 :             delta_layer_end_lsn = std::cmp::max(delta_layer_start_lsn, lsn);
    9968            2 :         }
    9969            2 : 
    9970            2 :         delta_layer_end_lsn = Lsn(delta_layer_end_lsn.0 + 1);
    9971            2 : 
    9972            2 :         assert!(
    9973            2 :             nested_image_layer_lsn > delta_layer_start_lsn
    9974            2 :                 && nested_image_layer_lsn < delta_layer_end_lsn
    9975            2 :         );
    9976            2 : 
    9977            2 :         let tline = tenant
    9978            2 :             .create_test_timeline_with_layers(
    9979            2 :                 TIMELINE_ID,
    9980            2 :                 baseline_image_layer_lsn,
    9981            2 :                 DEFAULT_PG_VERSION,
    9982            2 :                 &ctx,
    9983            2 :                 vec![DeltaLayerTestDesc::new_with_inferred_key_range(
    9984            2 :                     delta_layer_start_lsn..delta_layer_end_lsn,
    9985            2 :                     delta_layer_spec,
    9986            2 :                 )], // delta layers
    9987            2 :                 vec![
    9988            2 :                     (baseline_image_layer_lsn, baseline_img_layer),
    9989            2 :                     (nested_image_layer_lsn, nested_img_layer),
    9990            2 :                 ], // image layers
    9991            2 :                 delta_layer_end_lsn,
    9992            2 :             )
    9993            2 :             .await?;
    9994            2 : 
    9995            2 :         let keyspace = KeySpace::single(get_key(0)..get_key(10));
    9996            2 :         let results = tline
    9997            2 :             .get_vectored(keyspace, delta_layer_end_lsn, &ctx)
    9998            2 :             .await
    9999            2 :             .expect("No vectored errors");
   10000           22 :         for (key, res) in results {
   10001           20 :             let value = res.expect("No key errors");
   10002           20 :             let expected_value = expected_key_values.remove(&key).expect("No unknown keys");
   10003           20 :             assert_eq!(value, Bytes::from(expected_value));
   10004            2 :         }
   10005            2 : 
   10006            2 :         Ok(())
   10007            2 :     }
   10008              : 
   10009          214 :     fn sort_layer_key(k1: &PersistentLayerKey, k2: &PersistentLayerKey) -> std::cmp::Ordering {
   10010          214 :         (
   10011          214 :             k1.is_delta,
   10012          214 :             k1.key_range.start,
   10013          214 :             k1.key_range.end,
   10014          214 :             k1.lsn_range.start,
   10015          214 :             k1.lsn_range.end,
   10016          214 :         )
   10017          214 :             .cmp(&(
   10018          214 :                 k2.is_delta,
   10019          214 :                 k2.key_range.start,
   10020          214 :                 k2.key_range.end,
   10021          214 :                 k2.lsn_range.start,
   10022          214 :                 k2.lsn_range.end,
   10023          214 :             ))
   10024          214 :     }
   10025              : 
   10026           24 :     async fn inspect_and_sort(
   10027           24 :         tline: &Arc<Timeline>,
   10028           24 :         filter: Option<std::ops::Range<Key>>,
   10029           24 :     ) -> Vec<PersistentLayerKey> {
   10030           24 :         let mut all_layers = tline.inspect_historic_layers().await.unwrap();
   10031           24 :         if let Some(filter) = filter {
   10032          108 :             all_layers.retain(|layer| overlaps_with(&layer.key_range, &filter));
   10033           22 :         }
   10034           24 :         all_layers.sort_by(sort_layer_key);
   10035           24 :         all_layers
   10036           24 :     }
   10037              : 
   10038              :     #[cfg(feature = "testing")]
   10039           22 :     fn check_layer_map_key_eq(
   10040           22 :         mut left: Vec<PersistentLayerKey>,
   10041           22 :         mut right: Vec<PersistentLayerKey>,
   10042           22 :     ) {
   10043           22 :         left.sort_by(sort_layer_key);
   10044           22 :         right.sort_by(sort_layer_key);
   10045           22 :         if left != right {
   10046            0 :             eprintln!("---LEFT---");
   10047            0 :             for left in left.iter() {
   10048            0 :                 eprintln!("{}", left);
   10049            0 :             }
   10050            0 :             eprintln!("---RIGHT---");
   10051            0 :             for right in right.iter() {
   10052            0 :                 eprintln!("{}", right);
   10053            0 :             }
   10054            0 :             assert_eq!(left, right);
   10055           22 :         }
   10056           22 :     }
   10057              : 
   10058              :     #[cfg(feature = "testing")]
   10059              :     #[tokio::test]
   10060            2 :     async fn test_simple_partial_bottom_most_compaction() -> anyhow::Result<()> {
   10061            2 :         let harness = TenantHarness::create("test_simple_partial_bottom_most_compaction").await?;
   10062            2 :         let (tenant, ctx) = harness.load().await;
   10063            2 : 
   10064          182 :         fn get_key(id: u32) -> Key {
   10065          182 :             // using aux key here b/c they are guaranteed to be inside `collect_keyspace`.
   10066          182 :             let mut key = Key::from_hex("620000000033333333444444445500000000").unwrap();
   10067          182 :             key.field6 = id;
   10068          182 :             key
   10069          182 :         }
   10070            2 : 
   10071            2 :         // img layer at 0x10
   10072            2 :         let img_layer = (0..10)
   10073           20 :             .map(|id| (get_key(id), Bytes::from(format!("value {id}@0x10"))))
   10074            2 :             .collect_vec();
   10075            2 : 
   10076            2 :         let delta1 = vec![
   10077            2 :             (
   10078            2 :                 get_key(1),
   10079            2 :                 Lsn(0x20),
   10080            2 :                 Value::Image(Bytes::from("value 1@0x20")),
   10081            2 :             ),
   10082            2 :             (
   10083            2 :                 get_key(2),
   10084            2 :                 Lsn(0x30),
   10085            2 :                 Value::Image(Bytes::from("value 2@0x30")),
   10086            2 :             ),
   10087            2 :             (
   10088            2 :                 get_key(3),
   10089            2 :                 Lsn(0x40),
   10090            2 :                 Value::Image(Bytes::from("value 3@0x40")),
   10091            2 :             ),
   10092            2 :         ];
   10093            2 :         let delta2 = vec![
   10094            2 :             (
   10095            2 :                 get_key(5),
   10096            2 :                 Lsn(0x20),
   10097            2 :                 Value::Image(Bytes::from("value 5@0x20")),
   10098            2 :             ),
   10099            2 :             (
   10100            2 :                 get_key(6),
   10101            2 :                 Lsn(0x20),
   10102            2 :                 Value::Image(Bytes::from("value 6@0x20")),
   10103            2 :             ),
   10104            2 :         ];
   10105            2 :         let delta3 = vec![
   10106            2 :             (
   10107            2 :                 get_key(8),
   10108            2 :                 Lsn(0x48),
   10109            2 :                 Value::Image(Bytes::from("value 8@0x48")),
   10110            2 :             ),
   10111            2 :             (
   10112            2 :                 get_key(9),
   10113            2 :                 Lsn(0x48),
   10114            2 :                 Value::Image(Bytes::from("value 9@0x48")),
   10115            2 :             ),
   10116            2 :         ];
   10117            2 : 
   10118            2 :         let tline = tenant
   10119            2 :             .create_test_timeline_with_layers(
   10120            2 :                 TIMELINE_ID,
   10121            2 :                 Lsn(0x10),
   10122            2 :                 DEFAULT_PG_VERSION,
   10123            2 :                 &ctx,
   10124            2 :                 vec![
   10125            2 :                     DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x20)..Lsn(0x48), delta1),
   10126            2 :                     DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x20)..Lsn(0x48), delta2),
   10127            2 :                     DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x48)..Lsn(0x50), delta3),
   10128            2 :                 ], // delta layers
   10129            2 :                 vec![(Lsn(0x10), img_layer)], // image layers
   10130            2 :                 Lsn(0x50),
   10131            2 :             )
   10132            2 :             .await?;
   10133            2 : 
   10134            2 :         {
   10135            2 :             tline
   10136            2 :                 .latest_gc_cutoff_lsn
   10137            2 :                 .lock_for_write()
   10138            2 :                 .store_and_unlock(Lsn(0x30))
   10139            2 :                 .wait()
   10140            2 :                 .await;
   10141            2 :             // Update GC info
   10142            2 :             let mut guard = tline.gc_info.write().unwrap();
   10143            2 :             *guard = GcInfo {
   10144            2 :                 retain_lsns: vec![(Lsn(0x20), tline.timeline_id, MaybeOffloaded::No)],
   10145            2 :                 cutoffs: GcCutoffs {
   10146            2 :                     time: Lsn(0x30),
   10147            2 :                     space: Lsn(0x30),
   10148            2 :                 },
   10149            2 :                 leases: Default::default(),
   10150            2 :                 within_ancestor_pitr: false,
   10151            2 :             };
   10152            2 :         }
   10153            2 : 
   10154            2 :         let cancel = CancellationToken::new();
   10155            2 : 
   10156            2 :         // Do a partial compaction on key range 0..2
   10157            2 :         tline
   10158            2 :             .compact_with_gc(
   10159            2 :                 &cancel,
   10160            2 :                 CompactOptions {
   10161            2 :                     flags: EnumSet::new(),
   10162            2 :                     compact_key_range: Some((get_key(0)..get_key(2)).into()),
   10163            2 :                     ..Default::default()
   10164            2 :                 },
   10165            2 :                 &ctx,
   10166            2 :             )
   10167            2 :             .await
   10168            2 :             .unwrap();
   10169            2 :         let all_layers = inspect_and_sort(&tline, Some(get_key(0)..get_key(10))).await;
   10170            2 :         check_layer_map_key_eq(
   10171            2 :             all_layers,
   10172            2 :             vec![
   10173            2 :                 // newly-generated image layer for the partial compaction range 0-2
   10174            2 :                 PersistentLayerKey {
   10175            2 :                     key_range: get_key(0)..get_key(2),
   10176            2 :                     lsn_range: Lsn(0x20)..Lsn(0x21),
   10177            2 :                     is_delta: false,
   10178            2 :                 },
   10179            2 :                 PersistentLayerKey {
   10180            2 :                     key_range: get_key(0)..get_key(10),
   10181            2 :                     lsn_range: Lsn(0x10)..Lsn(0x11),
   10182            2 :                     is_delta: false,
   10183            2 :                 },
   10184            2 :                 // delta1 is split and the second part is rewritten
   10185            2 :                 PersistentLayerKey {
   10186            2 :                     key_range: get_key(2)..get_key(4),
   10187            2 :                     lsn_range: Lsn(0x20)..Lsn(0x48),
   10188            2 :                     is_delta: true,
   10189            2 :                 },
   10190            2 :                 PersistentLayerKey {
   10191            2 :                     key_range: get_key(5)..get_key(7),
   10192            2 :                     lsn_range: Lsn(0x20)..Lsn(0x48),
   10193            2 :                     is_delta: true,
   10194            2 :                 },
   10195            2 :                 PersistentLayerKey {
   10196            2 :                     key_range: get_key(8)..get_key(10),
   10197            2 :                     lsn_range: Lsn(0x48)..Lsn(0x50),
   10198            2 :                     is_delta: true,
   10199            2 :                 },
   10200            2 :             ],
   10201            2 :         );
   10202            2 : 
   10203            2 :         // Do a partial compaction on key range 2..4
   10204            2 :         tline
   10205            2 :             .compact_with_gc(
   10206            2 :                 &cancel,
   10207            2 :                 CompactOptions {
   10208            2 :                     flags: EnumSet::new(),
   10209            2 :                     compact_key_range: Some((get_key(2)..get_key(4)).into()),
   10210            2 :                     ..Default::default()
   10211            2 :                 },
   10212            2 :                 &ctx,
   10213            2 :             )
   10214            2 :             .await
   10215            2 :             .unwrap();
   10216            2 :         let all_layers = inspect_and_sort(&tline, Some(get_key(0)..get_key(10))).await;
   10217            2 :         check_layer_map_key_eq(
   10218            2 :             all_layers,
   10219            2 :             vec![
   10220            2 :                 PersistentLayerKey {
   10221            2 :                     key_range: get_key(0)..get_key(2),
   10222            2 :                     lsn_range: Lsn(0x20)..Lsn(0x21),
   10223            2 :                     is_delta: false,
   10224            2 :                 },
   10225            2 :                 PersistentLayerKey {
   10226            2 :                     key_range: get_key(0)..get_key(10),
   10227            2 :                     lsn_range: Lsn(0x10)..Lsn(0x11),
   10228            2 :                     is_delta: false,
   10229            2 :                 },
   10230            2 :                 // image layer generated for the compaction range 2-4
   10231            2 :                 PersistentLayerKey {
   10232            2 :                     key_range: get_key(2)..get_key(4),
   10233            2 :                     lsn_range: Lsn(0x20)..Lsn(0x21),
   10234            2 :                     is_delta: false,
   10235            2 :                 },
   10236            2 :                 // we have key2/key3 above the retain_lsn, so we still need this delta layer
   10237            2 :                 PersistentLayerKey {
   10238            2 :                     key_range: get_key(2)..get_key(4),
   10239            2 :                     lsn_range: Lsn(0x20)..Lsn(0x48),
   10240            2 :                     is_delta: true,
   10241            2 :                 },
   10242            2 :                 PersistentLayerKey {
   10243            2 :                     key_range: get_key(5)..get_key(7),
   10244            2 :                     lsn_range: Lsn(0x20)..Lsn(0x48),
   10245            2 :                     is_delta: true,
   10246            2 :                 },
   10247            2 :                 PersistentLayerKey {
   10248            2 :                     key_range: get_key(8)..get_key(10),
   10249            2 :                     lsn_range: Lsn(0x48)..Lsn(0x50),
   10250            2 :                     is_delta: true,
   10251            2 :                 },
   10252            2 :             ],
   10253            2 :         );
   10254            2 : 
   10255            2 :         // Do a partial compaction on key range 4..9
   10256            2 :         tline
   10257            2 :             .compact_with_gc(
   10258            2 :                 &cancel,
   10259            2 :                 CompactOptions {
   10260            2 :                     flags: EnumSet::new(),
   10261            2 :                     compact_key_range: Some((get_key(4)..get_key(9)).into()),
   10262            2 :                     ..Default::default()
   10263            2 :                 },
   10264            2 :                 &ctx,
   10265            2 :             )
   10266            2 :             .await
   10267            2 :             .unwrap();
   10268            2 :         let all_layers = inspect_and_sort(&tline, Some(get_key(0)..get_key(10))).await;
   10269            2 :         check_layer_map_key_eq(
   10270            2 :             all_layers,
   10271            2 :             vec![
   10272            2 :                 PersistentLayerKey {
   10273            2 :                     key_range: get_key(0)..get_key(2),
   10274            2 :                     lsn_range: Lsn(0x20)..Lsn(0x21),
   10275            2 :                     is_delta: false,
   10276            2 :                 },
   10277            2 :                 PersistentLayerKey {
   10278            2 :                     key_range: get_key(0)..get_key(10),
   10279            2 :                     lsn_range: Lsn(0x10)..Lsn(0x11),
   10280            2 :                     is_delta: false,
   10281            2 :                 },
   10282            2 :                 PersistentLayerKey {
   10283            2 :                     key_range: get_key(2)..get_key(4),
   10284            2 :                     lsn_range: Lsn(0x20)..Lsn(0x21),
   10285            2 :                     is_delta: false,
   10286            2 :                 },
   10287            2 :                 PersistentLayerKey {
   10288            2 :                     key_range: get_key(2)..get_key(4),
   10289            2 :                     lsn_range: Lsn(0x20)..Lsn(0x48),
   10290            2 :                     is_delta: true,
   10291            2 :                 },
   10292            2 :                 // image layer generated for this compaction range
   10293            2 :                 PersistentLayerKey {
   10294            2 :                     key_range: get_key(4)..get_key(9),
   10295            2 :                     lsn_range: Lsn(0x20)..Lsn(0x21),
   10296            2 :                     is_delta: false,
   10297            2 :                 },
   10298            2 :                 PersistentLayerKey {
   10299            2 :                     key_range: get_key(8)..get_key(10),
   10300            2 :                     lsn_range: Lsn(0x48)..Lsn(0x50),
   10301            2 :                     is_delta: true,
   10302            2 :                 },
   10303            2 :             ],
   10304            2 :         );
   10305            2 : 
   10306            2 :         // Do a partial compaction on key range 9..10
   10307            2 :         tline
   10308            2 :             .compact_with_gc(
   10309            2 :                 &cancel,
   10310            2 :                 CompactOptions {
   10311            2 :                     flags: EnumSet::new(),
   10312            2 :                     compact_key_range: Some((get_key(9)..get_key(10)).into()),
   10313            2 :                     ..Default::default()
   10314            2 :                 },
   10315            2 :                 &ctx,
   10316            2 :             )
   10317            2 :             .await
   10318            2 :             .unwrap();
   10319            2 :         let all_layers = inspect_and_sort(&tline, Some(get_key(0)..get_key(10))).await;
   10320            2 :         check_layer_map_key_eq(
   10321            2 :             all_layers,
   10322            2 :             vec![
   10323            2 :                 PersistentLayerKey {
   10324            2 :                     key_range: get_key(0)..get_key(2),
   10325            2 :                     lsn_range: Lsn(0x20)..Lsn(0x21),
   10326            2 :                     is_delta: false,
   10327            2 :                 },
   10328            2 :                 PersistentLayerKey {
   10329            2 :                     key_range: get_key(0)..get_key(10),
   10330            2 :                     lsn_range: Lsn(0x10)..Lsn(0x11),
   10331            2 :                     is_delta: false,
   10332            2 :                 },
   10333            2 :                 PersistentLayerKey {
   10334            2 :                     key_range: get_key(2)..get_key(4),
   10335            2 :                     lsn_range: Lsn(0x20)..Lsn(0x21),
   10336            2 :                     is_delta: false,
   10337            2 :                 },
   10338            2 :                 PersistentLayerKey {
   10339            2 :                     key_range: get_key(2)..get_key(4),
   10340            2 :                     lsn_range: Lsn(0x20)..Lsn(0x48),
   10341            2 :                     is_delta: true,
   10342            2 :                 },
   10343            2 :                 PersistentLayerKey {
   10344            2 :                     key_range: get_key(4)..get_key(9),
   10345            2 :                     lsn_range: Lsn(0x20)..Lsn(0x21),
   10346            2 :                     is_delta: false,
   10347            2 :                 },
   10348            2 :                 // image layer generated for the compaction range
   10349            2 :                 PersistentLayerKey {
   10350            2 :                     key_range: get_key(9)..get_key(10),
   10351            2 :                     lsn_range: Lsn(0x20)..Lsn(0x21),
   10352            2 :                     is_delta: false,
   10353            2 :                 },
   10354            2 :                 PersistentLayerKey {
   10355            2 :                     key_range: get_key(8)..get_key(10),
   10356            2 :                     lsn_range: Lsn(0x48)..Lsn(0x50),
   10357            2 :                     is_delta: true,
   10358            2 :                 },
   10359            2 :             ],
   10360            2 :         );
   10361            2 : 
   10362            2 :         // Do a partial compaction on key range 0..10, all image layers below LSN 20 can be replaced with new ones.
   10363            2 :         tline
   10364            2 :             .compact_with_gc(
   10365            2 :                 &cancel,
   10366            2 :                 CompactOptions {
   10367            2 :                     flags: EnumSet::new(),
   10368            2 :                     compact_key_range: Some((get_key(0)..get_key(10)).into()),
   10369            2 :                     ..Default::default()
   10370            2 :                 },
   10371            2 :                 &ctx,
   10372            2 :             )
   10373            2 :             .await
   10374            2 :             .unwrap();
   10375            2 :         let all_layers = inspect_and_sort(&tline, Some(get_key(0)..get_key(10))).await;
   10376            2 :         check_layer_map_key_eq(
   10377            2 :             all_layers,
   10378            2 :             vec![
   10379            2 :                 // aha, we removed all unnecessary image/delta layers and got a very clean layer map!
   10380            2 :                 PersistentLayerKey {
   10381            2 :                     key_range: get_key(0)..get_key(10),
   10382            2 :                     lsn_range: Lsn(0x20)..Lsn(0x21),
   10383            2 :                     is_delta: false,
   10384            2 :                 },
   10385            2 :                 PersistentLayerKey {
   10386            2 :                     key_range: get_key(2)..get_key(4),
   10387            2 :                     lsn_range: Lsn(0x20)..Lsn(0x48),
   10388            2 :                     is_delta: true,
   10389            2 :                 },
   10390            2 :                 PersistentLayerKey {
   10391            2 :                     key_range: get_key(8)..get_key(10),
   10392            2 :                     lsn_range: Lsn(0x48)..Lsn(0x50),
   10393            2 :                     is_delta: true,
   10394            2 :                 },
   10395            2 :             ],
   10396            2 :         );
   10397            2 :         Ok(())
   10398            2 :     }
   10399              : 
   10400              :     #[cfg(feature = "testing")]
   10401              :     #[tokio::test]
   10402            2 :     async fn test_timeline_offload_retain_lsn() -> anyhow::Result<()> {
   10403            2 :         let harness = TenantHarness::create("test_timeline_offload_retain_lsn")
   10404            2 :             .await
   10405            2 :             .unwrap();
   10406            2 :         let (tenant, ctx) = harness.load().await;
   10407            2 :         let tline_parent = tenant
   10408            2 :             .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx)
   10409            2 :             .await
   10410            2 :             .unwrap();
   10411            2 :         let tline_child = tenant
   10412            2 :             .branch_timeline_test(&tline_parent, NEW_TIMELINE_ID, Some(Lsn(0x20)), &ctx)
   10413            2 :             .await
   10414            2 :             .unwrap();
   10415            2 :         {
   10416            2 :             let gc_info_parent = tline_parent.gc_info.read().unwrap();
   10417            2 :             assert_eq!(
   10418            2 :                 gc_info_parent.retain_lsns,
   10419            2 :                 vec![(Lsn(0x20), tline_child.timeline_id, MaybeOffloaded::No)]
   10420            2 :             );
   10421            2 :         }
   10422            2 :         // We have to directly call the remote_client instead of using the archive function to avoid constructing broker client...
   10423            2 :         tline_child
   10424            2 :             .remote_client
   10425            2 :             .schedule_index_upload_for_timeline_archival_state(TimelineArchivalState::Archived)
   10426            2 :             .unwrap();
   10427            2 :         tline_child.remote_client.wait_completion().await.unwrap();
   10428            2 :         offload_timeline(&tenant, &tline_child)
   10429            2 :             .instrument(tracing::info_span!(parent: None, "offload_test", tenant_id=%"test", shard_id=%"test", timeline_id=%"test"))
   10430            2 :             .await.unwrap();
   10431            2 :         let child_timeline_id = tline_child.timeline_id;
   10432            2 :         Arc::try_unwrap(tline_child).unwrap();
   10433            2 : 
   10434            2 :         {
   10435            2 :             let gc_info_parent = tline_parent.gc_info.read().unwrap();
   10436            2 :             assert_eq!(
   10437            2 :                 gc_info_parent.retain_lsns,
   10438            2 :                 vec![(Lsn(0x20), child_timeline_id, MaybeOffloaded::Yes)]
   10439            2 :             );
   10440            2 :         }
   10441            2 : 
   10442            2 :         tenant
   10443            2 :             .get_offloaded_timeline(child_timeline_id)
   10444            2 :             .unwrap()
   10445            2 :             .defuse_for_tenant_drop();
   10446            2 : 
   10447            2 :         Ok(())
   10448            2 :     }
   10449              : 
   10450              :     #[cfg(feature = "testing")]
   10451              :     #[tokio::test]
   10452            2 :     async fn test_simple_bottom_most_compaction_above_lsn() -> anyhow::Result<()> {
   10453            2 :         let harness = TenantHarness::create("test_simple_bottom_most_compaction_above_lsn").await?;
   10454            2 :         let (tenant, ctx) = harness.load().await;
   10455            2 : 
   10456          296 :         fn get_key(id: u32) -> Key {
   10457          296 :             // using aux key here b/c they are guaranteed to be inside `collect_keyspace`.
   10458          296 :             let mut key = Key::from_hex("620000000033333333444444445500000000").unwrap();
   10459          296 :             key.field6 = id;
   10460          296 :             key
   10461          296 :         }
   10462            2 : 
   10463            2 :         let img_layer = (0..10)
   10464           20 :             .map(|id| (get_key(id), Bytes::from(format!("value {id}@0x10"))))
   10465            2 :             .collect_vec();
   10466            2 : 
   10467            2 :         let delta1 = vec![(
   10468            2 :             get_key(1),
   10469            2 :             Lsn(0x20),
   10470            2 :             Value::WalRecord(NeonWalRecord::wal_append("@0x20")),
   10471            2 :         )];
   10472            2 :         let delta4 = vec![(
   10473            2 :             get_key(1),
   10474            2 :             Lsn(0x28),
   10475            2 :             Value::WalRecord(NeonWalRecord::wal_append("@0x28")),
   10476            2 :         )];
   10477            2 :         let delta2 = vec![
   10478            2 :             (
   10479            2 :                 get_key(1),
   10480            2 :                 Lsn(0x30),
   10481            2 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x30")),
   10482            2 :             ),
   10483            2 :             (
   10484            2 :                 get_key(1),
   10485            2 :                 Lsn(0x38),
   10486            2 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x38")),
   10487            2 :             ),
   10488            2 :         ];
   10489            2 :         let delta3 = vec![
   10490            2 :             (
   10491            2 :                 get_key(8),
   10492            2 :                 Lsn(0x48),
   10493            2 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x48")),
   10494            2 :             ),
   10495            2 :             (
   10496            2 :                 get_key(9),
   10497            2 :                 Lsn(0x48),
   10498            2 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x48")),
   10499            2 :             ),
   10500            2 :         ];
   10501            2 : 
   10502            2 :         let tline = tenant
   10503            2 :             .create_test_timeline_with_layers(
   10504            2 :                 TIMELINE_ID,
   10505            2 :                 Lsn(0x10),
   10506            2 :                 DEFAULT_PG_VERSION,
   10507            2 :                 &ctx,
   10508            2 :                 vec![
   10509            2 :                     // delta1/2/4 only contain a single key but multiple updates
   10510            2 :                     DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x20)..Lsn(0x28), delta1),
   10511            2 :                     DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x30)..Lsn(0x50), delta2),
   10512            2 :                     DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x28)..Lsn(0x30), delta4),
   10513            2 :                     DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x30)..Lsn(0x50), delta3),
   10514            2 :                 ], // delta layers
   10515            2 :                 vec![(Lsn(0x10), img_layer)], // image layers
   10516            2 :                 Lsn(0x50),
   10517            2 :             )
   10518            2 :             .await?;
   10519            2 :         {
   10520            2 :             tline
   10521            2 :                 .latest_gc_cutoff_lsn
   10522            2 :                 .lock_for_write()
   10523            2 :                 .store_and_unlock(Lsn(0x30))
   10524            2 :                 .wait()
   10525            2 :                 .await;
   10526            2 :             // Update GC info
   10527            2 :             let mut guard = tline.gc_info.write().unwrap();
   10528            2 :             *guard = GcInfo {
   10529            2 :                 retain_lsns: vec![
   10530            2 :                     (Lsn(0x10), tline.timeline_id, MaybeOffloaded::No),
   10531            2 :                     (Lsn(0x20), tline.timeline_id, MaybeOffloaded::No),
   10532            2 :                 ],
   10533            2 :                 cutoffs: GcCutoffs {
   10534            2 :                     time: Lsn(0x30),
   10535            2 :                     space: Lsn(0x30),
   10536            2 :                 },
   10537            2 :                 leases: Default::default(),
   10538            2 :                 within_ancestor_pitr: false,
   10539            2 :             };
   10540            2 :         }
   10541            2 : 
   10542            2 :         let expected_result = [
   10543            2 :             Bytes::from_static(b"value 0@0x10"),
   10544            2 :             Bytes::from_static(b"value 1@0x10@0x20@0x28@0x30@0x38"),
   10545            2 :             Bytes::from_static(b"value 2@0x10"),
   10546            2 :             Bytes::from_static(b"value 3@0x10"),
   10547            2 :             Bytes::from_static(b"value 4@0x10"),
   10548            2 :             Bytes::from_static(b"value 5@0x10"),
   10549            2 :             Bytes::from_static(b"value 6@0x10"),
   10550            2 :             Bytes::from_static(b"value 7@0x10"),
   10551            2 :             Bytes::from_static(b"value 8@0x10@0x48"),
   10552            2 :             Bytes::from_static(b"value 9@0x10@0x48"),
   10553            2 :         ];
   10554            2 : 
   10555            2 :         let expected_result_at_gc_horizon = [
   10556            2 :             Bytes::from_static(b"value 0@0x10"),
   10557            2 :             Bytes::from_static(b"value 1@0x10@0x20@0x28@0x30"),
   10558            2 :             Bytes::from_static(b"value 2@0x10"),
   10559            2 :             Bytes::from_static(b"value 3@0x10"),
   10560            2 :             Bytes::from_static(b"value 4@0x10"),
   10561            2 :             Bytes::from_static(b"value 5@0x10"),
   10562            2 :             Bytes::from_static(b"value 6@0x10"),
   10563            2 :             Bytes::from_static(b"value 7@0x10"),
   10564            2 :             Bytes::from_static(b"value 8@0x10"),
   10565            2 :             Bytes::from_static(b"value 9@0x10"),
   10566            2 :         ];
   10567            2 : 
   10568            2 :         let expected_result_at_lsn_20 = [
   10569            2 :             Bytes::from_static(b"value 0@0x10"),
   10570            2 :             Bytes::from_static(b"value 1@0x10@0x20"),
   10571            2 :             Bytes::from_static(b"value 2@0x10"),
   10572            2 :             Bytes::from_static(b"value 3@0x10"),
   10573            2 :             Bytes::from_static(b"value 4@0x10"),
   10574            2 :             Bytes::from_static(b"value 5@0x10"),
   10575            2 :             Bytes::from_static(b"value 6@0x10"),
   10576            2 :             Bytes::from_static(b"value 7@0x10"),
   10577            2 :             Bytes::from_static(b"value 8@0x10"),
   10578            2 :             Bytes::from_static(b"value 9@0x10"),
   10579            2 :         ];
   10580            2 : 
   10581            2 :         let expected_result_at_lsn_10 = [
   10582            2 :             Bytes::from_static(b"value 0@0x10"),
   10583            2 :             Bytes::from_static(b"value 1@0x10"),
   10584            2 :             Bytes::from_static(b"value 2@0x10"),
   10585            2 :             Bytes::from_static(b"value 3@0x10"),
   10586            2 :             Bytes::from_static(b"value 4@0x10"),
   10587            2 :             Bytes::from_static(b"value 5@0x10"),
   10588            2 :             Bytes::from_static(b"value 6@0x10"),
   10589            2 :             Bytes::from_static(b"value 7@0x10"),
   10590            2 :             Bytes::from_static(b"value 8@0x10"),
   10591            2 :             Bytes::from_static(b"value 9@0x10"),
   10592            2 :         ];
   10593            2 : 
   10594            6 :         let verify_result = || async {
   10595            6 :             let gc_horizon = {
   10596            6 :                 let gc_info = tline.gc_info.read().unwrap();
   10597            6 :                 gc_info.cutoffs.time
   10598            2 :             };
   10599           66 :             for idx in 0..10 {
   10600           60 :                 assert_eq!(
   10601           60 :                     tline
   10602           60 :                         .get(get_key(idx as u32), Lsn(0x50), &ctx)
   10603           60 :                         .await
   10604           60 :                         .unwrap(),
   10605           60 :                     &expected_result[idx]
   10606            2 :                 );
   10607           60 :                 assert_eq!(
   10608           60 :                     tline
   10609           60 :                         .get(get_key(idx as u32), gc_horizon, &ctx)
   10610           60 :                         .await
   10611           60 :                         .unwrap(),
   10612           60 :                     &expected_result_at_gc_horizon[idx]
   10613            2 :                 );
   10614           60 :                 assert_eq!(
   10615           60 :                     tline
   10616           60 :                         .get(get_key(idx as u32), Lsn(0x20), &ctx)
   10617           60 :                         .await
   10618           60 :                         .unwrap(),
   10619           60 :                     &expected_result_at_lsn_20[idx]
   10620            2 :                 );
   10621           60 :                 assert_eq!(
   10622           60 :                     tline
   10623           60 :                         .get(get_key(idx as u32), Lsn(0x10), &ctx)
   10624           60 :                         .await
   10625           60 :                         .unwrap(),
   10626           60 :                     &expected_result_at_lsn_10[idx]
   10627            2 :                 );
   10628            2 :             }
   10629           12 :         };
   10630            2 : 
   10631            2 :         verify_result().await;
   10632            2 : 
   10633            2 :         let cancel = CancellationToken::new();
   10634            2 :         tline
   10635            2 :             .compact_with_gc(
   10636            2 :                 &cancel,
   10637            2 :                 CompactOptions {
   10638            2 :                     compact_lsn_range: Some(CompactLsnRange::above(Lsn(0x28))),
   10639            2 :                     ..Default::default()
   10640            2 :                 },
   10641            2 :                 &ctx,
   10642            2 :             )
   10643            2 :             .await
   10644            2 :             .unwrap();
   10645            2 :         verify_result().await;
   10646            2 : 
   10647            2 :         let all_layers = inspect_and_sort(&tline, Some(get_key(0)..get_key(10))).await;
   10648            2 :         check_layer_map_key_eq(
   10649            2 :             all_layers,
   10650            2 :             vec![
   10651            2 :                 // The original image layer, not compacted
   10652            2 :                 PersistentLayerKey {
   10653            2 :                     key_range: get_key(0)..get_key(10),
   10654            2 :                     lsn_range: Lsn(0x10)..Lsn(0x11),
   10655            2 :                     is_delta: false,
   10656            2 :                 },
   10657            2 :                 // Delta layer below the specified above_lsn not compacted
   10658            2 :                 PersistentLayerKey {
   10659            2 :                     key_range: get_key(1)..get_key(2),
   10660            2 :                     lsn_range: Lsn(0x20)..Lsn(0x28),
   10661            2 :                     is_delta: true,
   10662            2 :                 },
   10663            2 :                 // Delta layer compacted above the LSN
   10664            2 :                 PersistentLayerKey {
   10665            2 :                     key_range: get_key(1)..get_key(10),
   10666            2 :                     lsn_range: Lsn(0x28)..Lsn(0x50),
   10667            2 :                     is_delta: true,
   10668            2 :                 },
   10669            2 :             ],
   10670            2 :         );
   10671            2 : 
   10672            2 :         // compact again
   10673            2 :         tline
   10674            2 :             .compact_with_gc(&cancel, CompactOptions::default(), &ctx)
   10675            2 :             .await
   10676            2 :             .unwrap();
   10677            2 :         verify_result().await;
   10678            2 : 
   10679            2 :         let all_layers = inspect_and_sort(&tline, Some(get_key(0)..get_key(10))).await;
   10680            2 :         check_layer_map_key_eq(
   10681            2 :             all_layers,
   10682            2 :             vec![
   10683            2 :                 // The compacted image layer (full key range)
   10684            2 :                 PersistentLayerKey {
   10685            2 :                     key_range: Key::MIN..Key::MAX,
   10686            2 :                     lsn_range: Lsn(0x10)..Lsn(0x11),
   10687            2 :                     is_delta: false,
   10688            2 :                 },
   10689            2 :                 // All other data in the delta layer
   10690            2 :                 PersistentLayerKey {
   10691            2 :                     key_range: get_key(1)..get_key(10),
   10692            2 :                     lsn_range: Lsn(0x10)..Lsn(0x50),
   10693            2 :                     is_delta: true,
   10694            2 :                 },
   10695            2 :             ],
   10696            2 :         );
   10697            2 : 
   10698            2 :         Ok(())
   10699            2 :     }
   10700              : 
   10701              :     #[cfg(feature = "testing")]
   10702              :     #[tokio::test]
   10703            2 :     async fn test_simple_bottom_most_compaction_rectangle() -> anyhow::Result<()> {
   10704            2 :         let harness = TenantHarness::create("test_simple_bottom_most_compaction_rectangle").await?;
   10705            2 :         let (tenant, ctx) = harness.load().await;
   10706            2 : 
   10707          508 :         fn get_key(id: u32) -> Key {
   10708          508 :             // using aux key here b/c they are guaranteed to be inside `collect_keyspace`.
   10709          508 :             let mut key = Key::from_hex("620000000033333333444444445500000000").unwrap();
   10710          508 :             key.field6 = id;
   10711          508 :             key
   10712          508 :         }
   10713            2 : 
   10714            2 :         let img_layer = (0..10)
   10715           20 :             .map(|id| (get_key(id), Bytes::from(format!("value {id}@0x10"))))
   10716            2 :             .collect_vec();
   10717            2 : 
   10718            2 :         let delta1 = vec![(
   10719            2 :             get_key(1),
   10720            2 :             Lsn(0x20),
   10721            2 :             Value::WalRecord(NeonWalRecord::wal_append("@0x20")),
   10722            2 :         )];
   10723            2 :         let delta4 = vec![(
   10724            2 :             get_key(1),
   10725            2 :             Lsn(0x28),
   10726            2 :             Value::WalRecord(NeonWalRecord::wal_append("@0x28")),
   10727            2 :         )];
   10728            2 :         let delta2 = vec![
   10729            2 :             (
   10730            2 :                 get_key(1),
   10731            2 :                 Lsn(0x30),
   10732            2 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x30")),
   10733            2 :             ),
   10734            2 :             (
   10735            2 :                 get_key(1),
   10736            2 :                 Lsn(0x38),
   10737            2 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x38")),
   10738            2 :             ),
   10739            2 :         ];
   10740            2 :         let delta3 = vec![
   10741            2 :             (
   10742            2 :                 get_key(8),
   10743            2 :                 Lsn(0x48),
   10744            2 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x48")),
   10745            2 :             ),
   10746            2 :             (
   10747            2 :                 get_key(9),
   10748            2 :                 Lsn(0x48),
   10749            2 :                 Value::WalRecord(NeonWalRecord::wal_append("@0x48")),
   10750            2 :             ),
   10751            2 :         ];
   10752            2 : 
   10753            2 :         let tline = tenant
   10754            2 :             .create_test_timeline_with_layers(
   10755            2 :                 TIMELINE_ID,
   10756            2 :                 Lsn(0x10),
   10757            2 :                 DEFAULT_PG_VERSION,
   10758            2 :                 &ctx,
   10759            2 :                 vec![
   10760            2 :                     // delta1/2/4 only contain a single key but multiple updates
   10761            2 :                     DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x20)..Lsn(0x28), delta1),
   10762            2 :                     DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x30)..Lsn(0x50), delta2),
   10763            2 :                     DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x28)..Lsn(0x30), delta4),
   10764            2 :                     DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x30)..Lsn(0x50), delta3),
   10765            2 :                 ], // delta layers
   10766            2 :                 vec![(Lsn(0x10), img_layer)], // image layers
   10767            2 :                 Lsn(0x50),
   10768            2 :             )
   10769            2 :             .await?;
   10770            2 :         {
   10771            2 :             tline
   10772            2 :                 .latest_gc_cutoff_lsn
   10773            2 :                 .lock_for_write()
   10774            2 :                 .store_and_unlock(Lsn(0x30))
   10775            2 :                 .wait()
   10776            2 :                 .await;
   10777            2 :             // Update GC info
   10778            2 :             let mut guard = tline.gc_info.write().unwrap();
   10779            2 :             *guard = GcInfo {
   10780            2 :                 retain_lsns: vec![
   10781            2 :                     (Lsn(0x10), tline.timeline_id, MaybeOffloaded::No),
   10782            2 :                     (Lsn(0x20), tline.timeline_id, MaybeOffloaded::No),
   10783            2 :                 ],
   10784            2 :                 cutoffs: GcCutoffs {
   10785            2 :                     time: Lsn(0x30),
   10786            2 :                     space: Lsn(0x30),
   10787            2 :                 },
   10788            2 :                 leases: Default::default(),
   10789            2 :                 within_ancestor_pitr: false,
   10790            2 :             };
   10791            2 :         }
   10792            2 : 
   10793            2 :         let expected_result = [
   10794            2 :             Bytes::from_static(b"value 0@0x10"),
   10795            2 :             Bytes::from_static(b"value 1@0x10@0x20@0x28@0x30@0x38"),
   10796            2 :             Bytes::from_static(b"value 2@0x10"),
   10797            2 :             Bytes::from_static(b"value 3@0x10"),
   10798            2 :             Bytes::from_static(b"value 4@0x10"),
   10799            2 :             Bytes::from_static(b"value 5@0x10"),
   10800            2 :             Bytes::from_static(b"value 6@0x10"),
   10801            2 :             Bytes::from_static(b"value 7@0x10"),
   10802            2 :             Bytes::from_static(b"value 8@0x10@0x48"),
   10803            2 :             Bytes::from_static(b"value 9@0x10@0x48"),
   10804            2 :         ];
   10805            2 : 
   10806            2 :         let expected_result_at_gc_horizon = [
   10807            2 :             Bytes::from_static(b"value 0@0x10"),
   10808            2 :             Bytes::from_static(b"value 1@0x10@0x20@0x28@0x30"),
   10809            2 :             Bytes::from_static(b"value 2@0x10"),
   10810            2 :             Bytes::from_static(b"value 3@0x10"),
   10811            2 :             Bytes::from_static(b"value 4@0x10"),
   10812            2 :             Bytes::from_static(b"value 5@0x10"),
   10813            2 :             Bytes::from_static(b"value 6@0x10"),
   10814            2 :             Bytes::from_static(b"value 7@0x10"),
   10815            2 :             Bytes::from_static(b"value 8@0x10"),
   10816            2 :             Bytes::from_static(b"value 9@0x10"),
   10817            2 :         ];
   10818            2 : 
   10819            2 :         let expected_result_at_lsn_20 = [
   10820            2 :             Bytes::from_static(b"value 0@0x10"),
   10821            2 :             Bytes::from_static(b"value 1@0x10@0x20"),
   10822            2 :             Bytes::from_static(b"value 2@0x10"),
   10823            2 :             Bytes::from_static(b"value 3@0x10"),
   10824            2 :             Bytes::from_static(b"value 4@0x10"),
   10825            2 :             Bytes::from_static(b"value 5@0x10"),
   10826            2 :             Bytes::from_static(b"value 6@0x10"),
   10827            2 :             Bytes::from_static(b"value 7@0x10"),
   10828            2 :             Bytes::from_static(b"value 8@0x10"),
   10829            2 :             Bytes::from_static(b"value 9@0x10"),
   10830            2 :         ];
   10831            2 : 
   10832            2 :         let expected_result_at_lsn_10 = [
   10833            2 :             Bytes::from_static(b"value 0@0x10"),
   10834            2 :             Bytes::from_static(b"value 1@0x10"),
   10835            2 :             Bytes::from_static(b"value 2@0x10"),
   10836            2 :             Bytes::from_static(b"value 3@0x10"),
   10837            2 :             Bytes::from_static(b"value 4@0x10"),
   10838            2 :             Bytes::from_static(b"value 5@0x10"),
   10839            2 :             Bytes::from_static(b"value 6@0x10"),
   10840            2 :             Bytes::from_static(b"value 7@0x10"),
   10841            2 :             Bytes::from_static(b"value 8@0x10"),
   10842            2 :             Bytes::from_static(b"value 9@0x10"),
   10843            2 :         ];
   10844            2 : 
   10845           10 :         let verify_result = || async {
   10846           10 :             let gc_horizon = {
   10847           10 :                 let gc_info = tline.gc_info.read().unwrap();
   10848           10 :                 gc_info.cutoffs.time
   10849            2 :             };
   10850          110 :             for idx in 0..10 {
   10851          100 :                 assert_eq!(
   10852          100 :                     tline
   10853          100 :                         .get(get_key(idx as u32), Lsn(0x50), &ctx)
   10854          100 :                         .await
   10855          100 :                         .unwrap(),
   10856          100 :                     &expected_result[idx]
   10857            2 :                 );
   10858          100 :                 assert_eq!(
   10859          100 :                     tline
   10860          100 :                         .get(get_key(idx as u32), gc_horizon, &ctx)
   10861          100 :                         .await
   10862          100 :                         .unwrap(),
   10863          100 :                     &expected_result_at_gc_horizon[idx]
   10864            2 :                 );
   10865          100 :                 assert_eq!(
   10866          100 :                     tline
   10867          100 :                         .get(get_key(idx as u32), Lsn(0x20), &ctx)
   10868          100 :                         .await
   10869          100 :                         .unwrap(),
   10870          100 :                     &expected_result_at_lsn_20[idx]
   10871            2 :                 );
   10872          100 :                 assert_eq!(
   10873          100 :                     tline
   10874          100 :                         .get(get_key(idx as u32), Lsn(0x10), &ctx)
   10875          100 :                         .await
   10876          100 :                         .unwrap(),
   10877          100 :                     &expected_result_at_lsn_10[idx]
   10878            2 :                 );
   10879            2 :             }
   10880           20 :         };
   10881            2 : 
   10882            2 :         verify_result().await;
   10883            2 : 
   10884            2 :         let cancel = CancellationToken::new();
   10885            2 : 
   10886            2 :         tline
   10887            2 :             .compact_with_gc(
   10888            2 :                 &cancel,
   10889            2 :                 CompactOptions {
   10890            2 :                     compact_key_range: Some((get_key(0)..get_key(2)).into()),
   10891            2 :                     compact_lsn_range: Some((Lsn(0x20)..Lsn(0x28)).into()),
   10892            2 :                     ..Default::default()
   10893            2 :                 },
   10894            2 :                 &ctx,
   10895            2 :             )
   10896            2 :             .await
   10897            2 :             .unwrap();
   10898            2 :         verify_result().await;
   10899            2 : 
   10900            2 :         let all_layers = inspect_and_sort(&tline, Some(get_key(0)..get_key(10))).await;
   10901            2 :         check_layer_map_key_eq(
   10902            2 :             all_layers,
   10903            2 :             vec![
   10904            2 :                 // The original image layer, not compacted
   10905            2 :                 PersistentLayerKey {
   10906            2 :                     key_range: get_key(0)..get_key(10),
   10907            2 :                     lsn_range: Lsn(0x10)..Lsn(0x11),
   10908            2 :                     is_delta: false,
   10909            2 :                 },
   10910            2 :                 // According the selection logic, we select all layers with start key <= 0x28, so we would merge the layer 0x20-0x28 and
   10911            2 :                 // the layer 0x28-0x30 into one.
   10912            2 :                 PersistentLayerKey {
   10913            2 :                     key_range: get_key(1)..get_key(2),
   10914            2 :                     lsn_range: Lsn(0x20)..Lsn(0x30),
   10915            2 :                     is_delta: true,
   10916            2 :                 },
   10917            2 :                 // Above the upper bound and untouched
   10918            2 :                 PersistentLayerKey {
   10919            2 :                     key_range: get_key(1)..get_key(2),
   10920            2 :                     lsn_range: Lsn(0x30)..Lsn(0x50),
   10921            2 :                     is_delta: true,
   10922            2 :                 },
   10923            2 :                 // This layer is untouched
   10924            2 :                 PersistentLayerKey {
   10925            2 :                     key_range: get_key(8)..get_key(10),
   10926            2 :                     lsn_range: Lsn(0x30)..Lsn(0x50),
   10927            2 :                     is_delta: true,
   10928            2 :                 },
   10929            2 :             ],
   10930            2 :         );
   10931            2 : 
   10932            2 :         tline
   10933            2 :             .compact_with_gc(
   10934            2 :                 &cancel,
   10935            2 :                 CompactOptions {
   10936            2 :                     compact_key_range: Some((get_key(3)..get_key(8)).into()),
   10937            2 :                     compact_lsn_range: Some((Lsn(0x28)..Lsn(0x40)).into()),
   10938            2 :                     ..Default::default()
   10939            2 :                 },
   10940            2 :                 &ctx,
   10941            2 :             )
   10942            2 :             .await
   10943            2 :             .unwrap();
   10944            2 :         verify_result().await;
   10945            2 : 
   10946            2 :         let all_layers = inspect_and_sort(&tline, Some(get_key(0)..get_key(10))).await;
   10947            2 :         check_layer_map_key_eq(
   10948            2 :             all_layers,
   10949            2 :             vec![
   10950            2 :                 // The original image layer, not compacted
   10951            2 :                 PersistentLayerKey {
   10952            2 :                     key_range: get_key(0)..get_key(10),
   10953            2 :                     lsn_range: Lsn(0x10)..Lsn(0x11),
   10954            2 :                     is_delta: false,
   10955            2 :                 },
   10956            2 :                 // Not in the compaction key range, uncompacted
   10957            2 :                 PersistentLayerKey {
   10958            2 :                     key_range: get_key(1)..get_key(2),
   10959            2 :                     lsn_range: Lsn(0x20)..Lsn(0x30),
   10960            2 :                     is_delta: true,
   10961            2 :                 },
   10962            2 :                 // Not in the compaction key range, uncompacted but need rewrite because the delta layer overlaps with the range
   10963            2 :                 PersistentLayerKey {
   10964            2 :                     key_range: get_key(1)..get_key(2),
   10965            2 :                     lsn_range: Lsn(0x30)..Lsn(0x50),
   10966            2 :                     is_delta: true,
   10967            2 :                 },
   10968            2 :                 // Note that when we specify the LSN upper bound to be 0x40, the compaction algorithm will not try to cut the layer
   10969            2 :                 // horizontally in half. Instead, it will include all LSNs that overlap with 0x40. So the real max_lsn of the compaction
   10970            2 :                 // becomes 0x50.
   10971            2 :                 PersistentLayerKey {
   10972            2 :                     key_range: get_key(8)..get_key(10),
   10973            2 :                     lsn_range: Lsn(0x30)..Lsn(0x50),
   10974            2 :                     is_delta: true,
   10975            2 :                 },
   10976            2 :             ],
   10977            2 :         );
   10978            2 : 
   10979            2 :         // compact again
   10980            2 :         tline
   10981            2 :             .compact_with_gc(
   10982            2 :                 &cancel,
   10983            2 :                 CompactOptions {
   10984            2 :                     compact_key_range: Some((get_key(0)..get_key(5)).into()),
   10985            2 :                     compact_lsn_range: Some((Lsn(0x20)..Lsn(0x50)).into()),
   10986            2 :                     ..Default::default()
   10987            2 :                 },
   10988            2 :                 &ctx,
   10989            2 :             )
   10990            2 :             .await
   10991            2 :             .unwrap();
   10992            2 :         verify_result().await;
   10993            2 : 
   10994            2 :         let all_layers = inspect_and_sort(&tline, Some(get_key(0)..get_key(10))).await;
   10995            2 :         check_layer_map_key_eq(
   10996            2 :             all_layers,
   10997            2 :             vec![
   10998            2 :                 // The original image layer, not compacted
   10999            2 :                 PersistentLayerKey {
   11000            2 :                     key_range: get_key(0)..get_key(10),
   11001            2 :                     lsn_range: Lsn(0x10)..Lsn(0x11),
   11002            2 :                     is_delta: false,
   11003            2 :                 },
   11004            2 :                 // The range gets compacted
   11005            2 :                 PersistentLayerKey {
   11006            2 :                     key_range: get_key(1)..get_key(2),
   11007            2 :                     lsn_range: Lsn(0x20)..Lsn(0x50),
   11008            2 :                     is_delta: true,
   11009            2 :                 },
   11010            2 :                 // Not touched during this iteration of compaction
   11011            2 :                 PersistentLayerKey {
   11012            2 :                     key_range: get_key(8)..get_key(10),
   11013            2 :                     lsn_range: Lsn(0x30)..Lsn(0x50),
   11014            2 :                     is_delta: true,
   11015            2 :                 },
   11016            2 :             ],
   11017            2 :         );
   11018            2 : 
   11019            2 :         // final full compaction
   11020            2 :         tline
   11021            2 :             .compact_with_gc(&cancel, CompactOptions::default(), &ctx)
   11022            2 :             .await
   11023            2 :             .unwrap();
   11024            2 :         verify_result().await;
   11025            2 : 
   11026            2 :         let all_layers = inspect_and_sort(&tline, Some(get_key(0)..get_key(10))).await;
   11027            2 :         check_layer_map_key_eq(
   11028            2 :             all_layers,
   11029            2 :             vec![
   11030            2 :                 // The compacted image layer (full key range)
   11031            2 :                 PersistentLayerKey {
   11032            2 :                     key_range: Key::MIN..Key::MAX,
   11033            2 :                     lsn_range: Lsn(0x10)..Lsn(0x11),
   11034            2 :                     is_delta: false,
   11035            2 :                 },
   11036            2 :                 // All other data in the delta layer
   11037            2 :                 PersistentLayerKey {
   11038            2 :                     key_range: get_key(1)..get_key(10),
   11039            2 :                     lsn_range: Lsn(0x10)..Lsn(0x50),
   11040            2 :                     is_delta: true,
   11041            2 :                 },
   11042            2 :             ],
   11043            2 :         );
   11044            2 : 
   11045            2 :         Ok(())
   11046            2 :     }
   11047              : }
        

Generated by: LCOV version 2.1-beta