LCOV - code coverage report
Current view: top level - pageserver/src/tenant - timeline.rs (source / functions) Coverage Total Hit
Test: 1b0a6a0c05cee5a7de360813c8034804e105ce1c.info Lines: 63.0 % 4248 2678
Test Date: 2025-03-12 00:01:28 Functions: 59.7 % 357 213

            Line data    Source code
       1              : pub(crate) mod analysis;
       2              : pub(crate) mod compaction;
       3              : pub mod delete;
       4              : pub(crate) mod detach_ancestor;
       5              : mod eviction_task;
       6              : pub(crate) mod handle;
       7              : mod heatmap_layers_downloader;
       8              : pub(crate) mod import_pgdata;
       9              : mod init;
      10              : pub mod layer_manager;
      11              : pub(crate) mod logical_size;
      12              : pub mod offload;
      13              : pub mod span;
      14              : pub mod uninit;
      15              : mod walreceiver;
      16              : 
      17              : use std::array;
      18              : use std::cmp::{max, min};
      19              : use std::collections::btree_map::Entry;
      20              : use std::collections::{BTreeMap, HashMap, HashSet};
      21              : use std::ops::{ControlFlow, Deref, Range};
      22              : use std::sync::atomic::{AtomicBool, AtomicU64, Ordering as AtomicOrdering};
      23              : use std::sync::{Arc, Mutex, OnceLock, RwLock, Weak};
      24              : use std::time::{Duration, Instant, SystemTime};
      25              : 
      26              : use anyhow::{Context, Result, anyhow, bail, ensure};
      27              : use arc_swap::{ArcSwap, ArcSwapOption};
      28              : use bytes::Bytes;
      29              : use camino::Utf8Path;
      30              : use chrono::{DateTime, Utc};
      31              : use compaction::{CompactionOutcome, GcCompactionCombinedSettings};
      32              : use enumset::EnumSet;
      33              : use fail::fail_point;
      34              : use futures::stream::FuturesUnordered;
      35              : use futures::{FutureExt, StreamExt};
      36              : use handle::ShardTimelineId;
      37              : use layer_manager::Shutdown;
      38              : use offload::OffloadError;
      39              : use once_cell::sync::Lazy;
      40              : use pageserver_api::config::tenant_conf_defaults::DEFAULT_PITR_INTERVAL;
      41              : use pageserver_api::key::{
      42              :     KEY_SIZE, Key, METADATA_KEY_BEGIN_PREFIX, METADATA_KEY_END_PREFIX, NON_INHERITED_RANGE,
      43              :     SPARSE_RANGE,
      44              : };
      45              : use pageserver_api::keyspace::{KeySpaceAccum, KeySpaceRandomAccum, SparseKeyPartitioning};
      46              : use pageserver_api::models::{
      47              :     CompactKeyRange, CompactLsnRange, CompactionAlgorithm, CompactionAlgorithmSettings,
      48              :     DownloadRemoteLayersTaskInfo, DownloadRemoteLayersTaskSpawnRequest, EvictionPolicy,
      49              :     InMemoryLayerInfo, LayerMapInfo, LsnLease, PageTraceEvent, RelSizeMigration, TimelineState,
      50              : };
      51              : use pageserver_api::reltag::{BlockNumber, RelTag};
      52              : use pageserver_api::shard::{ShardIdentity, ShardIndex, ShardNumber, TenantShardId};
      53              : #[cfg(test)]
      54              : use pageserver_api::value::Value;
      55              : use postgres_connection::PgConnectionConfig;
      56              : use postgres_ffi::v14::xlog_utils;
      57              : use postgres_ffi::{WAL_SEGMENT_SIZE, to_pg_timestamp};
      58              : use rand::Rng;
      59              : use remote_storage::DownloadError;
      60              : use serde_with::serde_as;
      61              : use storage_broker::BrokerClientChannel;
      62              : use tokio::runtime::Handle;
      63              : use tokio::sync::mpsc::Sender;
      64              : use tokio::sync::{Notify, oneshot, watch};
      65              : use tokio_util::sync::CancellationToken;
      66              : use tracing::*;
      67              : use utils::generation::Generation;
      68              : use utils::guard_arc_swap::GuardArcSwap;
      69              : use utils::id::TimelineId;
      70              : use utils::lsn::{AtomicLsn, Lsn, RecordLsn};
      71              : use utils::postgres_client::PostgresClientProtocol;
      72              : use utils::rate_limit::RateLimit;
      73              : use utils::seqwait::SeqWait;
      74              : use utils::simple_rcu::{Rcu, RcuReadGuard};
      75              : use utils::sync::gate::{Gate, GateGuard};
      76              : use utils::{completion, critical, fs_ext, pausable_failpoint};
      77              : use wal_decoder::serialized_batch::{SerializedValueBatch, ValueMeta};
      78              : 
      79              : use self::delete::DeleteTimelineFlow;
      80              : pub(super) use self::eviction_task::EvictionTaskTenantState;
      81              : use self::eviction_task::EvictionTaskTimelineState;
      82              : use self::layer_manager::LayerManager;
      83              : use self::logical_size::LogicalSize;
      84              : use self::walreceiver::{WalReceiver, WalReceiverConf};
      85              : use super::config::TenantConf;
      86              : use super::remote_timeline_client::index::{GcCompactionState, IndexPart};
      87              : use super::remote_timeline_client::{RemoteTimelineClient, WaitCompletionError};
      88              : use super::secondary::heatmap::HeatMapLayer;
      89              : use super::storage_layer::{LayerFringe, LayerVisibilityHint, ReadableLayer};
      90              : use super::upload_queue::NotInitialized;
      91              : use super::{
      92              :     AttachedTenantConf, GcError, HeatMapTimeline, MaybeOffloaded,
      93              :     debug_assert_current_span_has_tenant_and_timeline_id,
      94              : };
      95              : use crate::aux_file::AuxFileSizeEstimator;
      96              : use crate::config::PageServerConf;
      97              : use crate::context::{DownloadBehavior, RequestContext};
      98              : use crate::disk_usage_eviction_task::{DiskUsageEvictionInfo, EvictionCandidate, finite_f32};
      99              : use crate::keyspace::{KeyPartitioning, KeySpace};
     100              : use crate::l0_flush::{self, L0FlushGlobalState};
     101              : use crate::metrics::{
     102              :     DELTAS_PER_READ_GLOBAL, LAYERS_PER_READ_AMORTIZED_GLOBAL, LAYERS_PER_READ_BATCH_GLOBAL,
     103              :     LAYERS_PER_READ_GLOBAL, ScanLatencyOngoingRecording, TimelineMetrics,
     104              : };
     105              : use crate::page_service::TenantManagerTypes;
     106              : use crate::pgdatadir_mapping::{
     107              :     CalculateLogicalSizeError, CollectKeySpaceError, DirectoryKind, LsnForTimestamp,
     108              :     MAX_AUX_FILE_V2_DELTAS, MetricsUpdate,
     109              : };
     110              : use crate::task_mgr::TaskKind;
     111              : use crate::tenant::config::{AttachmentMode, TenantConfOpt};
     112              : use crate::tenant::gc_result::GcResult;
     113              : use crate::tenant::layer_map::{LayerMap, SearchResult};
     114              : use crate::tenant::metadata::TimelineMetadata;
     115              : use crate::tenant::storage_layer::delta_layer::DeltaEntry;
     116              : use crate::tenant::storage_layer::inmemory_layer::IndexEntry;
     117              : use crate::tenant::storage_layer::{
     118              :     AsLayerDesc, BatchLayerWriter, DeltaLayerWriter, EvictionError, ImageLayerName,
     119              :     ImageLayerWriter, InMemoryLayer, IoConcurrency, Layer, LayerAccessStatsReset, LayerName,
     120              :     PersistentLayerDesc, PersistentLayerKey, ResidentLayer, ValueReconstructSituation,
     121              :     ValueReconstructState, ValuesReconstructState,
     122              : };
     123              : use crate::tenant::tasks::BackgroundLoopKind;
     124              : use crate::tenant::timeline::logical_size::CurrentLogicalSize;
     125              : use crate::virtual_file::{MaybeFatalIo, VirtualFile};
     126              : use crate::walingest::WalLagCooldown;
     127              : use crate::{ZERO_PAGE, task_mgr, walredo};
     128              : 
     129              : #[derive(Debug, PartialEq, Eq, Clone, Copy)]
     130              : pub(crate) enum FlushLoopState {
     131              :     NotStarted,
     132              :     Running {
     133              :         #[cfg(test)]
     134              :         expect_initdb_optimization: bool,
     135              :         #[cfg(test)]
     136              :         initdb_optimization_count: usize,
     137              :     },
     138              :     Exited,
     139              : }
     140              : 
     141              : #[derive(Debug, Copy, Clone, PartialEq, Eq)]
     142              : pub enum ImageLayerCreationMode {
     143              :     /// Try to create image layers based on `time_for_new_image_layer`. Used in compaction code path.
     144              :     Try,
     145              :     /// Force creating the image layers if possible. For now, no image layers will be created
     146              :     /// for metadata keys. Used in compaction code path with force flag enabled.
     147              :     Force,
     148              :     /// Initial ingestion of the data, and no data should be dropped in this function. This
     149              :     /// means that no metadata keys should be included in the partitions. Used in flush frozen layer
     150              :     /// code path.
     151              :     Initial,
     152              : }
     153              : 
     154              : #[derive(Clone, Debug, Default)]
     155              : pub enum LastImageLayerCreationStatus {
     156              :     Incomplete {
     157              :         /// The last key of the partition (exclusive) that was processed in the last
     158              :         /// image layer creation attempt. We will continue from this key in the next
     159              :         /// attempt.
     160              :         last_key: Key,
     161              :     },
     162              :     Complete,
     163              :     #[default]
     164              :     Initial,
     165              : }
     166              : 
     167              : impl std::fmt::Display for ImageLayerCreationMode {
     168         1148 :     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
     169         1148 :         write!(f, "{:?}", self)
     170         1148 :     }
     171              : }
     172              : 
     173              : /// Temporary function for immutable storage state refactor, ensures we are dropping mutex guard instead of other things.
     174              : /// Can be removed after all refactors are done.
     175           56 : fn drop_rlock<T>(rlock: tokio::sync::RwLockReadGuard<T>) {
     176           56 :     drop(rlock)
     177           56 : }
     178              : 
     179              : /// Temporary function for immutable storage state refactor, ensures we are dropping mutex guard instead of other things.
     180              : /// Can be removed after all refactors are done.
     181         1204 : fn drop_wlock<T>(rlock: tokio::sync::RwLockWriteGuard<'_, T>) {
     182         1204 :     drop(rlock)
     183         1204 : }
     184              : 
     185              : /// The outward-facing resources required to build a Timeline
     186              : pub struct TimelineResources {
     187              :     pub remote_client: RemoteTimelineClient,
     188              :     pub pagestream_throttle: Arc<crate::tenant::throttle::Throttle>,
     189              :     pub pagestream_throttle_metrics: Arc<crate::metrics::tenant_throttling::Pagestream>,
     190              :     pub l0_compaction_trigger: Arc<Notify>,
     191              :     pub l0_flush_global_state: l0_flush::L0FlushGlobalState,
     192              : }
     193              : 
     194              : /// The relation size cache caches relation sizes at the end of the timeline. It speeds up WAL
     195              : /// ingestion considerably, because WAL ingestion needs to check on most records if the record
     196              : /// implicitly extends the relation.  At startup, `complete_as_of` is initialized to the current end
     197              : /// of the timeline (disk_consistent_lsn).  It's used on reads of relation sizes to check if the
     198              : /// value can be used to also update the cache, see [`Timeline::update_cached_rel_size`].
     199              : pub(crate) struct RelSizeCache {
     200              :     pub(crate) complete_as_of: Lsn,
     201              :     pub(crate) map: HashMap<RelTag, (Lsn, BlockNumber)>,
     202              : }
     203              : 
     204              : pub struct Timeline {
     205              :     pub(crate) conf: &'static PageServerConf,
     206              :     tenant_conf: Arc<ArcSwap<AttachedTenantConf>>,
     207              : 
     208              :     myself: Weak<Self>,
     209              : 
     210              :     pub(crate) tenant_shard_id: TenantShardId,
     211              :     pub timeline_id: TimelineId,
     212              : 
     213              :     /// The generation of the tenant that instantiated us: this is used for safety when writing remote objects.
     214              :     /// Never changes for the lifetime of this [`Timeline`] object.
     215              :     ///
     216              :     /// This duplicates the generation stored in LocationConf, but that structure is mutable:
     217              :     /// this copy enforces the invariant that generatio doesn't change during a Tenant's lifetime.
     218              :     pub(crate) generation: Generation,
     219              : 
     220              :     /// The detailed sharding information from our parent Tenant.  This enables us to map keys
     221              :     /// to shards, and is constant through the lifetime of this Timeline.
     222              :     shard_identity: ShardIdentity,
     223              : 
     224              :     pub pg_version: u32,
     225              : 
     226              :     /// The tuple has two elements.
     227              :     /// 1. `LayerFileManager` keeps track of the various physical representations of the layer files (inmem, local, remote).
     228              :     /// 2. `LayerMap`, the acceleration data structure for `get_reconstruct_data`.
     229              :     ///
     230              :     /// `LayerMap` maps out the `(PAGE,LSN) / (KEY,LSN)` space, which is composed of `(KeyRange, LsnRange)` rectangles.
     231              :     /// We describe these rectangles through the `PersistentLayerDesc` struct.
     232              :     ///
     233              :     /// When we want to reconstruct a page, we first find the `PersistentLayerDesc`'s that we need for page reconstruction,
     234              :     /// using `LayerMap`. Then, we use `LayerFileManager` to get the `PersistentLayer`'s that correspond to the
     235              :     /// `PersistentLayerDesc`'s.
     236              :     ///
     237              :     /// Hence, it's important to keep things coherent. The `LayerFileManager` must always have an entry for all
     238              :     /// `PersistentLayerDesc`'s in the `LayerMap`. If it doesn't, `LayerFileManager::get_from_desc` will panic at
     239              :     /// runtime, e.g., during page reconstruction.
     240              :     ///
     241              :     /// In the future, we'll be able to split up the tuple of LayerMap and `LayerFileManager`,
     242              :     /// so that e.g. on-demand-download/eviction, and layer spreading, can operate just on `LayerFileManager`.
     243              :     pub(crate) layers: tokio::sync::RwLock<LayerManager>,
     244              : 
     245              :     last_freeze_at: AtomicLsn,
     246              :     // Atomic would be more appropriate here.
     247              :     last_freeze_ts: RwLock<Instant>,
     248              : 
     249              :     pub(crate) standby_horizon: AtomicLsn,
     250              : 
     251              :     // WAL redo manager. `None` only for broken tenants.
     252              :     walredo_mgr: Option<Arc<super::WalRedoManager>>,
     253              : 
     254              :     /// Remote storage client.
     255              :     /// See [`remote_timeline_client`](super::remote_timeline_client) module comment for details.
     256              :     pub(crate) remote_client: Arc<RemoteTimelineClient>,
     257              : 
     258              :     // What page versions do we hold in the repository? If we get a
     259              :     // request > last_record_lsn, we need to wait until we receive all
     260              :     // the WAL up to the request. The SeqWait provides functions for
     261              :     // that. TODO: If we get a request for an old LSN, such that the
     262              :     // versions have already been garbage collected away, we should
     263              :     // throw an error, but we don't track that currently.
     264              :     //
     265              :     // last_record_lsn.load().last points to the end of last processed WAL record.
     266              :     //
     267              :     // We also remember the starting point of the previous record in
     268              :     // 'last_record_lsn.load().prev'. It's used to set the xl_prev pointer of the
     269              :     // first WAL record when the node is started up. But here, we just
     270              :     // keep track of it.
     271              :     last_record_lsn: SeqWait<RecordLsn, Lsn>,
     272              : 
     273              :     // All WAL records have been processed and stored durably on files on
     274              :     // local disk, up to this LSN. On crash and restart, we need to re-process
     275              :     // the WAL starting from this point.
     276              :     //
     277              :     // Some later WAL records might have been processed and also flushed to disk
     278              :     // already, so don't be surprised to see some, but there's no guarantee on
     279              :     // them yet.
     280              :     disk_consistent_lsn: AtomicLsn,
     281              : 
     282              :     // Parent timeline that this timeline was branched from, and the LSN
     283              :     // of the branch point.
     284              :     ancestor_timeline: Option<Arc<Timeline>>,
     285              :     ancestor_lsn: Lsn,
     286              : 
     287              :     // The LSN of gc-compaction that was last applied to this timeline.
     288              :     gc_compaction_state: ArcSwap<Option<GcCompactionState>>,
     289              : 
     290              :     pub(crate) metrics: Arc<TimelineMetrics>,
     291              : 
     292              :     // `Timeline` doesn't write these metrics itself, but it manages the lifetime.  Code
     293              :     // in `crate::page_service` writes these metrics.
     294              :     pub(crate) query_metrics: crate::metrics::SmgrQueryTimePerTimeline,
     295              : 
     296              :     directory_metrics_inited: [AtomicBool; DirectoryKind::KINDS_NUM],
     297              :     directory_metrics: [AtomicU64; DirectoryKind::KINDS_NUM],
     298              : 
     299              :     /// Ensures layers aren't frozen by checkpointer between
     300              :     /// [`Timeline::get_layer_for_write`] and layer reads.
     301              :     /// Locked automatically by [`TimelineWriter`] and checkpointer.
     302              :     /// Must always be acquired before the layer map/individual layer lock
     303              :     /// to avoid deadlock.
     304              :     ///
     305              :     /// The state is cleared upon freezing.
     306              :     write_lock: tokio::sync::Mutex<Option<TimelineWriterState>>,
     307              : 
     308              :     /// Used to avoid multiple `flush_loop` tasks running
     309              :     pub(super) flush_loop_state: Mutex<FlushLoopState>,
     310              : 
     311              :     /// layer_flush_start_tx can be used to wake up the layer-flushing task.
     312              :     /// - The u64 value is a counter, incremented every time a new flush cycle is requested.
     313              :     ///   The flush cycle counter is sent back on the layer_flush_done channel when
     314              :     ///   the flush finishes. You can use that to wait for the flush to finish.
     315              :     /// - The LSN is updated to max() of its current value and the latest disk_consistent_lsn
     316              :     ///   read by whoever sends an update
     317              :     layer_flush_start_tx: tokio::sync::watch::Sender<(u64, Lsn)>,
     318              :     /// to be notified when layer flushing has finished, subscribe to the layer_flush_done channel
     319              :     layer_flush_done_tx: tokio::sync::watch::Sender<(u64, Result<(), FlushLayerError>)>,
     320              : 
     321              :     // The LSN at which we have executed GC: whereas [`Self::gc_info`] records the LSN at which
     322              :     // we _intend_ to GC (i.e. the PITR cutoff), this LSN records where we actually last did it.
     323              :     // Because PITR interval is mutable, it's possible for this LSN to be earlier or later than
     324              :     // the planned GC cutoff.
     325              :     pub applied_gc_cutoff_lsn: Rcu<Lsn>,
     326              : 
     327              :     pub(crate) gc_compaction_layer_update_lock: tokio::sync::RwLock<()>,
     328              : 
     329              :     // List of child timelines and their branch points. This is needed to avoid
     330              :     // garbage collecting data that is still needed by the child timelines.
     331              :     pub(crate) gc_info: std::sync::RwLock<GcInfo>,
     332              : 
     333              :     pub(crate) last_image_layer_creation_status: ArcSwap<LastImageLayerCreationStatus>,
     334              : 
     335              :     // It may change across major versions so for simplicity
     336              :     // keep it after running initdb for a timeline.
     337              :     // It is needed in checks when we want to error on some operations
     338              :     // when they are requested for pre-initdb lsn.
     339              :     // It can be unified with latest_gc_cutoff_lsn under some "first_valid_lsn",
     340              :     // though let's keep them both for better error visibility.
     341              :     pub initdb_lsn: Lsn,
     342              : 
     343              :     /// The repartitioning result. Allows a single writer and multiple readers.
     344              :     pub(crate) partitioning: GuardArcSwap<((KeyPartitioning, SparseKeyPartitioning), Lsn)>,
     345              : 
     346              :     /// Configuration: how often should the partitioning be recalculated.
     347              :     repartition_threshold: u64,
     348              : 
     349              :     last_image_layer_creation_check_at: AtomicLsn,
     350              :     last_image_layer_creation_check_instant: std::sync::Mutex<Option<Instant>>,
     351              : 
     352              :     /// Current logical size of the "datadir", at the last LSN.
     353              :     current_logical_size: LogicalSize,
     354              : 
     355              :     /// Information about the last processed message by the WAL receiver,
     356              :     /// or None if WAL receiver has not received anything for this timeline
     357              :     /// yet.
     358              :     pub last_received_wal: Mutex<Option<WalReceiverInfo>>,
     359              :     pub walreceiver: Mutex<Option<WalReceiver>>,
     360              : 
     361              :     /// Relation size cache
     362              :     pub(crate) rel_size_cache: RwLock<RelSizeCache>,
     363              : 
     364              :     download_all_remote_layers_task_info: RwLock<Option<DownloadRemoteLayersTaskInfo>>,
     365              : 
     366              :     state: watch::Sender<TimelineState>,
     367              : 
     368              :     /// Prevent two tasks from deleting the timeline at the same time. If held, the
     369              :     /// timeline is being deleted. If 'true', the timeline has already been deleted.
     370              :     pub delete_progress: TimelineDeleteProgress,
     371              : 
     372              :     eviction_task_timeline_state: tokio::sync::Mutex<EvictionTaskTimelineState>,
     373              : 
     374              :     /// Load or creation time information about the disk_consistent_lsn and when the loading
     375              :     /// happened. Used for consumption metrics.
     376              :     pub(crate) loaded_at: (Lsn, SystemTime),
     377              : 
     378              :     /// Gate to prevent shutdown completing while I/O is still happening to this timeline's data
     379              :     pub(crate) gate: Gate,
     380              : 
     381              :     /// Cancellation token scoped to this timeline: anything doing long-running work relating
     382              :     /// to the timeline should drop out when this token fires.
     383              :     pub(crate) cancel: CancellationToken,
     384              : 
     385              :     /// Make sure we only have one running compaction at a time in tests.
     386              :     ///
     387              :     /// Must only be taken in two places:
     388              :     /// - [`Timeline::compact`] (this file)
     389              :     /// - [`delete::delete_local_timeline_directory`]
     390              :     ///
     391              :     /// Timeline deletion will acquire both compaction and gc locks in whatever order.
     392              :     compaction_lock: tokio::sync::Mutex<()>,
     393              : 
     394              :     /// If true, the last compaction failed.
     395              :     compaction_failed: AtomicBool,
     396              : 
     397              :     /// Notifies the tenant compaction loop that there is pending L0 compaction work.
     398              :     l0_compaction_trigger: Arc<Notify>,
     399              : 
     400              :     /// Make sure we only have one running gc at a time.
     401              :     ///
     402              :     /// Must only be taken in two places:
     403              :     /// - [`Timeline::gc`] (this file)
     404              :     /// - [`delete::delete_local_timeline_directory`]
     405              :     ///
     406              :     /// Timeline deletion will acquire both compaction and gc locks in whatever order.
     407              :     gc_lock: tokio::sync::Mutex<()>,
     408              : 
     409              :     /// Cloned from [`super::Tenant::pagestream_throttle`] on construction.
     410              :     pub(crate) pagestream_throttle: Arc<crate::tenant::throttle::Throttle>,
     411              : 
     412              :     /// Size estimator for aux file v2
     413              :     pub(crate) aux_file_size_estimator: AuxFileSizeEstimator,
     414              : 
     415              :     /// Some test cases directly place keys into the timeline without actually modifying the directory
     416              :     /// keys (i.e., DB_DIR). The test cases creating such keys will put the keyspaces here, so that
     417              :     /// these keys won't get garbage-collected during compaction/GC. This field only modifies the dense
     418              :     /// keyspace return value of `collect_keyspace`. For sparse keyspaces, use AUX keys for testing, and
     419              :     /// in the future, add `extra_test_sparse_keyspace` if necessary.
     420              :     #[cfg(test)]
     421              :     pub(crate) extra_test_dense_keyspace: ArcSwap<KeySpace>,
     422              : 
     423              :     pub(crate) l0_flush_global_state: L0FlushGlobalState,
     424              : 
     425              :     pub(crate) handles: handle::PerTimelineState<TenantManagerTypes>,
     426              : 
     427              :     pub(crate) attach_wal_lag_cooldown: Arc<OnceLock<WalLagCooldown>>,
     428              : 
     429              :     /// Cf. [`crate::tenant::CreateTimelineIdempotency`].
     430              :     pub(crate) create_idempotency: crate::tenant::CreateTimelineIdempotency,
     431              : 
     432              :     /// If Some, collects GetPage metadata for an ongoing PageTrace.
     433              :     pub(crate) page_trace: ArcSwapOption<Sender<PageTraceEvent>>,
     434              : 
     435              :     pub(super) previous_heatmap: ArcSwapOption<PreviousHeatmap>,
     436              : 
     437              :     /// May host a background Tokio task which downloads all the layers from the current
     438              :     /// heatmap on demand.
     439              :     heatmap_layers_downloader: Mutex<Option<heatmap_layers_downloader::HeatmapLayersDownloader>>,
     440              : 
     441              :     pub(crate) rel_size_v2_status: ArcSwapOption<RelSizeMigration>,
     442              : }
     443              : 
     444              : pub(crate) enum PreviousHeatmap {
     445              :     Active {
     446              :         heatmap: HeatMapTimeline,
     447              :         read_at: std::time::Instant,
     448              :         // End LSN covered by the heatmap if known
     449              :         end_lsn: Option<Lsn>,
     450              :     },
     451              :     Obsolete,
     452              : }
     453              : 
     454              : pub type TimelineDeleteProgress = Arc<tokio::sync::Mutex<DeleteTimelineFlow>>;
     455              : 
     456              : pub struct WalReceiverInfo {
     457              :     pub wal_source_connconf: PgConnectionConfig,
     458              :     pub last_received_msg_lsn: Lsn,
     459              :     pub last_received_msg_ts: u128,
     460              : }
     461              : 
     462              : /// Information about how much history needs to be retained, needed by
     463              : /// Garbage Collection.
     464              : #[derive(Default)]
     465              : pub(crate) struct GcInfo {
     466              :     /// Specific LSNs that are needed.
     467              :     ///
     468              :     /// Currently, this includes all points where child branches have
     469              :     /// been forked off from. In the future, could also include
     470              :     /// explicit user-defined snapshot points.
     471              :     pub(crate) retain_lsns: Vec<(Lsn, TimelineId, MaybeOffloaded)>,
     472              : 
     473              :     /// The cutoff coordinates, which are combined by selecting the minimum.
     474              :     pub(crate) cutoffs: GcCutoffs,
     475              : 
     476              :     /// Leases granted to particular LSNs.
     477              :     pub(crate) leases: BTreeMap<Lsn, LsnLease>,
     478              : 
     479              :     /// Whether our branch point is within our ancestor's PITR interval (for cost estimation)
     480              :     pub(crate) within_ancestor_pitr: bool,
     481              : }
     482              : 
     483              : impl GcInfo {
     484          600 :     pub(crate) fn min_cutoff(&self) -> Lsn {
     485          600 :         self.cutoffs.select_min()
     486          600 :     }
     487              : 
     488          464 :     pub(super) fn insert_child(
     489          464 :         &mut self,
     490          464 :         child_id: TimelineId,
     491          464 :         child_lsn: Lsn,
     492          464 :         is_offloaded: MaybeOffloaded,
     493          464 :     ) {
     494          464 :         self.retain_lsns.push((child_lsn, child_id, is_offloaded));
     495          464 :         self.retain_lsns.sort_by_key(|i| i.0);
     496          464 :     }
     497              : 
     498            8 :     pub(super) fn remove_child_maybe_offloaded(
     499            8 :         &mut self,
     500            8 :         child_id: TimelineId,
     501            8 :         maybe_offloaded: MaybeOffloaded,
     502            8 :     ) -> bool {
     503            8 :         // Remove at most one element. Needed for correctness if there is two live `Timeline` objects referencing
     504            8 :         // the same timeline. Shouldn't but maybe can occur when Arc's live longer than intended.
     505            8 :         let mut removed = false;
     506           12 :         self.retain_lsns.retain(|i| {
     507           12 :             if removed {
     508            4 :                 return true;
     509            8 :             }
     510            8 :             let remove = i.1 == child_id && i.2 == maybe_offloaded;
     511            8 :             removed |= remove;
     512            8 :             !remove
     513           12 :         });
     514            8 :         removed
     515            8 :     }
     516              : 
     517            8 :     pub(super) fn remove_child_not_offloaded(&mut self, child_id: TimelineId) -> bool {
     518            8 :         self.remove_child_maybe_offloaded(child_id, MaybeOffloaded::No)
     519            8 :     }
     520              : 
     521            0 :     pub(super) fn remove_child_offloaded(&mut self, child_id: TimelineId) -> bool {
     522            0 :         self.remove_child_maybe_offloaded(child_id, MaybeOffloaded::Yes)
     523            0 :     }
     524          464 :     pub(crate) fn lsn_covered_by_lease(&self, lsn: Lsn) -> bool {
     525          464 :         self.leases.contains_key(&lsn)
     526          464 :     }
     527              : }
     528              : 
     529              : /// The `GcInfo` component describing which Lsns need to be retained.  Functionally, this
     530              : /// is a single number (the oldest LSN which we must retain), but it internally distinguishes
     531              : /// between time-based and space-based retention for observability and consumption metrics purposes.
     532              : #[derive(Debug, Clone)]
     533              : pub(crate) struct GcCutoffs {
     534              :     /// Calculated from the [`TenantConf::gc_horizon`], this LSN indicates how much
     535              :     /// history we must keep to retain a specified number of bytes of WAL.
     536              :     pub(crate) space: Lsn,
     537              : 
     538              :     /// Calculated from [`TenantConf::pitr_interval`], this LSN indicates how much
     539              :     /// history we must keep to enable reading back at least the PITR interval duration.
     540              :     pub(crate) time: Lsn,
     541              : }
     542              : 
     543              : impl Default for GcCutoffs {
     544          904 :     fn default() -> Self {
     545          904 :         Self {
     546          904 :             space: Lsn::INVALID,
     547          904 :             time: Lsn::INVALID,
     548          904 :         }
     549          904 :     }
     550              : }
     551              : 
     552              : impl GcCutoffs {
     553          600 :     fn select_min(&self) -> Lsn {
     554          600 :         std::cmp::min(self.space, self.time)
     555          600 :     }
     556              : }
     557              : 
     558              : pub(crate) struct TimelineVisitOutcome {
     559              :     completed_keyspace: KeySpace,
     560              :     image_covered_keyspace: KeySpace,
     561              : }
     562              : 
     563              : /// An error happened in a get() operation.
     564              : #[derive(thiserror::Error, Debug)]
     565              : pub(crate) enum PageReconstructError {
     566              :     #[error(transparent)]
     567              :     Other(anyhow::Error),
     568              : 
     569              :     #[error("Ancestor LSN wait error: {0}")]
     570              :     AncestorLsnTimeout(WaitLsnError),
     571              : 
     572              :     #[error("timeline shutting down")]
     573              :     Cancelled,
     574              : 
     575              :     /// An error happened replaying WAL records
     576              :     #[error(transparent)]
     577              :     WalRedo(anyhow::Error),
     578              : 
     579              :     #[error("{0}")]
     580              :     MissingKey(MissingKeyError),
     581              : }
     582              : 
     583              : impl From<anyhow::Error> for PageReconstructError {
     584            0 :     fn from(value: anyhow::Error) -> Self {
     585            0 :         // with walingest.rs many PageReconstructError are wrapped in as anyhow::Error
     586            0 :         match value.downcast::<PageReconstructError>() {
     587            0 :             Ok(pre) => pre,
     588            0 :             Err(other) => PageReconstructError::Other(other),
     589              :         }
     590            0 :     }
     591              : }
     592              : 
     593              : impl From<utils::bin_ser::DeserializeError> for PageReconstructError {
     594            0 :     fn from(value: utils::bin_ser::DeserializeError) -> Self {
     595            0 :         PageReconstructError::Other(anyhow::Error::new(value).context("deserialization failure"))
     596            0 :     }
     597              : }
     598              : 
     599              : impl From<layer_manager::Shutdown> for PageReconstructError {
     600            0 :     fn from(_: layer_manager::Shutdown) -> Self {
     601            0 :         PageReconstructError::Cancelled
     602            0 :     }
     603              : }
     604              : 
     605              : impl GetVectoredError {
     606              :     #[cfg(test)]
     607           12 :     pub(crate) fn is_missing_key_error(&self) -> bool {
     608           12 :         matches!(self, Self::MissingKey(_))
     609           12 :     }
     610              : }
     611              : 
     612              : impl From<layer_manager::Shutdown> for GetVectoredError {
     613            0 :     fn from(_: layer_manager::Shutdown) -> Self {
     614            0 :         GetVectoredError::Cancelled
     615            0 :     }
     616              : }
     617              : 
     618              : /// A layer identifier when used in the [`ReadPath`] structure. This enum is for observability purposes
     619              : /// only and not used by the "real read path".
     620              : pub enum ReadPathLayerId {
     621              :     PersistentLayer(PersistentLayerKey),
     622              :     InMemoryLayer(Range<Lsn>),
     623              : }
     624              : 
     625              : impl std::fmt::Display for ReadPathLayerId {
     626            0 :     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
     627            0 :         match self {
     628            0 :             ReadPathLayerId::PersistentLayer(key) => write!(f, "{}", key),
     629            0 :             ReadPathLayerId::InMemoryLayer(range) => {
     630            0 :                 write!(f, "in-mem {}..{}", range.start, range.end)
     631              :             }
     632              :         }
     633            0 :     }
     634              : }
     635              : pub struct ReadPath {
     636              :     keyspace: KeySpace,
     637              :     lsn: Lsn,
     638              :     path: Vec<(ReadPathLayerId, KeySpace, Range<Lsn>)>,
     639              : }
     640              : 
     641              : impl ReadPath {
     642      1255306 :     pub fn new(keyspace: KeySpace, lsn: Lsn) -> Self {
     643      1255306 :         Self {
     644      1255306 :             keyspace,
     645      1255306 :             lsn,
     646      1255306 :             path: Vec::new(),
     647      1255306 :         }
     648      1255306 :     }
     649              : 
     650      1693718 :     pub fn record_layer_visit(
     651      1693718 :         &mut self,
     652      1693718 :         layer_to_read: &ReadableLayer,
     653      1693718 :         keyspace_to_read: &KeySpace,
     654      1693718 :         lsn_range: &Range<Lsn>,
     655      1693718 :     ) {
     656      1693718 :         let id = match layer_to_read {
     657       480346 :             ReadableLayer::PersistentLayer(layer) => {
     658       480346 :                 ReadPathLayerId::PersistentLayer(layer.layer_desc().key())
     659              :             }
     660      1213372 :             ReadableLayer::InMemoryLayer(layer) => {
     661      1213372 :                 ReadPathLayerId::InMemoryLayer(layer.get_lsn_range())
     662              :             }
     663              :         };
     664      1693718 :         self.path
     665      1693718 :             .push((id, keyspace_to_read.clone(), lsn_range.clone()));
     666      1693718 :     }
     667              : }
     668              : 
     669              : impl std::fmt::Display for ReadPath {
     670            0 :     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
     671            0 :         writeln!(f, "Read path for {} at lsn {}:", self.keyspace, self.lsn)?;
     672            0 :         for (idx, (layer_id, keyspace, lsn_range)) in self.path.iter().enumerate() {
     673            0 :             writeln!(
     674            0 :                 f,
     675            0 :                 "{}: {} {}..{} {}",
     676            0 :                 idx, layer_id, lsn_range.start, lsn_range.end, keyspace
     677            0 :             )?;
     678              :         }
     679            0 :         Ok(())
     680            0 :     }
     681              : }
     682              : 
     683              : #[derive(thiserror::Error)]
     684              : pub struct MissingKeyError {
     685              :     key: Key,
     686              :     shard: ShardNumber,
     687              :     cont_lsn: Lsn,
     688              :     request_lsn: Lsn,
     689              :     ancestor_lsn: Option<Lsn>,
     690              :     /// Debug information about the read path if there's an error
     691              :     read_path: Option<ReadPath>,
     692              :     backtrace: Option<std::backtrace::Backtrace>,
     693              : }
     694              : 
     695              : impl std::fmt::Debug for MissingKeyError {
     696            0 :     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
     697            0 :         write!(f, "{}", self)
     698            0 :     }
     699              : }
     700              : 
     701              : impl std::fmt::Display for MissingKeyError {
     702            0 :     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
     703            0 :         write!(
     704            0 :             f,
     705            0 :             "could not find data for key {} (shard {:?}) at LSN {}, request LSN {}",
     706            0 :             self.key, self.shard, self.cont_lsn, self.request_lsn
     707            0 :         )?;
     708              : 
     709            0 :         if let Some(ref ancestor_lsn) = self.ancestor_lsn {
     710            0 :             write!(f, ", ancestor {}", ancestor_lsn)?;
     711            0 :         }
     712              : 
     713            0 :         if let Some(ref read_path) = self.read_path {
     714            0 :             write!(f, "\n{}", read_path)?;
     715            0 :         }
     716              : 
     717            0 :         if let Some(ref backtrace) = self.backtrace {
     718            0 :             write!(f, "\n{}", backtrace)?;
     719            0 :         }
     720              : 
     721            0 :         Ok(())
     722            0 :     }
     723              : }
     724              : 
     725              : impl PageReconstructError {
     726              :     /// Returns true if this error indicates a tenant/timeline shutdown alike situation
     727            0 :     pub(crate) fn is_stopping(&self) -> bool {
     728              :         use PageReconstructError::*;
     729            0 :         match self {
     730            0 :             Cancelled => true,
     731            0 :             Other(_) | AncestorLsnTimeout(_) | WalRedo(_) | MissingKey(_) => false,
     732              :         }
     733            0 :     }
     734              : }
     735              : 
     736              : #[derive(thiserror::Error, Debug)]
     737              : pub(crate) enum CreateImageLayersError {
     738              :     #[error("timeline shutting down")]
     739              :     Cancelled,
     740              : 
     741              :     #[error("read failed")]
     742              :     GetVectoredError(#[source] GetVectoredError),
     743              : 
     744              :     #[error("reconstruction failed")]
     745              :     PageReconstructError(#[source] PageReconstructError),
     746              : 
     747              :     #[error(transparent)]
     748              :     Other(#[from] anyhow::Error),
     749              : }
     750              : 
     751              : impl From<layer_manager::Shutdown> for CreateImageLayersError {
     752            0 :     fn from(_: layer_manager::Shutdown) -> Self {
     753            0 :         CreateImageLayersError::Cancelled
     754            0 :     }
     755              : }
     756              : 
     757              : #[derive(thiserror::Error, Debug, Clone)]
     758              : pub(crate) enum FlushLayerError {
     759              :     /// Timeline cancellation token was cancelled
     760              :     #[error("timeline shutting down")]
     761              :     Cancelled,
     762              : 
     763              :     /// We tried to flush a layer while the Timeline is in an unexpected state
     764              :     #[error("cannot flush frozen layers when flush_loop is not running, state is {0:?}")]
     765              :     NotRunning(FlushLoopState),
     766              : 
     767              :     // Arc<> the following non-clonable error types: we must be Clone-able because the flush error is propagated from the flush
     768              :     // loop via a watch channel, where we can only borrow it.
     769              :     #[error("create image layers (shared)")]
     770              :     CreateImageLayersError(Arc<CreateImageLayersError>),
     771              : 
     772              :     #[error("other (shared)")]
     773              :     Other(#[from] Arc<anyhow::Error>),
     774              : }
     775              : 
     776              : impl FlushLayerError {
     777              :     // When crossing from generic anyhow errors to this error type, we explicitly check
     778              :     // for timeline cancellation to avoid logging inoffensive shutdown errors as warn/err.
     779            0 :     fn from_anyhow(timeline: &Timeline, err: anyhow::Error) -> Self {
     780            0 :         let cancelled = timeline.cancel.is_cancelled()
     781              :             // The upload queue might have been shut down before the official cancellation of the timeline.
     782            0 :             || err
     783            0 :                 .downcast_ref::<NotInitialized>()
     784            0 :                 .map(NotInitialized::is_stopping)
     785            0 :                 .unwrap_or_default();
     786            0 :         if cancelled {
     787            0 :             Self::Cancelled
     788              :         } else {
     789            0 :             Self::Other(Arc::new(err))
     790              :         }
     791            0 :     }
     792              : }
     793              : 
     794              : impl From<layer_manager::Shutdown> for FlushLayerError {
     795            0 :     fn from(_: layer_manager::Shutdown) -> Self {
     796            0 :         FlushLayerError::Cancelled
     797            0 :     }
     798              : }
     799              : 
     800              : #[derive(thiserror::Error, Debug)]
     801              : pub(crate) enum GetVectoredError {
     802              :     #[error("timeline shutting down")]
     803              :     Cancelled,
     804              : 
     805              :     #[error("requested too many keys: {0} > {}", Timeline::MAX_GET_VECTORED_KEYS)]
     806              :     Oversized(u64),
     807              : 
     808              :     #[error("requested at invalid LSN: {0}")]
     809              :     InvalidLsn(Lsn),
     810              : 
     811              :     #[error("requested key not found: {0}")]
     812              :     MissingKey(MissingKeyError),
     813              : 
     814              :     #[error("ancestry walk")]
     815              :     GetReadyAncestorError(#[source] GetReadyAncestorError),
     816              : 
     817              :     #[error(transparent)]
     818              :     Other(#[from] anyhow::Error),
     819              : }
     820              : 
     821              : impl From<GetReadyAncestorError> for GetVectoredError {
     822            4 :     fn from(value: GetReadyAncestorError) -> Self {
     823              :         use GetReadyAncestorError::*;
     824            4 :         match value {
     825            0 :             Cancelled => GetVectoredError::Cancelled,
     826              :             AncestorLsnTimeout(_) | BadState { .. } => {
     827            4 :                 GetVectoredError::GetReadyAncestorError(value)
     828              :             }
     829              :         }
     830            4 :     }
     831              : }
     832              : 
     833              : #[derive(thiserror::Error, Debug)]
     834              : pub(crate) enum GetReadyAncestorError {
     835              :     #[error("ancestor LSN wait error")]
     836              :     AncestorLsnTimeout(#[from] WaitLsnError),
     837              : 
     838              :     #[error("bad state on timeline {timeline_id}: {state:?}")]
     839              :     BadState {
     840              :         timeline_id: TimelineId,
     841              :         state: TimelineState,
     842              :     },
     843              : 
     844              :     #[error("cancelled")]
     845              :     Cancelled,
     846              : }
     847              : 
     848              : #[derive(Clone, Copy)]
     849              : pub enum LogicalSizeCalculationCause {
     850              :     Initial,
     851              :     ConsumptionMetricsSyntheticSize,
     852              :     EvictionTaskImitation,
     853              :     TenantSizeHandler,
     854              : }
     855              : 
     856              : pub enum GetLogicalSizePriority {
     857              :     User,
     858              :     Background,
     859              : }
     860              : 
     861            0 : #[derive(Debug, enumset::EnumSetType)]
     862              : pub(crate) enum CompactFlags {
     863              :     ForceRepartition,
     864              :     ForceImageLayerCreation,
     865              :     ForceL0Compaction,
     866              :     OnlyL0Compaction,
     867              :     EnhancedGcBottomMostCompaction,
     868              :     DryRun,
     869              :     /// Disables compaction yielding e.g. due to high L0 count. This is set e.g. when requesting
     870              :     /// compaction via HTTP API.
     871              :     NoYield,
     872              : }
     873              : 
     874              : #[serde_with::serde_as]
     875            0 : #[derive(Debug, Clone, serde::Deserialize)]
     876              : pub(crate) struct CompactRequest {
     877              :     pub compact_key_range: Option<CompactKeyRange>,
     878              :     pub compact_lsn_range: Option<CompactLsnRange>,
     879              :     /// Whether the compaction job should be scheduled.
     880              :     #[serde(default)]
     881              :     pub scheduled: bool,
     882              :     /// Whether the compaction job should be split across key ranges.
     883              :     #[serde(default)]
     884              :     pub sub_compaction: bool,
     885              :     /// Max job size for each subcompaction job.
     886              :     pub sub_compaction_max_job_size_mb: Option<u64>,
     887              : }
     888              : 
     889              : #[derive(Debug, Clone, Default)]
     890              : pub(crate) struct CompactOptions {
     891              :     pub flags: EnumSet<CompactFlags>,
     892              :     /// If set, the compaction will only compact the key range specified by this option.
     893              :     /// This option is only used by GC compaction. For the full explanation, see [`compaction::GcCompactJob`].
     894              :     pub compact_key_range: Option<CompactKeyRange>,
     895              :     /// If set, the compaction will only compact the LSN within this value.
     896              :     /// This option is only used by GC compaction. For the full explanation, see [`compaction::GcCompactJob`].
     897              :     pub compact_lsn_range: Option<CompactLsnRange>,
     898              :     /// Enable sub-compaction (split compaction job across key ranges).
     899              :     /// This option is only used by GC compaction.
     900              :     pub sub_compaction: bool,
     901              :     /// Set job size for the GC compaction.
     902              :     /// This option is only used by GC compaction.
     903              :     pub sub_compaction_max_job_size_mb: Option<u64>,
     904              : }
     905              : 
     906              : impl std::fmt::Debug for Timeline {
     907            0 :     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
     908            0 :         write!(f, "Timeline<{}>", self.timeline_id)
     909            0 :     }
     910              : }
     911              : 
     912              : #[derive(thiserror::Error, Debug)]
     913              : pub(crate) enum WaitLsnError {
     914              :     // Called on a timeline which is shutting down
     915              :     #[error("Shutdown")]
     916              :     Shutdown,
     917              : 
     918              :     // Called on an timeline not in active state or shutting down
     919              :     #[error("Bad timeline state: {0:?}")]
     920              :     BadState(TimelineState),
     921              : 
     922              :     // Timeout expired while waiting for LSN to catch up with goal.
     923              :     #[error("{0}")]
     924              :     Timeout(String),
     925              : }
     926              : 
     927              : // The impls below achieve cancellation mapping for errors.
     928              : // Perhaps there's a way of achieving this with less cruft.
     929              : 
     930              : impl From<CreateImageLayersError> for CompactionError {
     931            0 :     fn from(e: CreateImageLayersError) -> Self {
     932            0 :         match e {
     933            0 :             CreateImageLayersError::Cancelled => CompactionError::ShuttingDown,
     934            0 :             CreateImageLayersError::Other(e) => {
     935            0 :                 CompactionError::Other(e.context("create image layers"))
     936              :             }
     937            0 :             _ => CompactionError::Other(e.into()),
     938              :         }
     939            0 :     }
     940              : }
     941              : 
     942              : impl From<CreateImageLayersError> for FlushLayerError {
     943            0 :     fn from(e: CreateImageLayersError) -> Self {
     944            0 :         match e {
     945            0 :             CreateImageLayersError::Cancelled => FlushLayerError::Cancelled,
     946            0 :             any => FlushLayerError::CreateImageLayersError(Arc::new(any)),
     947              :         }
     948            0 :     }
     949              : }
     950              : 
     951              : impl From<PageReconstructError> for CreateImageLayersError {
     952            0 :     fn from(e: PageReconstructError) -> Self {
     953            0 :         match e {
     954            0 :             PageReconstructError::Cancelled => CreateImageLayersError::Cancelled,
     955            0 :             _ => CreateImageLayersError::PageReconstructError(e),
     956              :         }
     957            0 :     }
     958              : }
     959              : 
     960              : impl From<GetVectoredError> for CreateImageLayersError {
     961            0 :     fn from(e: GetVectoredError) -> Self {
     962            0 :         match e {
     963            0 :             GetVectoredError::Cancelled => CreateImageLayersError::Cancelled,
     964            0 :             _ => CreateImageLayersError::GetVectoredError(e),
     965              :         }
     966            0 :     }
     967              : }
     968              : 
     969              : impl From<GetVectoredError> for PageReconstructError {
     970           12 :     fn from(e: GetVectoredError) -> Self {
     971           12 :         match e {
     972            0 :             GetVectoredError::Cancelled => PageReconstructError::Cancelled,
     973            0 :             GetVectoredError::InvalidLsn(_) => PageReconstructError::Other(anyhow!("Invalid LSN")),
     974            0 :             err @ GetVectoredError::Oversized(_) => PageReconstructError::Other(err.into()),
     975            8 :             GetVectoredError::MissingKey(err) => PageReconstructError::MissingKey(err),
     976            4 :             GetVectoredError::GetReadyAncestorError(err) => PageReconstructError::from(err),
     977            0 :             GetVectoredError::Other(err) => PageReconstructError::Other(err),
     978              :         }
     979           12 :     }
     980              : }
     981              : 
     982              : impl From<GetReadyAncestorError> for PageReconstructError {
     983            4 :     fn from(e: GetReadyAncestorError) -> Self {
     984              :         use GetReadyAncestorError::*;
     985            4 :         match e {
     986            0 :             AncestorLsnTimeout(wait_err) => PageReconstructError::AncestorLsnTimeout(wait_err),
     987            4 :             bad_state @ BadState { .. } => PageReconstructError::Other(anyhow::anyhow!(bad_state)),
     988            0 :             Cancelled => PageReconstructError::Cancelled,
     989              :         }
     990            4 :     }
     991              : }
     992              : 
     993              : pub(crate) enum WaitLsnTimeout {
     994              :     Custom(Duration),
     995              :     // Use the [`PageServerConf::wait_lsn_timeout`] default
     996              :     Default,
     997              : }
     998              : 
     999              : pub(crate) enum WaitLsnWaiter<'a> {
    1000              :     Timeline(&'a Timeline),
    1001              :     Tenant,
    1002              :     PageService,
    1003              :     HttpEndpoint,
    1004              : }
    1005              : 
    1006              : /// Argument to [`Timeline::shutdown`].
    1007              : #[derive(Debug, Clone, Copy)]
    1008              : pub(crate) enum ShutdownMode {
    1009              :     /// Graceful shutdown, may do a lot of I/O as we flush any open layers to disk and then
    1010              :     /// also to remote storage.  This method can easily take multiple seconds for a busy timeline.
    1011              :     ///
    1012              :     /// While we are flushing, we continue to accept read I/O for LSNs ingested before
    1013              :     /// the call to [`Timeline::shutdown`].
    1014              :     FreezeAndFlush,
    1015              :     /// Only flush the layers to the remote storage without freezing any open layers. Flush the deletion
    1016              :     /// queue. This is the mode used by ancestor detach and any other operations that reloads a tenant
    1017              :     /// but not increasing the generation number. Note that this mode cannot be used at tenant shutdown,
    1018              :     /// as flushing the deletion queue at that time will cause shutdown-in-progress errors.
    1019              :     Reload,
    1020              :     /// Shut down immediately, without waiting for any open layers to flush.
    1021              :     Hard,
    1022              : }
    1023              : 
    1024              : enum ImageLayerCreationOutcome {
    1025              :     /// We generated an image layer
    1026              :     Generated {
    1027              :         unfinished_image_layer: ImageLayerWriter,
    1028              :     },
    1029              :     /// The key range is empty
    1030              :     Empty,
    1031              :     /// (Only used in metadata image layer creation), after reading the metadata keys, we decide to skip
    1032              :     /// the image layer creation.
    1033              :     Skip,
    1034              : }
    1035              : 
    1036              : /// Public interface functions
    1037              : impl Timeline {
    1038              :     /// Get the LSN where this branch was created
    1039            8 :     pub(crate) fn get_ancestor_lsn(&self) -> Lsn {
    1040            8 :         self.ancestor_lsn
    1041            8 :     }
    1042              : 
    1043              :     /// Get the ancestor's timeline id
    1044           24 :     pub(crate) fn get_ancestor_timeline_id(&self) -> Option<TimelineId> {
    1045           24 :         self.ancestor_timeline
    1046           24 :             .as_ref()
    1047           24 :             .map(|ancestor| ancestor.timeline_id)
    1048           24 :     }
    1049              : 
    1050              :     /// Get the ancestor timeline
    1051            4 :     pub(crate) fn ancestor_timeline(&self) -> Option<&Arc<Timeline>> {
    1052            4 :         self.ancestor_timeline.as_ref()
    1053            4 :     }
    1054              : 
    1055              :     /// Get the bytes written since the PITR cutoff on this branch, and
    1056              :     /// whether this branch's ancestor_lsn is within its parent's PITR.
    1057            0 :     pub(crate) fn get_pitr_history_stats(&self) -> (u64, bool) {
    1058            0 :         let gc_info = self.gc_info.read().unwrap();
    1059            0 :         let history = self
    1060            0 :             .get_last_record_lsn()
    1061            0 :             .checked_sub(gc_info.cutoffs.time)
    1062            0 :             .unwrap_or(Lsn(0))
    1063            0 :             .0;
    1064            0 :         (history, gc_info.within_ancestor_pitr)
    1065            0 :     }
    1066              : 
    1067              :     /// Read timeline's GC cutoff: this is the LSN at which GC has started to happen
    1068      1704239 :     pub(crate) fn get_applied_gc_cutoff_lsn(&self) -> RcuReadGuard<Lsn> {
    1069      1704239 :         self.applied_gc_cutoff_lsn.read()
    1070      1704239 :     }
    1071              : 
    1072              :     /// Read timeline's planned GC cutoff: this is the logical end of history that users
    1073              :     /// are allowed to read (based on configured PITR), even if physically we have more history.
    1074            0 :     pub(crate) fn get_gc_cutoff_lsn(&self) -> Lsn {
    1075            0 :         self.gc_info.read().unwrap().cutoffs.time
    1076            0 :     }
    1077              : 
    1078              :     /// Look up given page version.
    1079              :     ///
    1080              :     /// If a remote layer file is needed, it is downloaded as part of this
    1081              :     /// call.
    1082              :     ///
    1083              :     /// This method enforces [`Self::pagestream_throttle`] internally.
    1084              :     ///
    1085              :     /// NOTE: It is considered an error to 'get' a key that doesn't exist. The
    1086              :     /// abstraction above this needs to store suitable metadata to track what
    1087              :     /// data exists with what keys, in separate metadata entries. If a
    1088              :     /// non-existent key is requested, we may incorrectly return a value from
    1089              :     /// an ancestor branch, for example, or waste a lot of cycles chasing the
    1090              :     /// non-existing key.
    1091              :     ///
    1092              :     /// # Cancel-Safety
    1093              :     ///
    1094              :     /// This method is cancellation-safe.
    1095              :     #[inline(always)]
    1096      1215130 :     pub(crate) async fn get(
    1097      1215130 :         &self,
    1098      1215130 :         key: Key,
    1099      1215130 :         lsn: Lsn,
    1100      1215130 :         ctx: &RequestContext,
    1101      1215130 :     ) -> Result<Bytes, PageReconstructError> {
    1102      1215130 :         if !lsn.is_valid() {
    1103            0 :             return Err(PageReconstructError::Other(anyhow::anyhow!("Invalid LSN")));
    1104      1215130 :         }
    1105      1215130 : 
    1106      1215130 :         // This check is debug-only because of the cost of hashing, and because it's a double-check: we
    1107      1215130 :         // already checked the key against the shard_identity when looking up the Timeline from
    1108      1215130 :         // page_service.
    1109      1215130 :         debug_assert!(!self.shard_identity.is_key_disposable(&key));
    1110              : 
    1111      1215130 :         let keyspace = KeySpace {
    1112      1215130 :             ranges: vec![key..key.next()],
    1113      1215130 :         };
    1114      1215130 : 
    1115      1215130 :         let mut reconstruct_state = ValuesReconstructState::new(IoConcurrency::sequential());
    1116              : 
    1117      1215130 :         let vectored_res = self
    1118      1215130 :             .get_vectored_impl(keyspace.clone(), lsn, &mut reconstruct_state, ctx)
    1119      1215130 :             .await;
    1120              : 
    1121      1215130 :         let key_value = vectored_res?.pop_first();
    1122      1215118 :         match key_value {
    1123      1215094 :             Some((got_key, value)) => {
    1124      1215094 :                 if got_key != key {
    1125            0 :                     error!(
    1126            0 :                         "Expected {}, but singular vectored get returned {}",
    1127              :                         key, got_key
    1128              :                     );
    1129            0 :                     Err(PageReconstructError::Other(anyhow!(
    1130            0 :                         "Singular vectored get returned wrong key"
    1131            0 :                     )))
    1132              :                 } else {
    1133      1215094 :                     value
    1134              :                 }
    1135              :             }
    1136           24 :             None => Err(PageReconstructError::MissingKey(MissingKeyError {
    1137           24 :                 key,
    1138           24 :                 shard: self.shard_identity.get_shard_number(&key),
    1139           24 :                 cont_lsn: Lsn(0),
    1140           24 :                 request_lsn: lsn,
    1141           24 :                 ancestor_lsn: None,
    1142           24 :                 backtrace: None,
    1143           24 :                 read_path: None,
    1144           24 :             })),
    1145              :         }
    1146      1215130 :     }
    1147              : 
    1148              :     pub(crate) const MAX_GET_VECTORED_KEYS: u64 = 32;
    1149              :     pub(crate) const LAYERS_VISITED_WARN_THRESHOLD: u32 = 100;
    1150              : 
    1151              :     /// Look up multiple page versions at a given LSN
    1152              :     ///
    1153              :     /// This naive implementation will be replaced with a more efficient one
    1154              :     /// which actually vectorizes the read path.
    1155        39432 :     pub(crate) async fn get_vectored(
    1156        39432 :         &self,
    1157        39432 :         keyspace: KeySpace,
    1158        39432 :         lsn: Lsn,
    1159        39432 :         io_concurrency: super::storage_layer::IoConcurrency,
    1160        39432 :         ctx: &RequestContext,
    1161        39432 :     ) -> Result<BTreeMap<Key, Result<Bytes, PageReconstructError>>, GetVectoredError> {
    1162        39432 :         if !lsn.is_valid() {
    1163            0 :             return Err(GetVectoredError::InvalidLsn(lsn));
    1164        39432 :         }
    1165        39432 : 
    1166        39432 :         let key_count = keyspace.total_raw_size().try_into().unwrap();
    1167        39432 :         if key_count > Timeline::MAX_GET_VECTORED_KEYS {
    1168            0 :             return Err(GetVectoredError::Oversized(key_count));
    1169        39432 :         }
    1170              : 
    1171        78864 :         for range in &keyspace.ranges {
    1172        39432 :             let mut key = range.start;
    1173        79380 :             while key != range.end {
    1174        39948 :                 assert!(!self.shard_identity.is_key_disposable(&key));
    1175        39948 :                 key = key.next();
    1176              :             }
    1177              :         }
    1178              : 
    1179        39432 :         trace!(
    1180            0 :             "get vectored request for {:?}@{} from task kind {:?}",
    1181            0 :             keyspace,
    1182            0 :             lsn,
    1183            0 :             ctx.task_kind(),
    1184              :         );
    1185              : 
    1186        39432 :         let start = crate::metrics::GET_VECTORED_LATENCY
    1187        39432 :             .for_task_kind(ctx.task_kind())
    1188        39432 :             .map(|metric| (metric, Instant::now()));
    1189              : 
    1190        39432 :         let res = self
    1191        39432 :             .get_vectored_impl(
    1192        39432 :                 keyspace.clone(),
    1193        39432 :                 lsn,
    1194        39432 :                 &mut ValuesReconstructState::new(io_concurrency),
    1195        39432 :                 ctx,
    1196        39432 :             )
    1197        39432 :             .await;
    1198              : 
    1199        39432 :         if let Some((metric, start)) = start {
    1200            0 :             let elapsed = start.elapsed();
    1201            0 :             metric.observe(elapsed.as_secs_f64());
    1202        39432 :         }
    1203              : 
    1204        39432 :         res
    1205        39432 :     }
    1206              : 
    1207              :     /// Scan the keyspace and return all existing key-values in the keyspace. This currently uses vectored
    1208              :     /// get underlying. Normal vectored get would throw an error when a key in the keyspace is not found
    1209              :     /// during the search, but for the scan interface, it returns all existing key-value pairs, and does
    1210              :     /// not expect each single key in the key space will be found. The semantics is closer to the RocksDB
    1211              :     /// scan iterator interface. We could optimize this interface later to avoid some checks in the vectored
    1212              :     /// get path to maintain and split the probing and to-be-probe keyspace. We also need to ensure that
    1213              :     /// the scan operation will not cause OOM in the future.
    1214           24 :     pub(crate) async fn scan(
    1215           24 :         &self,
    1216           24 :         keyspace: KeySpace,
    1217           24 :         lsn: Lsn,
    1218           24 :         ctx: &RequestContext,
    1219           24 :         io_concurrency: super::storage_layer::IoConcurrency,
    1220           24 :     ) -> Result<BTreeMap<Key, Result<Bytes, PageReconstructError>>, GetVectoredError> {
    1221           24 :         if !lsn.is_valid() {
    1222            0 :             return Err(GetVectoredError::InvalidLsn(lsn));
    1223           24 :         }
    1224           24 : 
    1225           24 :         trace!(
    1226            0 :             "key-value scan request for {:?}@{} from task kind {:?}",
    1227            0 :             keyspace,
    1228            0 :             lsn,
    1229            0 :             ctx.task_kind()
    1230              :         );
    1231              : 
    1232              :         // We should generalize this into Keyspace::contains in the future.
    1233           48 :         for range in &keyspace.ranges {
    1234           24 :             if range.start.field1 < METADATA_KEY_BEGIN_PREFIX
    1235           24 :                 || range.end.field1 > METADATA_KEY_END_PREFIX
    1236              :             {
    1237            0 :                 return Err(GetVectoredError::Other(anyhow::anyhow!(
    1238            0 :                     "only metadata keyspace can be scanned"
    1239            0 :                 )));
    1240           24 :             }
    1241              :         }
    1242              : 
    1243           24 :         let start = crate::metrics::SCAN_LATENCY
    1244           24 :             .for_task_kind(ctx.task_kind())
    1245           24 :             .map(ScanLatencyOngoingRecording::start_recording);
    1246              : 
    1247           24 :         let vectored_res = self
    1248           24 :             .get_vectored_impl(
    1249           24 :                 keyspace.clone(),
    1250           24 :                 lsn,
    1251           24 :                 &mut ValuesReconstructState::new(io_concurrency),
    1252           24 :                 ctx,
    1253           24 :             )
    1254           24 :             .await;
    1255              : 
    1256           24 :         if let Some(recording) = start {
    1257            0 :             recording.observe();
    1258           24 :         }
    1259              : 
    1260           24 :         vectored_res
    1261           24 :     }
    1262              : 
    1263      1255306 :     pub(super) async fn get_vectored_impl(
    1264      1255306 :         &self,
    1265      1255306 :         keyspace: KeySpace,
    1266      1255306 :         lsn: Lsn,
    1267      1255306 :         reconstruct_state: &mut ValuesReconstructState,
    1268      1255306 :         ctx: &RequestContext,
    1269      1255306 :     ) -> Result<BTreeMap<Key, Result<Bytes, PageReconstructError>>, GetVectoredError> {
    1270      1255306 :         let read_path = if self.conf.enable_read_path_debugging || ctx.read_path_debug() {
    1271      1255306 :             Some(ReadPath::new(keyspace.clone(), lsn))
    1272              :         } else {
    1273            0 :             None
    1274              :         };
    1275      1255306 :         reconstruct_state.read_path = read_path;
    1276              : 
    1277      1255306 :         let traversal_res: Result<(), _> = self
    1278      1255306 :             .get_vectored_reconstruct_data(keyspace.clone(), lsn, reconstruct_state, ctx)
    1279      1255306 :             .await;
    1280      1255306 :         if let Err(err) = traversal_res {
    1281              :             // Wait for all the spawned IOs to complete.
    1282              :             // See comments on `spawn_io` inside `storage_layer` for more details.
    1283           32 :             let mut collect_futs = std::mem::take(&mut reconstruct_state.keys)
    1284           32 :                 .into_values()
    1285           32 :                 .map(|state| state.collect_pending_ios())
    1286           32 :                 .collect::<FuturesUnordered<_>>();
    1287           32 :             while collect_futs.next().await.is_some() {}
    1288           32 :             return Err(err);
    1289      1255274 :         };
    1290      1255274 : 
    1291      1255274 :         let layers_visited = reconstruct_state.get_layers_visited();
    1292      1255274 : 
    1293      1255274 :         let futs = FuturesUnordered::new();
    1294      1335978 :         for (key, state) in std::mem::take(&mut reconstruct_state.keys) {
    1295      1335978 :             futs.push({
    1296      1335978 :                 let walredo_self = self.myself.upgrade().expect("&self method holds the arc");
    1297      1335978 :                 async move {
    1298      1335978 :                     assert_eq!(state.situation, ValueReconstructSituation::Complete);
    1299              : 
    1300      1335978 :                     let converted = match state.collect_pending_ios().await {
    1301      1335978 :                         Ok(ok) => ok,
    1302            0 :                         Err(err) => {
    1303            0 :                             return (key, Err(err));
    1304              :                         }
    1305              :                     };
    1306      1335978 :                     DELTAS_PER_READ_GLOBAL.observe(converted.num_deltas() as f64);
    1307      1335978 : 
    1308      1335978 :                     // The walredo module expects the records to be descending in terms of Lsn.
    1309      1335978 :                     // And we submit the IOs in that order, so, there shuold be no need to sort here.
    1310      1335978 :                     debug_assert!(
    1311      1335978 :                         converted
    1312      1335978 :                             .records
    1313      1335978 :                             .is_sorted_by_key(|(lsn, _)| std::cmp::Reverse(*lsn)),
    1314            0 :                         "{converted:?}"
    1315              :                     );
    1316              : 
    1317              :                     (
    1318      1335978 :                         key,
    1319      1335978 :                         walredo_self.reconstruct_value(key, lsn, converted).await,
    1320              :                     )
    1321      1335978 :                 }
    1322      1335978 :             });
    1323      1335978 :         }
    1324              : 
    1325      1255274 :         let results = futs
    1326      1255274 :             .collect::<BTreeMap<Key, Result<Bytes, PageReconstructError>>>()
    1327      1255274 :             .await;
    1328              : 
    1329              :         // For aux file keys (v1 or v2) the vectored read path does not return an error
    1330              :         // when they're missing. Instead they are omitted from the resulting btree
    1331              :         // (this is a requirement, not a bug). Skip updating the metric in these cases
    1332              :         // to avoid infinite results.
    1333      1255274 :         if !results.is_empty() {
    1334      1254782 :             if layers_visited >= Self::LAYERS_VISITED_WARN_THRESHOLD {
    1335            0 :                 static LOG_PACER: Lazy<Mutex<RateLimit>> =
    1336            0 :                     Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(60))));
    1337            0 :                 LOG_PACER.lock().unwrap().call(|| {
    1338            0 :                     let num_keys = keyspace.total_raw_size();
    1339            0 :                     let num_pages = results.len();
    1340            0 :                     tracing::info!(
    1341            0 :                       shard_id = %self.tenant_shard_id.shard_slug(),
    1342            0 :                       lsn = %lsn,
    1343            0 :                       "Vectored read for {keyspace} visited {layers_visited} layers. Returned {num_pages}/{num_keys} pages.",
    1344              :                     );
    1345            0 :                 });
    1346      1254782 :             }
    1347              : 
    1348              :             // Records the number of layers visited in a few different ways:
    1349              :             //
    1350              :             // * LAYERS_PER_READ: all layers count towards every read in the batch, because each
    1351              :             //   layer directly affects its observed latency.
    1352              :             //
    1353              :             // * LAYERS_PER_READ_BATCH: all layers count towards each batch, to get the per-batch
    1354              :             //   layer visits and access cost.
    1355              :             //
    1356              :             // * LAYERS_PER_READ_AMORTIZED: the average layer count per read, to get the amortized
    1357              :             //   read amplification after batching.
    1358      1254782 :             let layers_visited = layers_visited as f64;
    1359      1254782 :             let avg_layers_visited = layers_visited / results.len() as f64;
    1360      1254782 :             LAYERS_PER_READ_BATCH_GLOBAL.observe(layers_visited);
    1361      2590760 :             for _ in &results {
    1362      1335978 :                 self.metrics.layers_per_read.observe(layers_visited);
    1363      1335978 :                 LAYERS_PER_READ_GLOBAL.observe(layers_visited);
    1364      1335978 :                 LAYERS_PER_READ_AMORTIZED_GLOBAL.observe(avg_layers_visited);
    1365      1335978 :             }
    1366          492 :         }
    1367              : 
    1368      1255274 :         Ok(results)
    1369      1255306 :     }
    1370              : 
    1371              :     /// Get last or prev record separately. Same as get_last_record_rlsn().last/prev.
    1372       548924 :     pub(crate) fn get_last_record_lsn(&self) -> Lsn {
    1373       548924 :         self.last_record_lsn.load().last
    1374       548924 :     }
    1375              : 
    1376            0 :     pub(crate) fn get_prev_record_lsn(&self) -> Lsn {
    1377            0 :         self.last_record_lsn.load().prev
    1378            0 :     }
    1379              : 
    1380              :     /// Atomically get both last and prev.
    1381          456 :     pub(crate) fn get_last_record_rlsn(&self) -> RecordLsn {
    1382          456 :         self.last_record_lsn.load()
    1383          456 :     }
    1384              : 
    1385              :     /// Subscribe to callers of wait_lsn(). The value of the channel is None if there are no
    1386              :     /// wait_lsn() calls in progress, and Some(Lsn) if there is an active waiter for wait_lsn().
    1387            0 :     pub(crate) fn subscribe_for_wait_lsn_updates(&self) -> watch::Receiver<Option<Lsn>> {
    1388            0 :         self.last_record_lsn.status_receiver()
    1389            0 :     }
    1390              : 
    1391          896 :     pub(crate) fn get_disk_consistent_lsn(&self) -> Lsn {
    1392          896 :         self.disk_consistent_lsn.load()
    1393          896 :     }
    1394              : 
    1395              :     /// remote_consistent_lsn from the perspective of the tenant's current generation,
    1396              :     /// not validated with control plane yet.
    1397              :     /// See [`Self::get_remote_consistent_lsn_visible`].
    1398            0 :     pub(crate) fn get_remote_consistent_lsn_projected(&self) -> Option<Lsn> {
    1399            0 :         self.remote_client.remote_consistent_lsn_projected()
    1400            0 :     }
    1401              : 
    1402              :     /// remote_consistent_lsn which the tenant is guaranteed not to go backward from,
    1403              :     /// i.e. a value of remote_consistent_lsn_projected which has undergone
    1404              :     /// generation validation in the deletion queue.
    1405            0 :     pub(crate) fn get_remote_consistent_lsn_visible(&self) -> Option<Lsn> {
    1406            0 :         self.remote_client.remote_consistent_lsn_visible()
    1407            0 :     }
    1408              : 
    1409              :     /// The sum of the file size of all historic layers in the layer map.
    1410              :     /// This method makes no distinction between local and remote layers.
    1411              :     /// Hence, the result **does not represent local filesystem usage**.
    1412            0 :     pub(crate) async fn layer_size_sum(&self) -> u64 {
    1413            0 :         let guard = self.layers.read().await;
    1414            0 :         guard.layer_size_sum()
    1415            0 :     }
    1416              : 
    1417            0 :     pub(crate) fn resident_physical_size(&self) -> u64 {
    1418            0 :         self.metrics.resident_physical_size_get()
    1419            0 :     }
    1420              : 
    1421            0 :     pub(crate) fn get_directory_metrics(&self) -> [u64; DirectoryKind::KINDS_NUM] {
    1422            0 :         array::from_fn(|idx| self.directory_metrics[idx].load(AtomicOrdering::Relaxed))
    1423            0 :     }
    1424              : 
    1425              :     ///
    1426              :     /// Wait until WAL has been received and processed up to this LSN.
    1427              :     ///
    1428              :     /// You should call this before any of the other get_* or list_* functions. Calling
    1429              :     /// those functions with an LSN that has been processed yet is an error.
    1430              :     ///
    1431       448329 :     pub(crate) async fn wait_lsn(
    1432       448329 :         &self,
    1433       448329 :         lsn: Lsn,
    1434       448329 :         who_is_waiting: WaitLsnWaiter<'_>,
    1435       448329 :         timeout: WaitLsnTimeout,
    1436       448329 :         ctx: &RequestContext, /* Prepare for use by cancellation */
    1437       448329 :     ) -> Result<(), WaitLsnError> {
    1438       448329 :         let state = self.current_state();
    1439       448329 :         if self.cancel.is_cancelled() || matches!(state, TimelineState::Stopping) {
    1440            0 :             return Err(WaitLsnError::Shutdown);
    1441       448329 :         } else if !matches!(state, TimelineState::Active) {
    1442            0 :             return Err(WaitLsnError::BadState(state));
    1443       448329 :         }
    1444       448329 : 
    1445       448329 :         if cfg!(debug_assertions) {
    1446       448329 :             match ctx.task_kind() {
    1447              :                 TaskKind::WalReceiverManager
    1448              :                 | TaskKind::WalReceiverConnectionHandler
    1449              :                 | TaskKind::WalReceiverConnectionPoller => {
    1450            0 :                     let is_myself = match who_is_waiting {
    1451            0 :                         WaitLsnWaiter::Timeline(waiter) => {
    1452            0 :                             Weak::ptr_eq(&waiter.myself, &self.myself)
    1453              :                         }
    1454              :                         WaitLsnWaiter::Tenant
    1455              :                         | WaitLsnWaiter::PageService
    1456            0 :                         | WaitLsnWaiter::HttpEndpoint => unreachable!(
    1457            0 :                             "tenant or page_service context are not expected to have task kind {:?}",
    1458            0 :                             ctx.task_kind()
    1459            0 :                         ),
    1460              :                     };
    1461            0 :                     if is_myself {
    1462            0 :                         if let Err(current) = self.last_record_lsn.would_wait_for(lsn) {
    1463              :                             // walingest is the only one that can advance last_record_lsn; it should make sure to never reach here
    1464            0 :                             panic!(
    1465            0 :                                 "this timeline's walingest task is calling wait_lsn({lsn}) but we only have last_record_lsn={current}; would deadlock"
    1466            0 :                             );
    1467            0 :                         }
    1468            0 :                     } else {
    1469            0 :                         // if another  timeline's  is waiting for us, there's no deadlock risk because
    1470            0 :                         // our walreceiver task can make progress independent of theirs
    1471            0 :                     }
    1472              :                 }
    1473       448329 :                 _ => {}
    1474              :             }
    1475            0 :         }
    1476              : 
    1477       448329 :         let timeout = match timeout {
    1478            0 :             WaitLsnTimeout::Custom(t) => t,
    1479       448329 :             WaitLsnTimeout::Default => self.conf.wait_lsn_timeout,
    1480              :         };
    1481              : 
    1482       448329 :         let _timer = crate::metrics::WAIT_LSN_TIME.start_timer();
    1483       448329 : 
    1484       448329 :         match self.last_record_lsn.wait_for_timeout(lsn, timeout).await {
    1485       448329 :             Ok(()) => Ok(()),
    1486            0 :             Err(e) => {
    1487              :                 use utils::seqwait::SeqWaitError::*;
    1488            0 :                 match e {
    1489            0 :                     Shutdown => Err(WaitLsnError::Shutdown),
    1490              :                     Timeout => {
    1491              :                         // don't count the time spent waiting for lock below, and also in walreceiver.status(), towards the wait_lsn_time_histo
    1492            0 :                         drop(_timer);
    1493            0 :                         let walreceiver_status = self.walreceiver_status();
    1494            0 :                         Err(WaitLsnError::Timeout(format!(
    1495            0 :                             "Timed out while waiting for WAL record at LSN {} to arrive, last_record_lsn {} disk consistent LSN={}, WalReceiver status: {}",
    1496            0 :                             lsn,
    1497            0 :                             self.get_last_record_lsn(),
    1498            0 :                             self.get_disk_consistent_lsn(),
    1499            0 :                             walreceiver_status,
    1500            0 :                         )))
    1501              :                     }
    1502              :                 }
    1503              :             }
    1504              :         }
    1505       448329 :     }
    1506              : 
    1507            0 :     pub(crate) fn walreceiver_status(&self) -> String {
    1508            0 :         match &*self.walreceiver.lock().unwrap() {
    1509            0 :             None => "stopping or stopped".to_string(),
    1510            0 :             Some(walreceiver) => match walreceiver.status() {
    1511            0 :                 Some(status) => status.to_human_readable_string(),
    1512            0 :                 None => "Not active".to_string(),
    1513              :             },
    1514              :         }
    1515            0 :     }
    1516              : 
    1517              :     /// Check that it is valid to request operations with that lsn.
    1518          464 :     pub(crate) fn check_lsn_is_in_scope(
    1519          464 :         &self,
    1520          464 :         lsn: Lsn,
    1521          464 :         latest_gc_cutoff_lsn: &RcuReadGuard<Lsn>,
    1522          464 :     ) -> anyhow::Result<()> {
    1523          464 :         ensure!(
    1524          464 :             lsn >= **latest_gc_cutoff_lsn,
    1525            8 :             "LSN {} is earlier than latest GC cutoff {} (we might've already garbage collected needed data)",
    1526            8 :             lsn,
    1527            8 :             **latest_gc_cutoff_lsn,
    1528              :         );
    1529          456 :         Ok(())
    1530          464 :     }
    1531              : 
    1532              :     /// Initializes an LSN lease. The function will return an error if the requested LSN is less than the `latest_gc_cutoff_lsn`.
    1533           20 :     pub(crate) fn init_lsn_lease(
    1534           20 :         &self,
    1535           20 :         lsn: Lsn,
    1536           20 :         length: Duration,
    1537           20 :         ctx: &RequestContext,
    1538           20 :     ) -> anyhow::Result<LsnLease> {
    1539           20 :         self.make_lsn_lease(lsn, length, true, ctx)
    1540           20 :     }
    1541              : 
    1542              :     /// Renews a lease at a particular LSN. The requested LSN is not validated against the `latest_gc_cutoff_lsn` when we are in the grace period.
    1543            8 :     pub(crate) fn renew_lsn_lease(
    1544            8 :         &self,
    1545            8 :         lsn: Lsn,
    1546            8 :         length: Duration,
    1547            8 :         ctx: &RequestContext,
    1548            8 :     ) -> anyhow::Result<LsnLease> {
    1549            8 :         self.make_lsn_lease(lsn, length, false, ctx)
    1550            8 :     }
    1551              : 
    1552              :     /// Obtains a temporary lease blocking garbage collection for the given LSN.
    1553              :     ///
    1554              :     /// If we are in `AttachedSingle` mode and is not blocked by the lsn lease deadline, this function will error
    1555              :     /// if the requesting LSN is less than the `latest_gc_cutoff_lsn` and there is no existing request present.
    1556              :     ///
    1557              :     /// If there is an existing lease in the map, the lease will be renewed only if the request extends the lease.
    1558              :     /// The returned lease is therefore the maximum between the existing lease and the requesting lease.
    1559           28 :     fn make_lsn_lease(
    1560           28 :         &self,
    1561           28 :         lsn: Lsn,
    1562           28 :         length: Duration,
    1563           28 :         init: bool,
    1564           28 :         _ctx: &RequestContext,
    1565           28 :     ) -> anyhow::Result<LsnLease> {
    1566           24 :         let lease = {
    1567              :             // Normalize the requested LSN to be aligned, and move to the first record
    1568              :             // if it points to the beginning of the page (header).
    1569           28 :             let lsn = xlog_utils::normalize_lsn(lsn, WAL_SEGMENT_SIZE);
    1570           28 : 
    1571           28 :             let mut gc_info = self.gc_info.write().unwrap();
    1572           28 :             let planned_cutoff = gc_info.min_cutoff();
    1573           28 : 
    1574           28 :             let valid_until = SystemTime::now() + length;
    1575           28 : 
    1576           28 :             let entry = gc_info.leases.entry(lsn);
    1577           28 : 
    1578           28 :             match entry {
    1579           12 :                 Entry::Occupied(mut occupied) => {
    1580           12 :                     let existing_lease = occupied.get_mut();
    1581           12 :                     if valid_until > existing_lease.valid_until {
    1582            4 :                         existing_lease.valid_until = valid_until;
    1583            4 :                         let dt: DateTime<Utc> = valid_until.into();
    1584            4 :                         info!("lease extended to {}", dt);
    1585              :                     } else {
    1586            8 :                         let dt: DateTime<Utc> = existing_lease.valid_until.into();
    1587            8 :                         info!("existing lease covers greater length, valid until {}", dt);
    1588              :                     }
    1589              : 
    1590           12 :                     existing_lease.clone()
    1591              :                 }
    1592           16 :                 Entry::Vacant(vacant) => {
    1593              :                     // Reject already GC-ed LSN if we are in AttachedSingle and
    1594              :                     // not blocked by the lsn lease deadline.
    1595           16 :                     let validate = {
    1596           16 :                         let conf = self.tenant_conf.load();
    1597           16 :                         conf.location.attach_mode == AttachmentMode::Single
    1598           16 :                             && !conf.is_gc_blocked_by_lsn_lease_deadline()
    1599              :                     };
    1600              : 
    1601           16 :                     if init || validate {
    1602           16 :                         let latest_gc_cutoff_lsn = self.get_applied_gc_cutoff_lsn();
    1603           16 :                         if lsn < *latest_gc_cutoff_lsn {
    1604            4 :                             bail!(
    1605            4 :                                 "tried to request an lsn lease for an lsn below the latest gc cutoff. requested at {} gc cutoff {}",
    1606            4 :                                 lsn,
    1607            4 :                                 *latest_gc_cutoff_lsn
    1608            4 :                             );
    1609           12 :                         }
    1610           12 :                         if lsn < planned_cutoff {
    1611            0 :                             bail!(
    1612            0 :                                 "tried to request an lsn lease for an lsn below the planned gc cutoff. requested at {} planned gc cutoff {}",
    1613            0 :                                 lsn,
    1614            0 :                                 planned_cutoff
    1615            0 :                             );
    1616           12 :                         }
    1617            0 :                     }
    1618              : 
    1619           12 :                     let dt: DateTime<Utc> = valid_until.into();
    1620           12 :                     info!("lease created, valid until {}", dt);
    1621           12 :                     vacant.insert(LsnLease { valid_until }).clone()
    1622              :                 }
    1623              :             }
    1624              :         };
    1625              : 
    1626           24 :         Ok(lease)
    1627           28 :     }
    1628              : 
    1629              :     /// Freeze the current open in-memory layer. It will be written to disk on next iteration.
    1630              :     /// Returns the flush request ID which can be awaited with wait_flush_completion().
    1631              :     #[instrument(skip(self), fields(tenant_id=%self.tenant_shard_id.tenant_id, shard_id=%self.tenant_shard_id.shard_slug(), timeline_id=%self.timeline_id))]
    1632              :     pub(crate) async fn freeze(&self) -> Result<u64, FlushLayerError> {
    1633              :         self.freeze0().await
    1634              :     }
    1635              : 
    1636              :     /// Freeze and flush the open in-memory layer, waiting for it to be written to disk.
    1637              :     #[instrument(skip(self), fields(tenant_id=%self.tenant_shard_id.tenant_id, shard_id=%self.tenant_shard_id.shard_slug(), timeline_id=%self.timeline_id))]
    1638              :     pub(crate) async fn freeze_and_flush(&self) -> Result<(), FlushLayerError> {
    1639              :         self.freeze_and_flush0().await
    1640              :     }
    1641              : 
    1642              :     /// Freeze the current open in-memory layer. It will be written to disk on next iteration.
    1643              :     /// Returns the flush request ID which can be awaited with wait_flush_completion().
    1644         2252 :     pub(crate) async fn freeze0(&self) -> Result<u64, FlushLayerError> {
    1645         2252 :         let mut g = self.write_lock.lock().await;
    1646         2252 :         let to_lsn = self.get_last_record_lsn();
    1647         2252 :         self.freeze_inmem_layer_at(to_lsn, &mut g).await
    1648         2252 :     }
    1649              : 
    1650              :     // This exists to provide a non-span creating version of `freeze_and_flush` we can call without
    1651              :     // polluting the span hierarchy.
    1652         2252 :     pub(crate) async fn freeze_and_flush0(&self) -> Result<(), FlushLayerError> {
    1653         2252 :         let token = self.freeze0().await?;
    1654         2252 :         self.wait_flush_completion(token).await
    1655         2252 :     }
    1656              : 
    1657              :     // Check if an open ephemeral layer should be closed: this provides
    1658              :     // background enforcement of checkpoint interval if there is no active WAL receiver, to avoid keeping
    1659              :     // an ephemeral layer open forever when idle.  It also freezes layers if the global limit on
    1660              :     // ephemeral layer bytes has been breached.
    1661            0 :     pub(super) async fn maybe_freeze_ephemeral_layer(&self) {
    1662            0 :         let Ok(mut write_guard) = self.write_lock.try_lock() else {
    1663              :             // If the write lock is held, there is an active wal receiver: rolling open layers
    1664              :             // is their responsibility while they hold this lock.
    1665            0 :             return;
    1666              :         };
    1667              : 
    1668              :         // FIXME: why not early exit? because before #7927 the state would had been cleared every
    1669              :         // time, and this was missed.
    1670              :         // if write_guard.is_none() { return; }
    1671              : 
    1672            0 :         let Ok(layers_guard) = self.layers.try_read() else {
    1673              :             // Don't block if the layer lock is busy
    1674            0 :             return;
    1675              :         };
    1676              : 
    1677            0 :         let Ok(lm) = layers_guard.layer_map() else {
    1678            0 :             return;
    1679              :         };
    1680              : 
    1681            0 :         let Some(open_layer) = &lm.open_layer else {
    1682              :             // If there is no open layer, we have no layer freezing to do.  However, we might need to generate
    1683              :             // some updates to disk_consistent_lsn and remote_consistent_lsn, in case we ingested some WAL regions
    1684              :             // that didn't result in writes to this shard.
    1685              : 
    1686              :             // Must not hold the layers lock while waiting for a flush.
    1687            0 :             drop(layers_guard);
    1688            0 : 
    1689            0 :             let last_record_lsn = self.get_last_record_lsn();
    1690            0 :             let disk_consistent_lsn = self.get_disk_consistent_lsn();
    1691            0 :             if last_record_lsn > disk_consistent_lsn {
    1692              :                 // We have no open layer, but disk_consistent_lsn is behind the last record: this indicates
    1693              :                 // we are a sharded tenant and have skipped some WAL
    1694            0 :                 let last_freeze_ts = *self.last_freeze_ts.read().unwrap();
    1695            0 :                 if last_freeze_ts.elapsed() >= self.get_checkpoint_timeout() {
    1696              :                     // Only do this if have been layer-less longer than get_checkpoint_timeout, so that a shard
    1697              :                     // without any data ingested (yet) doesn't write a remote index as soon as it
    1698              :                     // sees its LSN advance: we only do this if we've been layer-less
    1699              :                     // for some time.
    1700            0 :                     tracing::debug!(
    1701            0 :                         "Advancing disk_consistent_lsn past WAL ingest gap {} -> {}",
    1702              :                         disk_consistent_lsn,
    1703              :                         last_record_lsn
    1704              :                     );
    1705              : 
    1706              :                     // The flush loop will update remote consistent LSN as well as disk consistent LSN.
    1707              :                     // We know there is no open layer, so we can request freezing without actually
    1708              :                     // freezing anything. This is true even if we have dropped the layers_guard, we
    1709              :                     // still hold the write_guard.
    1710            0 :                     let _ = async {
    1711            0 :                         let token = self
    1712            0 :                             .freeze_inmem_layer_at(last_record_lsn, &mut write_guard)
    1713            0 :                             .await?;
    1714            0 :                         self.wait_flush_completion(token).await
    1715            0 :                     }
    1716            0 :                     .await;
    1717            0 :                 }
    1718            0 :             }
    1719              : 
    1720            0 :             return;
    1721              :         };
    1722              : 
    1723            0 :         let Some(current_size) = open_layer.try_len() else {
    1724              :             // Unexpected: since we hold the write guard, nobody else should be writing to this layer, so
    1725              :             // read lock to get size should always succeed.
    1726            0 :             tracing::warn!("Lock conflict while reading size of open layer");
    1727            0 :             return;
    1728              :         };
    1729              : 
    1730            0 :         let current_lsn = self.get_last_record_lsn();
    1731              : 
    1732            0 :         let checkpoint_distance_override = open_layer.tick().await;
    1733              : 
    1734            0 :         if let Some(size_override) = checkpoint_distance_override {
    1735            0 :             if current_size > size_override {
    1736              :                 // This is not harmful, but it only happens in relatively rare cases where
    1737              :                 // time-based checkpoints are not happening fast enough to keep the amount of
    1738              :                 // ephemeral data within configured limits.  It's a sign of stress on the system.
    1739            0 :                 tracing::info!(
    1740            0 :                     "Early-rolling open layer at size {current_size} (limit {size_override}) due to dirty data pressure"
    1741              :                 );
    1742            0 :             }
    1743            0 :         }
    1744              : 
    1745            0 :         let checkpoint_distance =
    1746            0 :             checkpoint_distance_override.unwrap_or(self.get_checkpoint_distance());
    1747            0 : 
    1748            0 :         if self.should_roll(
    1749            0 :             current_size,
    1750            0 :             current_size,
    1751            0 :             checkpoint_distance,
    1752            0 :             self.get_last_record_lsn(),
    1753            0 :             self.last_freeze_at.load(),
    1754            0 :             open_layer.get_opened_at(),
    1755            0 :         ) {
    1756            0 :             match open_layer.info() {
    1757            0 :                 InMemoryLayerInfo::Frozen { lsn_start, lsn_end } => {
    1758            0 :                     // We may reach this point if the layer was already frozen by not yet flushed: flushing
    1759            0 :                     // happens asynchronously in the background.
    1760            0 :                     tracing::debug!(
    1761            0 :                         "Not freezing open layer, it's already frozen ({lsn_start}..{lsn_end})"
    1762              :                     );
    1763              :                 }
    1764              :                 InMemoryLayerInfo::Open { .. } => {
    1765              :                     // Upgrade to a write lock and freeze the layer
    1766            0 :                     drop(layers_guard);
    1767            0 :                     let res = self
    1768            0 :                         .freeze_inmem_layer_at(current_lsn, &mut write_guard)
    1769            0 :                         .await;
    1770              : 
    1771            0 :                     if let Err(e) = res {
    1772            0 :                         tracing::info!(
    1773            0 :                             "failed to flush frozen layer after background freeze: {e:#}"
    1774              :                         );
    1775            0 :                     }
    1776              :                 }
    1777              :             }
    1778            0 :         }
    1779            0 :     }
    1780              : 
    1781              :     /// Checks if the internal state of the timeline is consistent with it being able to be offloaded.
    1782              :     ///
    1783              :     /// This is neccessary but not sufficient for offloading of the timeline as it might have
    1784              :     /// child timelines that are not offloaded yet.
    1785            0 :     pub(crate) fn can_offload(&self) -> (bool, &'static str) {
    1786            0 :         if self.remote_client.is_archived() != Some(true) {
    1787            0 :             return (false, "the timeline is not archived");
    1788            0 :         }
    1789            0 :         if !self.remote_client.no_pending_work() {
    1790              :             // if the remote client is still processing some work, we can't offload
    1791            0 :             return (false, "the upload queue is not drained yet");
    1792            0 :         }
    1793            0 : 
    1794            0 :         (true, "ok")
    1795            0 :     }
    1796              : 
    1797              :     /// Outermost timeline compaction operation; downloads needed layers. Returns whether we have pending
    1798              :     /// compaction tasks.
    1799          728 :     pub(crate) async fn compact(
    1800          728 :         self: &Arc<Self>,
    1801          728 :         cancel: &CancellationToken,
    1802          728 :         flags: EnumSet<CompactFlags>,
    1803          728 :         ctx: &RequestContext,
    1804          728 :     ) -> Result<CompactionOutcome, CompactionError> {
    1805          728 :         self.compact_with_options(
    1806          728 :             cancel,
    1807          728 :             CompactOptions {
    1808          728 :                 flags,
    1809          728 :                 compact_key_range: None,
    1810          728 :                 compact_lsn_range: None,
    1811          728 :                 sub_compaction: false,
    1812          728 :                 sub_compaction_max_job_size_mb: None,
    1813          728 :             },
    1814          728 :             ctx,
    1815          728 :         )
    1816          728 :         .await
    1817          728 :     }
    1818              : 
    1819              :     /// Outermost timeline compaction operation; downloads needed layers.
    1820              :     ///
    1821              :     /// NB: the cancellation token is usually from a background task, but can also come from a
    1822              :     /// request task.
    1823          728 :     pub(crate) async fn compact_with_options(
    1824          728 :         self: &Arc<Self>,
    1825          728 :         cancel: &CancellationToken,
    1826          728 :         options: CompactOptions,
    1827          728 :         ctx: &RequestContext,
    1828          728 :     ) -> Result<CompactionOutcome, CompactionError> {
    1829          728 :         // Acquire the compaction lock and task semaphore.
    1830          728 :         //
    1831          728 :         // L0-only compaction uses a separate semaphore (if enabled) to make sure it isn't starved
    1832          728 :         // out by other background tasks (including image compaction). We request this via
    1833          728 :         // `BackgroundLoopKind::L0Compaction`.
    1834          728 :         //
    1835          728 :         // If this is a regular compaction pass, and L0-only compaction is enabled in the config,
    1836          728 :         // then we should yield for immediate L0 compaction if necessary while we're waiting for the
    1837          728 :         // background task semaphore. There's no point yielding otherwise, since we'd just end up
    1838          728 :         // right back here.
    1839          728 :         let is_l0_only = options.flags.contains(CompactFlags::OnlyL0Compaction);
    1840          728 :         let semaphore_kind = match is_l0_only && self.get_compaction_l0_semaphore() {
    1841            0 :             true => BackgroundLoopKind::L0Compaction,
    1842          728 :             false => BackgroundLoopKind::Compaction,
    1843              :         };
    1844          728 :         let yield_for_l0 = !is_l0_only
    1845          728 :             && self.get_compaction_l0_first()
    1846            0 :             && !options.flags.contains(CompactFlags::NoYield);
    1847              : 
    1848          728 :         let acquire = async move {
    1849          728 :             let guard = self.compaction_lock.lock().await;
    1850          728 :             let permit = super::tasks::acquire_concurrency_permit(semaphore_kind, ctx).await;
    1851          728 :             (guard, permit)
    1852          728 :         };
    1853              : 
    1854          728 :         let (_guard, _permit) = tokio::select! {
    1855          728 :             (guard, permit) = acquire => (guard, permit),
    1856          728 :             _ = self.l0_compaction_trigger.notified(), if yield_for_l0 => {
    1857            0 :                 return Ok(CompactionOutcome::YieldForL0);
    1858              :             }
    1859          728 :             _ = self.cancel.cancelled() => return Ok(CompactionOutcome::Skipped),
    1860          728 :             _ = cancel.cancelled() => return Ok(CompactionOutcome::Skipped),
    1861              :         };
    1862              : 
    1863          728 :         let last_record_lsn = self.get_last_record_lsn();
    1864          728 : 
    1865          728 :         // Last record Lsn could be zero in case the timeline was just created
    1866          728 :         if !last_record_lsn.is_valid() {
    1867            0 :             warn!(
    1868            0 :                 "Skipping compaction for potentially just initialized timeline, it has invalid last record lsn: {last_record_lsn}"
    1869              :             );
    1870            0 :             return Ok(CompactionOutcome::Skipped);
    1871          728 :         }
    1872              : 
    1873          728 :         let result = match self.get_compaction_algorithm_settings().kind {
    1874              :             CompactionAlgorithm::Tiered => {
    1875            0 :                 self.compact_tiered(cancel, ctx).await?;
    1876            0 :                 Ok(CompactionOutcome::Done)
    1877              :             }
    1878          728 :             CompactionAlgorithm::Legacy => self.compact_legacy(cancel, options, ctx).await,
    1879              :         };
    1880              : 
    1881              :         // Signal compaction failure to avoid L0 flush stalls when it's broken.
    1882            0 :         match &result {
    1883          728 :             Ok(_) => self.compaction_failed.store(false, AtomicOrdering::Relaxed),
    1884            0 :             Err(e) if e.is_cancel() => {}
    1885            0 :             Err(CompactionError::ShuttingDown) => {
    1886            0 :                 // Covered by the `Err(e) if e.is_cancel()` branch.
    1887            0 :             }
    1888            0 :             Err(CompactionError::AlreadyRunning(_)) => {
    1889            0 :                 // Covered by the `Err(e) if e.is_cancel()` branch.
    1890            0 :             }
    1891              :             Err(CompactionError::Other(_)) => {
    1892            0 :                 self.compaction_failed.store(true, AtomicOrdering::Relaxed)
    1893              :             }
    1894              :             Err(CompactionError::CollectKeySpaceError(_)) => {
    1895              :                 // Cancelled errors are covered by the `Err(e) if e.is_cancel()` branch.
    1896            0 :                 self.compaction_failed.store(true, AtomicOrdering::Relaxed)
    1897              :             }
    1898              :             // Don't change the current value on offload failure or shutdown. We don't want to
    1899              :             // abruptly stall nor resume L0 flushes in these cases.
    1900            0 :             Err(CompactionError::Offload(_)) => {}
    1901              :         };
    1902              : 
    1903          728 :         result
    1904          728 :     }
    1905              : 
    1906              :     /// Mutate the timeline with a [`TimelineWriter`].
    1907     10266392 :     pub(crate) async fn writer(&self) -> TimelineWriter<'_> {
    1908     10266392 :         TimelineWriter {
    1909     10266392 :             tl: self,
    1910     10266392 :             write_guard: self.write_lock.lock().await,
    1911              :         }
    1912     10266392 :     }
    1913              : 
    1914            0 :     pub(crate) fn activate(
    1915            0 :         self: &Arc<Self>,
    1916            0 :         parent: Arc<crate::tenant::Tenant>,
    1917            0 :         broker_client: BrokerClientChannel,
    1918            0 :         background_jobs_can_start: Option<&completion::Barrier>,
    1919            0 :         ctx: &RequestContext,
    1920            0 :     ) {
    1921            0 :         if self.tenant_shard_id.is_shard_zero() {
    1922            0 :             // Logical size is only maintained accurately on shard zero.
    1923            0 :             self.spawn_initial_logical_size_computation_task(ctx);
    1924            0 :         }
    1925            0 :         self.launch_wal_receiver(ctx, broker_client);
    1926            0 :         self.set_state(TimelineState::Active);
    1927            0 :         self.launch_eviction_task(parent, background_jobs_can_start);
    1928            0 :     }
    1929              : 
    1930              :     /// After this function returns, there are no timeline-scoped tasks are left running.
    1931              :     ///
    1932              :     /// The preferred pattern for is:
    1933              :     /// - in any spawned tasks, keep Timeline::guard open + Timeline::cancel / child token
    1934              :     /// - if early shutdown (not just cancellation) of a sub-tree of tasks is required,
    1935              :     ///   go the extra mile and keep track of JoinHandles
    1936              :     /// - Keep track of JoinHandles using a passed-down `Arc<Mutex<Option<JoinSet>>>` or similar,
    1937              :     ///   instead of spawning directly on a runtime. It is a more composable / testable pattern.
    1938              :     ///
    1939              :     /// For legacy reasons, we still have multiple tasks spawned using
    1940              :     /// `task_mgr::spawn(X, Some(tenant_id), Some(timeline_id))`.
    1941              :     /// We refer to these as "timeline-scoped task_mgr tasks".
    1942              :     /// Some of these tasks are already sensitive to Timeline::cancel while others are
    1943              :     /// not sensitive to Timeline::cancel and instead respect [`task_mgr::shutdown_token`]
    1944              :     /// or [`task_mgr::shutdown_watcher`].
    1945              :     /// We want to gradually convert the code base away from these.
    1946              :     ///
    1947              :     /// Here is an inventory of timeline-scoped task_mgr tasks that are still sensitive to
    1948              :     /// `task_mgr::shutdown_{token,watcher}` (there are also tenant-scoped and global-scoped
    1949              :     /// ones that aren't mentioned here):
    1950              :     /// - [`TaskKind::TimelineDeletionWorker`]
    1951              :     ///    - NB: also used for tenant deletion
    1952              :     /// - [`TaskKind::RemoteUploadTask`]`
    1953              :     /// - [`TaskKind::InitialLogicalSizeCalculation`]
    1954              :     /// - [`TaskKind::DownloadAllRemoteLayers`] (can we get rid of it?)
    1955              :     // Inventory of timeline-scoped task_mgr tasks that use spawn but aren't sensitive:
    1956              :     /// - [`TaskKind::Eviction`]
    1957              :     /// - [`TaskKind::LayerFlushTask`]
    1958              :     /// - [`TaskKind::OndemandLogicalSizeCalculation`]
    1959              :     /// - [`TaskKind::GarbageCollector`] (immediate_gc is timeline-scoped)
    1960           20 :     pub(crate) async fn shutdown(&self, mode: ShutdownMode) {
    1961           20 :         debug_assert_current_span_has_tenant_and_timeline_id();
    1962           20 : 
    1963           20 :         // Regardless of whether we're going to try_freeze_and_flush
    1964           20 :         // or not, stop ingesting any more data. Walreceiver only provides
    1965           20 :         // cancellation but no "wait until gone", because it uses the Timeline::gate.
    1966           20 :         // So, only after the self.gate.close() below will we know for sure that
    1967           20 :         // no walreceiver tasks are left.
    1968           20 :         // For `try_freeze_and_flush=true`, this means that we might still be ingesting
    1969           20 :         // data during the call to `self.freeze_and_flush()` below.
    1970           20 :         // That's not ideal, but, we don't have the concept of a ChildGuard,
    1971           20 :         // which is what we'd need to properly model early shutdown of the walreceiver
    1972           20 :         // task sub-tree before the other Timeline task sub-trees.
    1973           20 :         let walreceiver = self.walreceiver.lock().unwrap().take();
    1974           20 :         tracing::debug!(
    1975            0 :             is_some = walreceiver.is_some(),
    1976            0 :             "Waiting for WalReceiverManager..."
    1977              :         );
    1978           20 :         if let Some(walreceiver) = walreceiver {
    1979            0 :             walreceiver.cancel();
    1980           20 :         }
    1981              :         // ... and inform any waiters for newer LSNs that there won't be any.
    1982           20 :         self.last_record_lsn.shutdown();
    1983           20 : 
    1984           20 :         if let ShutdownMode::FreezeAndFlush = mode {
    1985           12 :             let do_flush = if let Some((open, frozen)) = self
    1986           12 :                 .layers
    1987           12 :                 .read()
    1988           12 :                 .await
    1989           12 :                 .layer_map()
    1990           12 :                 .map(|lm| (lm.open_layer.is_some(), lm.frozen_layers.len()))
    1991           12 :                 .ok()
    1992           12 :                 .filter(|(open, frozen)| *open || *frozen > 0)
    1993              :             {
    1994            0 :                 if self.remote_client.is_archived() == Some(true) {
    1995              :                     // No point flushing on shutdown for an archived timeline: it is not important
    1996              :                     // to have it nice and fresh after our restart, and trying to flush here might
    1997              :                     // race with trying to offload it (which also stops the flush loop)
    1998            0 :                     false
    1999              :                 } else {
    2000            0 :                     tracing::info!(?open, frozen, "flushing and freezing on shutdown");
    2001            0 :                     true
    2002              :                 }
    2003              :             } else {
    2004              :                 // this is double-shutdown, it'll be a no-op
    2005           12 :                 true
    2006              :             };
    2007              : 
    2008              :             // we shut down walreceiver above, so, we won't add anything more
    2009              :             // to the InMemoryLayer; freeze it and wait for all frozen layers
    2010              :             // to reach the disk & upload queue, then shut the upload queue and
    2011              :             // wait for it to drain.
    2012           12 :             if do_flush {
    2013           12 :                 match self.freeze_and_flush().await {
    2014              :                     Ok(_) => {
    2015              :                         // drain the upload queue
    2016              :                         // if we did not wait for completion here, it might be our shutdown process
    2017              :                         // didn't wait for remote uploads to complete at all, as new tasks can forever
    2018              :                         // be spawned.
    2019              :                         //
    2020              :                         // what is problematic is the shutting down of RemoteTimelineClient, because
    2021              :                         // obviously it does not make sense to stop while we wait for it, but what
    2022              :                         // about corner cases like s3 suddenly hanging up?
    2023           12 :                         self.remote_client.shutdown().await;
    2024              :                     }
    2025              :                     Err(FlushLayerError::Cancelled) => {
    2026              :                         // this is likely the second shutdown, ignore silently.
    2027              :                         // TODO: this can be removed once https://github.com/neondatabase/neon/issues/5080
    2028            0 :                         debug_assert!(self.cancel.is_cancelled());
    2029              :                     }
    2030            0 :                     Err(e) => {
    2031            0 :                         // Non-fatal.  Shutdown is infallible.  Failures to flush just mean that
    2032            0 :                         // we have some extra WAL replay to do next time the timeline starts.
    2033            0 :                         warn!("failed to freeze and flush: {e:#}");
    2034              :                     }
    2035              :                 }
    2036              : 
    2037              :                 // `self.remote_client.shutdown().await` above should have already flushed everything from the queue, but
    2038              :                 // we also do a final check here to ensure that the queue is empty.
    2039           12 :                 if !self.remote_client.no_pending_work() {
    2040            0 :                     warn!(
    2041            0 :                         "still have pending work in remote upload queue, but continuing shutting down anyways"
    2042              :                     );
    2043           12 :                 }
    2044            0 :             }
    2045            8 :         }
    2046              : 
    2047           20 :         if let ShutdownMode::Reload = mode {
    2048              :             // drain the upload queue
    2049            4 :             self.remote_client.shutdown().await;
    2050            4 :             if !self.remote_client.no_pending_work() {
    2051            0 :                 warn!(
    2052            0 :                     "still have pending work in remote upload queue, but continuing shutting down anyways"
    2053              :                 );
    2054            4 :             }
    2055           16 :         }
    2056              : 
    2057              :         // Signal any subscribers to our cancellation token to drop out
    2058           20 :         tracing::debug!("Cancelling CancellationToken");
    2059           20 :         self.cancel.cancel();
    2060           20 : 
    2061           20 :         // If we have a background task downloading heatmap layers stop it.
    2062           20 :         // The background downloads are sensitive to timeline cancellation (done above),
    2063           20 :         // so the drain will be immediate.
    2064           20 :         self.stop_and_drain_heatmap_layers_download().await;
    2065              : 
    2066              :         // Ensure Prevent new page service requests from starting.
    2067           20 :         self.handles.shutdown();
    2068           20 : 
    2069           20 :         // Transition the remote_client into a state where it's only useful for timeline deletion.
    2070           20 :         // (The deletion use case is why we can't just hook up remote_client to Self::cancel).)
    2071           20 :         self.remote_client.stop();
    2072           20 : 
    2073           20 :         // As documented in remote_client.stop()'s doc comment, it's our responsibility
    2074           20 :         // to shut down the upload queue tasks.
    2075           20 :         // TODO: fix that, task management should be encapsulated inside remote_client.
    2076           20 :         task_mgr::shutdown_tasks(
    2077           20 :             Some(TaskKind::RemoteUploadTask),
    2078           20 :             Some(self.tenant_shard_id),
    2079           20 :             Some(self.timeline_id),
    2080           20 :         )
    2081           20 :         .await;
    2082              : 
    2083              :         // TODO: work toward making this a no-op. See this function's doc comment for more context.
    2084           20 :         tracing::debug!("Waiting for tasks...");
    2085           20 :         task_mgr::shutdown_tasks(None, Some(self.tenant_shard_id), Some(self.timeline_id)).await;
    2086              : 
    2087              :         {
    2088              :             // Allow any remaining in-memory layers to do cleanup -- until that, they hold the gate
    2089              :             // open.
    2090           20 :             let mut write_guard = self.write_lock.lock().await;
    2091           20 :             self.layers.write().await.shutdown(&mut write_guard);
    2092           20 :         }
    2093           20 : 
    2094           20 :         // Finally wait until any gate-holders are complete.
    2095           20 :         //
    2096           20 :         // TODO: once above shutdown_tasks is a no-op, we can close the gate before calling shutdown_tasks
    2097           20 :         // and use a TBD variant of shutdown_tasks that asserts that there were no tasks left.
    2098           20 :         self.gate.close().await;
    2099              : 
    2100           20 :         self.metrics.shutdown();
    2101           20 :     }
    2102              : 
    2103          908 :     pub(crate) fn set_state(&self, new_state: TimelineState) {
    2104          908 :         match (self.current_state(), new_state) {
    2105          908 :             (equal_state_1, equal_state_2) if equal_state_1 == equal_state_2 => {
    2106            4 :                 info!("Ignoring new state, equal to the existing one: {equal_state_2:?}");
    2107              :             }
    2108            0 :             (st, TimelineState::Loading) => {
    2109            0 :                 error!("ignoring transition from {st:?} into Loading state");
    2110              :             }
    2111            0 :             (TimelineState::Broken { .. }, new_state) => {
    2112            0 :                 error!("Ignoring state update {new_state:?} for broken timeline");
    2113              :             }
    2114              :             (TimelineState::Stopping, TimelineState::Active) => {
    2115            0 :                 error!("Not activating a Stopping timeline");
    2116              :             }
    2117          904 :             (_, new_state) => {
    2118          904 :                 self.state.send_replace(new_state);
    2119          904 :             }
    2120              :         }
    2121          908 :     }
    2122              : 
    2123            4 :     pub(crate) fn set_broken(&self, reason: String) {
    2124            4 :         let backtrace_str: String = format!("{}", std::backtrace::Backtrace::force_capture());
    2125            4 :         let broken_state = TimelineState::Broken {
    2126            4 :             reason,
    2127            4 :             backtrace: backtrace_str,
    2128            4 :         };
    2129            4 :         self.set_state(broken_state);
    2130            4 : 
    2131            4 :         // Although the Broken state is not equivalent to shutdown() (shutdown will be called
    2132            4 :         // later when this tenant is detach or the process shuts down), firing the cancellation token
    2133            4 :         // here avoids the need for other tasks to watch for the Broken state explicitly.
    2134            4 :         self.cancel.cancel();
    2135            4 :     }
    2136              : 
    2137       450445 :     pub(crate) fn current_state(&self) -> TimelineState {
    2138       450445 :         self.state.borrow().clone()
    2139       450445 :     }
    2140              : 
    2141           12 :     pub(crate) fn is_broken(&self) -> bool {
    2142           12 :         matches!(&*self.state.borrow(), TimelineState::Broken { .. })
    2143           12 :     }
    2144              : 
    2145          472 :     pub(crate) fn is_active(&self) -> bool {
    2146          472 :         self.current_state() == TimelineState::Active
    2147          472 :     }
    2148              : 
    2149            0 :     pub(crate) fn is_archived(&self) -> Option<bool> {
    2150            0 :         self.remote_client.is_archived()
    2151            0 :     }
    2152              : 
    2153          736 :     pub(crate) fn is_stopping(&self) -> bool {
    2154          736 :         self.current_state() == TimelineState::Stopping
    2155          736 :     }
    2156              : 
    2157            0 :     pub(crate) fn subscribe_for_state_updates(&self) -> watch::Receiver<TimelineState> {
    2158            0 :         self.state.subscribe()
    2159            0 :     }
    2160              : 
    2161       448333 :     pub(crate) async fn wait_to_become_active(
    2162       448333 :         &self,
    2163       448333 :         _ctx: &RequestContext, // Prepare for use by cancellation
    2164       448333 :     ) -> Result<(), TimelineState> {
    2165       448333 :         let mut receiver = self.state.subscribe();
    2166              :         loop {
    2167       448333 :             let current_state = receiver.borrow().clone();
    2168       448333 :             match current_state {
    2169              :                 TimelineState::Loading => {
    2170            0 :                     receiver
    2171            0 :                         .changed()
    2172            0 :                         .await
    2173            0 :                         .expect("holding a reference to self");
    2174              :                 }
    2175              :                 TimelineState::Active { .. } => {
    2176       448329 :                     return Ok(());
    2177              :                 }
    2178              :                 TimelineState::Broken { .. } | TimelineState::Stopping => {
    2179              :                     // There's no chance the timeline can transition back into ::Active
    2180            4 :                     return Err(current_state);
    2181              :                 }
    2182              :             }
    2183              :         }
    2184       448333 :     }
    2185              : 
    2186            0 :     pub(crate) async fn layer_map_info(
    2187            0 :         &self,
    2188            0 :         reset: LayerAccessStatsReset,
    2189            0 :     ) -> Result<LayerMapInfo, layer_manager::Shutdown> {
    2190            0 :         let guard = self.layers.read().await;
    2191            0 :         let layer_map = guard.layer_map()?;
    2192            0 :         let mut in_memory_layers = Vec::with_capacity(layer_map.frozen_layers.len() + 1);
    2193            0 :         if let Some(open_layer) = &layer_map.open_layer {
    2194            0 :             in_memory_layers.push(open_layer.info());
    2195            0 :         }
    2196            0 :         for frozen_layer in &layer_map.frozen_layers {
    2197            0 :             in_memory_layers.push(frozen_layer.info());
    2198            0 :         }
    2199              : 
    2200            0 :         let historic_layers = layer_map
    2201            0 :             .iter_historic_layers()
    2202            0 :             .map(|desc| guard.get_from_desc(&desc).info(reset))
    2203            0 :             .collect();
    2204            0 : 
    2205            0 :         Ok(LayerMapInfo {
    2206            0 :             in_memory_layers,
    2207            0 :             historic_layers,
    2208            0 :         })
    2209            0 :     }
    2210              : 
    2211              :     #[instrument(skip_all, fields(tenant_id = %self.tenant_shard_id.tenant_id, shard_id = %self.tenant_shard_id.shard_slug(), timeline_id = %self.timeline_id))]
    2212              :     pub(crate) async fn download_layer(
    2213              :         &self,
    2214              :         layer_file_name: &LayerName,
    2215              :         ctx: &RequestContext,
    2216              :     ) -> Result<Option<bool>, super::storage_layer::layer::DownloadError> {
    2217              :         let Some(layer) = self
    2218              :             .find_layer(layer_file_name)
    2219              :             .await
    2220            0 :             .map_err(|e| match e {
    2221            0 :                 layer_manager::Shutdown => {
    2222            0 :                     super::storage_layer::layer::DownloadError::TimelineShutdown
    2223            0 :                 }
    2224            0 :             })?
    2225              :         else {
    2226              :             return Ok(None);
    2227              :         };
    2228              : 
    2229              :         layer.download(ctx).await?;
    2230              : 
    2231              :         Ok(Some(true))
    2232              :     }
    2233              : 
    2234              :     /// Evict just one layer.
    2235              :     ///
    2236              :     /// Returns `Ok(None)` in the case where the layer could not be found by its `layer_file_name`.
    2237            0 :     pub(crate) async fn evict_layer(
    2238            0 :         &self,
    2239            0 :         layer_file_name: &LayerName,
    2240            0 :     ) -> anyhow::Result<Option<bool>> {
    2241            0 :         let _gate = self
    2242            0 :             .gate
    2243            0 :             .enter()
    2244            0 :             .map_err(|_| anyhow::anyhow!("Shutting down"))?;
    2245              : 
    2246            0 :         let Some(local_layer) = self.find_layer(layer_file_name).await? else {
    2247            0 :             return Ok(None);
    2248              :         };
    2249              : 
    2250              :         // curl has this by default
    2251            0 :         let timeout = std::time::Duration::from_secs(120);
    2252            0 : 
    2253            0 :         match local_layer.evict_and_wait(timeout).await {
    2254            0 :             Ok(()) => Ok(Some(true)),
    2255            0 :             Err(EvictionError::NotFound) => Ok(Some(false)),
    2256            0 :             Err(EvictionError::Downloaded) => Ok(Some(false)),
    2257            0 :             Err(EvictionError::Timeout) => Ok(Some(false)),
    2258              :         }
    2259            0 :     }
    2260              : 
    2261      9606020 :     fn should_roll(
    2262      9606020 :         &self,
    2263      9606020 :         layer_size: u64,
    2264      9606020 :         projected_layer_size: u64,
    2265      9606020 :         checkpoint_distance: u64,
    2266      9606020 :         projected_lsn: Lsn,
    2267      9606020 :         last_freeze_at: Lsn,
    2268      9606020 :         opened_at: Instant,
    2269      9606020 :     ) -> bool {
    2270      9606020 :         let distance = projected_lsn.widening_sub(last_freeze_at);
    2271      9606020 : 
    2272      9606020 :         // Rolling the open layer can be triggered by:
    2273      9606020 :         // 1. The distance from the last LSN we rolled at. This bounds the amount of WAL that
    2274      9606020 :         //    the safekeepers need to store.  For sharded tenants, we multiply by shard count to
    2275      9606020 :         //    account for how writes are distributed across shards: we expect each node to consume
    2276      9606020 :         //    1/count of the LSN on average.
    2277      9606020 :         // 2. The size of the currently open layer.
    2278      9606020 :         // 3. The time since the last roll. It helps safekeepers to regard pageserver as caught
    2279      9606020 :         //    up and suspend activity.
    2280      9606020 :         if distance >= checkpoint_distance as i128 * self.shard_identity.count.count() as i128 {
    2281            0 :             info!(
    2282            0 :                 "Will roll layer at {} with layer size {} due to LSN distance ({})",
    2283              :                 projected_lsn, layer_size, distance
    2284              :             );
    2285              : 
    2286            0 :             true
    2287      9606020 :         } else if projected_layer_size >= checkpoint_distance {
    2288              :             // NB: this check is relied upon by:
    2289          160 :             let _ = IndexEntry::validate_checkpoint_distance;
    2290          160 :             info!(
    2291            0 :                 "Will roll layer at {} with layer size {} due to layer size ({})",
    2292              :                 projected_lsn, layer_size, projected_layer_size
    2293              :             );
    2294              : 
    2295          160 :             true
    2296      9605860 :         } else if distance > 0 && opened_at.elapsed() >= self.get_checkpoint_timeout() {
    2297            0 :             info!(
    2298            0 :                 "Will roll layer at {} with layer size {} due to time since first write to the layer ({:?})",
    2299            0 :                 projected_lsn,
    2300            0 :                 layer_size,
    2301            0 :                 opened_at.elapsed()
    2302              :             );
    2303              : 
    2304            0 :             true
    2305              :         } else {
    2306      9605860 :             false
    2307              :         }
    2308      9606020 :     }
    2309              : }
    2310              : 
    2311              : /// Number of times we will compute partition within a checkpoint distance.
    2312              : const REPARTITION_FREQ_IN_CHECKPOINT_DISTANCE: u64 = 10;
    2313              : 
    2314              : // Private functions
    2315              : impl Timeline {
    2316           24 :     pub(crate) fn get_lsn_lease_length(&self) -> Duration {
    2317           24 :         let tenant_conf = self.tenant_conf.load();
    2318           24 :         tenant_conf
    2319           24 :             .tenant_conf
    2320           24 :             .lsn_lease_length
    2321           24 :             .unwrap_or(self.conf.default_tenant_conf.lsn_lease_length)
    2322           24 :     }
    2323              : 
    2324            0 :     pub(crate) fn get_lsn_lease_length_for_ts(&self) -> Duration {
    2325            0 :         let tenant_conf = self.tenant_conf.load();
    2326            0 :         tenant_conf
    2327            0 :             .tenant_conf
    2328            0 :             .lsn_lease_length_for_ts
    2329            0 :             .unwrap_or(self.conf.default_tenant_conf.lsn_lease_length_for_ts)
    2330            0 :     }
    2331              : 
    2332            0 :     pub(crate) fn is_gc_blocked_by_lsn_lease_deadline(&self) -> bool {
    2333            0 :         let tenant_conf = self.tenant_conf.load();
    2334            0 :         tenant_conf.is_gc_blocked_by_lsn_lease_deadline()
    2335            0 :     }
    2336              : 
    2337            0 :     pub(crate) fn get_lazy_slru_download(&self) -> bool {
    2338            0 :         let tenant_conf = self.tenant_conf.load();
    2339            0 :         tenant_conf
    2340            0 :             .tenant_conf
    2341            0 :             .lazy_slru_download
    2342            0 :             .unwrap_or(self.conf.default_tenant_conf.lazy_slru_download)
    2343            0 :     }
    2344              : 
    2345      9609016 :     fn get_checkpoint_distance(&self) -> u64 {
    2346      9609016 :         let tenant_conf = self.tenant_conf.load();
    2347      9609016 :         tenant_conf
    2348      9609016 :             .tenant_conf
    2349      9609016 :             .checkpoint_distance
    2350      9609016 :             .unwrap_or(self.conf.default_tenant_conf.checkpoint_distance)
    2351      9609016 :     }
    2352              : 
    2353      9605860 :     fn get_checkpoint_timeout(&self) -> Duration {
    2354      9605860 :         let tenant_conf = self.tenant_conf.load();
    2355      9605860 :         tenant_conf
    2356      9605860 :             .tenant_conf
    2357      9605860 :             .checkpoint_timeout
    2358      9605860 :             .unwrap_or(self.conf.default_tenant_conf.checkpoint_timeout)
    2359      9605860 :     }
    2360              : 
    2361         5032 :     fn get_compaction_period(&self) -> Duration {
    2362         5032 :         let tenant_conf = self.tenant_conf.load().tenant_conf.clone();
    2363         5032 :         tenant_conf
    2364         5032 :             .compaction_period
    2365         5032 :             .unwrap_or(self.conf.default_tenant_conf.compaction_period)
    2366         5032 :     }
    2367              : 
    2368         1336 :     fn get_compaction_target_size(&self) -> u64 {
    2369         1336 :         let tenant_conf = self.tenant_conf.load();
    2370         1336 :         tenant_conf
    2371         1336 :             .tenant_conf
    2372         1336 :             .compaction_target_size
    2373         1336 :             .unwrap_or(self.conf.default_tenant_conf.compaction_target_size)
    2374         1336 :     }
    2375              : 
    2376         3140 :     fn get_compaction_threshold(&self) -> usize {
    2377         3140 :         let tenant_conf = self.tenant_conf.load();
    2378         3140 :         tenant_conf
    2379         3140 :             .tenant_conf
    2380         3140 :             .compaction_threshold
    2381         3140 :             .unwrap_or(self.conf.default_tenant_conf.compaction_threshold)
    2382         3140 :     }
    2383              : 
    2384              :     /// Returns `true` if the rel_size_v2 config is enabled. NOTE: the write path and read path
    2385              :     /// should look at `get_rel_size_v2_status()` to get the actual status of the timeline. It is
    2386              :     /// possible that the index part persists the state while the config doesn't get persisted.
    2387         3892 :     pub(crate) fn get_rel_size_v2_enabled(&self) -> bool {
    2388         3892 :         let tenant_conf = self.tenant_conf.load();
    2389         3892 :         tenant_conf
    2390         3892 :             .tenant_conf
    2391         3892 :             .rel_size_v2_enabled
    2392         3892 :             .unwrap_or(self.conf.default_tenant_conf.rel_size_v2_enabled)
    2393         3892 :     }
    2394              : 
    2395         4384 :     pub(crate) fn get_rel_size_v2_status(&self) -> RelSizeMigration {
    2396         4384 :         self.rel_size_v2_status
    2397         4384 :             .load()
    2398         4384 :             .as_ref()
    2399         4384 :             .map(|s| s.as_ref().clone())
    2400         4384 :             .unwrap_or(RelSizeMigration::Legacy)
    2401         4384 :     }
    2402              : 
    2403           56 :     fn get_compaction_upper_limit(&self) -> usize {
    2404           56 :         let tenant_conf = self.tenant_conf.load();
    2405           56 :         tenant_conf
    2406           56 :             .tenant_conf
    2407           56 :             .compaction_upper_limit
    2408           56 :             .unwrap_or(self.conf.default_tenant_conf.compaction_upper_limit)
    2409           56 :     }
    2410              : 
    2411          728 :     pub fn get_compaction_l0_first(&self) -> bool {
    2412          728 :         let tenant_conf = self.tenant_conf.load().tenant_conf.clone();
    2413          728 :         tenant_conf
    2414          728 :             .compaction_l0_first
    2415          728 :             .unwrap_or(self.conf.default_tenant_conf.compaction_l0_first)
    2416          728 :     }
    2417              : 
    2418            0 :     pub fn get_compaction_l0_semaphore(&self) -> bool {
    2419            0 :         let tenant_conf = self.tenant_conf.load().tenant_conf.clone();
    2420            0 :         tenant_conf
    2421            0 :             .compaction_l0_semaphore
    2422            0 :             .unwrap_or(self.conf.default_tenant_conf.compaction_l0_semaphore)
    2423            0 :     }
    2424              : 
    2425         2516 :     fn get_l0_flush_delay_threshold(&self) -> Option<usize> {
    2426              :         // Disable L0 flushes by default. This and compaction needs further tuning.
    2427              :         const DEFAULT_L0_FLUSH_DELAY_FACTOR: usize = 0; // TODO: default to e.g. 3
    2428              : 
    2429              :         // If compaction is disabled, don't delay.
    2430         2516 :         if self.get_compaction_period() == Duration::ZERO {
    2431         2516 :             return None;
    2432            0 :         }
    2433            0 : 
    2434            0 :         let compaction_threshold = self.get_compaction_threshold();
    2435            0 :         let tenant_conf = self.tenant_conf.load();
    2436            0 :         let l0_flush_delay_threshold = tenant_conf
    2437            0 :             .tenant_conf
    2438            0 :             .l0_flush_delay_threshold
    2439            0 :             .or(self.conf.default_tenant_conf.l0_flush_delay_threshold)
    2440            0 :             .unwrap_or(DEFAULT_L0_FLUSH_DELAY_FACTOR * compaction_threshold);
    2441            0 : 
    2442            0 :         // 0 disables backpressure.
    2443            0 :         if l0_flush_delay_threshold == 0 {
    2444            0 :             return None;
    2445            0 :         }
    2446            0 : 
    2447            0 :         // Clamp the flush delay threshold to the compaction threshold; it doesn't make sense to
    2448            0 :         // backpressure flushes below this.
    2449            0 :         // TODO: the tenant config should have validation to prevent this instead.
    2450            0 :         debug_assert!(l0_flush_delay_threshold >= compaction_threshold);
    2451            0 :         Some(max(l0_flush_delay_threshold, compaction_threshold))
    2452         2516 :     }
    2453              : 
    2454         2516 :     fn get_l0_flush_stall_threshold(&self) -> Option<usize> {
    2455              :         // Disable L0 stalls by default. In ingest benchmarks, we see image compaction take >10
    2456              :         // minutes, blocking L0 compaction, and we can't stall L0 flushes for that long.
    2457              :         const DEFAULT_L0_FLUSH_STALL_FACTOR: usize = 0; // TODO: default to e.g. 5
    2458              : 
    2459              :         // If compaction is disabled, don't stall.
    2460         2516 :         if self.get_compaction_period() == Duration::ZERO {
    2461         2516 :             return None;
    2462            0 :         }
    2463            0 : 
    2464            0 :         // If compaction is failing, don't stall and try to keep the tenant alive. This may not be a
    2465            0 :         // good idea: read amp can grow unbounded, leading to terrible performance, and we may take
    2466            0 :         // on unbounded compaction debt that can take a long time to fix once compaction comes back
    2467            0 :         // online. At least we'll delay flushes, slowing down the growth and buying some time.
    2468            0 :         if self.compaction_failed.load(AtomicOrdering::Relaxed) {
    2469            0 :             return None;
    2470            0 :         }
    2471            0 : 
    2472            0 :         let compaction_threshold = self.get_compaction_threshold();
    2473            0 :         let tenant_conf = self.tenant_conf.load();
    2474            0 :         let l0_flush_stall_threshold = tenant_conf
    2475            0 :             .tenant_conf
    2476            0 :             .l0_flush_stall_threshold
    2477            0 :             .or(self.conf.default_tenant_conf.l0_flush_stall_threshold);
    2478            0 : 
    2479            0 :         // Tests sometimes set compaction_threshold=1 to generate lots of layer files, and don't
    2480            0 :         // handle the 20-second compaction delay. Some (e.g. `test_backward_compatibility`) can't
    2481            0 :         // easily adjust the L0 backpressure settings, so just disable stalls in this case.
    2482            0 :         if cfg!(feature = "testing")
    2483            0 :             && compaction_threshold == 1
    2484            0 :             && l0_flush_stall_threshold.is_none()
    2485              :         {
    2486            0 :             return None;
    2487            0 :         }
    2488            0 : 
    2489            0 :         let l0_flush_stall_threshold = l0_flush_stall_threshold
    2490            0 :             .unwrap_or(DEFAULT_L0_FLUSH_STALL_FACTOR * compaction_threshold);
    2491            0 : 
    2492            0 :         // 0 disables backpressure.
    2493            0 :         if l0_flush_stall_threshold == 0 {
    2494            0 :             return None;
    2495            0 :         }
    2496            0 : 
    2497            0 :         // Clamp the flush stall threshold to the compaction threshold; it doesn't make sense to
    2498            0 :         // backpressure flushes below this.
    2499            0 :         // TODO: the tenant config should have validation to prevent this instead.
    2500            0 :         debug_assert!(l0_flush_stall_threshold >= compaction_threshold);
    2501            0 :         Some(max(l0_flush_stall_threshold, compaction_threshold))
    2502         2516 :     }
    2503              : 
    2504         2356 :     fn get_l0_flush_wait_upload(&self) -> bool {
    2505         2356 :         let tenant_conf = self.tenant_conf.load();
    2506         2356 :         tenant_conf
    2507         2356 :             .tenant_conf
    2508         2356 :             .l0_flush_wait_upload
    2509         2356 :             .unwrap_or(self.conf.default_tenant_conf.l0_flush_wait_upload)
    2510         2356 :     }
    2511              : 
    2512           28 :     fn get_image_creation_threshold(&self) -> usize {
    2513           28 :         let tenant_conf = self.tenant_conf.load();
    2514           28 :         tenant_conf
    2515           28 :             .tenant_conf
    2516           28 :             .image_creation_threshold
    2517           28 :             .unwrap_or(self.conf.default_tenant_conf.image_creation_threshold)
    2518           28 :     }
    2519              : 
    2520          728 :     fn get_compaction_algorithm_settings(&self) -> CompactionAlgorithmSettings {
    2521          728 :         let tenant_conf = &self.tenant_conf.load();
    2522          728 :         tenant_conf
    2523          728 :             .tenant_conf
    2524          728 :             .compaction_algorithm
    2525          728 :             .as_ref()
    2526          728 :             .unwrap_or(&self.conf.default_tenant_conf.compaction_algorithm)
    2527          728 :             .clone()
    2528          728 :     }
    2529              : 
    2530            0 :     fn get_eviction_policy(&self) -> EvictionPolicy {
    2531            0 :         let tenant_conf = self.tenant_conf.load();
    2532            0 :         tenant_conf
    2533            0 :             .tenant_conf
    2534            0 :             .eviction_policy
    2535            0 :             .unwrap_or(self.conf.default_tenant_conf.eviction_policy)
    2536            0 :     }
    2537              : 
    2538          904 :     fn get_evictions_low_residence_duration_metric_threshold(
    2539          904 :         tenant_conf: &TenantConfOpt,
    2540          904 :         default_tenant_conf: &TenantConf,
    2541          904 :     ) -> Duration {
    2542          904 :         tenant_conf
    2543          904 :             .evictions_low_residence_duration_metric_threshold
    2544          904 :             .unwrap_or(default_tenant_conf.evictions_low_residence_duration_metric_threshold)
    2545          904 :     }
    2546              : 
    2547         1148 :     fn get_image_layer_creation_check_threshold(&self) -> u8 {
    2548         1148 :         let tenant_conf = self.tenant_conf.load();
    2549         1148 :         tenant_conf
    2550         1148 :             .tenant_conf
    2551         1148 :             .image_layer_creation_check_threshold
    2552         1148 :             .unwrap_or(
    2553         1148 :                 self.conf
    2554         1148 :                     .default_tenant_conf
    2555         1148 :                     .image_layer_creation_check_threshold,
    2556         1148 :             )
    2557         1148 :     }
    2558              : 
    2559            0 :     fn get_gc_compaction_settings(&self) -> GcCompactionCombinedSettings {
    2560            0 :         let tenant_conf = &self.tenant_conf.load();
    2561            0 :         let gc_compaction_enabled = tenant_conf
    2562            0 :             .tenant_conf
    2563            0 :             .gc_compaction_enabled
    2564            0 :             .unwrap_or(self.conf.default_tenant_conf.gc_compaction_enabled);
    2565            0 :         let gc_compaction_initial_threshold_kb = tenant_conf
    2566            0 :             .tenant_conf
    2567            0 :             .gc_compaction_initial_threshold_kb
    2568            0 :             .unwrap_or(
    2569            0 :                 self.conf
    2570            0 :                     .default_tenant_conf
    2571            0 :                     .gc_compaction_initial_threshold_kb,
    2572            0 :             );
    2573            0 :         let gc_compaction_ratio_percent = tenant_conf
    2574            0 :             .tenant_conf
    2575            0 :             .gc_compaction_ratio_percent
    2576            0 :             .unwrap_or(self.conf.default_tenant_conf.gc_compaction_ratio_percent);
    2577            0 :         GcCompactionCombinedSettings {
    2578            0 :             gc_compaction_enabled,
    2579            0 :             gc_compaction_initial_threshold_kb,
    2580            0 :             gc_compaction_ratio_percent,
    2581            0 :         }
    2582            0 :     }
    2583              : 
    2584            0 :     fn get_image_creation_preempt_threshold(&self) -> usize {
    2585            0 :         let tenant_conf = self.tenant_conf.load();
    2586            0 :         tenant_conf
    2587            0 :             .tenant_conf
    2588            0 :             .image_creation_preempt_threshold
    2589            0 :             .unwrap_or(
    2590            0 :                 self.conf
    2591            0 :                     .default_tenant_conf
    2592            0 :                     .image_creation_preempt_threshold,
    2593            0 :             )
    2594            0 :     }
    2595              : 
    2596              :     /// Resolve the effective WAL receiver protocol to use for this tenant.
    2597              :     ///
    2598              :     /// Priority order is:
    2599              :     /// 1. Tenant config override
    2600              :     /// 2. Default value for tenant config override
    2601              :     /// 3. Pageserver config override
    2602              :     /// 4. Pageserver config default
    2603            0 :     pub fn resolve_wal_receiver_protocol(&self) -> PostgresClientProtocol {
    2604            0 :         let tenant_conf = self.tenant_conf.load().tenant_conf.clone();
    2605            0 :         tenant_conf
    2606            0 :             .wal_receiver_protocol_override
    2607            0 :             .or(self.conf.default_tenant_conf.wal_receiver_protocol_override)
    2608            0 :             .unwrap_or(self.conf.wal_receiver_protocol)
    2609            0 :     }
    2610              : 
    2611            0 :     pub(super) fn tenant_conf_updated(&self, new_conf: &AttachedTenantConf) {
    2612            0 :         // NB: Most tenant conf options are read by background loops, so,
    2613            0 :         // changes will automatically be picked up.
    2614            0 : 
    2615            0 :         // The threshold is embedded in the metric. So, we need to update it.
    2616            0 :         {
    2617            0 :             let new_threshold = Self::get_evictions_low_residence_duration_metric_threshold(
    2618            0 :                 &new_conf.tenant_conf,
    2619            0 :                 &self.conf.default_tenant_conf,
    2620            0 :             );
    2621            0 : 
    2622            0 :             let tenant_id_str = self.tenant_shard_id.tenant_id.to_string();
    2623            0 :             let shard_id_str = format!("{}", self.tenant_shard_id.shard_slug());
    2624            0 : 
    2625            0 :             let timeline_id_str = self.timeline_id.to_string();
    2626            0 : 
    2627            0 :             self.remote_client.update_config(&new_conf.location);
    2628            0 : 
    2629            0 :             self.metrics
    2630            0 :                 .evictions_with_low_residence_duration
    2631            0 :                 .write()
    2632            0 :                 .unwrap()
    2633            0 :                 .change_threshold(
    2634            0 :                     &tenant_id_str,
    2635            0 :                     &shard_id_str,
    2636            0 :                     &timeline_id_str,
    2637            0 :                     new_threshold,
    2638            0 :                 );
    2639            0 :         }
    2640            0 :     }
    2641              : 
    2642              :     /// Open a Timeline handle.
    2643              :     ///
    2644              :     /// Loads the metadata for the timeline into memory, but not the layer map.
    2645              :     #[allow(clippy::too_many_arguments)]
    2646          904 :     pub(super) fn new(
    2647          904 :         conf: &'static PageServerConf,
    2648          904 :         tenant_conf: Arc<ArcSwap<AttachedTenantConf>>,
    2649          904 :         metadata: &TimelineMetadata,
    2650          904 :         previous_heatmap: Option<PreviousHeatmap>,
    2651          904 :         ancestor: Option<Arc<Timeline>>,
    2652          904 :         timeline_id: TimelineId,
    2653          904 :         tenant_shard_id: TenantShardId,
    2654          904 :         generation: Generation,
    2655          904 :         shard_identity: ShardIdentity,
    2656          904 :         walredo_mgr: Option<Arc<super::WalRedoManager>>,
    2657          904 :         resources: TimelineResources,
    2658          904 :         pg_version: u32,
    2659          904 :         state: TimelineState,
    2660          904 :         attach_wal_lag_cooldown: Arc<OnceLock<WalLagCooldown>>,
    2661          904 :         create_idempotency: crate::tenant::CreateTimelineIdempotency,
    2662          904 :         gc_compaction_state: Option<GcCompactionState>,
    2663          904 :         rel_size_v2_status: Option<RelSizeMigration>,
    2664          904 :         cancel: CancellationToken,
    2665          904 :     ) -> Arc<Self> {
    2666          904 :         let disk_consistent_lsn = metadata.disk_consistent_lsn();
    2667          904 :         let (state, _) = watch::channel(state);
    2668          904 : 
    2669          904 :         let (layer_flush_start_tx, _) = tokio::sync::watch::channel((0, disk_consistent_lsn));
    2670          904 :         let (layer_flush_done_tx, _) = tokio::sync::watch::channel((0, Ok(())));
    2671          904 : 
    2672          904 :         let evictions_low_residence_duration_metric_threshold = {
    2673          904 :             let loaded_tenant_conf = tenant_conf.load();
    2674          904 :             Self::get_evictions_low_residence_duration_metric_threshold(
    2675          904 :                 &loaded_tenant_conf.tenant_conf,
    2676          904 :                 &conf.default_tenant_conf,
    2677          904 :             )
    2678              :         };
    2679              : 
    2680          904 :         if let Some(ancestor) = &ancestor {
    2681          460 :             let mut ancestor_gc_info = ancestor.gc_info.write().unwrap();
    2682          460 :             // If we construct an explicit timeline object, it's obviously not offloaded
    2683          460 :             let is_offloaded = MaybeOffloaded::No;
    2684          460 :             ancestor_gc_info.insert_child(timeline_id, metadata.ancestor_lsn(), is_offloaded);
    2685          460 :         }
    2686              : 
    2687          904 :         Arc::new_cyclic(|myself| {
    2688          904 :             let metrics = Arc::new(TimelineMetrics::new(
    2689          904 :                 &tenant_shard_id,
    2690          904 :                 &timeline_id,
    2691          904 :                 crate::metrics::EvictionsWithLowResidenceDurationBuilder::new(
    2692          904 :                     "mtime",
    2693          904 :                     evictions_low_residence_duration_metric_threshold,
    2694          904 :                 ),
    2695          904 :             ));
    2696          904 :             let aux_file_metrics = metrics.aux_file_size_gauge.clone();
    2697              : 
    2698          904 :             let mut result = Timeline {
    2699          904 :                 conf,
    2700          904 :                 tenant_conf,
    2701          904 :                 myself: myself.clone(),
    2702          904 :                 timeline_id,
    2703          904 :                 tenant_shard_id,
    2704          904 :                 generation,
    2705          904 :                 shard_identity,
    2706          904 :                 pg_version,
    2707          904 :                 layers: Default::default(),
    2708          904 :                 gc_compaction_layer_update_lock: tokio::sync::RwLock::new(()),
    2709          904 : 
    2710          904 :                 walredo_mgr,
    2711          904 :                 walreceiver: Mutex::new(None),
    2712          904 : 
    2713          904 :                 remote_client: Arc::new(resources.remote_client),
    2714          904 : 
    2715          904 :                 // initialize in-memory 'last_record_lsn' from 'disk_consistent_lsn'.
    2716          904 :                 last_record_lsn: SeqWait::new(RecordLsn {
    2717          904 :                     last: disk_consistent_lsn,
    2718          904 :                     prev: metadata.prev_record_lsn().unwrap_or(Lsn(0)),
    2719          904 :                 }),
    2720          904 :                 disk_consistent_lsn: AtomicLsn::new(disk_consistent_lsn.0),
    2721          904 : 
    2722          904 :                 gc_compaction_state: ArcSwap::new(Arc::new(gc_compaction_state)),
    2723          904 : 
    2724          904 :                 last_freeze_at: AtomicLsn::new(disk_consistent_lsn.0),
    2725          904 :                 last_freeze_ts: RwLock::new(Instant::now()),
    2726          904 : 
    2727          904 :                 loaded_at: (disk_consistent_lsn, SystemTime::now()),
    2728          904 : 
    2729          904 :                 ancestor_timeline: ancestor,
    2730          904 :                 ancestor_lsn: metadata.ancestor_lsn(),
    2731          904 : 
    2732          904 :                 metrics,
    2733          904 : 
    2734          904 :                 query_metrics: crate::metrics::SmgrQueryTimePerTimeline::new(
    2735          904 :                     &tenant_shard_id,
    2736          904 :                     &timeline_id,
    2737          904 :                     resources.pagestream_throttle_metrics,
    2738          904 :                 ),
    2739          904 : 
    2740         7232 :                 directory_metrics: array::from_fn(|_| AtomicU64::new(0)),
    2741         7232 :                 directory_metrics_inited: array::from_fn(|_| AtomicBool::new(false)),
    2742          904 : 
    2743          904 :                 flush_loop_state: Mutex::new(FlushLoopState::NotStarted),
    2744          904 : 
    2745          904 :                 layer_flush_start_tx,
    2746          904 :                 layer_flush_done_tx,
    2747          904 : 
    2748          904 :                 write_lock: tokio::sync::Mutex::new(None),
    2749          904 : 
    2750          904 :                 gc_info: std::sync::RwLock::new(GcInfo::default()),
    2751          904 : 
    2752          904 :                 last_image_layer_creation_status: ArcSwap::new(Arc::new(
    2753          904 :                     LastImageLayerCreationStatus::default(),
    2754          904 :                 )),
    2755          904 : 
    2756          904 :                 applied_gc_cutoff_lsn: Rcu::new(metadata.latest_gc_cutoff_lsn()),
    2757          904 :                 initdb_lsn: metadata.initdb_lsn(),
    2758          904 : 
    2759          904 :                 current_logical_size: if disk_consistent_lsn.is_valid() {
    2760              :                     // we're creating timeline data with some layer files existing locally,
    2761              :                     // need to recalculate timeline's logical size based on data in the layers.
    2762          468 :                     LogicalSize::deferred_initial(disk_consistent_lsn)
    2763              :                 } else {
    2764              :                     // we're creating timeline data without any layers existing locally,
    2765              :                     // initial logical size is 0.
    2766          436 :                     LogicalSize::empty_initial()
    2767              :                 },
    2768              : 
    2769          904 :                 partitioning: GuardArcSwap::new((
    2770          904 :                     (KeyPartitioning::new(), KeyPartitioning::new().into_sparse()),
    2771          904 :                     Lsn(0),
    2772          904 :                 )),
    2773          904 :                 repartition_threshold: 0,
    2774          904 :                 last_image_layer_creation_check_at: AtomicLsn::new(0),
    2775          904 :                 last_image_layer_creation_check_instant: Mutex::new(None),
    2776          904 : 
    2777          904 :                 last_received_wal: Mutex::new(None),
    2778          904 :                 rel_size_cache: RwLock::new(RelSizeCache {
    2779          904 :                     complete_as_of: disk_consistent_lsn,
    2780          904 :                     map: HashMap::new(),
    2781          904 :                 }),
    2782          904 : 
    2783          904 :                 download_all_remote_layers_task_info: RwLock::new(None),
    2784          904 : 
    2785          904 :                 state,
    2786          904 : 
    2787          904 :                 eviction_task_timeline_state: tokio::sync::Mutex::new(
    2788          904 :                     EvictionTaskTimelineState::default(),
    2789          904 :                 ),
    2790          904 :                 delete_progress: TimelineDeleteProgress::default(),
    2791          904 : 
    2792          904 :                 cancel,
    2793          904 :                 gate: Gate::default(),
    2794          904 : 
    2795          904 :                 compaction_lock: tokio::sync::Mutex::default(),
    2796          904 :                 compaction_failed: AtomicBool::default(),
    2797          904 :                 l0_compaction_trigger: resources.l0_compaction_trigger,
    2798          904 :                 gc_lock: tokio::sync::Mutex::default(),
    2799          904 : 
    2800          904 :                 standby_horizon: AtomicLsn::new(0),
    2801          904 : 
    2802          904 :                 pagestream_throttle: resources.pagestream_throttle,
    2803          904 : 
    2804          904 :                 aux_file_size_estimator: AuxFileSizeEstimator::new(aux_file_metrics),
    2805          904 : 
    2806          904 :                 #[cfg(test)]
    2807          904 :                 extra_test_dense_keyspace: ArcSwap::new(Arc::new(KeySpace::default())),
    2808          904 : 
    2809          904 :                 l0_flush_global_state: resources.l0_flush_global_state,
    2810          904 : 
    2811          904 :                 handles: Default::default(),
    2812          904 : 
    2813          904 :                 attach_wal_lag_cooldown,
    2814          904 : 
    2815          904 :                 create_idempotency,
    2816          904 : 
    2817          904 :                 page_trace: Default::default(),
    2818          904 : 
    2819          904 :                 previous_heatmap: ArcSwapOption::from_pointee(previous_heatmap),
    2820          904 : 
    2821          904 :                 heatmap_layers_downloader: Mutex::new(None),
    2822          904 : 
    2823          904 :                 rel_size_v2_status: ArcSwapOption::from_pointee(rel_size_v2_status),
    2824          904 :             };
    2825          904 : 
    2826          904 :             result.repartition_threshold =
    2827          904 :                 result.get_checkpoint_distance() / REPARTITION_FREQ_IN_CHECKPOINT_DISTANCE;
    2828          904 : 
    2829          904 :             result
    2830          904 :                 .metrics
    2831          904 :                 .last_record_lsn_gauge
    2832          904 :                 .set(disk_consistent_lsn.0 as i64);
    2833          904 :             result
    2834          904 :         })
    2835          904 :     }
    2836              : 
    2837         1312 :     pub(super) fn maybe_spawn_flush_loop(self: &Arc<Self>) {
    2838         1312 :         let Ok(guard) = self.gate.enter() else {
    2839            0 :             info!("cannot start flush loop when the timeline gate has already been closed");
    2840            0 :             return;
    2841              :         };
    2842         1312 :         let mut flush_loop_state = self.flush_loop_state.lock().unwrap();
    2843         1312 :         match *flush_loop_state {
    2844          892 :             FlushLoopState::NotStarted => (),
    2845              :             FlushLoopState::Running { .. } => {
    2846          420 :                 info!(
    2847            0 :                     "skipping attempt to start flush_loop twice {}/{}",
    2848            0 :                     self.tenant_shard_id, self.timeline_id
    2849              :                 );
    2850          420 :                 return;
    2851              :             }
    2852              :             FlushLoopState::Exited => {
    2853            0 :                 info!(
    2854            0 :                     "ignoring attempt to restart exited flush_loop {}/{}",
    2855            0 :                     self.tenant_shard_id, self.timeline_id
    2856              :                 );
    2857            0 :                 return;
    2858              :             }
    2859              :         }
    2860              : 
    2861          892 :         let layer_flush_start_rx = self.layer_flush_start_tx.subscribe();
    2862          892 :         let self_clone = Arc::clone(self);
    2863          892 : 
    2864          892 :         debug!("spawning flush loop");
    2865          892 :         *flush_loop_state = FlushLoopState::Running {
    2866          892 :             #[cfg(test)]
    2867          892 :             expect_initdb_optimization: false,
    2868          892 :             #[cfg(test)]
    2869          892 :             initdb_optimization_count: 0,
    2870          892 :         };
    2871          892 :         task_mgr::spawn(
    2872          892 :             task_mgr::BACKGROUND_RUNTIME.handle(),
    2873          892 :             task_mgr::TaskKind::LayerFlushTask,
    2874          892 :             self.tenant_shard_id,
    2875          892 :             Some(self.timeline_id),
    2876          892 :             "layer flush task",
    2877          892 :             async move {
    2878          892 :                 let _guard = guard;
    2879          892 :                 let background_ctx = RequestContext::todo_child(TaskKind::LayerFlushTask, DownloadBehavior::Error).with_scope_timeline(&self_clone);
    2880          892 :                 self_clone.flush_loop(layer_flush_start_rx, &background_ctx).await;
    2881           20 :                 let mut flush_loop_state = self_clone.flush_loop_state.lock().unwrap();
    2882           20 :                 assert!(matches!(*flush_loop_state, FlushLoopState::Running{..}));
    2883           20 :                 *flush_loop_state  = FlushLoopState::Exited;
    2884           20 :                 Ok(())
    2885           20 :             }
    2886          892 :             .instrument(info_span!(parent: None, "layer flush task", tenant_id = %self.tenant_shard_id.tenant_id, shard_id = %self.tenant_shard_id.shard_slug(), timeline_id = %self.timeline_id))
    2887              :         );
    2888         1312 :     }
    2889              : 
    2890            0 :     pub(crate) fn update_gc_compaction_state(
    2891            0 :         &self,
    2892            0 :         gc_compaction_state: GcCompactionState,
    2893            0 :     ) -> anyhow::Result<()> {
    2894            0 :         self.gc_compaction_state
    2895            0 :             .store(Arc::new(Some(gc_compaction_state.clone())));
    2896            0 :         self.remote_client
    2897            0 :             .schedule_index_upload_for_gc_compaction_state_update(gc_compaction_state)
    2898            0 :     }
    2899              : 
    2900            0 :     pub(crate) fn update_rel_size_v2_status(
    2901            0 :         &self,
    2902            0 :         rel_size_v2_status: RelSizeMigration,
    2903            0 :     ) -> anyhow::Result<()> {
    2904            0 :         self.rel_size_v2_status
    2905            0 :             .store(Some(Arc::new(rel_size_v2_status.clone())));
    2906            0 :         self.remote_client
    2907            0 :             .schedule_index_upload_for_rel_size_v2_status_update(rel_size_v2_status)
    2908            0 :     }
    2909              : 
    2910            0 :     pub(crate) fn get_gc_compaction_state(&self) -> Option<GcCompactionState> {
    2911            0 :         self.gc_compaction_state.load_full().as_ref().clone()
    2912            0 :     }
    2913              : 
    2914              :     /// Creates and starts the wal receiver.
    2915              :     ///
    2916              :     /// This function is expected to be called at most once per Timeline's lifecycle
    2917              :     /// when the timeline is activated.
    2918            0 :     fn launch_wal_receiver(
    2919            0 :         self: &Arc<Self>,
    2920            0 :         ctx: &RequestContext,
    2921            0 :         broker_client: BrokerClientChannel,
    2922            0 :     ) {
    2923            0 :         info!(
    2924            0 :             "launching WAL receiver for timeline {} of tenant {}",
    2925            0 :             self.timeline_id, self.tenant_shard_id
    2926              :         );
    2927              : 
    2928            0 :         let tenant_conf = self.tenant_conf.load();
    2929            0 :         let wal_connect_timeout = tenant_conf
    2930            0 :             .tenant_conf
    2931            0 :             .walreceiver_connect_timeout
    2932            0 :             .unwrap_or(self.conf.default_tenant_conf.walreceiver_connect_timeout);
    2933            0 :         let lagging_wal_timeout = tenant_conf
    2934            0 :             .tenant_conf
    2935            0 :             .lagging_wal_timeout
    2936            0 :             .unwrap_or(self.conf.default_tenant_conf.lagging_wal_timeout);
    2937            0 :         let max_lsn_wal_lag = tenant_conf
    2938            0 :             .tenant_conf
    2939            0 :             .max_lsn_wal_lag
    2940            0 :             .unwrap_or(self.conf.default_tenant_conf.max_lsn_wal_lag);
    2941            0 : 
    2942            0 :         let mut guard = self.walreceiver.lock().unwrap();
    2943            0 :         assert!(
    2944            0 :             guard.is_none(),
    2945            0 :             "multiple launches / re-launches of WAL receiver are not supported"
    2946              :         );
    2947            0 :         *guard = Some(WalReceiver::start(
    2948            0 :             Arc::clone(self),
    2949            0 :             WalReceiverConf {
    2950            0 :                 protocol: self.resolve_wal_receiver_protocol(),
    2951            0 :                 wal_connect_timeout,
    2952            0 :                 lagging_wal_timeout,
    2953            0 :                 max_lsn_wal_lag,
    2954            0 :                 auth_token: crate::config::SAFEKEEPER_AUTH_TOKEN.get().cloned(),
    2955            0 :                 availability_zone: self.conf.availability_zone.clone(),
    2956            0 :                 ingest_batch_size: self.conf.ingest_batch_size,
    2957            0 :                 validate_wal_contiguity: self.conf.validate_wal_contiguity,
    2958            0 :             },
    2959            0 :             broker_client,
    2960            0 :             ctx,
    2961            0 :         ));
    2962            0 :     }
    2963              : 
    2964              :     /// Initialize with an empty layer map. Used when creating a new timeline.
    2965          892 :     pub(super) fn init_empty_layer_map(&self, start_lsn: Lsn) {
    2966          892 :         let mut layers = self.layers.try_write().expect(
    2967          892 :             "in the context where we call this function, no other task has access to the object",
    2968          892 :         );
    2969          892 :         layers
    2970          892 :             .open_mut()
    2971          892 :             .expect("in this context the LayerManager must still be open")
    2972          892 :             .initialize_empty(Lsn(start_lsn.0));
    2973          892 :     }
    2974              : 
    2975              :     /// Scan the timeline directory, cleanup, populate the layer map, and schedule uploads for local-only
    2976              :     /// files.
    2977           12 :     pub(super) async fn load_layer_map(
    2978           12 :         &self,
    2979           12 :         disk_consistent_lsn: Lsn,
    2980           12 :         index_part: IndexPart,
    2981           12 :     ) -> anyhow::Result<()> {
    2982              :         use LayerName::*;
    2983              :         use init::Decision::*;
    2984              :         use init::{Discovered, DismissedLayer};
    2985              : 
    2986           12 :         let mut guard = self.layers.write().await;
    2987              : 
    2988           12 :         let timer = self.metrics.load_layer_map_histo.start_timer();
    2989           12 : 
    2990           12 :         // Scan timeline directory and create ImageLayerName and DeltaFilename
    2991           12 :         // structs representing all files on disk
    2992           12 :         let timeline_path = self
    2993           12 :             .conf
    2994           12 :             .timeline_path(&self.tenant_shard_id, &self.timeline_id);
    2995           12 :         let conf = self.conf;
    2996           12 :         let span = tracing::Span::current();
    2997           12 : 
    2998           12 :         // Copy to move into the task we're about to spawn
    2999           12 :         let this = self.myself.upgrade().expect("&self method holds the arc");
    3000              : 
    3001           12 :         let (loaded_layers, needs_cleanup, total_physical_size) = tokio::task::spawn_blocking({
    3002           12 :             move || {
    3003           12 :                 let _g = span.entered();
    3004           12 :                 let discovered = init::scan_timeline_dir(&timeline_path)?;
    3005           12 :                 let mut discovered_layers = Vec::with_capacity(discovered.len());
    3006           12 :                 let mut unrecognized_files = Vec::new();
    3007           12 : 
    3008           12 :                 let mut path = timeline_path;
    3009              : 
    3010           44 :                 for discovered in discovered {
    3011           32 :                     let (name, kind) = match discovered {
    3012           32 :                         Discovered::Layer(layer_file_name, local_metadata) => {
    3013           32 :                             discovered_layers.push((layer_file_name, local_metadata));
    3014           32 :                             continue;
    3015              :                         }
    3016            0 :                         Discovered::IgnoredBackup(path) => {
    3017            0 :                             std::fs::remove_file(path)
    3018            0 :                                 .or_else(fs_ext::ignore_not_found)
    3019            0 :                                 .fatal_err("Removing .old file");
    3020            0 :                             continue;
    3021              :                         }
    3022            0 :                         Discovered::Unknown(file_name) => {
    3023            0 :                             // we will later error if there are any
    3024            0 :                             unrecognized_files.push(file_name);
    3025            0 :                             continue;
    3026              :                         }
    3027            0 :                         Discovered::Ephemeral(name) => (name, "old ephemeral file"),
    3028            0 :                         Discovered::Temporary(name) => (name, "temporary timeline file"),
    3029            0 :                         Discovered::TemporaryDownload(name) => (name, "temporary download"),
    3030              :                     };
    3031            0 :                     path.push(Utf8Path::new(&name));
    3032            0 :                     init::cleanup(&path, kind)?;
    3033            0 :                     path.pop();
    3034              :                 }
    3035              : 
    3036           12 :                 if !unrecognized_files.is_empty() {
    3037              :                     // assume that if there are any there are many many.
    3038            0 :                     let n = unrecognized_files.len();
    3039            0 :                     let first = &unrecognized_files[..n.min(10)];
    3040            0 :                     anyhow::bail!(
    3041            0 :                         "unrecognized files in timeline dir (total {n}), first 10: {first:?}"
    3042            0 :                     );
    3043           12 :                 }
    3044           12 : 
    3045           12 :                 let decided = init::reconcile(discovered_layers, &index_part, disk_consistent_lsn);
    3046           12 : 
    3047           12 :                 let mut loaded_layers = Vec::new();
    3048           12 :                 let mut needs_cleanup = Vec::new();
    3049           12 :                 let mut total_physical_size = 0;
    3050              : 
    3051           44 :                 for (name, decision) in decided {
    3052           32 :                     let decision = match decision {
    3053           32 :                         Ok(decision) => decision,
    3054            0 :                         Err(DismissedLayer::Future { local }) => {
    3055            0 :                             if let Some(local) = local {
    3056            0 :                                 init::cleanup_future_layer(
    3057            0 :                                     &local.local_path,
    3058            0 :                                     &name,
    3059            0 :                                     disk_consistent_lsn,
    3060            0 :                                 )?;
    3061            0 :                             }
    3062            0 :                             needs_cleanup.push(name);
    3063            0 :                             continue;
    3064              :                         }
    3065            0 :                         Err(DismissedLayer::LocalOnly(local)) => {
    3066            0 :                             init::cleanup_local_only_file(&name, &local)?;
    3067              :                             // this file never existed remotely, we will have to do rework
    3068            0 :                             continue;
    3069              :                         }
    3070            0 :                         Err(DismissedLayer::BadMetadata(local)) => {
    3071            0 :                             init::cleanup_local_file_for_remote(&local)?;
    3072              :                             // this file never existed remotely, we will have to do rework
    3073            0 :                             continue;
    3074              :                         }
    3075              :                     };
    3076              : 
    3077           32 :                     match &name {
    3078           24 :                         Delta(d) => assert!(d.lsn_range.end <= disk_consistent_lsn + 1),
    3079            8 :                         Image(i) => assert!(i.lsn <= disk_consistent_lsn),
    3080              :                     }
    3081              : 
    3082           32 :                     tracing::debug!(layer=%name, ?decision, "applied");
    3083              : 
    3084           32 :                     let layer = match decision {
    3085           32 :                         Resident { local, remote } => {
    3086           32 :                             total_physical_size += local.file_size;
    3087           32 :                             Layer::for_resident(conf, &this, local.local_path, name, remote)
    3088           32 :                                 .drop_eviction_guard()
    3089              :                         }
    3090            0 :                         Evicted(remote) => Layer::for_evicted(conf, &this, name, remote),
    3091              :                     };
    3092              : 
    3093           32 :                     loaded_layers.push(layer);
    3094              :                 }
    3095           12 :                 Ok((loaded_layers, needs_cleanup, total_physical_size))
    3096           12 :             }
    3097           12 :         })
    3098           12 :         .await
    3099           12 :         .map_err(anyhow::Error::new)
    3100           12 :         .and_then(|x| x)?;
    3101              : 
    3102           12 :         let num_layers = loaded_layers.len();
    3103           12 : 
    3104           12 :         guard
    3105           12 :             .open_mut()
    3106           12 :             .expect("layermanager must be open during init")
    3107           12 :             .initialize_local_layers(loaded_layers, disk_consistent_lsn + 1);
    3108           12 : 
    3109           12 :         self.remote_client
    3110           12 :             .schedule_layer_file_deletion(&needs_cleanup)?;
    3111           12 :         self.remote_client
    3112           12 :             .schedule_index_upload_for_file_changes()?;
    3113              :         // This barrier orders above DELETEs before any later operations.
    3114              :         // This is critical because code executing after the barrier might
    3115              :         // create again objects with the same key that we just scheduled for deletion.
    3116              :         // For example, if we just scheduled deletion of an image layer "from the future",
    3117              :         // later compaction might run again and re-create the same image layer.
    3118              :         // "from the future" here means an image layer whose LSN is > IndexPart::disk_consistent_lsn.
    3119              :         // "same" here means same key range and LSN.
    3120              :         //
    3121              :         // Without a barrier between above DELETEs and the re-creation's PUTs,
    3122              :         // the upload queue may execute the PUT first, then the DELETE.
    3123              :         // In our example, we will end up with an IndexPart referencing a non-existent object.
    3124              :         //
    3125              :         // 1. a future image layer is created and uploaded
    3126              :         // 2. ps restart
    3127              :         // 3. the future layer from (1) is deleted during load layer map
    3128              :         // 4. image layer is re-created and uploaded
    3129              :         // 5. deletion queue would like to delete (1) but actually deletes (4)
    3130              :         // 6. delete by name works as expected, but it now deletes the wrong (later) version
    3131              :         //
    3132              :         // See https://github.com/neondatabase/neon/issues/5878
    3133              :         //
    3134              :         // NB: generation numbers naturally protect against this because they disambiguate
    3135              :         //     (1) and (4)
    3136              :         // TODO: this is basically a no-op now, should we remove it?
    3137           12 :         self.remote_client.schedule_barrier()?;
    3138              :         // Tenant::create_timeline will wait for these uploads to happen before returning, or
    3139              :         // on retry.
    3140              : 
    3141              :         // Now that we have the full layer map, we may calculate the visibility of layers within it (a global scan)
    3142           12 :         drop(guard); // drop write lock, update_layer_visibility will take a read lock.
    3143           12 :         self.update_layer_visibility().await?;
    3144              : 
    3145           12 :         info!(
    3146            0 :             "loaded layer map with {} layers at {}, total physical size: {}",
    3147              :             num_layers, disk_consistent_lsn, total_physical_size
    3148              :         );
    3149              : 
    3150           12 :         timer.stop_and_record();
    3151           12 :         Ok(())
    3152           12 :     }
    3153              : 
    3154              :     /// Retrieve current logical size of the timeline.
    3155              :     ///
    3156              :     /// The size could be lagging behind the actual number, in case
    3157              :     /// the initial size calculation has not been run (gets triggered on the first size access).
    3158              :     ///
    3159              :     /// return size and boolean flag that shows if the size is exact
    3160            0 :     pub(crate) fn get_current_logical_size(
    3161            0 :         self: &Arc<Self>,
    3162            0 :         priority: GetLogicalSizePriority,
    3163            0 :         ctx: &RequestContext,
    3164            0 :     ) -> logical_size::CurrentLogicalSize {
    3165            0 :         if !self.tenant_shard_id.is_shard_zero() {
    3166              :             // Logical size is only accurately maintained on shard zero: when called elsewhere, for example
    3167              :             // when HTTP API is serving a GET for timeline zero, return zero
    3168            0 :             return logical_size::CurrentLogicalSize::Approximate(logical_size::Approximate::zero());
    3169            0 :         }
    3170            0 : 
    3171            0 :         let current_size = self.current_logical_size.current_size();
    3172            0 :         debug!("Current size: {current_size:?}");
    3173              : 
    3174            0 :         match (current_size.accuracy(), priority) {
    3175            0 :             (logical_size::Accuracy::Exact, _) => (), // nothing to do
    3176            0 :             (logical_size::Accuracy::Approximate, GetLogicalSizePriority::Background) => {
    3177            0 :                 // background task will eventually deliver an exact value, we're in no rush
    3178            0 :             }
    3179              :             (logical_size::Accuracy::Approximate, GetLogicalSizePriority::User) => {
    3180              :                 // background task is not ready, but user is asking for it now;
    3181              :                 // => make the background task skip the line
    3182              :                 // (The alternative would be to calculate the size here, but,
    3183              :                 //  it can actually take a long time if the user has a lot of rels.
    3184              :                 //  And we'll inevitable need it again; So, let the background task do the work.)
    3185            0 :                 match self
    3186            0 :                     .current_logical_size
    3187            0 :                     .cancel_wait_for_background_loop_concurrency_limit_semaphore
    3188            0 :                     .get()
    3189              :                 {
    3190            0 :                     Some(cancel) => cancel.cancel(),
    3191              :                     None => {
    3192            0 :                         match self.current_state() {
    3193            0 :                             TimelineState::Broken { .. } | TimelineState::Stopping => {
    3194            0 :                                 // Can happen when timeline detail endpoint is used when deletion is ongoing (or its broken).
    3195            0 :                                 // Don't make noise.
    3196            0 :                             }
    3197              :                             TimelineState::Loading => {
    3198              :                                 // Import does not return an activated timeline.
    3199            0 :                                 info!(
    3200            0 :                                     "discarding priority boost for logical size calculation because timeline is not yet active"
    3201              :                                 );
    3202              :                             }
    3203              :                             TimelineState::Active => {
    3204              :                                 // activation should be setting the once cell
    3205            0 :                                 warn!(
    3206            0 :                                     "unexpected: cancel_wait_for_background_loop_concurrency_limit_semaphore not set, priority-boosting of logical size calculation will not work"
    3207              :                                 );
    3208            0 :                                 debug_assert!(false);
    3209              :                             }
    3210              :                         }
    3211              :                     }
    3212              :                 }
    3213              :             }
    3214              :         }
    3215              : 
    3216            0 :         if let CurrentLogicalSize::Approximate(_) = &current_size {
    3217            0 :             if ctx.task_kind() == TaskKind::WalReceiverConnectionHandler {
    3218            0 :                 let first = self
    3219            0 :                     .current_logical_size
    3220            0 :                     .did_return_approximate_to_walreceiver
    3221            0 :                     .compare_exchange(
    3222            0 :                         false,
    3223            0 :                         true,
    3224            0 :                         AtomicOrdering::Relaxed,
    3225            0 :                         AtomicOrdering::Relaxed,
    3226            0 :                     )
    3227            0 :                     .is_ok();
    3228            0 :                 if first {
    3229            0 :                     crate::metrics::initial_logical_size::TIMELINES_WHERE_WALRECEIVER_GOT_APPROXIMATE_SIZE.inc();
    3230            0 :                 }
    3231            0 :             }
    3232            0 :         }
    3233              : 
    3234            0 :         current_size
    3235            0 :     }
    3236              : 
    3237            0 :     fn spawn_initial_logical_size_computation_task(self: &Arc<Self>, ctx: &RequestContext) {
    3238            0 :         let Some(initial_part_end) = self.current_logical_size.initial_part_end else {
    3239              :             // nothing to do for freshly created timelines;
    3240            0 :             assert_eq!(
    3241            0 :                 self.current_logical_size.current_size().accuracy(),
    3242            0 :                 logical_size::Accuracy::Exact,
    3243            0 :             );
    3244            0 :             self.current_logical_size.initialized.add_permits(1);
    3245            0 :             return;
    3246              :         };
    3247              : 
    3248            0 :         let cancel_wait_for_background_loop_concurrency_limit_semaphore = CancellationToken::new();
    3249            0 :         let token = cancel_wait_for_background_loop_concurrency_limit_semaphore.clone();
    3250            0 :         self.current_logical_size
    3251            0 :             .cancel_wait_for_background_loop_concurrency_limit_semaphore.set(token)
    3252            0 :             .expect("initial logical size calculation task must be spawned exactly once per Timeline object");
    3253            0 : 
    3254            0 :         let self_clone = Arc::clone(self);
    3255            0 :         let background_ctx = ctx.detached_child(
    3256            0 :             TaskKind::InitialLogicalSizeCalculation,
    3257            0 :             DownloadBehavior::Download,
    3258            0 :         );
    3259            0 :         task_mgr::spawn(
    3260            0 :             task_mgr::BACKGROUND_RUNTIME.handle(),
    3261            0 :             task_mgr::TaskKind::InitialLogicalSizeCalculation,
    3262            0 :             self.tenant_shard_id,
    3263            0 :             Some(self.timeline_id),
    3264            0 :             "initial size calculation",
    3265              :             // NB: don't log errors here, task_mgr will do that.
    3266            0 :             async move {
    3267            0 :                 self_clone
    3268            0 :                     .initial_logical_size_calculation_task(
    3269            0 :                         initial_part_end,
    3270            0 :                         cancel_wait_for_background_loop_concurrency_limit_semaphore,
    3271            0 :                         background_ctx,
    3272            0 :                     )
    3273            0 :                     .await;
    3274            0 :                 Ok(())
    3275            0 :             }
    3276            0 :             .instrument(info_span!(parent: None, "initial_size_calculation", tenant_id=%self.tenant_shard_id.tenant_id, shard_id=%self.tenant_shard_id.shard_slug(), timeline_id=%self.timeline_id)),
    3277              :         );
    3278            0 :     }
    3279              : 
    3280              :     /// # Cancellation
    3281              :     ///
    3282              :     /// This method is sensitive to `Timeline::cancel`.
    3283              :     ///
    3284              :     /// It is _not_ sensitive to task_mgr::shutdown_token().
    3285              :     ///
    3286              :     /// # Cancel-Safety
    3287              :     ///
    3288              :     /// It does Timeline IO, hence this should be polled to completion because
    3289              :     /// we could be leaving in-flight IOs behind, which is safe, but annoying
    3290              :     /// to reason about.
    3291            0 :     async fn initial_logical_size_calculation_task(
    3292            0 :         self: Arc<Self>,
    3293            0 :         initial_part_end: Lsn,
    3294            0 :         skip_concurrency_limiter: CancellationToken,
    3295            0 :         background_ctx: RequestContext,
    3296            0 :     ) {
    3297            0 :         scopeguard::defer! {
    3298            0 :             // Irrespective of the outcome of this operation, we should unblock anyone waiting for it.
    3299            0 :             self.current_logical_size.initialized.add_permits(1);
    3300            0 :         }
    3301            0 : 
    3302            0 :         let try_once = |attempt: usize| {
    3303            0 :             let background_ctx = &background_ctx;
    3304            0 :             let self_ref = &self;
    3305            0 :             let skip_concurrency_limiter = &skip_concurrency_limiter;
    3306            0 :             async move {
    3307            0 :                 let wait_for_permit = super::tasks::acquire_concurrency_permit(
    3308            0 :                     BackgroundLoopKind::InitialLogicalSizeCalculation,
    3309            0 :                     background_ctx,
    3310            0 :                 );
    3311              : 
    3312              :                 use crate::metrics::initial_logical_size::StartCircumstances;
    3313            0 :                 let (_maybe_permit, circumstances) = tokio::select! {
    3314            0 :                     permit = wait_for_permit => {
    3315            0 :                         (Some(permit), StartCircumstances::AfterBackgroundTasksRateLimit)
    3316              :                     }
    3317            0 :                     _ = self_ref.cancel.cancelled() => {
    3318            0 :                         return Err(CalculateLogicalSizeError::Cancelled);
    3319              :                     }
    3320            0 :                     () = skip_concurrency_limiter.cancelled() => {
    3321              :                         // Some action that is part of a end user interaction requested logical size
    3322              :                         // => break out of the rate limit
    3323              :                         // TODO: ideally we'd not run on BackgroundRuntime but the requester's runtime;
    3324              :                         // but then again what happens if they cancel; also, we should just be using
    3325              :                         // one runtime across the entire process, so, let's leave this for now.
    3326            0 :                         (None, StartCircumstances::SkippedConcurrencyLimiter)
    3327              :                     }
    3328              :                 };
    3329              : 
    3330            0 :                 let metrics_guard = if attempt == 1 {
    3331            0 :                     crate::metrics::initial_logical_size::START_CALCULATION.first(circumstances)
    3332              :                 } else {
    3333            0 :                     crate::metrics::initial_logical_size::START_CALCULATION.retry(circumstances)
    3334              :                 };
    3335              : 
    3336            0 :                 let io_concurrency = IoConcurrency::spawn_from_conf(
    3337            0 :                     self_ref.conf,
    3338            0 :                     self_ref
    3339            0 :                         .gate
    3340            0 :                         .enter()
    3341            0 :                         .map_err(|_| CalculateLogicalSizeError::Cancelled)?,
    3342              :                 );
    3343              : 
    3344            0 :                 let calculated_size = self_ref
    3345            0 :                     .logical_size_calculation_task(
    3346            0 :                         initial_part_end,
    3347            0 :                         LogicalSizeCalculationCause::Initial,
    3348            0 :                         background_ctx,
    3349            0 :                     )
    3350            0 :                     .await?;
    3351              : 
    3352            0 :                 self_ref
    3353            0 :                     .trigger_aux_file_size_computation(
    3354            0 :                         initial_part_end,
    3355            0 :                         background_ctx,
    3356            0 :                         io_concurrency,
    3357            0 :                     )
    3358            0 :                     .await?;
    3359              : 
    3360              :                 // TODO: add aux file size to logical size
    3361              : 
    3362            0 :                 Ok((calculated_size, metrics_guard))
    3363            0 :             }
    3364            0 :         };
    3365              : 
    3366            0 :         let retrying = async {
    3367            0 :             let mut attempt = 0;
    3368              :             loop {
    3369            0 :                 attempt += 1;
    3370            0 : 
    3371            0 :                 match try_once(attempt).await {
    3372            0 :                     Ok(res) => return ControlFlow::Continue(res),
    3373            0 :                     Err(CalculateLogicalSizeError::Cancelled) => return ControlFlow::Break(()),
    3374              :                     Err(
    3375            0 :                         e @ (CalculateLogicalSizeError::Decode(_)
    3376            0 :                         | CalculateLogicalSizeError::PageRead(_)),
    3377            0 :                     ) => {
    3378            0 :                         warn!(attempt, "initial size calculation failed: {e:?}");
    3379              :                         // exponential back-off doesn't make sense at these long intervals;
    3380              :                         // use fixed retry interval with generous jitter instead
    3381            0 :                         let sleep_duration = Duration::from_secs(
    3382            0 :                             u64::try_from(
    3383            0 :                                 // 1hour base
    3384            0 :                                 (60_i64 * 60_i64)
    3385            0 :                                     // 10min jitter
    3386            0 :                                     + rand::thread_rng().gen_range(-10 * 60..10 * 60),
    3387            0 :                             )
    3388            0 :                             .expect("10min < 1hour"),
    3389            0 :                         );
    3390            0 :                         tokio::select! {
    3391            0 :                             _ = tokio::time::sleep(sleep_duration) => {}
    3392            0 :                             _ = self.cancel.cancelled() => return ControlFlow::Break(()),
    3393              :                         }
    3394              :                     }
    3395              :                 }
    3396              :             }
    3397            0 :         };
    3398              : 
    3399            0 :         let (calculated_size, metrics_guard) = match retrying.await {
    3400            0 :             ControlFlow::Continue(calculated_size) => calculated_size,
    3401            0 :             ControlFlow::Break(()) => return,
    3402              :         };
    3403              : 
    3404              :         // we cannot query current_logical_size.current_size() to know the current
    3405              :         // *negative* value, only truncated to u64.
    3406            0 :         let added = self
    3407            0 :             .current_logical_size
    3408            0 :             .size_added_after_initial
    3409            0 :             .load(AtomicOrdering::Relaxed);
    3410            0 : 
    3411            0 :         let sum = calculated_size.saturating_add_signed(added);
    3412            0 : 
    3413            0 :         // set the gauge value before it can be set in `update_current_logical_size`.
    3414            0 :         self.metrics.current_logical_size_gauge.set(sum);
    3415            0 : 
    3416            0 :         self.current_logical_size
    3417            0 :             .initial_logical_size
    3418            0 :             .set((calculated_size, metrics_guard.calculation_result_saved()))
    3419            0 :             .ok()
    3420            0 :             .expect("only this task sets it");
    3421            0 :     }
    3422              : 
    3423            0 :     pub(crate) fn spawn_ondemand_logical_size_calculation(
    3424            0 :         self: &Arc<Self>,
    3425            0 :         lsn: Lsn,
    3426            0 :         cause: LogicalSizeCalculationCause,
    3427            0 :         ctx: RequestContext,
    3428            0 :     ) -> oneshot::Receiver<Result<u64, CalculateLogicalSizeError>> {
    3429            0 :         let (sender, receiver) = oneshot::channel();
    3430            0 :         let self_clone = Arc::clone(self);
    3431            0 :         // XXX if our caller loses interest, i.e., ctx is cancelled,
    3432            0 :         // we should stop the size calculation work and return an error.
    3433            0 :         // That would require restructuring this function's API to
    3434            0 :         // return the result directly, instead of a Receiver for the result.
    3435            0 :         let ctx = ctx.detached_child(
    3436            0 :             TaskKind::OndemandLogicalSizeCalculation,
    3437            0 :             DownloadBehavior::Download,
    3438            0 :         );
    3439            0 :         task_mgr::spawn(
    3440            0 :             task_mgr::BACKGROUND_RUNTIME.handle(),
    3441            0 :             task_mgr::TaskKind::OndemandLogicalSizeCalculation,
    3442            0 :             self.tenant_shard_id,
    3443            0 :             Some(self.timeline_id),
    3444            0 :             "ondemand logical size calculation",
    3445            0 :             async move {
    3446            0 :                 let res = self_clone
    3447            0 :                     .logical_size_calculation_task(lsn, cause, &ctx)
    3448            0 :                     .await;
    3449            0 :                 let _ = sender.send(res).ok();
    3450            0 :                 Ok(()) // Receiver is responsible for handling errors
    3451            0 :             }
    3452            0 :             .in_current_span(),
    3453            0 :         );
    3454            0 :         receiver
    3455            0 :     }
    3456              : 
    3457              :     #[instrument(skip_all)]
    3458              :     async fn logical_size_calculation_task(
    3459              :         self: &Arc<Self>,
    3460              :         lsn: Lsn,
    3461              :         cause: LogicalSizeCalculationCause,
    3462              :         ctx: &RequestContext,
    3463              :     ) -> Result<u64, CalculateLogicalSizeError> {
    3464              :         crate::span::debug_assert_current_span_has_tenant_and_timeline_id();
    3465              :         // We should never be calculating logical sizes on shard !=0, because these shards do not have
    3466              :         // accurate relation sizes, and they do not emit consumption metrics.
    3467              :         debug_assert!(self.tenant_shard_id.is_shard_zero());
    3468              : 
    3469              :         let guard = self
    3470              :             .gate
    3471              :             .enter()
    3472            0 :             .map_err(|_| CalculateLogicalSizeError::Cancelled)?;
    3473              : 
    3474              :         self.calculate_logical_size(lsn, cause, &guard, ctx).await
    3475              :     }
    3476              : 
    3477              :     /// Calculate the logical size of the database at the latest LSN.
    3478              :     ///
    3479              :     /// NOTE: counted incrementally, includes ancestors. This can be a slow operation,
    3480              :     /// especially if we need to download remote layers.
    3481            0 :     async fn calculate_logical_size(
    3482            0 :         &self,
    3483            0 :         up_to_lsn: Lsn,
    3484            0 :         cause: LogicalSizeCalculationCause,
    3485            0 :         _guard: &GateGuard,
    3486            0 :         ctx: &RequestContext,
    3487            0 :     ) -> Result<u64, CalculateLogicalSizeError> {
    3488            0 :         info!(
    3489            0 :             "Calculating logical size for timeline {} at {}",
    3490              :             self.timeline_id, up_to_lsn
    3491              :         );
    3492              : 
    3493            0 :         if let Err(()) = pausable_failpoint!("timeline-calculate-logical-size-pause", &self.cancel)
    3494              :         {
    3495            0 :             return Err(CalculateLogicalSizeError::Cancelled);
    3496            0 :         }
    3497              : 
    3498              :         // See if we've already done the work for initial size calculation.
    3499              :         // This is a short-cut for timelines that are mostly unused.
    3500            0 :         if let Some(size) = self.current_logical_size.initialized_size(up_to_lsn) {
    3501            0 :             return Ok(size);
    3502            0 :         }
    3503            0 :         let storage_time_metrics = match cause {
    3504              :             LogicalSizeCalculationCause::Initial
    3505              :             | LogicalSizeCalculationCause::ConsumptionMetricsSyntheticSize
    3506            0 :             | LogicalSizeCalculationCause::TenantSizeHandler => &self.metrics.logical_size_histo,
    3507              :             LogicalSizeCalculationCause::EvictionTaskImitation => {
    3508            0 :                 &self.metrics.imitate_logical_size_histo
    3509              :             }
    3510              :         };
    3511            0 :         let timer = storage_time_metrics.start_timer();
    3512            0 :         let logical_size = self
    3513            0 :             .get_current_logical_size_non_incremental(up_to_lsn, ctx)
    3514            0 :             .await?;
    3515            0 :         debug!("calculated logical size: {logical_size}");
    3516            0 :         timer.stop_and_record();
    3517            0 :         Ok(logical_size)
    3518            0 :     }
    3519              : 
    3520              :     /// Update current logical size, adding `delta' to the old value.
    3521       541140 :     fn update_current_logical_size(&self, delta: i64) {
    3522       541140 :         let logical_size = &self.current_logical_size;
    3523       541140 :         logical_size.increment_size(delta);
    3524       541140 : 
    3525       541140 :         // Also set the value in the prometheus gauge. Note that
    3526       541140 :         // there is a race condition here: if this is is called by two
    3527       541140 :         // threads concurrently, the prometheus gauge might be set to
    3528       541140 :         // one value while current_logical_size is set to the
    3529       541140 :         // other.
    3530       541140 :         match logical_size.current_size() {
    3531       541140 :             CurrentLogicalSize::Exact(ref new_current_size) => self
    3532       541140 :                 .metrics
    3533       541140 :                 .current_logical_size_gauge
    3534       541140 :                 .set(new_current_size.into()),
    3535            0 :             CurrentLogicalSize::Approximate(_) => {
    3536            0 :                 // don't update the gauge yet, this allows us not to update the gauge back and
    3537            0 :                 // forth between the initial size calculation task.
    3538            0 :             }
    3539              :         }
    3540       541140 :     }
    3541              : 
    3542         5988 :     pub(crate) fn update_directory_entries_count(&self, kind: DirectoryKind, count: MetricsUpdate) {
    3543         5988 :         // TODO: this directory metrics is not correct -- we could have multiple reldirs in the system
    3544         5988 :         // for each of the database, but we only store one value, and therefore each pgdirmodification
    3545         5988 :         // would overwrite the previous value if they modify different databases.
    3546         5988 : 
    3547         5988 :         match count {
    3548         2144 :             MetricsUpdate::Set(count) => {
    3549         2144 :                 self.directory_metrics[kind.offset()].store(count, AtomicOrdering::Relaxed);
    3550         2144 :                 self.directory_metrics_inited[kind.offset()].store(true, AtomicOrdering::Relaxed);
    3551         2144 :             }
    3552         3840 :             MetricsUpdate::Add(count) => {
    3553         3840 :                 // TODO: these operations are not atomic; but we only have one writer to the metrics, so
    3554         3840 :                 // it's fine.
    3555         3840 :                 if self.directory_metrics_inited[kind.offset()].load(AtomicOrdering::Relaxed) {
    3556         3840 :                     // The metrics has been initialized with `MetricsUpdate::Set` before, so we can add/sub
    3557         3840 :                     // the value reliably.
    3558         3840 :                     self.directory_metrics[kind.offset()].fetch_add(count, AtomicOrdering::Relaxed);
    3559         3840 :                 }
    3560              :                 // Otherwise, ignore this update
    3561              :             }
    3562            4 :             MetricsUpdate::Sub(count) => {
    3563            4 :                 // TODO: these operations are not atomic; but we only have one writer to the metrics, so
    3564            4 :                 // it's fine.
    3565            4 :                 if self.directory_metrics_inited[kind.offset()].load(AtomicOrdering::Relaxed) {
    3566            4 :                     // The metrics has been initialized with `MetricsUpdate::Set` before.
    3567            4 :                     // The operation could overflow so we need to normalize the value.
    3568            4 :                     let prev_val =
    3569            4 :                         self.directory_metrics[kind.offset()].load(AtomicOrdering::Relaxed);
    3570            4 :                     let res = prev_val.saturating_sub(count);
    3571            4 :                     self.directory_metrics[kind.offset()].store(res, AtomicOrdering::Relaxed);
    3572            4 :                 }
    3573              :                 // Otherwise, ignore this update
    3574              :             }
    3575              :         };
    3576              : 
    3577              :         // TODO: remove this, there's no place in the code that updates this aux metrics.
    3578         5988 :         let aux_metric =
    3579         5988 :             self.directory_metrics[DirectoryKind::AuxFiles.offset()].load(AtomicOrdering::Relaxed);
    3580         5988 : 
    3581         5988 :         let sum_of_entries = self
    3582         5988 :             .directory_metrics
    3583         5988 :             .iter()
    3584        47904 :             .map(|v| v.load(AtomicOrdering::Relaxed))
    3585         5988 :             .sum();
    3586              :         // Set a high general threshold and a lower threshold for the auxiliary files,
    3587              :         // as we can have large numbers of relations in the db directory.
    3588              :         const SUM_THRESHOLD: u64 = 5000;
    3589              :         const AUX_THRESHOLD: u64 = 1000;
    3590         5988 :         if sum_of_entries >= SUM_THRESHOLD || aux_metric >= AUX_THRESHOLD {
    3591            0 :             self.metrics
    3592            0 :                 .directory_entries_count_gauge
    3593            0 :                 .set(sum_of_entries);
    3594         5988 :         } else if let Some(metric) = Lazy::get(&self.metrics.directory_entries_count_gauge) {
    3595            0 :             metric.set(sum_of_entries);
    3596         5988 :         }
    3597         5988 :     }
    3598              : 
    3599            0 :     async fn find_layer(
    3600            0 :         &self,
    3601            0 :         layer_name: &LayerName,
    3602            0 :     ) -> Result<Option<Layer>, layer_manager::Shutdown> {
    3603            0 :         let guard = self.layers.read().await;
    3604            0 :         let layer = guard
    3605            0 :             .layer_map()?
    3606            0 :             .iter_historic_layers()
    3607            0 :             .find(|l| &l.layer_name() == layer_name)
    3608            0 :             .map(|found| guard.get_from_desc(&found));
    3609            0 :         Ok(layer)
    3610            0 :     }
    3611              : 
    3612            0 :     pub(super) fn should_keep_previous_heatmap(&self, new_heatmap_end_lsn: Lsn) -> bool {
    3613            0 :         let crnt = self.previous_heatmap.load();
    3614            0 :         match crnt.as_deref() {
    3615            0 :             Some(PreviousHeatmap::Active { end_lsn, .. }) => match end_lsn {
    3616            0 :                 Some(crnt_end_lsn) => *crnt_end_lsn > new_heatmap_end_lsn,
    3617            0 :                 None => true,
    3618              :             },
    3619            0 :             Some(PreviousHeatmap::Obsolete) => false,
    3620            0 :             None => false,
    3621              :         }
    3622            0 :     }
    3623              : 
    3624              :     /// The timeline heatmap is a hint to secondary locations from the primary location,
    3625              :     /// indicating which layers are currently on-disk on the primary.
    3626              :     ///
    3627              :     /// None is returned if the Timeline is in a state where uploading a heatmap
    3628              :     /// doesn't make sense, such as shutting down or initializing.  The caller
    3629              :     /// should treat this as a cue to simply skip doing any heatmap uploading
    3630              :     /// for this timeline.
    3631           32 :     pub(crate) async fn generate_heatmap(&self) -> Option<HeatMapTimeline> {
    3632           32 :         if !self.is_active() {
    3633            0 :             return None;
    3634           32 :         }
    3635              : 
    3636           32 :         let guard = self.layers.read().await;
    3637              : 
    3638              :         // Firstly, if there's any heatmap left over from when this location
    3639              :         // was a secondary, take that into account. Keep layers that are:
    3640              :         // * present in the layer map
    3641              :         // * visible
    3642              :         // * non-resident
    3643              :         // * not evicted since we read the heatmap
    3644              :         //
    3645              :         // Without this, a new cold, attached location would clobber the previous
    3646              :         // heatamp.
    3647           32 :         let previous_heatmap = self.previous_heatmap.load();
    3648           32 :         let visible_non_resident = match previous_heatmap.as_deref() {
    3649              :             Some(PreviousHeatmap::Active {
    3650           24 :                 heatmap, read_at, ..
    3651           92 :             }) => Some(heatmap.all_layers().filter_map(|hl| {
    3652           92 :                 let desc: PersistentLayerDesc = hl.name.clone().into();
    3653           92 :                 let layer = guard.try_get_from_key(&desc.key())?;
    3654              : 
    3655           92 :                 if layer.visibility() == LayerVisibilityHint::Covered {
    3656            0 :                     return None;
    3657           92 :                 }
    3658           92 : 
    3659           92 :                 if layer.is_likely_resident() {
    3660           40 :                     return None;
    3661           52 :                 }
    3662           52 : 
    3663           52 :                 if layer.last_evicted_at().happened_after(*read_at) {
    3664           12 :                     return None;
    3665           40 :                 }
    3666           40 : 
    3667           40 :                 Some((desc, hl.metadata.clone(), hl.access_time, hl.cold))
    3668           92 :             })),
    3669            0 :             Some(PreviousHeatmap::Obsolete) => None,
    3670            8 :             None => None,
    3671              :         };
    3672              : 
    3673              :         // Secondly, all currently visible, resident layers are included.
    3674           72 :         let resident = guard.likely_resident_layers().filter_map(|layer| {
    3675           72 :             match layer.visibility() {
    3676              :                 LayerVisibilityHint::Visible => {
    3677              :                     // Layer is visible to one or more read LSNs: elegible for inclusion in layer map
    3678           68 :                     let last_activity_ts = layer.latest_activity();
    3679           68 :                     Some((
    3680           68 :                         layer.layer_desc().clone(),
    3681           68 :                         layer.metadata(),
    3682           68 :                         last_activity_ts,
    3683           68 :                         false, // these layers are not cold
    3684           68 :                     ))
    3685              :                 }
    3686              :                 LayerVisibilityHint::Covered => {
    3687              :                     // Layer is resident but unlikely to be read: not elegible for inclusion in heatmap.
    3688            4 :                     None
    3689              :                 }
    3690              :             }
    3691           72 :         });
    3692              : 
    3693           32 :         let mut layers = match visible_non_resident {
    3694           24 :             Some(non_resident) => {
    3695           24 :                 let mut non_resident = non_resident.peekable();
    3696           24 :                 if non_resident.peek().is_none() {
    3697            8 :                     tracing::info!(timeline_id=%self.timeline_id, "Previous heatmap now obsolete");
    3698            8 :                     self.previous_heatmap
    3699            8 :                         .store(Some(PreviousHeatmap::Obsolete.into()));
    3700           16 :                 }
    3701              : 
    3702           24 :                 non_resident.chain(resident).collect::<Vec<_>>()
    3703              :             }
    3704            8 :             None => resident.collect::<Vec<_>>(),
    3705              :         };
    3706              : 
    3707              :         // Sort layers in order of which to download first.  For a large set of layers to download, we
    3708              :         // want to prioritize those layers which are most likely to still be in the resident many minutes
    3709              :         // or hours later:
    3710              :         // - Cold layers go last for convenience when a human inspects the heatmap.
    3711              :         // - Download L0s last, because they churn the fastest: L0s on a fast-writing tenant might
    3712              :         //   only exist for a few minutes before being compacted into L1s.
    3713              :         // - For L1 & image layers, download most recent LSNs first: the older the LSN, the sooner
    3714              :         //   the layer is likely to be covered by an image layer during compaction.
    3715          240 :         layers.sort_by_key(|(desc, _meta, _atime, cold)| {
    3716          240 :             std::cmp::Reverse((
    3717          240 :                 *cold,
    3718          240 :                 !LayerMap::is_l0(&desc.key_range, desc.is_delta),
    3719          240 :                 desc.lsn_range.end,
    3720          240 :             ))
    3721          240 :         });
    3722           32 : 
    3723           32 :         let layers = layers
    3724           32 :             .into_iter()
    3725          108 :             .map(|(desc, meta, atime, cold)| {
    3726          108 :                 HeatMapLayer::new(desc.layer_name(), meta, atime, cold)
    3727          108 :             })
    3728           32 :             .collect();
    3729           32 : 
    3730           32 :         Some(HeatMapTimeline::new(self.timeline_id, layers))
    3731           32 :     }
    3732              : 
    3733            0 :     pub(super) async fn generate_unarchival_heatmap(&self, end_lsn: Lsn) -> PreviousHeatmap {
    3734            0 :         let guard = self.layers.read().await;
    3735              : 
    3736            0 :         let now = SystemTime::now();
    3737            0 :         let mut heatmap_layers = Vec::default();
    3738            0 :         for vl in guard.visible_layers() {
    3739            0 :             if vl.layer_desc().get_lsn_range().start >= end_lsn {
    3740            0 :                 continue;
    3741            0 :             }
    3742            0 : 
    3743            0 :             let hl = HeatMapLayer {
    3744            0 :                 name: vl.layer_desc().layer_name(),
    3745            0 :                 metadata: vl.metadata(),
    3746            0 :                 access_time: now,
    3747            0 :                 cold: true,
    3748            0 :             };
    3749            0 :             heatmap_layers.push(hl);
    3750              :         }
    3751              : 
    3752            0 :         tracing::info!(
    3753            0 :             "Generating unarchival heatmap with {} layers",
    3754            0 :             heatmap_layers.len()
    3755              :         );
    3756              : 
    3757            0 :         let heatmap = HeatMapTimeline::new(self.timeline_id, heatmap_layers);
    3758            0 :         PreviousHeatmap::Active {
    3759            0 :             heatmap,
    3760            0 :             read_at: Instant::now(),
    3761            0 :             end_lsn: Some(end_lsn),
    3762            0 :         }
    3763            0 :     }
    3764              : 
    3765              :     /// Returns true if the given lsn is or was an ancestor branchpoint.
    3766            0 :     pub(crate) fn is_ancestor_lsn(&self, lsn: Lsn) -> bool {
    3767            0 :         // upon timeline detach, we set the ancestor_lsn to Lsn::INVALID and the store the original
    3768            0 :         // branchpoint in the value in IndexPart::lineage
    3769            0 :         self.ancestor_lsn == lsn
    3770            0 :             || (self.ancestor_lsn == Lsn::INVALID
    3771            0 :                 && self.remote_client.is_previous_ancestor_lsn(lsn))
    3772            0 :     }
    3773              : }
    3774              : 
    3775              : impl Timeline {
    3776              :     #[allow(clippy::doc_lazy_continuation)]
    3777              :     /// Get the data needed to reconstruct all keys in the provided keyspace
    3778              :     ///
    3779              :     /// The algorithm is as follows:
    3780              :     /// 1.   While some keys are still not done and there's a timeline to visit:
    3781              :     /// 2.   Visit the timeline (see [`Timeline::get_vectored_reconstruct_data_timeline`]:
    3782              :     /// 2.1: Build the fringe for the current keyspace
    3783              :     /// 2.2  Visit the newest layer from the fringe to collect all values for the range it
    3784              :     ///      intersects
    3785              :     /// 2.3. Pop the timeline from the fringe
    3786              :     /// 2.4. If the fringe is empty, go back to 1
    3787      1255306 :     async fn get_vectored_reconstruct_data(
    3788      1255306 :         &self,
    3789      1255306 :         mut keyspace: KeySpace,
    3790      1255306 :         request_lsn: Lsn,
    3791      1255306 :         reconstruct_state: &mut ValuesReconstructState,
    3792      1255306 :         ctx: &RequestContext,
    3793      1255306 :     ) -> Result<(), GetVectoredError> {
    3794      1255306 :         let mut timeline_owned: Arc<Timeline>;
    3795      1255306 :         let mut timeline = self;
    3796      1255306 : 
    3797      1255306 :         let mut cont_lsn = Lsn(request_lsn.0 + 1);
    3798              : 
    3799      1255302 :         let missing_keyspace = loop {
    3800      1703635 :             if self.cancel.is_cancelled() {
    3801            0 :                 return Err(GetVectoredError::Cancelled);
    3802      1703635 :             }
    3803              : 
    3804              :             let TimelineVisitOutcome {
    3805      1703635 :                 completed_keyspace: completed,
    3806      1703635 :                 image_covered_keyspace,
    3807      1703635 :             } = Self::get_vectored_reconstruct_data_timeline(
    3808      1703635 :                 timeline,
    3809      1703635 :                 keyspace.clone(),
    3810      1703635 :                 cont_lsn,
    3811      1703635 :                 reconstruct_state,
    3812      1703635 :                 &self.cancel,
    3813      1703635 :                 ctx,
    3814      1703635 :             )
    3815      1703635 :             .await?;
    3816              : 
    3817      1703635 :             keyspace.remove_overlapping_with(&completed);
    3818      1703635 : 
    3819      1703635 :             // Do not descend into the ancestor timeline for aux files.
    3820      1703635 :             // We don't return a blanket [`GetVectoredError::MissingKey`] to avoid
    3821      1703635 :             // stalling compaction.
    3822      1703635 :             keyspace.remove_overlapping_with(&KeySpace {
    3823      1703635 :                 ranges: vec![NON_INHERITED_RANGE, Key::sparse_non_inherited_keyspace()],
    3824      1703635 :             });
    3825      1703635 : 
    3826      1703635 :             // Keyspace is fully retrieved
    3827      1703635 :             if keyspace.is_empty() {
    3828      1254794 :                 break None;
    3829       448841 :             }
    3830              : 
    3831       448841 :             let Some(ancestor_timeline) = timeline.ancestor_timeline.as_ref() else {
    3832              :                 // Not fully retrieved but no ancestor timeline.
    3833          508 :                 break Some(keyspace);
    3834              :             };
    3835              : 
    3836              :             // Now we see if there are keys covered by the image layer but does not exist in the
    3837              :             // image layer, which means that the key does not exist.
    3838              : 
    3839              :             // The block below will stop the vectored search if any of the keys encountered an image layer
    3840              :             // which did not contain a snapshot for said key. Since we have already removed all completed
    3841              :             // keys from `keyspace`, we expect there to be no overlap between it and the image covered key
    3842              :             // space. If that's not the case, we had at least one key encounter a gap in the image layer
    3843              :             // and stop the search as a result of that.
    3844       448333 :             let mut removed = keyspace.remove_overlapping_with(&image_covered_keyspace);
    3845       448333 :             // Do not fire missing key error and end early for sparse keys. Note that we hava already removed
    3846       448333 :             // non-inherited keyspaces before, so we can safely do a full `SPARSE_RANGE` remove instead of
    3847       448333 :             // figuring out what is the inherited key range and do a fine-grained pruning.
    3848       448333 :             removed.remove_overlapping_with(&KeySpace {
    3849       448333 :                 ranges: vec![SPARSE_RANGE],
    3850       448333 :             });
    3851       448333 :             if !removed.is_empty() {
    3852            0 :                 break Some(removed);
    3853       448333 :             }
    3854       448333 :             // If we reached this point, `remove_overlapping_with` should not have made any change to the
    3855       448333 :             // keyspace.
    3856       448333 : 
    3857       448333 :             // Take the min to avoid reconstructing a page with data newer than request Lsn.
    3858       448333 :             cont_lsn = std::cmp::min(Lsn(request_lsn.0 + 1), Lsn(timeline.ancestor_lsn.0 + 1));
    3859       448333 :             timeline_owned = timeline
    3860       448333 :                 .get_ready_ancestor_timeline(ancestor_timeline, ctx)
    3861       448333 :                 .await?;
    3862       448329 :             timeline = &*timeline_owned;
    3863              :         };
    3864              : 
    3865              :         // Remove sparse keys from the keyspace so that it doesn't fire errors.
    3866      1255302 :         let missing_keyspace = if let Some(missing_keyspace) = missing_keyspace {
    3867          508 :             let mut missing_keyspace = missing_keyspace;
    3868          508 :             missing_keyspace.remove_overlapping_with(&KeySpace {
    3869          508 :                 ranges: vec![SPARSE_RANGE],
    3870          508 :             });
    3871          508 :             if missing_keyspace.is_empty() {
    3872          480 :                 None
    3873              :             } else {
    3874           28 :                 Some(missing_keyspace)
    3875              :             }
    3876              :         } else {
    3877      1254794 :             None
    3878              :         };
    3879              : 
    3880      1255302 :         if let Some(missing_keyspace) = missing_keyspace {
    3881           28 :             return Err(GetVectoredError::MissingKey(MissingKeyError {
    3882           28 :                 key: missing_keyspace.start().unwrap(), /* better if we can store the full keyspace */
    3883           28 :                 shard: self
    3884           28 :                     .shard_identity
    3885           28 :                     .get_shard_number(&missing_keyspace.start().unwrap()),
    3886           28 :                 cont_lsn,
    3887           28 :                 request_lsn,
    3888           28 :                 ancestor_lsn: Some(timeline.ancestor_lsn),
    3889           28 :                 backtrace: None,
    3890           28 :                 read_path: std::mem::take(&mut reconstruct_state.read_path),
    3891           28 :             }));
    3892      1255274 :         }
    3893      1255274 : 
    3894      1255274 :         Ok(())
    3895      1255306 :     }
    3896              : 
    3897              :     /// Collect the reconstruct data for a keyspace from the specified timeline.
    3898              :     ///
    3899              :     /// Maintain a fringe [`LayerFringe`] which tracks all the layers that intersect
    3900              :     /// the current keyspace. The current keyspace of the search at any given timeline
    3901              :     /// is the original keyspace minus all the keys that have been completed minus
    3902              :     /// any keys for which we couldn't find an intersecting layer. It's not tracked explicitly,
    3903              :     /// but if you merge all the keyspaces in the fringe, you get the "current keyspace".
    3904              :     ///
    3905              :     /// This is basically a depth-first search visitor implementation where a vertex
    3906              :     /// is the (layer, lsn range, key space) tuple. The fringe acts as the stack.
    3907              :     ///
    3908              :     /// At each iteration pop the top of the fringe (the layer with the highest Lsn)
    3909              :     /// and get all the required reconstruct data from the layer in one go.
    3910              :     ///
    3911              :     /// Returns the completed keyspace and the keyspaces with image coverage. The caller
    3912              :     /// decides how to deal with these two keyspaces.
    3913      1703635 :     async fn get_vectored_reconstruct_data_timeline(
    3914      1703635 :         timeline: &Timeline,
    3915      1703635 :         keyspace: KeySpace,
    3916      1703635 :         mut cont_lsn: Lsn,
    3917      1703635 :         reconstruct_state: &mut ValuesReconstructState,
    3918      1703635 :         cancel: &CancellationToken,
    3919      1703635 :         ctx: &RequestContext,
    3920      1703635 :     ) -> Result<TimelineVisitOutcome, GetVectoredError> {
    3921      1703635 :         let mut unmapped_keyspace = keyspace.clone();
    3922      1703635 :         let mut fringe = LayerFringe::new();
    3923      1703635 : 
    3924      1703635 :         let mut completed_keyspace = KeySpace::default();
    3925      1703635 :         let mut image_covered_keyspace = KeySpaceRandomAccum::new();
    3926      1703635 : 
    3927      1703635 :         // Prevent GC from progressing while visiting the current timeline.
    3928      1703635 :         // If we are GC-ing because a new image layer was added while traversing
    3929      1703635 :         // the timeline, then it will remove layers that are required for fulfilling
    3930      1703635 :         // the current get request (read-path cannot "look back" and notice the new
    3931      1703635 :         // image layer).
    3932      1703635 :         let _gc_cutoff_holder = timeline.get_applied_gc_cutoff_lsn();
    3933              : 
    3934              :         // See `compaction::compact_with_gc` for why we need this.
    3935      1703635 :         let _guard = timeline.gc_compaction_layer_update_lock.read().await;
    3936              : 
    3937              :         loop {
    3938      3397353 :             if cancel.is_cancelled() {
    3939            0 :                 return Err(GetVectoredError::Cancelled);
    3940      3397353 :             }
    3941      3397353 : 
    3942      3397353 :             let (keys_done_last_step, keys_with_image_coverage) =
    3943      3397353 :                 reconstruct_state.consume_done_keys();
    3944      3397353 :             unmapped_keyspace.remove_overlapping_with(&keys_done_last_step);
    3945      3397353 :             completed_keyspace.merge(&keys_done_last_step);
    3946      3397353 :             if let Some(keys_with_image_coverage) = keys_with_image_coverage {
    3947        45100 :                 unmapped_keyspace
    3948        45100 :                     .remove_overlapping_with(&KeySpace::single(keys_with_image_coverage.clone()));
    3949        45100 :                 image_covered_keyspace.add_range(keys_with_image_coverage);
    3950      3352253 :             }
    3951              : 
    3952              :             // Do not descent any further if the last layer we visited
    3953              :             // completed all keys in the keyspace it inspected. This is not
    3954              :             // required for correctness, but avoids visiting extra layers
    3955              :             // which turns out to be a perf bottleneck in some cases.
    3956      3397353 :             if !unmapped_keyspace.is_empty() {
    3957      2146599 :                 let guard = timeline.layers.read().await;
    3958      2146599 :                 let layers = guard.layer_map()?;
    3959              : 
    3960      2147995 :                 for range in unmapped_keyspace.ranges.iter() {
    3961      2147995 :                     let results = layers.range_search(range.clone(), cont_lsn);
    3962      2147995 : 
    3963      2147995 :                     results
    3964      2147995 :                         .found
    3965      2147995 :                         .into_iter()
    3966      2147995 :                         .map(|(SearchResult { layer, lsn_floor }, keyspace_accum)| {
    3967      1694654 :                             (
    3968      1694654 :                                 guard.upgrade(layer),
    3969      1694654 :                                 keyspace_accum.to_keyspace(),
    3970      1694654 :                                 lsn_floor..cont_lsn,
    3971      1694654 :                             )
    3972      2147995 :                         })
    3973      2147995 :                         .for_each(|(layer, keyspace, lsn_range)| {
    3974      1694654 :                             fringe.update(layer, keyspace, lsn_range)
    3975      2147995 :                         });
    3976      2147995 :                 }
    3977              : 
    3978              :                 // It's safe to drop the layer map lock after planning the next round of reads.
    3979              :                 // The fringe keeps readable handles for the layers which are safe to read even
    3980              :                 // if layers were compacted or flushed.
    3981              :                 //
    3982              :                 // The more interesting consideration is: "Why is the read algorithm still correct
    3983              :                 // if the layer map changes while it is operating?". Doing a vectored read on a
    3984              :                 // timeline boils down to pushing an imaginary lsn boundary downwards for each range
    3985              :                 // covered by the read. The layer map tells us how to move the lsn downwards for a
    3986              :                 // range at *a particular point in time*. It is fine for the answer to be different
    3987              :                 // at two different time points.
    3988      2146599 :                 drop(guard);
    3989      1250754 :             }
    3990              : 
    3991      3397353 :             if let Some((layer_to_read, keyspace_to_read, lsn_range)) = fringe.next_layer() {
    3992      1693718 :                 if let Some(ref mut read_path) = reconstruct_state.read_path {
    3993      1693718 :                     read_path.record_layer_visit(&layer_to_read, &keyspace_to_read, &lsn_range);
    3994      1693718 :                 }
    3995      1693718 :                 let next_cont_lsn = lsn_range.start;
    3996      1693718 :                 layer_to_read
    3997      1693718 :                     .get_values_reconstruct_data(
    3998      1693718 :                         keyspace_to_read.clone(),
    3999      1693718 :                         lsn_range,
    4000      1693718 :                         reconstruct_state,
    4001      1693718 :                         ctx,
    4002      1693718 :                     )
    4003      1693718 :                     .await?;
    4004              : 
    4005      1693718 :                 unmapped_keyspace = keyspace_to_read;
    4006      1693718 :                 cont_lsn = next_cont_lsn;
    4007      1693718 : 
    4008      1693718 :                 reconstruct_state.on_layer_visited(&layer_to_read);
    4009              :             } else {
    4010      1703635 :                 break;
    4011      1703635 :             }
    4012      1703635 :         }
    4013      1703635 : 
    4014      1703635 :         Ok(TimelineVisitOutcome {
    4015      1703635 :             completed_keyspace,
    4016      1703635 :             image_covered_keyspace: image_covered_keyspace.consume_keyspace(),
    4017      1703635 :         })
    4018      1703635 :     }
    4019              : 
    4020       448333 :     async fn get_ready_ancestor_timeline(
    4021       448333 :         &self,
    4022       448333 :         ancestor: &Arc<Timeline>,
    4023       448333 :         ctx: &RequestContext,
    4024       448333 :     ) -> Result<Arc<Timeline>, GetReadyAncestorError> {
    4025       448333 :         // It's possible that the ancestor timeline isn't active yet, or
    4026       448333 :         // is active but hasn't yet caught up to the branch point. Wait
    4027       448333 :         // for it.
    4028       448333 :         //
    4029       448333 :         // This cannot happen while the pageserver is running normally,
    4030       448333 :         // because you cannot create a branch from a point that isn't
    4031       448333 :         // present in the pageserver yet. However, we don't wait for the
    4032       448333 :         // branch point to be uploaded to cloud storage before creating
    4033       448333 :         // a branch. I.e., the branch LSN need not be remote consistent
    4034       448333 :         // for the branching operation to succeed.
    4035       448333 :         //
    4036       448333 :         // Hence, if we try to load a tenant in such a state where
    4037       448333 :         // 1. the existence of the branch was persisted (in IndexPart and/or locally)
    4038       448333 :         // 2. but the ancestor state is behind branch_lsn because it was not yet persisted
    4039       448333 :         // then we will need to wait for the ancestor timeline to
    4040       448333 :         // re-stream WAL up to branch_lsn before we access it.
    4041       448333 :         //
    4042       448333 :         // How can a tenant get in such a state?
    4043       448333 :         // - ungraceful pageserver process exit
    4044       448333 :         // - detach+attach => this is a bug, https://github.com/neondatabase/neon/issues/4219
    4045       448333 :         //
    4046       448333 :         // NB: this could be avoided by requiring
    4047       448333 :         //   branch_lsn >= remote_consistent_lsn
    4048       448333 :         // during branch creation.
    4049       448333 :         match ancestor.wait_to_become_active(ctx).await {
    4050       448329 :             Ok(()) => {}
    4051              :             Err(TimelineState::Stopping) => {
    4052              :                 // If an ancestor is stopping, it means the tenant is stopping: handle this the same as if this timeline was stopping.
    4053            0 :                 return Err(GetReadyAncestorError::Cancelled);
    4054              :             }
    4055            4 :             Err(state) => {
    4056            4 :                 return Err(GetReadyAncestorError::BadState {
    4057            4 :                     timeline_id: ancestor.timeline_id,
    4058            4 :                     state,
    4059            4 :                 });
    4060              :             }
    4061              :         }
    4062       448329 :         ancestor
    4063       448329 :             .wait_lsn(
    4064       448329 :                 self.ancestor_lsn,
    4065       448329 :                 WaitLsnWaiter::Timeline(self),
    4066       448329 :                 WaitLsnTimeout::Default,
    4067       448329 :                 ctx,
    4068       448329 :             )
    4069       448329 :             .await
    4070       448329 :             .map_err(|e| match e {
    4071            0 :                 e @ WaitLsnError::Timeout(_) => GetReadyAncestorError::AncestorLsnTimeout(e),
    4072            0 :                 WaitLsnError::Shutdown => GetReadyAncestorError::Cancelled,
    4073            0 :                 WaitLsnError::BadState(state) => GetReadyAncestorError::BadState {
    4074            0 :                     timeline_id: ancestor.timeline_id,
    4075            0 :                     state,
    4076            0 :                 },
    4077       448329 :             })?;
    4078              : 
    4079       448329 :         Ok(ancestor.clone())
    4080       448333 :     }
    4081              : 
    4082       594312 :     pub(crate) fn get_shard_identity(&self) -> &ShardIdentity {
    4083       594312 :         &self.shard_identity
    4084       594312 :     }
    4085              : 
    4086              :     #[inline(always)]
    4087            0 :     pub(crate) fn shard_timeline_id(&self) -> ShardTimelineId {
    4088            0 :         ShardTimelineId {
    4089            0 :             shard_index: ShardIndex {
    4090            0 :                 shard_number: self.shard_identity.number,
    4091            0 :                 shard_count: self.shard_identity.count,
    4092            0 :             },
    4093            0 :             timeline_id: self.timeline_id,
    4094            0 :         }
    4095            0 :     }
    4096              : 
    4097              :     /// Returns a non-frozen open in-memory layer for ingestion.
    4098              :     ///
    4099              :     /// Takes a witness of timeline writer state lock being held, because it makes no sense to call
    4100              :     /// this function without holding the mutex.
    4101         2604 :     async fn get_layer_for_write(
    4102         2604 :         &self,
    4103         2604 :         lsn: Lsn,
    4104         2604 :         _guard: &tokio::sync::MutexGuard<'_, Option<TimelineWriterState>>,
    4105         2604 :         ctx: &RequestContext,
    4106         2604 :     ) -> anyhow::Result<Arc<InMemoryLayer>> {
    4107         2604 :         let mut guard = self.layers.write().await;
    4108              : 
    4109         2604 :         let last_record_lsn = self.get_last_record_lsn();
    4110         2604 :         ensure!(
    4111         2604 :             lsn > last_record_lsn,
    4112            0 :             "cannot modify relation after advancing last_record_lsn (incoming_lsn={}, last_record_lsn={})",
    4113              :             lsn,
    4114              :             last_record_lsn,
    4115              :         );
    4116              : 
    4117         2604 :         let layer = guard
    4118         2604 :             .open_mut()?
    4119         2604 :             .get_layer_for_write(
    4120         2604 :                 lsn,
    4121         2604 :                 self.conf,
    4122         2604 :                 self.timeline_id,
    4123         2604 :                 self.tenant_shard_id,
    4124         2604 :                 &self.gate,
    4125         2604 :                 ctx,
    4126         2604 :             )
    4127         2604 :             .await?;
    4128         2604 :         Ok(layer)
    4129         2604 :     }
    4130              : 
    4131     10558196 :     pub(crate) fn finish_write(&self, new_lsn: Lsn) {
    4132     10558196 :         assert!(new_lsn.is_aligned());
    4133              : 
    4134     10558196 :         self.metrics.last_record_lsn_gauge.set(new_lsn.0 as i64);
    4135     10558196 :         self.last_record_lsn.advance(new_lsn);
    4136     10558196 :     }
    4137              : 
    4138              :     /// Freeze any existing open in-memory layer and unconditionally notify the flush loop.
    4139              :     ///
    4140              :     /// Unconditional flush loop notification is given because in sharded cases we will want to
    4141              :     /// leave an Lsn gap. Unsharded tenants do not have Lsn gaps.
    4142         2412 :     async fn freeze_inmem_layer_at(
    4143         2412 :         &self,
    4144         2412 :         at: Lsn,
    4145         2412 :         write_lock: &mut tokio::sync::MutexGuard<'_, Option<TimelineWriterState>>,
    4146         2412 :     ) -> Result<u64, FlushLayerError> {
    4147         2412 :         let frozen = {
    4148         2412 :             let mut guard = self.layers.write().await;
    4149         2412 :             guard
    4150         2412 :                 .open_mut()?
    4151         2412 :                 .try_freeze_in_memory_layer(at, &self.last_freeze_at, write_lock, &self.metrics)
    4152         2412 :                 .await
    4153              :         };
    4154              : 
    4155         2412 :         if frozen {
    4156         2356 :             let now = Instant::now();
    4157         2356 :             *(self.last_freeze_ts.write().unwrap()) = now;
    4158         2356 :         }
    4159              : 
    4160              :         // Increment the flush cycle counter and wake up the flush task.
    4161              :         // Remember the new value, so that when we listen for the flush
    4162              :         // to finish, we know when the flush that we initiated has
    4163              :         // finished, instead of some other flush that was started earlier.
    4164         2412 :         let mut my_flush_request = 0;
    4165         2412 : 
    4166         2412 :         let flush_loop_state = { *self.flush_loop_state.lock().unwrap() };
    4167         2412 :         if !matches!(flush_loop_state, FlushLoopState::Running { .. }) {
    4168            0 :             return Err(FlushLayerError::NotRunning(flush_loop_state));
    4169         2412 :         }
    4170         2412 : 
    4171         2412 :         self.layer_flush_start_tx.send_modify(|(counter, lsn)| {
    4172         2412 :             my_flush_request = *counter + 1;
    4173         2412 :             *counter = my_flush_request;
    4174         2412 :             *lsn = std::cmp::max(at, *lsn);
    4175         2412 :         });
    4176         2412 : 
    4177         2412 :         assert_ne!(my_flush_request, 0);
    4178              : 
    4179         2412 :         Ok(my_flush_request)
    4180         2412 :     }
    4181              : 
    4182              :     /// Layer flusher task's main loop.
    4183          892 :     async fn flush_loop(
    4184          892 :         self: &Arc<Self>,
    4185          892 :         mut layer_flush_start_rx: tokio::sync::watch::Receiver<(u64, Lsn)>,
    4186          892 :         ctx: &RequestContext,
    4187          892 :     ) {
    4188              :         // Subscribe to L0 delta layer updates, for compaction backpressure.
    4189          892 :         let mut watch_l0 = match self.layers.read().await.layer_map() {
    4190          892 :             Ok(lm) => lm.watch_level0_deltas(),
    4191            0 :             Err(Shutdown) => return,
    4192              :         };
    4193              : 
    4194          892 :         info!("started flush loop");
    4195              :         loop {
    4196         3242 :             tokio::select! {
    4197         3242 :                 _ = self.cancel.cancelled() => {
    4198           20 :                     info!("shutting down layer flush task due to Timeline::cancel");
    4199           20 :                     break;
    4200              :                 },
    4201         3242 :                 _ = layer_flush_start_rx.changed() => {}
    4202         2350 :             }
    4203         2350 :             trace!("waking up");
    4204         2350 :             let (flush_counter, frozen_to_lsn) = *layer_flush_start_rx.borrow();
    4205         2350 : 
    4206         2350 :             // The highest LSN to which we flushed in the loop over frozen layers
    4207         2350 :             let mut flushed_to_lsn = Lsn(0);
    4208              : 
    4209         2350 :             let result = loop {
    4210         4706 :                 if self.cancel.is_cancelled() {
    4211            0 :                     info!("dropping out of flush loop for timeline shutdown");
    4212              :                     // Note: we do not bother transmitting into [`layer_flush_done_tx`], because
    4213              :                     // anyone waiting on that will respect self.cancel as well: they will stop
    4214              :                     // waiting at the same time we as drop out of this loop.
    4215            0 :                     return;
    4216         4706 :                 }
    4217         4706 : 
    4218         4706 :                 // Break to notify potential waiters as soon as we've flushed the requested LSN. If
    4219         4706 :                 // more requests have arrived in the meanwhile, we'll resume flushing afterwards.
    4220         4706 :                 if flushed_to_lsn >= frozen_to_lsn {
    4221         2294 :                     break Ok(());
    4222         2412 :                 }
    4223              : 
    4224              :                 // Fetch the next layer to flush, if any.
    4225         2412 :                 let (layer, l0_count, frozen_count, frozen_size) = {
    4226         2412 :                     let layers = self.layers.read().await;
    4227         2412 :                     let Ok(lm) = layers.layer_map() else {
    4228            0 :                         info!("dropping out of flush loop for timeline shutdown");
    4229            0 :                         return;
    4230              :                     };
    4231         2412 :                     let l0_count = lm.level0_deltas().len();
    4232         2412 :                     let frozen_count = lm.frozen_layers.len();
    4233         2412 :                     let frozen_size: u64 = lm
    4234         2412 :                         .frozen_layers
    4235         2412 :                         .iter()
    4236         2420 :                         .map(|l| l.estimated_in_mem_size())
    4237         2412 :                         .sum();
    4238         2412 :                     let layer = lm.frozen_layers.front().cloned();
    4239         2412 :                     (layer, l0_count, frozen_count, frozen_size)
    4240         2412 :                     // drop 'layers' lock
    4241         2412 :                 };
    4242         2412 :                 let Some(layer) = layer else {
    4243           56 :                     break Ok(());
    4244              :                 };
    4245              : 
    4246              :                 // Stall flushes to backpressure if compaction can't keep up. This is propagated up
    4247              :                 // to WAL ingestion by having ephemeral layer rolls wait for flushes.
    4248         2356 :                 if let Some(stall_threshold) = self.get_l0_flush_stall_threshold() {
    4249            0 :                     if l0_count >= stall_threshold {
    4250            0 :                         warn!(
    4251            0 :                             "stalling layer flushes for compaction backpressure at {l0_count} \
    4252            0 :                             L0 layers ({frozen_count} frozen layers with {frozen_size} bytes)"
    4253              :                         );
    4254            0 :                         let stall_timer = self
    4255            0 :                             .metrics
    4256            0 :                             .flush_delay_histo
    4257            0 :                             .start_timer()
    4258            0 :                             .record_on_drop();
    4259            0 :                         tokio::select! {
    4260            0 :                             result = watch_l0.wait_for(|l0| *l0 < stall_threshold) => {
    4261            0 :                                 if let Ok(l0) = result.as_deref() {
    4262            0 :                                     let delay = stall_timer.elapsed().as_secs_f64();
    4263            0 :                                     info!("resuming layer flushes at {l0} L0 layers after {delay:.3}s");
    4264            0 :                                 }
    4265              :                             },
    4266            0 :                             _ = self.cancel.cancelled() => {},
    4267              :                         }
    4268            0 :                         continue; // check again
    4269            0 :                     }
    4270         2356 :                 }
    4271              : 
    4272              :                 // Flush the layer.
    4273         2356 :                 let flush_timer = self.metrics.flush_time_histo.start_timer();
    4274         2356 :                 match self.flush_frozen_layer(layer, ctx).await {
    4275         2356 :                     Ok(layer_lsn) => flushed_to_lsn = max(flushed_to_lsn, layer_lsn),
    4276              :                     Err(FlushLayerError::Cancelled) => {
    4277            0 :                         info!("dropping out of flush loop for timeline shutdown");
    4278            0 :                         return;
    4279              :                     }
    4280            0 :                     err @ Err(
    4281            0 :                         FlushLayerError::NotRunning(_)
    4282            0 :                         | FlushLayerError::Other(_)
    4283            0 :                         | FlushLayerError::CreateImageLayersError(_),
    4284            0 :                     ) => {
    4285            0 :                         error!("could not flush frozen layer: {err:?}");
    4286            0 :                         break err.map(|_| ());
    4287              :                     }
    4288              :                 }
    4289         2356 :                 let flush_duration = flush_timer.stop_and_record();
    4290         2356 : 
    4291         2356 :                 // Notify the tenant compaction loop if L0 compaction is needed.
    4292         2356 :                 let l0_count = *watch_l0.borrow();
    4293         2356 :                 if l0_count >= self.get_compaction_threshold() {
    4294          956 :                     self.l0_compaction_trigger.notify_one();
    4295         1400 :                 }
    4296              : 
    4297              :                 // Delay the next flush to backpressure if compaction can't keep up. We delay by the
    4298              :                 // flush duration such that the flush takes 2x as long. This is propagated up to WAL
    4299              :                 // ingestion by having ephemeral layer rolls wait for flushes.
    4300         2356 :                 if let Some(delay_threshold) = self.get_l0_flush_delay_threshold() {
    4301            0 :                     if l0_count >= delay_threshold {
    4302            0 :                         let delay = flush_duration.as_secs_f64();
    4303            0 :                         info!(
    4304            0 :                             "delaying layer flush by {delay:.3}s for compaction backpressure at \
    4305            0 :                             {l0_count} L0 layers ({frozen_count} frozen layers with {frozen_size} bytes)"
    4306              :                         );
    4307            0 :                         let _delay_timer = self
    4308            0 :                             .metrics
    4309            0 :                             .flush_delay_histo
    4310            0 :                             .start_timer()
    4311            0 :                             .record_on_drop();
    4312            0 :                         tokio::select! {
    4313            0 :                             _ = tokio::time::sleep(flush_duration) => {},
    4314            0 :                             _ = watch_l0.wait_for(|l0| *l0 < delay_threshold) => {},
    4315            0 :                             _ = self.cancel.cancelled() => {},
    4316              :                         }
    4317            0 :                     }
    4318         2356 :                 }
    4319              :             };
    4320              : 
    4321              :             // Unsharded tenants should never advance their LSN beyond the end of the
    4322              :             // highest layer they write: such gaps between layer data and the frozen LSN
    4323              :             // are only legal on sharded tenants.
    4324         2350 :             debug_assert!(
    4325         2350 :                 self.shard_identity.count.count() > 1
    4326         2350 :                     || flushed_to_lsn >= frozen_to_lsn
    4327           56 :                     || !flushed_to_lsn.is_valid()
    4328              :             );
    4329              : 
    4330         2350 :             if flushed_to_lsn < frozen_to_lsn && self.shard_identity.count.count() > 1 {
    4331              :                 // If our layer flushes didn't carry disk_consistent_lsn up to the `to_lsn` advertised
    4332              :                 // to us via layer_flush_start_rx, then advance it here.
    4333              :                 //
    4334              :                 // This path is only taken for tenants with multiple shards: single sharded tenants should
    4335              :                 // never encounter a gap in the wal.
    4336            0 :                 let old_disk_consistent_lsn = self.disk_consistent_lsn.load();
    4337            0 :                 tracing::debug!(
    4338            0 :                     "Advancing disk_consistent_lsn across layer gap {old_disk_consistent_lsn}->{frozen_to_lsn}"
    4339              :                 );
    4340            0 :                 if self.set_disk_consistent_lsn(frozen_to_lsn) {
    4341            0 :                     if let Err(e) = self.schedule_uploads(frozen_to_lsn, vec![]) {
    4342            0 :                         tracing::warn!(
    4343            0 :                             "Failed to schedule metadata upload after updating disk_consistent_lsn: {e}"
    4344              :                         );
    4345            0 :                     }
    4346            0 :                 }
    4347         2350 :             }
    4348              : 
    4349              :             // Notify any listeners that we're done
    4350         2350 :             let _ = self
    4351         2350 :                 .layer_flush_done_tx
    4352         2350 :                 .send_replace((flush_counter, result));
    4353              :         }
    4354           20 :     }
    4355              : 
    4356              :     /// Waits any flush request created by [`Self::freeze_inmem_layer_at`] to complete.
    4357         2252 :     async fn wait_flush_completion(&self, request: u64) -> Result<(), FlushLayerError> {
    4358         2252 :         let mut rx = self.layer_flush_done_tx.subscribe();
    4359              :         loop {
    4360              :             {
    4361         4576 :                 let (last_result_counter, last_result) = &*rx.borrow();
    4362         4576 :                 if *last_result_counter >= request {
    4363         2252 :                     if let Err(err) = last_result {
    4364              :                         // We already logged the original error in
    4365              :                         // flush_loop. We cannot propagate it to the caller
    4366              :                         // here, because it might not be Cloneable
    4367            0 :                         return Err(err.clone());
    4368              :                     } else {
    4369         2252 :                         return Ok(());
    4370              :                     }
    4371         2324 :                 }
    4372         2324 :             }
    4373         2324 :             trace!("waiting for flush to complete");
    4374         2324 :             tokio::select! {
    4375         2324 :                 rx_e = rx.changed() => {
    4376         2324 :                     rx_e.map_err(|_| FlushLayerError::NotRunning(*self.flush_loop_state.lock().unwrap()))?;
    4377              :                 },
    4378              :                 // Cancellation safety: we are not leaving an I/O in-flight for the flush, we're just ignoring
    4379              :                 // the notification from [`flush_loop`] that it completed.
    4380         2324 :                 _ = self.cancel.cancelled() => {
    4381            0 :                     tracing::info!("Cancelled layer flush due on timeline shutdown");
    4382            0 :                     return Ok(())
    4383              :                 }
    4384              :             };
    4385         2324 :             trace!("done")
    4386              :         }
    4387         2252 :     }
    4388              : 
    4389              :     /// Flush one frozen in-memory layer to disk, as a new delta layer.
    4390              :     ///
    4391              :     /// Return value is the last lsn (inclusive) of the layer that was frozen.
    4392              :     #[instrument(skip_all, fields(layer=%frozen_layer))]
    4393              :     async fn flush_frozen_layer(
    4394              :         self: &Arc<Self>,
    4395              :         frozen_layer: Arc<InMemoryLayer>,
    4396              :         ctx: &RequestContext,
    4397              :     ) -> Result<Lsn, FlushLayerError> {
    4398              :         debug_assert_current_span_has_tenant_and_timeline_id();
    4399              : 
    4400              :         // As a special case, when we have just imported an image into the repository,
    4401              :         // instead of writing out a L0 delta layer, we directly write out image layer
    4402              :         // files instead. This is possible as long as *all* the data imported into the
    4403              :         // repository have the same LSN.
    4404              :         let lsn_range = frozen_layer.get_lsn_range();
    4405              : 
    4406              :         // Whether to directly create image layers for this flush, or flush them as delta layers
    4407              :         let create_image_layer =
    4408              :             lsn_range.start == self.initdb_lsn && lsn_range.end == Lsn(self.initdb_lsn.0 + 1);
    4409              : 
    4410              :         #[cfg(test)]
    4411              :         {
    4412              :             match &mut *self.flush_loop_state.lock().unwrap() {
    4413              :                 FlushLoopState::NotStarted | FlushLoopState::Exited => {
    4414              :                     panic!("flush loop not running")
    4415              :                 }
    4416              :                 FlushLoopState::Running {
    4417              :                     expect_initdb_optimization,
    4418              :                     initdb_optimization_count,
    4419              :                     ..
    4420              :                 } => {
    4421              :                     if create_image_layer {
    4422              :                         *initdb_optimization_count += 1;
    4423              :                     } else {
    4424              :                         assert!(!*expect_initdb_optimization, "expected initdb optimization");
    4425              :                     }
    4426              :                 }
    4427              :             }
    4428              :         }
    4429              : 
    4430              :         let (layers_to_upload, delta_layer_to_add) = if create_image_layer {
    4431              :             // Note: The 'ctx' in use here has DownloadBehavior::Error. We should not
    4432              :             // require downloading anything during initial import.
    4433              :             let ((rel_partition, metadata_partition), _lsn) = self
    4434              :                 .repartition(
    4435              :                     self.initdb_lsn,
    4436              :                     self.get_compaction_target_size(),
    4437              :                     EnumSet::empty(),
    4438              :                     ctx,
    4439              :                 )
    4440              :                 .await
    4441            0 :                 .map_err(|e| FlushLayerError::from_anyhow(self, e.into()))?;
    4442              : 
    4443              :             if self.cancel.is_cancelled() {
    4444              :                 return Err(FlushLayerError::Cancelled);
    4445              :             }
    4446              : 
    4447              :             // Ensure that we have a single call to `create_image_layers` with a combined dense keyspace.
    4448              :             // So that the key ranges don't overlap.
    4449              :             let mut partitions = KeyPartitioning::default();
    4450              :             partitions.parts.extend(rel_partition.parts);
    4451              :             if !metadata_partition.parts.is_empty() {
    4452              :                 assert_eq!(
    4453              :                     metadata_partition.parts.len(),
    4454              :                     1,
    4455              :                     "currently sparse keyspace should only contain a single metadata keyspace"
    4456              :                 );
    4457              :                 // Safety: create_image_layers treat sparse keyspaces differently that it does not scan
    4458              :                 // every single key within the keyspace, and therefore, it's safe to force converting it
    4459              :                 // into a dense keyspace before calling this function.
    4460              :                 partitions
    4461              :                     .parts
    4462              :                     .extend(metadata_partition.into_dense().parts);
    4463              :             }
    4464              : 
    4465              :             let mut layers_to_upload = Vec::new();
    4466              :             let (generated_image_layers, is_complete) = self
    4467              :                 .create_image_layers(
    4468              :                     &partitions,
    4469              :                     self.initdb_lsn,
    4470              :                     ImageLayerCreationMode::Initial,
    4471              :                     ctx,
    4472              :                     LastImageLayerCreationStatus::Initial,
    4473              :                     false, // don't yield for L0, we're flushing L0
    4474              :                 )
    4475              :                 .await?;
    4476              :             debug_assert!(
    4477              :                 matches!(is_complete, LastImageLayerCreationStatus::Complete),
    4478              :                 "init image generation mode must fully cover the keyspace"
    4479              :             );
    4480              :             layers_to_upload.extend(generated_image_layers);
    4481              : 
    4482              :             (layers_to_upload, None)
    4483              :         } else {
    4484              :             // Normal case, write out a L0 delta layer file.
    4485              :             // `create_delta_layer` will not modify the layer map.
    4486              :             // We will remove frozen layer and add delta layer in one atomic operation later.
    4487              :             let Some(layer) = self
    4488              :                 .create_delta_layer(&frozen_layer, None, ctx)
    4489              :                 .await
    4490            0 :                 .map_err(|e| FlushLayerError::from_anyhow(self, e))?
    4491              :             else {
    4492              :                 panic!("delta layer cannot be empty if no filter is applied");
    4493              :             };
    4494              :             (
    4495              :                 // FIXME: even though we have a single image and single delta layer assumption
    4496              :                 // we push them to vec
    4497              :                 vec![layer.clone()],
    4498              :                 Some(layer),
    4499              :             )
    4500              :         };
    4501              : 
    4502              :         pausable_failpoint!("flush-layer-cancel-after-writing-layer-out-pausable");
    4503              : 
    4504              :         if self.cancel.is_cancelled() {
    4505              :             return Err(FlushLayerError::Cancelled);
    4506              :         }
    4507              : 
    4508              :         let disk_consistent_lsn = Lsn(lsn_range.end.0 - 1);
    4509              : 
    4510              :         // The new on-disk layers are now in the layer map. We can remove the
    4511              :         // in-memory layer from the map now. The flushed layer is stored in
    4512              :         // the mapping in `create_delta_layer`.
    4513              :         {
    4514              :             let mut guard = self.layers.write().await;
    4515              : 
    4516              :             guard.open_mut()?.finish_flush_l0_layer(
    4517              :                 delta_layer_to_add.as_ref(),
    4518              :                 &frozen_layer,
    4519              :                 &self.metrics,
    4520              :             );
    4521              : 
    4522              :             if self.set_disk_consistent_lsn(disk_consistent_lsn) {
    4523              :                 // Schedule remote uploads that will reflect our new disk_consistent_lsn
    4524              :                 self.schedule_uploads(disk_consistent_lsn, layers_to_upload)
    4525            0 :                     .map_err(|e| FlushLayerError::from_anyhow(self, e))?;
    4526              :             }
    4527              :             // release lock on 'layers'
    4528              :         };
    4529              : 
    4530              :         // Backpressure mechanism: wait with continuation of the flush loop until we have uploaded all layer files.
    4531              :         // This makes us refuse ingest until the new layers have been persisted to the remote
    4532              :         // TODO: remove this, and rely on l0_flush_{delay,stall}_threshold instead.
    4533              :         if self.get_l0_flush_wait_upload() {
    4534              :             let start = Instant::now();
    4535              :             self.remote_client
    4536              :                 .wait_completion()
    4537              :                 .await
    4538            0 :                 .map_err(|e| match e {
    4539              :                     WaitCompletionError::UploadQueueShutDownOrStopped
    4540              :                     | WaitCompletionError::NotInitialized(
    4541              :                         NotInitialized::ShuttingDown | NotInitialized::Stopped,
    4542            0 :                     ) => FlushLayerError::Cancelled,
    4543              :                     WaitCompletionError::NotInitialized(NotInitialized::Uninitialized) => {
    4544            0 :                         FlushLayerError::Other(anyhow!(e).into())
    4545              :                     }
    4546            0 :                 })?;
    4547              :             let duration = start.elapsed().as_secs_f64();
    4548              :             self.metrics.flush_wait_upload_time_gauge_add(duration);
    4549              :         }
    4550              : 
    4551              :         // FIXME: between create_delta_layer and the scheduling of the upload in `update_metadata_file`,
    4552              :         // a compaction can delete the file and then it won't be available for uploads any more.
    4553              :         // We still schedule the upload, resulting in an error, but ideally we'd somehow avoid this
    4554              :         // race situation.
    4555              :         // See https://github.com/neondatabase/neon/issues/4526
    4556              :         pausable_failpoint!("flush-frozen-pausable");
    4557              : 
    4558              :         // This failpoint is used by another test case `test_pageserver_recovery`.
    4559              :         fail_point!("flush-frozen-exit");
    4560              : 
    4561              :         Ok(Lsn(lsn_range.end.0 - 1))
    4562              :     }
    4563              : 
    4564              :     /// Return true if the value changed
    4565              :     ///
    4566              :     /// This function must only be used from the layer flush task.
    4567         2356 :     fn set_disk_consistent_lsn(&self, new_value: Lsn) -> bool {
    4568         2356 :         let old_value = self.disk_consistent_lsn.fetch_max(new_value);
    4569         2356 :         assert!(
    4570         2356 :             new_value >= old_value,
    4571            0 :             "disk_consistent_lsn must be growing monotonously at runtime; current {old_value}, offered {new_value}"
    4572              :         );
    4573              : 
    4574         2356 :         self.metrics
    4575         2356 :             .disk_consistent_lsn_gauge
    4576         2356 :             .set(new_value.0 as i64);
    4577         2356 :         new_value != old_value
    4578         2356 :     }
    4579              : 
    4580              :     /// Update metadata file
    4581         2456 :     fn schedule_uploads(
    4582         2456 :         &self,
    4583         2456 :         disk_consistent_lsn: Lsn,
    4584         2456 :         layers_to_upload: impl IntoIterator<Item = ResidentLayer>,
    4585         2456 :     ) -> anyhow::Result<()> {
    4586         2456 :         // We can only save a valid 'prev_record_lsn' value on disk if we
    4587         2456 :         // flushed *all* in-memory changes to disk. We only track
    4588         2456 :         // 'prev_record_lsn' in memory for the latest processed record, so we
    4589         2456 :         // don't remember what the correct value that corresponds to some old
    4590         2456 :         // LSN is. But if we flush everything, then the value corresponding
    4591         2456 :         // current 'last_record_lsn' is correct and we can store it on disk.
    4592         2456 :         let RecordLsn {
    4593         2456 :             last: last_record_lsn,
    4594         2456 :             prev: prev_record_lsn,
    4595         2456 :         } = self.last_record_lsn.load();
    4596         2456 :         let ondisk_prev_record_lsn = if disk_consistent_lsn == last_record_lsn {
    4597         2201 :             Some(prev_record_lsn)
    4598              :         } else {
    4599          255 :             None
    4600              :         };
    4601              : 
    4602         2456 :         let update = crate::tenant::metadata::MetadataUpdate::new(
    4603         2456 :             disk_consistent_lsn,
    4604         2456 :             ondisk_prev_record_lsn,
    4605         2456 :             *self.applied_gc_cutoff_lsn.read(),
    4606         2456 :         );
    4607         2456 : 
    4608         2456 :         fail_point!("checkpoint-before-saving-metadata", |x| bail!(
    4609            0 :             "{}",
    4610            0 :             x.unwrap()
    4611         2456 :         ));
    4612              : 
    4613         4836 :         for layer in layers_to_upload {
    4614         2380 :             self.remote_client.schedule_layer_file_upload(layer)?;
    4615              :         }
    4616         2456 :         self.remote_client
    4617         2456 :             .schedule_index_upload_for_metadata_update(&update)?;
    4618              : 
    4619         2456 :         Ok(())
    4620         2456 :     }
    4621              : 
    4622            0 :     pub(crate) async fn preserve_initdb_archive(&self) -> anyhow::Result<()> {
    4623            0 :         self.remote_client
    4624            0 :             .preserve_initdb_archive(
    4625            0 :                 &self.tenant_shard_id.tenant_id,
    4626            0 :                 &self.timeline_id,
    4627            0 :                 &self.cancel,
    4628            0 :             )
    4629            0 :             .await
    4630            0 :     }
    4631              : 
    4632              :     // Write out the given frozen in-memory layer as a new L0 delta file. This L0 file will not be tracked
    4633              :     // in layer map immediately. The caller is responsible to put it into the layer map.
    4634         1936 :     async fn create_delta_layer(
    4635         1936 :         self: &Arc<Self>,
    4636         1936 :         frozen_layer: &Arc<InMemoryLayer>,
    4637         1936 :         key_range: Option<Range<Key>>,
    4638         1936 :         ctx: &RequestContext,
    4639         1936 :     ) -> anyhow::Result<Option<ResidentLayer>> {
    4640         1936 :         let self_clone = Arc::clone(self);
    4641         1936 :         let frozen_layer = Arc::clone(frozen_layer);
    4642         1936 :         let ctx = ctx.attached_child();
    4643         1936 :         let work = async move {
    4644         1936 :             let Some((desc, path)) = frozen_layer
    4645         1936 :                 .write_to_disk(&ctx, key_range, self_clone.l0_flush_global_state.inner())
    4646         1936 :                 .await?
    4647              :             else {
    4648            0 :                 return Ok(None);
    4649              :             };
    4650         1936 :             let new_delta = Layer::finish_creating(self_clone.conf, &self_clone, desc, &path)?;
    4651              : 
    4652              :             // The write_to_disk() above calls writer.finish() which already did the fsync of the inodes.
    4653              :             // We just need to fsync the directory in which these inodes are linked,
    4654              :             // which we know to be the timeline directory.
    4655              :             //
    4656              :             // We use fatal_err() below because the after write_to_disk returns with success,
    4657              :             // the in-memory state of the filesystem already has the layer file in its final place,
    4658              :             // and subsequent pageserver code could think it's durable while it really isn't.
    4659         1936 :             let timeline_dir = VirtualFile::open(
    4660         1936 :                 &self_clone
    4661         1936 :                     .conf
    4662         1936 :                     .timeline_path(&self_clone.tenant_shard_id, &self_clone.timeline_id),
    4663         1936 :                 &ctx,
    4664         1936 :             )
    4665         1936 :             .await
    4666         1936 :             .fatal_err("VirtualFile::open for timeline dir fsync");
    4667         1936 :             timeline_dir
    4668         1936 :                 .sync_all()
    4669         1936 :                 .await
    4670         1936 :                 .fatal_err("VirtualFile::sync_all timeline dir");
    4671         1936 :             anyhow::Ok(Some(new_delta))
    4672         1936 :         };
    4673              :         // Before tokio-epoll-uring, we ran write_to_disk & the sync_all inside spawn_blocking.
    4674              :         // Preserve that behavior to maintain the same behavior for `virtual_file_io_engine=std-fs`.
    4675              :         use crate::virtual_file::io_engine::IoEngine;
    4676         1936 :         match crate::virtual_file::io_engine::get() {
    4677            0 :             IoEngine::NotSet => panic!("io engine not set"),
    4678              :             IoEngine::StdFs => {
    4679          968 :                 let span = tracing::info_span!("blocking");
    4680          968 :                 tokio::task::spawn_blocking({
    4681          968 :                     move || Handle::current().block_on(work.instrument(span))
    4682          968 :                 })
    4683          968 :                 .await
    4684          968 :                 .context("spawn_blocking")
    4685          968 :                 .and_then(|x| x)
    4686              :             }
    4687              :             #[cfg(target_os = "linux")]
    4688          968 :             IoEngine::TokioEpollUring => work.await,
    4689              :         }
    4690         1936 :     }
    4691              : 
    4692         1148 :     async fn repartition(
    4693         1148 :         &self,
    4694         1148 :         lsn: Lsn,
    4695         1148 :         partition_size: u64,
    4696         1148 :         flags: EnumSet<CompactFlags>,
    4697         1148 :         ctx: &RequestContext,
    4698         1148 :     ) -> Result<((KeyPartitioning, SparseKeyPartitioning), Lsn), CompactionError> {
    4699         1148 :         let Ok(mut guard) = self.partitioning.try_write_guard() else {
    4700              :             // NB: there are two callers, one is the compaction task, of which there is only one per struct Tenant and hence Timeline.
    4701              :             // The other is the initdb optimization in flush_frozen_layer, used by `boostrap_timeline`, which runs before `.activate()`
    4702              :             // and hence before the compaction task starts.
    4703            0 :             return Err(CompactionError::Other(anyhow!(
    4704            0 :                 "repartition() called concurrently"
    4705            0 :             )));
    4706              :         };
    4707         1148 :         let ((dense_partition, sparse_partition), partition_lsn) = &*guard.read();
    4708         1148 :         if lsn < *partition_lsn {
    4709            0 :             return Err(CompactionError::Other(anyhow!(
    4710            0 :                 "repartition() called with LSN going backwards, this should not happen"
    4711            0 :             )));
    4712         1148 :         }
    4713         1148 : 
    4714         1148 :         let distance = lsn.0 - partition_lsn.0;
    4715         1148 :         if *partition_lsn != Lsn(0)
    4716          524 :             && distance <= self.repartition_threshold
    4717          524 :             && !flags.contains(CompactFlags::ForceRepartition)
    4718              :         {
    4719          496 :             debug!(
    4720              :                 distance,
    4721              :                 threshold = self.repartition_threshold,
    4722            0 :                 "no repartitioning needed"
    4723              :             );
    4724          496 :             return Ok((
    4725          496 :                 (dense_partition.clone(), sparse_partition.clone()),
    4726          496 :                 *partition_lsn,
    4727          496 :             ));
    4728          652 :         }
    4729              : 
    4730          652 :         let (dense_ks, sparse_ks) = self.collect_keyspace(lsn, ctx).await?;
    4731          652 :         let dense_partitioning = dense_ks.partition(&self.shard_identity, partition_size);
    4732          652 :         let sparse_partitioning = SparseKeyPartitioning {
    4733          652 :             parts: vec![sparse_ks],
    4734          652 :         }; // no partitioning for metadata keys for now
    4735          652 :         let result = ((dense_partitioning, sparse_partitioning), lsn);
    4736          652 :         guard.write(result.clone());
    4737          652 :         Ok(result)
    4738         1148 :     }
    4739              : 
    4740              :     // Is it time to create a new image layer for the given partition? True if we want to generate.
    4741           28 :     async fn time_for_new_image_layer(&self, partition: &KeySpace, lsn: Lsn) -> bool {
    4742           28 :         let threshold = self.get_image_creation_threshold();
    4743              : 
    4744           28 :         let guard = self.layers.read().await;
    4745           28 :         let Ok(layers) = guard.layer_map() else {
    4746            0 :             return false;
    4747              :         };
    4748              : 
    4749           28 :         let mut max_deltas = 0;
    4750           56 :         for part_range in &partition.ranges {
    4751           28 :             let image_coverage = layers.image_coverage(part_range, lsn);
    4752           56 :             for (img_range, last_img) in image_coverage {
    4753           28 :                 let img_lsn = if let Some(last_img) = last_img {
    4754            0 :                     last_img.get_lsn_range().end
    4755              :                 } else {
    4756           28 :                     Lsn(0)
    4757              :                 };
    4758              :                 // Let's consider an example:
    4759              :                 //
    4760              :                 // delta layer with LSN range 71-81
    4761              :                 // delta layer with LSN range 81-91
    4762              :                 // delta layer with LSN range 91-101
    4763              :                 // image layer at LSN 100
    4764              :                 //
    4765              :                 // If 'lsn' is still 100, i.e. no new WAL has been processed since the last image layer,
    4766              :                 // there's no need to create a new one. We check this case explicitly, to avoid passing
    4767              :                 // a bogus range to count_deltas below, with start > end. It's even possible that there
    4768              :                 // are some delta layers *later* than current 'lsn', if more WAL was processed and flushed
    4769              :                 // after we read last_record_lsn, which is passed here in the 'lsn' argument.
    4770           28 :                 if img_lsn < lsn {
    4771           28 :                     let num_deltas =
    4772           28 :                         layers.count_deltas(&img_range, &(img_lsn..lsn), Some(threshold));
    4773           28 : 
    4774           28 :                     max_deltas = max_deltas.max(num_deltas);
    4775           28 :                     if num_deltas >= threshold {
    4776            0 :                         debug!(
    4777            0 :                             "key range {}-{}, has {} deltas on this timeline in LSN range {}..{}",
    4778              :                             img_range.start, img_range.end, num_deltas, img_lsn, lsn
    4779              :                         );
    4780            0 :                         return true;
    4781           28 :                     }
    4782            0 :                 }
    4783              :             }
    4784              :         }
    4785              : 
    4786           28 :         debug!(
    4787              :             max_deltas,
    4788            0 :             "none of the partitioned ranges had >= {threshold} deltas"
    4789              :         );
    4790           28 :         false
    4791           28 :     }
    4792              : 
    4793              :     /// Create image layers for Postgres data. Assumes the caller passes a partition that is not too large,
    4794              :     /// so that at most one image layer will be produced from this function.
    4795              :     #[allow(clippy::too_many_arguments)]
    4796          472 :     async fn create_image_layer_for_rel_blocks(
    4797          472 :         self: &Arc<Self>,
    4798          472 :         partition: &KeySpace,
    4799          472 :         mut image_layer_writer: ImageLayerWriter,
    4800          472 :         lsn: Lsn,
    4801          472 :         ctx: &RequestContext,
    4802          472 :         img_range: Range<Key>,
    4803          472 :         io_concurrency: IoConcurrency,
    4804          472 :     ) -> Result<ImageLayerCreationOutcome, CreateImageLayersError> {
    4805          472 :         let mut wrote_keys = false;
    4806          472 : 
    4807          472 :         let mut key_request_accum = KeySpaceAccum::new();
    4808         3128 :         for range in &partition.ranges {
    4809         2656 :             let mut key = range.start;
    4810         5756 :             while key < range.end {
    4811              :                 // Decide whether to retain this key: usually we do, but sharded tenants may
    4812              :                 // need to drop keys that don't belong to them.  If we retain the key, add it
    4813              :                 // to `key_request_accum` for later issuing a vectored get
    4814         3100 :                 if self.shard_identity.is_key_disposable(&key) {
    4815            0 :                     debug!(
    4816            0 :                         "Dropping key {} during compaction (it belongs on shard {:?})",
    4817            0 :                         key,
    4818            0 :                         self.shard_identity.get_shard_number(&key)
    4819              :                     );
    4820         3100 :                 } else {
    4821         3100 :                     key_request_accum.add_key(key);
    4822         3100 :                 }
    4823              : 
    4824         3100 :                 let last_key_in_range = key.next() == range.end;
    4825         3100 :                 key = key.next();
    4826         3100 : 
    4827         3100 :                 // Maybe flush `key_rest_accum`
    4828         3100 :                 if key_request_accum.raw_size() >= Timeline::MAX_GET_VECTORED_KEYS
    4829         3100 :                     || (last_key_in_range && key_request_accum.raw_size() > 0)
    4830              :                 {
    4831         2656 :                     let results = self
    4832         2656 :                         .get_vectored(
    4833         2656 :                             key_request_accum.consume_keyspace(),
    4834         2656 :                             lsn,
    4835         2656 :                             io_concurrency.clone(),
    4836         2656 :                             ctx,
    4837         2656 :                         )
    4838         2656 :                         .await?;
    4839              : 
    4840         2656 :                     if self.cancel.is_cancelled() {
    4841            0 :                         return Err(CreateImageLayersError::Cancelled);
    4842         2656 :                     }
    4843              : 
    4844         5756 :                     for (img_key, img) in results {
    4845         3100 :                         let img = match img {
    4846         3100 :                             Ok(img) => img,
    4847            0 :                             Err(err) => {
    4848            0 :                                 // If we fail to reconstruct a VM or FSM page, we can zero the
    4849            0 :                                 // page without losing any actual user data. That seems better
    4850            0 :                                 // than failing repeatedly and getting stuck.
    4851            0 :                                 //
    4852            0 :                                 // We had a bug at one point, where we truncated the FSM and VM
    4853            0 :                                 // in the pageserver, but the Postgres didn't know about that
    4854            0 :                                 // and continued to generate incremental WAL records for pages
    4855            0 :                                 // that didn't exist in the pageserver. Trying to replay those
    4856            0 :                                 // WAL records failed to find the previous image of the page.
    4857            0 :                                 // This special case allows us to recover from that situation.
    4858            0 :                                 // See https://github.com/neondatabase/neon/issues/2601.
    4859            0 :                                 //
    4860            0 :                                 // Unfortunately we cannot do this for the main fork, or for
    4861            0 :                                 // any metadata keys, keys, as that would lead to actual data
    4862            0 :                                 // loss.
    4863            0 :                                 if img_key.is_rel_fsm_block_key() || img_key.is_rel_vm_block_key() {
    4864            0 :                                     warn!(
    4865            0 :                                         "could not reconstruct FSM or VM key {img_key}, filling with zeros: {err:?}"
    4866              :                                     );
    4867            0 :                                     ZERO_PAGE.clone()
    4868              :                                 } else {
    4869            0 :                                     return Err(CreateImageLayersError::from(err));
    4870              :                                 }
    4871              :                             }
    4872              :                         };
    4873              : 
    4874              :                         // Write all the keys we just read into our new image layer.
    4875         3100 :                         image_layer_writer.put_image(img_key, img, ctx).await?;
    4876         3100 :                         wrote_keys = true;
    4877              :                     }
    4878          444 :                 }
    4879              :             }
    4880              :         }
    4881              : 
    4882          472 :         if wrote_keys {
    4883              :             // Normal path: we have written some data into the new image layer for this
    4884              :             // partition, so flush it to disk.
    4885          472 :             info!(
    4886            0 :                 "produced image layer for rel {}",
    4887            0 :                 ImageLayerName {
    4888            0 :                     key_range: img_range.clone(),
    4889            0 :                     lsn
    4890            0 :                 },
    4891              :             );
    4892          472 :             Ok(ImageLayerCreationOutcome::Generated {
    4893          472 :                 unfinished_image_layer: image_layer_writer,
    4894          472 :             })
    4895              :         } else {
    4896            0 :             tracing::debug!("no data in range {}-{}", img_range.start, img_range.end);
    4897            0 :             Ok(ImageLayerCreationOutcome::Empty)
    4898              :         }
    4899          472 :     }
    4900              : 
    4901              :     /// Create an image layer for metadata keys. This function produces one image layer for all metadata
    4902              :     /// keys for now. Because metadata keys cannot exceed basebackup size limit, the image layer for it
    4903              :     /// would not be too large to fit in a single image layer.
    4904              :     ///
    4905              :     /// Creating image layers for metadata keys are different from relational keys. Firstly, instead of
    4906              :     /// iterating each key and get an image for each of them, we do a `vectored_get` scan over the sparse
    4907              :     /// keyspace to get all images in one run. Secondly, we use a different image layer generation metrics
    4908              :     /// for metadata keys than relational keys, which is the number of delta files visited during the scan.
    4909              :     #[allow(clippy::too_many_arguments)]
    4910          452 :     async fn create_image_layer_for_metadata_keys(
    4911          452 :         self: &Arc<Self>,
    4912          452 :         partition: &KeySpace,
    4913          452 :         mut image_layer_writer: ImageLayerWriter,
    4914          452 :         lsn: Lsn,
    4915          452 :         ctx: &RequestContext,
    4916          452 :         img_range: Range<Key>,
    4917          452 :         mode: ImageLayerCreationMode,
    4918          452 :         io_concurrency: IoConcurrency,
    4919          452 :     ) -> Result<ImageLayerCreationOutcome, CreateImageLayersError> {
    4920          452 :         // Metadata keys image layer creation.
    4921          452 :         let mut reconstruct_state = ValuesReconstructState::new(io_concurrency);
    4922          452 :         let begin = Instant::now();
    4923              :         // Directly use `get_vectored_impl` to skip the max_vectored_read_key limit check. Note that the keyspace should
    4924              :         // not contain too many keys, otherwise this takes a lot of memory.
    4925          452 :         let data = self
    4926          452 :             .get_vectored_impl(partition.clone(), lsn, &mut reconstruct_state, ctx)
    4927          452 :             .await?;
    4928          452 :         let (data, total_kb_retrieved, total_keys_retrieved) = {
    4929          452 :             let mut new_data = BTreeMap::new();
    4930          452 :             let mut total_kb_retrieved = 0;
    4931          452 :             let mut total_keys_retrieved = 0;
    4932        20476 :             for (k, v) in data {
    4933        20024 :                 let v = v?;
    4934        20024 :                 total_kb_retrieved += KEY_SIZE + v.len();
    4935        20024 :                 total_keys_retrieved += 1;
    4936        20024 :                 new_data.insert(k, v);
    4937              :             }
    4938          452 :             (new_data, total_kb_retrieved / 1024, total_keys_retrieved)
    4939          452 :         };
    4940          452 :         let delta_files_accessed = reconstruct_state.get_delta_layers_visited();
    4941          452 :         let elapsed = begin.elapsed();
    4942          452 : 
    4943          452 :         let trigger_generation = delta_files_accessed as usize >= MAX_AUX_FILE_V2_DELTAS;
    4944          452 :         info!(
    4945            0 :             "metadata key compaction: trigger_generation={trigger_generation}, delta_files_accessed={delta_files_accessed}, total_kb_retrieved={total_kb_retrieved}, total_keys_retrieved={total_keys_retrieved}, read_time={}s",
    4946            0 :             elapsed.as_secs_f64()
    4947              :         );
    4948              : 
    4949          452 :         if !trigger_generation && mode == ImageLayerCreationMode::Try {
    4950            4 :             return Ok(ImageLayerCreationOutcome::Skip);
    4951          448 :         }
    4952          448 :         if self.cancel.is_cancelled() {
    4953            0 :             return Err(CreateImageLayersError::Cancelled);
    4954          448 :         }
    4955          448 :         let mut wrote_any_image = false;
    4956        20472 :         for (k, v) in data {
    4957        20024 :             if v.is_empty() {
    4958              :                 // the key has been deleted, it does not need an image
    4959              :                 // in metadata keyspace, an empty image == tombstone
    4960           16 :                 continue;
    4961        20008 :             }
    4962        20008 :             wrote_any_image = true;
    4963        20008 : 
    4964        20008 :             // No need to handle sharding b/c metadata keys are always on the 0-th shard.
    4965        20008 : 
    4966        20008 :             // TODO: split image layers to avoid too large layer files. Too large image files are not handled
    4967        20008 :             // on the normal data path either.
    4968        20008 :             image_layer_writer.put_image(k, v, ctx).await?;
    4969              :         }
    4970              : 
    4971          448 :         if wrote_any_image {
    4972              :             // Normal path: we have written some data into the new image layer for this
    4973              :             // partition, so flush it to disk.
    4974           24 :             info!(
    4975            0 :                 "created image layer for metadata {}",
    4976            0 :                 ImageLayerName {
    4977            0 :                     key_range: img_range.clone(),
    4978            0 :                     lsn
    4979            0 :                 }
    4980              :             );
    4981           24 :             Ok(ImageLayerCreationOutcome::Generated {
    4982           24 :                 unfinished_image_layer: image_layer_writer,
    4983           24 :             })
    4984              :         } else {
    4985          424 :             tracing::debug!("no data in range {}-{}", img_range.start, img_range.end);
    4986          424 :             Ok(ImageLayerCreationOutcome::Empty)
    4987              :         }
    4988          452 :     }
    4989              : 
    4990              :     /// Predicate function which indicates whether we should check if new image layers
    4991              :     /// are required. Since checking if new image layers are required is expensive in
    4992              :     /// terms of CPU, we only do it in the following cases:
    4993              :     /// 1. If the timeline has ingested sufficient WAL to justify the cost
    4994              :     /// 2. If enough time has passed since the last check:
    4995              :     ///     1. For large tenants, we wish to perform the check more often since they
    4996              :     ///        suffer from the lack of image layers
    4997              :     ///     2. For small tenants (that can mostly fit in RAM), we use a much longer interval
    4998         1148 :     fn should_check_if_image_layers_required(self: &Arc<Timeline>, lsn: Lsn) -> bool {
    4999              :         const LARGE_TENANT_THRESHOLD: u64 = 2 * 1024 * 1024 * 1024;
    5000              : 
    5001         1148 :         let last_checks_at = self.last_image_layer_creation_check_at.load();
    5002         1148 :         let distance = lsn
    5003         1148 :             .checked_sub(last_checks_at)
    5004         1148 :             .expect("Attempt to compact with LSN going backwards");
    5005         1148 :         let min_distance =
    5006         1148 :             self.get_image_layer_creation_check_threshold() as u64 * self.get_checkpoint_distance();
    5007         1148 : 
    5008         1148 :         let distance_based_decision = distance.0 >= min_distance;
    5009         1148 : 
    5010         1148 :         let mut time_based_decision = false;
    5011         1148 :         let mut last_check_instant = self.last_image_layer_creation_check_instant.lock().unwrap();
    5012         1148 :         if let CurrentLogicalSize::Exact(logical_size) = self.current_logical_size.current_size() {
    5013          944 :             let check_required_after = if Into::<u64>::into(&logical_size) >= LARGE_TENANT_THRESHOLD
    5014              :             {
    5015            0 :                 self.get_checkpoint_timeout()
    5016              :             } else {
    5017          944 :                 Duration::from_secs(3600 * 48)
    5018              :             };
    5019              : 
    5020          944 :             time_based_decision = match *last_check_instant {
    5021          524 :                 Some(last_check) => {
    5022          524 :                     let elapsed = last_check.elapsed();
    5023          524 :                     elapsed >= check_required_after
    5024              :                 }
    5025          420 :                 None => true,
    5026              :             };
    5027          204 :         }
    5028              : 
    5029              :         // Do the expensive delta layer counting only if this timeline has ingested sufficient
    5030              :         // WAL since the last check or a checkpoint timeout interval has elapsed since the last
    5031              :         // check.
    5032         1148 :         let decision = distance_based_decision || time_based_decision;
    5033              : 
    5034         1148 :         if decision {
    5035          424 :             self.last_image_layer_creation_check_at.store(lsn);
    5036          424 :             *last_check_instant = Some(Instant::now());
    5037          724 :         }
    5038              : 
    5039         1148 :         decision
    5040         1148 :     }
    5041              : 
    5042              :     /// Returns the image layers generated and an enum indicating whether the process is fully completed.
    5043              :     /// true = we have generate all image layers, false = we preempt the process for L0 compaction.
    5044              :     #[tracing::instrument(skip_all, fields(%lsn, %mode))]
    5045              :     async fn create_image_layers(
    5046              :         self: &Arc<Timeline>,
    5047              :         partitioning: &KeyPartitioning,
    5048              :         lsn: Lsn,
    5049              :         mode: ImageLayerCreationMode,
    5050              :         ctx: &RequestContext,
    5051              :         last_status: LastImageLayerCreationStatus,
    5052              :         yield_for_l0: bool,
    5053              :     ) -> Result<(Vec<ResidentLayer>, LastImageLayerCreationStatus), CreateImageLayersError> {
    5054              :         let timer = self.metrics.create_images_time_histo.start_timer();
    5055              : 
    5056              :         if partitioning.parts.is_empty() {
    5057              :             warn!("no partitions to create image layers for");
    5058              :             return Ok((vec![], LastImageLayerCreationStatus::Complete));
    5059              :         }
    5060              : 
    5061              :         // We need to avoid holes between generated image layers.
    5062              :         // Otherwise LayerMap::image_layer_exists will return false if key range of some layer is covered by more than one
    5063              :         // image layer with hole between them. In this case such layer can not be utilized by GC.
    5064              :         //
    5065              :         // How such hole between partitions can appear?
    5066              :         // if we have relation with relid=1 and size 100 and relation with relid=2 with size 200 then result of
    5067              :         // KeySpace::partition may contain partitions <100000000..100000099> and <200000000..200000199>.
    5068              :         // If there is delta layer <100000000..300000000> then it never be garbage collected because
    5069              :         // image layers  <100000000..100000099> and <200000000..200000199> are not completely covering it.
    5070              :         let mut start = Key::MIN;
    5071              : 
    5072              :         let check_for_image_layers =
    5073              :             if let LastImageLayerCreationStatus::Incomplete { last_key } = last_status {
    5074              :                 info!(
    5075              :                     "resuming image layer creation: last_status=incomplete, continue from {}",
    5076              :                     last_key
    5077              :                 );
    5078              :                 true
    5079              :             } else {
    5080              :                 self.should_check_if_image_layers_required(lsn)
    5081              :             };
    5082              : 
    5083              :         let mut batch_image_writer = BatchLayerWriter::new(self.conf).await?;
    5084              : 
    5085              :         let mut all_generated = true;
    5086              : 
    5087              :         let mut partition_processed = 0;
    5088              :         let mut total_partitions = partitioning.parts.len();
    5089              :         let mut last_partition_processed = None;
    5090              :         let mut partition_parts = partitioning.parts.clone();
    5091              : 
    5092              :         if let LastImageLayerCreationStatus::Incomplete { last_key } = last_status {
    5093              :             // We need to skip the partitions that have already been processed.
    5094              :             let mut found = false;
    5095              :             for (i, partition) in partition_parts.iter().enumerate() {
    5096              :                 if last_key <= partition.end().unwrap() {
    5097              :                     // ```plain
    5098              :                     // |------|--------|----------|------|
    5099              :                     //              ^last_key
    5100              :                     //                    ^start from this partition
    5101              :                     // ```
    5102              :                     // Why `i+1` instead of `i`?
    5103              :                     // It is possible that the user did some writes after the previous image layer creation attempt so that
    5104              :                     // a relation grows in size, and the last_key is now in the middle of the partition. In this case, we
    5105              :                     // still want to skip this partition, so that we can make progress and avoid generating image layers over
    5106              :                     // the same partition. Doing a mod to ensure we don't end up with an empty vec.
    5107              :                     if i + 1 >= total_partitions {
    5108              :                         // In general, this case should not happen -- if last_key is on the last partition, the previous
    5109              :                         // iteration of image layer creation should return a complete status.
    5110              :                         break; // with found=false
    5111              :                     }
    5112              :                     partition_parts = partition_parts.split_off(i + 1); // Remove the first i + 1 elements
    5113              :                     total_partitions = partition_parts.len();
    5114              :                     // Update the start key to the partition start.
    5115              :                     start = partition_parts[0].start().unwrap();
    5116              :                     found = true;
    5117              :                     break;
    5118              :                 }
    5119              :             }
    5120              :             if !found {
    5121              :                 // Last key is within the last partition, or larger than all partitions.
    5122              :                 return Ok((vec![], LastImageLayerCreationStatus::Complete));
    5123              :             }
    5124              :         }
    5125              : 
    5126              :         for partition in partition_parts.iter() {
    5127              :             if self.cancel.is_cancelled() {
    5128              :                 return Err(CreateImageLayersError::Cancelled);
    5129              :             }
    5130              :             partition_processed += 1;
    5131              :             let img_range = start..partition.ranges.last().unwrap().end;
    5132              :             let compact_metadata = partition.overlaps(&Key::metadata_key_range());
    5133              :             if compact_metadata {
    5134              :                 for range in &partition.ranges {
    5135              :                     assert!(
    5136              :                         range.start.field1 >= METADATA_KEY_BEGIN_PREFIX
    5137              :                             && range.end.field1 <= METADATA_KEY_END_PREFIX,
    5138              :                         "metadata keys must be partitioned separately"
    5139              :                     );
    5140              :                 }
    5141              :                 if mode == ImageLayerCreationMode::Try && !check_for_image_layers {
    5142              :                     // Skip compaction if there are not enough updates. Metadata compaction will do a scan and
    5143              :                     // might mess up with evictions.
    5144              :                     start = img_range.end;
    5145              :                     continue;
    5146              :                 }
    5147              :                 // For initial and force modes, we always generate image layers for metadata keys.
    5148              :             } else if let ImageLayerCreationMode::Try = mode {
    5149              :                 // check_for_image_layers = false -> skip
    5150              :                 // check_for_image_layers = true -> check time_for_new_image_layer -> skip/generate
    5151              :                 if !check_for_image_layers || !self.time_for_new_image_layer(partition, lsn).await {
    5152              :                     start = img_range.end;
    5153              :                     continue;
    5154              :                 }
    5155              :             }
    5156              :             if let ImageLayerCreationMode::Force = mode {
    5157              :                 // When forced to create image layers, we might try and create them where they already
    5158              :                 // exist.  This mode is only used in tests/debug.
    5159              :                 let layers = self.layers.read().await;
    5160              :                 if layers.contains_key(&PersistentLayerKey {
    5161              :                     key_range: img_range.clone(),
    5162              :                     lsn_range: PersistentLayerDesc::image_layer_lsn_range(lsn),
    5163              :                     is_delta: false,
    5164              :                 }) {
    5165              :                     // TODO: this can be processed with the BatchLayerWriter::finish_with_discard
    5166              :                     // in the future.
    5167              :                     tracing::info!(
    5168              :                         "Skipping image layer at {lsn} {}..{}, already exists",
    5169              :                         img_range.start,
    5170              :                         img_range.end
    5171              :                     );
    5172              :                     start = img_range.end;
    5173              :                     continue;
    5174              :                 }
    5175              :             }
    5176              : 
    5177              :             let image_layer_writer = ImageLayerWriter::new(
    5178              :                 self.conf,
    5179              :                 self.timeline_id,
    5180              :                 self.tenant_shard_id,
    5181              :                 &img_range,
    5182              :                 lsn,
    5183              :                 ctx,
    5184              :             )
    5185              :             .await?;
    5186              : 
    5187            0 :             fail_point!("image-layer-writer-fail-before-finish", |_| {
    5188            0 :                 Err(CreateImageLayersError::Other(anyhow::anyhow!(
    5189            0 :                     "failpoint image-layer-writer-fail-before-finish"
    5190            0 :                 )))
    5191            0 :             });
    5192              : 
    5193              :             let io_concurrency = IoConcurrency::spawn_from_conf(
    5194              :                 self.conf,
    5195              :                 self.gate
    5196              :                     .enter()
    5197            0 :                     .map_err(|_| CreateImageLayersError::Cancelled)?,
    5198              :             );
    5199              : 
    5200              :             let outcome = if !compact_metadata {
    5201              :                 self.create_image_layer_for_rel_blocks(
    5202              :                     partition,
    5203              :                     image_layer_writer,
    5204              :                     lsn,
    5205              :                     ctx,
    5206              :                     img_range.clone(),
    5207              :                     io_concurrency,
    5208              :                 )
    5209              :                 .await?
    5210              :             } else {
    5211              :                 self.create_image_layer_for_metadata_keys(
    5212              :                     partition,
    5213              :                     image_layer_writer,
    5214              :                     lsn,
    5215              :                     ctx,
    5216              :                     img_range.clone(),
    5217              :                     mode,
    5218              :                     io_concurrency,
    5219              :                 )
    5220              :                 .await?
    5221              :             };
    5222              :             match outcome {
    5223              :                 ImageLayerCreationOutcome::Empty => {
    5224              :                     // No data in this partition, so we don't need to create an image layer (for now).
    5225              :                     // The next image layer should cover this key range, so we don't advance the `start`
    5226              :                     // key.
    5227              :                 }
    5228              :                 ImageLayerCreationOutcome::Generated {
    5229              :                     unfinished_image_layer,
    5230              :                 } => {
    5231              :                     batch_image_writer.add_unfinished_image_writer(
    5232              :                         unfinished_image_layer,
    5233              :                         img_range.clone(),
    5234              :                         lsn,
    5235              :                     );
    5236              :                     // The next image layer should be generated right after this one.
    5237              :                     start = img_range.end;
    5238              :                 }
    5239              :                 ImageLayerCreationOutcome::Skip => {
    5240              :                     // We don't need to create an image layer for this partition.
    5241              :                     // The next image layer should NOT cover this range, otherwise
    5242              :                     // the keyspace becomes empty (reads don't go past image layers).
    5243              :                     start = img_range.end;
    5244              :                 }
    5245              :             }
    5246              : 
    5247              :             if let ImageLayerCreationMode::Try = mode {
    5248              :                 // We have at least made some progress
    5249              :                 if yield_for_l0 && batch_image_writer.pending_layer_num() >= 1 {
    5250              :                     // The `Try` mode is currently only used on the compaction path. We want to avoid
    5251              :                     // image layer generation taking too long time and blocking L0 compaction. So in this
    5252              :                     // mode, we also inspect the current number of L0 layers and skip image layer generation
    5253              :                     // if there are too many of them.
    5254              :                     let image_preempt_threshold = self.get_image_creation_preempt_threshold()
    5255              :                         * self.get_compaction_threshold();
    5256              :                     // TODO: currently we do not respect `get_image_creation_preempt_threshold` and always yield
    5257              :                     // when there is a single timeline with more than L0 threshold L0 layers. As long as the
    5258              :                     // `get_image_creation_preempt_threshold` is set to a value greater than 0, we will yield for L0 compaction.
    5259              :                     if image_preempt_threshold != 0 {
    5260              :                         let should_yield = self
    5261              :                             .l0_compaction_trigger
    5262              :                             .notified()
    5263              :                             .now_or_never()
    5264              :                             .is_some();
    5265              :                         if should_yield {
    5266              :                             tracing::info!(
    5267              :                                 "preempt image layer generation at {lsn} when processing partition {}..{}: too many L0 layers",
    5268              :                                 partition.start().unwrap(),
    5269              :                                 partition.end().unwrap()
    5270              :                             );
    5271              :                             last_partition_processed = Some(partition.clone());
    5272              :                             all_generated = false;
    5273              :                             break;
    5274              :                         }
    5275              :                     }
    5276              :                 }
    5277              :             }
    5278              :         }
    5279              : 
    5280              :         let image_layers = batch_image_writer.finish(self, ctx).await?;
    5281              : 
    5282              :         let mut guard = self.layers.write().await;
    5283              : 
    5284              :         // FIXME: we could add the images to be uploaded *before* returning from here, but right
    5285              :         // now they are being scheduled outside of write lock; current way is inconsistent with
    5286              :         // compaction lock order.
    5287              :         guard
    5288              :             .open_mut()?
    5289              :             .track_new_image_layers(&image_layers, &self.metrics);
    5290              :         drop_wlock(guard);
    5291              :         let duration = timer.stop_and_record();
    5292              : 
    5293              :         // Creating image layers may have caused some previously visible layers to be covered
    5294              :         if !image_layers.is_empty() {
    5295              :             self.update_layer_visibility().await?;
    5296              :         }
    5297              : 
    5298              :         let total_layer_size = image_layers
    5299              :             .iter()
    5300          496 :             .map(|l| l.metadata().file_size)
    5301              :             .sum::<u64>();
    5302              : 
    5303              :         if !image_layers.is_empty() {
    5304              :             info!(
    5305              :                 "created {} image layers ({} bytes) in {}s, processed {} out of {} partitions",
    5306              :                 image_layers.len(),
    5307              :                 total_layer_size,
    5308              :                 duration.as_secs_f64(),
    5309              :                 partition_processed,
    5310              :                 total_partitions
    5311              :             );
    5312              :         }
    5313              : 
    5314              :         Ok((
    5315              :             image_layers,
    5316              :             if all_generated {
    5317              :                 LastImageLayerCreationStatus::Complete
    5318              :             } else {
    5319              :                 LastImageLayerCreationStatus::Incomplete {
    5320              :                     last_key: if let Some(last_partition_processed) = last_partition_processed {
    5321              :                         last_partition_processed.end().unwrap_or(Key::MIN)
    5322              :                     } else {
    5323              :                         // This branch should be unreachable, but in case it happens, we can just return the start key.
    5324              :                         Key::MIN
    5325              :                     },
    5326              :                 }
    5327              :             },
    5328              :         ))
    5329              :     }
    5330              : 
    5331              :     /// Wait until the background initial logical size calculation is complete, or
    5332              :     /// this Timeline is shut down.  Calling this function will cause the initial
    5333              :     /// logical size calculation to skip waiting for the background jobs barrier.
    5334            0 :     pub(crate) async fn await_initial_logical_size(self: Arc<Self>) {
    5335            0 :         if !self.shard_identity.is_shard_zero() {
    5336              :             // We don't populate logical size on shard >0: skip waiting for it.
    5337            0 :             return;
    5338            0 :         }
    5339            0 : 
    5340            0 :         if self.remote_client.is_deleting() {
    5341              :             // The timeline was created in a deletion-resume state, we don't expect logical size to be populated
    5342            0 :             return;
    5343            0 :         }
    5344            0 : 
    5345            0 :         if self.current_logical_size.current_size().is_exact() {
    5346              :             // root timelines are initialized with exact count, but never start the background
    5347              :             // calculation
    5348            0 :             return;
    5349            0 :         }
    5350              : 
    5351            0 :         if let Some(await_bg_cancel) = self
    5352            0 :             .current_logical_size
    5353            0 :             .cancel_wait_for_background_loop_concurrency_limit_semaphore
    5354            0 :             .get()
    5355            0 :         {
    5356            0 :             await_bg_cancel.cancel();
    5357            0 :         } else {
    5358              :             // We should not wait if we were not able to explicitly instruct
    5359              :             // the logical size cancellation to skip the concurrency limit semaphore.
    5360              :             // TODO: this is an unexpected case.  We should restructure so that it
    5361              :             // can't happen.
    5362            0 :             tracing::warn!(
    5363            0 :                 "await_initial_logical_size: can't get semaphore cancel token, skipping"
    5364              :             );
    5365            0 :             debug_assert!(false);
    5366              :         }
    5367              : 
    5368            0 :         tokio::select!(
    5369            0 :             _ = self.current_logical_size.initialized.acquire() => {},
    5370            0 :             _ = self.cancel.cancelled() => {}
    5371              :         )
    5372            0 :     }
    5373              : 
    5374              :     /// Detach this timeline from its ancestor by copying all of ancestors layers as this
    5375              :     /// Timelines layers up to the ancestor_lsn.
    5376              :     ///
    5377              :     /// Requires a timeline that:
    5378              :     /// - has an ancestor to detach from
    5379              :     /// - the ancestor does not have an ancestor -- follows from the original RFC limitations, not
    5380              :     ///   a technical requirement
    5381              :     ///
    5382              :     /// After the operation has been started, it cannot be canceled. Upon restart it needs to be
    5383              :     /// polled again until completion.
    5384              :     ///
    5385              :     /// During the operation all timelines sharing the data with this timeline will be reparented
    5386              :     /// from our ancestor to be branches of this timeline.
    5387            0 :     pub(crate) async fn prepare_to_detach_from_ancestor(
    5388            0 :         self: &Arc<Timeline>,
    5389            0 :         tenant: &crate::tenant::Tenant,
    5390            0 :         options: detach_ancestor::Options,
    5391            0 :         ctx: &RequestContext,
    5392            0 :     ) -> Result<detach_ancestor::Progress, detach_ancestor::Error> {
    5393            0 :         detach_ancestor::prepare(self, tenant, options, ctx).await
    5394            0 :     }
    5395              : 
    5396              :     /// Second step of detach from ancestor; detaches the `self` from it's current ancestor and
    5397              :     /// reparents any reparentable children of previous ancestor.
    5398              :     ///
    5399              :     /// This method is to be called while holding the TenantManager's tenant slot, so during this
    5400              :     /// method we cannot be deleted nor can any timeline be deleted. After this method returns
    5401              :     /// successfully, tenant must be reloaded.
    5402              :     ///
    5403              :     /// Final step will be to [`Self::complete_detaching_timeline_ancestor`] after optionally
    5404              :     /// resetting the tenant.
    5405            0 :     pub(crate) async fn detach_from_ancestor_and_reparent(
    5406            0 :         self: &Arc<Timeline>,
    5407            0 :         tenant: &crate::tenant::Tenant,
    5408            0 :         prepared: detach_ancestor::PreparedTimelineDetach,
    5409            0 :         ctx: &RequestContext,
    5410            0 :     ) -> Result<detach_ancestor::DetachingAndReparenting, detach_ancestor::Error> {
    5411            0 :         detach_ancestor::detach_and_reparent(self, tenant, prepared, ctx).await
    5412            0 :     }
    5413              : 
    5414              :     /// Final step which unblocks the GC.
    5415              :     ///
    5416              :     /// The tenant must've been reset if ancestry was modified previously (in tenant manager).
    5417            0 :     pub(crate) async fn complete_detaching_timeline_ancestor(
    5418            0 :         self: &Arc<Timeline>,
    5419            0 :         tenant: &crate::tenant::Tenant,
    5420            0 :         attempt: detach_ancestor::Attempt,
    5421            0 :         ctx: &RequestContext,
    5422            0 :     ) -> Result<(), detach_ancestor::Error> {
    5423            0 :         detach_ancestor::complete(self, tenant, attempt, ctx).await
    5424            0 :     }
    5425              : }
    5426              : 
    5427              : impl Drop for Timeline {
    5428           20 :     fn drop(&mut self) {
    5429           20 :         if let Some(ancestor) = &self.ancestor_timeline {
    5430              :             // This lock should never be poisoned, but in case it is we do a .map() instead of
    5431              :             // an unwrap(), to avoid panicking in a destructor and thereby aborting the process.
    5432            8 :             if let Ok(mut gc_info) = ancestor.gc_info.write() {
    5433            8 :                 if !gc_info.remove_child_not_offloaded(self.timeline_id) {
    5434            0 :                     tracing::error!(tenant_id = %self.tenant_shard_id.tenant_id, shard_id = %self.tenant_shard_id.shard_slug(), timeline_id = %self.timeline_id,
    5435            0 :                         "Couldn't remove retain_lsn entry from offloaded timeline's parent: already removed");
    5436            8 :                 }
    5437            0 :             }
    5438           12 :         }
    5439           20 :         info!(
    5440            0 :             "Timeline {} for tenant {} is being dropped",
    5441              :             self.timeline_id, self.tenant_shard_id.tenant_id
    5442              :         );
    5443           20 :     }
    5444              : }
    5445              : 
    5446              : /// Top-level failure to compact.
    5447              : #[derive(Debug, thiserror::Error)]
    5448              : pub(crate) enum CompactionError {
    5449              :     #[error("The timeline or pageserver is shutting down")]
    5450              :     ShuttingDown,
    5451              :     /// Compaction tried to offload a timeline and failed
    5452              :     #[error("Failed to offload timeline: {0}")]
    5453              :     Offload(OffloadError),
    5454              :     /// Compaction cannot be done right now; page reconstruction and so on.
    5455              :     #[error("Failed to collect keyspace: {0}")]
    5456              :     CollectKeySpaceError(#[from] CollectKeySpaceError),
    5457              :     #[error(transparent)]
    5458              :     Other(anyhow::Error),
    5459              :     #[error("Compaction already running: {0}")]
    5460              :     AlreadyRunning(&'static str),
    5461              : }
    5462              : 
    5463              : impl CompactionError {
    5464              :     /// Errors that can be ignored, i.e., cancel and shutdown.
    5465            0 :     pub fn is_cancel(&self) -> bool {
    5466            0 :         matches!(
    5467            0 :             self,
    5468              :             Self::ShuttingDown
    5469              :                 | Self::AlreadyRunning(_)
    5470              :                 | Self::CollectKeySpaceError(CollectKeySpaceError::Cancelled)
    5471              :                 | Self::CollectKeySpaceError(CollectKeySpaceError::PageRead(
    5472              :                     PageReconstructError::Cancelled
    5473              :                 ))
    5474              :                 | Self::Offload(OffloadError::Cancelled)
    5475              :         )
    5476            0 :     }
    5477              : 
    5478              :     /// Critical errors that indicate data corruption.
    5479            0 :     pub fn is_critical(&self) -> bool {
    5480            0 :         matches!(
    5481            0 :             self,
    5482              :             Self::CollectKeySpaceError(
    5483              :                 CollectKeySpaceError::Decode(_)
    5484              :                     | CollectKeySpaceError::PageRead(
    5485              :                         PageReconstructError::MissingKey(_) | PageReconstructError::WalRedo(_),
    5486              :                     )
    5487              :             )
    5488              :         )
    5489            0 :     }
    5490              : }
    5491              : 
    5492              : impl From<OffloadError> for CompactionError {
    5493            0 :     fn from(e: OffloadError) -> Self {
    5494            0 :         match e {
    5495            0 :             OffloadError::Cancelled => Self::ShuttingDown,
    5496            0 :             _ => Self::Offload(e),
    5497              :         }
    5498            0 :     }
    5499              : }
    5500              : 
    5501              : impl From<super::upload_queue::NotInitialized> for CompactionError {
    5502            0 :     fn from(value: super::upload_queue::NotInitialized) -> Self {
    5503            0 :         match value {
    5504              :             super::upload_queue::NotInitialized::Uninitialized => {
    5505            0 :                 CompactionError::Other(anyhow::anyhow!(value))
    5506              :             }
    5507              :             super::upload_queue::NotInitialized::ShuttingDown
    5508            0 :             | super::upload_queue::NotInitialized::Stopped => CompactionError::ShuttingDown,
    5509              :         }
    5510            0 :     }
    5511              : }
    5512              : 
    5513              : impl From<super::storage_layer::layer::DownloadError> for CompactionError {
    5514            0 :     fn from(e: super::storage_layer::layer::DownloadError) -> Self {
    5515            0 :         match e {
    5516              :             super::storage_layer::layer::DownloadError::TimelineShutdown
    5517              :             | super::storage_layer::layer::DownloadError::DownloadCancelled => {
    5518            0 :                 CompactionError::ShuttingDown
    5519              :             }
    5520              :             super::storage_layer::layer::DownloadError::ContextAndConfigReallyDeniesDownloads
    5521              :             | super::storage_layer::layer::DownloadError::DownloadRequired
    5522              :             | super::storage_layer::layer::DownloadError::NotFile(_)
    5523              :             | super::storage_layer::layer::DownloadError::DownloadFailed
    5524              :             | super::storage_layer::layer::DownloadError::PreStatFailed(_) => {
    5525            0 :                 CompactionError::Other(anyhow::anyhow!(e))
    5526              :             }
    5527              :             #[cfg(test)]
    5528              :             super::storage_layer::layer::DownloadError::Failpoint(_) => {
    5529            0 :                 CompactionError::Other(anyhow::anyhow!(e))
    5530              :             }
    5531              :         }
    5532            0 :     }
    5533              : }
    5534              : 
    5535              : impl From<layer_manager::Shutdown> for CompactionError {
    5536            0 :     fn from(_: layer_manager::Shutdown) -> Self {
    5537            0 :         CompactionError::ShuttingDown
    5538            0 :     }
    5539              : }
    5540              : 
    5541              : #[serde_as]
    5542          392 : #[derive(serde::Serialize)]
    5543              : struct RecordedDuration(#[serde_as(as = "serde_with::DurationMicroSeconds")] Duration);
    5544              : 
    5545              : #[derive(Default)]
    5546              : enum DurationRecorder {
    5547              :     #[default]
    5548              :     NotStarted,
    5549              :     Recorded(RecordedDuration, tokio::time::Instant),
    5550              : }
    5551              : 
    5552              : impl DurationRecorder {
    5553         1008 :     fn till_now(&self) -> DurationRecorder {
    5554         1008 :         match self {
    5555              :             DurationRecorder::NotStarted => {
    5556            0 :                 panic!("must only call on recorded measurements")
    5557              :             }
    5558         1008 :             DurationRecorder::Recorded(_, ended) => {
    5559         1008 :                 let now = tokio::time::Instant::now();
    5560         1008 :                 DurationRecorder::Recorded(RecordedDuration(now - *ended), now)
    5561         1008 :             }
    5562         1008 :         }
    5563         1008 :     }
    5564          392 :     fn into_recorded(self) -> Option<RecordedDuration> {
    5565          392 :         match self {
    5566            0 :             DurationRecorder::NotStarted => None,
    5567          392 :             DurationRecorder::Recorded(recorded, _) => Some(recorded),
    5568              :         }
    5569          392 :     }
    5570              : }
    5571              : 
    5572              : /// Descriptor for a delta layer used in testing infra. The start/end key/lsn range of the
    5573              : /// delta layer might be different from the min/max key/lsn in the delta layer. Therefore,
    5574              : /// the layer descriptor requires the user to provide the ranges, which should cover all
    5575              : /// keys specified in the `data` field.
    5576              : #[cfg(test)]
    5577              : #[derive(Clone)]
    5578              : pub struct DeltaLayerTestDesc {
    5579              :     pub lsn_range: Range<Lsn>,
    5580              :     pub key_range: Range<Key>,
    5581              :     pub data: Vec<(Key, Lsn, Value)>,
    5582              : }
    5583              : 
    5584              : #[cfg(test)]
    5585              : #[derive(Clone)]
    5586              : pub struct InMemoryLayerTestDesc {
    5587              :     pub lsn_range: Range<Lsn>,
    5588              :     pub data: Vec<(Key, Lsn, Value)>,
    5589              :     pub is_open: bool,
    5590              : }
    5591              : 
    5592              : #[cfg(test)]
    5593              : impl DeltaLayerTestDesc {
    5594            8 :     pub fn new(lsn_range: Range<Lsn>, key_range: Range<Key>, data: Vec<(Key, Lsn, Value)>) -> Self {
    5595            8 :         Self {
    5596            8 :             lsn_range,
    5597            8 :             key_range,
    5598            8 :             data,
    5599            8 :         }
    5600            8 :     }
    5601              : 
    5602          176 :     pub fn new_with_inferred_key_range(
    5603          176 :         lsn_range: Range<Lsn>,
    5604          176 :         data: Vec<(Key, Lsn, Value)>,
    5605          176 :     ) -> Self {
    5606          440 :         let key_min = data.iter().map(|(key, _, _)| key).min().unwrap();
    5607          440 :         let key_max = data.iter().map(|(key, _, _)| key).max().unwrap();
    5608          176 :         Self {
    5609          176 :             key_range: (*key_min)..(key_max.next()),
    5610          176 :             lsn_range,
    5611          176 :             data,
    5612          176 :         }
    5613          176 :     }
    5614              : 
    5615           20 :     pub(crate) fn layer_name(&self) -> LayerName {
    5616           20 :         LayerName::Delta(super::storage_layer::DeltaLayerName {
    5617           20 :             key_range: self.key_range.clone(),
    5618           20 :             lsn_range: self.lsn_range.clone(),
    5619           20 :         })
    5620           20 :     }
    5621              : }
    5622              : 
    5623              : impl Timeline {
    5624           56 :     async fn finish_compact_batch(
    5625           56 :         self: &Arc<Self>,
    5626           56 :         new_deltas: &[ResidentLayer],
    5627           56 :         new_images: &[ResidentLayer],
    5628           56 :         layers_to_remove: &[Layer],
    5629           56 :     ) -> Result<(), CompactionError> {
    5630           56 :         let mut guard = tokio::select! {
    5631           56 :             guard = self.layers.write() => guard,
    5632           56 :             _ = self.cancel.cancelled() => {
    5633            0 :                 return Err(CompactionError::ShuttingDown);
    5634              :             }
    5635              :         };
    5636              : 
    5637           56 :         let mut duplicated_layers = HashSet::new();
    5638           56 : 
    5639           56 :         let mut insert_layers = Vec::with_capacity(new_deltas.len());
    5640              : 
    5641          672 :         for l in new_deltas {
    5642          616 :             if guard.contains(l.as_ref()) {
    5643              :                 // expected in tests
    5644            0 :                 tracing::error!(layer=%l, "duplicated L1 layer");
    5645              : 
    5646              :                 // good ways to cause a duplicate: we repeatedly error after taking the writelock
    5647              :                 // `guard`  on self.layers. as of writing this, there are no error returns except
    5648              :                 // for compact_level0_phase1 creating an L0, which does not happen in practice
    5649              :                 // because we have not implemented L0 => L0 compaction.
    5650            0 :                 duplicated_layers.insert(l.layer_desc().key());
    5651          616 :             } else if LayerMap::is_l0(&l.layer_desc().key_range, l.layer_desc().is_delta) {
    5652            0 :                 return Err(CompactionError::Other(anyhow::anyhow!(
    5653            0 :                     "compaction generates a L0 layer file as output, which will cause infinite compaction."
    5654            0 :                 )));
    5655          616 :             } else {
    5656          616 :                 insert_layers.push(l.clone());
    5657          616 :             }
    5658              :         }
    5659              : 
    5660              :         // only remove those inputs which were not outputs
    5661           56 :         let remove_layers: Vec<Layer> = layers_to_remove
    5662           56 :             .iter()
    5663          804 :             .filter(|l| !duplicated_layers.contains(&l.layer_desc().key()))
    5664           56 :             .cloned()
    5665           56 :             .collect();
    5666           56 : 
    5667           56 :         if !new_images.is_empty() {
    5668            0 :             guard
    5669            0 :                 .open_mut()?
    5670            0 :                 .track_new_image_layers(new_images, &self.metrics);
    5671           56 :         }
    5672              : 
    5673           56 :         guard
    5674           56 :             .open_mut()?
    5675           56 :             .finish_compact_l0(&remove_layers, &insert_layers, &self.metrics);
    5676           56 : 
    5677           56 :         self.remote_client
    5678           56 :             .schedule_compaction_update(&remove_layers, new_deltas)?;
    5679              : 
    5680           56 :         drop_wlock(guard);
    5681           56 : 
    5682           56 :         Ok(())
    5683           56 :     }
    5684              : 
    5685            0 :     async fn rewrite_layers(
    5686            0 :         self: &Arc<Self>,
    5687            0 :         mut replace_layers: Vec<(Layer, ResidentLayer)>,
    5688            0 :         mut drop_layers: Vec<Layer>,
    5689            0 :     ) -> Result<(), CompactionError> {
    5690            0 :         let mut guard = self.layers.write().await;
    5691              : 
    5692              :         // Trim our lists in case our caller (compaction) raced with someone else (GC) removing layers: we want
    5693              :         // to avoid double-removing, and avoid rewriting something that was removed.
    5694            0 :         replace_layers.retain(|(l, _)| guard.contains(l));
    5695            0 :         drop_layers.retain(|l| guard.contains(l));
    5696            0 : 
    5697            0 :         guard
    5698            0 :             .open_mut()?
    5699            0 :             .rewrite_layers(&replace_layers, &drop_layers, &self.metrics);
    5700            0 : 
    5701            0 :         let upload_layers: Vec<_> = replace_layers.into_iter().map(|r| r.1).collect();
    5702            0 : 
    5703            0 :         self.remote_client
    5704            0 :             .schedule_compaction_update(&drop_layers, &upload_layers)?;
    5705              : 
    5706            0 :         Ok(())
    5707            0 :     }
    5708              : 
    5709              :     /// Schedules the uploads of the given image layers
    5710          728 :     fn upload_new_image_layers(
    5711          728 :         self: &Arc<Self>,
    5712          728 :         new_images: impl IntoIterator<Item = ResidentLayer>,
    5713          728 :     ) -> Result<(), super::upload_queue::NotInitialized> {
    5714          780 :         for layer in new_images {
    5715           52 :             self.remote_client.schedule_layer_file_upload(layer)?;
    5716              :         }
    5717              :         // should any new image layer been created, not uploading index_part will
    5718              :         // result in a mismatch between remote_physical_size and layermap calculated
    5719              :         // size, which will fail some tests, but should not be an issue otherwise.
    5720          728 :         self.remote_client
    5721          728 :             .schedule_index_upload_for_file_changes()?;
    5722          728 :         Ok(())
    5723          728 :     }
    5724              : 
    5725            0 :     async fn find_gc_time_cutoff(
    5726            0 :         &self,
    5727            0 :         now: SystemTime,
    5728            0 :         pitr: Duration,
    5729            0 :         cancel: &CancellationToken,
    5730            0 :         ctx: &RequestContext,
    5731            0 :     ) -> Result<Option<Lsn>, PageReconstructError> {
    5732            0 :         debug_assert_current_span_has_tenant_and_timeline_id();
    5733            0 :         if self.shard_identity.is_shard_zero() {
    5734              :             // Shard Zero has SLRU data and can calculate the PITR time -> LSN mapping itself
    5735            0 :             let time_range = if pitr == Duration::ZERO {
    5736            0 :                 humantime::parse_duration(DEFAULT_PITR_INTERVAL).expect("constant is invalid")
    5737              :             } else {
    5738            0 :                 pitr
    5739              :             };
    5740              : 
    5741              :             // If PITR is so large or `now` is so small that this underflows, we will retain no history (highly unexpected case)
    5742            0 :             let time_cutoff = now.checked_sub(time_range).unwrap_or(now);
    5743            0 :             let timestamp = to_pg_timestamp(time_cutoff);
    5744              : 
    5745            0 :             let time_cutoff = match self.find_lsn_for_timestamp(timestamp, cancel, ctx).await? {
    5746            0 :                 LsnForTimestamp::Present(lsn) => Some(lsn),
    5747            0 :                 LsnForTimestamp::Future(lsn) => {
    5748            0 :                     // The timestamp is in the future. That sounds impossible,
    5749            0 :                     // but what it really means is that there hasn't been
    5750            0 :                     // any commits since the cutoff timestamp.
    5751            0 :                     //
    5752            0 :                     // In this case we should use the LSN of the most recent commit,
    5753            0 :                     // which is implicitly the last LSN in the log.
    5754            0 :                     debug!("future({})", lsn);
    5755            0 :                     Some(self.get_last_record_lsn())
    5756              :                 }
    5757            0 :                 LsnForTimestamp::Past(lsn) => {
    5758            0 :                     debug!("past({})", lsn);
    5759            0 :                     None
    5760              :                 }
    5761            0 :                 LsnForTimestamp::NoData(lsn) => {
    5762            0 :                     debug!("nodata({})", lsn);
    5763            0 :                     None
    5764              :                 }
    5765              :             };
    5766            0 :             Ok(time_cutoff)
    5767              :         } else {
    5768              :             // Shards other than shard zero cannot do timestamp->lsn lookups, and must instead learn their GC cutoff
    5769              :             // from shard zero's index.  The index doesn't explicitly tell us the time cutoff, but we may assume that
    5770              :             // the point up to which shard zero's last_gc_cutoff has advanced will either be the time cutoff, or a
    5771              :             // space cutoff that we would also have respected ourselves.
    5772            0 :             match self
    5773            0 :                 .remote_client
    5774            0 :                 .download_foreign_index(ShardNumber(0), cancel)
    5775            0 :                 .await
    5776              :             {
    5777            0 :                 Ok((index_part, index_generation, _index_mtime)) => {
    5778            0 :                     tracing::info!(
    5779            0 :                         "GC loaded shard zero metadata (gen {index_generation:?}): latest_gc_cutoff_lsn: {}",
    5780            0 :                         index_part.metadata.latest_gc_cutoff_lsn()
    5781              :                     );
    5782            0 :                     Ok(Some(index_part.metadata.latest_gc_cutoff_lsn()))
    5783              :                 }
    5784              :                 Err(DownloadError::NotFound) => {
    5785              :                     // This is unexpected, because during timeline creations shard zero persists to remote
    5786              :                     // storage before other shards are called, and during timeline deletion non-zeroth shards are
    5787              :                     // deleted before the zeroth one.  However, it should be harmless: if we somehow end up in this
    5788              :                     // state, then shard zero should _eventually_ write an index when it GCs.
    5789            0 :                     tracing::warn!("GC couldn't find shard zero's index for timeline");
    5790            0 :                     Ok(None)
    5791              :                 }
    5792            0 :                 Err(e) => {
    5793            0 :                     // TODO: this function should return a different error type than page reconstruct error
    5794            0 :                     Err(PageReconstructError::Other(anyhow::anyhow!(e)))
    5795              :                 }
    5796              :             }
    5797              : 
    5798              :             // TODO: after reading shard zero's GC cutoff, we should validate its generation with the storage
    5799              :             // controller.  Otherwise, it is possible that we see the GC cutoff go backwards while shard zero
    5800              :             // is going through a migration if we read the old location's index and it has GC'd ahead of the
    5801              :             // new location.  This is legal in principle, but problematic in practice because it might result
    5802              :             // in a timeline creation succeeding on shard zero ('s new location) but then failing on other shards
    5803              :             // because they have GC'd past the branch point.
    5804              :         }
    5805            0 :     }
    5806              : 
    5807              :     /// Find the Lsns above which layer files need to be retained on
    5808              :     /// garbage collection.
    5809              :     ///
    5810              :     /// We calculate two cutoffs, one based on time and one based on WAL size.  `pitr`
    5811              :     /// controls the time cutoff (or ZERO to disable time-based retention), and `space_cutoff` controls
    5812              :     /// the space-based retention.
    5813              :     ///
    5814              :     /// This function doesn't simply to calculate time & space based retention: it treats time-based
    5815              :     /// retention as authoritative if enabled, and falls back to space-based retention if calculating
    5816              :     /// the LSN for a time point isn't possible.  Therefore the GcCutoffs::horizon in the response might
    5817              :     /// be different to the `space_cutoff` input.  Callers should treat the min() of the two cutoffs
    5818              :     /// in the response as the GC cutoff point for the timeline.
    5819              :     #[instrument(skip_all, fields(timeline_id=%self.timeline_id))]
    5820              :     pub(super) async fn find_gc_cutoffs(
    5821              :         &self,
    5822              :         now: SystemTime,
    5823              :         space_cutoff: Lsn,
    5824              :         pitr: Duration,
    5825              :         cancel: &CancellationToken,
    5826              :         ctx: &RequestContext,
    5827              :     ) -> Result<GcCutoffs, PageReconstructError> {
    5828              :         let _timer = self
    5829              :             .metrics
    5830              :             .find_gc_cutoffs_histo
    5831              :             .start_timer()
    5832              :             .record_on_drop();
    5833              : 
    5834              :         pausable_failpoint!("Timeline::find_gc_cutoffs-pausable");
    5835              : 
    5836              :         if cfg!(test) {
    5837              :             // Unit tests which specify zero PITR interval expect to avoid doing any I/O for timestamp lookup
    5838              :             if pitr == Duration::ZERO {
    5839              :                 return Ok(GcCutoffs {
    5840              :                     time: self.get_last_record_lsn(),
    5841              :                     space: space_cutoff,
    5842              :                 });
    5843              :             }
    5844              :         }
    5845              : 
    5846              :         // Calculate a time-based limit on how much to retain:
    5847              :         // - if PITR interval is set, then this is our cutoff.
    5848              :         // - if PITR interval is not set, then we do a lookup
    5849              :         //   based on DEFAULT_PITR_INTERVAL, so that size-based retention does not result in keeping history around permanently on idle databases.
    5850              :         let time_cutoff = self.find_gc_time_cutoff(now, pitr, cancel, ctx).await?;
    5851              : 
    5852              :         Ok(match (pitr, time_cutoff) {
    5853              :             (Duration::ZERO, Some(time_cutoff)) => {
    5854              :                 // PITR is not set. Retain the size-based limit, or the default time retention,
    5855              :                 // whichever requires less data.
    5856              :                 GcCutoffs {
    5857              :                     time: self.get_last_record_lsn(),
    5858              :                     space: std::cmp::max(time_cutoff, space_cutoff),
    5859              :                 }
    5860              :             }
    5861              :             (Duration::ZERO, None) => {
    5862              :                 // PITR is not set, and time lookup failed
    5863              :                 GcCutoffs {
    5864              :                     time: self.get_last_record_lsn(),
    5865              :                     space: space_cutoff,
    5866              :                 }
    5867              :             }
    5868              :             (_, None) => {
    5869              :                 // PITR interval is set & we didn't look up a timestamp successfully.  Conservatively assume PITR
    5870              :                 // cannot advance beyond what was already GC'd, and respect space-based retention
    5871              :                 GcCutoffs {
    5872              :                     time: *self.get_applied_gc_cutoff_lsn(),
    5873              :                     space: space_cutoff,
    5874              :                 }
    5875              :             }
    5876              :             (_, Some(time_cutoff)) => {
    5877              :                 // PITR interval is set and we looked up timestamp successfully.  Ignore
    5878              :                 // size based retention and make time cutoff authoritative
    5879              :                 GcCutoffs {
    5880              :                     time: time_cutoff,
    5881              :                     space: time_cutoff,
    5882              :                 }
    5883              :             }
    5884              :         })
    5885              :     }
    5886              : 
    5887              :     /// Garbage collect layer files on a timeline that are no longer needed.
    5888              :     ///
    5889              :     /// Currently, we don't make any attempt at removing unneeded page versions
    5890              :     /// within a layer file. We can only remove the whole file if it's fully
    5891              :     /// obsolete.
    5892            8 :     pub(super) async fn gc(&self) -> Result<GcResult, GcError> {
    5893              :         // this is most likely the background tasks, but it might be the spawned task from
    5894              :         // immediate_gc
    5895            8 :         let _g = tokio::select! {
    5896            8 :             guard = self.gc_lock.lock() => guard,
    5897            8 :             _ = self.cancel.cancelled() => return Ok(GcResult::default()),
    5898              :         };
    5899            8 :         let timer = self.metrics.garbage_collect_histo.start_timer();
    5900            8 : 
    5901            8 :         fail_point!("before-timeline-gc");
    5902            8 : 
    5903            8 :         // Is the timeline being deleted?
    5904            8 :         if self.is_stopping() {
    5905            0 :             return Err(GcError::TimelineCancelled);
    5906            8 :         }
    5907            8 : 
    5908            8 :         let (space_cutoff, time_cutoff, retain_lsns, max_lsn_with_valid_lease) = {
    5909            8 :             let gc_info = self.gc_info.read().unwrap();
    5910            8 : 
    5911            8 :             let space_cutoff = min(gc_info.cutoffs.space, self.get_disk_consistent_lsn());
    5912            8 :             let time_cutoff = gc_info.cutoffs.time;
    5913            8 :             let retain_lsns = gc_info
    5914            8 :                 .retain_lsns
    5915            8 :                 .iter()
    5916            8 :                 .map(|(lsn, _child_id, _is_offloaded)| *lsn)
    5917            8 :                 .collect();
    5918            8 : 
    5919            8 :             // Gets the maximum LSN that holds the valid lease.
    5920            8 :             //
    5921            8 :             // Caveat: `refresh_gc_info` is in charged of updating the lease map.
    5922            8 :             // Here, we do not check for stale leases again.
    5923            8 :             let max_lsn_with_valid_lease = gc_info.leases.last_key_value().map(|(lsn, _)| *lsn);
    5924            8 : 
    5925            8 :             (
    5926            8 :                 space_cutoff,
    5927            8 :                 time_cutoff,
    5928            8 :                 retain_lsns,
    5929            8 :                 max_lsn_with_valid_lease,
    5930            8 :             )
    5931            8 :         };
    5932            8 : 
    5933            8 :         let mut new_gc_cutoff = Lsn::min(space_cutoff, time_cutoff);
    5934            8 :         let standby_horizon = self.standby_horizon.load();
    5935            8 :         // Hold GC for the standby, but as a safety guard do it only within some
    5936            8 :         // reasonable lag.
    5937            8 :         if standby_horizon != Lsn::INVALID {
    5938            0 :             if let Some(standby_lag) = new_gc_cutoff.checked_sub(standby_horizon) {
    5939              :                 const MAX_ALLOWED_STANDBY_LAG: u64 = 10u64 << 30; // 10 GB
    5940            0 :                 if standby_lag.0 < MAX_ALLOWED_STANDBY_LAG {
    5941            0 :                     new_gc_cutoff = Lsn::min(standby_horizon, new_gc_cutoff);
    5942            0 :                     trace!("holding off GC for standby apply LSN {}", standby_horizon);
    5943              :                 } else {
    5944            0 :                     warn!(
    5945            0 :                         "standby is lagging for more than {}MB, not holding gc for it",
    5946            0 :                         MAX_ALLOWED_STANDBY_LAG / 1024 / 1024
    5947              :                     )
    5948              :                 }
    5949            0 :             }
    5950            8 :         }
    5951              : 
    5952              :         // Reset standby horizon to ignore it if it is not updated till next GC.
    5953              :         // It is an easy way to unset it when standby disappears without adding
    5954              :         // more conf options.
    5955            8 :         self.standby_horizon.store(Lsn::INVALID);
    5956            8 :         self.metrics
    5957            8 :             .standby_horizon_gauge
    5958            8 :             .set(Lsn::INVALID.0 as i64);
    5959              : 
    5960            8 :         let res = self
    5961            8 :             .gc_timeline(
    5962            8 :                 space_cutoff,
    5963            8 :                 time_cutoff,
    5964            8 :                 retain_lsns,
    5965            8 :                 max_lsn_with_valid_lease,
    5966            8 :                 new_gc_cutoff,
    5967            8 :             )
    5968            8 :             .instrument(
    5969            8 :                 info_span!("gc_timeline", timeline_id = %self.timeline_id, cutoff = %new_gc_cutoff),
    5970              :             )
    5971            8 :             .await?;
    5972              : 
    5973              :         // only record successes
    5974            8 :         timer.stop_and_record();
    5975            8 : 
    5976            8 :         Ok(res)
    5977            8 :     }
    5978              : 
    5979            8 :     async fn gc_timeline(
    5980            8 :         &self,
    5981            8 :         space_cutoff: Lsn,
    5982            8 :         time_cutoff: Lsn,
    5983            8 :         retain_lsns: Vec<Lsn>,
    5984            8 :         max_lsn_with_valid_lease: Option<Lsn>,
    5985            8 :         new_gc_cutoff: Lsn,
    5986            8 :     ) -> Result<GcResult, GcError> {
    5987            8 :         // FIXME: if there is an ongoing detach_from_ancestor, we should just skip gc
    5988            8 : 
    5989            8 :         let now = SystemTime::now();
    5990            8 :         let mut result: GcResult = GcResult::default();
    5991            8 : 
    5992            8 :         // Nothing to GC. Return early.
    5993            8 :         let latest_gc_cutoff = *self.get_applied_gc_cutoff_lsn();
    5994            8 :         if latest_gc_cutoff >= new_gc_cutoff {
    5995            0 :             info!(
    5996            0 :                 "Nothing to GC: new_gc_cutoff_lsn {new_gc_cutoff}, latest_gc_cutoff_lsn {latest_gc_cutoff}",
    5997              :             );
    5998            0 :             return Ok(result);
    5999            8 :         }
    6000              : 
    6001              :         // We need to ensure that no one tries to read page versions or create
    6002              :         // branches at a point before latest_gc_cutoff_lsn. See branch_timeline()
    6003              :         // for details. This will block until the old value is no longer in use.
    6004              :         //
    6005              :         // The GC cutoff should only ever move forwards.
    6006            8 :         let waitlist = {
    6007            8 :             let write_guard = self.applied_gc_cutoff_lsn.lock_for_write();
    6008            8 :             if *write_guard > new_gc_cutoff {
    6009            0 :                 return Err(GcError::BadLsn {
    6010            0 :                     why: format!(
    6011            0 :                         "Cannot move GC cutoff LSN backwards (was {}, new {})",
    6012            0 :                         *write_guard, new_gc_cutoff
    6013            0 :                     ),
    6014            0 :                 });
    6015            8 :             }
    6016            8 : 
    6017            8 :             write_guard.store_and_unlock(new_gc_cutoff)
    6018            8 :         };
    6019            8 :         waitlist.wait().await;
    6020              : 
    6021            8 :         info!("GC starting");
    6022              : 
    6023            8 :         debug!("retain_lsns: {:?}", retain_lsns);
    6024              : 
    6025            8 :         let mut layers_to_remove = Vec::new();
    6026              : 
    6027              :         // Scan all layers in the timeline (remote or on-disk).
    6028              :         //
    6029              :         // Garbage collect the layer if all conditions are satisfied:
    6030              :         // 1. it is older than cutoff LSN;
    6031              :         // 2. it is older than PITR interval;
    6032              :         // 3. it doesn't need to be retained for 'retain_lsns';
    6033              :         // 4. it does not need to be kept for LSNs holding valid leases.
    6034              :         // 5. newer on-disk image layers cover the layer's whole key range
    6035              :         //
    6036              :         // TODO holding a write lock is too agressive and avoidable
    6037            8 :         let mut guard = self.layers.write().await;
    6038            8 :         let layers = guard.layer_map()?;
    6039           48 :         'outer: for l in layers.iter_historic_layers() {
    6040           48 :             result.layers_total += 1;
    6041           48 : 
    6042           48 :             // 1. Is it newer than GC horizon cutoff point?
    6043           48 :             if l.get_lsn_range().end > space_cutoff {
    6044            4 :                 info!(
    6045            0 :                     "keeping {} because it's newer than space_cutoff {}",
    6046            0 :                     l.layer_name(),
    6047              :                     space_cutoff,
    6048              :                 );
    6049            4 :                 result.layers_needed_by_cutoff += 1;
    6050            4 :                 continue 'outer;
    6051           44 :             }
    6052           44 : 
    6053           44 :             // 2. It is newer than PiTR cutoff point?
    6054           44 :             if l.get_lsn_range().end > time_cutoff {
    6055            0 :                 info!(
    6056            0 :                     "keeping {} because it's newer than time_cutoff {}",
    6057            0 :                     l.layer_name(),
    6058              :                     time_cutoff,
    6059              :                 );
    6060            0 :                 result.layers_needed_by_pitr += 1;
    6061            0 :                 continue 'outer;
    6062           44 :             }
    6063              : 
    6064              :             // 3. Is it needed by a child branch?
    6065              :             // NOTE With that we would keep data that
    6066              :             // might be referenced by child branches forever.
    6067              :             // We can track this in child timeline GC and delete parent layers when
    6068              :             // they are no longer needed. This might be complicated with long inheritance chains.
    6069              :             //
    6070              :             // TODO Vec is not a great choice for `retain_lsns`
    6071           44 :             for retain_lsn in &retain_lsns {
    6072              :                 // start_lsn is inclusive
    6073            0 :                 if &l.get_lsn_range().start <= retain_lsn {
    6074            0 :                     info!(
    6075            0 :                         "keeping {} because it's still might be referenced by child branch forked at {} is_dropped: xx is_incremental: {}",
    6076            0 :                         l.layer_name(),
    6077            0 :                         retain_lsn,
    6078            0 :                         l.is_incremental(),
    6079              :                     );
    6080            0 :                     result.layers_needed_by_branches += 1;
    6081            0 :                     continue 'outer;
    6082            0 :                 }
    6083              :             }
    6084              : 
    6085              :             // 4. Is there a valid lease that requires us to keep this layer?
    6086           44 :             if let Some(lsn) = &max_lsn_with_valid_lease {
    6087              :                 // keep if layer start <= any of the lease
    6088           36 :                 if &l.get_lsn_range().start <= lsn {
    6089           28 :                     info!(
    6090            0 :                         "keeping {} because there is a valid lease preventing GC at {}",
    6091            0 :                         l.layer_name(),
    6092              :                         lsn,
    6093              :                     );
    6094           28 :                     result.layers_needed_by_leases += 1;
    6095           28 :                     continue 'outer;
    6096            8 :                 }
    6097            8 :             }
    6098              : 
    6099              :             // 5. Is there a later on-disk layer for this relation?
    6100              :             //
    6101              :             // The end-LSN is exclusive, while disk_consistent_lsn is
    6102              :             // inclusive. For example, if disk_consistent_lsn is 100, it is
    6103              :             // OK for a delta layer to have end LSN 101, but if the end LSN
    6104              :             // is 102, then it might not have been fully flushed to disk
    6105              :             // before crash.
    6106              :             //
    6107              :             // For example, imagine that the following layers exist:
    6108              :             //
    6109              :             // 1000      - image (A)
    6110              :             // 1000-2000 - delta (B)
    6111              :             // 2000      - image (C)
    6112              :             // 2000-3000 - delta (D)
    6113              :             // 3000      - image (E)
    6114              :             //
    6115              :             // If GC horizon is at 2500, we can remove layers A and B, but
    6116              :             // we cannot remove C, even though it's older than 2500, because
    6117              :             // the delta layer 2000-3000 depends on it.
    6118           16 :             if !layers
    6119           16 :                 .image_layer_exists(&l.get_key_range(), &(l.get_lsn_range().end..new_gc_cutoff))
    6120              :             {
    6121           12 :                 info!("keeping {} because it is the latest layer", l.layer_name());
    6122           12 :                 result.layers_not_updated += 1;
    6123           12 :                 continue 'outer;
    6124            4 :             }
    6125            4 : 
    6126            4 :             // We didn't find any reason to keep this file, so remove it.
    6127            4 :             info!(
    6128            0 :                 "garbage collecting {} is_dropped: xx is_incremental: {}",
    6129            0 :                 l.layer_name(),
    6130            0 :                 l.is_incremental(),
    6131              :             );
    6132            4 :             layers_to_remove.push(l);
    6133              :         }
    6134              : 
    6135            8 :         if !layers_to_remove.is_empty() {
    6136              :             // Persist the new GC cutoff value before we actually remove anything.
    6137              :             // This unconditionally schedules also an index_part.json update, even though, we will
    6138              :             // be doing one a bit later with the unlinked gc'd layers.
    6139            4 :             let disk_consistent_lsn = self.disk_consistent_lsn.load();
    6140            4 :             self.schedule_uploads(disk_consistent_lsn, None)
    6141            4 :                 .map_err(|e| {
    6142            0 :                     if self.cancel.is_cancelled() {
    6143            0 :                         GcError::TimelineCancelled
    6144              :                     } else {
    6145            0 :                         GcError::Remote(e)
    6146              :                     }
    6147            4 :                 })?;
    6148              : 
    6149            4 :             let gc_layers = layers_to_remove
    6150            4 :                 .iter()
    6151            4 :                 .map(|x| guard.get_from_desc(x))
    6152            4 :                 .collect::<Vec<Layer>>();
    6153            4 : 
    6154            4 :             result.layers_removed = gc_layers.len() as u64;
    6155            4 : 
    6156            4 :             self.remote_client.schedule_gc_update(&gc_layers)?;
    6157              : 
    6158            4 :             guard.open_mut()?.finish_gc_timeline(&gc_layers);
    6159            4 : 
    6160            4 :             #[cfg(feature = "testing")]
    6161            4 :             {
    6162            4 :                 result.doomed_layers = gc_layers;
    6163            4 :             }
    6164            4 :         }
    6165              : 
    6166            8 :         info!(
    6167            0 :             "GC completed removing {} layers, cutoff {}",
    6168              :             result.layers_removed, new_gc_cutoff
    6169              :         );
    6170              : 
    6171            8 :         result.elapsed = now.elapsed().unwrap_or(Duration::ZERO);
    6172            8 :         Ok(result)
    6173            8 :     }
    6174              : 
    6175              :     /// Reconstruct a value, using the given base image and WAL records in 'data'.
    6176      1340058 :     async fn reconstruct_value(
    6177      1340058 :         &self,
    6178      1340058 :         key: Key,
    6179      1340058 :         request_lsn: Lsn,
    6180      1340058 :         mut data: ValueReconstructState,
    6181      1340058 :     ) -> Result<Bytes, PageReconstructError> {
    6182      1340058 :         // Perform WAL redo if needed
    6183      1340058 :         data.records.reverse();
    6184      1340058 : 
    6185      1340058 :         // If we have a page image, and no WAL, we're all set
    6186      1340058 :         if data.records.is_empty() {
    6187      1338382 :             if let Some((img_lsn, img)) = &data.img {
    6188      1338382 :                 trace!(
    6189            0 :                     "found page image for key {} at {}, no WAL redo required, req LSN {}",
    6190              :                     key, img_lsn, request_lsn,
    6191              :                 );
    6192      1338382 :                 Ok(img.clone())
    6193              :             } else {
    6194            0 :                 Err(PageReconstructError::from(anyhow!(
    6195            0 :                     "base image for {key} at {request_lsn} not found"
    6196            0 :                 )))
    6197              :             }
    6198              :         } else {
    6199              :             // We need to do WAL redo.
    6200              :             //
    6201              :             // If we don't have a base image, then the oldest WAL record better initialize
    6202              :             // the page
    6203         1676 :             if data.img.is_none() && !data.records.first().unwrap().1.will_init() {
    6204            0 :                 Err(PageReconstructError::from(anyhow!(
    6205            0 :                     "Base image for {} at {} not found, but got {} WAL records",
    6206            0 :                     key,
    6207            0 :                     request_lsn,
    6208            0 :                     data.records.len()
    6209            0 :                 )))
    6210              :             } else {
    6211         1676 :                 if data.img.is_some() {
    6212         1536 :                     trace!(
    6213            0 :                         "found {} WAL records and a base image for {} at {}, performing WAL redo",
    6214            0 :                         data.records.len(),
    6215              :                         key,
    6216              :                         request_lsn
    6217              :                     );
    6218              :                 } else {
    6219          140 :                     trace!(
    6220            0 :                         "found {} WAL records that will init the page for {} at {}, performing WAL redo",
    6221            0 :                         data.records.len(),
    6222              :                         key,
    6223              :                         request_lsn
    6224              :                     );
    6225              :                 };
    6226         1676 :                 let res = self
    6227         1676 :                     .walredo_mgr
    6228         1676 :                     .as_ref()
    6229         1676 :                     .context("timeline has no walredo manager")
    6230         1676 :                     .map_err(PageReconstructError::WalRedo)?
    6231         1676 :                     .request_redo(key, request_lsn, data.img, data.records, self.pg_version)
    6232         1676 :                     .await;
    6233         1676 :                 let img = match res {
    6234         1676 :                     Ok(img) => img,
    6235            0 :                     Err(walredo::Error::Cancelled) => return Err(PageReconstructError::Cancelled),
    6236            0 :                     Err(walredo::Error::Other(err)) => {
    6237            0 :                         critical!("walredo failure during page reconstruction: {err:?}");
    6238            0 :                         return Err(PageReconstructError::WalRedo(
    6239            0 :                             err.context("reconstruct a page image"),
    6240            0 :                         ));
    6241              :                     }
    6242              :                 };
    6243         1676 :                 Ok(img)
    6244              :             }
    6245              :         }
    6246      1340058 :     }
    6247              : 
    6248            0 :     pub(crate) async fn spawn_download_all_remote_layers(
    6249            0 :         self: Arc<Self>,
    6250            0 :         request: DownloadRemoteLayersTaskSpawnRequest,
    6251            0 :         ctx: &RequestContext,
    6252            0 :     ) -> Result<DownloadRemoteLayersTaskInfo, DownloadRemoteLayersTaskInfo> {
    6253              :         use pageserver_api::models::DownloadRemoteLayersTaskState;
    6254              : 
    6255              :         // this is not really needed anymore; it has tests which really check the return value from
    6256              :         // http api. it would be better not to maintain this anymore.
    6257              : 
    6258            0 :         let mut status_guard = self.download_all_remote_layers_task_info.write().unwrap();
    6259            0 :         if let Some(st) = &*status_guard {
    6260            0 :             match &st.state {
    6261              :                 DownloadRemoteLayersTaskState::Running => {
    6262            0 :                     return Err(st.clone());
    6263              :                 }
    6264              :                 DownloadRemoteLayersTaskState::ShutDown
    6265            0 :                 | DownloadRemoteLayersTaskState::Completed => {
    6266            0 :                     *status_guard = None;
    6267            0 :                 }
    6268              :             }
    6269            0 :         }
    6270              : 
    6271            0 :         let self_clone = Arc::clone(&self);
    6272            0 :         let task_ctx = ctx.detached_child(
    6273            0 :             TaskKind::DownloadAllRemoteLayers,
    6274            0 :             DownloadBehavior::Download,
    6275            0 :         );
    6276            0 :         let task_id = task_mgr::spawn(
    6277            0 :             task_mgr::BACKGROUND_RUNTIME.handle(),
    6278            0 :             task_mgr::TaskKind::DownloadAllRemoteLayers,
    6279            0 :             self.tenant_shard_id,
    6280            0 :             Some(self.timeline_id),
    6281            0 :             "download all remote layers task",
    6282            0 :             async move {
    6283            0 :                 self_clone.download_all_remote_layers(request, &task_ctx).await;
    6284            0 :                 let mut status_guard = self_clone.download_all_remote_layers_task_info.write().unwrap();
    6285            0 :                  match &mut *status_guard {
    6286              :                     None => {
    6287            0 :                         warn!("tasks status is supposed to be Some(), since we are running");
    6288              :                     }
    6289            0 :                     Some(st) => {
    6290            0 :                         let exp_task_id = format!("{}", task_mgr::current_task_id().unwrap());
    6291            0 :                         if st.task_id != exp_task_id {
    6292            0 :                             warn!("task id changed while we were still running, expecting {} but have {}", exp_task_id, st.task_id);
    6293            0 :                         } else {
    6294            0 :                             st.state = DownloadRemoteLayersTaskState::Completed;
    6295            0 :                         }
    6296              :                     }
    6297              :                 };
    6298            0 :                 Ok(())
    6299            0 :             }
    6300            0 :             .instrument(info_span!(parent: None, "download_all_remote_layers", tenant_id = %self.tenant_shard_id.tenant_id, shard_id = %self.tenant_shard_id.shard_slug(), timeline_id = %self.timeline_id))
    6301              :         );
    6302              : 
    6303            0 :         let initial_info = DownloadRemoteLayersTaskInfo {
    6304            0 :             task_id: format!("{task_id}"),
    6305            0 :             state: DownloadRemoteLayersTaskState::Running,
    6306            0 :             total_layer_count: 0,
    6307            0 :             successful_download_count: 0,
    6308            0 :             failed_download_count: 0,
    6309            0 :         };
    6310            0 :         *status_guard = Some(initial_info.clone());
    6311            0 : 
    6312            0 :         Ok(initial_info)
    6313            0 :     }
    6314              : 
    6315            0 :     async fn download_all_remote_layers(
    6316            0 :         self: &Arc<Self>,
    6317            0 :         request: DownloadRemoteLayersTaskSpawnRequest,
    6318            0 :         ctx: &RequestContext,
    6319            0 :     ) {
    6320              :         use pageserver_api::models::DownloadRemoteLayersTaskState;
    6321              : 
    6322            0 :         let remaining = {
    6323            0 :             let guard = self.layers.read().await;
    6324            0 :             let Ok(lm) = guard.layer_map() else {
    6325              :                 // technically here we could look into iterating accessible layers, but downloading
    6326              :                 // all layers of a shutdown timeline makes no sense regardless.
    6327            0 :                 tracing::info!("attempted to download all layers of shutdown timeline");
    6328            0 :                 return;
    6329              :             };
    6330            0 :             lm.iter_historic_layers()
    6331            0 :                 .map(|desc| guard.get_from_desc(&desc))
    6332            0 :                 .collect::<Vec<_>>()
    6333            0 :         };
    6334            0 :         let total_layer_count = remaining.len();
    6335              : 
    6336              :         macro_rules! lock_status {
    6337              :             ($st:ident) => {
    6338              :                 let mut st = self.download_all_remote_layers_task_info.write().unwrap();
    6339              :                 let st = st
    6340              :                     .as_mut()
    6341              :                     .expect("this function is only called after the task has been spawned");
    6342              :                 assert_eq!(
    6343              :                     st.task_id,
    6344              :                     format!(
    6345              :                         "{}",
    6346              :                         task_mgr::current_task_id().expect("we run inside a task_mgr task")
    6347              :                     )
    6348              :                 );
    6349              :                 let $st = st;
    6350              :             };
    6351              :         }
    6352              : 
    6353              :         {
    6354            0 :             lock_status!(st);
    6355            0 :             st.total_layer_count = total_layer_count as u64;
    6356            0 :         }
    6357            0 : 
    6358            0 :         let mut remaining = remaining.into_iter();
    6359            0 :         let mut have_remaining = true;
    6360            0 :         let mut js = tokio::task::JoinSet::new();
    6361            0 : 
    6362            0 :         let cancel = task_mgr::shutdown_token();
    6363            0 : 
    6364            0 :         let limit = request.max_concurrent_downloads;
    6365              : 
    6366              :         loop {
    6367            0 :             while js.len() < limit.get() && have_remaining && !cancel.is_cancelled() {
    6368            0 :                 let Some(next) = remaining.next() else {
    6369            0 :                     have_remaining = false;
    6370            0 :                     break;
    6371              :                 };
    6372              : 
    6373            0 :                 let span = tracing::info_span!("download", layer = %next);
    6374              : 
    6375            0 :                 let ctx = ctx.attached_child();
    6376            0 :                 js.spawn(
    6377            0 :                     async move {
    6378            0 :                         let res = next.download(&ctx).await;
    6379            0 :                         (next, res)
    6380            0 :                     }
    6381            0 :                     .instrument(span),
    6382            0 :                 );
    6383            0 :             }
    6384              : 
    6385            0 :             while let Some(res) = js.join_next().await {
    6386            0 :                 match res {
    6387              :                     Ok((_, Ok(_))) => {
    6388            0 :                         lock_status!(st);
    6389            0 :                         st.successful_download_count += 1;
    6390              :                     }
    6391            0 :                     Ok((layer, Err(e))) => {
    6392            0 :                         tracing::error!(%layer, "download failed: {e:#}");
    6393            0 :                         lock_status!(st);
    6394            0 :                         st.failed_download_count += 1;
    6395              :                     }
    6396            0 :                     Err(je) if je.is_cancelled() => unreachable!("not used here"),
    6397            0 :                     Err(je) if je.is_panic() => {
    6398            0 :                         lock_status!(st);
    6399            0 :                         st.failed_download_count += 1;
    6400              :                     }
    6401            0 :                     Err(je) => tracing::warn!("unknown joinerror: {je:?}"),
    6402              :                 }
    6403              :             }
    6404              : 
    6405            0 :             if js.is_empty() && (!have_remaining || cancel.is_cancelled()) {
    6406            0 :                 break;
    6407            0 :             }
    6408              :         }
    6409              : 
    6410              :         {
    6411            0 :             lock_status!(st);
    6412            0 :             st.state = DownloadRemoteLayersTaskState::Completed;
    6413              :         }
    6414            0 :     }
    6415              : 
    6416            0 :     pub(crate) fn get_download_all_remote_layers_task_info(
    6417            0 :         &self,
    6418            0 :     ) -> Option<DownloadRemoteLayersTaskInfo> {
    6419            0 :         self.download_all_remote_layers_task_info
    6420            0 :             .read()
    6421            0 :             .unwrap()
    6422            0 :             .clone()
    6423            0 :     }
    6424              : }
    6425              : 
    6426              : impl Timeline {
    6427              :     /// Returns non-remote layers for eviction.
    6428            0 :     pub(crate) async fn get_local_layers_for_disk_usage_eviction(&self) -> DiskUsageEvictionInfo {
    6429            0 :         let guard = self.layers.read().await;
    6430            0 :         let mut max_layer_size: Option<u64> = None;
    6431            0 : 
    6432            0 :         let resident_layers = guard
    6433            0 :             .likely_resident_layers()
    6434            0 :             .map(|layer| {
    6435            0 :                 let file_size = layer.layer_desc().file_size;
    6436            0 :                 max_layer_size = max_layer_size.map_or(Some(file_size), |m| Some(m.max(file_size)));
    6437            0 : 
    6438            0 :                 let last_activity_ts = layer.latest_activity();
    6439            0 : 
    6440            0 :                 EvictionCandidate {
    6441            0 :                     layer: layer.to_owned().into(),
    6442            0 :                     last_activity_ts,
    6443            0 :                     relative_last_activity: finite_f32::FiniteF32::ZERO,
    6444            0 :                     visibility: layer.visibility(),
    6445            0 :                 }
    6446            0 :             })
    6447            0 :             .collect();
    6448            0 : 
    6449            0 :         DiskUsageEvictionInfo {
    6450            0 :             max_layer_size,
    6451            0 :             resident_layers,
    6452            0 :         }
    6453            0 :     }
    6454              : 
    6455         3748 :     pub(crate) fn get_shard_index(&self) -> ShardIndex {
    6456         3748 :         ShardIndex {
    6457         3748 :             shard_number: self.tenant_shard_id.shard_number,
    6458         3748 :             shard_count: self.tenant_shard_id.shard_count,
    6459         3748 :         }
    6460         3748 :     }
    6461              : 
    6462              :     /// Persistently blocks gc for `Manual` reason.
    6463              :     ///
    6464              :     /// Returns true if no such block existed before, false otherwise.
    6465            0 :     pub(crate) async fn block_gc(&self, tenant: &super::Tenant) -> anyhow::Result<bool> {
    6466              :         use crate::tenant::remote_timeline_client::index::GcBlockingReason;
    6467            0 :         assert_eq!(self.tenant_shard_id, tenant.tenant_shard_id);
    6468            0 :         tenant.gc_block.insert(self, GcBlockingReason::Manual).await
    6469            0 :     }
    6470              : 
    6471              :     /// Persistently unblocks gc for `Manual` reason.
    6472            0 :     pub(crate) async fn unblock_gc(&self, tenant: &super::Tenant) -> anyhow::Result<()> {
    6473              :         use crate::tenant::remote_timeline_client::index::GcBlockingReason;
    6474            0 :         assert_eq!(self.tenant_shard_id, tenant.tenant_shard_id);
    6475            0 :         tenant.gc_block.remove(self, GcBlockingReason::Manual).await
    6476            0 :     }
    6477              : 
    6478              :     #[cfg(test)]
    6479          100 :     pub(super) fn force_advance_lsn(self: &Arc<Timeline>, new_lsn: Lsn) {
    6480          100 :         self.last_record_lsn.advance(new_lsn);
    6481          100 :     }
    6482              : 
    6483              :     #[cfg(test)]
    6484            4 :     pub(super) fn force_set_disk_consistent_lsn(&self, new_value: Lsn) {
    6485            4 :         self.disk_consistent_lsn.store(new_value);
    6486            4 :     }
    6487              : 
    6488              :     /// Force create an image layer and place it into the layer map.
    6489              :     ///
    6490              :     /// DO NOT use this function directly. Use [`Tenant::branch_timeline_test_with_layers`]
    6491              :     /// or [`Tenant::create_test_timeline_with_layers`] to ensure all these layers are
    6492              :     /// placed into the layer map in one run AND be validated.
    6493              :     #[cfg(test)]
    6494          128 :     pub(super) async fn force_create_image_layer(
    6495          128 :         self: &Arc<Timeline>,
    6496          128 :         lsn: Lsn,
    6497          128 :         mut images: Vec<(Key, Bytes)>,
    6498          128 :         check_start_lsn: Option<Lsn>,
    6499          128 :         ctx: &RequestContext,
    6500          128 :     ) -> anyhow::Result<()> {
    6501          128 :         let last_record_lsn = self.get_last_record_lsn();
    6502          128 :         assert!(
    6503          128 :             lsn <= last_record_lsn,
    6504            0 :             "advance last record lsn before inserting a layer, lsn={lsn}, last_record_lsn={last_record_lsn}"
    6505              :         );
    6506          128 :         if let Some(check_start_lsn) = check_start_lsn {
    6507          128 :             assert!(lsn >= check_start_lsn);
    6508            0 :         }
    6509          380 :         images.sort_unstable_by(|(ka, _), (kb, _)| ka.cmp(kb));
    6510          128 :         let min_key = *images.first().map(|(k, _)| k).unwrap();
    6511          128 :         let end_key = images.last().map(|(k, _)| k).unwrap().next();
    6512          128 :         let mut image_layer_writer = ImageLayerWriter::new(
    6513          128 :             self.conf,
    6514          128 :             self.timeline_id,
    6515          128 :             self.tenant_shard_id,
    6516          128 :             &(min_key..end_key),
    6517          128 :             lsn,
    6518          128 :             ctx,
    6519          128 :         )
    6520          128 :         .await?;
    6521          636 :         for (key, img) in images {
    6522          508 :             image_layer_writer.put_image(key, img, ctx).await?;
    6523              :         }
    6524          128 :         let (desc, path) = image_layer_writer.finish(ctx).await?;
    6525          128 :         let image_layer = Layer::finish_creating(self.conf, self, desc, &path)?;
    6526          128 :         info!("force created image layer {}", image_layer.local_path());
    6527              :         {
    6528          128 :             let mut guard = self.layers.write().await;
    6529          128 :             guard
    6530          128 :                 .open_mut()
    6531          128 :                 .unwrap()
    6532          128 :                 .force_insert_layer(image_layer.clone());
    6533          128 :         }
    6534          128 : 
    6535          128 :         // Update remote_timeline_client state to reflect existence of this layer
    6536          128 :         self.remote_client
    6537          128 :             .schedule_layer_file_upload(image_layer)
    6538          128 :             .unwrap();
    6539          128 : 
    6540          128 :         Ok(())
    6541          128 :     }
    6542              : 
    6543              :     /// Force create a delta layer and place it into the layer map.
    6544              :     ///
    6545              :     /// DO NOT use this function directly. Use [`Tenant::branch_timeline_test_with_layers`]
    6546              :     /// or [`Tenant::create_test_timeline_with_layers`] to ensure all these layers are
    6547              :     /// placed into the layer map in one run AND be validated.
    6548              :     #[cfg(test)]
    6549          184 :     pub(super) async fn force_create_delta_layer(
    6550          184 :         self: &Arc<Timeline>,
    6551          184 :         mut deltas: DeltaLayerTestDesc,
    6552          184 :         check_start_lsn: Option<Lsn>,
    6553          184 :         ctx: &RequestContext,
    6554          184 :     ) -> anyhow::Result<()> {
    6555          184 :         let last_record_lsn = self.get_last_record_lsn();
    6556          184 :         deltas
    6557          184 :             .data
    6558          264 :             .sort_unstable_by(|(ka, la, _), (kb, lb, _)| (ka, la).cmp(&(kb, lb)));
    6559          184 :         assert!(deltas.data.first().unwrap().0 >= deltas.key_range.start);
    6560          184 :         assert!(deltas.data.last().unwrap().0 < deltas.key_range.end);
    6561          632 :         for (_, lsn, _) in &deltas.data {
    6562          448 :             assert!(deltas.lsn_range.start <= *lsn && *lsn < deltas.lsn_range.end);
    6563              :         }
    6564          184 :         assert!(
    6565          184 :             deltas.lsn_range.end <= last_record_lsn,
    6566            0 :             "advance last record lsn before inserting a layer, end_lsn={}, last_record_lsn={}",
    6567              :             deltas.lsn_range.end,
    6568              :             last_record_lsn
    6569              :         );
    6570          184 :         if let Some(check_start_lsn) = check_start_lsn {
    6571          184 :             assert!(deltas.lsn_range.start >= check_start_lsn);
    6572            0 :         }
    6573          184 :         let mut delta_layer_writer = DeltaLayerWriter::new(
    6574          184 :             self.conf,
    6575          184 :             self.timeline_id,
    6576          184 :             self.tenant_shard_id,
    6577          184 :             deltas.key_range.start,
    6578          184 :             deltas.lsn_range,
    6579          184 :             ctx,
    6580          184 :         )
    6581          184 :         .await?;
    6582          632 :         for (key, lsn, val) in deltas.data {
    6583          448 :             delta_layer_writer.put_value(key, lsn, val, ctx).await?;
    6584              :         }
    6585          184 :         let (desc, path) = delta_layer_writer.finish(deltas.key_range.end, ctx).await?;
    6586          184 :         let delta_layer = Layer::finish_creating(self.conf, self, desc, &path)?;
    6587          184 :         info!("force created delta layer {}", delta_layer.local_path());
    6588              :         {
    6589          184 :             let mut guard = self.layers.write().await;
    6590          184 :             guard
    6591          184 :                 .open_mut()
    6592          184 :                 .unwrap()
    6593          184 :                 .force_insert_layer(delta_layer.clone());
    6594          184 :         }
    6595          184 : 
    6596          184 :         // Update remote_timeline_client state to reflect existence of this layer
    6597          184 :         self.remote_client
    6598          184 :             .schedule_layer_file_upload(delta_layer)
    6599          184 :             .unwrap();
    6600          184 : 
    6601          184 :         Ok(())
    6602          184 :     }
    6603              : 
    6604              :     /// Force create an in-memory layer and place them into the layer map.
    6605              :     #[cfg(test)]
    6606            8 :     pub(super) async fn force_create_in_memory_layer(
    6607            8 :         self: &Arc<Timeline>,
    6608            8 :         mut in_memory: InMemoryLayerTestDesc,
    6609            8 :         check_start_lsn: Option<Lsn>,
    6610            8 :         ctx: &RequestContext,
    6611            8 :     ) -> anyhow::Result<()> {
    6612              :         use utils::bin_ser::BeSer;
    6613              : 
    6614              :         // Validate LSNs
    6615            8 :         if let Some(check_start_lsn) = check_start_lsn {
    6616            8 :             assert!(in_memory.lsn_range.start >= check_start_lsn);
    6617            0 :         }
    6618              : 
    6619            8 :         let last_record_lsn = self.get_last_record_lsn();
    6620            8 :         let layer_end_lsn = if in_memory.is_open {
    6621            4 :             in_memory
    6622            4 :                 .data
    6623            4 :                 .iter()
    6624           40 :                 .map(|(_key, lsn, _value)| lsn)
    6625            4 :                 .max()
    6626            4 :                 .cloned()
    6627              :         } else {
    6628            4 :             Some(in_memory.lsn_range.end)
    6629              :         };
    6630              : 
    6631            8 :         if let Some(end) = layer_end_lsn {
    6632            8 :             assert!(
    6633            8 :                 end <= last_record_lsn,
    6634            0 :                 "advance last record lsn before inserting a layer, end_lsn={}, last_record_lsn={}",
    6635              :                 end,
    6636              :                 last_record_lsn,
    6637              :             );
    6638            0 :         }
    6639              : 
    6640           80 :         in_memory.data.iter().for_each(|(_key, lsn, _value)| {
    6641           80 :             assert!(*lsn >= in_memory.lsn_range.start);
    6642           80 :             assert!(*lsn < in_memory.lsn_range.end);
    6643           80 :         });
    6644            8 : 
    6645            8 :         // Build the batch
    6646            8 :         in_memory
    6647            8 :             .data
    6648           72 :             .sort_unstable_by(|(ka, la, _), (kb, lb, _)| (ka, la).cmp(&(kb, lb)));
    6649            8 : 
    6650            8 :         let data = in_memory
    6651            8 :             .data
    6652            8 :             .into_iter()
    6653           80 :             .map(|(key, lsn, value)| {
    6654           80 :                 let value_size = value.serialized_size().unwrap() as usize;
    6655           80 :                 (key.to_compact(), lsn, value_size, value)
    6656           80 :             })
    6657            8 :             .collect::<Vec<_>>();
    6658            8 : 
    6659            8 :         let batch = SerializedValueBatch::from_values(data);
    6660              : 
    6661              :         // Create the in-memory layer and write the batch into it
    6662            8 :         let layer = InMemoryLayer::create(
    6663            8 :             self.conf,
    6664            8 :             self.timeline_id,
    6665            8 :             self.tenant_shard_id,
    6666            8 :             in_memory.lsn_range.start,
    6667            8 :             &self.gate,
    6668            8 :             ctx,
    6669            8 :         )
    6670            8 :         .await
    6671            8 :         .unwrap();
    6672            8 : 
    6673            8 :         layer.put_batch(batch, ctx).await.unwrap();
    6674            8 :         if !in_memory.is_open {
    6675            4 :             layer.freeze(in_memory.lsn_range.end).await;
    6676            4 :         }
    6677              : 
    6678            8 :         info!("force created in-memory layer {:?}", in_memory.lsn_range);
    6679              : 
    6680              :         // Link the layer to the layer map
    6681              :         {
    6682            8 :             let mut guard = self.layers.write().await;
    6683            8 :             let layer_map = guard.open_mut().unwrap();
    6684            8 :             layer_map.force_insert_in_memory_layer(Arc::new(layer));
    6685            8 :         }
    6686            8 : 
    6687            8 :         Ok(())
    6688            8 :     }
    6689              : 
    6690              :     /// Return all keys at the LSN in the image layers
    6691              :     #[cfg(test)]
    6692           12 :     pub(crate) async fn inspect_image_layers(
    6693           12 :         self: &Arc<Timeline>,
    6694           12 :         lsn: Lsn,
    6695           12 :         ctx: &RequestContext,
    6696           12 :         io_concurrency: IoConcurrency,
    6697           12 :     ) -> anyhow::Result<Vec<(Key, Bytes)>> {
    6698           12 :         let mut all_data = Vec::new();
    6699           12 :         let guard = self.layers.read().await;
    6700           68 :         for layer in guard.layer_map()?.iter_historic_layers() {
    6701           68 :             if !layer.is_delta() && layer.image_layer_lsn() == lsn {
    6702           16 :                 let layer = guard.get_from_desc(&layer);
    6703           16 :                 let mut reconstruct_data = ValuesReconstructState::new(io_concurrency.clone());
    6704           16 :                 layer
    6705           16 :                     .get_values_reconstruct_data(
    6706           16 :                         KeySpace::single(Key::MIN..Key::MAX),
    6707           16 :                         lsn..Lsn(lsn.0 + 1),
    6708           16 :                         &mut reconstruct_data,
    6709           16 :                         ctx,
    6710           16 :                     )
    6711           16 :                     .await?;
    6712          132 :                 for (k, v) in std::mem::take(&mut reconstruct_data.keys) {
    6713          132 :                     let v = v.collect_pending_ios().await?;
    6714          132 :                     all_data.push((k, v.img.unwrap().1));
    6715              :                 }
    6716           52 :             }
    6717              :         }
    6718           12 :         all_data.sort();
    6719           12 :         Ok(all_data)
    6720           12 :     }
    6721              : 
    6722              :     /// Get all historic layer descriptors in the layer map
    6723              :     #[cfg(test)]
    6724           48 :     pub(crate) async fn inspect_historic_layers(
    6725           48 :         self: &Arc<Timeline>,
    6726           48 :     ) -> anyhow::Result<Vec<super::storage_layer::PersistentLayerKey>> {
    6727           48 :         let mut layers = Vec::new();
    6728           48 :         let guard = self.layers.read().await;
    6729          228 :         for layer in guard.layer_map()?.iter_historic_layers() {
    6730          228 :             layers.push(layer.key());
    6731          228 :         }
    6732           48 :         Ok(layers)
    6733           48 :     }
    6734              : 
    6735              :     #[cfg(test)]
    6736           20 :     pub(crate) fn add_extra_test_dense_keyspace(&self, ks: KeySpace) {
    6737           20 :         let mut keyspace = self.extra_test_dense_keyspace.load().as_ref().clone();
    6738           20 :         keyspace.merge(&ks);
    6739           20 :         self.extra_test_dense_keyspace.store(Arc::new(keyspace));
    6740           20 :     }
    6741              : }
    6742              : 
    6743              : /// Tracking writes ingestion does to a particular in-memory layer.
    6744              : ///
    6745              : /// Cleared upon freezing a layer.
    6746              : pub(crate) struct TimelineWriterState {
    6747              :     open_layer: Arc<InMemoryLayer>,
    6748              :     current_size: u64,
    6749              :     // Previous Lsn which passed through
    6750              :     prev_lsn: Option<Lsn>,
    6751              :     // Largest Lsn which passed through the current writer
    6752              :     max_lsn: Option<Lsn>,
    6753              :     // Cached details of the last freeze. Avoids going trough the atomic/lock on every put.
    6754              :     cached_last_freeze_at: Lsn,
    6755              : }
    6756              : 
    6757              : impl TimelineWriterState {
    6758         2604 :     fn new(open_layer: Arc<InMemoryLayer>, current_size: u64, last_freeze_at: Lsn) -> Self {
    6759         2604 :         Self {
    6760         2604 :             open_layer,
    6761         2604 :             current_size,
    6762         2604 :             prev_lsn: None,
    6763         2604 :             max_lsn: None,
    6764         2604 :             cached_last_freeze_at: last_freeze_at,
    6765         2604 :         }
    6766         2604 :     }
    6767              : }
    6768              : 
    6769              : /// Various functions to mutate the timeline.
    6770              : // TODO Currently, Deref is used to allow easy access to read methods from this trait.
    6771              : // This is probably considered a bad practice in Rust and should be fixed eventually,
    6772              : // but will cause large code changes.
    6773              : pub(crate) struct TimelineWriter<'a> {
    6774              :     tl: &'a Timeline,
    6775              :     write_guard: tokio::sync::MutexGuard<'a, Option<TimelineWriterState>>,
    6776              : }
    6777              : 
    6778              : impl Deref for TimelineWriter<'_> {
    6779              :     type Target = Timeline;
    6780              : 
    6781     19796508 :     fn deref(&self) -> &Self::Target {
    6782     19796508 :         self.tl
    6783     19796508 :     }
    6784              : }
    6785              : 
    6786              : #[derive(PartialEq)]
    6787              : enum OpenLayerAction {
    6788              :     Roll,
    6789              :     Open,
    6790              :     None,
    6791              : }
    6792              : 
    6793              : impl TimelineWriter<'_> {
    6794      9608476 :     async fn handle_open_layer_action(
    6795      9608476 :         &mut self,
    6796      9608476 :         at: Lsn,
    6797      9608476 :         action: OpenLayerAction,
    6798      9608476 :         ctx: &RequestContext,
    6799      9608476 :     ) -> anyhow::Result<&Arc<InMemoryLayer>> {
    6800      9608476 :         match action {
    6801              :             OpenLayerAction::Roll => {
    6802          160 :                 let freeze_at = self.write_guard.as_ref().unwrap().max_lsn.unwrap();
    6803          160 :                 self.roll_layer(freeze_at).await?;
    6804          160 :                 self.open_layer(at, ctx).await?;
    6805              :             }
    6806         2444 :             OpenLayerAction::Open => self.open_layer(at, ctx).await?,
    6807              :             OpenLayerAction::None => {
    6808      9605872 :                 assert!(self.write_guard.is_some());
    6809              :             }
    6810              :         }
    6811              : 
    6812      9608476 :         Ok(&self.write_guard.as_ref().unwrap().open_layer)
    6813      9608476 :     }
    6814              : 
    6815         2604 :     async fn open_layer(&mut self, at: Lsn, ctx: &RequestContext) -> anyhow::Result<()> {
    6816         2604 :         let layer = self
    6817         2604 :             .tl
    6818         2604 :             .get_layer_for_write(at, &self.write_guard, ctx)
    6819         2604 :             .await?;
    6820         2604 :         let initial_size = layer.size().await?;
    6821              : 
    6822         2604 :         let last_freeze_at = self.last_freeze_at.load();
    6823         2604 :         self.write_guard.replace(TimelineWriterState::new(
    6824         2604 :             layer,
    6825         2604 :             initial_size,
    6826         2604 :             last_freeze_at,
    6827         2604 :         ));
    6828         2604 : 
    6829         2604 :         Ok(())
    6830         2604 :     }
    6831              : 
    6832          160 :     async fn roll_layer(&mut self, freeze_at: Lsn) -> Result<(), FlushLayerError> {
    6833          160 :         let current_size = self.write_guard.as_ref().unwrap().current_size;
    6834              : 
    6835              :         // If layer flushes are backpressured due to compaction not keeping up, wait for the flush
    6836              :         // to propagate the backpressure up into WAL ingestion.
    6837          160 :         let l0_count = self
    6838          160 :             .tl
    6839          160 :             .layers
    6840          160 :             .read()
    6841          160 :             .await
    6842          160 :             .layer_map()?
    6843          160 :             .level0_deltas()
    6844          160 :             .len();
    6845          160 :         let wait_thresholds = [
    6846          160 :             self.get_l0_flush_delay_threshold(),
    6847          160 :             self.get_l0_flush_stall_threshold(),
    6848          160 :         ];
    6849          160 :         let wait_threshold = wait_thresholds.into_iter().flatten().min();
    6850              : 
    6851              :         // self.write_guard will be taken by the freezing
    6852          160 :         let flush_id = self
    6853          160 :             .tl
    6854          160 :             .freeze_inmem_layer_at(freeze_at, &mut self.write_guard)
    6855          160 :             .await?;
    6856              : 
    6857          160 :         assert!(self.write_guard.is_none());
    6858              : 
    6859          160 :         if let Some(wait_threshold) = wait_threshold {
    6860            0 :             if l0_count >= wait_threshold {
    6861            0 :                 debug!(
    6862            0 :                     "layer roll waiting for flush due to compaction backpressure at {l0_count} L0 layers"
    6863              :                 );
    6864            0 :                 self.tl.wait_flush_completion(flush_id).await?;
    6865            0 :             }
    6866          160 :         }
    6867              : 
    6868          160 :         if current_size >= self.get_checkpoint_distance() * 2 {
    6869            0 :             warn!("Flushed oversized open layer with size {}", current_size)
    6870          160 :         }
    6871              : 
    6872          160 :         Ok(())
    6873          160 :     }
    6874              : 
    6875      9608476 :     fn get_open_layer_action(&self, lsn: Lsn, new_value_size: u64) -> OpenLayerAction {
    6876      9608476 :         let state = &*self.write_guard;
    6877      9608476 :         let Some(state) = &state else {
    6878         2444 :             return OpenLayerAction::Open;
    6879              :         };
    6880              : 
    6881              :         #[cfg(feature = "testing")]
    6882      9606032 :         if state.cached_last_freeze_at < self.tl.last_freeze_at.load() {
    6883              :             // this check and assertion are not really needed because
    6884              :             // LayerManager::try_freeze_in_memory_layer will always clear out the
    6885              :             // TimelineWriterState if something is frozen. however, we can advance last_freeze_at when there
    6886              :             // is no TimelineWriterState.
    6887            0 :             assert!(
    6888            0 :                 state.open_layer.end_lsn.get().is_some(),
    6889            0 :                 "our open_layer must be outdated"
    6890              :             );
    6891              : 
    6892              :             // this would be a memory leak waiting to happen because the in-memory layer always has
    6893              :             // an index
    6894            0 :             panic!("BUG: TimelineWriterState held on to frozen in-memory layer.");
    6895      9606032 :         }
    6896      9606032 : 
    6897      9606032 :         if state.prev_lsn == Some(lsn) {
    6898              :             // Rolling mid LSN is not supported by [downstream code].
    6899              :             // Hence, only roll at LSN boundaries.
    6900              :             //
    6901              :             // [downstream code]: https://github.com/neondatabase/neon/pull/7993#discussion_r1633345422
    6902           12 :             return OpenLayerAction::None;
    6903      9606020 :         }
    6904      9606020 : 
    6905      9606020 :         if state.current_size == 0 {
    6906              :             // Don't roll empty layers
    6907            0 :             return OpenLayerAction::None;
    6908      9606020 :         }
    6909      9606020 : 
    6910      9606020 :         if self.tl.should_roll(
    6911      9606020 :             state.current_size,
    6912      9606020 :             state.current_size + new_value_size,
    6913      9606020 :             self.get_checkpoint_distance(),
    6914      9606020 :             lsn,
    6915      9606020 :             state.cached_last_freeze_at,
    6916      9606020 :             state.open_layer.get_opened_at(),
    6917      9606020 :         ) {
    6918          160 :             OpenLayerAction::Roll
    6919              :         } else {
    6920      9605860 :             OpenLayerAction::None
    6921              :         }
    6922      9608476 :     }
    6923              : 
    6924              :     /// Put a batch of keys at the specified Lsns.
    6925      9608472 :     pub(crate) async fn put_batch(
    6926      9608472 :         &mut self,
    6927      9608472 :         batch: SerializedValueBatch,
    6928      9608472 :         ctx: &RequestContext,
    6929      9608472 :     ) -> anyhow::Result<()> {
    6930      9608472 :         if !batch.has_data() {
    6931            0 :             return Ok(());
    6932      9608472 :         }
    6933      9608472 : 
    6934      9608472 :         // In debug builds, assert that we don't write any keys that don't belong to this shard.
    6935      9608472 :         // We don't assert this in release builds, since key ownership policies may change over
    6936      9608472 :         // time. Stray keys will be removed during compaction.
    6937      9608472 :         if cfg!(debug_assertions) {
    6938     19789888 :             for metadata in &batch.metadata {
    6939     10181416 :                 if let ValueMeta::Serialized(metadata) = metadata {
    6940     10181416 :                     let key = Key::from_compact(metadata.key);
    6941     10181416 :                     assert!(
    6942     10181416 :                         self.shard_identity.is_key_local(&key)
    6943            0 :                             || self.shard_identity.is_key_global(&key),
    6944            0 :                         "key {key} does not belong on shard {}",
    6945            0 :                         self.shard_identity.shard_index()
    6946              :                     );
    6947            0 :                 }
    6948              :             }
    6949            0 :         }
    6950              : 
    6951      9608472 :         let batch_max_lsn = batch.max_lsn;
    6952      9608472 :         let buf_size: u64 = batch.buffer_size() as u64;
    6953      9608472 : 
    6954      9608472 :         let action = self.get_open_layer_action(batch_max_lsn, buf_size);
    6955      9608472 :         let layer = self
    6956      9608472 :             .handle_open_layer_action(batch_max_lsn, action, ctx)
    6957      9608472 :             .await?;
    6958              : 
    6959      9608472 :         let res = layer.put_batch(batch, ctx).await;
    6960              : 
    6961      9608472 :         if res.is_ok() {
    6962      9608472 :             // Update the current size only when the entire write was ok.
    6963      9608472 :             // In case of failures, we may have had partial writes which
    6964      9608472 :             // render the size tracking out of sync. That's ok because
    6965      9608472 :             // the checkpoint distance should be significantly smaller
    6966      9608472 :             // than the S3 single shot upload limit of 5GiB.
    6967      9608472 :             let state = self.write_guard.as_mut().unwrap();
    6968      9608472 : 
    6969      9608472 :             state.current_size += buf_size;
    6970      9608472 :             state.prev_lsn = Some(batch_max_lsn);
    6971      9608472 :             state.max_lsn = std::cmp::max(state.max_lsn, Some(batch_max_lsn));
    6972      9608472 :         }
    6973              : 
    6974      9608472 :         res
    6975      9608472 :     }
    6976              : 
    6977              :     #[cfg(test)]
    6978              :     /// Test helper, for tests that would like to poke individual values without composing a batch
    6979      8780308 :     pub(crate) async fn put(
    6980      8780308 :         &mut self,
    6981      8780308 :         key: Key,
    6982      8780308 :         lsn: Lsn,
    6983      8780308 :         value: &Value,
    6984      8780308 :         ctx: &RequestContext,
    6985      8780308 :     ) -> anyhow::Result<()> {
    6986              :         use utils::bin_ser::BeSer;
    6987      8780308 :         if !key.is_valid_key_on_write_path() {
    6988            0 :             bail!(
    6989            0 :                 "the request contains data not supported by pageserver at TimelineWriter::put: {}",
    6990            0 :                 key
    6991            0 :             );
    6992      8780308 :         }
    6993      8780308 :         let val_ser_size = value.serialized_size().unwrap() as usize;
    6994      8780308 :         let batch = SerializedValueBatch::from_values(vec![(
    6995      8780308 :             key.to_compact(),
    6996      8780308 :             lsn,
    6997      8780308 :             val_ser_size,
    6998      8780308 :             value.clone(),
    6999      8780308 :         )]);
    7000      8780308 : 
    7001      8780308 :         self.put_batch(batch, ctx).await
    7002      8780308 :     }
    7003              : 
    7004            4 :     pub(crate) async fn delete_batch(
    7005            4 :         &mut self,
    7006            4 :         batch: &[(Range<Key>, Lsn)],
    7007            4 :         ctx: &RequestContext,
    7008            4 :     ) -> anyhow::Result<()> {
    7009            4 :         if let Some((_, lsn)) = batch.first() {
    7010            4 :             let action = self.get_open_layer_action(*lsn, 0);
    7011            4 :             let layer = self.handle_open_layer_action(*lsn, action, ctx).await?;
    7012            4 :             layer.put_tombstones(batch).await?;
    7013            0 :         }
    7014              : 
    7015            4 :         Ok(())
    7016            4 :     }
    7017              : 
    7018              :     /// Track the end of the latest digested WAL record.
    7019              :     /// Remember the (end of) last valid WAL record remembered in the timeline.
    7020              :     ///
    7021              :     /// Call this after you have finished writing all the WAL up to 'lsn'.
    7022              :     ///
    7023              :     /// 'lsn' must be aligned. This wakes up any wait_lsn() callers waiting for
    7024              :     /// the 'lsn' or anything older. The previous last record LSN is stored alongside
    7025              :     /// the latest and can be read.
    7026     10558196 :     pub(crate) fn finish_write(&self, new_lsn: Lsn) {
    7027     10558196 :         self.tl.finish_write(new_lsn);
    7028     10558196 :     }
    7029              : 
    7030       541140 :     pub(crate) fn update_current_logical_size(&self, delta: i64) {
    7031       541140 :         self.tl.update_current_logical_size(delta)
    7032       541140 :     }
    7033              : }
    7034              : 
    7035              : // We need TimelineWriter to be send in upcoming conversion of
    7036              : // Timeline::layers to tokio::sync::RwLock.
    7037              : #[test]
    7038            4 : fn is_send() {
    7039            4 :     fn _assert_send<T: Send>() {}
    7040            4 :     _assert_send::<TimelineWriter<'_>>();
    7041            4 : }
    7042              : 
    7043              : #[cfg(test)]
    7044              : mod tests {
    7045              :     use std::sync::Arc;
    7046              : 
    7047              :     use pageserver_api::key::Key;
    7048              :     use pageserver_api::value::Value;
    7049              :     use std::iter::Iterator;
    7050              :     use tracing::Instrument;
    7051              :     use utils::id::TimelineId;
    7052              :     use utils::lsn::Lsn;
    7053              : 
    7054              :     use super::HeatMapTimeline;
    7055              :     use crate::context::RequestContextBuilder;
    7056              :     use crate::tenant::harness::{TenantHarness, test_img};
    7057              :     use crate::tenant::layer_map::LayerMap;
    7058              :     use crate::tenant::storage_layer::{Layer, LayerName, LayerVisibilityHint};
    7059              :     use crate::tenant::timeline::{DeltaLayerTestDesc, EvictionError};
    7060              :     use crate::tenant::{PreviousHeatmap, Timeline};
    7061              : 
    7062           20 :     fn assert_heatmaps_have_same_layers(lhs: &HeatMapTimeline, rhs: &HeatMapTimeline) {
    7063           20 :         assert_eq!(lhs.all_layers().count(), rhs.all_layers().count());
    7064           20 :         let lhs_rhs = lhs.all_layers().zip(rhs.all_layers());
    7065          100 :         for (l, r) in lhs_rhs {
    7066           80 :             assert_eq!(l.name, r.name);
    7067           80 :             assert_eq!(l.metadata, r.metadata);
    7068              :         }
    7069           20 :     }
    7070              : 
    7071              :     #[tokio::test]
    7072            4 :     async fn test_heatmap_generation() {
    7073            4 :         let harness = TenantHarness::create("heatmap_generation").await.unwrap();
    7074            4 : 
    7075            4 :         let covered_delta = DeltaLayerTestDesc::new_with_inferred_key_range(
    7076            4 :             Lsn(0x10)..Lsn(0x20),
    7077            4 :             vec![(
    7078            4 :                 Key::from_hex("620000000033333333444444445500000000").unwrap(),
    7079            4 :                 Lsn(0x11),
    7080            4 :                 Value::Image(test_img("foo")),
    7081            4 :             )],
    7082            4 :         );
    7083            4 :         let visible_delta = DeltaLayerTestDesc::new_with_inferred_key_range(
    7084            4 :             Lsn(0x10)..Lsn(0x20),
    7085            4 :             vec![(
    7086            4 :                 Key::from_hex("720000000033333333444444445500000000").unwrap(),
    7087            4 :                 Lsn(0x11),
    7088            4 :                 Value::Image(test_img("foo")),
    7089            4 :             )],
    7090            4 :         );
    7091            4 :         let l0_delta = DeltaLayerTestDesc::new(
    7092            4 :             Lsn(0x20)..Lsn(0x30),
    7093            4 :             Key::from_hex("000000000000000000000000000000000000").unwrap()
    7094            4 :                 ..Key::from_hex("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF").unwrap(),
    7095            4 :             vec![(
    7096            4 :                 Key::from_hex("720000000033333333444444445500000000").unwrap(),
    7097            4 :                 Lsn(0x25),
    7098            4 :                 Value::Image(test_img("foo")),
    7099            4 :             )],
    7100            4 :         );
    7101            4 :         let delta_layers = vec![
    7102            4 :             covered_delta.clone(),
    7103            4 :             visible_delta.clone(),
    7104            4 :             l0_delta.clone(),
    7105            4 :         ];
    7106            4 : 
    7107            4 :         let image_layer = (
    7108            4 :             Lsn(0x40),
    7109            4 :             vec![(
    7110            4 :                 Key::from_hex("620000000033333333444444445500000000").unwrap(),
    7111            4 :                 test_img("bar"),
    7112            4 :             )],
    7113            4 :         );
    7114            4 :         let image_layers = vec![image_layer];
    7115            4 : 
    7116            4 :         let (tenant, ctx) = harness.load().await;
    7117            4 :         let timeline = tenant
    7118            4 :             .create_test_timeline_with_layers(
    7119            4 :                 TimelineId::generate(),
    7120            4 :                 Lsn(0x10),
    7121            4 :                 14,
    7122            4 :                 &ctx,
    7123            4 :                 Vec::new(), // in-memory layers
    7124            4 :                 delta_layers,
    7125            4 :                 image_layers,
    7126            4 :                 Lsn(0x100),
    7127            4 :             )
    7128            4 :             .await
    7129            4 :             .unwrap();
    7130            4 :         let ctx = &ctx.with_scope_timeline(&timeline);
    7131            4 : 
    7132            4 :         // Layer visibility is an input to heatmap generation, so refresh it first
    7133            4 :         timeline.update_layer_visibility().await.unwrap();
    7134            4 : 
    7135            4 :         let heatmap = timeline
    7136            4 :             .generate_heatmap()
    7137            4 :             .await
    7138            4 :             .expect("Infallible while timeline is not shut down");
    7139            4 : 
    7140            4 :         assert_eq!(heatmap.timeline_id, timeline.timeline_id);
    7141            4 : 
    7142            4 :         // L0 should come last
    7143            4 :         let heatmap_layers = heatmap.all_layers().collect::<Vec<_>>();
    7144            4 :         assert_eq!(heatmap_layers.last().unwrap().name, l0_delta.layer_name());
    7145            4 : 
    7146            4 :         let mut last_lsn = Lsn::MAX;
    7147           20 :         for layer in heatmap_layers {
    7148            4 :             // Covered layer should be omitted
    7149           16 :             assert!(layer.name != covered_delta.layer_name());
    7150            4 : 
    7151           16 :             let layer_lsn = match &layer.name {
    7152            8 :                 LayerName::Delta(d) => d.lsn_range.end,
    7153            8 :                 LayerName::Image(i) => i.lsn,
    7154            4 :             };
    7155            4 : 
    7156            4 :             // Apart from L0s, newest Layers should come first
    7157           16 :             if !LayerMap::is_l0(layer.name.key_range(), layer.name.is_delta()) {
    7158           12 :                 assert!(layer_lsn <= last_lsn);
    7159           12 :                 last_lsn = layer_lsn;
    7160            4 :             }
    7161            4 :         }
    7162            4 : 
    7163            4 :         // Evict all the layers and stash the old heatmap in the timeline.
    7164            4 :         // This simulates a migration to a cold secondary location.
    7165            4 : 
    7166            4 :         let guard = timeline.layers.read().await;
    7167            4 :         let mut all_layers = Vec::new();
    7168            4 :         let forever = std::time::Duration::from_secs(120);
    7169           20 :         for layer in guard.likely_resident_layers() {
    7170           20 :             all_layers.push(layer.clone());
    7171           20 :             layer.evict_and_wait(forever).await.unwrap();
    7172            4 :         }
    7173            4 :         drop(guard);
    7174            4 : 
    7175            4 :         timeline
    7176            4 :             .previous_heatmap
    7177            4 :             .store(Some(Arc::new(PreviousHeatmap::Active {
    7178            4 :                 heatmap: heatmap.clone(),
    7179            4 :                 read_at: std::time::Instant::now(),
    7180            4 :                 end_lsn: None,
    7181            4 :             })));
    7182            4 : 
    7183            4 :         // Generate a new heatmap and assert that it contains the same layers as the old one.
    7184            4 :         let post_migration_heatmap = timeline.generate_heatmap().await.unwrap();
    7185            4 :         assert_heatmaps_have_same_layers(&heatmap, &post_migration_heatmap);
    7186            4 : 
    7187            4 :         // Download each layer one by one. Generate the heatmap at each step and check
    7188            4 :         // that it's stable.
    7189           24 :         for layer in all_layers {
    7190           20 :             if layer.visibility() == LayerVisibilityHint::Covered {
    7191            4 :                 continue;
    7192           16 :             }
    7193           16 : 
    7194           16 :             eprintln!("Downloading {layer} and re-generating heatmap");
    7195           16 : 
    7196           16 :             let ctx = &RequestContextBuilder::extend(ctx)
    7197           16 :                 .download_behavior(crate::context::DownloadBehavior::Download)
    7198           16 :                 .build();
    7199            4 : 
    7200           16 :             let _resident = layer
    7201           16 :                 .download_and_keep_resident(ctx)
    7202           16 :                 .instrument(tracing::info_span!(
    7203           16 :                     parent: None,
    7204            4 :                     "download_layer",
    7205            4 :                     tenant_id = %timeline.tenant_shard_id.tenant_id,
    7206            0 :                     shard_id = %timeline.tenant_shard_id.shard_slug(),
    7207            0 :                     timeline_id = %timeline.timeline_id
    7208            4 :                 ))
    7209           16 :                 .await
    7210           16 :                 .unwrap();
    7211            4 : 
    7212           16 :             let post_download_heatmap = timeline.generate_heatmap().await.unwrap();
    7213           16 :             assert_heatmaps_have_same_layers(&heatmap, &post_download_heatmap);
    7214            4 :         }
    7215            4 : 
    7216            4 :         // Everything from the post-migration heatmap is now resident.
    7217            4 :         // Check that we drop it from memory.
    7218            4 :         assert!(matches!(
    7219            4 :             timeline.previous_heatmap.load().as_deref(),
    7220            4 :             Some(PreviousHeatmap::Obsolete)
    7221            4 :         ));
    7222            4 :     }
    7223              : 
    7224              :     #[tokio::test]
    7225            4 :     async fn test_previous_heatmap_obsoletion() {
    7226            4 :         let harness = TenantHarness::create("heatmap_previous_heatmap_obsoletion")
    7227            4 :             .await
    7228            4 :             .unwrap();
    7229            4 : 
    7230            4 :         let l0_delta = DeltaLayerTestDesc::new(
    7231            4 :             Lsn(0x20)..Lsn(0x30),
    7232            4 :             Key::from_hex("000000000000000000000000000000000000").unwrap()
    7233            4 :                 ..Key::from_hex("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF").unwrap(),
    7234            4 :             vec![(
    7235            4 :                 Key::from_hex("720000000033333333444444445500000000").unwrap(),
    7236            4 :                 Lsn(0x25),
    7237            4 :                 Value::Image(test_img("foo")),
    7238            4 :             )],
    7239            4 :         );
    7240            4 : 
    7241            4 :         let image_layer = (
    7242            4 :             Lsn(0x40),
    7243            4 :             vec![(
    7244            4 :                 Key::from_hex("620000000033333333444444445500000000").unwrap(),
    7245            4 :                 test_img("bar"),
    7246            4 :             )],
    7247            4 :         );
    7248            4 : 
    7249            4 :         let delta_layers = vec![l0_delta];
    7250            4 :         let image_layers = vec![image_layer];
    7251            4 : 
    7252            4 :         let (tenant, ctx) = harness.load().await;
    7253            4 :         let timeline = tenant
    7254            4 :             .create_test_timeline_with_layers(
    7255            4 :                 TimelineId::generate(),
    7256            4 :                 Lsn(0x10),
    7257            4 :                 14,
    7258            4 :                 &ctx,
    7259            4 :                 Vec::new(), // in-memory layers
    7260            4 :                 delta_layers,
    7261            4 :                 image_layers,
    7262            4 :                 Lsn(0x100),
    7263            4 :             )
    7264            4 :             .await
    7265            4 :             .unwrap();
    7266            4 : 
    7267            4 :         // Layer visibility is an input to heatmap generation, so refresh it first
    7268            4 :         timeline.update_layer_visibility().await.unwrap();
    7269            4 : 
    7270            4 :         let heatmap = timeline
    7271            4 :             .generate_heatmap()
    7272            4 :             .await
    7273            4 :             .expect("Infallible while timeline is not shut down");
    7274            4 : 
    7275            4 :         // Both layers should be in the heatmap
    7276            4 :         assert!(heatmap.all_layers().count() > 0);
    7277            4 : 
    7278            4 :         // Now simulate a migration.
    7279            4 :         timeline
    7280            4 :             .previous_heatmap
    7281            4 :             .store(Some(Arc::new(PreviousHeatmap::Active {
    7282            4 :                 heatmap: heatmap.clone(),
    7283            4 :                 read_at: std::time::Instant::now(),
    7284            4 :                 end_lsn: None,
    7285            4 :             })));
    7286            4 : 
    7287            4 :         // Evict all the layers in the previous heatmap
    7288            4 :         let guard = timeline.layers.read().await;
    7289            4 :         let forever = std::time::Duration::from_secs(120);
    7290           12 :         for layer in guard.likely_resident_layers() {
    7291           12 :             layer.evict_and_wait(forever).await.unwrap();
    7292            4 :         }
    7293            4 :         drop(guard);
    7294            4 : 
    7295            4 :         // Generate a new heatmap and check that the previous heatmap
    7296            4 :         // has been marked obsolete.
    7297            4 :         let post_eviction_heatmap = timeline
    7298            4 :             .generate_heatmap()
    7299            4 :             .await
    7300            4 :             .expect("Infallible while timeline is not shut down");
    7301            4 : 
    7302            4 :         assert_eq!(post_eviction_heatmap.all_layers().count(), 0);
    7303            4 :         assert!(matches!(
    7304            4 :             timeline.previous_heatmap.load().as_deref(),
    7305            4 :             Some(PreviousHeatmap::Obsolete)
    7306            4 :         ));
    7307            4 :     }
    7308              : 
    7309              :     #[tokio::test]
    7310            4 :     async fn two_layer_eviction_attempts_at_the_same_time() {
    7311            4 :         let harness = TenantHarness::create("two_layer_eviction_attempts_at_the_same_time")
    7312            4 :             .await
    7313            4 :             .unwrap();
    7314            4 : 
    7315            4 :         let (tenant, ctx) = harness.load().await;
    7316            4 :         let timeline = tenant
    7317            4 :             .create_test_timeline(TimelineId::generate(), Lsn(0x10), 14, &ctx)
    7318            4 :             .await
    7319            4 :             .unwrap();
    7320            4 : 
    7321            4 :         let layer = find_some_layer(&timeline).await;
    7322            4 :         let layer = layer
    7323            4 :             .keep_resident()
    7324            4 :             .await
    7325            4 :             .expect("no download => no downloading errors")
    7326            4 :             .drop_eviction_guard();
    7327            4 : 
    7328            4 :         let forever = std::time::Duration::from_secs(120);
    7329            4 : 
    7330            4 :         let first = layer.evict_and_wait(forever);
    7331            4 :         let second = layer.evict_and_wait(forever);
    7332            4 : 
    7333            4 :         let (first, second) = tokio::join!(first, second);
    7334            4 : 
    7335            4 :         let res = layer.keep_resident().await;
    7336            4 :         assert!(res.is_none(), "{res:?}");
    7337            4 : 
    7338            4 :         match (first, second) {
    7339            4 :             (Ok(()), Ok(())) => {
    7340            4 :                 // because there are no more timeline locks being taken on eviction path, we can
    7341            4 :                 // witness all three outcomes here.
    7342            4 :             }
    7343            4 :             (Ok(()), Err(EvictionError::NotFound)) | (Err(EvictionError::NotFound), Ok(())) => {
    7344            0 :                 // if one completes before the other, this is fine just as well.
    7345            0 :             }
    7346            4 :             other => unreachable!("unexpected {:?}", other),
    7347            4 :         }
    7348            4 :     }
    7349              : 
    7350            4 :     async fn find_some_layer(timeline: &Timeline) -> Layer {
    7351            4 :         let layers = timeline.layers.read().await;
    7352            4 :         let desc = layers
    7353            4 :             .layer_map()
    7354            4 :             .unwrap()
    7355            4 :             .iter_historic_layers()
    7356            4 :             .next()
    7357            4 :             .expect("must find one layer to evict");
    7358            4 : 
    7359            4 :         layers.get_from_desc(&desc)
    7360            4 :     }
    7361              : }
        

Generated by: LCOV version 2.1-beta