LCOV - code coverage report
Current view: top level - pageserver/src/tenant - timeline.rs (source / functions) Coverage Total Hit
Test: f2bfe5dc5ab550768e936d6bc7b94d9b2e2d4cc9.info Lines: 62.6 % 3676 2302
Test Date: 2025-01-27 20:39:28 Functions: 59.8 % 323 193

            Line data    Source code
       1              : pub(crate) mod analysis;
       2              : pub(crate) mod compaction;
       3              : pub mod delete;
       4              : pub(crate) mod detach_ancestor;
       5              : mod eviction_task;
       6              : pub(crate) mod handle;
       7              : pub(crate) mod import_pgdata;
       8              : mod init;
       9              : pub mod layer_manager;
      10              : pub(crate) mod logical_size;
      11              : pub mod offload;
      12              : pub mod span;
      13              : pub mod uninit;
      14              : mod walreceiver;
      15              : 
      16              : use anyhow::{anyhow, bail, ensure, Context, Result};
      17              : use arc_swap::{ArcSwap, ArcSwapOption};
      18              : use bytes::Bytes;
      19              : use camino::Utf8Path;
      20              : use chrono::{DateTime, Utc};
      21              : use enumset::EnumSet;
      22              : use fail::fail_point;
      23              : use futures::{stream::FuturesUnordered, StreamExt};
      24              : use handle::ShardTimelineId;
      25              : use layer_manager::Shutdown;
      26              : use offload::OffloadError;
      27              : use once_cell::sync::Lazy;
      28              : use pageserver_api::models::PageTraceEvent;
      29              : use pageserver_api::{
      30              :     key::{
      31              :         KEY_SIZE, METADATA_KEY_BEGIN_PREFIX, METADATA_KEY_END_PREFIX, NON_INHERITED_RANGE,
      32              :         SPARSE_RANGE,
      33              :     },
      34              :     keyspace::{KeySpaceAccum, KeySpaceRandomAccum, SparseKeyPartitioning},
      35              :     models::{
      36              :         CompactKeyRange, CompactLsnRange, CompactionAlgorithm, CompactionAlgorithmSettings,
      37              :         DownloadRemoteLayersTaskInfo, DownloadRemoteLayersTaskSpawnRequest, EvictionPolicy,
      38              :         InMemoryLayerInfo, LayerMapInfo, LsnLease, TimelineState,
      39              :     },
      40              :     reltag::BlockNumber,
      41              :     shard::{ShardIdentity, ShardNumber, TenantShardId},
      42              : };
      43              : use rand::Rng;
      44              : use remote_storage::DownloadError;
      45              : use serde_with::serde_as;
      46              : use storage_broker::BrokerClientChannel;
      47              : use tokio::sync::mpsc::Sender;
      48              : use tokio::{
      49              :     runtime::Handle,
      50              :     sync::{oneshot, watch},
      51              : };
      52              : use tokio_util::sync::CancellationToken;
      53              : use tracing::*;
      54              : use utils::{
      55              :     fs_ext,
      56              :     guard_arc_swap::GuardArcSwap,
      57              :     pausable_failpoint,
      58              :     postgres_client::PostgresClientProtocol,
      59              :     sync::gate::{Gate, GateGuard},
      60              : };
      61              : use wal_decoder::serialized_batch::{SerializedValueBatch, ValueMeta};
      62              : 
      63              : use std::array;
      64              : use std::cmp::{max, min};
      65              : use std::collections::btree_map::Entry;
      66              : use std::collections::{BTreeMap, HashMap, HashSet};
      67              : use std::ops::{ControlFlow, Deref, Range};
      68              : use std::sync::atomic::{AtomicBool, AtomicU64, Ordering as AtomicOrdering};
      69              : use std::sync::{Arc, Mutex, OnceLock, RwLock, Weak};
      70              : use std::time::{Duration, Instant, SystemTime};
      71              : 
      72              : use crate::l0_flush::{self, L0FlushGlobalState};
      73              : use crate::{
      74              :     aux_file::AuxFileSizeEstimator,
      75              :     page_service::TenantManagerTypes,
      76              :     tenant::{
      77              :         config::AttachmentMode,
      78              :         layer_map::{LayerMap, SearchResult},
      79              :         metadata::TimelineMetadata,
      80              :         storage_layer::{
      81              :             inmemory_layer::IndexEntry, IoConcurrency, PersistentLayerDesc,
      82              :             ValueReconstructSituation,
      83              :         },
      84              :     },
      85              :     walingest::WalLagCooldown,
      86              :     walredo,
      87              : };
      88              : use crate::{
      89              :     context::{DownloadBehavior, RequestContext},
      90              :     disk_usage_eviction_task::DiskUsageEvictionInfo,
      91              :     pgdatadir_mapping::CollectKeySpaceError,
      92              : };
      93              : use crate::{
      94              :     disk_usage_eviction_task::finite_f32,
      95              :     tenant::storage_layer::{
      96              :         AsLayerDesc, DeltaLayerWriter, EvictionError, ImageLayerWriter, InMemoryLayer, Layer,
      97              :         LayerAccessStatsReset, LayerName, ResidentLayer, ValueReconstructState,
      98              :         ValuesReconstructState,
      99              :     },
     100              : };
     101              : use crate::{
     102              :     disk_usage_eviction_task::EvictionCandidate, tenant::storage_layer::delta_layer::DeltaEntry,
     103              : };
     104              : use crate::{
     105              :     metrics::ScanLatencyOngoingRecording, tenant::timeline::logical_size::CurrentLogicalSize,
     106              : };
     107              : use crate::{
     108              :     pgdatadir_mapping::DirectoryKind,
     109              :     virtual_file::{MaybeFatalIo, VirtualFile},
     110              : };
     111              : use crate::{pgdatadir_mapping::LsnForTimestamp, tenant::tasks::BackgroundLoopKind};
     112              : use crate::{pgdatadir_mapping::MAX_AUX_FILE_V2_DELTAS, tenant::storage_layer::PersistentLayerKey};
     113              : use pageserver_api::config::tenant_conf_defaults::DEFAULT_PITR_INTERVAL;
     114              : 
     115              : use crate::config::PageServerConf;
     116              : use crate::keyspace::{KeyPartitioning, KeySpace};
     117              : use crate::metrics::TimelineMetrics;
     118              : use crate::pgdatadir_mapping::CalculateLogicalSizeError;
     119              : use crate::tenant::config::TenantConfOpt;
     120              : use pageserver_api::reltag::RelTag;
     121              : use pageserver_api::shard::ShardIndex;
     122              : 
     123              : use postgres_connection::PgConnectionConfig;
     124              : use postgres_ffi::{to_pg_timestamp, v14::xlog_utils, WAL_SEGMENT_SIZE};
     125              : use utils::{
     126              :     completion,
     127              :     generation::Generation,
     128              :     id::TimelineId,
     129              :     lsn::{AtomicLsn, Lsn, RecordLsn},
     130              :     seqwait::SeqWait,
     131              :     simple_rcu::{Rcu, RcuReadGuard},
     132              : };
     133              : 
     134              : use crate::task_mgr;
     135              : use crate::task_mgr::TaskKind;
     136              : use crate::tenant::gc_result::GcResult;
     137              : use crate::ZERO_PAGE;
     138              : use pageserver_api::key::Key;
     139              : 
     140              : use self::delete::DeleteTimelineFlow;
     141              : pub(super) use self::eviction_task::EvictionTaskTenantState;
     142              : use self::eviction_task::EvictionTaskTimelineState;
     143              : use self::layer_manager::LayerManager;
     144              : use self::logical_size::LogicalSize;
     145              : use self::walreceiver::{WalReceiver, WalReceiverConf};
     146              : 
     147              : use super::config::TenantConf;
     148              : use super::remote_timeline_client::index::IndexPart;
     149              : use super::remote_timeline_client::RemoteTimelineClient;
     150              : use super::secondary::heatmap::{HeatMapLayer, HeatMapTimeline};
     151              : use super::storage_layer::{LayerFringe, LayerVisibilityHint, ReadableLayer};
     152              : use super::upload_queue::NotInitialized;
     153              : use super::GcError;
     154              : use super::{
     155              :     debug_assert_current_span_has_tenant_and_timeline_id, AttachedTenantConf, MaybeOffloaded,
     156              : };
     157              : 
     158              : #[cfg(test)]
     159              : use pageserver_api::value::Value;
     160              : 
     161              : #[derive(Debug, PartialEq, Eq, Clone, Copy)]
     162              : pub(crate) enum FlushLoopState {
     163              :     NotStarted,
     164              :     Running {
     165              :         #[cfg(test)]
     166              :         expect_initdb_optimization: bool,
     167              :         #[cfg(test)]
     168              :         initdb_optimization_count: usize,
     169              :     },
     170              :     Exited,
     171              : }
     172              : 
     173              : #[derive(Debug, Copy, Clone, PartialEq, Eq)]
     174              : pub enum ImageLayerCreationMode {
     175              :     /// Try to create image layers based on `time_for_new_image_layer`. Used in compaction code path.
     176              :     Try,
     177              :     /// Force creating the image layers if possible. For now, no image layers will be created
     178              :     /// for metadata keys. Used in compaction code path with force flag enabled.
     179              :     Force,
     180              :     /// Initial ingestion of the data, and no data should be dropped in this function. This
     181              :     /// means that no metadata keys should be included in the partitions. Used in flush frozen layer
     182              :     /// code path.
     183              :     Initial,
     184              : }
     185              : 
     186              : impl std::fmt::Display for ImageLayerCreationMode {
     187         1136 :     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
     188         1136 :         write!(f, "{:?}", self)
     189         1136 :     }
     190              : }
     191              : 
     192              : /// Temporary function for immutable storage state refactor, ensures we are dropping mutex guard instead of other things.
     193              : /// Can be removed after all refactors are done.
     194           56 : fn drop_rlock<T>(rlock: tokio::sync::RwLockReadGuard<T>) {
     195           56 :     drop(rlock)
     196           56 : }
     197              : 
     198              : /// Temporary function for immutable storage state refactor, ensures we are dropping mutex guard instead of other things.
     199              : /// Can be removed after all refactors are done.
     200         1192 : fn drop_wlock<T>(rlock: tokio::sync::RwLockWriteGuard<'_, T>) {
     201         1192 :     drop(rlock)
     202         1192 : }
     203              : 
     204              : /// The outward-facing resources required to build a Timeline
     205              : pub struct TimelineResources {
     206              :     pub remote_client: RemoteTimelineClient,
     207              :     pub pagestream_throttle: Arc<crate::tenant::throttle::Throttle>,
     208              :     pub pagestream_throttle_metrics: Arc<crate::metrics::tenant_throttling::Pagestream>,
     209              :     pub l0_flush_global_state: l0_flush::L0FlushGlobalState,
     210              : }
     211              : 
     212              : /// The relation size cache caches relation sizes at the end of the timeline. It speeds up WAL
     213              : /// ingestion considerably, because WAL ingestion needs to check on most records if the record
     214              : /// implicitly extends the relation.  At startup, `complete_as_of` is initialized to the current end
     215              : /// of the timeline (disk_consistent_lsn).  It's used on reads of relation sizes to check if the
     216              : /// value can be used to also update the cache, see [`Timeline::update_cached_rel_size`].
     217              : pub(crate) struct RelSizeCache {
     218              :     pub(crate) complete_as_of: Lsn,
     219              :     pub(crate) map: HashMap<RelTag, (Lsn, BlockNumber)>,
     220              : }
     221              : 
     222              : pub struct Timeline {
     223              :     pub(crate) conf: &'static PageServerConf,
     224              :     tenant_conf: Arc<ArcSwap<AttachedTenantConf>>,
     225              : 
     226              :     myself: Weak<Self>,
     227              : 
     228              :     pub(crate) tenant_shard_id: TenantShardId,
     229              :     pub timeline_id: TimelineId,
     230              : 
     231              :     /// The generation of the tenant that instantiated us: this is used for safety when writing remote objects.
     232              :     /// Never changes for the lifetime of this [`Timeline`] object.
     233              :     ///
     234              :     /// This duplicates the generation stored in LocationConf, but that structure is mutable:
     235              :     /// this copy enforces the invariant that generatio doesn't change during a Tenant's lifetime.
     236              :     pub(crate) generation: Generation,
     237              : 
     238              :     /// The detailed sharding information from our parent Tenant.  This enables us to map keys
     239              :     /// to shards, and is constant through the lifetime of this Timeline.
     240              :     shard_identity: ShardIdentity,
     241              : 
     242              :     pub pg_version: u32,
     243              : 
     244              :     /// The tuple has two elements.
     245              :     /// 1. `LayerFileManager` keeps track of the various physical representations of the layer files (inmem, local, remote).
     246              :     /// 2. `LayerMap`, the acceleration data structure for `get_reconstruct_data`.
     247              :     ///
     248              :     /// `LayerMap` maps out the `(PAGE,LSN) / (KEY,LSN)` space, which is composed of `(KeyRange, LsnRange)` rectangles.
     249              :     /// We describe these rectangles through the `PersistentLayerDesc` struct.
     250              :     ///
     251              :     /// When we want to reconstruct a page, we first find the `PersistentLayerDesc`'s that we need for page reconstruction,
     252              :     /// using `LayerMap`. Then, we use `LayerFileManager` to get the `PersistentLayer`'s that correspond to the
     253              :     /// `PersistentLayerDesc`'s.
     254              :     ///
     255              :     /// Hence, it's important to keep things coherent. The `LayerFileManager` must always have an entry for all
     256              :     /// `PersistentLayerDesc`'s in the `LayerMap`. If it doesn't, `LayerFileManager::get_from_desc` will panic at
     257              :     /// runtime, e.g., during page reconstruction.
     258              :     ///
     259              :     /// In the future, we'll be able to split up the tuple of LayerMap and `LayerFileManager`,
     260              :     /// so that e.g. on-demand-download/eviction, and layer spreading, can operate just on `LayerFileManager`.
     261              :     pub(crate) layers: tokio::sync::RwLock<LayerManager>,
     262              : 
     263              :     last_freeze_at: AtomicLsn,
     264              :     // Atomic would be more appropriate here.
     265              :     last_freeze_ts: RwLock<Instant>,
     266              : 
     267              :     pub(crate) standby_horizon: AtomicLsn,
     268              : 
     269              :     // WAL redo manager. `None` only for broken tenants.
     270              :     walredo_mgr: Option<Arc<super::WalRedoManager>>,
     271              : 
     272              :     /// Remote storage client.
     273              :     /// See [`remote_timeline_client`](super::remote_timeline_client) module comment for details.
     274              :     pub(crate) remote_client: Arc<RemoteTimelineClient>,
     275              : 
     276              :     // What page versions do we hold in the repository? If we get a
     277              :     // request > last_record_lsn, we need to wait until we receive all
     278              :     // the WAL up to the request. The SeqWait provides functions for
     279              :     // that. TODO: If we get a request for an old LSN, such that the
     280              :     // versions have already been garbage collected away, we should
     281              :     // throw an error, but we don't track that currently.
     282              :     //
     283              :     // last_record_lsn.load().last points to the end of last processed WAL record.
     284              :     //
     285              :     // We also remember the starting point of the previous record in
     286              :     // 'last_record_lsn.load().prev'. It's used to set the xl_prev pointer of the
     287              :     // first WAL record when the node is started up. But here, we just
     288              :     // keep track of it.
     289              :     last_record_lsn: SeqWait<RecordLsn, Lsn>,
     290              : 
     291              :     // All WAL records have been processed and stored durably on files on
     292              :     // local disk, up to this LSN. On crash and restart, we need to re-process
     293              :     // the WAL starting from this point.
     294              :     //
     295              :     // Some later WAL records might have been processed and also flushed to disk
     296              :     // already, so don't be surprised to see some, but there's no guarantee on
     297              :     // them yet.
     298              :     disk_consistent_lsn: AtomicLsn,
     299              : 
     300              :     // Parent timeline that this timeline was branched from, and the LSN
     301              :     // of the branch point.
     302              :     ancestor_timeline: Option<Arc<Timeline>>,
     303              :     ancestor_lsn: Lsn,
     304              : 
     305              :     pub(super) metrics: TimelineMetrics,
     306              : 
     307              :     // `Timeline` doesn't write these metrics itself, but it manages the lifetime.  Code
     308              :     // in `crate::page_service` writes these metrics.
     309              :     pub(crate) query_metrics: crate::metrics::SmgrQueryTimePerTimeline,
     310              : 
     311              :     directory_metrics: [AtomicU64; DirectoryKind::KINDS_NUM],
     312              : 
     313              :     /// Ensures layers aren't frozen by checkpointer between
     314              :     /// [`Timeline::get_layer_for_write`] and layer reads.
     315              :     /// Locked automatically by [`TimelineWriter`] and checkpointer.
     316              :     /// Must always be acquired before the layer map/individual layer lock
     317              :     /// to avoid deadlock.
     318              :     ///
     319              :     /// The state is cleared upon freezing.
     320              :     write_lock: tokio::sync::Mutex<Option<TimelineWriterState>>,
     321              : 
     322              :     /// Used to avoid multiple `flush_loop` tasks running
     323              :     pub(super) flush_loop_state: Mutex<FlushLoopState>,
     324              : 
     325              :     /// layer_flush_start_tx can be used to wake up the layer-flushing task.
     326              :     /// - The u64 value is a counter, incremented every time a new flush cycle is requested.
     327              :     ///   The flush cycle counter is sent back on the layer_flush_done channel when
     328              :     ///   the flush finishes. You can use that to wait for the flush to finish.
     329              :     /// - The LSN is updated to max() of its current value and the latest disk_consistent_lsn
     330              :     ///   read by whoever sends an update
     331              :     layer_flush_start_tx: tokio::sync::watch::Sender<(u64, Lsn)>,
     332              :     /// to be notified when layer flushing has finished, subscribe to the layer_flush_done channel
     333              :     layer_flush_done_tx: tokio::sync::watch::Sender<(u64, Result<(), FlushLayerError>)>,
     334              : 
     335              :     // Needed to ensure that we can't create a branch at a point that was already garbage collected
     336              :     pub latest_gc_cutoff_lsn: Rcu<Lsn>,
     337              : 
     338              :     // List of child timelines and their branch points. This is needed to avoid
     339              :     // garbage collecting data that is still needed by the child timelines.
     340              :     pub(crate) gc_info: std::sync::RwLock<GcInfo>,
     341              : 
     342              :     // It may change across major versions so for simplicity
     343              :     // keep it after running initdb for a timeline.
     344              :     // It is needed in checks when we want to error on some operations
     345              :     // when they are requested for pre-initdb lsn.
     346              :     // It can be unified with latest_gc_cutoff_lsn under some "first_valid_lsn",
     347              :     // though let's keep them both for better error visibility.
     348              :     pub initdb_lsn: Lsn,
     349              : 
     350              :     /// The repartitioning result. Allows a single writer and multiple readers.
     351              :     pub(crate) partitioning: GuardArcSwap<((KeyPartitioning, SparseKeyPartitioning), Lsn)>,
     352              : 
     353              :     /// Configuration: how often should the partitioning be recalculated.
     354              :     repartition_threshold: u64,
     355              : 
     356              :     last_image_layer_creation_check_at: AtomicLsn,
     357              :     last_image_layer_creation_check_instant: std::sync::Mutex<Option<Instant>>,
     358              : 
     359              :     /// Current logical size of the "datadir", at the last LSN.
     360              :     current_logical_size: LogicalSize,
     361              : 
     362              :     /// Information about the last processed message by the WAL receiver,
     363              :     /// or None if WAL receiver has not received anything for this timeline
     364              :     /// yet.
     365              :     pub last_received_wal: Mutex<Option<WalReceiverInfo>>,
     366              :     pub walreceiver: Mutex<Option<WalReceiver>>,
     367              : 
     368              :     /// Relation size cache
     369              :     pub(crate) rel_size_cache: RwLock<RelSizeCache>,
     370              : 
     371              :     download_all_remote_layers_task_info: RwLock<Option<DownloadRemoteLayersTaskInfo>>,
     372              : 
     373              :     state: watch::Sender<TimelineState>,
     374              : 
     375              :     /// Prevent two tasks from deleting the timeline at the same time. If held, the
     376              :     /// timeline is being deleted. If 'true', the timeline has already been deleted.
     377              :     pub delete_progress: TimelineDeleteProgress,
     378              : 
     379              :     eviction_task_timeline_state: tokio::sync::Mutex<EvictionTaskTimelineState>,
     380              : 
     381              :     /// Load or creation time information about the disk_consistent_lsn and when the loading
     382              :     /// happened. Used for consumption metrics.
     383              :     pub(crate) loaded_at: (Lsn, SystemTime),
     384              : 
     385              :     /// Gate to prevent shutdown completing while I/O is still happening to this timeline's data
     386              :     pub(crate) gate: Gate,
     387              : 
     388              :     /// Cancellation token scoped to this timeline: anything doing long-running work relating
     389              :     /// to the timeline should drop out when this token fires.
     390              :     pub(crate) cancel: CancellationToken,
     391              : 
     392              :     /// Make sure we only have one running compaction at a time in tests.
     393              :     ///
     394              :     /// Must only be taken in two places:
     395              :     /// - [`Timeline::compact`] (this file)
     396              :     /// - [`delete::delete_local_timeline_directory`]
     397              :     ///
     398              :     /// Timeline deletion will acquire both compaction and gc locks in whatever order.
     399              :     compaction_lock: tokio::sync::Mutex<()>,
     400              : 
     401              :     /// If true, the last compaction failed.
     402              :     compaction_failed: AtomicBool,
     403              : 
     404              :     /// Make sure we only have one running gc at a time.
     405              :     ///
     406              :     /// Must only be taken in two places:
     407              :     /// - [`Timeline::gc`] (this file)
     408              :     /// - [`delete::delete_local_timeline_directory`]
     409              :     ///
     410              :     /// Timeline deletion will acquire both compaction and gc locks in whatever order.
     411              :     gc_lock: tokio::sync::Mutex<()>,
     412              : 
     413              :     /// Cloned from [`super::Tenant::pagestream_throttle`] on construction.
     414              :     pub(crate) pagestream_throttle: Arc<crate::tenant::throttle::Throttle>,
     415              : 
     416              :     /// Size estimator for aux file v2
     417              :     pub(crate) aux_file_size_estimator: AuxFileSizeEstimator,
     418              : 
     419              :     /// Some test cases directly place keys into the timeline without actually modifying the directory
     420              :     /// keys (i.e., DB_DIR). The test cases creating such keys will put the keyspaces here, so that
     421              :     /// these keys won't get garbage-collected during compaction/GC. This field only modifies the dense
     422              :     /// keyspace return value of `collect_keyspace`. For sparse keyspaces, use AUX keys for testing, and
     423              :     /// in the future, add `extra_test_sparse_keyspace` if necessary.
     424              :     #[cfg(test)]
     425              :     pub(crate) extra_test_dense_keyspace: ArcSwap<KeySpace>,
     426              : 
     427              :     pub(crate) l0_flush_global_state: L0FlushGlobalState,
     428              : 
     429              :     pub(crate) handles: handle::PerTimelineState<TenantManagerTypes>,
     430              : 
     431              :     pub(crate) attach_wal_lag_cooldown: Arc<OnceLock<WalLagCooldown>>,
     432              : 
     433              :     /// Cf. [`crate::tenant::CreateTimelineIdempotency`].
     434              :     pub(crate) create_idempotency: crate::tenant::CreateTimelineIdempotency,
     435              : 
     436              :     /// If Some, collects GetPage metadata for an ongoing PageTrace.
     437              :     pub(crate) page_trace: ArcSwapOption<Sender<PageTraceEvent>>,
     438              : }
     439              : 
     440              : pub type TimelineDeleteProgress = Arc<tokio::sync::Mutex<DeleteTimelineFlow>>;
     441              : 
     442              : pub struct WalReceiverInfo {
     443              :     pub wal_source_connconf: PgConnectionConfig,
     444              :     pub last_received_msg_lsn: Lsn,
     445              :     pub last_received_msg_ts: u128,
     446              : }
     447              : 
     448              : /// Information about how much history needs to be retained, needed by
     449              : /// Garbage Collection.
     450              : #[derive(Default)]
     451              : pub(crate) struct GcInfo {
     452              :     /// Specific LSNs that are needed.
     453              :     ///
     454              :     /// Currently, this includes all points where child branches have
     455              :     /// been forked off from. In the future, could also include
     456              :     /// explicit user-defined snapshot points.
     457              :     pub(crate) retain_lsns: Vec<(Lsn, TimelineId, MaybeOffloaded)>,
     458              : 
     459              :     /// The cutoff coordinates, which are combined by selecting the minimum.
     460              :     pub(crate) cutoffs: GcCutoffs,
     461              : 
     462              :     /// Leases granted to particular LSNs.
     463              :     pub(crate) leases: BTreeMap<Lsn, LsnLease>,
     464              : 
     465              :     /// Whether our branch point is within our ancestor's PITR interval (for cost estimation)
     466              :     pub(crate) within_ancestor_pitr: bool,
     467              : }
     468              : 
     469              : impl GcInfo {
     470          564 :     pub(crate) fn min_cutoff(&self) -> Lsn {
     471          564 :         self.cutoffs.select_min()
     472          564 :     }
     473              : 
     474          464 :     pub(super) fn insert_child(
     475          464 :         &mut self,
     476          464 :         child_id: TimelineId,
     477          464 :         child_lsn: Lsn,
     478          464 :         is_offloaded: MaybeOffloaded,
     479          464 :     ) {
     480          464 :         self.retain_lsns.push((child_lsn, child_id, is_offloaded));
     481          464 :         self.retain_lsns.sort_by_key(|i| i.0);
     482          464 :     }
     483              : 
     484            8 :     pub(super) fn remove_child_maybe_offloaded(
     485            8 :         &mut self,
     486            8 :         child_id: TimelineId,
     487            8 :         maybe_offloaded: MaybeOffloaded,
     488            8 :     ) -> bool {
     489            8 :         // Remove at most one element. Needed for correctness if there is two live `Timeline` objects referencing
     490            8 :         // the same timeline. Shouldn't but maybe can occur when Arc's live longer than intended.
     491            8 :         let mut removed = false;
     492           12 :         self.retain_lsns.retain(|i| {
     493           12 :             if removed {
     494            4 :                 return true;
     495            8 :             }
     496            8 :             let remove = i.1 == child_id && i.2 == maybe_offloaded;
     497            8 :             removed |= remove;
     498            8 :             !remove
     499           12 :         });
     500            8 :         removed
     501            8 :     }
     502              : 
     503            8 :     pub(super) fn remove_child_not_offloaded(&mut self, child_id: TimelineId) -> bool {
     504            8 :         self.remove_child_maybe_offloaded(child_id, MaybeOffloaded::No)
     505            8 :     }
     506              : 
     507            0 :     pub(super) fn remove_child_offloaded(&mut self, child_id: TimelineId) -> bool {
     508            0 :         self.remove_child_maybe_offloaded(child_id, MaybeOffloaded::Yes)
     509            0 :     }
     510              : }
     511              : 
     512              : /// The `GcInfo` component describing which Lsns need to be retained.  Functionally, this
     513              : /// is a single number (the oldest LSN which we must retain), but it internally distinguishes
     514              : /// between time-based and space-based retention for observability and consumption metrics purposes.
     515              : #[derive(Debug, Clone)]
     516              : pub(crate) struct GcCutoffs {
     517              :     /// Calculated from the [`TenantConf::gc_horizon`], this LSN indicates how much
     518              :     /// history we must keep to retain a specified number of bytes of WAL.
     519              :     pub(crate) space: Lsn,
     520              : 
     521              :     /// Calculated from [`TenantConf::pitr_interval`], this LSN indicates how much
     522              :     /// history we must keep to enable reading back at least the PITR interval duration.
     523              :     pub(crate) time: Lsn,
     524              : }
     525              : 
     526              : impl Default for GcCutoffs {
     527          892 :     fn default() -> Self {
     528          892 :         Self {
     529          892 :             space: Lsn::INVALID,
     530          892 :             time: Lsn::INVALID,
     531          892 :         }
     532          892 :     }
     533              : }
     534              : 
     535              : impl GcCutoffs {
     536          564 :     fn select_min(&self) -> Lsn {
     537          564 :         std::cmp::min(self.space, self.time)
     538          564 :     }
     539              : }
     540              : 
     541              : pub(crate) struct TimelineVisitOutcome {
     542              :     completed_keyspace: KeySpace,
     543              :     image_covered_keyspace: KeySpace,
     544              : }
     545              : 
     546              : /// An error happened in a get() operation.
     547              : #[derive(thiserror::Error, Debug)]
     548              : pub(crate) enum PageReconstructError {
     549              :     #[error(transparent)]
     550              :     Other(anyhow::Error),
     551              : 
     552              :     #[error("Ancestor LSN wait error: {0}")]
     553              :     AncestorLsnTimeout(WaitLsnError),
     554              : 
     555              :     #[error("timeline shutting down")]
     556              :     Cancelled,
     557              : 
     558              :     /// An error happened replaying WAL records
     559              :     #[error(transparent)]
     560              :     WalRedo(anyhow::Error),
     561              : 
     562              :     #[error("{0}")]
     563              :     MissingKey(MissingKeyError),
     564              : }
     565              : 
     566              : impl From<anyhow::Error> for PageReconstructError {
     567            0 :     fn from(value: anyhow::Error) -> Self {
     568            0 :         // with walingest.rs many PageReconstructError are wrapped in as anyhow::Error
     569            0 :         match value.downcast::<PageReconstructError>() {
     570            0 :             Ok(pre) => pre,
     571            0 :             Err(other) => PageReconstructError::Other(other),
     572              :         }
     573            0 :     }
     574              : }
     575              : 
     576              : impl From<utils::bin_ser::DeserializeError> for PageReconstructError {
     577            0 :     fn from(value: utils::bin_ser::DeserializeError) -> Self {
     578            0 :         PageReconstructError::Other(anyhow::Error::new(value).context("deserialization failure"))
     579            0 :     }
     580              : }
     581              : 
     582              : impl From<layer_manager::Shutdown> for PageReconstructError {
     583            0 :     fn from(_: layer_manager::Shutdown) -> Self {
     584            0 :         PageReconstructError::Cancelled
     585            0 :     }
     586              : }
     587              : 
     588              : impl GetVectoredError {
     589              :     #[cfg(test)]
     590           12 :     pub(crate) fn is_missing_key_error(&self) -> bool {
     591           12 :         matches!(self, Self::MissingKey(_))
     592           12 :     }
     593              : }
     594              : 
     595              : impl From<layer_manager::Shutdown> for GetVectoredError {
     596            0 :     fn from(_: layer_manager::Shutdown) -> Self {
     597            0 :         GetVectoredError::Cancelled
     598            0 :     }
     599              : }
     600              : 
     601              : #[derive(thiserror::Error)]
     602              : pub struct MissingKeyError {
     603              :     key: Key,
     604              :     shard: ShardNumber,
     605              :     cont_lsn: Lsn,
     606              :     request_lsn: Lsn,
     607              :     ancestor_lsn: Option<Lsn>,
     608              :     backtrace: Option<std::backtrace::Backtrace>,
     609              : }
     610              : 
     611              : impl std::fmt::Debug for MissingKeyError {
     612            0 :     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
     613            0 :         write!(f, "{}", self)
     614            0 :     }
     615              : }
     616              : 
     617              : impl std::fmt::Display for MissingKeyError {
     618            0 :     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
     619            0 :         write!(
     620            0 :             f,
     621            0 :             "could not find data for key {} (shard {:?}) at LSN {}, request LSN {}",
     622            0 :             self.key, self.shard, self.cont_lsn, self.request_lsn
     623            0 :         )?;
     624            0 :         if let Some(ref ancestor_lsn) = self.ancestor_lsn {
     625            0 :             write!(f, ", ancestor {}", ancestor_lsn)?;
     626            0 :         }
     627              : 
     628            0 :         if let Some(ref backtrace) = self.backtrace {
     629            0 :             write!(f, "\n{}", backtrace)?;
     630            0 :         }
     631              : 
     632            0 :         Ok(())
     633            0 :     }
     634              : }
     635              : 
     636              : impl PageReconstructError {
     637              :     /// Returns true if this error indicates a tenant/timeline shutdown alike situation
     638            0 :     pub(crate) fn is_stopping(&self) -> bool {
     639              :         use PageReconstructError::*;
     640            0 :         match self {
     641            0 :             Cancelled => true,
     642            0 :             Other(_) | AncestorLsnTimeout(_) | WalRedo(_) | MissingKey(_) => false,
     643              :         }
     644            0 :     }
     645              : }
     646              : 
     647              : #[derive(thiserror::Error, Debug)]
     648              : pub(crate) enum CreateImageLayersError {
     649              :     #[error("timeline shutting down")]
     650              :     Cancelled,
     651              : 
     652              :     #[error("read failed")]
     653              :     GetVectoredError(#[source] GetVectoredError),
     654              : 
     655              :     #[error("reconstruction failed")]
     656              :     PageReconstructError(#[source] PageReconstructError),
     657              : 
     658              :     #[error(transparent)]
     659              :     Other(#[from] anyhow::Error),
     660              : }
     661              : 
     662              : impl From<layer_manager::Shutdown> for CreateImageLayersError {
     663            0 :     fn from(_: layer_manager::Shutdown) -> Self {
     664            0 :         CreateImageLayersError::Cancelled
     665            0 :     }
     666              : }
     667              : 
     668              : #[derive(thiserror::Error, Debug, Clone)]
     669              : pub(crate) enum FlushLayerError {
     670              :     /// Timeline cancellation token was cancelled
     671              :     #[error("timeline shutting down")]
     672              :     Cancelled,
     673              : 
     674              :     /// We tried to flush a layer while the Timeline is in an unexpected state
     675              :     #[error("cannot flush frozen layers when flush_loop is not running, state is {0:?}")]
     676              :     NotRunning(FlushLoopState),
     677              : 
     678              :     // Arc<> the following non-clonable error types: we must be Clone-able because the flush error is propagated from the flush
     679              :     // loop via a watch channel, where we can only borrow it.
     680              :     #[error("create image layers (shared)")]
     681              :     CreateImageLayersError(Arc<CreateImageLayersError>),
     682              : 
     683              :     #[error("other (shared)")]
     684              :     Other(#[from] Arc<anyhow::Error>),
     685              : }
     686              : 
     687              : impl FlushLayerError {
     688              :     // When crossing from generic anyhow errors to this error type, we explicitly check
     689              :     // for timeline cancellation to avoid logging inoffensive shutdown errors as warn/err.
     690            0 :     fn from_anyhow(timeline: &Timeline, err: anyhow::Error) -> Self {
     691            0 :         let cancelled = timeline.cancel.is_cancelled()
     692              :             // The upload queue might have been shut down before the official cancellation of the timeline.
     693            0 :             || err
     694            0 :                 .downcast_ref::<NotInitialized>()
     695            0 :                 .map(NotInitialized::is_stopping)
     696            0 :                 .unwrap_or_default();
     697            0 :         if cancelled {
     698            0 :             Self::Cancelled
     699              :         } else {
     700            0 :             Self::Other(Arc::new(err))
     701              :         }
     702            0 :     }
     703              : }
     704              : 
     705              : impl From<layer_manager::Shutdown> for FlushLayerError {
     706            0 :     fn from(_: layer_manager::Shutdown) -> Self {
     707            0 :         FlushLayerError::Cancelled
     708            0 :     }
     709              : }
     710              : 
     711              : #[derive(thiserror::Error, Debug)]
     712              : pub(crate) enum GetVectoredError {
     713              :     #[error("timeline shutting down")]
     714              :     Cancelled,
     715              : 
     716              :     #[error("requested too many keys: {0} > {}", Timeline::MAX_GET_VECTORED_KEYS)]
     717              :     Oversized(u64),
     718              : 
     719              :     #[error("requested at invalid LSN: {0}")]
     720              :     InvalidLsn(Lsn),
     721              : 
     722              :     #[error("requested key not found: {0}")]
     723              :     MissingKey(MissingKeyError),
     724              : 
     725              :     #[error("ancestry walk")]
     726              :     GetReadyAncestorError(#[source] GetReadyAncestorError),
     727              : 
     728              :     #[error(transparent)]
     729              :     Other(#[from] anyhow::Error),
     730              : }
     731              : 
     732              : impl From<GetReadyAncestorError> for GetVectoredError {
     733            4 :     fn from(value: GetReadyAncestorError) -> Self {
     734              :         use GetReadyAncestorError::*;
     735            4 :         match value {
     736            0 :             Cancelled => GetVectoredError::Cancelled,
     737              :             AncestorLsnTimeout(_) | BadState { .. } => {
     738            4 :                 GetVectoredError::GetReadyAncestorError(value)
     739              :             }
     740              :         }
     741            4 :     }
     742              : }
     743              : 
     744              : #[derive(thiserror::Error, Debug)]
     745              : pub(crate) enum GetReadyAncestorError {
     746              :     #[error("ancestor LSN wait error")]
     747              :     AncestorLsnTimeout(#[from] WaitLsnError),
     748              : 
     749              :     #[error("bad state on timeline {timeline_id}: {state:?}")]
     750              :     BadState {
     751              :         timeline_id: TimelineId,
     752              :         state: TimelineState,
     753              :     },
     754              : 
     755              :     #[error("cancelled")]
     756              :     Cancelled,
     757              : }
     758              : 
     759              : #[derive(Clone, Copy)]
     760              : pub enum LogicalSizeCalculationCause {
     761              :     Initial,
     762              :     ConsumptionMetricsSyntheticSize,
     763              :     EvictionTaskImitation,
     764              :     TenantSizeHandler,
     765              : }
     766              : 
     767              : pub enum GetLogicalSizePriority {
     768              :     User,
     769              :     Background,
     770              : }
     771              : 
     772            0 : #[derive(Debug, enumset::EnumSetType)]
     773              : pub(crate) enum CompactFlags {
     774              :     ForceRepartition,
     775              :     ForceImageLayerCreation,
     776              :     ForceL0Compaction,
     777              :     EnhancedGcBottomMostCompaction,
     778              :     DryRun,
     779              : }
     780              : 
     781              : #[serde_with::serde_as]
     782            0 : #[derive(Debug, Clone, serde::Deserialize)]
     783              : pub(crate) struct CompactRequest {
     784              :     pub compact_key_range: Option<CompactKeyRange>,
     785              :     pub compact_lsn_range: Option<CompactLsnRange>,
     786              :     /// Whether the compaction job should be scheduled.
     787              :     #[serde(default)]
     788              :     pub scheduled: bool,
     789              :     /// Whether the compaction job should be split across key ranges.
     790              :     #[serde(default)]
     791              :     pub sub_compaction: bool,
     792              :     /// Max job size for each subcompaction job.
     793              :     pub sub_compaction_max_job_size_mb: Option<u64>,
     794              : }
     795              : 
     796              : #[derive(Debug, Clone, Default)]
     797              : pub(crate) struct CompactOptions {
     798              :     pub flags: EnumSet<CompactFlags>,
     799              :     /// If set, the compaction will only compact the key range specified by this option.
     800              :     /// This option is only used by GC compaction. For the full explanation, see [`compaction::GcCompactJob`].
     801              :     pub compact_key_range: Option<CompactKeyRange>,
     802              :     /// If set, the compaction will only compact the LSN within this value.
     803              :     /// This option is only used by GC compaction. For the full explanation, see [`compaction::GcCompactJob`].
     804              :     pub compact_lsn_range: Option<CompactLsnRange>,
     805              :     /// Enable sub-compaction (split compaction job across key ranges).
     806              :     /// This option is only used by GC compaction.
     807              :     pub sub_compaction: bool,
     808              :     /// Set job size for the GC compaction.
     809              :     /// This option is only used by GC compaction.
     810              :     pub sub_compaction_max_job_size_mb: Option<u64>,
     811              : }
     812              : 
     813              : impl std::fmt::Debug for Timeline {
     814            0 :     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
     815            0 :         write!(f, "Timeline<{}>", self.timeline_id)
     816            0 :     }
     817              : }
     818              : 
     819              : #[derive(thiserror::Error, Debug)]
     820              : pub(crate) enum WaitLsnError {
     821              :     // Called on a timeline which is shutting down
     822              :     #[error("Shutdown")]
     823              :     Shutdown,
     824              : 
     825              :     // Called on an timeline not in active state or shutting down
     826              :     #[error("Bad timeline state: {0:?}")]
     827              :     BadState(TimelineState),
     828              : 
     829              :     // Timeout expired while waiting for LSN to catch up with goal.
     830              :     #[error("{0}")]
     831              :     Timeout(String),
     832              : }
     833              : 
     834              : // The impls below achieve cancellation mapping for errors.
     835              : // Perhaps there's a way of achieving this with less cruft.
     836              : 
     837              : impl From<CreateImageLayersError> for CompactionError {
     838            0 :     fn from(e: CreateImageLayersError) -> Self {
     839            0 :         match e {
     840            0 :             CreateImageLayersError::Cancelled => CompactionError::ShuttingDown,
     841            0 :             CreateImageLayersError::Other(e) => {
     842            0 :                 CompactionError::Other(e.context("create image layers"))
     843              :             }
     844            0 :             _ => CompactionError::Other(e.into()),
     845              :         }
     846            0 :     }
     847              : }
     848              : 
     849              : impl From<CreateImageLayersError> for FlushLayerError {
     850            0 :     fn from(e: CreateImageLayersError) -> Self {
     851            0 :         match e {
     852            0 :             CreateImageLayersError::Cancelled => FlushLayerError::Cancelled,
     853            0 :             any => FlushLayerError::CreateImageLayersError(Arc::new(any)),
     854              :         }
     855            0 :     }
     856              : }
     857              : 
     858              : impl From<PageReconstructError> for CreateImageLayersError {
     859            0 :     fn from(e: PageReconstructError) -> Self {
     860            0 :         match e {
     861            0 :             PageReconstructError::Cancelled => CreateImageLayersError::Cancelled,
     862            0 :             _ => CreateImageLayersError::PageReconstructError(e),
     863              :         }
     864            0 :     }
     865              : }
     866              : 
     867              : impl From<GetVectoredError> for CreateImageLayersError {
     868            0 :     fn from(e: GetVectoredError) -> Self {
     869            0 :         match e {
     870            0 :             GetVectoredError::Cancelled => CreateImageLayersError::Cancelled,
     871            0 :             _ => CreateImageLayersError::GetVectoredError(e),
     872              :         }
     873            0 :     }
     874              : }
     875              : 
     876              : impl From<GetVectoredError> for PageReconstructError {
     877           12 :     fn from(e: GetVectoredError) -> Self {
     878           12 :         match e {
     879            0 :             GetVectoredError::Cancelled => PageReconstructError::Cancelled,
     880            0 :             GetVectoredError::InvalidLsn(_) => PageReconstructError::Other(anyhow!("Invalid LSN")),
     881            0 :             err @ GetVectoredError::Oversized(_) => PageReconstructError::Other(err.into()),
     882            8 :             GetVectoredError::MissingKey(err) => PageReconstructError::MissingKey(err),
     883            4 :             GetVectoredError::GetReadyAncestorError(err) => PageReconstructError::from(err),
     884            0 :             GetVectoredError::Other(err) => PageReconstructError::Other(err),
     885              :         }
     886           12 :     }
     887              : }
     888              : 
     889              : impl From<GetReadyAncestorError> for PageReconstructError {
     890            4 :     fn from(e: GetReadyAncestorError) -> Self {
     891              :         use GetReadyAncestorError::*;
     892            4 :         match e {
     893            0 :             AncestorLsnTimeout(wait_err) => PageReconstructError::AncestorLsnTimeout(wait_err),
     894            4 :             bad_state @ BadState { .. } => PageReconstructError::Other(anyhow::anyhow!(bad_state)),
     895            0 :             Cancelled => PageReconstructError::Cancelled,
     896              :         }
     897            4 :     }
     898              : }
     899              : 
     900              : pub(crate) enum WaitLsnWaiter<'a> {
     901              :     Timeline(&'a Timeline),
     902              :     Tenant,
     903              :     PageService,
     904              : }
     905              : 
     906              : /// Argument to [`Timeline::shutdown`].
     907              : #[derive(Debug, Clone, Copy)]
     908              : pub(crate) enum ShutdownMode {
     909              :     /// Graceful shutdown, may do a lot of I/O as we flush any open layers to disk and then
     910              :     /// also to remote storage.  This method can easily take multiple seconds for a busy timeline.
     911              :     ///
     912              :     /// While we are flushing, we continue to accept read I/O for LSNs ingested before
     913              :     /// the call to [`Timeline::shutdown`].
     914              :     FreezeAndFlush,
     915              :     /// Only flush the layers to the remote storage without freezing any open layers. Flush the deletion
     916              :     /// queue. This is the mode used by ancestor detach and any other operations that reloads a tenant
     917              :     /// but not increasing the generation number. Note that this mode cannot be used at tenant shutdown,
     918              :     /// as flushing the deletion queue at that time will cause shutdown-in-progress errors.
     919              :     Reload,
     920              :     /// Shut down immediately, without waiting for any open layers to flush.
     921              :     Hard,
     922              : }
     923              : 
     924              : struct ImageLayerCreationOutcome {
     925              :     image: Option<ResidentLayer>,
     926              :     next_start_key: Key,
     927              : }
     928              : 
     929              : /// Public interface functions
     930              : impl Timeline {
     931              :     /// Get the LSN where this branch was created
     932            8 :     pub(crate) fn get_ancestor_lsn(&self) -> Lsn {
     933            8 :         self.ancestor_lsn
     934            8 :     }
     935              : 
     936              :     /// Get the ancestor's timeline id
     937           24 :     pub(crate) fn get_ancestor_timeline_id(&self) -> Option<TimelineId> {
     938           24 :         self.ancestor_timeline
     939           24 :             .as_ref()
     940           24 :             .map(|ancestor| ancestor.timeline_id)
     941           24 :     }
     942              : 
     943              :     /// Get the ancestor timeline
     944            4 :     pub(crate) fn ancestor_timeline(&self) -> Option<&Arc<Timeline>> {
     945            4 :         self.ancestor_timeline.as_ref()
     946            4 :     }
     947              : 
     948              :     /// Get the bytes written since the PITR cutoff on this branch, and
     949              :     /// whether this branch's ancestor_lsn is within its parent's PITR.
     950            0 :     pub(crate) fn get_pitr_history_stats(&self) -> (u64, bool) {
     951            0 :         let gc_info = self.gc_info.read().unwrap();
     952            0 :         let history = self
     953            0 :             .get_last_record_lsn()
     954            0 :             .checked_sub(gc_info.cutoffs.time)
     955            0 :             .unwrap_or(Lsn(0))
     956            0 :             .0;
     957            0 :         (history, gc_info.within_ancestor_pitr)
     958            0 :     }
     959              : 
     960              :     /// Lock and get timeline's GC cutoff
     961          604 :     pub(crate) fn get_latest_gc_cutoff_lsn(&self) -> RcuReadGuard<Lsn> {
     962          604 :         self.latest_gc_cutoff_lsn.read()
     963          604 :     }
     964              : 
     965              :     /// Look up given page version.
     966              :     ///
     967              :     /// If a remote layer file is needed, it is downloaded as part of this
     968              :     /// call.
     969              :     ///
     970              :     /// This method enforces [`Self::pagestream_throttle`] internally.
     971              :     ///
     972              :     /// NOTE: It is considered an error to 'get' a key that doesn't exist. The
     973              :     /// abstraction above this needs to store suitable metadata to track what
     974              :     /// data exists with what keys, in separate metadata entries. If a
     975              :     /// non-existent key is requested, we may incorrectly return a value from
     976              :     /// an ancestor branch, for example, or waste a lot of cycles chasing the
     977              :     /// non-existing key.
     978              :     ///
     979              :     /// # Cancel-Safety
     980              :     ///
     981              :     /// This method is cancellation-safe.
     982              :     #[inline(always)]
     983      1215181 :     pub(crate) async fn get(
     984      1215181 :         &self,
     985      1215181 :         key: Key,
     986      1215181 :         lsn: Lsn,
     987      1215181 :         ctx: &RequestContext,
     988      1215181 :     ) -> Result<Bytes, PageReconstructError> {
     989      1215181 :         if !lsn.is_valid() {
     990            0 :             return Err(PageReconstructError::Other(anyhow::anyhow!("Invalid LSN")));
     991      1215181 :         }
     992      1215181 : 
     993      1215181 :         // This check is debug-only because of the cost of hashing, and because it's a double-check: we
     994      1215181 :         // already checked the key against the shard_identity when looking up the Timeline from
     995      1215181 :         // page_service.
     996      1215181 :         debug_assert!(!self.shard_identity.is_key_disposable(&key));
     997              : 
     998      1215181 :         let keyspace = KeySpace {
     999      1215181 :             ranges: vec![key..key.next()],
    1000      1215181 :         };
    1001      1215181 : 
    1002      1215181 :         let mut reconstruct_state = ValuesReconstructState::new(IoConcurrency::sequential());
    1003              : 
    1004      1215181 :         let vectored_res = self
    1005      1215181 :             .get_vectored_impl(keyspace.clone(), lsn, &mut reconstruct_state, ctx)
    1006      1215181 :             .await;
    1007              : 
    1008      1215181 :         let key_value = vectored_res?.pop_first();
    1009      1215169 :         match key_value {
    1010      1215145 :             Some((got_key, value)) => {
    1011      1215145 :                 if got_key != key {
    1012            0 :                     error!(
    1013            0 :                         "Expected {}, but singular vectored get returned {}",
    1014              :                         key, got_key
    1015              :                     );
    1016            0 :                     Err(PageReconstructError::Other(anyhow!(
    1017            0 :                         "Singular vectored get returned wrong key"
    1018            0 :                     )))
    1019              :                 } else {
    1020      1215145 :                     value
    1021              :                 }
    1022              :             }
    1023           24 :             None => Err(PageReconstructError::MissingKey(MissingKeyError {
    1024           24 :                 key,
    1025           24 :                 shard: self.shard_identity.get_shard_number(&key),
    1026           24 :                 cont_lsn: Lsn(0),
    1027           24 :                 request_lsn: lsn,
    1028           24 :                 ancestor_lsn: None,
    1029           24 :                 backtrace: None,
    1030           24 :             })),
    1031              :         }
    1032      1215181 :     }
    1033              : 
    1034              :     pub(crate) const MAX_GET_VECTORED_KEYS: u64 = 32;
    1035              :     pub(crate) const VEC_GET_LAYERS_VISITED_WARN_THRESH: f64 = 512.0;
    1036              : 
    1037              :     /// Look up multiple page versions at a given LSN
    1038              :     ///
    1039              :     /// This naive implementation will be replaced with a more efficient one
    1040              :     /// which actually vectorizes the read path.
    1041        39356 :     pub(crate) async fn get_vectored(
    1042        39356 :         &self,
    1043        39356 :         keyspace: KeySpace,
    1044        39356 :         lsn: Lsn,
    1045        39356 :         io_concurrency: super::storage_layer::IoConcurrency,
    1046        39356 :         ctx: &RequestContext,
    1047        39356 :     ) -> Result<BTreeMap<Key, Result<Bytes, PageReconstructError>>, GetVectoredError> {
    1048        39356 :         if !lsn.is_valid() {
    1049            0 :             return Err(GetVectoredError::InvalidLsn(lsn));
    1050        39356 :         }
    1051        39356 : 
    1052        39356 :         let key_count = keyspace.total_raw_size().try_into().unwrap();
    1053        39356 :         if key_count > Timeline::MAX_GET_VECTORED_KEYS {
    1054            0 :             return Err(GetVectoredError::Oversized(key_count));
    1055        39356 :         }
    1056              : 
    1057        78712 :         for range in &keyspace.ranges {
    1058        39356 :             let mut key = range.start;
    1059        79180 :             while key != range.end {
    1060        39824 :                 assert!(!self.shard_identity.is_key_disposable(&key));
    1061        39824 :                 key = key.next();
    1062              :             }
    1063              :         }
    1064              : 
    1065        39356 :         trace!(
    1066            0 :             "get vectored request for {:?}@{} from task kind {:?}",
    1067            0 :             keyspace,
    1068            0 :             lsn,
    1069            0 :             ctx.task_kind(),
    1070              :         );
    1071              : 
    1072        39356 :         let start = crate::metrics::GET_VECTORED_LATENCY
    1073        39356 :             .for_task_kind(ctx.task_kind())
    1074        39356 :             .map(|metric| (metric, Instant::now()));
    1075              : 
    1076        39356 :         let res = self
    1077        39356 :             .get_vectored_impl(
    1078        39356 :                 keyspace.clone(),
    1079        39356 :                 lsn,
    1080        39356 :                 &mut ValuesReconstructState::new(io_concurrency),
    1081        39356 :                 ctx,
    1082        39356 :             )
    1083        39356 :             .await;
    1084              : 
    1085        39356 :         if let Some((metric, start)) = start {
    1086            0 :             let elapsed = start.elapsed();
    1087            0 :             metric.observe(elapsed.as_secs_f64());
    1088        39356 :         }
    1089              : 
    1090        39356 :         res
    1091        39356 :     }
    1092              : 
    1093              :     /// Scan the keyspace and return all existing key-values in the keyspace. This currently uses vectored
    1094              :     /// get underlying. Normal vectored get would throw an error when a key in the keyspace is not found
    1095              :     /// during the search, but for the scan interface, it returns all existing key-value pairs, and does
    1096              :     /// not expect each single key in the key space will be found. The semantics is closer to the RocksDB
    1097              :     /// scan iterator interface. We could optimize this interface later to avoid some checks in the vectored
    1098              :     /// get path to maintain and split the probing and to-be-probe keyspace. We also need to ensure that
    1099              :     /// the scan operation will not cause OOM in the future.
    1100           24 :     pub(crate) async fn scan(
    1101           24 :         &self,
    1102           24 :         keyspace: KeySpace,
    1103           24 :         lsn: Lsn,
    1104           24 :         ctx: &RequestContext,
    1105           24 :         io_concurrency: super::storage_layer::IoConcurrency,
    1106           24 :     ) -> Result<BTreeMap<Key, Result<Bytes, PageReconstructError>>, GetVectoredError> {
    1107           24 :         if !lsn.is_valid() {
    1108            0 :             return Err(GetVectoredError::InvalidLsn(lsn));
    1109           24 :         }
    1110           24 : 
    1111           24 :         trace!(
    1112            0 :             "key-value scan request for {:?}@{} from task kind {:?}",
    1113            0 :             keyspace,
    1114            0 :             lsn,
    1115            0 :             ctx.task_kind()
    1116              :         );
    1117              : 
    1118              :         // We should generalize this into Keyspace::contains in the future.
    1119           48 :         for range in &keyspace.ranges {
    1120           24 :             if range.start.field1 < METADATA_KEY_BEGIN_PREFIX
    1121           24 :                 || range.end.field1 > METADATA_KEY_END_PREFIX
    1122              :             {
    1123            0 :                 return Err(GetVectoredError::Other(anyhow::anyhow!(
    1124            0 :                     "only metadata keyspace can be scanned"
    1125            0 :                 )));
    1126           24 :             }
    1127              :         }
    1128              : 
    1129           24 :         let start = crate::metrics::SCAN_LATENCY
    1130           24 :             .for_task_kind(ctx.task_kind())
    1131           24 :             .map(ScanLatencyOngoingRecording::start_recording);
    1132              : 
    1133           24 :         let vectored_res = self
    1134           24 :             .get_vectored_impl(
    1135           24 :                 keyspace.clone(),
    1136           24 :                 lsn,
    1137           24 :                 &mut ValuesReconstructState::new(io_concurrency),
    1138           24 :                 ctx,
    1139           24 :             )
    1140           24 :             .await;
    1141              : 
    1142           24 :         if let Some(recording) = start {
    1143            0 :             recording.observe();
    1144           24 :         }
    1145              : 
    1146           24 :         vectored_res
    1147           24 :     }
    1148              : 
    1149      1255269 :     pub(super) async fn get_vectored_impl(
    1150      1255269 :         &self,
    1151      1255269 :         keyspace: KeySpace,
    1152      1255269 :         lsn: Lsn,
    1153      1255269 :         reconstruct_state: &mut ValuesReconstructState,
    1154      1255269 :         ctx: &RequestContext,
    1155      1255269 :     ) -> Result<BTreeMap<Key, Result<Bytes, PageReconstructError>>, GetVectoredError> {
    1156      1255269 :         let traversal_res: Result<(), _> = self
    1157      1255269 :             .get_vectored_reconstruct_data(keyspace.clone(), lsn, reconstruct_state, ctx)
    1158      1255269 :             .await;
    1159      1255269 :         if let Err(err) = traversal_res {
    1160              :             // Wait for all the spawned IOs to complete.
    1161              :             // See comments on `spawn_io` inside `storage_layer` for more details.
    1162           32 :             let mut collect_futs = std::mem::take(&mut reconstruct_state.keys)
    1163           32 :                 .into_values()
    1164           32 :                 .map(|state| state.collect_pending_ios())
    1165           32 :                 .collect::<FuturesUnordered<_>>();
    1166           32 :             while collect_futs.next().await.is_some() {}
    1167           32 :             return Err(err);
    1168      1255237 :         };
    1169      1255237 : 
    1170      1255237 :         let layers_visited = reconstruct_state.get_layers_visited();
    1171      1255237 : 
    1172      1255237 :         let futs = FuturesUnordered::new();
    1173      1335905 :         for (key, state) in std::mem::take(&mut reconstruct_state.keys) {
    1174      1335905 :             futs.push({
    1175      1335905 :                 let walredo_self = self.myself.upgrade().expect("&self method holds the arc");
    1176      1335905 :                 async move {
    1177      1335905 :                     assert_eq!(state.situation, ValueReconstructSituation::Complete);
    1178              : 
    1179      1335905 :                     let converted = match state.collect_pending_ios().await {
    1180      1335905 :                         Ok(ok) => ok,
    1181            0 :                         Err(err) => {
    1182            0 :                             return (key, Err(err));
    1183              :                         }
    1184              :                     };
    1185              : 
    1186              :                     // The walredo module expects the records to be descending in terms of Lsn.
    1187              :                     // And we submit the IOs in that order, so, there shuold be no need to sort here.
    1188      1335905 :                     debug_assert!(
    1189      1335905 :                         converted
    1190      1335905 :                             .records
    1191      1335905 :                             .is_sorted_by_key(|(lsn, _)| std::cmp::Reverse(*lsn)),
    1192            0 :                         "{converted:?}"
    1193              :                     );
    1194              : 
    1195              :                     (
    1196      1335905 :                         key,
    1197      1335905 :                         walredo_self.reconstruct_value(key, lsn, converted).await,
    1198              :                     )
    1199      1335905 :                 }
    1200      1335905 :             });
    1201      1335905 :         }
    1202              : 
    1203      1255237 :         let results = futs
    1204      1255237 :             .collect::<BTreeMap<Key, Result<Bytes, PageReconstructError>>>()
    1205      1255237 :             .await;
    1206              : 
    1207              :         // For aux file keys (v1 or v2) the vectored read path does not return an error
    1208              :         // when they're missing. Instead they are omitted from the resulting btree
    1209              :         // (this is a requirement, not a bug). Skip updating the metric in these cases
    1210              :         // to avoid infinite results.
    1211      1255237 :         if !results.is_empty() {
    1212      1254757 :             let avg = layers_visited as f64 / results.len() as f64;
    1213      1254757 :             if avg >= Self::VEC_GET_LAYERS_VISITED_WARN_THRESH {
    1214            0 :                 use utils::rate_limit::RateLimit;
    1215            0 :                 static LOGGED: Lazy<Mutex<RateLimit>> =
    1216            0 :                     Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(60))));
    1217            0 :                 let mut rate_limit = LOGGED.lock().unwrap();
    1218            0 :                 rate_limit.call(|| {
    1219            0 :                     tracing::info!(
    1220            0 :                       shard_id = %self.tenant_shard_id.shard_slug(),
    1221            0 :                       lsn = %lsn,
    1222            0 :                       "Vectored read for {} visited {} layers on average per key and {} in total. {}/{} pages were returned",
    1223            0 :                       keyspace, avg, layers_visited, results.len(), keyspace.total_raw_size());
    1224            0 :                 });
    1225      1254757 :             }
    1226              : 
    1227              :             // Note that this is an approximation. Tracking the exact number of layers visited
    1228              :             // per key requires virtually unbounded memory usage and is inefficient
    1229              :             // (i.e. segment tree tracking each range queried from a layer)
    1230      1254757 :             crate::metrics::VEC_READ_NUM_LAYERS_VISITED.observe(avg);
    1231          480 :         }
    1232              : 
    1233      1255237 :         Ok(results)
    1234      1255269 :     }
    1235              : 
    1236              :     /// Get last or prev record separately. Same as get_last_record_rlsn().last/prev.
    1237       548848 :     pub(crate) fn get_last_record_lsn(&self) -> Lsn {
    1238       548848 :         self.last_record_lsn.load().last
    1239       548848 :     }
    1240              : 
    1241            0 :     pub(crate) fn get_prev_record_lsn(&self) -> Lsn {
    1242            0 :         self.last_record_lsn.load().prev
    1243            0 :     }
    1244              : 
    1245              :     /// Atomically get both last and prev.
    1246          456 :     pub(crate) fn get_last_record_rlsn(&self) -> RecordLsn {
    1247          456 :         self.last_record_lsn.load()
    1248          456 :     }
    1249              : 
    1250              :     /// Subscribe to callers of wait_lsn(). The value of the channel is None if there are no
    1251              :     /// wait_lsn() calls in progress, and Some(Lsn) if there is an active waiter for wait_lsn().
    1252            0 :     pub(crate) fn subscribe_for_wait_lsn_updates(&self) -> watch::Receiver<Option<Lsn>> {
    1253            0 :         self.last_record_lsn.status_receiver()
    1254            0 :     }
    1255              : 
    1256          884 :     pub(crate) fn get_disk_consistent_lsn(&self) -> Lsn {
    1257          884 :         self.disk_consistent_lsn.load()
    1258          884 :     }
    1259              : 
    1260              :     /// remote_consistent_lsn from the perspective of the tenant's current generation,
    1261              :     /// not validated with control plane yet.
    1262              :     /// See [`Self::get_remote_consistent_lsn_visible`].
    1263            0 :     pub(crate) fn get_remote_consistent_lsn_projected(&self) -> Option<Lsn> {
    1264            0 :         self.remote_client.remote_consistent_lsn_projected()
    1265            0 :     }
    1266              : 
    1267              :     /// remote_consistent_lsn which the tenant is guaranteed not to go backward from,
    1268              :     /// i.e. a value of remote_consistent_lsn_projected which has undergone
    1269              :     /// generation validation in the deletion queue.
    1270            0 :     pub(crate) fn get_remote_consistent_lsn_visible(&self) -> Option<Lsn> {
    1271            0 :         self.remote_client.remote_consistent_lsn_visible()
    1272            0 :     }
    1273              : 
    1274              :     /// The sum of the file size of all historic layers in the layer map.
    1275              :     /// This method makes no distinction between local and remote layers.
    1276              :     /// Hence, the result **does not represent local filesystem usage**.
    1277            0 :     pub(crate) async fn layer_size_sum(&self) -> u64 {
    1278            0 :         let guard = self.layers.read().await;
    1279            0 :         guard.layer_size_sum()
    1280            0 :     }
    1281              : 
    1282            0 :     pub(crate) fn resident_physical_size(&self) -> u64 {
    1283            0 :         self.metrics.resident_physical_size_get()
    1284            0 :     }
    1285              : 
    1286            0 :     pub(crate) fn get_directory_metrics(&self) -> [u64; DirectoryKind::KINDS_NUM] {
    1287            0 :         array::from_fn(|idx| self.directory_metrics[idx].load(AtomicOrdering::Relaxed))
    1288            0 :     }
    1289              : 
    1290              :     ///
    1291              :     /// Wait until WAL has been received and processed up to this LSN.
    1292              :     ///
    1293              :     /// You should call this before any of the other get_* or list_* functions. Calling
    1294              :     /// those functions with an LSN that has been processed yet is an error.
    1295              :     ///
    1296       451167 :     pub(crate) async fn wait_lsn(
    1297       451167 :         &self,
    1298       451167 :         lsn: Lsn,
    1299       451167 :         who_is_waiting: WaitLsnWaiter<'_>,
    1300       451167 :         ctx: &RequestContext, /* Prepare for use by cancellation */
    1301       451167 :     ) -> Result<(), WaitLsnError> {
    1302       451167 :         let state = self.current_state();
    1303       451167 :         if self.cancel.is_cancelled() || matches!(state, TimelineState::Stopping) {
    1304            0 :             return Err(WaitLsnError::Shutdown);
    1305       451167 :         } else if !matches!(state, TimelineState::Active) {
    1306            0 :             return Err(WaitLsnError::BadState(state));
    1307       451167 :         }
    1308       451167 : 
    1309       451167 :         if cfg!(debug_assertions) {
    1310       451167 :             match ctx.task_kind() {
    1311              :                 TaskKind::WalReceiverManager
    1312              :                 | TaskKind::WalReceiverConnectionHandler
    1313              :                 | TaskKind::WalReceiverConnectionPoller => {
    1314            0 :                     let is_myself = match who_is_waiting {
    1315            0 :                         WaitLsnWaiter::Timeline(waiter) => Weak::ptr_eq(&waiter.myself, &self.myself),
    1316            0 :                         WaitLsnWaiter::Tenant | WaitLsnWaiter::PageService => unreachable!("tenant or page_service context are not expected to have task kind {:?}", ctx.task_kind()),
    1317              :                     };
    1318            0 :                     if is_myself {
    1319            0 :                         if let Err(current) = self.last_record_lsn.would_wait_for(lsn) {
    1320              :                             // walingest is the only one that can advance last_record_lsn; it should make sure to never reach here
    1321            0 :                             panic!("this timeline's walingest task is calling wait_lsn({lsn}) but we only have last_record_lsn={current}; would deadlock");
    1322            0 :                         }
    1323            0 :                     } else {
    1324            0 :                         // if another  timeline's  is waiting for us, there's no deadlock risk because
    1325            0 :                         // our walreceiver task can make progress independent of theirs
    1326            0 :                     }
    1327              :                 }
    1328       451167 :                 _ => {}
    1329              :             }
    1330            0 :         }
    1331              : 
    1332       451167 :         let _timer = crate::metrics::WAIT_LSN_TIME.start_timer();
    1333       451167 : 
    1334       451167 :         match self
    1335       451167 :             .last_record_lsn
    1336       451167 :             .wait_for_timeout(lsn, self.conf.wait_lsn_timeout)
    1337       451167 :             .await
    1338              :         {
    1339       451167 :             Ok(()) => Ok(()),
    1340            0 :             Err(e) => {
    1341              :                 use utils::seqwait::SeqWaitError::*;
    1342            0 :                 match e {
    1343            0 :                     Shutdown => Err(WaitLsnError::Shutdown),
    1344              :                     Timeout => {
    1345              :                         // don't count the time spent waiting for lock below, and also in walreceiver.status(), towards the wait_lsn_time_histo
    1346            0 :                         drop(_timer);
    1347            0 :                         let walreceiver_status = self.walreceiver_status();
    1348            0 :                         Err(WaitLsnError::Timeout(format!(
    1349            0 :                         "Timed out while waiting for WAL record at LSN {} to arrive, last_record_lsn {} disk consistent LSN={}, WalReceiver status: {}",
    1350            0 :                         lsn,
    1351            0 :                         self.get_last_record_lsn(),
    1352            0 :                         self.get_disk_consistent_lsn(),
    1353            0 :                         walreceiver_status,
    1354            0 :                     )))
    1355              :                     }
    1356              :                 }
    1357              :             }
    1358              :         }
    1359       451167 :     }
    1360              : 
    1361            0 :     pub(crate) fn walreceiver_status(&self) -> String {
    1362            0 :         match &*self.walreceiver.lock().unwrap() {
    1363            0 :             None => "stopping or stopped".to_string(),
    1364            0 :             Some(walreceiver) => match walreceiver.status() {
    1365            0 :                 Some(status) => status.to_human_readable_string(),
    1366            0 :                 None => "Not active".to_string(),
    1367              :             },
    1368              :         }
    1369            0 :     }
    1370              : 
    1371              :     /// Check that it is valid to request operations with that lsn.
    1372          464 :     pub(crate) fn check_lsn_is_in_scope(
    1373          464 :         &self,
    1374          464 :         lsn: Lsn,
    1375          464 :         latest_gc_cutoff_lsn: &RcuReadGuard<Lsn>,
    1376          464 :     ) -> anyhow::Result<()> {
    1377          464 :         ensure!(
    1378          464 :             lsn >= **latest_gc_cutoff_lsn,
    1379            8 :             "LSN {} is earlier than latest GC cutoff {} (we might've already garbage collected needed data)",
    1380            8 :             lsn,
    1381            8 :             **latest_gc_cutoff_lsn,
    1382              :         );
    1383          456 :         Ok(())
    1384          464 :     }
    1385              : 
    1386              :     /// Initializes an LSN lease. The function will return an error if the requested LSN is less than the `latest_gc_cutoff_lsn`.
    1387           20 :     pub(crate) fn init_lsn_lease(
    1388           20 :         &self,
    1389           20 :         lsn: Lsn,
    1390           20 :         length: Duration,
    1391           20 :         ctx: &RequestContext,
    1392           20 :     ) -> anyhow::Result<LsnLease> {
    1393           20 :         self.make_lsn_lease(lsn, length, true, ctx)
    1394           20 :     }
    1395              : 
    1396              :     /// Renews a lease at a particular LSN. The requested LSN is not validated against the `latest_gc_cutoff_lsn` when we are in the grace period.
    1397            8 :     pub(crate) fn renew_lsn_lease(
    1398            8 :         &self,
    1399            8 :         lsn: Lsn,
    1400            8 :         length: Duration,
    1401            8 :         ctx: &RequestContext,
    1402            8 :     ) -> anyhow::Result<LsnLease> {
    1403            8 :         self.make_lsn_lease(lsn, length, false, ctx)
    1404            8 :     }
    1405              : 
    1406              :     /// Obtains a temporary lease blocking garbage collection for the given LSN.
    1407              :     ///
    1408              :     /// If we are in `AttachedSingle` mode and is not blocked by the lsn lease deadline, this function will error
    1409              :     /// if the requesting LSN is less than the `latest_gc_cutoff_lsn` and there is no existing request present.
    1410              :     ///
    1411              :     /// If there is an existing lease in the map, the lease will be renewed only if the request extends the lease.
    1412              :     /// The returned lease is therefore the maximum between the existing lease and the requesting lease.
    1413           28 :     fn make_lsn_lease(
    1414           28 :         &self,
    1415           28 :         lsn: Lsn,
    1416           28 :         length: Duration,
    1417           28 :         init: bool,
    1418           28 :         _ctx: &RequestContext,
    1419           28 :     ) -> anyhow::Result<LsnLease> {
    1420           24 :         let lease = {
    1421              :             // Normalize the requested LSN to be aligned, and move to the first record
    1422              :             // if it points to the beginning of the page (header).
    1423           28 :             let lsn = xlog_utils::normalize_lsn(lsn, WAL_SEGMENT_SIZE);
    1424           28 : 
    1425           28 :             let mut gc_info = self.gc_info.write().unwrap();
    1426           28 : 
    1427           28 :             let valid_until = SystemTime::now() + length;
    1428           28 : 
    1429           28 :             let entry = gc_info.leases.entry(lsn);
    1430           28 : 
    1431           28 :             match entry {
    1432           12 :                 Entry::Occupied(mut occupied) => {
    1433           12 :                     let existing_lease = occupied.get_mut();
    1434           12 :                     if valid_until > existing_lease.valid_until {
    1435            4 :                         existing_lease.valid_until = valid_until;
    1436            4 :                         let dt: DateTime<Utc> = valid_until.into();
    1437            4 :                         info!("lease extended to {}", dt);
    1438              :                     } else {
    1439            8 :                         let dt: DateTime<Utc> = existing_lease.valid_until.into();
    1440            8 :                         info!("existing lease covers greater length, valid until {}", dt);
    1441              :                     }
    1442              : 
    1443           12 :                     existing_lease.clone()
    1444              :                 }
    1445           16 :                 Entry::Vacant(vacant) => {
    1446              :                     // Reject already GC-ed LSN (lsn < latest_gc_cutoff) if we are in AttachedSingle and
    1447              :                     // not blocked by the lsn lease deadline.
    1448           16 :                     let validate = {
    1449           16 :                         let conf = self.tenant_conf.load();
    1450           16 :                         conf.location.attach_mode == AttachmentMode::Single
    1451           16 :                             && !conf.is_gc_blocked_by_lsn_lease_deadline()
    1452              :                     };
    1453              : 
    1454           16 :                     if init || validate {
    1455           16 :                         let latest_gc_cutoff_lsn = self.get_latest_gc_cutoff_lsn();
    1456           16 :                         if lsn < *latest_gc_cutoff_lsn {
    1457            4 :                             bail!("tried to request a page version that was garbage collected. requested at {} gc cutoff {}", lsn, *latest_gc_cutoff_lsn);
    1458           12 :                         }
    1459            0 :                     }
    1460              : 
    1461           12 :                     let dt: DateTime<Utc> = valid_until.into();
    1462           12 :                     info!("lease created, valid until {}", dt);
    1463           12 :                     vacant.insert(LsnLease { valid_until }).clone()
    1464              :                 }
    1465              :             }
    1466              :         };
    1467              : 
    1468           24 :         Ok(lease)
    1469           28 :     }
    1470              : 
    1471              :     /// Freeze the current open in-memory layer. It will be written to disk on next iteration.
    1472              :     /// Returns the flush request ID which can be awaited with wait_flush_completion().
    1473              :     #[instrument(skip(self), fields(tenant_id=%self.tenant_shard_id.tenant_id, shard_id=%self.tenant_shard_id.shard_slug(), timeline_id=%self.timeline_id))]
    1474              :     pub(crate) async fn freeze(&self) -> Result<u64, FlushLayerError> {
    1475              :         self.freeze0().await
    1476              :     }
    1477              : 
    1478              :     /// Freeze and flush the open in-memory layer, waiting for it to be written to disk.
    1479              :     #[instrument(skip(self), fields(tenant_id=%self.tenant_shard_id.tenant_id, shard_id=%self.tenant_shard_id.shard_slug(), timeline_id=%self.timeline_id))]
    1480              :     pub(crate) async fn freeze_and_flush(&self) -> Result<(), FlushLayerError> {
    1481              :         self.freeze_and_flush0().await
    1482              :     }
    1483              : 
    1484              :     /// Freeze the current open in-memory layer. It will be written to disk on next iteration.
    1485              :     /// Returns the flush request ID which can be awaited with wait_flush_completion().
    1486         2240 :     pub(crate) async fn freeze0(&self) -> Result<u64, FlushLayerError> {
    1487         2240 :         let mut g = self.write_lock.lock().await;
    1488         2240 :         let to_lsn = self.get_last_record_lsn();
    1489         2240 :         self.freeze_inmem_layer_at(to_lsn, &mut g).await
    1490         2240 :     }
    1491              : 
    1492              :     // This exists to provide a non-span creating version of `freeze_and_flush` we can call without
    1493              :     // polluting the span hierarchy.
    1494         2240 :     pub(crate) async fn freeze_and_flush0(&self) -> Result<(), FlushLayerError> {
    1495         2240 :         let token = self.freeze0().await?;
    1496         2240 :         self.wait_flush_completion(token).await
    1497         2240 :     }
    1498              : 
    1499              :     // Check if an open ephemeral layer should be closed: this provides
    1500              :     // background enforcement of checkpoint interval if there is no active WAL receiver, to avoid keeping
    1501              :     // an ephemeral layer open forever when idle.  It also freezes layers if the global limit on
    1502              :     // ephemeral layer bytes has been breached.
    1503            0 :     pub(super) async fn maybe_freeze_ephemeral_layer(&self) {
    1504            0 :         let Ok(mut write_guard) = self.write_lock.try_lock() else {
    1505              :             // If the write lock is held, there is an active wal receiver: rolling open layers
    1506              :             // is their responsibility while they hold this lock.
    1507            0 :             return;
    1508              :         };
    1509              : 
    1510              :         // FIXME: why not early exit? because before #7927 the state would had been cleared every
    1511              :         // time, and this was missed.
    1512              :         // if write_guard.is_none() { return; }
    1513              : 
    1514            0 :         let Ok(layers_guard) = self.layers.try_read() else {
    1515              :             // Don't block if the layer lock is busy
    1516            0 :             return;
    1517              :         };
    1518              : 
    1519            0 :         let Ok(lm) = layers_guard.layer_map() else {
    1520            0 :             return;
    1521              :         };
    1522              : 
    1523            0 :         let Some(open_layer) = &lm.open_layer else {
    1524              :             // If there is no open layer, we have no layer freezing to do.  However, we might need to generate
    1525              :             // some updates to disk_consistent_lsn and remote_consistent_lsn, in case we ingested some WAL regions
    1526              :             // that didn't result in writes to this shard.
    1527              : 
    1528              :             // Must not hold the layers lock while waiting for a flush.
    1529            0 :             drop(layers_guard);
    1530            0 : 
    1531            0 :             let last_record_lsn = self.get_last_record_lsn();
    1532            0 :             let disk_consistent_lsn = self.get_disk_consistent_lsn();
    1533            0 :             if last_record_lsn > disk_consistent_lsn {
    1534              :                 // We have no open layer, but disk_consistent_lsn is behind the last record: this indicates
    1535              :                 // we are a sharded tenant and have skipped some WAL
    1536            0 :                 let last_freeze_ts = *self.last_freeze_ts.read().unwrap();
    1537            0 :                 if last_freeze_ts.elapsed() >= self.get_checkpoint_timeout() {
    1538              :                     // Only do this if have been layer-less longer than get_checkpoint_timeout, so that a shard
    1539              :                     // without any data ingested (yet) doesn't write a remote index as soon as it
    1540              :                     // sees its LSN advance: we only do this if we've been layer-less
    1541              :                     // for some time.
    1542            0 :                     tracing::debug!(
    1543            0 :                         "Advancing disk_consistent_lsn past WAL ingest gap {} -> {}",
    1544              :                         disk_consistent_lsn,
    1545              :                         last_record_lsn
    1546              :                     );
    1547              : 
    1548              :                     // The flush loop will update remote consistent LSN as well as disk consistent LSN.
    1549              :                     // We know there is no open layer, so we can request freezing without actually
    1550              :                     // freezing anything. This is true even if we have dropped the layers_guard, we
    1551              :                     // still hold the write_guard.
    1552            0 :                     let _ = async {
    1553            0 :                         let token = self
    1554            0 :                             .freeze_inmem_layer_at(last_record_lsn, &mut write_guard)
    1555            0 :                             .await?;
    1556            0 :                         self.wait_flush_completion(token).await
    1557            0 :                     }
    1558            0 :                     .await;
    1559            0 :                 }
    1560            0 :             }
    1561              : 
    1562            0 :             return;
    1563              :         };
    1564              : 
    1565            0 :         let Some(current_size) = open_layer.try_len() else {
    1566              :             // Unexpected: since we hold the write guard, nobody else should be writing to this layer, so
    1567              :             // read lock to get size should always succeed.
    1568            0 :             tracing::warn!("Lock conflict while reading size of open layer");
    1569            0 :             return;
    1570              :         };
    1571              : 
    1572            0 :         let current_lsn = self.get_last_record_lsn();
    1573              : 
    1574            0 :         let checkpoint_distance_override = open_layer.tick().await;
    1575              : 
    1576            0 :         if let Some(size_override) = checkpoint_distance_override {
    1577            0 :             if current_size > size_override {
    1578              :                 // This is not harmful, but it only happens in relatively rare cases where
    1579              :                 // time-based checkpoints are not happening fast enough to keep the amount of
    1580              :                 // ephemeral data within configured limits.  It's a sign of stress on the system.
    1581            0 :                 tracing::info!("Early-rolling open layer at size {current_size} (limit {size_override}) due to dirty data pressure");
    1582            0 :             }
    1583            0 :         }
    1584              : 
    1585            0 :         let checkpoint_distance =
    1586            0 :             checkpoint_distance_override.unwrap_or(self.get_checkpoint_distance());
    1587            0 : 
    1588            0 :         if self.should_roll(
    1589            0 :             current_size,
    1590            0 :             current_size,
    1591            0 :             checkpoint_distance,
    1592            0 :             self.get_last_record_lsn(),
    1593            0 :             self.last_freeze_at.load(),
    1594            0 :             open_layer.get_opened_at(),
    1595            0 :         ) {
    1596            0 :             match open_layer.info() {
    1597            0 :                 InMemoryLayerInfo::Frozen { lsn_start, lsn_end } => {
    1598            0 :                     // We may reach this point if the layer was already frozen by not yet flushed: flushing
    1599            0 :                     // happens asynchronously in the background.
    1600            0 :                     tracing::debug!(
    1601            0 :                         "Not freezing open layer, it's already frozen ({lsn_start}..{lsn_end})"
    1602              :                     );
    1603              :                 }
    1604              :                 InMemoryLayerInfo::Open { .. } => {
    1605              :                     // Upgrade to a write lock and freeze the layer
    1606            0 :                     drop(layers_guard);
    1607            0 :                     let res = self
    1608            0 :                         .freeze_inmem_layer_at(current_lsn, &mut write_guard)
    1609            0 :                         .await;
    1610              : 
    1611            0 :                     if let Err(e) = res {
    1612            0 :                         tracing::info!(
    1613            0 :                             "failed to flush frozen layer after background freeze: {e:#}"
    1614              :                         );
    1615            0 :                     }
    1616              :                 }
    1617              :             }
    1618            0 :         }
    1619            0 :     }
    1620              : 
    1621              :     /// Checks if the internal state of the timeline is consistent with it being able to be offloaded.
    1622              :     ///
    1623              :     /// This is neccessary but not sufficient for offloading of the timeline as it might have
    1624              :     /// child timelines that are not offloaded yet.
    1625            0 :     pub(crate) fn can_offload(&self) -> (bool, &'static str) {
    1626            0 :         if self.remote_client.is_archived() != Some(true) {
    1627            0 :             return (false, "the timeline is not archived");
    1628            0 :         }
    1629            0 :         if !self.remote_client.no_pending_work() {
    1630              :             // if the remote client is still processing some work, we can't offload
    1631            0 :             return (false, "the upload queue is not drained yet");
    1632            0 :         }
    1633            0 : 
    1634            0 :         (true, "ok")
    1635            0 :     }
    1636              : 
    1637              :     /// Outermost timeline compaction operation; downloads needed layers. Returns whether we have pending
    1638              :     /// compaction tasks.
    1639          728 :     pub(crate) async fn compact(
    1640          728 :         self: &Arc<Self>,
    1641          728 :         cancel: &CancellationToken,
    1642          728 :         flags: EnumSet<CompactFlags>,
    1643          728 :         ctx: &RequestContext,
    1644          728 :     ) -> Result<bool, CompactionError> {
    1645          728 :         self.compact_with_options(
    1646          728 :             cancel,
    1647          728 :             CompactOptions {
    1648          728 :                 flags,
    1649          728 :                 compact_key_range: None,
    1650          728 :                 compact_lsn_range: None,
    1651          728 :                 sub_compaction: false,
    1652          728 :                 sub_compaction_max_job_size_mb: None,
    1653          728 :             },
    1654          728 :             ctx,
    1655          728 :         )
    1656          728 :         .await
    1657          728 :     }
    1658              : 
    1659              :     /// Outermost timeline compaction operation; downloads needed layers. Returns whether we have pending
    1660              :     /// compaction tasks.
    1661          728 :     pub(crate) async fn compact_with_options(
    1662          728 :         self: &Arc<Self>,
    1663          728 :         cancel: &CancellationToken,
    1664          728 :         options: CompactOptions,
    1665          728 :         ctx: &RequestContext,
    1666          728 :     ) -> Result<bool, CompactionError> {
    1667          728 :         // most likely the cancellation token is from background task, but in tests it could be the
    1668          728 :         // request task as well.
    1669          728 : 
    1670          728 :         let prepare = async move {
    1671          728 :             let guard = self.compaction_lock.lock().await;
    1672              : 
    1673          728 :             let permit = super::tasks::concurrent_background_tasks_rate_limit_permit(
    1674          728 :                 BackgroundLoopKind::Compaction,
    1675          728 :                 ctx,
    1676          728 :             )
    1677          728 :             .await;
    1678              : 
    1679          728 :             (guard, permit)
    1680          728 :         };
    1681              : 
    1682              :         // this wait probably never needs any "long time spent" logging, because we already nag if
    1683              :         // compaction task goes over it's period (20s) which is quite often in production.
    1684          728 :         let (_guard, _permit) = tokio::select! {
    1685          728 :             tuple = prepare => { tuple },
    1686          728 :             _ = self.cancel.cancelled() => return Ok(false),
    1687          728 :             _ = cancel.cancelled() => return Ok(false),
    1688              :         };
    1689              : 
    1690          728 :         let last_record_lsn = self.get_last_record_lsn();
    1691          728 : 
    1692          728 :         // Last record Lsn could be zero in case the timeline was just created
    1693          728 :         if !last_record_lsn.is_valid() {
    1694            0 :             warn!("Skipping compaction for potentially just initialized timeline, it has invalid last record lsn: {last_record_lsn}");
    1695            0 :             return Ok(false);
    1696          728 :         }
    1697              : 
    1698          728 :         let result = match self.get_compaction_algorithm_settings().kind {
    1699              :             CompactionAlgorithm::Tiered => {
    1700            0 :                 self.compact_tiered(cancel, ctx).await?;
    1701            0 :                 Ok(false)
    1702              :             }
    1703          728 :             CompactionAlgorithm::Legacy => self.compact_legacy(cancel, options, ctx).await,
    1704              :         };
    1705              : 
    1706              :         // Signal compaction failure to avoid L0 flush stalls when it's broken.
    1707            0 :         match result {
    1708          728 :             Ok(_) => self.compaction_failed.store(false, AtomicOrdering::Relaxed),
    1709              :             Err(CompactionError::Other(_)) => {
    1710            0 :                 self.compaction_failed.store(true, AtomicOrdering::Relaxed)
    1711              :             }
    1712              :             // Don't change the current value on offload failure or shutdown. We don't want to
    1713              :             // abruptly stall nor resume L0 flushes in these cases.
    1714            0 :             Err(CompactionError::Offload(_)) => {}
    1715            0 :             Err(CompactionError::ShuttingDown) => {}
    1716              :         };
    1717              : 
    1718          728 :         result
    1719          728 :     }
    1720              : 
    1721              :     /// Mutate the timeline with a [`TimelineWriter`].
    1722     10266380 :     pub(crate) async fn writer(&self) -> TimelineWriter<'_> {
    1723     10266380 :         TimelineWriter {
    1724     10266380 :             tl: self,
    1725     10266380 :             write_guard: self.write_lock.lock().await,
    1726              :         }
    1727     10266380 :     }
    1728              : 
    1729            0 :     pub(crate) fn activate(
    1730            0 :         self: &Arc<Self>,
    1731            0 :         parent: Arc<crate::tenant::Tenant>,
    1732            0 :         broker_client: BrokerClientChannel,
    1733            0 :         background_jobs_can_start: Option<&completion::Barrier>,
    1734            0 :         ctx: &RequestContext,
    1735            0 :     ) {
    1736            0 :         if self.tenant_shard_id.is_shard_zero() {
    1737            0 :             // Logical size is only maintained accurately on shard zero.
    1738            0 :             self.spawn_initial_logical_size_computation_task(ctx);
    1739            0 :         }
    1740            0 :         self.launch_wal_receiver(ctx, broker_client);
    1741            0 :         self.set_state(TimelineState::Active);
    1742            0 :         self.launch_eviction_task(parent, background_jobs_can_start);
    1743            0 :     }
    1744              : 
    1745              :     /// After this function returns, there are no timeline-scoped tasks are left running.
    1746              :     ///
    1747              :     /// The preferred pattern for is:
    1748              :     /// - in any spawned tasks, keep Timeline::guard open + Timeline::cancel / child token
    1749              :     /// - if early shutdown (not just cancellation) of a sub-tree of tasks is required,
    1750              :     ///   go the extra mile and keep track of JoinHandles
    1751              :     /// - Keep track of JoinHandles using a passed-down `Arc<Mutex<Option<JoinSet>>>` or similar,
    1752              :     ///   instead of spawning directly on a runtime. It is a more composable / testable pattern.
    1753              :     ///
    1754              :     /// For legacy reasons, we still have multiple tasks spawned using
    1755              :     /// `task_mgr::spawn(X, Some(tenant_id), Some(timeline_id))`.
    1756              :     /// We refer to these as "timeline-scoped task_mgr tasks".
    1757              :     /// Some of these tasks are already sensitive to Timeline::cancel while others are
    1758              :     /// not sensitive to Timeline::cancel and instead respect [`task_mgr::shutdown_token`]
    1759              :     /// or [`task_mgr::shutdown_watcher`].
    1760              :     /// We want to gradually convert the code base away from these.
    1761              :     ///
    1762              :     /// Here is an inventory of timeline-scoped task_mgr tasks that are still sensitive to
    1763              :     /// `task_mgr::shutdown_{token,watcher}` (there are also tenant-scoped and global-scoped
    1764              :     /// ones that aren't mentioned here):
    1765              :     /// - [`TaskKind::TimelineDeletionWorker`]
    1766              :     ///    - NB: also used for tenant deletion
    1767              :     /// - [`TaskKind::RemoteUploadTask`]`
    1768              :     /// - [`TaskKind::InitialLogicalSizeCalculation`]
    1769              :     /// - [`TaskKind::DownloadAllRemoteLayers`] (can we get rid of it?)
    1770              :     // Inventory of timeline-scoped task_mgr tasks that use spawn but aren't sensitive:
    1771              :     /// - [`TaskKind::Eviction`]
    1772              :     /// - [`TaskKind::LayerFlushTask`]
    1773              :     /// - [`TaskKind::OndemandLogicalSizeCalculation`]
    1774              :     /// - [`TaskKind::GarbageCollector`] (immediate_gc is timeline-scoped)
    1775           20 :     pub(crate) async fn shutdown(&self, mode: ShutdownMode) {
    1776           20 :         debug_assert_current_span_has_tenant_and_timeline_id();
    1777           20 : 
    1778           20 :         // Regardless of whether we're going to try_freeze_and_flush
    1779           20 :         // or not, stop ingesting any more data. Walreceiver only provides
    1780           20 :         // cancellation but no "wait until gone", because it uses the Timeline::gate.
    1781           20 :         // So, only after the self.gate.close() below will we know for sure that
    1782           20 :         // no walreceiver tasks are left.
    1783           20 :         // For `try_freeze_and_flush=true`, this means that we might still be ingesting
    1784           20 :         // data during the call to `self.freeze_and_flush()` below.
    1785           20 :         // That's not ideal, but, we don't have the concept of a ChildGuard,
    1786           20 :         // which is what we'd need to properly model early shutdown of the walreceiver
    1787           20 :         // task sub-tree before the other Timeline task sub-trees.
    1788           20 :         let walreceiver = self.walreceiver.lock().unwrap().take();
    1789           20 :         tracing::debug!(
    1790            0 :             is_some = walreceiver.is_some(),
    1791            0 :             "Waiting for WalReceiverManager..."
    1792              :         );
    1793           20 :         if let Some(walreceiver) = walreceiver {
    1794            0 :             walreceiver.cancel();
    1795           20 :         }
    1796              :         // ... and inform any waiters for newer LSNs that there won't be any.
    1797           20 :         self.last_record_lsn.shutdown();
    1798           20 : 
    1799           20 :         if let ShutdownMode::FreezeAndFlush = mode {
    1800           12 :             if let Some((open, frozen)) = self
    1801           12 :                 .layers
    1802           12 :                 .read()
    1803           12 :                 .await
    1804           12 :                 .layer_map()
    1805           12 :                 .map(|lm| (lm.open_layer.is_some(), lm.frozen_layers.len()))
    1806           12 :                 .ok()
    1807           12 :                 .filter(|(open, frozen)| *open || *frozen > 0)
    1808              :             {
    1809            0 :                 tracing::info!(?open, frozen, "flushing and freezing on shutdown");
    1810           12 :             } else {
    1811           12 :                 // this is double-shutdown, ignore it
    1812           12 :             }
    1813              : 
    1814              :             // we shut down walreceiver above, so, we won't add anything more
    1815              :             // to the InMemoryLayer; freeze it and wait for all frozen layers
    1816              :             // to reach the disk & upload queue, then shut the upload queue and
    1817              :             // wait for it to drain.
    1818           12 :             match self.freeze_and_flush().await {
    1819              :                 Ok(_) => {
    1820              :                     // drain the upload queue
    1821              :                     // if we did not wait for completion here, it might be our shutdown process
    1822              :                     // didn't wait for remote uploads to complete at all, as new tasks can forever
    1823              :                     // be spawned.
    1824              :                     //
    1825              :                     // what is problematic is the shutting down of RemoteTimelineClient, because
    1826              :                     // obviously it does not make sense to stop while we wait for it, but what
    1827              :                     // about corner cases like s3 suddenly hanging up?
    1828           12 :                     self.remote_client.shutdown().await;
    1829              :                 }
    1830              :                 Err(FlushLayerError::Cancelled) => {
    1831              :                     // this is likely the second shutdown, ignore silently.
    1832              :                     // TODO: this can be removed once https://github.com/neondatabase/neon/issues/5080
    1833            0 :                     debug_assert!(self.cancel.is_cancelled());
    1834              :                 }
    1835            0 :                 Err(e) => {
    1836            0 :                     // Non-fatal.  Shutdown is infallible.  Failures to flush just mean that
    1837            0 :                     // we have some extra WAL replay to do next time the timeline starts.
    1838            0 :                     warn!("failed to freeze and flush: {e:#}");
    1839              :                 }
    1840              :             }
    1841              : 
    1842              :             // `self.remote_client.shutdown().await` above should have already flushed everything from the queue, but
    1843              :             // we also do a final check here to ensure that the queue is empty.
    1844           12 :             if !self.remote_client.no_pending_work() {
    1845            0 :                 warn!("still have pending work in remote upload queue, but continuing shutting down anyways");
    1846           12 :             }
    1847            8 :         }
    1848              : 
    1849           20 :         if let ShutdownMode::Reload = mode {
    1850              :             // drain the upload queue
    1851            4 :             self.remote_client.shutdown().await;
    1852            4 :             if !self.remote_client.no_pending_work() {
    1853            0 :                 warn!("still have pending work in remote upload queue, but continuing shutting down anyways");
    1854            4 :             }
    1855           16 :         }
    1856              : 
    1857              :         // Signal any subscribers to our cancellation token to drop out
    1858           20 :         tracing::debug!("Cancelling CancellationToken");
    1859           20 :         self.cancel.cancel();
    1860           20 : 
    1861           20 :         // Ensure Prevent new page service requests from starting.
    1862           20 :         self.handles.shutdown();
    1863           20 : 
    1864           20 :         // Transition the remote_client into a state where it's only useful for timeline deletion.
    1865           20 :         // (The deletion use case is why we can't just hook up remote_client to Self::cancel).)
    1866           20 :         self.remote_client.stop();
    1867           20 : 
    1868           20 :         // As documented in remote_client.stop()'s doc comment, it's our responsibility
    1869           20 :         // to shut down the upload queue tasks.
    1870           20 :         // TODO: fix that, task management should be encapsulated inside remote_client.
    1871           20 :         task_mgr::shutdown_tasks(
    1872           20 :             Some(TaskKind::RemoteUploadTask),
    1873           20 :             Some(self.tenant_shard_id),
    1874           20 :             Some(self.timeline_id),
    1875           20 :         )
    1876           20 :         .await;
    1877              : 
    1878              :         // TODO: work toward making this a no-op. See this function's doc comment for more context.
    1879           20 :         tracing::debug!("Waiting for tasks...");
    1880           20 :         task_mgr::shutdown_tasks(None, Some(self.tenant_shard_id), Some(self.timeline_id)).await;
    1881              : 
    1882              :         {
    1883              :             // Allow any remaining in-memory layers to do cleanup -- until that, they hold the gate
    1884              :             // open.
    1885           20 :             let mut write_guard = self.write_lock.lock().await;
    1886           20 :             self.layers.write().await.shutdown(&mut write_guard);
    1887           20 :         }
    1888           20 : 
    1889           20 :         // Finally wait until any gate-holders are complete.
    1890           20 :         //
    1891           20 :         // TODO: once above shutdown_tasks is a no-op, we can close the gate before calling shutdown_tasks
    1892           20 :         // and use a TBD variant of shutdown_tasks that asserts that there were no tasks left.
    1893           20 :         self.gate.close().await;
    1894              : 
    1895           20 :         self.metrics.shutdown();
    1896           20 :     }
    1897              : 
    1898          896 :     pub(crate) fn set_state(&self, new_state: TimelineState) {
    1899          896 :         match (self.current_state(), new_state) {
    1900          896 :             (equal_state_1, equal_state_2) if equal_state_1 == equal_state_2 => {
    1901            4 :                 info!("Ignoring new state, equal to the existing one: {equal_state_2:?}");
    1902              :             }
    1903            0 :             (st, TimelineState::Loading) => {
    1904            0 :                 error!("ignoring transition from {st:?} into Loading state");
    1905              :             }
    1906            0 :             (TimelineState::Broken { .. }, new_state) => {
    1907            0 :                 error!("Ignoring state update {new_state:?} for broken timeline");
    1908              :             }
    1909              :             (TimelineState::Stopping, TimelineState::Active) => {
    1910            0 :                 error!("Not activating a Stopping timeline");
    1911              :             }
    1912          892 :             (_, new_state) => {
    1913          892 :                 self.state.send_replace(new_state);
    1914          892 :             }
    1915              :         }
    1916          896 :     }
    1917              : 
    1918            4 :     pub(crate) fn set_broken(&self, reason: String) {
    1919            4 :         let backtrace_str: String = format!("{}", std::backtrace::Backtrace::force_capture());
    1920            4 :         let broken_state = TimelineState::Broken {
    1921            4 :             reason,
    1922            4 :             backtrace: backtrace_str,
    1923            4 :         };
    1924            4 :         self.set_state(broken_state);
    1925            4 : 
    1926            4 :         // Although the Broken state is not equivalent to shutdown() (shutdown will be called
    1927            4 :         // later when this tenant is detach or the process shuts down), firing the cancellation token
    1928            4 :         // here avoids the need for other tasks to watch for the Broken state explicitly.
    1929            4 :         self.cancel.cancel();
    1930            4 :     }
    1931              : 
    1932       453243 :     pub(crate) fn current_state(&self) -> TimelineState {
    1933       453243 :         self.state.borrow().clone()
    1934       453243 :     }
    1935              : 
    1936           12 :     pub(crate) fn is_broken(&self) -> bool {
    1937           12 :         matches!(&*self.state.borrow(), TimelineState::Broken { .. })
    1938           12 :     }
    1939              : 
    1940          444 :     pub(crate) fn is_active(&self) -> bool {
    1941          444 :         self.current_state() == TimelineState::Active
    1942          444 :     }
    1943              : 
    1944            0 :     pub(crate) fn is_archived(&self) -> Option<bool> {
    1945            0 :         self.remote_client.is_archived()
    1946            0 :     }
    1947              : 
    1948          736 :     pub(crate) fn is_stopping(&self) -> bool {
    1949          736 :         self.current_state() == TimelineState::Stopping
    1950          736 :     }
    1951              : 
    1952            0 :     pub(crate) fn subscribe_for_state_updates(&self) -> watch::Receiver<TimelineState> {
    1953            0 :         self.state.subscribe()
    1954            0 :     }
    1955              : 
    1956       451171 :     pub(crate) async fn wait_to_become_active(
    1957       451171 :         &self,
    1958       451171 :         _ctx: &RequestContext, // Prepare for use by cancellation
    1959       451171 :     ) -> Result<(), TimelineState> {
    1960       451171 :         let mut receiver = self.state.subscribe();
    1961              :         loop {
    1962       451171 :             let current_state = receiver.borrow().clone();
    1963       451171 :             match current_state {
    1964              :                 TimelineState::Loading => {
    1965            0 :                     receiver
    1966            0 :                         .changed()
    1967            0 :                         .await
    1968            0 :                         .expect("holding a reference to self");
    1969              :                 }
    1970              :                 TimelineState::Active { .. } => {
    1971       451167 :                     return Ok(());
    1972              :                 }
    1973              :                 TimelineState::Broken { .. } | TimelineState::Stopping => {
    1974              :                     // There's no chance the timeline can transition back into ::Active
    1975            4 :                     return Err(current_state);
    1976              :                 }
    1977              :             }
    1978              :         }
    1979       451171 :     }
    1980              : 
    1981            0 :     pub(crate) async fn layer_map_info(
    1982            0 :         &self,
    1983            0 :         reset: LayerAccessStatsReset,
    1984            0 :     ) -> Result<LayerMapInfo, layer_manager::Shutdown> {
    1985            0 :         let guard = self.layers.read().await;
    1986            0 :         let layer_map = guard.layer_map()?;
    1987            0 :         let mut in_memory_layers = Vec::with_capacity(layer_map.frozen_layers.len() + 1);
    1988            0 :         if let Some(open_layer) = &layer_map.open_layer {
    1989            0 :             in_memory_layers.push(open_layer.info());
    1990            0 :         }
    1991            0 :         for frozen_layer in &layer_map.frozen_layers {
    1992            0 :             in_memory_layers.push(frozen_layer.info());
    1993            0 :         }
    1994              : 
    1995            0 :         let historic_layers = layer_map
    1996            0 :             .iter_historic_layers()
    1997            0 :             .map(|desc| guard.get_from_desc(&desc).info(reset))
    1998            0 :             .collect();
    1999            0 : 
    2000            0 :         Ok(LayerMapInfo {
    2001            0 :             in_memory_layers,
    2002            0 :             historic_layers,
    2003            0 :         })
    2004            0 :     }
    2005              : 
    2006              :     #[instrument(skip_all, fields(tenant_id = %self.tenant_shard_id.tenant_id, shard_id = %self.tenant_shard_id.shard_slug(), timeline_id = %self.timeline_id))]
    2007              :     pub(crate) async fn download_layer(
    2008              :         &self,
    2009              :         layer_file_name: &LayerName,
    2010              :     ) -> anyhow::Result<Option<bool>> {
    2011              :         let Some(layer) = self.find_layer(layer_file_name).await? else {
    2012              :             return Ok(None);
    2013              :         };
    2014              : 
    2015              :         layer.download().await?;
    2016              : 
    2017              :         Ok(Some(true))
    2018              :     }
    2019              : 
    2020              :     /// Evict just one layer.
    2021              :     ///
    2022              :     /// Returns `Ok(None)` in the case where the layer could not be found by its `layer_file_name`.
    2023            0 :     pub(crate) async fn evict_layer(
    2024            0 :         &self,
    2025            0 :         layer_file_name: &LayerName,
    2026            0 :     ) -> anyhow::Result<Option<bool>> {
    2027            0 :         let _gate = self
    2028            0 :             .gate
    2029            0 :             .enter()
    2030            0 :             .map_err(|_| anyhow::anyhow!("Shutting down"))?;
    2031              : 
    2032            0 :         let Some(local_layer) = self.find_layer(layer_file_name).await? else {
    2033            0 :             return Ok(None);
    2034              :         };
    2035              : 
    2036              :         // curl has this by default
    2037            0 :         let timeout = std::time::Duration::from_secs(120);
    2038            0 : 
    2039            0 :         match local_layer.evict_and_wait(timeout).await {
    2040            0 :             Ok(()) => Ok(Some(true)),
    2041            0 :             Err(EvictionError::NotFound) => Ok(Some(false)),
    2042            0 :             Err(EvictionError::Downloaded) => Ok(Some(false)),
    2043            0 :             Err(EvictionError::Timeout) => Ok(Some(false)),
    2044              :         }
    2045            0 :     }
    2046              : 
    2047      9606020 :     fn should_roll(
    2048      9606020 :         &self,
    2049      9606020 :         layer_size: u64,
    2050      9606020 :         projected_layer_size: u64,
    2051      9606020 :         checkpoint_distance: u64,
    2052      9606020 :         projected_lsn: Lsn,
    2053      9606020 :         last_freeze_at: Lsn,
    2054      9606020 :         opened_at: Instant,
    2055      9606020 :     ) -> bool {
    2056      9606020 :         let distance = projected_lsn.widening_sub(last_freeze_at);
    2057      9606020 : 
    2058      9606020 :         // Rolling the open layer can be triggered by:
    2059      9606020 :         // 1. The distance from the last LSN we rolled at. This bounds the amount of WAL that
    2060      9606020 :         //    the safekeepers need to store.  For sharded tenants, we multiply by shard count to
    2061      9606020 :         //    account for how writes are distributed across shards: we expect each node to consume
    2062      9606020 :         //    1/count of the LSN on average.
    2063      9606020 :         // 2. The size of the currently open layer.
    2064      9606020 :         // 3. The time since the last roll. It helps safekeepers to regard pageserver as caught
    2065      9606020 :         //    up and suspend activity.
    2066      9606020 :         if distance >= checkpoint_distance as i128 * self.shard_identity.count.count() as i128 {
    2067            0 :             info!(
    2068            0 :                 "Will roll layer at {} with layer size {} due to LSN distance ({})",
    2069              :                 projected_lsn, layer_size, distance
    2070              :             );
    2071              : 
    2072            0 :             true
    2073      9606020 :         } else if projected_layer_size >= checkpoint_distance {
    2074              :             // NB: this check is relied upon by:
    2075          160 :             let _ = IndexEntry::validate_checkpoint_distance;
    2076          160 :             info!(
    2077            0 :                 "Will roll layer at {} with layer size {} due to layer size ({})",
    2078              :                 projected_lsn, layer_size, projected_layer_size
    2079              :             );
    2080              : 
    2081          160 :             true
    2082      9605860 :         } else if distance > 0 && opened_at.elapsed() >= self.get_checkpoint_timeout() {
    2083            0 :             info!(
    2084            0 :                 "Will roll layer at {} with layer size {} due to time since first write to the layer ({:?})",
    2085            0 :                 projected_lsn,
    2086            0 :                 layer_size,
    2087            0 :                 opened_at.elapsed()
    2088              :             );
    2089              : 
    2090            0 :             true
    2091              :         } else {
    2092      9605860 :             false
    2093              :         }
    2094      9606020 :     }
    2095              : }
    2096              : 
    2097              : /// Number of times we will compute partition within a checkpoint distance.
    2098              : const REPARTITION_FREQ_IN_CHECKPOINT_DISTANCE: u64 = 10;
    2099              : 
    2100              : // Private functions
    2101              : impl Timeline {
    2102           24 :     pub(crate) fn get_lsn_lease_length(&self) -> Duration {
    2103           24 :         let tenant_conf = self.tenant_conf.load();
    2104           24 :         tenant_conf
    2105           24 :             .tenant_conf
    2106           24 :             .lsn_lease_length
    2107           24 :             .unwrap_or(self.conf.default_tenant_conf.lsn_lease_length)
    2108           24 :     }
    2109              : 
    2110            0 :     pub(crate) fn get_lsn_lease_length_for_ts(&self) -> Duration {
    2111            0 :         let tenant_conf = self.tenant_conf.load();
    2112            0 :         tenant_conf
    2113            0 :             .tenant_conf
    2114            0 :             .lsn_lease_length_for_ts
    2115            0 :             .unwrap_or(self.conf.default_tenant_conf.lsn_lease_length_for_ts)
    2116            0 :     }
    2117              : 
    2118            0 :     pub(crate) fn is_gc_blocked_by_lsn_lease_deadline(&self) -> bool {
    2119            0 :         let tenant_conf = self.tenant_conf.load();
    2120            0 :         tenant_conf.is_gc_blocked_by_lsn_lease_deadline()
    2121            0 :     }
    2122              : 
    2123            0 :     pub(crate) fn get_lazy_slru_download(&self) -> bool {
    2124            0 :         let tenant_conf = self.tenant_conf.load();
    2125            0 :         tenant_conf
    2126            0 :             .tenant_conf
    2127            0 :             .lazy_slru_download
    2128            0 :             .unwrap_or(self.conf.default_tenant_conf.lazy_slru_download)
    2129            0 :     }
    2130              : 
    2131      9608992 :     fn get_checkpoint_distance(&self) -> u64 {
    2132      9608992 :         let tenant_conf = self.tenant_conf.load();
    2133      9608992 :         tenant_conf
    2134      9608992 :             .tenant_conf
    2135      9608992 :             .checkpoint_distance
    2136      9608992 :             .unwrap_or(self.conf.default_tenant_conf.checkpoint_distance)
    2137      9608992 :     }
    2138              : 
    2139      9605860 :     fn get_checkpoint_timeout(&self) -> Duration {
    2140      9605860 :         let tenant_conf = self.tenant_conf.load();
    2141      9605860 :         tenant_conf
    2142      9605860 :             .tenant_conf
    2143      9605860 :             .checkpoint_timeout
    2144      9605860 :             .unwrap_or(self.conf.default_tenant_conf.checkpoint_timeout)
    2145      9605860 :     }
    2146              : 
    2147         5008 :     fn get_compaction_period(&self) -> Duration {
    2148         5008 :         let tenant_conf = self.tenant_conf.load().tenant_conf.clone();
    2149         5008 :         tenant_conf
    2150         5008 :             .compaction_period
    2151         5008 :             .unwrap_or(self.conf.default_tenant_conf.compaction_period)
    2152         5008 :     }
    2153              : 
    2154         1324 :     fn get_compaction_target_size(&self) -> u64 {
    2155         1324 :         let tenant_conf = self.tenant_conf.load();
    2156         1324 :         tenant_conf
    2157         1324 :             .tenant_conf
    2158         1324 :             .compaction_target_size
    2159         1324 :             .unwrap_or(self.conf.default_tenant_conf.compaction_target_size)
    2160         1324 :     }
    2161              : 
    2162          784 :     fn get_compaction_threshold(&self) -> usize {
    2163          784 :         let tenant_conf = self.tenant_conf.load();
    2164          784 :         tenant_conf
    2165          784 :             .tenant_conf
    2166          784 :             .compaction_threshold
    2167          784 :             .unwrap_or(self.conf.default_tenant_conf.compaction_threshold)
    2168          784 :     }
    2169              : 
    2170         2504 :     fn get_l0_flush_delay_threshold(&self) -> Option<usize> {
    2171              :         // Default to delay L0 flushes at 3x compaction threshold.
    2172              :         const DEFAULT_L0_FLUSH_DELAY_FACTOR: usize = 3;
    2173              : 
    2174              :         // If compaction is disabled, don't delay.
    2175         2504 :         if self.get_compaction_period() == Duration::ZERO {
    2176         2504 :             return None;
    2177            0 :         }
    2178            0 : 
    2179            0 :         let compaction_threshold = self.get_compaction_threshold();
    2180            0 :         let tenant_conf = self.tenant_conf.load();
    2181            0 :         let l0_flush_delay_threshold = tenant_conf
    2182            0 :             .tenant_conf
    2183            0 :             .l0_flush_delay_threshold
    2184            0 :             .or(self.conf.default_tenant_conf.l0_flush_delay_threshold)
    2185            0 :             .unwrap_or(DEFAULT_L0_FLUSH_DELAY_FACTOR * compaction_threshold);
    2186            0 : 
    2187            0 :         // 0 disables backpressure.
    2188            0 :         if l0_flush_delay_threshold == 0 {
    2189            0 :             return None;
    2190            0 :         }
    2191            0 : 
    2192            0 :         // Clamp the flush delay threshold to the compaction threshold; it doesn't make sense to
    2193            0 :         // backpressure flushes below this.
    2194            0 :         // TODO: the tenant config should have validation to prevent this instead.
    2195            0 :         debug_assert!(l0_flush_delay_threshold >= compaction_threshold);
    2196            0 :         Some(max(l0_flush_delay_threshold, compaction_threshold))
    2197         2504 :     }
    2198              : 
    2199         2504 :     fn get_l0_flush_stall_threshold(&self) -> Option<usize> {
    2200              :         // Default to stall L0 flushes at 5x compaction threshold.
    2201              :         // TODO: stalls are temporarily disabled by default, see below.
    2202              :         #[allow(unused)]
    2203              :         const DEFAULT_L0_FLUSH_STALL_FACTOR: usize = 5;
    2204              : 
    2205              :         // If compaction is disabled, don't stall.
    2206         2504 :         if self.get_compaction_period() == Duration::ZERO {
    2207         2504 :             return None;
    2208            0 :         }
    2209            0 : 
    2210            0 :         // If compaction is failing, don't stall and try to keep the tenant alive. This may not be a
    2211            0 :         // good idea: read amp can grow unbounded, leading to terrible performance, and we may take
    2212            0 :         // on unbounded compaction debt that can take a long time to fix once compaction comes back
    2213            0 :         // online. At least we'll delay flushes, slowing down the growth and buying some time.
    2214            0 :         if self.compaction_failed.load(AtomicOrdering::Relaxed) {
    2215            0 :             return None;
    2216            0 :         }
    2217            0 : 
    2218            0 :         let compaction_threshold = self.get_compaction_threshold();
    2219            0 :         let tenant_conf = self.tenant_conf.load();
    2220            0 :         let l0_flush_stall_threshold = tenant_conf
    2221            0 :             .tenant_conf
    2222            0 :             .l0_flush_stall_threshold
    2223            0 :             .or(self.conf.default_tenant_conf.l0_flush_stall_threshold);
    2224            0 : 
    2225            0 :         // Tests sometimes set compaction_threshold=1 to generate lots of layer files, and don't
    2226            0 :         // handle the 20-second compaction delay. Some (e.g. `test_backward_compatibility`) can't
    2227            0 :         // easily adjust the L0 backpressure settings, so just disable stalls in this case.
    2228            0 :         if cfg!(feature = "testing")
    2229            0 :             && compaction_threshold == 1
    2230            0 :             && l0_flush_stall_threshold.is_none()
    2231              :         {
    2232            0 :             return None;
    2233            0 :         }
    2234              : 
    2235              :         // Disable stalls by default. In ingest benchmarks, we see image compaction take >10
    2236              :         // minutes, blocking L0 compaction, and we can't stall L0 flushes for that long.
    2237              :         //
    2238              :         // TODO: fix this.
    2239              :         // let l0_flush_stall_threshold = l0_flush_stall_threshold
    2240              :         //    .unwrap_or(DEFAULT_L0_FLUSH_STALL_FACTOR * compaction_threshold);
    2241            0 :         let l0_flush_stall_threshold = l0_flush_stall_threshold?;
    2242              : 
    2243              :         // 0 disables backpressure.
    2244            0 :         if l0_flush_stall_threshold == 0 {
    2245            0 :             return None;
    2246            0 :         }
    2247            0 : 
    2248            0 :         // Clamp the flush stall threshold to the compaction threshold; it doesn't make sense to
    2249            0 :         // backpressure flushes below this.
    2250            0 :         // TODO: the tenant config should have validation to prevent this instead.
    2251            0 :         debug_assert!(l0_flush_stall_threshold >= compaction_threshold);
    2252            0 :         Some(max(l0_flush_stall_threshold, compaction_threshold))
    2253         2504 :     }
    2254              : 
    2255           28 :     fn get_image_creation_threshold(&self) -> usize {
    2256           28 :         let tenant_conf = self.tenant_conf.load();
    2257           28 :         tenant_conf
    2258           28 :             .tenant_conf
    2259           28 :             .image_creation_threshold
    2260           28 :             .unwrap_or(self.conf.default_tenant_conf.image_creation_threshold)
    2261           28 :     }
    2262              : 
    2263          728 :     fn get_compaction_algorithm_settings(&self) -> CompactionAlgorithmSettings {
    2264          728 :         let tenant_conf = &self.tenant_conf.load();
    2265          728 :         tenant_conf
    2266          728 :             .tenant_conf
    2267          728 :             .compaction_algorithm
    2268          728 :             .as_ref()
    2269          728 :             .unwrap_or(&self.conf.default_tenant_conf.compaction_algorithm)
    2270          728 :             .clone()
    2271          728 :     }
    2272              : 
    2273            0 :     fn get_eviction_policy(&self) -> EvictionPolicy {
    2274            0 :         let tenant_conf = self.tenant_conf.load();
    2275            0 :         tenant_conf
    2276            0 :             .tenant_conf
    2277            0 :             .eviction_policy
    2278            0 :             .unwrap_or(self.conf.default_tenant_conf.eviction_policy)
    2279            0 :     }
    2280              : 
    2281          892 :     fn get_evictions_low_residence_duration_metric_threshold(
    2282          892 :         tenant_conf: &TenantConfOpt,
    2283          892 :         default_tenant_conf: &TenantConf,
    2284          892 :     ) -> Duration {
    2285          892 :         tenant_conf
    2286          892 :             .evictions_low_residence_duration_metric_threshold
    2287          892 :             .unwrap_or(default_tenant_conf.evictions_low_residence_duration_metric_threshold)
    2288          892 :     }
    2289              : 
    2290         1136 :     fn get_image_layer_creation_check_threshold(&self) -> u8 {
    2291         1136 :         let tenant_conf = self.tenant_conf.load();
    2292         1136 :         tenant_conf
    2293         1136 :             .tenant_conf
    2294         1136 :             .image_layer_creation_check_threshold
    2295         1136 :             .unwrap_or(
    2296         1136 :                 self.conf
    2297         1136 :                     .default_tenant_conf
    2298         1136 :                     .image_layer_creation_check_threshold,
    2299         1136 :             )
    2300         1136 :     }
    2301              : 
    2302              :     /// Resolve the effective WAL receiver protocol to use for this tenant.
    2303              :     ///
    2304              :     /// Priority order is:
    2305              :     /// 1. Tenant config override
    2306              :     /// 2. Default value for tenant config override
    2307              :     /// 3. Pageserver config override
    2308              :     /// 4. Pageserver config default
    2309            0 :     pub fn resolve_wal_receiver_protocol(&self) -> PostgresClientProtocol {
    2310            0 :         let tenant_conf = self.tenant_conf.load().tenant_conf.clone();
    2311            0 :         tenant_conf
    2312            0 :             .wal_receiver_protocol_override
    2313            0 :             .or(self.conf.default_tenant_conf.wal_receiver_protocol_override)
    2314            0 :             .unwrap_or(self.conf.wal_receiver_protocol)
    2315            0 :     }
    2316              : 
    2317            0 :     pub(super) fn tenant_conf_updated(&self, new_conf: &AttachedTenantConf) {
    2318            0 :         // NB: Most tenant conf options are read by background loops, so,
    2319            0 :         // changes will automatically be picked up.
    2320            0 : 
    2321            0 :         // The threshold is embedded in the metric. So, we need to update it.
    2322            0 :         {
    2323            0 :             let new_threshold = Self::get_evictions_low_residence_duration_metric_threshold(
    2324            0 :                 &new_conf.tenant_conf,
    2325            0 :                 &self.conf.default_tenant_conf,
    2326            0 :             );
    2327            0 : 
    2328            0 :             let tenant_id_str = self.tenant_shard_id.tenant_id.to_string();
    2329            0 :             let shard_id_str = format!("{}", self.tenant_shard_id.shard_slug());
    2330            0 : 
    2331            0 :             let timeline_id_str = self.timeline_id.to_string();
    2332            0 : 
    2333            0 :             self.remote_client.update_config(&new_conf.location);
    2334            0 : 
    2335            0 :             self.metrics
    2336            0 :                 .evictions_with_low_residence_duration
    2337            0 :                 .write()
    2338            0 :                 .unwrap()
    2339            0 :                 .change_threshold(
    2340            0 :                     &tenant_id_str,
    2341            0 :                     &shard_id_str,
    2342            0 :                     &timeline_id_str,
    2343            0 :                     new_threshold,
    2344            0 :                 );
    2345            0 :         }
    2346            0 :     }
    2347              : 
    2348              :     /// Open a Timeline handle.
    2349              :     ///
    2350              :     /// Loads the metadata for the timeline into memory, but not the layer map.
    2351              :     #[allow(clippy::too_many_arguments)]
    2352          892 :     pub(super) fn new(
    2353          892 :         conf: &'static PageServerConf,
    2354          892 :         tenant_conf: Arc<ArcSwap<AttachedTenantConf>>,
    2355          892 :         metadata: &TimelineMetadata,
    2356          892 :         ancestor: Option<Arc<Timeline>>,
    2357          892 :         timeline_id: TimelineId,
    2358          892 :         tenant_shard_id: TenantShardId,
    2359          892 :         generation: Generation,
    2360          892 :         shard_identity: ShardIdentity,
    2361          892 :         walredo_mgr: Option<Arc<super::WalRedoManager>>,
    2362          892 :         resources: TimelineResources,
    2363          892 :         pg_version: u32,
    2364          892 :         state: TimelineState,
    2365          892 :         attach_wal_lag_cooldown: Arc<OnceLock<WalLagCooldown>>,
    2366          892 :         create_idempotency: crate::tenant::CreateTimelineIdempotency,
    2367          892 :         cancel: CancellationToken,
    2368          892 :     ) -> Arc<Self> {
    2369          892 :         let disk_consistent_lsn = metadata.disk_consistent_lsn();
    2370          892 :         let (state, _) = watch::channel(state);
    2371          892 : 
    2372          892 :         let (layer_flush_start_tx, _) = tokio::sync::watch::channel((0, disk_consistent_lsn));
    2373          892 :         let (layer_flush_done_tx, _) = tokio::sync::watch::channel((0, Ok(())));
    2374          892 : 
    2375          892 :         let evictions_low_residence_duration_metric_threshold = {
    2376          892 :             let loaded_tenant_conf = tenant_conf.load();
    2377          892 :             Self::get_evictions_low_residence_duration_metric_threshold(
    2378          892 :                 &loaded_tenant_conf.tenant_conf,
    2379          892 :                 &conf.default_tenant_conf,
    2380          892 :             )
    2381              :         };
    2382              : 
    2383          892 :         if let Some(ancestor) = &ancestor {
    2384          460 :             let mut ancestor_gc_info = ancestor.gc_info.write().unwrap();
    2385          460 :             // If we construct an explicit timeline object, it's obviously not offloaded
    2386          460 :             let is_offloaded = MaybeOffloaded::No;
    2387          460 :             ancestor_gc_info.insert_child(timeline_id, metadata.ancestor_lsn(), is_offloaded);
    2388          460 :         }
    2389              : 
    2390          892 :         Arc::new_cyclic(|myself| {
    2391          892 :             let metrics = TimelineMetrics::new(
    2392          892 :                 &tenant_shard_id,
    2393          892 :                 &timeline_id,
    2394          892 :                 crate::metrics::EvictionsWithLowResidenceDurationBuilder::new(
    2395          892 :                     "mtime",
    2396          892 :                     evictions_low_residence_duration_metric_threshold,
    2397          892 :                 ),
    2398          892 :             );
    2399          892 :             let aux_file_metrics = metrics.aux_file_size_gauge.clone();
    2400              : 
    2401          892 :             let mut result = Timeline {
    2402          892 :                 conf,
    2403          892 :                 tenant_conf,
    2404          892 :                 myself: myself.clone(),
    2405          892 :                 timeline_id,
    2406          892 :                 tenant_shard_id,
    2407          892 :                 generation,
    2408          892 :                 shard_identity,
    2409          892 :                 pg_version,
    2410          892 :                 layers: Default::default(),
    2411          892 : 
    2412          892 :                 walredo_mgr,
    2413          892 :                 walreceiver: Mutex::new(None),
    2414          892 : 
    2415          892 :                 remote_client: Arc::new(resources.remote_client),
    2416          892 : 
    2417          892 :                 // initialize in-memory 'last_record_lsn' from 'disk_consistent_lsn'.
    2418          892 :                 last_record_lsn: SeqWait::new(RecordLsn {
    2419          892 :                     last: disk_consistent_lsn,
    2420          892 :                     prev: metadata.prev_record_lsn().unwrap_or(Lsn(0)),
    2421          892 :                 }),
    2422          892 :                 disk_consistent_lsn: AtomicLsn::new(disk_consistent_lsn.0),
    2423          892 : 
    2424          892 :                 last_freeze_at: AtomicLsn::new(disk_consistent_lsn.0),
    2425          892 :                 last_freeze_ts: RwLock::new(Instant::now()),
    2426          892 : 
    2427          892 :                 loaded_at: (disk_consistent_lsn, SystemTime::now()),
    2428          892 : 
    2429          892 :                 ancestor_timeline: ancestor,
    2430          892 :                 ancestor_lsn: metadata.ancestor_lsn(),
    2431          892 : 
    2432          892 :                 metrics,
    2433          892 : 
    2434          892 :                 query_metrics: crate::metrics::SmgrQueryTimePerTimeline::new(
    2435          892 :                     &tenant_shard_id,
    2436          892 :                     &timeline_id,
    2437          892 :                     resources.pagestream_throttle_metrics,
    2438          892 :                 ),
    2439          892 : 
    2440         6244 :                 directory_metrics: array::from_fn(|_| AtomicU64::new(0)),
    2441          892 : 
    2442          892 :                 flush_loop_state: Mutex::new(FlushLoopState::NotStarted),
    2443          892 : 
    2444          892 :                 layer_flush_start_tx,
    2445          892 :                 layer_flush_done_tx,
    2446          892 : 
    2447          892 :                 write_lock: tokio::sync::Mutex::new(None),
    2448          892 : 
    2449          892 :                 gc_info: std::sync::RwLock::new(GcInfo::default()),
    2450          892 : 
    2451          892 :                 latest_gc_cutoff_lsn: Rcu::new(metadata.latest_gc_cutoff_lsn()),
    2452          892 :                 initdb_lsn: metadata.initdb_lsn(),
    2453          892 : 
    2454          892 :                 current_logical_size: if disk_consistent_lsn.is_valid() {
    2455              :                     // we're creating timeline data with some layer files existing locally,
    2456              :                     // need to recalculate timeline's logical size based on data in the layers.
    2457          468 :                     LogicalSize::deferred_initial(disk_consistent_lsn)
    2458              :                 } else {
    2459              :                     // we're creating timeline data without any layers existing locally,
    2460              :                     // initial logical size is 0.
    2461          424 :                     LogicalSize::empty_initial()
    2462              :                 },
    2463              : 
    2464          892 :                 partitioning: GuardArcSwap::new((
    2465          892 :                     (KeyPartitioning::new(), KeyPartitioning::new().into_sparse()),
    2466          892 :                     Lsn(0),
    2467          892 :                 )),
    2468          892 :                 repartition_threshold: 0,
    2469          892 :                 last_image_layer_creation_check_at: AtomicLsn::new(0),
    2470          892 :                 last_image_layer_creation_check_instant: Mutex::new(None),
    2471          892 : 
    2472          892 :                 last_received_wal: Mutex::new(None),
    2473          892 :                 rel_size_cache: RwLock::new(RelSizeCache {
    2474          892 :                     complete_as_of: disk_consistent_lsn,
    2475          892 :                     map: HashMap::new(),
    2476          892 :                 }),
    2477          892 : 
    2478          892 :                 download_all_remote_layers_task_info: RwLock::new(None),
    2479          892 : 
    2480          892 :                 state,
    2481          892 : 
    2482          892 :                 eviction_task_timeline_state: tokio::sync::Mutex::new(
    2483          892 :                     EvictionTaskTimelineState::default(),
    2484          892 :                 ),
    2485          892 :                 delete_progress: TimelineDeleteProgress::default(),
    2486          892 : 
    2487          892 :                 cancel,
    2488          892 :                 gate: Gate::default(),
    2489          892 : 
    2490          892 :                 compaction_lock: tokio::sync::Mutex::default(),
    2491          892 :                 compaction_failed: AtomicBool::default(),
    2492          892 :                 gc_lock: tokio::sync::Mutex::default(),
    2493          892 : 
    2494          892 :                 standby_horizon: AtomicLsn::new(0),
    2495          892 : 
    2496          892 :                 pagestream_throttle: resources.pagestream_throttle,
    2497          892 : 
    2498          892 :                 aux_file_size_estimator: AuxFileSizeEstimator::new(aux_file_metrics),
    2499          892 : 
    2500          892 :                 #[cfg(test)]
    2501          892 :                 extra_test_dense_keyspace: ArcSwap::new(Arc::new(KeySpace::default())),
    2502          892 : 
    2503          892 :                 l0_flush_global_state: resources.l0_flush_global_state,
    2504          892 : 
    2505          892 :                 handles: Default::default(),
    2506          892 : 
    2507          892 :                 attach_wal_lag_cooldown,
    2508          892 : 
    2509          892 :                 create_idempotency,
    2510          892 : 
    2511          892 :                 page_trace: Default::default(),
    2512          892 :             };
    2513          892 : 
    2514          892 :             result.repartition_threshold =
    2515          892 :                 result.get_checkpoint_distance() / REPARTITION_FREQ_IN_CHECKPOINT_DISTANCE;
    2516          892 : 
    2517          892 :             result
    2518          892 :                 .metrics
    2519          892 :                 .last_record_lsn_gauge
    2520          892 :                 .set(disk_consistent_lsn.0 as i64);
    2521          892 :             result
    2522          892 :         })
    2523          892 :     }
    2524              : 
    2525         1288 :     pub(super) fn maybe_spawn_flush_loop(self: &Arc<Self>) {
    2526         1288 :         let Ok(guard) = self.gate.enter() else {
    2527            0 :             info!("cannot start flush loop when the timeline gate has already been closed");
    2528            0 :             return;
    2529              :         };
    2530         1288 :         let mut flush_loop_state = self.flush_loop_state.lock().unwrap();
    2531         1288 :         match *flush_loop_state {
    2532          880 :             FlushLoopState::NotStarted => (),
    2533              :             FlushLoopState::Running { .. } => {
    2534          408 :                 info!(
    2535            0 :                     "skipping attempt to start flush_loop twice {}/{}",
    2536            0 :                     self.tenant_shard_id, self.timeline_id
    2537              :                 );
    2538          408 :                 return;
    2539              :             }
    2540              :             FlushLoopState::Exited => {
    2541            0 :                 warn!(
    2542            0 :                     "ignoring attempt to restart exited flush_loop {}/{}",
    2543            0 :                     self.tenant_shard_id, self.timeline_id
    2544              :                 );
    2545            0 :                 return;
    2546              :             }
    2547              :         }
    2548              : 
    2549          880 :         let layer_flush_start_rx = self.layer_flush_start_tx.subscribe();
    2550          880 :         let self_clone = Arc::clone(self);
    2551          880 : 
    2552          880 :         debug!("spawning flush loop");
    2553          880 :         *flush_loop_state = FlushLoopState::Running {
    2554          880 :             #[cfg(test)]
    2555          880 :             expect_initdb_optimization: false,
    2556          880 :             #[cfg(test)]
    2557          880 :             initdb_optimization_count: 0,
    2558          880 :         };
    2559          880 :         task_mgr::spawn(
    2560          880 :             task_mgr::BACKGROUND_RUNTIME.handle(),
    2561          880 :             task_mgr::TaskKind::LayerFlushTask,
    2562          880 :             self.tenant_shard_id,
    2563          880 :             Some(self.timeline_id),
    2564          880 :             "layer flush task",
    2565          880 :             async move {
    2566          880 :                 let _guard = guard;
    2567          880 :                 let background_ctx = RequestContext::todo_child(TaskKind::LayerFlushTask, DownloadBehavior::Error);
    2568          880 :                 self_clone.flush_loop(layer_flush_start_rx, &background_ctx).await;
    2569           20 :                 let mut flush_loop_state = self_clone.flush_loop_state.lock().unwrap();
    2570           20 :                 assert!(matches!(*flush_loop_state, FlushLoopState::Running{..}));
    2571           20 :                 *flush_loop_state  = FlushLoopState::Exited;
    2572           20 :                 Ok(())
    2573           20 :             }
    2574          880 :             .instrument(info_span!(parent: None, "layer flush task", tenant_id = %self.tenant_shard_id.tenant_id, shard_id = %self.tenant_shard_id.shard_slug(), timeline_id = %self.timeline_id))
    2575              :         );
    2576         1288 :     }
    2577              : 
    2578              :     /// Creates and starts the wal receiver.
    2579              :     ///
    2580              :     /// This function is expected to be called at most once per Timeline's lifecycle
    2581              :     /// when the timeline is activated.
    2582            0 :     fn launch_wal_receiver(
    2583            0 :         self: &Arc<Self>,
    2584            0 :         ctx: &RequestContext,
    2585            0 :         broker_client: BrokerClientChannel,
    2586            0 :     ) {
    2587            0 :         info!(
    2588            0 :             "launching WAL receiver for timeline {} of tenant {}",
    2589            0 :             self.timeline_id, self.tenant_shard_id
    2590              :         );
    2591              : 
    2592            0 :         let tenant_conf = self.tenant_conf.load();
    2593            0 :         let wal_connect_timeout = tenant_conf
    2594            0 :             .tenant_conf
    2595            0 :             .walreceiver_connect_timeout
    2596            0 :             .unwrap_or(self.conf.default_tenant_conf.walreceiver_connect_timeout);
    2597            0 :         let lagging_wal_timeout = tenant_conf
    2598            0 :             .tenant_conf
    2599            0 :             .lagging_wal_timeout
    2600            0 :             .unwrap_or(self.conf.default_tenant_conf.lagging_wal_timeout);
    2601            0 :         let max_lsn_wal_lag = tenant_conf
    2602            0 :             .tenant_conf
    2603            0 :             .max_lsn_wal_lag
    2604            0 :             .unwrap_or(self.conf.default_tenant_conf.max_lsn_wal_lag);
    2605            0 : 
    2606            0 :         let mut guard = self.walreceiver.lock().unwrap();
    2607            0 :         assert!(
    2608            0 :             guard.is_none(),
    2609            0 :             "multiple launches / re-launches of WAL receiver are not supported"
    2610              :         );
    2611            0 :         *guard = Some(WalReceiver::start(
    2612            0 :             Arc::clone(self),
    2613            0 :             WalReceiverConf {
    2614            0 :                 protocol: self.resolve_wal_receiver_protocol(),
    2615            0 :                 wal_connect_timeout,
    2616            0 :                 lagging_wal_timeout,
    2617            0 :                 max_lsn_wal_lag,
    2618            0 :                 auth_token: crate::config::SAFEKEEPER_AUTH_TOKEN.get().cloned(),
    2619            0 :                 availability_zone: self.conf.availability_zone.clone(),
    2620            0 :                 ingest_batch_size: self.conf.ingest_batch_size,
    2621            0 :             },
    2622            0 :             broker_client,
    2623            0 :             ctx,
    2624            0 :         ));
    2625            0 :     }
    2626              : 
    2627              :     /// Initialize with an empty layer map. Used when creating a new timeline.
    2628          880 :     pub(super) fn init_empty_layer_map(&self, start_lsn: Lsn) {
    2629          880 :         let mut layers = self.layers.try_write().expect(
    2630          880 :             "in the context where we call this function, no other task has access to the object",
    2631          880 :         );
    2632          880 :         layers
    2633          880 :             .open_mut()
    2634          880 :             .expect("in this context the LayerManager must still be open")
    2635          880 :             .initialize_empty(Lsn(start_lsn.0));
    2636          880 :     }
    2637              : 
    2638              :     /// Scan the timeline directory, cleanup, populate the layer map, and schedule uploads for local-only
    2639              :     /// files.
    2640           12 :     pub(super) async fn load_layer_map(
    2641           12 :         &self,
    2642           12 :         disk_consistent_lsn: Lsn,
    2643           12 :         index_part: IndexPart,
    2644           12 :     ) -> anyhow::Result<()> {
    2645              :         use init::{Decision::*, Discovered, DismissedLayer};
    2646              :         use LayerName::*;
    2647              : 
    2648           12 :         let mut guard = self.layers.write().await;
    2649              : 
    2650           12 :         let timer = self.metrics.load_layer_map_histo.start_timer();
    2651           12 : 
    2652           12 :         // Scan timeline directory and create ImageLayerName and DeltaFilename
    2653           12 :         // structs representing all files on disk
    2654           12 :         let timeline_path = self
    2655           12 :             .conf
    2656           12 :             .timeline_path(&self.tenant_shard_id, &self.timeline_id);
    2657           12 :         let conf = self.conf;
    2658           12 :         let span = tracing::Span::current();
    2659           12 : 
    2660           12 :         // Copy to move into the task we're about to spawn
    2661           12 :         let this = self.myself.upgrade().expect("&self method holds the arc");
    2662              : 
    2663           12 :         let (loaded_layers, needs_cleanup, total_physical_size) = tokio::task::spawn_blocking({
    2664           12 :             move || {
    2665           12 :                 let _g = span.entered();
    2666           12 :                 let discovered = init::scan_timeline_dir(&timeline_path)?;
    2667           12 :                 let mut discovered_layers = Vec::with_capacity(discovered.len());
    2668           12 :                 let mut unrecognized_files = Vec::new();
    2669           12 : 
    2670           12 :                 let mut path = timeline_path;
    2671              : 
    2672           44 :                 for discovered in discovered {
    2673           32 :                     let (name, kind) = match discovered {
    2674           32 :                         Discovered::Layer(layer_file_name, local_metadata) => {
    2675           32 :                             discovered_layers.push((layer_file_name, local_metadata));
    2676           32 :                             continue;
    2677              :                         }
    2678            0 :                         Discovered::IgnoredBackup(path) => {
    2679            0 :                             std::fs::remove_file(path)
    2680            0 :                                 .or_else(fs_ext::ignore_not_found)
    2681            0 :                                 .fatal_err("Removing .old file");
    2682            0 :                             continue;
    2683              :                         }
    2684            0 :                         Discovered::Unknown(file_name) => {
    2685            0 :                             // we will later error if there are any
    2686            0 :                             unrecognized_files.push(file_name);
    2687            0 :                             continue;
    2688              :                         }
    2689            0 :                         Discovered::Ephemeral(name) => (name, "old ephemeral file"),
    2690            0 :                         Discovered::Temporary(name) => (name, "temporary timeline file"),
    2691            0 :                         Discovered::TemporaryDownload(name) => (name, "temporary download"),
    2692              :                     };
    2693            0 :                     path.push(Utf8Path::new(&name));
    2694            0 :                     init::cleanup(&path, kind)?;
    2695            0 :                     path.pop();
    2696              :                 }
    2697              : 
    2698           12 :                 if !unrecognized_files.is_empty() {
    2699              :                     // assume that if there are any there are many many.
    2700            0 :                     let n = unrecognized_files.len();
    2701            0 :                     let first = &unrecognized_files[..n.min(10)];
    2702            0 :                     anyhow::bail!(
    2703            0 :                         "unrecognized files in timeline dir (total {n}), first 10: {first:?}"
    2704            0 :                     );
    2705           12 :                 }
    2706           12 : 
    2707           12 :                 let decided = init::reconcile(discovered_layers, &index_part, disk_consistent_lsn);
    2708           12 : 
    2709           12 :                 let mut loaded_layers = Vec::new();
    2710           12 :                 let mut needs_cleanup = Vec::new();
    2711           12 :                 let mut total_physical_size = 0;
    2712              : 
    2713           44 :                 for (name, decision) in decided {
    2714           32 :                     let decision = match decision {
    2715           32 :                         Ok(decision) => decision,
    2716            0 :                         Err(DismissedLayer::Future { local }) => {
    2717            0 :                             if let Some(local) = local {
    2718            0 :                                 init::cleanup_future_layer(
    2719            0 :                                     &local.local_path,
    2720            0 :                                     &name,
    2721            0 :                                     disk_consistent_lsn,
    2722            0 :                                 )?;
    2723            0 :                             }
    2724            0 :                             needs_cleanup.push(name);
    2725            0 :                             continue;
    2726              :                         }
    2727            0 :                         Err(DismissedLayer::LocalOnly(local)) => {
    2728            0 :                             init::cleanup_local_only_file(&name, &local)?;
    2729              :                             // this file never existed remotely, we will have to do rework
    2730            0 :                             continue;
    2731              :                         }
    2732            0 :                         Err(DismissedLayer::BadMetadata(local)) => {
    2733            0 :                             init::cleanup_local_file_for_remote(&local)?;
    2734              :                             // this file never existed remotely, we will have to do rework
    2735            0 :                             continue;
    2736              :                         }
    2737              :                     };
    2738              : 
    2739           32 :                     match &name {
    2740           24 :                         Delta(d) => assert!(d.lsn_range.end <= disk_consistent_lsn + 1),
    2741            8 :                         Image(i) => assert!(i.lsn <= disk_consistent_lsn),
    2742              :                     }
    2743              : 
    2744           32 :                     tracing::debug!(layer=%name, ?decision, "applied");
    2745              : 
    2746           32 :                     let layer = match decision {
    2747           32 :                         Resident { local, remote } => {
    2748           32 :                             total_physical_size += local.file_size;
    2749           32 :                             Layer::for_resident(conf, &this, local.local_path, name, remote)
    2750           32 :                                 .drop_eviction_guard()
    2751              :                         }
    2752            0 :                         Evicted(remote) => Layer::for_evicted(conf, &this, name, remote),
    2753              :                     };
    2754              : 
    2755           32 :                     loaded_layers.push(layer);
    2756              :                 }
    2757           12 :                 Ok((loaded_layers, needs_cleanup, total_physical_size))
    2758           12 :             }
    2759           12 :         })
    2760           12 :         .await
    2761           12 :         .map_err(anyhow::Error::new)
    2762           12 :         .and_then(|x| x)?;
    2763              : 
    2764           12 :         let num_layers = loaded_layers.len();
    2765           12 : 
    2766           12 :         guard
    2767           12 :             .open_mut()
    2768           12 :             .expect("layermanager must be open during init")
    2769           12 :             .initialize_local_layers(loaded_layers, disk_consistent_lsn + 1);
    2770           12 : 
    2771           12 :         self.remote_client
    2772           12 :             .schedule_layer_file_deletion(&needs_cleanup)?;
    2773           12 :         self.remote_client
    2774           12 :             .schedule_index_upload_for_file_changes()?;
    2775              :         // This barrier orders above DELETEs before any later operations.
    2776              :         // This is critical because code executing after the barrier might
    2777              :         // create again objects with the same key that we just scheduled for deletion.
    2778              :         // For example, if we just scheduled deletion of an image layer "from the future",
    2779              :         // later compaction might run again and re-create the same image layer.
    2780              :         // "from the future" here means an image layer whose LSN is > IndexPart::disk_consistent_lsn.
    2781              :         // "same" here means same key range and LSN.
    2782              :         //
    2783              :         // Without a barrier between above DELETEs and the re-creation's PUTs,
    2784              :         // the upload queue may execute the PUT first, then the DELETE.
    2785              :         // In our example, we will end up with an IndexPart referencing a non-existent object.
    2786              :         //
    2787              :         // 1. a future image layer is created and uploaded
    2788              :         // 2. ps restart
    2789              :         // 3. the future layer from (1) is deleted during load layer map
    2790              :         // 4. image layer is re-created and uploaded
    2791              :         // 5. deletion queue would like to delete (1) but actually deletes (4)
    2792              :         // 6. delete by name works as expected, but it now deletes the wrong (later) version
    2793              :         //
    2794              :         // See https://github.com/neondatabase/neon/issues/5878
    2795              :         //
    2796              :         // NB: generation numbers naturally protect against this because they disambiguate
    2797              :         //     (1) and (4)
    2798              :         // TODO: this is basically a no-op now, should we remove it?
    2799           12 :         self.remote_client.schedule_barrier()?;
    2800              :         // Tenant::create_timeline will wait for these uploads to happen before returning, or
    2801              :         // on retry.
    2802              : 
    2803              :         // Now that we have the full layer map, we may calculate the visibility of layers within it (a global scan)
    2804           12 :         drop(guard); // drop write lock, update_layer_visibility will take a read lock.
    2805           12 :         self.update_layer_visibility().await?;
    2806              : 
    2807           12 :         info!(
    2808            0 :             "loaded layer map with {} layers at {}, total physical size: {}",
    2809              :             num_layers, disk_consistent_lsn, total_physical_size
    2810              :         );
    2811              : 
    2812           12 :         timer.stop_and_record();
    2813           12 :         Ok(())
    2814           12 :     }
    2815              : 
    2816              :     /// Retrieve current logical size of the timeline.
    2817              :     ///
    2818              :     /// The size could be lagging behind the actual number, in case
    2819              :     /// the initial size calculation has not been run (gets triggered on the first size access).
    2820              :     ///
    2821              :     /// return size and boolean flag that shows if the size is exact
    2822            0 :     pub(crate) fn get_current_logical_size(
    2823            0 :         self: &Arc<Self>,
    2824            0 :         priority: GetLogicalSizePriority,
    2825            0 :         ctx: &RequestContext,
    2826            0 :     ) -> logical_size::CurrentLogicalSize {
    2827            0 :         if !self.tenant_shard_id.is_shard_zero() {
    2828              :             // Logical size is only accurately maintained on shard zero: when called elsewhere, for example
    2829              :             // when HTTP API is serving a GET for timeline zero, return zero
    2830            0 :             return logical_size::CurrentLogicalSize::Approximate(logical_size::Approximate::zero());
    2831            0 :         }
    2832            0 : 
    2833            0 :         let current_size = self.current_logical_size.current_size();
    2834            0 :         debug!("Current size: {current_size:?}");
    2835              : 
    2836            0 :         match (current_size.accuracy(), priority) {
    2837            0 :             (logical_size::Accuracy::Exact, _) => (), // nothing to do
    2838            0 :             (logical_size::Accuracy::Approximate, GetLogicalSizePriority::Background) => {
    2839            0 :                 // background task will eventually deliver an exact value, we're in no rush
    2840            0 :             }
    2841              :             (logical_size::Accuracy::Approximate, GetLogicalSizePriority::User) => {
    2842              :                 // background task is not ready, but user is asking for it now;
    2843              :                 // => make the background task skip the line
    2844              :                 // (The alternative would be to calculate the size here, but,
    2845              :                 //  it can actually take a long time if the user has a lot of rels.
    2846              :                 //  And we'll inevitable need it again; So, let the background task do the work.)
    2847            0 :                 match self
    2848            0 :                     .current_logical_size
    2849            0 :                     .cancel_wait_for_background_loop_concurrency_limit_semaphore
    2850            0 :                     .get()
    2851              :                 {
    2852            0 :                     Some(cancel) => cancel.cancel(),
    2853              :                     None => {
    2854            0 :                         match self.current_state() {
    2855            0 :                             TimelineState::Broken { .. } | TimelineState::Stopping => {
    2856            0 :                                 // Can happen when timeline detail endpoint is used when deletion is ongoing (or its broken).
    2857            0 :                                 // Don't make noise.
    2858            0 :                             }
    2859              :                             TimelineState::Loading => {
    2860              :                                 // Import does not return an activated timeline.
    2861            0 :                                 info!("discarding priority boost for logical size calculation because timeline is not yet active");
    2862              :                             }
    2863              :                             TimelineState::Active => {
    2864              :                                 // activation should be setting the once cell
    2865            0 :                                 warn!("unexpected: cancel_wait_for_background_loop_concurrency_limit_semaphore not set, priority-boosting of logical size calculation will not work");
    2866            0 :                                 debug_assert!(false);
    2867              :                             }
    2868              :                         }
    2869              :                     }
    2870              :                 }
    2871              :             }
    2872              :         }
    2873              : 
    2874            0 :         if let CurrentLogicalSize::Approximate(_) = &current_size {
    2875            0 :             if ctx.task_kind() == TaskKind::WalReceiverConnectionHandler {
    2876            0 :                 let first = self
    2877            0 :                     .current_logical_size
    2878            0 :                     .did_return_approximate_to_walreceiver
    2879            0 :                     .compare_exchange(
    2880            0 :                         false,
    2881            0 :                         true,
    2882            0 :                         AtomicOrdering::Relaxed,
    2883            0 :                         AtomicOrdering::Relaxed,
    2884            0 :                     )
    2885            0 :                     .is_ok();
    2886            0 :                 if first {
    2887            0 :                     crate::metrics::initial_logical_size::TIMELINES_WHERE_WALRECEIVER_GOT_APPROXIMATE_SIZE.inc();
    2888            0 :                 }
    2889            0 :             }
    2890            0 :         }
    2891              : 
    2892            0 :         current_size
    2893            0 :     }
    2894              : 
    2895            0 :     fn spawn_initial_logical_size_computation_task(self: &Arc<Self>, ctx: &RequestContext) {
    2896            0 :         let Some(initial_part_end) = self.current_logical_size.initial_part_end else {
    2897              :             // nothing to do for freshly created timelines;
    2898            0 :             assert_eq!(
    2899            0 :                 self.current_logical_size.current_size().accuracy(),
    2900            0 :                 logical_size::Accuracy::Exact,
    2901            0 :             );
    2902            0 :             self.current_logical_size.initialized.add_permits(1);
    2903            0 :             return;
    2904              :         };
    2905              : 
    2906            0 :         let cancel_wait_for_background_loop_concurrency_limit_semaphore = CancellationToken::new();
    2907            0 :         let token = cancel_wait_for_background_loop_concurrency_limit_semaphore.clone();
    2908            0 :         self.current_logical_size
    2909            0 :             .cancel_wait_for_background_loop_concurrency_limit_semaphore.set(token)
    2910            0 :             .expect("initial logical size calculation task must be spawned exactly once per Timeline object");
    2911            0 : 
    2912            0 :         let self_clone = Arc::clone(self);
    2913            0 :         let background_ctx = ctx.detached_child(
    2914            0 :             TaskKind::InitialLogicalSizeCalculation,
    2915            0 :             DownloadBehavior::Download,
    2916            0 :         );
    2917            0 :         task_mgr::spawn(
    2918            0 :             task_mgr::BACKGROUND_RUNTIME.handle(),
    2919            0 :             task_mgr::TaskKind::InitialLogicalSizeCalculation,
    2920            0 :             self.tenant_shard_id,
    2921            0 :             Some(self.timeline_id),
    2922            0 :             "initial size calculation",
    2923              :             // NB: don't log errors here, task_mgr will do that.
    2924            0 :             async move {
    2925            0 :                 self_clone
    2926            0 :                     .initial_logical_size_calculation_task(
    2927            0 :                         initial_part_end,
    2928            0 :                         cancel_wait_for_background_loop_concurrency_limit_semaphore,
    2929            0 :                         background_ctx,
    2930            0 :                     )
    2931            0 :                     .await;
    2932            0 :                 Ok(())
    2933            0 :             }
    2934            0 :             .instrument(info_span!(parent: None, "initial_size_calculation", tenant_id=%self.tenant_shard_id.tenant_id, shard_id=%self.tenant_shard_id.shard_slug(), timeline_id=%self.timeline_id)),
    2935              :         );
    2936            0 :     }
    2937              : 
    2938              :     /// # Cancellation
    2939              :     ///
    2940              :     /// This method is sensitive to `Timeline::cancel`.
    2941              :     ///
    2942              :     /// It is _not_ sensitive to task_mgr::shutdown_token().
    2943              :     ///
    2944              :     /// # Cancel-Safety
    2945              :     ///
    2946              :     /// It does Timeline IO, hence this should be polled to completion because
    2947              :     /// we could be leaving in-flight IOs behind, which is safe, but annoying
    2948              :     /// to reason about.
    2949            0 :     async fn initial_logical_size_calculation_task(
    2950            0 :         self: Arc<Self>,
    2951            0 :         initial_part_end: Lsn,
    2952            0 :         skip_concurrency_limiter: CancellationToken,
    2953            0 :         background_ctx: RequestContext,
    2954            0 :     ) {
    2955            0 :         scopeguard::defer! {
    2956            0 :             // Irrespective of the outcome of this operation, we should unblock anyone waiting for it.
    2957            0 :             self.current_logical_size.initialized.add_permits(1);
    2958            0 :         }
    2959            0 : 
    2960            0 :         let try_once = |attempt: usize| {
    2961            0 :             let background_ctx = &background_ctx;
    2962            0 :             let self_ref = &self;
    2963            0 :             let skip_concurrency_limiter = &skip_concurrency_limiter;
    2964            0 :             async move {
    2965            0 :                 let wait_for_permit = super::tasks::concurrent_background_tasks_rate_limit_permit(
    2966            0 :                     BackgroundLoopKind::InitialLogicalSizeCalculation,
    2967            0 :                     background_ctx,
    2968            0 :                 );
    2969              : 
    2970              :                 use crate::metrics::initial_logical_size::StartCircumstances;
    2971            0 :                 let (_maybe_permit, circumstances) = tokio::select! {
    2972            0 :                     permit = wait_for_permit => {
    2973            0 :                         (Some(permit), StartCircumstances::AfterBackgroundTasksRateLimit)
    2974              :                     }
    2975            0 :                     _ = self_ref.cancel.cancelled() => {
    2976            0 :                         return Err(CalculateLogicalSizeError::Cancelled);
    2977              :                     }
    2978            0 :                     () = skip_concurrency_limiter.cancelled() => {
    2979              :                         // Some action that is part of a end user interaction requested logical size
    2980              :                         // => break out of the rate limit
    2981              :                         // TODO: ideally we'd not run on BackgroundRuntime but the requester's runtime;
    2982              :                         // but then again what happens if they cancel; also, we should just be using
    2983              :                         // one runtime across the entire process, so, let's leave this for now.
    2984            0 :                         (None, StartCircumstances::SkippedConcurrencyLimiter)
    2985              :                     }
    2986              :                 };
    2987              : 
    2988            0 :                 let metrics_guard = if attempt == 1 {
    2989            0 :                     crate::metrics::initial_logical_size::START_CALCULATION.first(circumstances)
    2990              :                 } else {
    2991            0 :                     crate::metrics::initial_logical_size::START_CALCULATION.retry(circumstances)
    2992              :                 };
    2993              : 
    2994            0 :                 let io_concurrency = IoConcurrency::spawn_from_conf(
    2995            0 :                     self_ref.conf,
    2996            0 :                     self_ref
    2997            0 :                         .gate
    2998            0 :                         .enter()
    2999            0 :                         .map_err(|_| CalculateLogicalSizeError::Cancelled)?,
    3000              :                 );
    3001              : 
    3002            0 :                 let calculated_size = self_ref
    3003            0 :                     .logical_size_calculation_task(
    3004            0 :                         initial_part_end,
    3005            0 :                         LogicalSizeCalculationCause::Initial,
    3006            0 :                         background_ctx,
    3007            0 :                     )
    3008            0 :                     .await?;
    3009              : 
    3010            0 :                 self_ref
    3011            0 :                     .trigger_aux_file_size_computation(
    3012            0 :                         initial_part_end,
    3013            0 :                         background_ctx,
    3014            0 :                         io_concurrency,
    3015            0 :                     )
    3016            0 :                     .await?;
    3017              : 
    3018              :                 // TODO: add aux file size to logical size
    3019              : 
    3020            0 :                 Ok((calculated_size, metrics_guard))
    3021            0 :             }
    3022            0 :         };
    3023              : 
    3024            0 :         let retrying = async {
    3025            0 :             let mut attempt = 0;
    3026              :             loop {
    3027            0 :                 attempt += 1;
    3028            0 : 
    3029            0 :                 match try_once(attempt).await {
    3030            0 :                     Ok(res) => return ControlFlow::Continue(res),
    3031            0 :                     Err(CalculateLogicalSizeError::Cancelled) => return ControlFlow::Break(()),
    3032              :                     Err(
    3033            0 :                         e @ (CalculateLogicalSizeError::Decode(_)
    3034            0 :                         | CalculateLogicalSizeError::PageRead(_)),
    3035            0 :                     ) => {
    3036            0 :                         warn!(attempt, "initial size calculation failed: {e:?}");
    3037              :                         // exponential back-off doesn't make sense at these long intervals;
    3038              :                         // use fixed retry interval with generous jitter instead
    3039            0 :                         let sleep_duration = Duration::from_secs(
    3040            0 :                             u64::try_from(
    3041            0 :                                 // 1hour base
    3042            0 :                                 (60_i64 * 60_i64)
    3043            0 :                                     // 10min jitter
    3044            0 :                                     + rand::thread_rng().gen_range(-10 * 60..10 * 60),
    3045            0 :                             )
    3046            0 :                             .expect("10min < 1hour"),
    3047            0 :                         );
    3048            0 :                         tokio::select! {
    3049            0 :                             _ = tokio::time::sleep(sleep_duration) => {}
    3050            0 :                             _ = self.cancel.cancelled() => return ControlFlow::Break(()),
    3051              :                         }
    3052              :                     }
    3053              :                 }
    3054              :             }
    3055            0 :         };
    3056              : 
    3057            0 :         let (calculated_size, metrics_guard) = match retrying.await {
    3058            0 :             ControlFlow::Continue(calculated_size) => calculated_size,
    3059            0 :             ControlFlow::Break(()) => return,
    3060              :         };
    3061              : 
    3062              :         // we cannot query current_logical_size.current_size() to know the current
    3063              :         // *negative* value, only truncated to u64.
    3064            0 :         let added = self
    3065            0 :             .current_logical_size
    3066            0 :             .size_added_after_initial
    3067            0 :             .load(AtomicOrdering::Relaxed);
    3068            0 : 
    3069            0 :         let sum = calculated_size.saturating_add_signed(added);
    3070            0 : 
    3071            0 :         // set the gauge value before it can be set in `update_current_logical_size`.
    3072            0 :         self.metrics.current_logical_size_gauge.set(sum);
    3073            0 : 
    3074            0 :         self.current_logical_size
    3075            0 :             .initial_logical_size
    3076            0 :             .set((calculated_size, metrics_guard.calculation_result_saved()))
    3077            0 :             .ok()
    3078            0 :             .expect("only this task sets it");
    3079            0 :     }
    3080              : 
    3081            0 :     pub(crate) fn spawn_ondemand_logical_size_calculation(
    3082            0 :         self: &Arc<Self>,
    3083            0 :         lsn: Lsn,
    3084            0 :         cause: LogicalSizeCalculationCause,
    3085            0 :         ctx: RequestContext,
    3086            0 :     ) -> oneshot::Receiver<Result<u64, CalculateLogicalSizeError>> {
    3087            0 :         let (sender, receiver) = oneshot::channel();
    3088            0 :         let self_clone = Arc::clone(self);
    3089            0 :         // XXX if our caller loses interest, i.e., ctx is cancelled,
    3090            0 :         // we should stop the size calculation work and return an error.
    3091            0 :         // That would require restructuring this function's API to
    3092            0 :         // return the result directly, instead of a Receiver for the result.
    3093            0 :         let ctx = ctx.detached_child(
    3094            0 :             TaskKind::OndemandLogicalSizeCalculation,
    3095            0 :             DownloadBehavior::Download,
    3096            0 :         );
    3097            0 :         task_mgr::spawn(
    3098            0 :             task_mgr::BACKGROUND_RUNTIME.handle(),
    3099            0 :             task_mgr::TaskKind::OndemandLogicalSizeCalculation,
    3100            0 :             self.tenant_shard_id,
    3101            0 :             Some(self.timeline_id),
    3102            0 :             "ondemand logical size calculation",
    3103            0 :             async move {
    3104            0 :                 let res = self_clone
    3105            0 :                     .logical_size_calculation_task(lsn, cause, &ctx)
    3106            0 :                     .await;
    3107            0 :                 let _ = sender.send(res).ok();
    3108            0 :                 Ok(()) // Receiver is responsible for handling errors
    3109            0 :             }
    3110            0 :             .in_current_span(),
    3111            0 :         );
    3112            0 :         receiver
    3113            0 :     }
    3114              : 
    3115              :     #[instrument(skip_all)]
    3116              :     async fn logical_size_calculation_task(
    3117              :         self: &Arc<Self>,
    3118              :         lsn: Lsn,
    3119              :         cause: LogicalSizeCalculationCause,
    3120              :         ctx: &RequestContext,
    3121              :     ) -> Result<u64, CalculateLogicalSizeError> {
    3122              :         crate::span::debug_assert_current_span_has_tenant_and_timeline_id();
    3123              :         // We should never be calculating logical sizes on shard !=0, because these shards do not have
    3124              :         // accurate relation sizes, and they do not emit consumption metrics.
    3125              :         debug_assert!(self.tenant_shard_id.is_shard_zero());
    3126              : 
    3127              :         let guard = self
    3128              :             .gate
    3129              :             .enter()
    3130            0 :             .map_err(|_| CalculateLogicalSizeError::Cancelled)?;
    3131              : 
    3132              :         self.calculate_logical_size(lsn, cause, &guard, ctx).await
    3133              :     }
    3134              : 
    3135              :     /// Calculate the logical size of the database at the latest LSN.
    3136              :     ///
    3137              :     /// NOTE: counted incrementally, includes ancestors. This can be a slow operation,
    3138              :     /// especially if we need to download remote layers.
    3139            0 :     async fn calculate_logical_size(
    3140            0 :         &self,
    3141            0 :         up_to_lsn: Lsn,
    3142            0 :         cause: LogicalSizeCalculationCause,
    3143            0 :         _guard: &GateGuard,
    3144            0 :         ctx: &RequestContext,
    3145            0 :     ) -> Result<u64, CalculateLogicalSizeError> {
    3146            0 :         info!(
    3147            0 :             "Calculating logical size for timeline {} at {}",
    3148              :             self.timeline_id, up_to_lsn
    3149              :         );
    3150              : 
    3151            0 :         if let Err(()) = pausable_failpoint!("timeline-calculate-logical-size-pause", &self.cancel)
    3152              :         {
    3153            0 :             return Err(CalculateLogicalSizeError::Cancelled);
    3154            0 :         }
    3155              : 
    3156              :         // See if we've already done the work for initial size calculation.
    3157              :         // This is a short-cut for timelines that are mostly unused.
    3158            0 :         if let Some(size) = self.current_logical_size.initialized_size(up_to_lsn) {
    3159            0 :             return Ok(size);
    3160            0 :         }
    3161            0 :         let storage_time_metrics = match cause {
    3162              :             LogicalSizeCalculationCause::Initial
    3163              :             | LogicalSizeCalculationCause::ConsumptionMetricsSyntheticSize
    3164            0 :             | LogicalSizeCalculationCause::TenantSizeHandler => &self.metrics.logical_size_histo,
    3165              :             LogicalSizeCalculationCause::EvictionTaskImitation => {
    3166            0 :                 &self.metrics.imitate_logical_size_histo
    3167              :             }
    3168              :         };
    3169            0 :         let timer = storage_time_metrics.start_timer();
    3170            0 :         let logical_size = self
    3171            0 :             .get_current_logical_size_non_incremental(up_to_lsn, ctx)
    3172            0 :             .await?;
    3173            0 :         debug!("calculated logical size: {logical_size}");
    3174            0 :         timer.stop_and_record();
    3175            0 :         Ok(logical_size)
    3176            0 :     }
    3177              : 
    3178              :     /// Update current logical size, adding `delta' to the old value.
    3179       541140 :     fn update_current_logical_size(&self, delta: i64) {
    3180       541140 :         let logical_size = &self.current_logical_size;
    3181       541140 :         logical_size.increment_size(delta);
    3182       541140 : 
    3183       541140 :         // Also set the value in the prometheus gauge. Note that
    3184       541140 :         // there is a race condition here: if this is is called by two
    3185       541140 :         // threads concurrently, the prometheus gauge might be set to
    3186       541140 :         // one value while current_logical_size is set to the
    3187       541140 :         // other.
    3188       541140 :         match logical_size.current_size() {
    3189       541140 :             CurrentLogicalSize::Exact(ref new_current_size) => self
    3190       541140 :                 .metrics
    3191       541140 :                 .current_logical_size_gauge
    3192       541140 :                 .set(new_current_size.into()),
    3193            0 :             CurrentLogicalSize::Approximate(_) => {
    3194            0 :                 // don't update the gauge yet, this allows us not to update the gauge back and
    3195            0 :                 // forth between the initial size calculation task.
    3196            0 :             }
    3197              :         }
    3198       541140 :     }
    3199              : 
    3200         5912 :     pub(crate) fn update_directory_entries_count(&self, kind: DirectoryKind, count: u64) {
    3201         5912 :         self.directory_metrics[kind.offset()].store(count, AtomicOrdering::Relaxed);
    3202         5912 :         let aux_metric =
    3203         5912 :             self.directory_metrics[DirectoryKind::AuxFiles.offset()].load(AtomicOrdering::Relaxed);
    3204         5912 : 
    3205         5912 :         let sum_of_entries = self
    3206         5912 :             .directory_metrics
    3207         5912 :             .iter()
    3208        41384 :             .map(|v| v.load(AtomicOrdering::Relaxed))
    3209         5912 :             .sum();
    3210              :         // Set a high general threshold and a lower threshold for the auxiliary files,
    3211              :         // as we can have large numbers of relations in the db directory.
    3212              :         const SUM_THRESHOLD: u64 = 5000;
    3213              :         const AUX_THRESHOLD: u64 = 1000;
    3214         5912 :         if sum_of_entries >= SUM_THRESHOLD || aux_metric >= AUX_THRESHOLD {
    3215            0 :             self.metrics
    3216            0 :                 .directory_entries_count_gauge
    3217            0 :                 .set(sum_of_entries);
    3218         5912 :         } else if let Some(metric) = Lazy::get(&self.metrics.directory_entries_count_gauge) {
    3219            0 :             metric.set(sum_of_entries);
    3220         5912 :         }
    3221         5912 :     }
    3222              : 
    3223            0 :     async fn find_layer(
    3224            0 :         &self,
    3225            0 :         layer_name: &LayerName,
    3226            0 :     ) -> Result<Option<Layer>, layer_manager::Shutdown> {
    3227            0 :         let guard = self.layers.read().await;
    3228            0 :         let layer = guard
    3229            0 :             .layer_map()?
    3230            0 :             .iter_historic_layers()
    3231            0 :             .find(|l| &l.layer_name() == layer_name)
    3232            0 :             .map(|found| guard.get_from_desc(&found));
    3233            0 :         Ok(layer)
    3234            0 :     }
    3235              : 
    3236              :     /// The timeline heatmap is a hint to secondary locations from the primary location,
    3237              :     /// indicating which layers are currently on-disk on the primary.
    3238              :     ///
    3239              :     /// None is returned if the Timeline is in a state where uploading a heatmap
    3240              :     /// doesn't make sense, such as shutting down or initializing.  The caller
    3241              :     /// should treat this as a cue to simply skip doing any heatmap uploading
    3242              :     /// for this timeline.
    3243            4 :     pub(crate) async fn generate_heatmap(&self) -> Option<HeatMapTimeline> {
    3244            4 :         if !self.is_active() {
    3245            0 :             return None;
    3246            4 :         }
    3247              : 
    3248            4 :         let guard = self.layers.read().await;
    3249              : 
    3250           20 :         let resident = guard.likely_resident_layers().filter_map(|layer| {
    3251           20 :             match layer.visibility() {
    3252              :                 LayerVisibilityHint::Visible => {
    3253              :                     // Layer is visible to one or more read LSNs: elegible for inclusion in layer map
    3254           16 :                     let last_activity_ts = layer.latest_activity();
    3255           16 :                     Some((layer.layer_desc(), layer.metadata(), last_activity_ts))
    3256              :                 }
    3257              :                 LayerVisibilityHint::Covered => {
    3258              :                     // Layer is resident but unlikely to be read: not elegible for inclusion in heatmap.
    3259            4 :                     None
    3260              :                 }
    3261              :             }
    3262           20 :         });
    3263            4 : 
    3264            4 :         let mut layers = resident.collect::<Vec<_>>();
    3265            4 : 
    3266            4 :         // Sort layers in order of which to download first.  For a large set of layers to download, we
    3267            4 :         // want to prioritize those layers which are most likely to still be in the resident many minutes
    3268            4 :         // or hours later:
    3269            4 :         // - Download L0s last, because they churn the fastest: L0s on a fast-writing tenant might
    3270            4 :         //   only exist for a few minutes before being compacted into L1s.
    3271            4 :         // - For L1 & image layers, download most recent LSNs first: the older the LSN, the sooner
    3272            4 :         //   the layer is likely to be covered by an image layer during compaction.
    3273           42 :         layers.sort_by_key(|(desc, _meta, _atime)| {
    3274           42 :             std::cmp::Reverse((
    3275           42 :                 !LayerMap::is_l0(&desc.key_range, desc.is_delta),
    3276           42 :                 desc.lsn_range.end,
    3277           42 :             ))
    3278           42 :         });
    3279            4 : 
    3280            4 :         let layers = layers
    3281            4 :             .into_iter()
    3282           16 :             .map(|(desc, meta, atime)| HeatMapLayer::new(desc.layer_name(), meta, atime))
    3283            4 :             .collect();
    3284            4 : 
    3285            4 :         Some(HeatMapTimeline::new(self.timeline_id, layers))
    3286            4 :     }
    3287              : 
    3288              :     /// Returns true if the given lsn is or was an ancestor branchpoint.
    3289            0 :     pub(crate) fn is_ancestor_lsn(&self, lsn: Lsn) -> bool {
    3290            0 :         // upon timeline detach, we set the ancestor_lsn to Lsn::INVALID and the store the original
    3291            0 :         // branchpoint in the value in IndexPart::lineage
    3292            0 :         self.ancestor_lsn == lsn
    3293            0 :             || (self.ancestor_lsn == Lsn::INVALID
    3294            0 :                 && self.remote_client.is_previous_ancestor_lsn(lsn))
    3295            0 :     }
    3296              : }
    3297              : 
    3298              : impl Timeline {
    3299              :     #[allow(clippy::doc_lazy_continuation)]
    3300              :     /// Get the data needed to reconstruct all keys in the provided keyspace
    3301              :     ///
    3302              :     /// The algorithm is as follows:
    3303              :     /// 1.   While some keys are still not done and there's a timeline to visit:
    3304              :     /// 2.   Visit the timeline (see [`Timeline::get_vectored_reconstruct_data_timeline`]:
    3305              :     /// 2.1: Build the fringe for the current keyspace
    3306              :     /// 2.2  Visit the newest layer from the fringe to collect all values for the range it
    3307              :     ///      intersects
    3308              :     /// 2.3. Pop the timeline from the fringe
    3309              :     /// 2.4. If the fringe is empty, go back to 1
    3310      1255269 :     async fn get_vectored_reconstruct_data(
    3311      1255269 :         &self,
    3312      1255269 :         mut keyspace: KeySpace,
    3313      1255269 :         request_lsn: Lsn,
    3314      1255269 :         reconstruct_state: &mut ValuesReconstructState,
    3315      1255269 :         ctx: &RequestContext,
    3316      1255269 :     ) -> Result<(), GetVectoredError> {
    3317      1255269 :         let mut timeline_owned: Arc<Timeline>;
    3318      1255269 :         let mut timeline = self;
    3319      1255269 : 
    3320      1255269 :         let mut cont_lsn = Lsn(request_lsn.0 + 1);
    3321              : 
    3322      1255265 :         let missing_keyspace = loop {
    3323      1706436 :             if self.cancel.is_cancelled() {
    3324            0 :                 return Err(GetVectoredError::Cancelled);
    3325      1706436 :             }
    3326              : 
    3327              :             let TimelineVisitOutcome {
    3328      1706436 :                 completed_keyspace: completed,
    3329      1706436 :                 image_covered_keyspace,
    3330      1706436 :             } = Self::get_vectored_reconstruct_data_timeline(
    3331      1706436 :                 timeline,
    3332      1706436 :                 keyspace.clone(),
    3333      1706436 :                 cont_lsn,
    3334      1706436 :                 reconstruct_state,
    3335      1706436 :                 &self.cancel,
    3336      1706436 :                 ctx,
    3337      1706436 :             )
    3338      1706436 :             .await?;
    3339              : 
    3340      1706436 :             keyspace.remove_overlapping_with(&completed);
    3341      1706436 : 
    3342      1706436 :             // Do not descend into the ancestor timeline for aux files.
    3343      1706436 :             // We don't return a blanket [`GetVectoredError::MissingKey`] to avoid
    3344      1706436 :             // stalling compaction.
    3345      1706436 :             keyspace.remove_overlapping_with(&KeySpace {
    3346      1706436 :                 ranges: vec![NON_INHERITED_RANGE, Key::sparse_non_inherited_keyspace()],
    3347      1706436 :             });
    3348      1706436 : 
    3349      1706436 :             // Keyspace is fully retrieved
    3350      1706436 :             if keyspace.is_empty() {
    3351      1255209 :                 break None;
    3352       451227 :             }
    3353              : 
    3354       451227 :             let Some(ancestor_timeline) = timeline.ancestor_timeline.as_ref() else {
    3355              :                 // Not fully retrieved but no ancestor timeline.
    3356           56 :                 break Some(keyspace);
    3357              :             };
    3358              : 
    3359              :             // Now we see if there are keys covered by the image layer but does not exist in the
    3360              :             // image layer, which means that the key does not exist.
    3361              : 
    3362              :             // The block below will stop the vectored search if any of the keys encountered an image layer
    3363              :             // which did not contain a snapshot for said key. Since we have already removed all completed
    3364              :             // keys from `keyspace`, we expect there to be no overlap between it and the image covered key
    3365              :             // space. If that's not the case, we had at least one key encounter a gap in the image layer
    3366              :             // and stop the search as a result of that.
    3367       451171 :             let mut removed = keyspace.remove_overlapping_with(&image_covered_keyspace);
    3368       451171 :             // Do not fire missing key error for sparse keys.
    3369       451171 :             removed.remove_overlapping_with(&KeySpace {
    3370       451171 :                 ranges: vec![SPARSE_RANGE],
    3371       451171 :             });
    3372       451171 :             if !removed.is_empty() {
    3373            0 :                 break Some(removed);
    3374       451171 :             }
    3375       451171 :             // If we reached this point, `remove_overlapping_with` should not have made any change to the
    3376       451171 :             // keyspace.
    3377       451171 : 
    3378       451171 :             // Take the min to avoid reconstructing a page with data newer than request Lsn.
    3379       451171 :             cont_lsn = std::cmp::min(Lsn(request_lsn.0 + 1), Lsn(timeline.ancestor_lsn.0 + 1));
    3380       451171 :             timeline_owned = timeline
    3381       451171 :                 .get_ready_ancestor_timeline(ancestor_timeline, ctx)
    3382       451171 :                 .await?;
    3383       451167 :             timeline = &*timeline_owned;
    3384              :         };
    3385              : 
    3386              :         // Remove sparse keys from the keyspace so that it doesn't fire errors.
    3387      1255265 :         let missing_keyspace = if let Some(missing_keyspace) = missing_keyspace {
    3388           56 :             let mut missing_keyspace = missing_keyspace;
    3389           56 :             missing_keyspace.remove_overlapping_with(&KeySpace {
    3390           56 :                 ranges: vec![SPARSE_RANGE],
    3391           56 :             });
    3392           56 :             if missing_keyspace.is_empty() {
    3393           28 :                 None
    3394              :             } else {
    3395           28 :                 Some(missing_keyspace)
    3396              :             }
    3397              :         } else {
    3398      1255209 :             None
    3399              :         };
    3400              : 
    3401      1255265 :         if let Some(missing_keyspace) = missing_keyspace {
    3402           28 :             return Err(GetVectoredError::MissingKey(MissingKeyError {
    3403           28 :                 key: missing_keyspace.start().unwrap(), /* better if we can store the full keyspace */
    3404           28 :                 shard: self
    3405           28 :                     .shard_identity
    3406           28 :                     .get_shard_number(&missing_keyspace.start().unwrap()),
    3407           28 :                 cont_lsn,
    3408           28 :                 request_lsn,
    3409           28 :                 ancestor_lsn: Some(timeline.ancestor_lsn),
    3410           28 :                 backtrace: None,
    3411           28 :             }));
    3412      1255237 :         }
    3413      1255237 : 
    3414      1255237 :         Ok(())
    3415      1255269 :     }
    3416              : 
    3417              :     /// Collect the reconstruct data for a keyspace from the specified timeline.
    3418              :     ///
    3419              :     /// Maintain a fringe [`LayerFringe`] which tracks all the layers that intersect
    3420              :     /// the current keyspace. The current keyspace of the search at any given timeline
    3421              :     /// is the original keyspace minus all the keys that have been completed minus
    3422              :     /// any keys for which we couldn't find an intersecting layer. It's not tracked explicitly,
    3423              :     /// but if you merge all the keyspaces in the fringe, you get the "current keyspace".
    3424              :     ///
    3425              :     /// This is basically a depth-first search visitor implementation where a vertex
    3426              :     /// is the (layer, lsn range, key space) tuple. The fringe acts as the stack.
    3427              :     ///
    3428              :     /// At each iteration pop the top of the fringe (the layer with the highest Lsn)
    3429              :     /// and get all the required reconstruct data from the layer in one go.
    3430              :     ///
    3431              :     /// Returns the completed keyspace and the keyspaces with image coverage. The caller
    3432              :     /// decides how to deal with these two keyspaces.
    3433      1706436 :     async fn get_vectored_reconstruct_data_timeline(
    3434      1706436 :         timeline: &Timeline,
    3435      1706436 :         keyspace: KeySpace,
    3436      1706436 :         mut cont_lsn: Lsn,
    3437      1706436 :         reconstruct_state: &mut ValuesReconstructState,
    3438      1706436 :         cancel: &CancellationToken,
    3439      1706436 :         ctx: &RequestContext,
    3440      1706436 :     ) -> Result<TimelineVisitOutcome, GetVectoredError> {
    3441      1706436 :         let mut unmapped_keyspace = keyspace.clone();
    3442      1706436 :         let mut fringe = LayerFringe::new();
    3443      1706436 : 
    3444      1706436 :         let mut completed_keyspace = KeySpace::default();
    3445      1706436 :         let mut image_covered_keyspace = KeySpaceRandomAccum::new();
    3446              : 
    3447              :         loop {
    3448      3399834 :             if cancel.is_cancelled() {
    3449            0 :                 return Err(GetVectoredError::Cancelled);
    3450      3399834 :             }
    3451      3399834 : 
    3452      3399834 :             let (keys_done_last_step, keys_with_image_coverage) =
    3453      3399834 :                 reconstruct_state.consume_done_keys();
    3454      3399834 :             unmapped_keyspace.remove_overlapping_with(&keys_done_last_step);
    3455      3399834 :             completed_keyspace.merge(&keys_done_last_step);
    3456      3399834 :             if let Some(keys_with_image_coverage) = keys_with_image_coverage {
    3457        45092 :                 unmapped_keyspace
    3458        45092 :                     .remove_overlapping_with(&KeySpace::single(keys_with_image_coverage.clone()));
    3459        45092 :                 image_covered_keyspace.add_range(keys_with_image_coverage);
    3460      3354742 :             }
    3461              : 
    3462              :             // Do not descent any further if the last layer we visited
    3463              :             // completed all keys in the keyspace it inspected. This is not
    3464              :             // required for correctness, but avoids visiting extra layers
    3465              :             // which turns out to be a perf bottleneck in some cases.
    3466      3399834 :             if !unmapped_keyspace.is_empty() {
    3467      2149109 :                 let guard = timeline.layers.read().await;
    3468      2149109 :                 let layers = guard.layer_map()?;
    3469              : 
    3470      2149109 :                 let in_memory_layer = layers.find_in_memory_layer(|l| {
    3471      1827625 :                     let start_lsn = l.get_lsn_range().start;
    3472      1827625 :                     cont_lsn > start_lsn
    3473      2149109 :                 });
    3474      2149109 : 
    3475      2149109 :                 match in_memory_layer {
    3476      1213232 :                     Some(l) => {
    3477      1213232 :                         let lsn_range = l.get_lsn_range().start..cont_lsn;
    3478      1213232 :                         fringe.update(
    3479      1213232 :                             ReadableLayer::InMemoryLayer(l),
    3480      1213232 :                             unmapped_keyspace.clone(),
    3481      1213232 :                             lsn_range,
    3482      1213232 :                         );
    3483      1213232 :                     }
    3484              :                     None => {
    3485       935925 :                         for range in unmapped_keyspace.ranges.iter() {
    3486       935925 :                             let results = layers.range_search(range.clone(), cont_lsn);
    3487       935925 : 
    3488       935925 :                             results
    3489       935925 :                                 .found
    3490       935925 :                                 .into_iter()
    3491       935925 :                                 .map(|(SearchResult { layer, lsn_floor }, keyspace_accum)| {
    3492       480194 :                                     (
    3493       480194 :                                         ReadableLayer::PersistentLayer(guard.get_from_desc(&layer)),
    3494       480194 :                                         keyspace_accum.to_keyspace(),
    3495       480194 :                                         lsn_floor..cont_lsn,
    3496       480194 :                                     )
    3497       935925 :                                 })
    3498       935925 :                                 .for_each(|(layer, keyspace, lsn_range)| {
    3499       480194 :                                     fringe.update(layer, keyspace, lsn_range)
    3500       935925 :                                 });
    3501       935925 :                         }
    3502              :                     }
    3503              :                 }
    3504              : 
    3505              :                 // It's safe to drop the layer map lock after planning the next round of reads.
    3506              :                 // The fringe keeps readable handles for the layers which are safe to read even
    3507              :                 // if layers were compacted or flushed.
    3508              :                 //
    3509              :                 // The more interesting consideration is: "Why is the read algorithm still correct
    3510              :                 // if the layer map changes while it is operating?". Doing a vectored read on a
    3511              :                 // timeline boils down to pushing an imaginary lsn boundary downwards for each range
    3512              :                 // covered by the read. The layer map tells us how to move the lsn downwards for a
    3513              :                 // range at *a particular point in time*. It is fine for the answer to be different
    3514              :                 // at two different time points.
    3515      2149109 :                 drop(guard);
    3516      1250725 :             }
    3517              : 
    3518      3399834 :             if let Some((layer_to_read, keyspace_to_read, lsn_range)) = fringe.next_layer() {
    3519      1693398 :                 let next_cont_lsn = lsn_range.start;
    3520      1693398 :                 layer_to_read
    3521      1693398 :                     .get_values_reconstruct_data(
    3522      1693398 :                         keyspace_to_read.clone(),
    3523      1693398 :                         lsn_range,
    3524      1693398 :                         reconstruct_state,
    3525      1693398 :                         ctx,
    3526      1693398 :                     )
    3527      1693398 :                     .await?;
    3528              : 
    3529      1693398 :                 unmapped_keyspace = keyspace_to_read;
    3530      1693398 :                 cont_lsn = next_cont_lsn;
    3531      1693398 : 
    3532      1693398 :                 reconstruct_state.on_layer_visited(&layer_to_read);
    3533              :             } else {
    3534      1706436 :                 break;
    3535      1706436 :             }
    3536      1706436 :         }
    3537      1706436 : 
    3538      1706436 :         Ok(TimelineVisitOutcome {
    3539      1706436 :             completed_keyspace,
    3540      1706436 :             image_covered_keyspace: image_covered_keyspace.consume_keyspace(),
    3541      1706436 :         })
    3542      1706436 :     }
    3543              : 
    3544       451171 :     async fn get_ready_ancestor_timeline(
    3545       451171 :         &self,
    3546       451171 :         ancestor: &Arc<Timeline>,
    3547       451171 :         ctx: &RequestContext,
    3548       451171 :     ) -> Result<Arc<Timeline>, GetReadyAncestorError> {
    3549       451171 :         // It's possible that the ancestor timeline isn't active yet, or
    3550       451171 :         // is active but hasn't yet caught up to the branch point. Wait
    3551       451171 :         // for it.
    3552       451171 :         //
    3553       451171 :         // This cannot happen while the pageserver is running normally,
    3554       451171 :         // because you cannot create a branch from a point that isn't
    3555       451171 :         // present in the pageserver yet. However, we don't wait for the
    3556       451171 :         // branch point to be uploaded to cloud storage before creating
    3557       451171 :         // a branch. I.e., the branch LSN need not be remote consistent
    3558       451171 :         // for the branching operation to succeed.
    3559       451171 :         //
    3560       451171 :         // Hence, if we try to load a tenant in such a state where
    3561       451171 :         // 1. the existence of the branch was persisted (in IndexPart and/or locally)
    3562       451171 :         // 2. but the ancestor state is behind branch_lsn because it was not yet persisted
    3563       451171 :         // then we will need to wait for the ancestor timeline to
    3564       451171 :         // re-stream WAL up to branch_lsn before we access it.
    3565       451171 :         //
    3566       451171 :         // How can a tenant get in such a state?
    3567       451171 :         // - ungraceful pageserver process exit
    3568       451171 :         // - detach+attach => this is a bug, https://github.com/neondatabase/neon/issues/4219
    3569       451171 :         //
    3570       451171 :         // NB: this could be avoided by requiring
    3571       451171 :         //   branch_lsn >= remote_consistent_lsn
    3572       451171 :         // during branch creation.
    3573       451171 :         match ancestor.wait_to_become_active(ctx).await {
    3574       451167 :             Ok(()) => {}
    3575              :             Err(TimelineState::Stopping) => {
    3576              :                 // If an ancestor is stopping, it means the tenant is stopping: handle this the same as if this timeline was stopping.
    3577            0 :                 return Err(GetReadyAncestorError::Cancelled);
    3578              :             }
    3579            4 :             Err(state) => {
    3580            4 :                 return Err(GetReadyAncestorError::BadState {
    3581            4 :                     timeline_id: ancestor.timeline_id,
    3582            4 :                     state,
    3583            4 :                 });
    3584              :             }
    3585              :         }
    3586       451167 :         ancestor
    3587       451167 :             .wait_lsn(self.ancestor_lsn, WaitLsnWaiter::Timeline(self), ctx)
    3588       451167 :             .await
    3589       451167 :             .map_err(|e| match e {
    3590            0 :                 e @ WaitLsnError::Timeout(_) => GetReadyAncestorError::AncestorLsnTimeout(e),
    3591            0 :                 WaitLsnError::Shutdown => GetReadyAncestorError::Cancelled,
    3592            0 :                 WaitLsnError::BadState(state) => GetReadyAncestorError::BadState {
    3593            0 :                     timeline_id: ancestor.timeline_id,
    3594            0 :                     state,
    3595            0 :                 },
    3596       451167 :             })?;
    3597              : 
    3598       451167 :         Ok(ancestor.clone())
    3599       451171 :     }
    3600              : 
    3601       594312 :     pub(crate) fn get_shard_identity(&self) -> &ShardIdentity {
    3602       594312 :         &self.shard_identity
    3603       594312 :     }
    3604              : 
    3605              :     #[inline(always)]
    3606            0 :     pub(crate) fn shard_timeline_id(&self) -> ShardTimelineId {
    3607            0 :         ShardTimelineId {
    3608            0 :             shard_index: ShardIndex {
    3609            0 :                 shard_number: self.shard_identity.number,
    3610            0 :                 shard_count: self.shard_identity.count,
    3611            0 :             },
    3612            0 :             timeline_id: self.timeline_id,
    3613            0 :         }
    3614            0 :     }
    3615              : 
    3616              :     /// Returns a non-frozen open in-memory layer for ingestion.
    3617              :     ///
    3618              :     /// Takes a witness of timeline writer state lock being held, because it makes no sense to call
    3619              :     /// this function without holding the mutex.
    3620         2592 :     async fn get_layer_for_write(
    3621         2592 :         &self,
    3622         2592 :         lsn: Lsn,
    3623         2592 :         _guard: &tokio::sync::MutexGuard<'_, Option<TimelineWriterState>>,
    3624         2592 :         ctx: &RequestContext,
    3625         2592 :     ) -> anyhow::Result<Arc<InMemoryLayer>> {
    3626         2592 :         let mut guard = self.layers.write().await;
    3627              : 
    3628         2592 :         let last_record_lsn = self.get_last_record_lsn();
    3629         2592 :         ensure!(
    3630         2592 :             lsn > last_record_lsn,
    3631            0 :             "cannot modify relation after advancing last_record_lsn (incoming_lsn={}, last_record_lsn={})",
    3632              :             lsn,
    3633              :             last_record_lsn,
    3634              :         );
    3635              : 
    3636         2592 :         let layer = guard
    3637         2592 :             .open_mut()?
    3638         2592 :             .get_layer_for_write(
    3639         2592 :                 lsn,
    3640         2592 :                 self.conf,
    3641         2592 :                 self.timeline_id,
    3642         2592 :                 self.tenant_shard_id,
    3643         2592 :                 &self.gate,
    3644         2592 :                 ctx,
    3645         2592 :             )
    3646         2592 :             .await?;
    3647         2592 :         Ok(layer)
    3648         2592 :     }
    3649              : 
    3650     10558184 :     pub(crate) fn finish_write(&self, new_lsn: Lsn) {
    3651     10558184 :         assert!(new_lsn.is_aligned());
    3652              : 
    3653     10558184 :         self.metrics.last_record_lsn_gauge.set(new_lsn.0 as i64);
    3654     10558184 :         self.last_record_lsn.advance(new_lsn);
    3655     10558184 :     }
    3656              : 
    3657              :     /// Freeze any existing open in-memory layer and unconditionally notify the flush loop.
    3658              :     ///
    3659              :     /// Unconditional flush loop notification is given because in sharded cases we will want to
    3660              :     /// leave an Lsn gap. Unsharded tenants do not have Lsn gaps.
    3661         2400 :     async fn freeze_inmem_layer_at(
    3662         2400 :         &self,
    3663         2400 :         at: Lsn,
    3664         2400 :         write_lock: &mut tokio::sync::MutexGuard<'_, Option<TimelineWriterState>>,
    3665         2400 :     ) -> Result<u64, FlushLayerError> {
    3666         2400 :         let frozen = {
    3667         2400 :             let mut guard = self.layers.write().await;
    3668         2400 :             guard
    3669         2400 :                 .open_mut()?
    3670         2400 :                 .try_freeze_in_memory_layer(at, &self.last_freeze_at, write_lock)
    3671         2400 :                 .await
    3672              :         };
    3673              : 
    3674         2400 :         if frozen {
    3675         2344 :             let now = Instant::now();
    3676         2344 :             *(self.last_freeze_ts.write().unwrap()) = now;
    3677         2344 :         }
    3678              : 
    3679              :         // Increment the flush cycle counter and wake up the flush task.
    3680              :         // Remember the new value, so that when we listen for the flush
    3681              :         // to finish, we know when the flush that we initiated has
    3682              :         // finished, instead of some other flush that was started earlier.
    3683         2400 :         let mut my_flush_request = 0;
    3684         2400 : 
    3685         2400 :         let flush_loop_state = { *self.flush_loop_state.lock().unwrap() };
    3686         2400 :         if !matches!(flush_loop_state, FlushLoopState::Running { .. }) {
    3687            0 :             return Err(FlushLayerError::NotRunning(flush_loop_state));
    3688         2400 :         }
    3689         2400 : 
    3690         2400 :         self.layer_flush_start_tx.send_modify(|(counter, lsn)| {
    3691         2400 :             my_flush_request = *counter + 1;
    3692         2400 :             *counter = my_flush_request;
    3693         2400 :             *lsn = std::cmp::max(at, *lsn);
    3694         2400 :         });
    3695         2400 : 
    3696         2400 :         assert_ne!(my_flush_request, 0);
    3697              : 
    3698         2400 :         Ok(my_flush_request)
    3699         2400 :     }
    3700              : 
    3701              :     /// Layer flusher task's main loop.
    3702          880 :     async fn flush_loop(
    3703          880 :         self: &Arc<Self>,
    3704          880 :         mut layer_flush_start_rx: tokio::sync::watch::Receiver<(u64, Lsn)>,
    3705          880 :         ctx: &RequestContext,
    3706          880 :     ) {
    3707              :         // Subscribe to L0 delta layer updates, for compaction backpressure.
    3708          880 :         let mut watch_l0 = match self.layers.read().await.layer_map() {
    3709          880 :             Ok(lm) => lm.watch_level0_deltas(),
    3710            0 :             Err(Shutdown) => return,
    3711              :         };
    3712              : 
    3713          880 :         info!("started flush loop");
    3714              :         loop {
    3715         3244 :             tokio::select! {
    3716         3244 :                 _ = self.cancel.cancelled() => {
    3717           20 :                     info!("shutting down layer flush task due to Timeline::cancel");
    3718           20 :                     break;
    3719              :                 },
    3720         3244 :                 _ = layer_flush_start_rx.changed() => {}
    3721         2364 :             }
    3722         2364 :             trace!("waking up");
    3723         2364 :             let (flush_counter, frozen_to_lsn) = *layer_flush_start_rx.borrow();
    3724         2364 : 
    3725         2364 :             // The highest LSN to which we flushed in the loop over frozen layers
    3726         2364 :             let mut flushed_to_lsn = Lsn(0);
    3727              : 
    3728         2364 :             let result = loop {
    3729         4708 :                 if self.cancel.is_cancelled() {
    3730            0 :                     info!("dropping out of flush loop for timeline shutdown");
    3731              :                     // Note: we do not bother transmitting into [`layer_flush_done_tx`], because
    3732              :                     // anyone waiting on that will respect self.cancel as well: they will stop
    3733              :                     // waiting at the same time we as drop out of this loop.
    3734            0 :                     return;
    3735         4708 :                 }
    3736         4708 : 
    3737         4708 :                 // Break to notify potential waiters as soon as we've flushed the requested LSN. If
    3738         4708 :                 // more requests have arrived in the meanwhile, we'll resume flushing afterwards.
    3739         4708 :                 if flushed_to_lsn >= frozen_to_lsn {
    3740         2308 :                     break Ok(());
    3741         2400 :                 }
    3742              : 
    3743              :                 // Fetch the next layer to flush, if any.
    3744         2400 :                 let (layer, l0_count, frozen_count, frozen_size) = {
    3745         2400 :                     let layers = self.layers.read().await;
    3746         2400 :                     let Ok(lm) = layers.layer_map() else {
    3747            0 :                         info!("dropping out of flush loop for timeline shutdown");
    3748            0 :                         return;
    3749              :                     };
    3750         2400 :                     let l0_count = lm.level0_deltas().len();
    3751         2400 :                     let frozen_count = lm.frozen_layers.len();
    3752         2400 :                     let frozen_size: u64 = lm
    3753         2400 :                         .frozen_layers
    3754         2400 :                         .iter()
    3755         2400 :                         .map(|l| l.estimated_in_mem_size())
    3756         2400 :                         .sum();
    3757         2400 :                     let layer = lm.frozen_layers.front().cloned();
    3758         2400 :                     (layer, l0_count, frozen_count, frozen_size)
    3759         2400 :                     // drop 'layers' lock
    3760         2400 :                 };
    3761         2400 :                 let Some(layer) = layer else {
    3762           56 :                     break Ok(());
    3763              :                 };
    3764              : 
    3765              :                 // Stall flushes to backpressure if compaction can't keep up. This is propagated up
    3766              :                 // to WAL ingestion by having ephemeral layer rolls wait for flushes.
    3767              :                 //
    3768              :                 // NB: the compaction loop only checks `compaction_threshold` every 20 seconds, so
    3769              :                 // we can end up stalling before compaction even starts. Consider making it more
    3770              :                 // responsive (e.g. via `watch_level0_deltas`).
    3771         2344 :                 if let Some(stall_threshold) = self.get_l0_flush_stall_threshold() {
    3772            0 :                     if l0_count >= stall_threshold {
    3773            0 :                         warn!(
    3774            0 :                             "stalling layer flushes for compaction backpressure at {l0_count} \
    3775            0 :                             L0 layers ({frozen_count} frozen layers with {frozen_size} bytes)"
    3776              :                         );
    3777            0 :                         let stall_timer = self
    3778            0 :                             .metrics
    3779            0 :                             .flush_delay_histo
    3780            0 :                             .start_timer()
    3781            0 :                             .record_on_drop();
    3782            0 :                         tokio::select! {
    3783            0 :                             result = watch_l0.wait_for(|l0| *l0 < stall_threshold) => {
    3784            0 :                                 if let Ok(l0) = result.as_deref() {
    3785            0 :                                     let delay = stall_timer.elapsed().as_secs_f64();
    3786            0 :                                     info!("resuming layer flushes at {l0} L0 layers after {delay:.3}s");
    3787            0 :                                 }
    3788              :                             },
    3789            0 :                             _ = self.cancel.cancelled() => {},
    3790              :                         }
    3791            0 :                         continue; // check again
    3792            0 :                     }
    3793         2344 :                 }
    3794              : 
    3795              :                 // Flush the layer.
    3796         2344 :                 let flush_timer = self.metrics.flush_time_histo.start_timer();
    3797         2344 :                 match self.flush_frozen_layer(layer, ctx).await {
    3798         2344 :                     Ok(layer_lsn) => flushed_to_lsn = max(flushed_to_lsn, layer_lsn),
    3799              :                     Err(FlushLayerError::Cancelled) => {
    3800            0 :                         info!("dropping out of flush loop for timeline shutdown");
    3801            0 :                         return;
    3802              :                     }
    3803            0 :                     err @ Err(
    3804            0 :                         FlushLayerError::NotRunning(_)
    3805            0 :                         | FlushLayerError::Other(_)
    3806            0 :                         | FlushLayerError::CreateImageLayersError(_),
    3807            0 :                     ) => {
    3808            0 :                         error!("could not flush frozen layer: {err:?}");
    3809            0 :                         break err.map(|_| ());
    3810              :                     }
    3811              :                 }
    3812         2344 :                 let flush_duration = flush_timer.stop_and_record();
    3813              : 
    3814              :                 // Delay the next flush to backpressure if compaction can't keep up. We delay by the
    3815              :                 // flush duration such that the flush takes 2x as long. This is propagated up to WAL
    3816              :                 // ingestion by having ephemeral layer rolls wait for flushes.
    3817         2344 :                 if let Some(delay_threshold) = self.get_l0_flush_delay_threshold() {
    3818            0 :                     if l0_count >= delay_threshold {
    3819            0 :                         let delay = flush_duration.as_secs_f64();
    3820            0 :                         info!(
    3821            0 :                             "delaying layer flush by {delay:.3}s for compaction backpressure at \
    3822            0 :                             {l0_count} L0 layers ({frozen_count} frozen layers with {frozen_size} bytes)"
    3823              :                         );
    3824            0 :                         let _delay_timer = self
    3825            0 :                             .metrics
    3826            0 :                             .flush_delay_histo
    3827            0 :                             .start_timer()
    3828            0 :                             .record_on_drop();
    3829            0 :                         tokio::select! {
    3830            0 :                             _ = tokio::time::sleep(flush_duration) => {},
    3831            0 :                             _ = watch_l0.wait_for(|l0| *l0 < delay_threshold) => {},
    3832            0 :                             _ = self.cancel.cancelled() => {},
    3833              :                         }
    3834            0 :                     }
    3835         2344 :                 }
    3836              :             };
    3837              : 
    3838              :             // Unsharded tenants should never advance their LSN beyond the end of the
    3839              :             // highest layer they write: such gaps between layer data and the frozen LSN
    3840              :             // are only legal on sharded tenants.
    3841         2364 :             debug_assert!(
    3842         2364 :                 self.shard_identity.count.count() > 1
    3843         2364 :                     || flushed_to_lsn >= frozen_to_lsn
    3844           56 :                     || !flushed_to_lsn.is_valid()
    3845              :             );
    3846              : 
    3847         2364 :             if flushed_to_lsn < frozen_to_lsn && self.shard_identity.count.count() > 1 {
    3848              :                 // If our layer flushes didn't carry disk_consistent_lsn up to the `to_lsn` advertised
    3849              :                 // to us via layer_flush_start_rx, then advance it here.
    3850              :                 //
    3851              :                 // This path is only taken for tenants with multiple shards: single sharded tenants should
    3852              :                 // never encounter a gap in the wal.
    3853            0 :                 let old_disk_consistent_lsn = self.disk_consistent_lsn.load();
    3854            0 :                 tracing::debug!("Advancing disk_consistent_lsn across layer gap {old_disk_consistent_lsn}->{frozen_to_lsn}");
    3855            0 :                 if self.set_disk_consistent_lsn(frozen_to_lsn) {
    3856            0 :                     if let Err(e) = self.schedule_uploads(frozen_to_lsn, vec![]) {
    3857            0 :                         tracing::warn!("Failed to schedule metadata upload after updating disk_consistent_lsn: {e}");
    3858            0 :                     }
    3859            0 :                 }
    3860         2364 :             }
    3861              : 
    3862              :             // Notify any listeners that we're done
    3863         2364 :             let _ = self
    3864         2364 :                 .layer_flush_done_tx
    3865         2364 :                 .send_replace((flush_counter, result));
    3866              :         }
    3867           20 :     }
    3868              : 
    3869              :     /// Waits any flush request created by [`Self::freeze_inmem_layer_at`] to complete.
    3870         2240 :     async fn wait_flush_completion(&self, request: u64) -> Result<(), FlushLayerError> {
    3871         2240 :         let mut rx = self.layer_flush_done_tx.subscribe();
    3872              :         loop {
    3873              :             {
    3874         4549 :                 let (last_result_counter, last_result) = &*rx.borrow();
    3875         4549 :                 if *last_result_counter >= request {
    3876         2240 :                     if let Err(err) = last_result {
    3877              :                         // We already logged the original error in
    3878              :                         // flush_loop. We cannot propagate it to the caller
    3879              :                         // here, because it might not be Cloneable
    3880            0 :                         return Err(err.clone());
    3881              :                     } else {
    3882         2240 :                         return Ok(());
    3883              :                     }
    3884         2309 :                 }
    3885         2309 :             }
    3886         2309 :             trace!("waiting for flush to complete");
    3887         2309 :             tokio::select! {
    3888         2309 :                 rx_e = rx.changed() => {
    3889         2309 :                     rx_e.map_err(|_| FlushLayerError::NotRunning(*self.flush_loop_state.lock().unwrap()))?;
    3890              :                 },
    3891              :                 // Cancellation safety: we are not leaving an I/O in-flight for the flush, we're just ignoring
    3892              :                 // the notification from [`flush_loop`] that it completed.
    3893         2309 :                 _ = self.cancel.cancelled() => {
    3894            0 :                     tracing::info!("Cancelled layer flush due on timeline shutdown");
    3895            0 :                     return Ok(())
    3896              :                 }
    3897              :             };
    3898         2309 :             trace!("done")
    3899              :         }
    3900         2240 :     }
    3901              : 
    3902              :     /// Flush one frozen in-memory layer to disk, as a new delta layer.
    3903              :     ///
    3904              :     /// Return value is the last lsn (inclusive) of the layer that was frozen.
    3905              :     #[instrument(skip_all, fields(layer=%frozen_layer))]
    3906              :     async fn flush_frozen_layer(
    3907              :         self: &Arc<Self>,
    3908              :         frozen_layer: Arc<InMemoryLayer>,
    3909              :         ctx: &RequestContext,
    3910              :     ) -> Result<Lsn, FlushLayerError> {
    3911              :         debug_assert_current_span_has_tenant_and_timeline_id();
    3912              : 
    3913              :         // As a special case, when we have just imported an image into the repository,
    3914              :         // instead of writing out a L0 delta layer, we directly write out image layer
    3915              :         // files instead. This is possible as long as *all* the data imported into the
    3916              :         // repository have the same LSN.
    3917              :         let lsn_range = frozen_layer.get_lsn_range();
    3918              : 
    3919              :         // Whether to directly create image layers for this flush, or flush them as delta layers
    3920              :         let create_image_layer =
    3921              :             lsn_range.start == self.initdb_lsn && lsn_range.end == Lsn(self.initdb_lsn.0 + 1);
    3922              : 
    3923              :         #[cfg(test)]
    3924              :         {
    3925              :             match &mut *self.flush_loop_state.lock().unwrap() {
    3926              :                 FlushLoopState::NotStarted | FlushLoopState::Exited => {
    3927              :                     panic!("flush loop not running")
    3928              :                 }
    3929              :                 FlushLoopState::Running {
    3930              :                     expect_initdb_optimization,
    3931              :                     initdb_optimization_count,
    3932              :                     ..
    3933              :                 } => {
    3934              :                     if create_image_layer {
    3935              :                         *initdb_optimization_count += 1;
    3936              :                     } else {
    3937              :                         assert!(!*expect_initdb_optimization, "expected initdb optimization");
    3938              :                     }
    3939              :                 }
    3940              :             }
    3941              :         }
    3942              : 
    3943              :         let (layers_to_upload, delta_layer_to_add) = if create_image_layer {
    3944              :             // Note: The 'ctx' in use here has DownloadBehavior::Error. We should not
    3945              :             // require downloading anything during initial import.
    3946              :             let ((rel_partition, metadata_partition), _lsn) = self
    3947              :                 .repartition(
    3948              :                     self.initdb_lsn,
    3949              :                     self.get_compaction_target_size(),
    3950              :                     EnumSet::empty(),
    3951              :                     ctx,
    3952              :                 )
    3953              :                 .await
    3954            0 :                 .map_err(|e| FlushLayerError::from_anyhow(self, e.into()))?;
    3955              : 
    3956              :             if self.cancel.is_cancelled() {
    3957              :                 return Err(FlushLayerError::Cancelled);
    3958              :             }
    3959              : 
    3960              :             // Ensure that we have a single call to `create_image_layers` with a combined dense keyspace.
    3961              :             // So that the key ranges don't overlap.
    3962              :             let mut partitions = KeyPartitioning::default();
    3963              :             partitions.parts.extend(rel_partition.parts);
    3964              :             if !metadata_partition.parts.is_empty() {
    3965              :                 assert_eq!(
    3966              :                     metadata_partition.parts.len(),
    3967              :                     1,
    3968              :                     "currently sparse keyspace should only contain a single metadata keyspace"
    3969              :                 );
    3970              :                 // Safety: create_image_layers treat sparse keyspaces differently that it does not scan
    3971              :                 // every single key within the keyspace, and therefore, it's safe to force converting it
    3972              :                 // into a dense keyspace before calling this function.
    3973              :                 partitions
    3974              :                     .parts
    3975              :                     .extend(metadata_partition.into_dense().parts);
    3976              :             }
    3977              : 
    3978              :             let mut layers_to_upload = Vec::new();
    3979              :             layers_to_upload.extend(
    3980              :                 self.create_image_layers(
    3981              :                     &partitions,
    3982              :                     self.initdb_lsn,
    3983              :                     ImageLayerCreationMode::Initial,
    3984              :                     ctx,
    3985              :                 )
    3986              :                 .await?,
    3987              :             );
    3988              : 
    3989              :             (layers_to_upload, None)
    3990              :         } else {
    3991              :             // Normal case, write out a L0 delta layer file.
    3992              :             // `create_delta_layer` will not modify the layer map.
    3993              :             // We will remove frozen layer and add delta layer in one atomic operation later.
    3994              :             let Some(layer) = self
    3995              :                 .create_delta_layer(&frozen_layer, None, ctx)
    3996              :                 .await
    3997            0 :                 .map_err(|e| FlushLayerError::from_anyhow(self, e))?
    3998              :             else {
    3999              :                 panic!("delta layer cannot be empty if no filter is applied");
    4000              :             };
    4001              :             (
    4002              :                 // FIXME: even though we have a single image and single delta layer assumption
    4003              :                 // we push them to vec
    4004              :                 vec![layer.clone()],
    4005              :                 Some(layer),
    4006              :             )
    4007              :         };
    4008              : 
    4009              :         pausable_failpoint!("flush-layer-cancel-after-writing-layer-out-pausable");
    4010              : 
    4011              :         if self.cancel.is_cancelled() {
    4012              :             return Err(FlushLayerError::Cancelled);
    4013              :         }
    4014              : 
    4015              :         let disk_consistent_lsn = Lsn(lsn_range.end.0 - 1);
    4016              : 
    4017              :         // The new on-disk layers are now in the layer map. We can remove the
    4018              :         // in-memory layer from the map now. The flushed layer is stored in
    4019              :         // the mapping in `create_delta_layer`.
    4020              :         {
    4021              :             let mut guard = self.layers.write().await;
    4022              : 
    4023              :             guard.open_mut()?.finish_flush_l0_layer(
    4024              :                 delta_layer_to_add.as_ref(),
    4025              :                 &frozen_layer,
    4026              :                 &self.metrics,
    4027              :             );
    4028              : 
    4029              :             if self.set_disk_consistent_lsn(disk_consistent_lsn) {
    4030              :                 // Schedule remote uploads that will reflect our new disk_consistent_lsn
    4031              :                 self.schedule_uploads(disk_consistent_lsn, layers_to_upload)
    4032            0 :                     .map_err(|e| FlushLayerError::from_anyhow(self, e))?;
    4033              :             }
    4034              :             // release lock on 'layers'
    4035              :         };
    4036              : 
    4037              :         // FIXME: between create_delta_layer and the scheduling of the upload in `update_metadata_file`,
    4038              :         // a compaction can delete the file and then it won't be available for uploads any more.
    4039              :         // We still schedule the upload, resulting in an error, but ideally we'd somehow avoid this
    4040              :         // race situation.
    4041              :         // See https://github.com/neondatabase/neon/issues/4526
    4042              :         pausable_failpoint!("flush-frozen-pausable");
    4043              : 
    4044              :         // This failpoint is used by another test case `test_pageserver_recovery`.
    4045              :         fail_point!("flush-frozen-exit");
    4046              : 
    4047              :         Ok(Lsn(lsn_range.end.0 - 1))
    4048              :     }
    4049              : 
    4050              :     /// Return true if the value changed
    4051              :     ///
    4052              :     /// This function must only be used from the layer flush task.
    4053         2344 :     fn set_disk_consistent_lsn(&self, new_value: Lsn) -> bool {
    4054         2344 :         let old_value = self.disk_consistent_lsn.fetch_max(new_value);
    4055         2344 :         assert!(new_value >= old_value, "disk_consistent_lsn must be growing monotonously at runtime; current {old_value}, offered {new_value}");
    4056              : 
    4057         2344 :         self.metrics
    4058         2344 :             .disk_consistent_lsn_gauge
    4059         2344 :             .set(new_value.0 as i64);
    4060         2344 :         new_value != old_value
    4061         2344 :     }
    4062              : 
    4063              :     /// Update metadata file
    4064         2444 :     fn schedule_uploads(
    4065         2444 :         &self,
    4066         2444 :         disk_consistent_lsn: Lsn,
    4067         2444 :         layers_to_upload: impl IntoIterator<Item = ResidentLayer>,
    4068         2444 :     ) -> anyhow::Result<()> {
    4069         2444 :         // We can only save a valid 'prev_record_lsn' value on disk if we
    4070         2444 :         // flushed *all* in-memory changes to disk. We only track
    4071         2444 :         // 'prev_record_lsn' in memory for the latest processed record, so we
    4072         2444 :         // don't remember what the correct value that corresponds to some old
    4073         2444 :         // LSN is. But if we flush everything, then the value corresponding
    4074         2444 :         // current 'last_record_lsn' is correct and we can store it on disk.
    4075         2444 :         let RecordLsn {
    4076         2444 :             last: last_record_lsn,
    4077         2444 :             prev: prev_record_lsn,
    4078         2444 :         } = self.last_record_lsn.load();
    4079         2444 :         let ondisk_prev_record_lsn = if disk_consistent_lsn == last_record_lsn {
    4080         2188 :             Some(prev_record_lsn)
    4081              :         } else {
    4082          256 :             None
    4083              :         };
    4084              : 
    4085         2444 :         let update = crate::tenant::metadata::MetadataUpdate::new(
    4086         2444 :             disk_consistent_lsn,
    4087         2444 :             ondisk_prev_record_lsn,
    4088         2444 :             *self.latest_gc_cutoff_lsn.read(),
    4089         2444 :         );
    4090         2444 : 
    4091         2444 :         fail_point!("checkpoint-before-saving-metadata", |x| bail!(
    4092            0 :             "{}",
    4093            0 :             x.unwrap()
    4094         2444 :         ));
    4095              : 
    4096         4812 :         for layer in layers_to_upload {
    4097         2368 :             self.remote_client.schedule_layer_file_upload(layer)?;
    4098              :         }
    4099         2444 :         self.remote_client
    4100         2444 :             .schedule_index_upload_for_metadata_update(&update)?;
    4101              : 
    4102         2444 :         Ok(())
    4103         2444 :     }
    4104              : 
    4105            0 :     pub(crate) async fn preserve_initdb_archive(&self) -> anyhow::Result<()> {
    4106            0 :         self.remote_client
    4107            0 :             .preserve_initdb_archive(
    4108            0 :                 &self.tenant_shard_id.tenant_id,
    4109            0 :                 &self.timeline_id,
    4110            0 :                 &self.cancel,
    4111            0 :             )
    4112            0 :             .await
    4113            0 :     }
    4114              : 
    4115              :     // Write out the given frozen in-memory layer as a new L0 delta file. This L0 file will not be tracked
    4116              :     // in layer map immediately. The caller is responsible to put it into the layer map.
    4117         1936 :     async fn create_delta_layer(
    4118         1936 :         self: &Arc<Self>,
    4119         1936 :         frozen_layer: &Arc<InMemoryLayer>,
    4120         1936 :         key_range: Option<Range<Key>>,
    4121         1936 :         ctx: &RequestContext,
    4122         1936 :     ) -> anyhow::Result<Option<ResidentLayer>> {
    4123         1936 :         let self_clone = Arc::clone(self);
    4124         1936 :         let frozen_layer = Arc::clone(frozen_layer);
    4125         1936 :         let ctx = ctx.attached_child();
    4126         1936 :         let work = async move {
    4127         1936 :             let Some((desc, path)) = frozen_layer
    4128         1936 :                 .write_to_disk(&ctx, key_range, self_clone.l0_flush_global_state.inner())
    4129         1936 :                 .await?
    4130              :             else {
    4131            0 :                 return Ok(None);
    4132              :             };
    4133         1936 :             let new_delta = Layer::finish_creating(self_clone.conf, &self_clone, desc, &path)?;
    4134              : 
    4135              :             // The write_to_disk() above calls writer.finish() which already did the fsync of the inodes.
    4136              :             // We just need to fsync the directory in which these inodes are linked,
    4137              :             // which we know to be the timeline directory.
    4138              :             //
    4139              :             // We use fatal_err() below because the after write_to_disk returns with success,
    4140              :             // the in-memory state of the filesystem already has the layer file in its final place,
    4141              :             // and subsequent pageserver code could think it's durable while it really isn't.
    4142         1936 :             let timeline_dir = VirtualFile::open(
    4143         1936 :                 &self_clone
    4144         1936 :                     .conf
    4145         1936 :                     .timeline_path(&self_clone.tenant_shard_id, &self_clone.timeline_id),
    4146         1936 :                 &ctx,
    4147         1936 :             )
    4148         1936 :             .await
    4149         1936 :             .fatal_err("VirtualFile::open for timeline dir fsync");
    4150         1936 :             timeline_dir
    4151         1936 :                 .sync_all()
    4152         1936 :                 .await
    4153         1936 :                 .fatal_err("VirtualFile::sync_all timeline dir");
    4154         1936 :             anyhow::Ok(Some(new_delta))
    4155         1936 :         };
    4156              :         // Before tokio-epoll-uring, we ran write_to_disk & the sync_all inside spawn_blocking.
    4157              :         // Preserve that behavior to maintain the same behavior for `virtual_file_io_engine=std-fs`.
    4158              :         use crate::virtual_file::io_engine::IoEngine;
    4159         1936 :         match crate::virtual_file::io_engine::get() {
    4160            0 :             IoEngine::NotSet => panic!("io engine not set"),
    4161              :             IoEngine::StdFs => {
    4162          968 :                 let span = tracing::info_span!("blocking");
    4163          968 :                 tokio::task::spawn_blocking({
    4164          968 :                     move || Handle::current().block_on(work.instrument(span))
    4165          968 :                 })
    4166          968 :                 .await
    4167          968 :                 .context("spawn_blocking")
    4168          968 :                 .and_then(|x| x)
    4169              :             }
    4170              :             #[cfg(target_os = "linux")]
    4171          968 :             IoEngine::TokioEpollUring => work.await,
    4172              :         }
    4173         1936 :     }
    4174              : 
    4175         1136 :     async fn repartition(
    4176         1136 :         &self,
    4177         1136 :         lsn: Lsn,
    4178         1136 :         partition_size: u64,
    4179         1136 :         flags: EnumSet<CompactFlags>,
    4180         1136 :         ctx: &RequestContext,
    4181         1136 :     ) -> Result<((KeyPartitioning, SparseKeyPartitioning), Lsn), CompactionError> {
    4182         1136 :         let Ok(mut guard) = self.partitioning.try_write_guard() else {
    4183              :             // NB: there are two callers, one is the compaction task, of which there is only one per struct Tenant and hence Timeline.
    4184              :             // The other is the initdb optimization in flush_frozen_layer, used by `boostrap_timeline`, which runs before `.activate()`
    4185              :             // and hence before the compaction task starts.
    4186            0 :             return Err(CompactionError::Other(anyhow!(
    4187            0 :                 "repartition() called concurrently"
    4188            0 :             )));
    4189              :         };
    4190         1136 :         let ((dense_partition, sparse_partition), partition_lsn) = &*guard.read();
    4191         1136 :         if lsn < *partition_lsn {
    4192            0 :             return Err(CompactionError::Other(anyhow!(
    4193            0 :                 "repartition() called with LSN going backwards, this should not happen"
    4194            0 :             )));
    4195         1136 :         }
    4196         1136 : 
    4197         1136 :         let distance = lsn.0 - partition_lsn.0;
    4198         1136 :         if *partition_lsn != Lsn(0)
    4199          524 :             && distance <= self.repartition_threshold
    4200          524 :             && !flags.contains(CompactFlags::ForceRepartition)
    4201              :         {
    4202          496 :             debug!(
    4203              :                 distance,
    4204              :                 threshold = self.repartition_threshold,
    4205            0 :                 "no repartitioning needed"
    4206              :             );
    4207          496 :             return Ok((
    4208          496 :                 (dense_partition.clone(), sparse_partition.clone()),
    4209          496 :                 *partition_lsn,
    4210          496 :             ));
    4211          640 :         }
    4212              : 
    4213          640 :         let (dense_ks, sparse_ks) = self.collect_keyspace(lsn, ctx).await?;
    4214          640 :         let dense_partitioning = dense_ks.partition(&self.shard_identity, partition_size);
    4215          640 :         let sparse_partitioning = SparseKeyPartitioning {
    4216          640 :             parts: vec![sparse_ks],
    4217          640 :         }; // no partitioning for metadata keys for now
    4218          640 :         let result = ((dense_partitioning, sparse_partitioning), lsn);
    4219          640 :         guard.write(result.clone());
    4220          640 :         Ok(result)
    4221         1136 :     }
    4222              : 
    4223              :     // Is it time to create a new image layer for the given partition?
    4224           28 :     async fn time_for_new_image_layer(&self, partition: &KeySpace, lsn: Lsn) -> bool {
    4225           28 :         let threshold = self.get_image_creation_threshold();
    4226              : 
    4227           28 :         let guard = self.layers.read().await;
    4228           28 :         let Ok(layers) = guard.layer_map() else {
    4229            0 :             return false;
    4230              :         };
    4231              : 
    4232           28 :         let mut max_deltas = 0;
    4233           56 :         for part_range in &partition.ranges {
    4234           28 :             let image_coverage = layers.image_coverage(part_range, lsn);
    4235           56 :             for (img_range, last_img) in image_coverage {
    4236           28 :                 let img_lsn = if let Some(last_img) = last_img {
    4237            0 :                     last_img.get_lsn_range().end
    4238              :                 } else {
    4239           28 :                     Lsn(0)
    4240              :                 };
    4241              :                 // Let's consider an example:
    4242              :                 //
    4243              :                 // delta layer with LSN range 71-81
    4244              :                 // delta layer with LSN range 81-91
    4245              :                 // delta layer with LSN range 91-101
    4246              :                 // image layer at LSN 100
    4247              :                 //
    4248              :                 // If 'lsn' is still 100, i.e. no new WAL has been processed since the last image layer,
    4249              :                 // there's no need to create a new one. We check this case explicitly, to avoid passing
    4250              :                 // a bogus range to count_deltas below, with start > end. It's even possible that there
    4251              :                 // are some delta layers *later* than current 'lsn', if more WAL was processed and flushed
    4252              :                 // after we read last_record_lsn, which is passed here in the 'lsn' argument.
    4253           28 :                 if img_lsn < lsn {
    4254           28 :                     let num_deltas =
    4255           28 :                         layers.count_deltas(&img_range, &(img_lsn..lsn), Some(threshold));
    4256           28 : 
    4257           28 :                     max_deltas = max_deltas.max(num_deltas);
    4258           28 :                     if num_deltas >= threshold {
    4259            0 :                         debug!(
    4260            0 :                             "key range {}-{}, has {} deltas on this timeline in LSN range {}..{}",
    4261              :                             img_range.start, img_range.end, num_deltas, img_lsn, lsn
    4262              :                         );
    4263            0 :                         return true;
    4264           28 :                     }
    4265            0 :                 }
    4266              :             }
    4267              :         }
    4268              : 
    4269           28 :         debug!(
    4270              :             max_deltas,
    4271            0 :             "none of the partitioned ranges had >= {threshold} deltas"
    4272              :         );
    4273           28 :         false
    4274           28 :     }
    4275              : 
    4276              :     /// Create image layers for Postgres data. Assumes the caller passes a partition that is not too large,
    4277              :     /// so that at most one image layer will be produced from this function.
    4278              :     #[allow(clippy::too_many_arguments)]
    4279          460 :     async fn create_image_layer_for_rel_blocks(
    4280          460 :         self: &Arc<Self>,
    4281          460 :         partition: &KeySpace,
    4282          460 :         mut image_layer_writer: ImageLayerWriter,
    4283          460 :         lsn: Lsn,
    4284          460 :         ctx: &RequestContext,
    4285          460 :         img_range: Range<Key>,
    4286          460 :         start: Key,
    4287          460 :         io_concurrency: IoConcurrency,
    4288          460 :     ) -> Result<ImageLayerCreationOutcome, CreateImageLayersError> {
    4289          460 :         let mut wrote_keys = false;
    4290          460 : 
    4291          460 :         let mut key_request_accum = KeySpaceAccum::new();
    4292         3044 :         for range in &partition.ranges {
    4293         2584 :             let mut key = range.start;
    4294         5600 :             while key < range.end {
    4295              :                 // Decide whether to retain this key: usually we do, but sharded tenants may
    4296              :                 // need to drop keys that don't belong to them.  If we retain the key, add it
    4297              :                 // to `key_request_accum` for later issuing a vectored get
    4298         3016 :                 if self.shard_identity.is_key_disposable(&key) {
    4299            0 :                     debug!(
    4300            0 :                         "Dropping key {} during compaction (it belongs on shard {:?})",
    4301            0 :                         key,
    4302            0 :                         self.shard_identity.get_shard_number(&key)
    4303              :                     );
    4304         3016 :                 } else {
    4305         3016 :                     key_request_accum.add_key(key);
    4306         3016 :                 }
    4307              : 
    4308         3016 :                 let last_key_in_range = key.next() == range.end;
    4309         3016 :                 key = key.next();
    4310         3016 : 
    4311         3016 :                 // Maybe flush `key_rest_accum`
    4312         3016 :                 if key_request_accum.raw_size() >= Timeline::MAX_GET_VECTORED_KEYS
    4313         3016 :                     || (last_key_in_range && key_request_accum.raw_size() > 0)
    4314              :                 {
    4315         2584 :                     let results = self
    4316         2584 :                         .get_vectored(
    4317         2584 :                             key_request_accum.consume_keyspace(),
    4318         2584 :                             lsn,
    4319         2584 :                             io_concurrency.clone(),
    4320         2584 :                             ctx,
    4321         2584 :                         )
    4322         2584 :                         .await?;
    4323              : 
    4324         2584 :                     if self.cancel.is_cancelled() {
    4325            0 :                         return Err(CreateImageLayersError::Cancelled);
    4326         2584 :                     }
    4327              : 
    4328         5600 :                     for (img_key, img) in results {
    4329         3016 :                         let img = match img {
    4330         3016 :                             Ok(img) => img,
    4331            0 :                             Err(err) => {
    4332            0 :                                 // If we fail to reconstruct a VM or FSM page, we can zero the
    4333            0 :                                 // page without losing any actual user data. That seems better
    4334            0 :                                 // than failing repeatedly and getting stuck.
    4335            0 :                                 //
    4336            0 :                                 // We had a bug at one point, where we truncated the FSM and VM
    4337            0 :                                 // in the pageserver, but the Postgres didn't know about that
    4338            0 :                                 // and continued to generate incremental WAL records for pages
    4339            0 :                                 // that didn't exist in the pageserver. Trying to replay those
    4340            0 :                                 // WAL records failed to find the previous image of the page.
    4341            0 :                                 // This special case allows us to recover from that situation.
    4342            0 :                                 // See https://github.com/neondatabase/neon/issues/2601.
    4343            0 :                                 //
    4344            0 :                                 // Unfortunately we cannot do this for the main fork, or for
    4345            0 :                                 // any metadata keys, keys, as that would lead to actual data
    4346            0 :                                 // loss.
    4347            0 :                                 if img_key.is_rel_fsm_block_key() || img_key.is_rel_vm_block_key() {
    4348            0 :                                     warn!("could not reconstruct FSM or VM key {img_key}, filling with zeros: {err:?}");
    4349            0 :                                     ZERO_PAGE.clone()
    4350              :                                 } else {
    4351            0 :                                     return Err(CreateImageLayersError::from(err));
    4352              :                                 }
    4353              :                             }
    4354              :                         };
    4355              : 
    4356              :                         // Write all the keys we just read into our new image layer.
    4357         3016 :                         image_layer_writer.put_image(img_key, img, ctx).await?;
    4358         3016 :                         wrote_keys = true;
    4359              :                     }
    4360          432 :                 }
    4361              :             }
    4362              :         }
    4363              : 
    4364          460 :         if wrote_keys {
    4365              :             // Normal path: we have written some data into the new image layer for this
    4366              :             // partition, so flush it to disk.
    4367          460 :             let (desc, path) = image_layer_writer.finish(ctx).await?;
    4368          460 :             let image_layer = Layer::finish_creating(self.conf, self, desc, &path)?;
    4369          460 :             info!("created image layer for rel {}", image_layer.local_path());
    4370          460 :             Ok(ImageLayerCreationOutcome {
    4371          460 :                 image: Some(image_layer),
    4372          460 :                 next_start_key: img_range.end,
    4373          460 :             })
    4374              :         } else {
    4375              :             // Special case: the image layer may be empty if this is a sharded tenant and the
    4376              :             // partition does not cover any keys owned by this shard.  In this case, to ensure
    4377              :             // we don't leave gaps between image layers, leave `start` where it is, so that the next
    4378              :             // layer we write will cover the key range that we just scanned.
    4379            0 :             tracing::debug!("no data in range {}-{}", img_range.start, img_range.end);
    4380            0 :             Ok(ImageLayerCreationOutcome {
    4381            0 :                 image: None,
    4382            0 :                 next_start_key: start,
    4383            0 :             })
    4384              :         }
    4385          460 :     }
    4386              : 
    4387              :     /// Create an image layer for metadata keys. This function produces one image layer for all metadata
    4388              :     /// keys for now. Because metadata keys cannot exceed basebackup size limit, the image layer for it
    4389              :     /// would not be too large to fit in a single image layer.
    4390              :     #[allow(clippy::too_many_arguments)]
    4391          440 :     async fn create_image_layer_for_metadata_keys(
    4392          440 :         self: &Arc<Self>,
    4393          440 :         partition: &KeySpace,
    4394          440 :         mut image_layer_writer: ImageLayerWriter,
    4395          440 :         lsn: Lsn,
    4396          440 :         ctx: &RequestContext,
    4397          440 :         img_range: Range<Key>,
    4398          440 :         mode: ImageLayerCreationMode,
    4399          440 :         start: Key,
    4400          440 :         io_concurrency: IoConcurrency,
    4401          440 :     ) -> Result<ImageLayerCreationOutcome, CreateImageLayersError> {
    4402          440 :         // Metadata keys image layer creation.
    4403          440 :         let mut reconstruct_state = ValuesReconstructState::new(io_concurrency);
    4404          440 :         let begin = Instant::now();
    4405          440 :         let data = self
    4406          440 :             .get_vectored_impl(partition.clone(), lsn, &mut reconstruct_state, ctx)
    4407          440 :             .await?;
    4408          440 :         let (data, total_kb_retrieved, total_keys_retrieved) = {
    4409          440 :             let mut new_data = BTreeMap::new();
    4410          440 :             let mut total_kb_retrieved = 0;
    4411          440 :             let mut total_keys_retrieved = 0;
    4412        20464 :             for (k, v) in data {
    4413        20024 :                 let v = v?;
    4414        20024 :                 total_kb_retrieved += KEY_SIZE + v.len();
    4415        20024 :                 total_keys_retrieved += 1;
    4416        20024 :                 new_data.insert(k, v);
    4417              :             }
    4418          440 :             (new_data, total_kb_retrieved / 1024, total_keys_retrieved)
    4419          440 :         };
    4420          440 :         let delta_files_accessed = reconstruct_state.get_delta_layers_visited();
    4421          440 :         let elapsed = begin.elapsed();
    4422          440 : 
    4423          440 :         let trigger_generation = delta_files_accessed as usize >= MAX_AUX_FILE_V2_DELTAS;
    4424          440 :         info!(
    4425            0 :             "metadata key compaction: trigger_generation={trigger_generation}, delta_files_accessed={delta_files_accessed}, total_kb_retrieved={total_kb_retrieved}, total_keys_retrieved={total_keys_retrieved}, read_time={}s", elapsed.as_secs_f64()
    4426              :         );
    4427              : 
    4428          440 :         if !trigger_generation && mode == ImageLayerCreationMode::Try {
    4429            4 :             return Ok(ImageLayerCreationOutcome {
    4430            4 :                 image: None,
    4431            4 :                 next_start_key: img_range.end,
    4432            4 :             });
    4433          436 :         }
    4434          436 :         if self.cancel.is_cancelled() {
    4435            0 :             return Err(CreateImageLayersError::Cancelled);
    4436          436 :         }
    4437          436 :         let mut wrote_any_image = false;
    4438        20460 :         for (k, v) in data {
    4439        20024 :             if v.is_empty() {
    4440              :                 // the key has been deleted, it does not need an image
    4441              :                 // in metadata keyspace, an empty image == tombstone
    4442           16 :                 continue;
    4443        20008 :             }
    4444        20008 :             wrote_any_image = true;
    4445        20008 : 
    4446        20008 :             // No need to handle sharding b/c metadata keys are always on the 0-th shard.
    4447        20008 : 
    4448        20008 :             // TODO: split image layers to avoid too large layer files. Too large image files are not handled
    4449        20008 :             // on the normal data path either.
    4450        20008 :             image_layer_writer.put_image(k, v, ctx).await?;
    4451              :         }
    4452              : 
    4453          436 :         if wrote_any_image {
    4454              :             // Normal path: we have written some data into the new image layer for this
    4455              :             // partition, so flush it to disk.
    4456           24 :             let (desc, path) = image_layer_writer.finish(ctx).await?;
    4457           24 :             let image_layer = Layer::finish_creating(self.conf, self, desc, &path)?;
    4458           24 :             info!(
    4459            0 :                 "created image layer for metadata {}",
    4460            0 :                 image_layer.local_path()
    4461              :             );
    4462           24 :             Ok(ImageLayerCreationOutcome {
    4463           24 :                 image: Some(image_layer),
    4464           24 :                 next_start_key: img_range.end,
    4465           24 :             })
    4466              :         } else {
    4467              :             // Special case: the image layer may be empty if this is a sharded tenant and the
    4468              :             // partition does not cover any keys owned by this shard. In this case, to ensure
    4469              :             // we don't leave gaps between image layers, leave `start` where it is, so that the next
    4470              :             // layer we write will cover the key range that we just scanned.
    4471          412 :             tracing::debug!("no data in range {}-{}", img_range.start, img_range.end);
    4472          412 :             Ok(ImageLayerCreationOutcome {
    4473          412 :                 image: None,
    4474          412 :                 next_start_key: start,
    4475          412 :             })
    4476              :         }
    4477          440 :     }
    4478              : 
    4479              :     /// Predicate function which indicates whether we should check if new image layers
    4480              :     /// are required. Since checking if new image layers are required is expensive in
    4481              :     /// terms of CPU, we only do it in the following cases:
    4482              :     /// 1. If the timeline has ingested sufficient WAL to justify the cost
    4483              :     /// 2. If enough time has passed since the last check:
    4484              :     ///     1. For large tenants, we wish to perform the check more often since they
    4485              :     ///        suffer from the lack of image layers
    4486              :     ///     2. For small tenants (that can mostly fit in RAM), we use a much longer interval
    4487         1136 :     fn should_check_if_image_layers_required(self: &Arc<Timeline>, lsn: Lsn) -> bool {
    4488              :         const LARGE_TENANT_THRESHOLD: u64 = 2 * 1024 * 1024 * 1024;
    4489              : 
    4490         1136 :         let last_checks_at = self.last_image_layer_creation_check_at.load();
    4491         1136 :         let distance = lsn
    4492         1136 :             .checked_sub(last_checks_at)
    4493         1136 :             .expect("Attempt to compact with LSN going backwards");
    4494         1136 :         let min_distance =
    4495         1136 :             self.get_image_layer_creation_check_threshold() as u64 * self.get_checkpoint_distance();
    4496         1136 : 
    4497         1136 :         let distance_based_decision = distance.0 >= min_distance;
    4498         1136 : 
    4499         1136 :         let mut time_based_decision = false;
    4500         1136 :         let mut last_check_instant = self.last_image_layer_creation_check_instant.lock().unwrap();
    4501         1136 :         if let CurrentLogicalSize::Exact(logical_size) = self.current_logical_size.current_size() {
    4502          932 :             let check_required_after = if Into::<u64>::into(&logical_size) >= LARGE_TENANT_THRESHOLD
    4503              :             {
    4504            0 :                 self.get_checkpoint_timeout()
    4505              :             } else {
    4506          932 :                 Duration::from_secs(3600 * 48)
    4507              :             };
    4508              : 
    4509          932 :             time_based_decision = match *last_check_instant {
    4510          524 :                 Some(last_check) => {
    4511          524 :                     let elapsed = last_check.elapsed();
    4512          524 :                     elapsed >= check_required_after
    4513              :                 }
    4514          408 :                 None => true,
    4515              :             };
    4516          204 :         }
    4517              : 
    4518              :         // Do the expensive delta layer counting only if this timeline has ingested sufficient
    4519              :         // WAL since the last check or a checkpoint timeout interval has elapsed since the last
    4520              :         // check.
    4521         1136 :         let decision = distance_based_decision || time_based_decision;
    4522              : 
    4523         1136 :         if decision {
    4524          412 :             self.last_image_layer_creation_check_at.store(lsn);
    4525          412 :             *last_check_instant = Some(Instant::now());
    4526          724 :         }
    4527              : 
    4528         1136 :         decision
    4529         1136 :     }
    4530              : 
    4531              :     #[tracing::instrument(skip_all, fields(%lsn, %mode))]
    4532              :     async fn create_image_layers(
    4533              :         self: &Arc<Timeline>,
    4534              :         partitioning: &KeyPartitioning,
    4535              :         lsn: Lsn,
    4536              :         mode: ImageLayerCreationMode,
    4537              :         ctx: &RequestContext,
    4538              :     ) -> Result<Vec<ResidentLayer>, CreateImageLayersError> {
    4539              :         let timer = self.metrics.create_images_time_histo.start_timer();
    4540              :         let mut image_layers = Vec::new();
    4541              : 
    4542              :         // We need to avoid holes between generated image layers.
    4543              :         // Otherwise LayerMap::image_layer_exists will return false if key range of some layer is covered by more than one
    4544              :         // image layer with hole between them. In this case such layer can not be utilized by GC.
    4545              :         //
    4546              :         // How such hole between partitions can appear?
    4547              :         // if we have relation with relid=1 and size 100 and relation with relid=2 with size 200 then result of
    4548              :         // KeySpace::partition may contain partitions <100000000..100000099> and <200000000..200000199>.
    4549              :         // If there is delta layer <100000000..300000000> then it never be garbage collected because
    4550              :         // image layers  <100000000..100000099> and <200000000..200000199> are not completely covering it.
    4551              :         let mut start = Key::MIN;
    4552              : 
    4553              :         let check_for_image_layers = self.should_check_if_image_layers_required(lsn);
    4554              : 
    4555              :         for partition in partitioning.parts.iter() {
    4556              :             if self.cancel.is_cancelled() {
    4557              :                 return Err(CreateImageLayersError::Cancelled);
    4558              :             }
    4559              : 
    4560              :             let img_range = start..partition.ranges.last().unwrap().end;
    4561              :             let compact_metadata = partition.overlaps(&Key::metadata_key_range());
    4562              :             if compact_metadata {
    4563              :                 for range in &partition.ranges {
    4564              :                     assert!(
    4565              :                         range.start.field1 >= METADATA_KEY_BEGIN_PREFIX
    4566              :                             && range.end.field1 <= METADATA_KEY_END_PREFIX,
    4567              :                         "metadata keys must be partitioned separately"
    4568              :                     );
    4569              :                 }
    4570              :                 if mode == ImageLayerCreationMode::Try && !check_for_image_layers {
    4571              :                     // Skip compaction if there are not enough updates. Metadata compaction will do a scan and
    4572              :                     // might mess up with evictions.
    4573              :                     start = img_range.end;
    4574              :                     continue;
    4575              :                 }
    4576              :                 // For initial and force modes, we always generate image layers for metadata keys.
    4577              :             } else if let ImageLayerCreationMode::Try = mode {
    4578              :                 // check_for_image_layers = false -> skip
    4579              :                 // check_for_image_layers = true -> check time_for_new_image_layer -> skip/generate
    4580              :                 if !check_for_image_layers || !self.time_for_new_image_layer(partition, lsn).await {
    4581              :                     start = img_range.end;
    4582              :                     continue;
    4583              :                 }
    4584              :             }
    4585              :             if let ImageLayerCreationMode::Force = mode {
    4586              :                 // When forced to create image layers, we might try and create them where they already
    4587              :                 // exist.  This mode is only used in tests/debug.
    4588              :                 let layers = self.layers.read().await;
    4589              :                 if layers.contains_key(&PersistentLayerKey {
    4590              :                     key_range: img_range.clone(),
    4591              :                     lsn_range: PersistentLayerDesc::image_layer_lsn_range(lsn),
    4592              :                     is_delta: false,
    4593              :                 }) {
    4594              :                     tracing::info!(
    4595              :                         "Skipping image layer at {lsn} {}..{}, already exists",
    4596              :                         img_range.start,
    4597              :                         img_range.end
    4598              :                     );
    4599              :                     start = img_range.end;
    4600              :                     continue;
    4601              :                 }
    4602              :             }
    4603              : 
    4604              :             let image_layer_writer = ImageLayerWriter::new(
    4605              :                 self.conf,
    4606              :                 self.timeline_id,
    4607              :                 self.tenant_shard_id,
    4608              :                 &img_range,
    4609              :                 lsn,
    4610              :                 ctx,
    4611              :             )
    4612              :             .await?;
    4613              : 
    4614            0 :             fail_point!("image-layer-writer-fail-before-finish", |_| {
    4615            0 :                 Err(CreateImageLayersError::Other(anyhow::anyhow!(
    4616            0 :                     "failpoint image-layer-writer-fail-before-finish"
    4617            0 :                 )))
    4618            0 :             });
    4619              : 
    4620              :             let io_concurrency = IoConcurrency::spawn_from_conf(
    4621              :                 self.conf,
    4622              :                 self.gate
    4623              :                     .enter()
    4624            0 :                     .map_err(|_| CreateImageLayersError::Cancelled)?,
    4625              :             );
    4626              : 
    4627              :             if !compact_metadata {
    4628              :                 let ImageLayerCreationOutcome {
    4629              :                     image,
    4630              :                     next_start_key,
    4631              :                 } = self
    4632              :                     .create_image_layer_for_rel_blocks(
    4633              :                         partition,
    4634              :                         image_layer_writer,
    4635              :                         lsn,
    4636              :                         ctx,
    4637              :                         img_range,
    4638              :                         start,
    4639              :                         io_concurrency,
    4640              :                     )
    4641              :                     .await?;
    4642              : 
    4643              :                 start = next_start_key;
    4644              :                 image_layers.extend(image);
    4645              :             } else {
    4646              :                 let ImageLayerCreationOutcome {
    4647              :                     image,
    4648              :                     next_start_key,
    4649              :                 } = self
    4650              :                     .create_image_layer_for_metadata_keys(
    4651              :                         partition,
    4652              :                         image_layer_writer,
    4653              :                         lsn,
    4654              :                         ctx,
    4655              :                         img_range,
    4656              :                         mode,
    4657              :                         start,
    4658              :                         io_concurrency,
    4659              :                     )
    4660              :                     .await?;
    4661              :                 start = next_start_key;
    4662              :                 image_layers.extend(image);
    4663              :             }
    4664              :         }
    4665              : 
    4666              :         let mut guard = self.layers.write().await;
    4667              : 
    4668              :         // FIXME: we could add the images to be uploaded *before* returning from here, but right
    4669              :         // now they are being scheduled outside of write lock; current way is inconsistent with
    4670              :         // compaction lock order.
    4671              :         guard
    4672              :             .open_mut()?
    4673              :             .track_new_image_layers(&image_layers, &self.metrics);
    4674              :         drop_wlock(guard);
    4675              :         timer.stop_and_record();
    4676              : 
    4677              :         // Creating image layers may have caused some previously visible layers to be covered
    4678              :         if !image_layers.is_empty() {
    4679              :             self.update_layer_visibility().await?;
    4680              :         }
    4681              : 
    4682              :         Ok(image_layers)
    4683              :     }
    4684              : 
    4685              :     /// Wait until the background initial logical size calculation is complete, or
    4686              :     /// this Timeline is shut down.  Calling this function will cause the initial
    4687              :     /// logical size calculation to skip waiting for the background jobs barrier.
    4688            0 :     pub(crate) async fn await_initial_logical_size(self: Arc<Self>) {
    4689            0 :         if !self.shard_identity.is_shard_zero() {
    4690              :             // We don't populate logical size on shard >0: skip waiting for it.
    4691            0 :             return;
    4692            0 :         }
    4693            0 : 
    4694            0 :         if self.remote_client.is_deleting() {
    4695              :             // The timeline was created in a deletion-resume state, we don't expect logical size to be populated
    4696            0 :             return;
    4697            0 :         }
    4698            0 : 
    4699            0 :         if self.current_logical_size.current_size().is_exact() {
    4700              :             // root timelines are initialized with exact count, but never start the background
    4701              :             // calculation
    4702            0 :             return;
    4703            0 :         }
    4704              : 
    4705            0 :         if let Some(await_bg_cancel) = self
    4706            0 :             .current_logical_size
    4707            0 :             .cancel_wait_for_background_loop_concurrency_limit_semaphore
    4708            0 :             .get()
    4709            0 :         {
    4710            0 :             await_bg_cancel.cancel();
    4711            0 :         } else {
    4712              :             // We should not wait if we were not able to explicitly instruct
    4713              :             // the logical size cancellation to skip the concurrency limit semaphore.
    4714              :             // TODO: this is an unexpected case.  We should restructure so that it
    4715              :             // can't happen.
    4716            0 :             tracing::warn!(
    4717            0 :                 "await_initial_logical_size: can't get semaphore cancel token, skipping"
    4718              :             );
    4719            0 :             debug_assert!(false);
    4720              :         }
    4721              : 
    4722            0 :         tokio::select!(
    4723            0 :             _ = self.current_logical_size.initialized.acquire() => {},
    4724            0 :             _ = self.cancel.cancelled() => {}
    4725              :         )
    4726            0 :     }
    4727              : 
    4728              :     /// Detach this timeline from its ancestor by copying all of ancestors layers as this
    4729              :     /// Timelines layers up to the ancestor_lsn.
    4730              :     ///
    4731              :     /// Requires a timeline that:
    4732              :     /// - has an ancestor to detach from
    4733              :     /// - the ancestor does not have an ancestor -- follows from the original RFC limitations, not
    4734              :     ///   a technical requirement
    4735              :     ///
    4736              :     /// After the operation has been started, it cannot be canceled. Upon restart it needs to be
    4737              :     /// polled again until completion.
    4738              :     ///
    4739              :     /// During the operation all timelines sharing the data with this timeline will be reparented
    4740              :     /// from our ancestor to be branches of this timeline.
    4741            0 :     pub(crate) async fn prepare_to_detach_from_ancestor(
    4742            0 :         self: &Arc<Timeline>,
    4743            0 :         tenant: &crate::tenant::Tenant,
    4744            0 :         options: detach_ancestor::Options,
    4745            0 :         ctx: &RequestContext,
    4746            0 :     ) -> Result<detach_ancestor::Progress, detach_ancestor::Error> {
    4747            0 :         detach_ancestor::prepare(self, tenant, options, ctx).await
    4748            0 :     }
    4749              : 
    4750              :     /// Second step of detach from ancestor; detaches the `self` from it's current ancestor and
    4751              :     /// reparents any reparentable children of previous ancestor.
    4752              :     ///
    4753              :     /// This method is to be called while holding the TenantManager's tenant slot, so during this
    4754              :     /// method we cannot be deleted nor can any timeline be deleted. After this method returns
    4755              :     /// successfully, tenant must be reloaded.
    4756              :     ///
    4757              :     /// Final step will be to [`Self::complete_detaching_timeline_ancestor`] after optionally
    4758              :     /// resetting the tenant.
    4759            0 :     pub(crate) async fn detach_from_ancestor_and_reparent(
    4760            0 :         self: &Arc<Timeline>,
    4761            0 :         tenant: &crate::tenant::Tenant,
    4762            0 :         prepared: detach_ancestor::PreparedTimelineDetach,
    4763            0 :         ctx: &RequestContext,
    4764            0 :     ) -> Result<detach_ancestor::DetachingAndReparenting, detach_ancestor::Error> {
    4765            0 :         detach_ancestor::detach_and_reparent(self, tenant, prepared, ctx).await
    4766            0 :     }
    4767              : 
    4768              :     /// Final step which unblocks the GC.
    4769              :     ///
    4770              :     /// The tenant must've been reset if ancestry was modified previously (in tenant manager).
    4771            0 :     pub(crate) async fn complete_detaching_timeline_ancestor(
    4772            0 :         self: &Arc<Timeline>,
    4773            0 :         tenant: &crate::tenant::Tenant,
    4774            0 :         attempt: detach_ancestor::Attempt,
    4775            0 :         ctx: &RequestContext,
    4776            0 :     ) -> Result<(), detach_ancestor::Error> {
    4777            0 :         detach_ancestor::complete(self, tenant, attempt, ctx).await
    4778            0 :     }
    4779              : }
    4780              : 
    4781              : impl Drop for Timeline {
    4782           20 :     fn drop(&mut self) {
    4783           20 :         if let Some(ancestor) = &self.ancestor_timeline {
    4784              :             // This lock should never be poisoned, but in case it is we do a .map() instead of
    4785              :             // an unwrap(), to avoid panicking in a destructor and thereby aborting the process.
    4786            8 :             if let Ok(mut gc_info) = ancestor.gc_info.write() {
    4787            8 :                 if !gc_info.remove_child_not_offloaded(self.timeline_id) {
    4788            0 :                     tracing::error!(tenant_id = %self.tenant_shard_id.tenant_id, shard_id = %self.tenant_shard_id.shard_slug(), timeline_id = %self.timeline_id,
    4789            0 :                         "Couldn't remove retain_lsn entry from offloaded timeline's parent: already removed");
    4790            8 :                 }
    4791            0 :             }
    4792           12 :         }
    4793           20 :         info!(
    4794            0 :             "Timeline {} for tenant {} is being dropped",
    4795              :             self.timeline_id, self.tenant_shard_id.tenant_id
    4796              :         );
    4797           20 :     }
    4798              : }
    4799              : 
    4800              : /// Top-level failure to compact.
    4801              : #[derive(Debug, thiserror::Error)]
    4802              : pub(crate) enum CompactionError {
    4803              :     #[error("The timeline or pageserver is shutting down")]
    4804              :     ShuttingDown,
    4805              :     /// Compaction tried to offload a timeline and failed
    4806              :     #[error("Failed to offload timeline: {0}")]
    4807              :     Offload(OffloadError),
    4808              :     /// Compaction cannot be done right now; page reconstruction and so on.
    4809              :     #[error(transparent)]
    4810              :     Other(anyhow::Error),
    4811              : }
    4812              : 
    4813              : impl From<OffloadError> for CompactionError {
    4814            0 :     fn from(e: OffloadError) -> Self {
    4815            0 :         match e {
    4816            0 :             OffloadError::Cancelled => Self::ShuttingDown,
    4817            0 :             _ => Self::Offload(e),
    4818              :         }
    4819            0 :     }
    4820              : }
    4821              : 
    4822              : impl CompactionError {
    4823            0 :     pub fn is_cancelled(&self) -> bool {
    4824            0 :         matches!(self, CompactionError::ShuttingDown)
    4825            0 :     }
    4826              : }
    4827              : 
    4828              : impl From<CollectKeySpaceError> for CompactionError {
    4829            0 :     fn from(err: CollectKeySpaceError) -> Self {
    4830            0 :         match err {
    4831              :             CollectKeySpaceError::Cancelled
    4832              :             | CollectKeySpaceError::PageRead(PageReconstructError::Cancelled) => {
    4833            0 :                 CompactionError::ShuttingDown
    4834              :             }
    4835            0 :             e => CompactionError::Other(e.into()),
    4836              :         }
    4837            0 :     }
    4838              : }
    4839              : 
    4840              : impl From<super::upload_queue::NotInitialized> for CompactionError {
    4841            0 :     fn from(value: super::upload_queue::NotInitialized) -> Self {
    4842            0 :         match value {
    4843              :             super::upload_queue::NotInitialized::Uninitialized => {
    4844            0 :                 CompactionError::Other(anyhow::anyhow!(value))
    4845              :             }
    4846              :             super::upload_queue::NotInitialized::ShuttingDown
    4847            0 :             | super::upload_queue::NotInitialized::Stopped => CompactionError::ShuttingDown,
    4848              :         }
    4849            0 :     }
    4850              : }
    4851              : 
    4852              : impl From<super::storage_layer::layer::DownloadError> for CompactionError {
    4853            0 :     fn from(e: super::storage_layer::layer::DownloadError) -> Self {
    4854            0 :         match e {
    4855              :             super::storage_layer::layer::DownloadError::TimelineShutdown
    4856              :             | super::storage_layer::layer::DownloadError::DownloadCancelled => {
    4857            0 :                 CompactionError::ShuttingDown
    4858              :             }
    4859              :             super::storage_layer::layer::DownloadError::ContextAndConfigReallyDeniesDownloads
    4860              :             | super::storage_layer::layer::DownloadError::DownloadRequired
    4861              :             | super::storage_layer::layer::DownloadError::NotFile(_)
    4862              :             | super::storage_layer::layer::DownloadError::DownloadFailed
    4863              :             | super::storage_layer::layer::DownloadError::PreStatFailed(_) => {
    4864            0 :                 CompactionError::Other(anyhow::anyhow!(e))
    4865              :             }
    4866              :             #[cfg(test)]
    4867              :             super::storage_layer::layer::DownloadError::Failpoint(_) => {
    4868            0 :                 CompactionError::Other(anyhow::anyhow!(e))
    4869              :             }
    4870              :         }
    4871            0 :     }
    4872              : }
    4873              : 
    4874              : impl From<layer_manager::Shutdown> for CompactionError {
    4875            0 :     fn from(_: layer_manager::Shutdown) -> Self {
    4876            0 :         CompactionError::ShuttingDown
    4877            0 :     }
    4878              : }
    4879              : 
    4880              : #[serde_as]
    4881          392 : #[derive(serde::Serialize)]
    4882              : struct RecordedDuration(#[serde_as(as = "serde_with::DurationMicroSeconds")] Duration);
    4883              : 
    4884              : #[derive(Default)]
    4885              : enum DurationRecorder {
    4886              :     #[default]
    4887              :     NotStarted,
    4888              :     Recorded(RecordedDuration, tokio::time::Instant),
    4889              : }
    4890              : 
    4891              : impl DurationRecorder {
    4892         1008 :     fn till_now(&self) -> DurationRecorder {
    4893         1008 :         match self {
    4894              :             DurationRecorder::NotStarted => {
    4895            0 :                 panic!("must only call on recorded measurements")
    4896              :             }
    4897         1008 :             DurationRecorder::Recorded(_, ended) => {
    4898         1008 :                 let now = tokio::time::Instant::now();
    4899         1008 :                 DurationRecorder::Recorded(RecordedDuration(now - *ended), now)
    4900         1008 :             }
    4901         1008 :         }
    4902         1008 :     }
    4903          392 :     fn into_recorded(self) -> Option<RecordedDuration> {
    4904          392 :         match self {
    4905            0 :             DurationRecorder::NotStarted => None,
    4906          392 :             DurationRecorder::Recorded(recorded, _) => Some(recorded),
    4907              :         }
    4908          392 :     }
    4909              : }
    4910              : 
    4911              : /// Descriptor for a delta layer used in testing infra. The start/end key/lsn range of the
    4912              : /// delta layer might be different from the min/max key/lsn in the delta layer. Therefore,
    4913              : /// the layer descriptor requires the user to provide the ranges, which should cover all
    4914              : /// keys specified in the `data` field.
    4915              : #[cfg(test)]
    4916              : #[derive(Clone)]
    4917              : pub struct DeltaLayerTestDesc {
    4918              :     pub lsn_range: Range<Lsn>,
    4919              :     pub key_range: Range<Key>,
    4920              :     pub data: Vec<(Key, Lsn, Value)>,
    4921              : }
    4922              : 
    4923              : #[cfg(test)]
    4924              : impl DeltaLayerTestDesc {
    4925            4 :     pub fn new(lsn_range: Range<Lsn>, key_range: Range<Key>, data: Vec<(Key, Lsn, Value)>) -> Self {
    4926            4 :         Self {
    4927            4 :             lsn_range,
    4928            4 :             key_range,
    4929            4 :             data,
    4930            4 :         }
    4931            4 :     }
    4932              : 
    4933          176 :     pub fn new_with_inferred_key_range(
    4934          176 :         lsn_range: Range<Lsn>,
    4935          176 :         data: Vec<(Key, Lsn, Value)>,
    4936          176 :     ) -> Self {
    4937          440 :         let key_min = data.iter().map(|(key, _, _)| key).min().unwrap();
    4938          440 :         let key_max = data.iter().map(|(key, _, _)| key).max().unwrap();
    4939          176 :         Self {
    4940          176 :             key_range: (*key_min)..(key_max.next()),
    4941          176 :             lsn_range,
    4942          176 :             data,
    4943          176 :         }
    4944          176 :     }
    4945              : 
    4946           20 :     pub(crate) fn layer_name(&self) -> LayerName {
    4947           20 :         LayerName::Delta(super::storage_layer::DeltaLayerName {
    4948           20 :             key_range: self.key_range.clone(),
    4949           20 :             lsn_range: self.lsn_range.clone(),
    4950           20 :         })
    4951           20 :     }
    4952              : }
    4953              : 
    4954              : impl Timeline {
    4955           56 :     async fn finish_compact_batch(
    4956           56 :         self: &Arc<Self>,
    4957           56 :         new_deltas: &[ResidentLayer],
    4958           56 :         new_images: &[ResidentLayer],
    4959           56 :         layers_to_remove: &[Layer],
    4960           56 :     ) -> Result<(), CompactionError> {
    4961           56 :         let mut guard = tokio::select! {
    4962           56 :             guard = self.layers.write() => guard,
    4963           56 :             _ = self.cancel.cancelled() => {
    4964            0 :                 return Err(CompactionError::ShuttingDown);
    4965              :             }
    4966              :         };
    4967              : 
    4968           56 :         let mut duplicated_layers = HashSet::new();
    4969           56 : 
    4970           56 :         let mut insert_layers = Vec::with_capacity(new_deltas.len());
    4971              : 
    4972          672 :         for l in new_deltas {
    4973          616 :             if guard.contains(l.as_ref()) {
    4974              :                 // expected in tests
    4975            0 :                 tracing::error!(layer=%l, "duplicated L1 layer");
    4976              : 
    4977              :                 // good ways to cause a duplicate: we repeatedly error after taking the writelock
    4978              :                 // `guard`  on self.layers. as of writing this, there are no error returns except
    4979              :                 // for compact_level0_phase1 creating an L0, which does not happen in practice
    4980              :                 // because we have not implemented L0 => L0 compaction.
    4981            0 :                 duplicated_layers.insert(l.layer_desc().key());
    4982          616 :             } else if LayerMap::is_l0(&l.layer_desc().key_range, l.layer_desc().is_delta) {
    4983            0 :                 return Err(CompactionError::Other(anyhow::anyhow!("compaction generates a L0 layer file as output, which will cause infinite compaction.")));
    4984          616 :             } else {
    4985          616 :                 insert_layers.push(l.clone());
    4986          616 :             }
    4987              :         }
    4988              : 
    4989              :         // only remove those inputs which were not outputs
    4990           56 :         let remove_layers: Vec<Layer> = layers_to_remove
    4991           56 :             .iter()
    4992          804 :             .filter(|l| !duplicated_layers.contains(&l.layer_desc().key()))
    4993           56 :             .cloned()
    4994           56 :             .collect();
    4995           56 : 
    4996           56 :         if !new_images.is_empty() {
    4997            0 :             guard
    4998            0 :                 .open_mut()?
    4999            0 :                 .track_new_image_layers(new_images, &self.metrics);
    5000           56 :         }
    5001              : 
    5002           56 :         guard
    5003           56 :             .open_mut()?
    5004           56 :             .finish_compact_l0(&remove_layers, &insert_layers, &self.metrics);
    5005           56 : 
    5006           56 :         self.remote_client
    5007           56 :             .schedule_compaction_update(&remove_layers, new_deltas)?;
    5008              : 
    5009           56 :         drop_wlock(guard);
    5010           56 : 
    5011           56 :         Ok(())
    5012           56 :     }
    5013              : 
    5014            0 :     async fn rewrite_layers(
    5015            0 :         self: &Arc<Self>,
    5016            0 :         mut replace_layers: Vec<(Layer, ResidentLayer)>,
    5017            0 :         mut drop_layers: Vec<Layer>,
    5018            0 :     ) -> Result<(), CompactionError> {
    5019            0 :         let mut guard = self.layers.write().await;
    5020              : 
    5021              :         // Trim our lists in case our caller (compaction) raced with someone else (GC) removing layers: we want
    5022              :         // to avoid double-removing, and avoid rewriting something that was removed.
    5023            0 :         replace_layers.retain(|(l, _)| guard.contains(l));
    5024            0 :         drop_layers.retain(|l| guard.contains(l));
    5025            0 : 
    5026            0 :         guard
    5027            0 :             .open_mut()?
    5028            0 :             .rewrite_layers(&replace_layers, &drop_layers, &self.metrics);
    5029            0 : 
    5030            0 :         let upload_layers: Vec<_> = replace_layers.into_iter().map(|r| r.1).collect();
    5031            0 : 
    5032            0 :         self.remote_client
    5033            0 :             .schedule_compaction_update(&drop_layers, &upload_layers)?;
    5034              : 
    5035            0 :         Ok(())
    5036            0 :     }
    5037              : 
    5038              :     /// Schedules the uploads of the given image layers
    5039          728 :     fn upload_new_image_layers(
    5040          728 :         self: &Arc<Self>,
    5041          728 :         new_images: impl IntoIterator<Item = ResidentLayer>,
    5042          728 :     ) -> Result<(), super::upload_queue::NotInitialized> {
    5043          780 :         for layer in new_images {
    5044           52 :             self.remote_client.schedule_layer_file_upload(layer)?;
    5045              :         }
    5046              :         // should any new image layer been created, not uploading index_part will
    5047              :         // result in a mismatch between remote_physical_size and layermap calculated
    5048              :         // size, which will fail some tests, but should not be an issue otherwise.
    5049          728 :         self.remote_client
    5050          728 :             .schedule_index_upload_for_file_changes()?;
    5051          728 :         Ok(())
    5052          728 :     }
    5053              : 
    5054            0 :     async fn find_gc_time_cutoff(
    5055            0 :         &self,
    5056            0 :         now: SystemTime,
    5057            0 :         pitr: Duration,
    5058            0 :         cancel: &CancellationToken,
    5059            0 :         ctx: &RequestContext,
    5060            0 :     ) -> Result<Option<Lsn>, PageReconstructError> {
    5061            0 :         debug_assert_current_span_has_tenant_and_timeline_id();
    5062            0 :         if self.shard_identity.is_shard_zero() {
    5063              :             // Shard Zero has SLRU data and can calculate the PITR time -> LSN mapping itself
    5064            0 :             let time_range = if pitr == Duration::ZERO {
    5065            0 :                 humantime::parse_duration(DEFAULT_PITR_INTERVAL).expect("constant is invalid")
    5066              :             } else {
    5067            0 :                 pitr
    5068              :             };
    5069              : 
    5070              :             // If PITR is so large or `now` is so small that this underflows, we will retain no history (highly unexpected case)
    5071            0 :             let time_cutoff = now.checked_sub(time_range).unwrap_or(now);
    5072            0 :             let timestamp = to_pg_timestamp(time_cutoff);
    5073              : 
    5074            0 :             let time_cutoff = match self.find_lsn_for_timestamp(timestamp, cancel, ctx).await? {
    5075            0 :                 LsnForTimestamp::Present(lsn) => Some(lsn),
    5076            0 :                 LsnForTimestamp::Future(lsn) => {
    5077            0 :                     // The timestamp is in the future. That sounds impossible,
    5078            0 :                     // but what it really means is that there hasn't been
    5079            0 :                     // any commits since the cutoff timestamp.
    5080            0 :                     //
    5081            0 :                     // In this case we should use the LSN of the most recent commit,
    5082            0 :                     // which is implicitly the last LSN in the log.
    5083            0 :                     debug!("future({})", lsn);
    5084            0 :                     Some(self.get_last_record_lsn())
    5085              :                 }
    5086            0 :                 LsnForTimestamp::Past(lsn) => {
    5087            0 :                     debug!("past({})", lsn);
    5088            0 :                     None
    5089              :                 }
    5090            0 :                 LsnForTimestamp::NoData(lsn) => {
    5091            0 :                     debug!("nodata({})", lsn);
    5092            0 :                     None
    5093              :                 }
    5094              :             };
    5095            0 :             Ok(time_cutoff)
    5096              :         } else {
    5097              :             // Shards other than shard zero cannot do timestamp->lsn lookups, and must instead learn their GC cutoff
    5098              :             // from shard zero's index.  The index doesn't explicitly tell us the time cutoff, but we may assume that
    5099              :             // the point up to which shard zero's last_gc_cutoff has advanced will either be the time cutoff, or a
    5100              :             // space cutoff that we would also have respected ourselves.
    5101            0 :             match self
    5102            0 :                 .remote_client
    5103            0 :                 .download_foreign_index(ShardNumber(0), cancel)
    5104            0 :                 .await
    5105              :             {
    5106            0 :                 Ok((index_part, index_generation, _index_mtime)) => {
    5107            0 :                     tracing::info!("GC loaded shard zero metadata (gen {index_generation:?}): latest_gc_cutoff_lsn: {}",
    5108            0 :                         index_part.metadata.latest_gc_cutoff_lsn());
    5109            0 :                     Ok(Some(index_part.metadata.latest_gc_cutoff_lsn()))
    5110              :                 }
    5111              :                 Err(DownloadError::NotFound) => {
    5112              :                     // This is unexpected, because during timeline creations shard zero persists to remote
    5113              :                     // storage before other shards are called, and during timeline deletion non-zeroth shards are
    5114              :                     // deleted before the zeroth one.  However, it should be harmless: if we somehow end up in this
    5115              :                     // state, then shard zero should _eventually_ write an index when it GCs.
    5116            0 :                     tracing::warn!("GC couldn't find shard zero's index for timeline");
    5117            0 :                     Ok(None)
    5118              :                 }
    5119            0 :                 Err(e) => {
    5120            0 :                     // TODO: this function should return a different error type than page reconstruct error
    5121            0 :                     Err(PageReconstructError::Other(anyhow::anyhow!(e)))
    5122              :                 }
    5123              :             }
    5124              : 
    5125              :             // TODO: after reading shard zero's GC cutoff, we should validate its generation with the storage
    5126              :             // controller.  Otherwise, it is possible that we see the GC cutoff go backwards while shard zero
    5127              :             // is going through a migration if we read the old location's index and it has GC'd ahead of the
    5128              :             // new location.  This is legal in principle, but problematic in practice because it might result
    5129              :             // in a timeline creation succeeding on shard zero ('s new location) but then failing on other shards
    5130              :             // because they have GC'd past the branch point.
    5131              :         }
    5132            0 :     }
    5133              : 
    5134              :     /// Find the Lsns above which layer files need to be retained on
    5135              :     /// garbage collection.
    5136              :     ///
    5137              :     /// We calculate two cutoffs, one based on time and one based on WAL size.  `pitr`
    5138              :     /// controls the time cutoff (or ZERO to disable time-based retention), and `space_cutoff` controls
    5139              :     /// the space-based retention.
    5140              :     ///
    5141              :     /// This function doesn't simply to calculate time & space based retention: it treats time-based
    5142              :     /// retention as authoritative if enabled, and falls back to space-based retention if calculating
    5143              :     /// the LSN for a time point isn't possible.  Therefore the GcCutoffs::horizon in the response might
    5144              :     /// be different to the `space_cutoff` input.  Callers should treat the min() of the two cutoffs
    5145              :     /// in the response as the GC cutoff point for the timeline.
    5146              :     #[instrument(skip_all, fields(timeline_id=%self.timeline_id))]
    5147              :     pub(super) async fn find_gc_cutoffs(
    5148              :         &self,
    5149              :         now: SystemTime,
    5150              :         space_cutoff: Lsn,
    5151              :         pitr: Duration,
    5152              :         cancel: &CancellationToken,
    5153              :         ctx: &RequestContext,
    5154              :     ) -> Result<GcCutoffs, PageReconstructError> {
    5155              :         let _timer = self
    5156              :             .metrics
    5157              :             .find_gc_cutoffs_histo
    5158              :             .start_timer()
    5159              :             .record_on_drop();
    5160              : 
    5161              :         pausable_failpoint!("Timeline::find_gc_cutoffs-pausable");
    5162              : 
    5163              :         if cfg!(test) {
    5164              :             // Unit tests which specify zero PITR interval expect to avoid doing any I/O for timestamp lookup
    5165              :             if pitr == Duration::ZERO {
    5166              :                 return Ok(GcCutoffs {
    5167              :                     time: self.get_last_record_lsn(),
    5168              :                     space: space_cutoff,
    5169              :                 });
    5170              :             }
    5171              :         }
    5172              : 
    5173              :         // Calculate a time-based limit on how much to retain:
    5174              :         // - if PITR interval is set, then this is our cutoff.
    5175              :         // - if PITR interval is not set, then we do a lookup
    5176              :         //   based on DEFAULT_PITR_INTERVAL, so that size-based retention does not result in keeping history around permanently on idle databases.
    5177              :         let time_cutoff = self.find_gc_time_cutoff(now, pitr, cancel, ctx).await?;
    5178              : 
    5179              :         Ok(match (pitr, time_cutoff) {
    5180              :             (Duration::ZERO, Some(time_cutoff)) => {
    5181              :                 // PITR is not set. Retain the size-based limit, or the default time retention,
    5182              :                 // whichever requires less data.
    5183              :                 GcCutoffs {
    5184              :                     time: self.get_last_record_lsn(),
    5185              :                     space: std::cmp::max(time_cutoff, space_cutoff),
    5186              :                 }
    5187              :             }
    5188              :             (Duration::ZERO, None) => {
    5189              :                 // PITR is not set, and time lookup failed
    5190              :                 GcCutoffs {
    5191              :                     time: self.get_last_record_lsn(),
    5192              :                     space: space_cutoff,
    5193              :                 }
    5194              :             }
    5195              :             (_, None) => {
    5196              :                 // PITR interval is set & we didn't look up a timestamp successfully.  Conservatively assume PITR
    5197              :                 // cannot advance beyond what was already GC'd, and respect space-based retention
    5198              :                 GcCutoffs {
    5199              :                     time: *self.get_latest_gc_cutoff_lsn(),
    5200              :                     space: space_cutoff,
    5201              :                 }
    5202              :             }
    5203              :             (_, Some(time_cutoff)) => {
    5204              :                 // PITR interval is set and we looked up timestamp successfully.  Ignore
    5205              :                 // size based retention and make time cutoff authoritative
    5206              :                 GcCutoffs {
    5207              :                     time: time_cutoff,
    5208              :                     space: time_cutoff,
    5209              :                 }
    5210              :             }
    5211              :         })
    5212              :     }
    5213              : 
    5214              :     /// Garbage collect layer files on a timeline that are no longer needed.
    5215              :     ///
    5216              :     /// Currently, we don't make any attempt at removing unneeded page versions
    5217              :     /// within a layer file. We can only remove the whole file if it's fully
    5218              :     /// obsolete.
    5219            8 :     pub(super) async fn gc(&self) -> Result<GcResult, GcError> {
    5220              :         // this is most likely the background tasks, but it might be the spawned task from
    5221              :         // immediate_gc
    5222            8 :         let _g = tokio::select! {
    5223            8 :             guard = self.gc_lock.lock() => guard,
    5224            8 :             _ = self.cancel.cancelled() => return Ok(GcResult::default()),
    5225              :         };
    5226            8 :         let timer = self.metrics.garbage_collect_histo.start_timer();
    5227            8 : 
    5228            8 :         fail_point!("before-timeline-gc");
    5229            8 : 
    5230            8 :         // Is the timeline being deleted?
    5231            8 :         if self.is_stopping() {
    5232            0 :             return Err(GcError::TimelineCancelled);
    5233            8 :         }
    5234            8 : 
    5235            8 :         let (space_cutoff, time_cutoff, retain_lsns, max_lsn_with_valid_lease) = {
    5236            8 :             let gc_info = self.gc_info.read().unwrap();
    5237            8 : 
    5238            8 :             let space_cutoff = min(gc_info.cutoffs.space, self.get_disk_consistent_lsn());
    5239            8 :             let time_cutoff = gc_info.cutoffs.time;
    5240            8 :             let retain_lsns = gc_info
    5241            8 :                 .retain_lsns
    5242            8 :                 .iter()
    5243            8 :                 .map(|(lsn, _child_id, _is_offloaded)| *lsn)
    5244            8 :                 .collect();
    5245            8 : 
    5246            8 :             // Gets the maximum LSN that holds the valid lease.
    5247            8 :             //
    5248            8 :             // Caveat: `refresh_gc_info` is in charged of updating the lease map.
    5249            8 :             // Here, we do not check for stale leases again.
    5250            8 :             let max_lsn_with_valid_lease = gc_info.leases.last_key_value().map(|(lsn, _)| *lsn);
    5251            8 : 
    5252            8 :             (
    5253            8 :                 space_cutoff,
    5254            8 :                 time_cutoff,
    5255            8 :                 retain_lsns,
    5256            8 :                 max_lsn_with_valid_lease,
    5257            8 :             )
    5258            8 :         };
    5259            8 : 
    5260            8 :         let mut new_gc_cutoff = Lsn::min(space_cutoff, time_cutoff);
    5261            8 :         let standby_horizon = self.standby_horizon.load();
    5262            8 :         // Hold GC for the standby, but as a safety guard do it only within some
    5263            8 :         // reasonable lag.
    5264            8 :         if standby_horizon != Lsn::INVALID {
    5265            0 :             if let Some(standby_lag) = new_gc_cutoff.checked_sub(standby_horizon) {
    5266              :                 const MAX_ALLOWED_STANDBY_LAG: u64 = 10u64 << 30; // 10 GB
    5267            0 :                 if standby_lag.0 < MAX_ALLOWED_STANDBY_LAG {
    5268            0 :                     new_gc_cutoff = Lsn::min(standby_horizon, new_gc_cutoff);
    5269            0 :                     trace!("holding off GC for standby apply LSN {}", standby_horizon);
    5270              :                 } else {
    5271            0 :                     warn!(
    5272            0 :                         "standby is lagging for more than {}MB, not holding gc for it",
    5273            0 :                         MAX_ALLOWED_STANDBY_LAG / 1024 / 1024
    5274              :                     )
    5275              :                 }
    5276            0 :             }
    5277            8 :         }
    5278              : 
    5279              :         // Reset standby horizon to ignore it if it is not updated till next GC.
    5280              :         // It is an easy way to unset it when standby disappears without adding
    5281              :         // more conf options.
    5282            8 :         self.standby_horizon.store(Lsn::INVALID);
    5283            8 :         self.metrics
    5284            8 :             .standby_horizon_gauge
    5285            8 :             .set(Lsn::INVALID.0 as i64);
    5286              : 
    5287            8 :         let res = self
    5288            8 :             .gc_timeline(
    5289            8 :                 space_cutoff,
    5290            8 :                 time_cutoff,
    5291            8 :                 retain_lsns,
    5292            8 :                 max_lsn_with_valid_lease,
    5293            8 :                 new_gc_cutoff,
    5294            8 :             )
    5295            8 :             .instrument(
    5296            8 :                 info_span!("gc_timeline", timeline_id = %self.timeline_id, cutoff = %new_gc_cutoff),
    5297              :             )
    5298            8 :             .await?;
    5299              : 
    5300              :         // only record successes
    5301            8 :         timer.stop_and_record();
    5302            8 : 
    5303            8 :         Ok(res)
    5304            8 :     }
    5305              : 
    5306            8 :     async fn gc_timeline(
    5307            8 :         &self,
    5308            8 :         space_cutoff: Lsn,
    5309            8 :         time_cutoff: Lsn,
    5310            8 :         retain_lsns: Vec<Lsn>,
    5311            8 :         max_lsn_with_valid_lease: Option<Lsn>,
    5312            8 :         new_gc_cutoff: Lsn,
    5313            8 :     ) -> Result<GcResult, GcError> {
    5314            8 :         // FIXME: if there is an ongoing detach_from_ancestor, we should just skip gc
    5315            8 : 
    5316            8 :         let now = SystemTime::now();
    5317            8 :         let mut result: GcResult = GcResult::default();
    5318            8 : 
    5319            8 :         // Nothing to GC. Return early.
    5320            8 :         let latest_gc_cutoff = *self.get_latest_gc_cutoff_lsn();
    5321            8 :         if latest_gc_cutoff >= new_gc_cutoff {
    5322            0 :             info!(
    5323            0 :                 "Nothing to GC: new_gc_cutoff_lsn {new_gc_cutoff}, latest_gc_cutoff_lsn {latest_gc_cutoff}",
    5324              :             );
    5325            0 :             return Ok(result);
    5326            8 :         }
    5327              : 
    5328              :         // We need to ensure that no one tries to read page versions or create
    5329              :         // branches at a point before latest_gc_cutoff_lsn. See branch_timeline()
    5330              :         // for details. This will block until the old value is no longer in use.
    5331              :         //
    5332              :         // The GC cutoff should only ever move forwards.
    5333            8 :         let waitlist = {
    5334            8 :             let write_guard = self.latest_gc_cutoff_lsn.lock_for_write();
    5335            8 :             if *write_guard > new_gc_cutoff {
    5336            0 :                 return Err(GcError::BadLsn {
    5337            0 :                     why: format!(
    5338            0 :                         "Cannot move GC cutoff LSN backwards (was {}, new {})",
    5339            0 :                         *write_guard, new_gc_cutoff
    5340            0 :                     ),
    5341            0 :                 });
    5342            8 :             }
    5343            8 : 
    5344            8 :             write_guard.store_and_unlock(new_gc_cutoff)
    5345            8 :         };
    5346            8 :         waitlist.wait().await;
    5347              : 
    5348            8 :         info!("GC starting");
    5349              : 
    5350            8 :         debug!("retain_lsns: {:?}", retain_lsns);
    5351              : 
    5352            8 :         let mut layers_to_remove = Vec::new();
    5353              : 
    5354              :         // Scan all layers in the timeline (remote or on-disk).
    5355              :         //
    5356              :         // Garbage collect the layer if all conditions are satisfied:
    5357              :         // 1. it is older than cutoff LSN;
    5358              :         // 2. it is older than PITR interval;
    5359              :         // 3. it doesn't need to be retained for 'retain_lsns';
    5360              :         // 4. it does not need to be kept for LSNs holding valid leases.
    5361              :         // 5. newer on-disk image layers cover the layer's whole key range
    5362              :         //
    5363              :         // TODO holding a write lock is too agressive and avoidable
    5364            8 :         let mut guard = self.layers.write().await;
    5365            8 :         let layers = guard.layer_map()?;
    5366           48 :         'outer: for l in layers.iter_historic_layers() {
    5367           48 :             result.layers_total += 1;
    5368           48 : 
    5369           48 :             // 1. Is it newer than GC horizon cutoff point?
    5370           48 :             if l.get_lsn_range().end > space_cutoff {
    5371            4 :                 info!(
    5372            0 :                     "keeping {} because it's newer than space_cutoff {}",
    5373            0 :                     l.layer_name(),
    5374              :                     space_cutoff,
    5375              :                 );
    5376            4 :                 result.layers_needed_by_cutoff += 1;
    5377            4 :                 continue 'outer;
    5378           44 :             }
    5379           44 : 
    5380           44 :             // 2. It is newer than PiTR cutoff point?
    5381           44 :             if l.get_lsn_range().end > time_cutoff {
    5382            0 :                 info!(
    5383            0 :                     "keeping {} because it's newer than time_cutoff {}",
    5384            0 :                     l.layer_name(),
    5385              :                     time_cutoff,
    5386              :                 );
    5387            0 :                 result.layers_needed_by_pitr += 1;
    5388            0 :                 continue 'outer;
    5389           44 :             }
    5390              : 
    5391              :             // 3. Is it needed by a child branch?
    5392              :             // NOTE With that we would keep data that
    5393              :             // might be referenced by child branches forever.
    5394              :             // We can track this in child timeline GC and delete parent layers when
    5395              :             // they are no longer needed. This might be complicated with long inheritance chains.
    5396              :             //
    5397              :             // TODO Vec is not a great choice for `retain_lsns`
    5398           44 :             for retain_lsn in &retain_lsns {
    5399              :                 // start_lsn is inclusive
    5400            0 :                 if &l.get_lsn_range().start <= retain_lsn {
    5401            0 :                     info!(
    5402            0 :                         "keeping {} because it's still might be referenced by child branch forked at {} is_dropped: xx is_incremental: {}",
    5403            0 :                         l.layer_name(),
    5404            0 :                         retain_lsn,
    5405            0 :                         l.is_incremental(),
    5406              :                     );
    5407            0 :                     result.layers_needed_by_branches += 1;
    5408            0 :                     continue 'outer;
    5409            0 :                 }
    5410              :             }
    5411              : 
    5412              :             // 4. Is there a valid lease that requires us to keep this layer?
    5413           44 :             if let Some(lsn) = &max_lsn_with_valid_lease {
    5414              :                 // keep if layer start <= any of the lease
    5415           36 :                 if &l.get_lsn_range().start <= lsn {
    5416           28 :                     info!(
    5417            0 :                         "keeping {} because there is a valid lease preventing GC at {}",
    5418            0 :                         l.layer_name(),
    5419              :                         lsn,
    5420              :                     );
    5421           28 :                     result.layers_needed_by_leases += 1;
    5422           28 :                     continue 'outer;
    5423            8 :                 }
    5424            8 :             }
    5425              : 
    5426              :             // 5. Is there a later on-disk layer for this relation?
    5427              :             //
    5428              :             // The end-LSN is exclusive, while disk_consistent_lsn is
    5429              :             // inclusive. For example, if disk_consistent_lsn is 100, it is
    5430              :             // OK for a delta layer to have end LSN 101, but if the end LSN
    5431              :             // is 102, then it might not have been fully flushed to disk
    5432              :             // before crash.
    5433              :             //
    5434              :             // For example, imagine that the following layers exist:
    5435              :             //
    5436              :             // 1000      - image (A)
    5437              :             // 1000-2000 - delta (B)
    5438              :             // 2000      - image (C)
    5439              :             // 2000-3000 - delta (D)
    5440              :             // 3000      - image (E)
    5441              :             //
    5442              :             // If GC horizon is at 2500, we can remove layers A and B, but
    5443              :             // we cannot remove C, even though it's older than 2500, because
    5444              :             // the delta layer 2000-3000 depends on it.
    5445           16 :             if !layers
    5446           16 :                 .image_layer_exists(&l.get_key_range(), &(l.get_lsn_range().end..new_gc_cutoff))
    5447              :             {
    5448           12 :                 info!("keeping {} because it is the latest layer", l.layer_name());
    5449           12 :                 result.layers_not_updated += 1;
    5450           12 :                 continue 'outer;
    5451            4 :             }
    5452            4 : 
    5453            4 :             // We didn't find any reason to keep this file, so remove it.
    5454            4 :             info!(
    5455            0 :                 "garbage collecting {} is_dropped: xx is_incremental: {}",
    5456            0 :                 l.layer_name(),
    5457            0 :                 l.is_incremental(),
    5458              :             );
    5459            4 :             layers_to_remove.push(l);
    5460              :         }
    5461              : 
    5462            8 :         if !layers_to_remove.is_empty() {
    5463              :             // Persist the new GC cutoff value before we actually remove anything.
    5464              :             // This unconditionally schedules also an index_part.json update, even though, we will
    5465              :             // be doing one a bit later with the unlinked gc'd layers.
    5466            4 :             let disk_consistent_lsn = self.disk_consistent_lsn.load();
    5467            4 :             self.schedule_uploads(disk_consistent_lsn, None)
    5468            4 :                 .map_err(|e| {
    5469            0 :                     if self.cancel.is_cancelled() {
    5470            0 :                         GcError::TimelineCancelled
    5471              :                     } else {
    5472            0 :                         GcError::Remote(e)
    5473              :                     }
    5474            4 :                 })?;
    5475              : 
    5476            4 :             let gc_layers = layers_to_remove
    5477            4 :                 .iter()
    5478            4 :                 .map(|x| guard.get_from_desc(x))
    5479            4 :                 .collect::<Vec<Layer>>();
    5480            4 : 
    5481            4 :             result.layers_removed = gc_layers.len() as u64;
    5482            4 : 
    5483            4 :             self.remote_client.schedule_gc_update(&gc_layers)?;
    5484              : 
    5485            4 :             guard.open_mut()?.finish_gc_timeline(&gc_layers);
    5486            4 : 
    5487            4 :             #[cfg(feature = "testing")]
    5488            4 :             {
    5489            4 :                 result.doomed_layers = gc_layers;
    5490            4 :             }
    5491            4 :         }
    5492              : 
    5493            8 :         info!(
    5494            0 :             "GC completed removing {} layers, cutoff {}",
    5495              :             result.layers_removed, new_gc_cutoff
    5496              :         );
    5497              : 
    5498            8 :         result.elapsed = now.elapsed().unwrap_or(Duration::ZERO);
    5499            8 :         Ok(result)
    5500            8 :     }
    5501              : 
    5502              :     /// Reconstruct a value, using the given base image and WAL records in 'data'.
    5503      1339985 :     async fn reconstruct_value(
    5504      1339985 :         &self,
    5505      1339985 :         key: Key,
    5506      1339985 :         request_lsn: Lsn,
    5507      1339985 :         mut data: ValueReconstructState,
    5508      1339985 :     ) -> Result<Bytes, PageReconstructError> {
    5509      1339985 :         // Perform WAL redo if needed
    5510      1339985 :         data.records.reverse();
    5511      1339985 : 
    5512      1339985 :         // If we have a page image, and no WAL, we're all set
    5513      1339985 :         if data.records.is_empty() {
    5514      1338349 :             if let Some((img_lsn, img)) = &data.img {
    5515      1338349 :                 trace!(
    5516            0 :                     "found page image for key {} at {}, no WAL redo required, req LSN {}",
    5517              :                     key,
    5518              :                     img_lsn,
    5519              :                     request_lsn,
    5520              :                 );
    5521      1338349 :                 Ok(img.clone())
    5522              :             } else {
    5523            0 :                 Err(PageReconstructError::from(anyhow!(
    5524            0 :                     "base image for {key} at {request_lsn} not found"
    5525            0 :                 )))
    5526              :             }
    5527              :         } else {
    5528              :             // We need to do WAL redo.
    5529              :             //
    5530              :             // If we don't have a base image, then the oldest WAL record better initialize
    5531              :             // the page
    5532         1636 :             if data.img.is_none() && !data.records.first().unwrap().1.will_init() {
    5533            0 :                 Err(PageReconstructError::from(anyhow!(
    5534            0 :                     "Base image for {} at {} not found, but got {} WAL records",
    5535            0 :                     key,
    5536            0 :                     request_lsn,
    5537            0 :                     data.records.len()
    5538            0 :                 )))
    5539              :             } else {
    5540         1636 :                 if data.img.is_some() {
    5541         1504 :                     trace!(
    5542            0 :                         "found {} WAL records and a base image for {} at {}, performing WAL redo",
    5543            0 :                         data.records.len(),
    5544              :                         key,
    5545              :                         request_lsn
    5546              :                     );
    5547              :                 } else {
    5548          132 :                     trace!("found {} WAL records that will init the page for {} at {}, performing WAL redo", data.records.len(), key, request_lsn);
    5549              :                 };
    5550         1636 :                 let res = self
    5551         1636 :                     .walredo_mgr
    5552         1636 :                     .as_ref()
    5553         1636 :                     .context("timeline has no walredo manager")
    5554         1636 :                     .map_err(PageReconstructError::WalRedo)?
    5555         1636 :                     .request_redo(key, request_lsn, data.img, data.records, self.pg_version)
    5556         1636 :                     .await;
    5557         1636 :                 let img = match res {
    5558         1636 :                     Ok(img) => img,
    5559            0 :                     Err(walredo::Error::Cancelled) => return Err(PageReconstructError::Cancelled),
    5560            0 :                     Err(walredo::Error::Other(e)) => {
    5561            0 :                         return Err(PageReconstructError::WalRedo(
    5562            0 :                             e.context("reconstruct a page image"),
    5563            0 :                         ))
    5564              :                     }
    5565              :                 };
    5566         1636 :                 Ok(img)
    5567              :             }
    5568              :         }
    5569      1339985 :     }
    5570              : 
    5571            0 :     pub(crate) async fn spawn_download_all_remote_layers(
    5572            0 :         self: Arc<Self>,
    5573            0 :         request: DownloadRemoteLayersTaskSpawnRequest,
    5574            0 :     ) -> Result<DownloadRemoteLayersTaskInfo, DownloadRemoteLayersTaskInfo> {
    5575              :         use pageserver_api::models::DownloadRemoteLayersTaskState;
    5576              : 
    5577              :         // this is not really needed anymore; it has tests which really check the return value from
    5578              :         // http api. it would be better not to maintain this anymore.
    5579              : 
    5580            0 :         let mut status_guard = self.download_all_remote_layers_task_info.write().unwrap();
    5581            0 :         if let Some(st) = &*status_guard {
    5582            0 :             match &st.state {
    5583              :                 DownloadRemoteLayersTaskState::Running => {
    5584            0 :                     return Err(st.clone());
    5585              :                 }
    5586              :                 DownloadRemoteLayersTaskState::ShutDown
    5587            0 :                 | DownloadRemoteLayersTaskState::Completed => {
    5588            0 :                     *status_guard = None;
    5589            0 :                 }
    5590              :             }
    5591            0 :         }
    5592              : 
    5593            0 :         let self_clone = Arc::clone(&self);
    5594            0 :         let task_id = task_mgr::spawn(
    5595            0 :             task_mgr::BACKGROUND_RUNTIME.handle(),
    5596            0 :             task_mgr::TaskKind::DownloadAllRemoteLayers,
    5597            0 :             self.tenant_shard_id,
    5598            0 :             Some(self.timeline_id),
    5599            0 :             "download all remote layers task",
    5600            0 :             async move {
    5601            0 :                 self_clone.download_all_remote_layers(request).await;
    5602            0 :                 let mut status_guard = self_clone.download_all_remote_layers_task_info.write().unwrap();
    5603            0 :                  match &mut *status_guard {
    5604              :                     None => {
    5605            0 :                         warn!("tasks status is supposed to be Some(), since we are running");
    5606              :                     }
    5607            0 :                     Some(st) => {
    5608            0 :                         let exp_task_id = format!("{}", task_mgr::current_task_id().unwrap());
    5609            0 :                         if st.task_id != exp_task_id {
    5610            0 :                             warn!("task id changed while we were still running, expecting {} but have {}", exp_task_id, st.task_id);
    5611            0 :                         } else {
    5612            0 :                             st.state = DownloadRemoteLayersTaskState::Completed;
    5613            0 :                         }
    5614              :                     }
    5615              :                 };
    5616            0 :                 Ok(())
    5617            0 :             }
    5618            0 :             .instrument(info_span!(parent: None, "download_all_remote_layers", tenant_id = %self.tenant_shard_id.tenant_id, shard_id = %self.tenant_shard_id.shard_slug(), timeline_id = %self.timeline_id))
    5619              :         );
    5620              : 
    5621            0 :         let initial_info = DownloadRemoteLayersTaskInfo {
    5622            0 :             task_id: format!("{task_id}"),
    5623            0 :             state: DownloadRemoteLayersTaskState::Running,
    5624            0 :             total_layer_count: 0,
    5625            0 :             successful_download_count: 0,
    5626            0 :             failed_download_count: 0,
    5627            0 :         };
    5628            0 :         *status_guard = Some(initial_info.clone());
    5629            0 : 
    5630            0 :         Ok(initial_info)
    5631            0 :     }
    5632              : 
    5633            0 :     async fn download_all_remote_layers(
    5634            0 :         self: &Arc<Self>,
    5635            0 :         request: DownloadRemoteLayersTaskSpawnRequest,
    5636            0 :     ) {
    5637              :         use pageserver_api::models::DownloadRemoteLayersTaskState;
    5638              : 
    5639            0 :         let remaining = {
    5640            0 :             let guard = self.layers.read().await;
    5641            0 :             let Ok(lm) = guard.layer_map() else {
    5642              :                 // technically here we could look into iterating accessible layers, but downloading
    5643              :                 // all layers of a shutdown timeline makes no sense regardless.
    5644            0 :                 tracing::info!("attempted to download all layers of shutdown timeline");
    5645            0 :                 return;
    5646              :             };
    5647            0 :             lm.iter_historic_layers()
    5648            0 :                 .map(|desc| guard.get_from_desc(&desc))
    5649            0 :                 .collect::<Vec<_>>()
    5650            0 :         };
    5651            0 :         let total_layer_count = remaining.len();
    5652              : 
    5653              :         macro_rules! lock_status {
    5654              :             ($st:ident) => {
    5655              :                 let mut st = self.download_all_remote_layers_task_info.write().unwrap();
    5656              :                 let st = st
    5657              :                     .as_mut()
    5658              :                     .expect("this function is only called after the task has been spawned");
    5659              :                 assert_eq!(
    5660              :                     st.task_id,
    5661              :                     format!(
    5662              :                         "{}",
    5663              :                         task_mgr::current_task_id().expect("we run inside a task_mgr task")
    5664              :                     )
    5665              :                 );
    5666              :                 let $st = st;
    5667              :             };
    5668              :         }
    5669              : 
    5670              :         {
    5671            0 :             lock_status!(st);
    5672            0 :             st.total_layer_count = total_layer_count as u64;
    5673            0 :         }
    5674            0 : 
    5675            0 :         let mut remaining = remaining.into_iter();
    5676            0 :         let mut have_remaining = true;
    5677            0 :         let mut js = tokio::task::JoinSet::new();
    5678            0 : 
    5679            0 :         let cancel = task_mgr::shutdown_token();
    5680            0 : 
    5681            0 :         let limit = request.max_concurrent_downloads;
    5682              : 
    5683              :         loop {
    5684            0 :             while js.len() < limit.get() && have_remaining && !cancel.is_cancelled() {
    5685            0 :                 let Some(next) = remaining.next() else {
    5686            0 :                     have_remaining = false;
    5687            0 :                     break;
    5688              :                 };
    5689              : 
    5690            0 :                 let span = tracing::info_span!("download", layer = %next);
    5691              : 
    5692            0 :                 js.spawn(
    5693            0 :                     async move {
    5694            0 :                         let res = next.download().await;
    5695            0 :                         (next, res)
    5696            0 :                     }
    5697            0 :                     .instrument(span),
    5698            0 :                 );
    5699            0 :             }
    5700              : 
    5701            0 :             while let Some(res) = js.join_next().await {
    5702            0 :                 match res {
    5703              :                     Ok((_, Ok(_))) => {
    5704            0 :                         lock_status!(st);
    5705            0 :                         st.successful_download_count += 1;
    5706              :                     }
    5707            0 :                     Ok((layer, Err(e))) => {
    5708            0 :                         tracing::error!(%layer, "download failed: {e:#}");
    5709            0 :                         lock_status!(st);
    5710            0 :                         st.failed_download_count += 1;
    5711              :                     }
    5712            0 :                     Err(je) if je.is_cancelled() => unreachable!("not used here"),
    5713            0 :                     Err(je) if je.is_panic() => {
    5714            0 :                         lock_status!(st);
    5715            0 :                         st.failed_download_count += 1;
    5716              :                     }
    5717            0 :                     Err(je) => tracing::warn!("unknown joinerror: {je:?}"),
    5718              :                 }
    5719              :             }
    5720              : 
    5721            0 :             if js.is_empty() && (!have_remaining || cancel.is_cancelled()) {
    5722            0 :                 break;
    5723            0 :             }
    5724              :         }
    5725              : 
    5726              :         {
    5727            0 :             lock_status!(st);
    5728            0 :             st.state = DownloadRemoteLayersTaskState::Completed;
    5729              :         }
    5730            0 :     }
    5731              : 
    5732            0 :     pub(crate) fn get_download_all_remote_layers_task_info(
    5733            0 :         &self,
    5734            0 :     ) -> Option<DownloadRemoteLayersTaskInfo> {
    5735            0 :         self.download_all_remote_layers_task_info
    5736            0 :             .read()
    5737            0 :             .unwrap()
    5738            0 :             .clone()
    5739            0 :     }
    5740              : }
    5741              : 
    5742              : impl Timeline {
    5743              :     /// Returns non-remote layers for eviction.
    5744            0 :     pub(crate) async fn get_local_layers_for_disk_usage_eviction(&self) -> DiskUsageEvictionInfo {
    5745            0 :         let guard = self.layers.read().await;
    5746            0 :         let mut max_layer_size: Option<u64> = None;
    5747            0 : 
    5748            0 :         let resident_layers = guard
    5749            0 :             .likely_resident_layers()
    5750            0 :             .map(|layer| {
    5751            0 :                 let file_size = layer.layer_desc().file_size;
    5752            0 :                 max_layer_size = max_layer_size.map_or(Some(file_size), |m| Some(m.max(file_size)));
    5753            0 : 
    5754            0 :                 let last_activity_ts = layer.latest_activity();
    5755            0 : 
    5756            0 :                 EvictionCandidate {
    5757            0 :                     layer: layer.to_owned().into(),
    5758            0 :                     last_activity_ts,
    5759            0 :                     relative_last_activity: finite_f32::FiniteF32::ZERO,
    5760            0 :                     visibility: layer.visibility(),
    5761            0 :                 }
    5762            0 :             })
    5763            0 :             .collect();
    5764            0 : 
    5765            0 :         DiskUsageEvictionInfo {
    5766            0 :             max_layer_size,
    5767            0 :             resident_layers,
    5768            0 :         }
    5769            0 :     }
    5770              : 
    5771         3720 :     pub(crate) fn get_shard_index(&self) -> ShardIndex {
    5772         3720 :         ShardIndex {
    5773         3720 :             shard_number: self.tenant_shard_id.shard_number,
    5774         3720 :             shard_count: self.tenant_shard_id.shard_count,
    5775         3720 :         }
    5776         3720 :     }
    5777              : 
    5778              :     /// Persistently blocks gc for `Manual` reason.
    5779              :     ///
    5780              :     /// Returns true if no such block existed before, false otherwise.
    5781            0 :     pub(crate) async fn block_gc(&self, tenant: &super::Tenant) -> anyhow::Result<bool> {
    5782              :         use crate::tenant::remote_timeline_client::index::GcBlockingReason;
    5783            0 :         assert_eq!(self.tenant_shard_id, tenant.tenant_shard_id);
    5784            0 :         tenant.gc_block.insert(self, GcBlockingReason::Manual).await
    5785            0 :     }
    5786              : 
    5787              :     /// Persistently unblocks gc for `Manual` reason.
    5788            0 :     pub(crate) async fn unblock_gc(&self, tenant: &super::Tenant) -> anyhow::Result<()> {
    5789              :         use crate::tenant::remote_timeline_client::index::GcBlockingReason;
    5790            0 :         assert_eq!(self.tenant_shard_id, tenant.tenant_shard_id);
    5791            0 :         tenant.gc_block.remove(self, GcBlockingReason::Manual).await
    5792            0 :     }
    5793              : 
    5794              :     #[cfg(test)]
    5795           92 :     pub(super) fn force_advance_lsn(self: &Arc<Timeline>, new_lsn: Lsn) {
    5796           92 :         self.last_record_lsn.advance(new_lsn);
    5797           92 :     }
    5798              : 
    5799              :     #[cfg(test)]
    5800            4 :     pub(super) fn force_set_disk_consistent_lsn(&self, new_value: Lsn) {
    5801            4 :         self.disk_consistent_lsn.store(new_value);
    5802            4 :     }
    5803              : 
    5804              :     /// Force create an image layer and place it into the layer map.
    5805              :     ///
    5806              :     /// DO NOT use this function directly. Use [`Tenant::branch_timeline_test_with_layers`]
    5807              :     /// or [`Tenant::create_test_timeline_with_layers`] to ensure all these layers are
    5808              :     /// placed into the layer map in one run AND be validated.
    5809              :     #[cfg(test)]
    5810          116 :     pub(super) async fn force_create_image_layer(
    5811          116 :         self: &Arc<Timeline>,
    5812          116 :         lsn: Lsn,
    5813          116 :         mut images: Vec<(Key, Bytes)>,
    5814          116 :         check_start_lsn: Option<Lsn>,
    5815          116 :         ctx: &RequestContext,
    5816          116 :     ) -> anyhow::Result<()> {
    5817          116 :         let last_record_lsn = self.get_last_record_lsn();
    5818          116 :         assert!(
    5819          116 :             lsn <= last_record_lsn,
    5820            0 :             "advance last record lsn before inserting a layer, lsn={lsn}, last_record_lsn={last_record_lsn}"
    5821              :         );
    5822          116 :         if let Some(check_start_lsn) = check_start_lsn {
    5823          116 :             assert!(lsn >= check_start_lsn);
    5824            0 :         }
    5825          348 :         images.sort_unstable_by(|(ka, _), (kb, _)| ka.cmp(kb));
    5826          116 :         let min_key = *images.first().map(|(k, _)| k).unwrap();
    5827          116 :         let end_key = images.last().map(|(k, _)| k).unwrap().next();
    5828          116 :         let mut image_layer_writer = ImageLayerWriter::new(
    5829          116 :             self.conf,
    5830          116 :             self.timeline_id,
    5831          116 :             self.tenant_shard_id,
    5832          116 :             &(min_key..end_key),
    5833          116 :             lsn,
    5834          116 :             ctx,
    5835          116 :         )
    5836          116 :         .await?;
    5837          580 :         for (key, img) in images {
    5838          464 :             image_layer_writer.put_image(key, img, ctx).await?;
    5839              :         }
    5840          116 :         let (desc, path) = image_layer_writer.finish(ctx).await?;
    5841          116 :         let image_layer = Layer::finish_creating(self.conf, self, desc, &path)?;
    5842          116 :         info!("force created image layer {}", image_layer.local_path());
    5843              :         {
    5844          116 :             let mut guard = self.layers.write().await;
    5845          116 :             guard
    5846          116 :                 .open_mut()
    5847          116 :                 .unwrap()
    5848          116 :                 .force_insert_layer(image_layer.clone());
    5849          116 :         }
    5850          116 : 
    5851          116 :         // Update remote_timeline_client state to reflect existence of this layer
    5852          116 :         self.remote_client
    5853          116 :             .schedule_layer_file_upload(image_layer)
    5854          116 :             .unwrap();
    5855          116 : 
    5856          116 :         Ok(())
    5857          116 :     }
    5858              : 
    5859              :     /// Force create a delta layer and place it into the layer map.
    5860              :     ///
    5861              :     /// DO NOT use this function directly. Use [`Tenant::branch_timeline_test_with_layers`]
    5862              :     /// or [`Tenant::create_test_timeline_with_layers`] to ensure all these layers are
    5863              :     /// placed into the layer map in one run AND be validated.
    5864              :     #[cfg(test)]
    5865          180 :     pub(super) async fn force_create_delta_layer(
    5866          180 :         self: &Arc<Timeline>,
    5867          180 :         mut deltas: DeltaLayerTestDesc,
    5868          180 :         check_start_lsn: Option<Lsn>,
    5869          180 :         ctx: &RequestContext,
    5870          180 :     ) -> anyhow::Result<()> {
    5871          180 :         let last_record_lsn = self.get_last_record_lsn();
    5872          180 :         deltas
    5873          180 :             .data
    5874          264 :             .sort_unstable_by(|(ka, la, _), (kb, lb, _)| (ka, la).cmp(&(kb, lb)));
    5875          180 :         assert!(deltas.data.first().unwrap().0 >= deltas.key_range.start);
    5876          180 :         assert!(deltas.data.last().unwrap().0 < deltas.key_range.end);
    5877          624 :         for (_, lsn, _) in &deltas.data {
    5878          444 :             assert!(deltas.lsn_range.start <= *lsn && *lsn < deltas.lsn_range.end);
    5879              :         }
    5880          180 :         assert!(
    5881          180 :             deltas.lsn_range.end <= last_record_lsn,
    5882            0 :             "advance last record lsn before inserting a layer, end_lsn={}, last_record_lsn={}",
    5883              :             deltas.lsn_range.end,
    5884              :             last_record_lsn
    5885              :         );
    5886          180 :         if let Some(check_start_lsn) = check_start_lsn {
    5887          180 :             assert!(deltas.lsn_range.start >= check_start_lsn);
    5888            0 :         }
    5889          180 :         let mut delta_layer_writer = DeltaLayerWriter::new(
    5890          180 :             self.conf,
    5891          180 :             self.timeline_id,
    5892          180 :             self.tenant_shard_id,
    5893          180 :             deltas.key_range.start,
    5894          180 :             deltas.lsn_range,
    5895          180 :             ctx,
    5896          180 :         )
    5897          180 :         .await?;
    5898          624 :         for (key, lsn, val) in deltas.data {
    5899          444 :             delta_layer_writer.put_value(key, lsn, val, ctx).await?;
    5900              :         }
    5901          180 :         let (desc, path) = delta_layer_writer.finish(deltas.key_range.end, ctx).await?;
    5902          180 :         let delta_layer = Layer::finish_creating(self.conf, self, desc, &path)?;
    5903          180 :         info!("force created delta layer {}", delta_layer.local_path());
    5904              :         {
    5905          180 :             let mut guard = self.layers.write().await;
    5906          180 :             guard
    5907          180 :                 .open_mut()
    5908          180 :                 .unwrap()
    5909          180 :                 .force_insert_layer(delta_layer.clone());
    5910          180 :         }
    5911          180 : 
    5912          180 :         // Update remote_timeline_client state to reflect existence of this layer
    5913          180 :         self.remote_client
    5914          180 :             .schedule_layer_file_upload(delta_layer)
    5915          180 :             .unwrap();
    5916          180 : 
    5917          180 :         Ok(())
    5918          180 :     }
    5919              : 
    5920              :     /// Return all keys at the LSN in the image layers
    5921              :     #[cfg(test)]
    5922           12 :     pub(crate) async fn inspect_image_layers(
    5923           12 :         self: &Arc<Timeline>,
    5924           12 :         lsn: Lsn,
    5925           12 :         ctx: &RequestContext,
    5926           12 :         io_concurrency: IoConcurrency,
    5927           12 :     ) -> anyhow::Result<Vec<(Key, Bytes)>> {
    5928           12 :         let mut all_data = Vec::new();
    5929           12 :         let guard = self.layers.read().await;
    5930           68 :         for layer in guard.layer_map()?.iter_historic_layers() {
    5931           68 :             if !layer.is_delta() && layer.image_layer_lsn() == lsn {
    5932           16 :                 let layer = guard.get_from_desc(&layer);
    5933           16 :                 let mut reconstruct_data = ValuesReconstructState::new(io_concurrency.clone());
    5934           16 :                 layer
    5935           16 :                     .get_values_reconstruct_data(
    5936           16 :                         KeySpace::single(Key::MIN..Key::MAX),
    5937           16 :                         lsn..Lsn(lsn.0 + 1),
    5938           16 :                         &mut reconstruct_data,
    5939           16 :                         ctx,
    5940           16 :                     )
    5941           16 :                     .await?;
    5942          132 :                 for (k, v) in std::mem::take(&mut reconstruct_data.keys) {
    5943          132 :                     let v = v.collect_pending_ios().await?;
    5944          132 :                     all_data.push((k, v.img.unwrap().1));
    5945              :                 }
    5946           52 :             }
    5947              :         }
    5948           12 :         all_data.sort();
    5949           12 :         Ok(all_data)
    5950           12 :     }
    5951              : 
    5952              :     /// Get all historic layer descriptors in the layer map
    5953              :     #[cfg(test)]
    5954           48 :     pub(crate) async fn inspect_historic_layers(
    5955           48 :         self: &Arc<Timeline>,
    5956           48 :     ) -> anyhow::Result<Vec<super::storage_layer::PersistentLayerKey>> {
    5957           48 :         let mut layers = Vec::new();
    5958           48 :         let guard = self.layers.read().await;
    5959          228 :         for layer in guard.layer_map()?.iter_historic_layers() {
    5960          228 :             layers.push(layer.key());
    5961          228 :         }
    5962           48 :         Ok(layers)
    5963           48 :     }
    5964              : 
    5965              :     #[cfg(test)]
    5966           20 :     pub(crate) fn add_extra_test_dense_keyspace(&self, ks: KeySpace) {
    5967           20 :         let mut keyspace = self.extra_test_dense_keyspace.load().as_ref().clone();
    5968           20 :         keyspace.merge(&ks);
    5969           20 :         self.extra_test_dense_keyspace.store(Arc::new(keyspace));
    5970           20 :     }
    5971              : }
    5972              : 
    5973              : /// Tracking writes ingestion does to a particular in-memory layer.
    5974              : ///
    5975              : /// Cleared upon freezing a layer.
    5976              : pub(crate) struct TimelineWriterState {
    5977              :     open_layer: Arc<InMemoryLayer>,
    5978              :     current_size: u64,
    5979              :     // Previous Lsn which passed through
    5980              :     prev_lsn: Option<Lsn>,
    5981              :     // Largest Lsn which passed through the current writer
    5982              :     max_lsn: Option<Lsn>,
    5983              :     // Cached details of the last freeze. Avoids going trough the atomic/lock on every put.
    5984              :     cached_last_freeze_at: Lsn,
    5985              : }
    5986              : 
    5987              : impl TimelineWriterState {
    5988         2592 :     fn new(open_layer: Arc<InMemoryLayer>, current_size: u64, last_freeze_at: Lsn) -> Self {
    5989         2592 :         Self {
    5990         2592 :             open_layer,
    5991         2592 :             current_size,
    5992         2592 :             prev_lsn: None,
    5993         2592 :             max_lsn: None,
    5994         2592 :             cached_last_freeze_at: last_freeze_at,
    5995         2592 :         }
    5996         2592 :     }
    5997              : }
    5998              : 
    5999              : /// Various functions to mutate the timeline.
    6000              : // TODO Currently, Deref is used to allow easy access to read methods from this trait.
    6001              : // This is probably considered a bad practice in Rust and should be fixed eventually,
    6002              : // but will cause large code changes.
    6003              : pub(crate) struct TimelineWriter<'a> {
    6004              :     tl: &'a Timeline,
    6005              :     write_guard: tokio::sync::MutexGuard<'a, Option<TimelineWriterState>>,
    6006              : }
    6007              : 
    6008              : impl Deref for TimelineWriter<'_> {
    6009              :     type Target = Timeline;
    6010              : 
    6011     19796336 :     fn deref(&self) -> &Self::Target {
    6012     19796336 :         self.tl
    6013     19796336 :     }
    6014              : }
    6015              : 
    6016              : #[derive(PartialEq)]
    6017              : enum OpenLayerAction {
    6018              :     Roll,
    6019              :     Open,
    6020              :     None,
    6021              : }
    6022              : 
    6023              : impl TimelineWriter<'_> {
    6024      9608464 :     async fn handle_open_layer_action(
    6025      9608464 :         &mut self,
    6026      9608464 :         at: Lsn,
    6027      9608464 :         action: OpenLayerAction,
    6028      9608464 :         ctx: &RequestContext,
    6029      9608464 :     ) -> anyhow::Result<&Arc<InMemoryLayer>> {
    6030      9608464 :         match action {
    6031              :             OpenLayerAction::Roll => {
    6032          160 :                 let freeze_at = self.write_guard.as_ref().unwrap().max_lsn.unwrap();
    6033          160 :                 self.roll_layer(freeze_at).await?;
    6034          160 :                 self.open_layer(at, ctx).await?;
    6035              :             }
    6036         2432 :             OpenLayerAction::Open => self.open_layer(at, ctx).await?,
    6037              :             OpenLayerAction::None => {
    6038      9605872 :                 assert!(self.write_guard.is_some());
    6039              :             }
    6040              :         }
    6041              : 
    6042      9608464 :         Ok(&self.write_guard.as_ref().unwrap().open_layer)
    6043      9608464 :     }
    6044              : 
    6045         2592 :     async fn open_layer(&mut self, at: Lsn, ctx: &RequestContext) -> anyhow::Result<()> {
    6046         2592 :         let layer = self
    6047         2592 :             .tl
    6048         2592 :             .get_layer_for_write(at, &self.write_guard, ctx)
    6049         2592 :             .await?;
    6050         2592 :         let initial_size = layer.size().await?;
    6051              : 
    6052         2592 :         let last_freeze_at = self.last_freeze_at.load();
    6053         2592 :         self.write_guard.replace(TimelineWriterState::new(
    6054         2592 :             layer,
    6055         2592 :             initial_size,
    6056         2592 :             last_freeze_at,
    6057         2592 :         ));
    6058         2592 : 
    6059         2592 :         Ok(())
    6060         2592 :     }
    6061              : 
    6062          160 :     async fn roll_layer(&mut self, freeze_at: Lsn) -> Result<(), FlushLayerError> {
    6063          160 :         let current_size = self.write_guard.as_ref().unwrap().current_size;
    6064              : 
    6065              :         // If layer flushes are backpressured due to compaction not keeping up, wait for the flush
    6066              :         // to propagate the backpressure up into WAL ingestion.
    6067          160 :         let l0_count = self
    6068          160 :             .tl
    6069          160 :             .layers
    6070          160 :             .read()
    6071          160 :             .await
    6072          160 :             .layer_map()?
    6073          160 :             .level0_deltas()
    6074          160 :             .len();
    6075          160 :         let wait_thresholds = [
    6076          160 :             self.get_l0_flush_delay_threshold(),
    6077          160 :             self.get_l0_flush_stall_threshold(),
    6078          160 :         ];
    6079          160 :         let wait_threshold = wait_thresholds.into_iter().flatten().min();
    6080              : 
    6081              :         // self.write_guard will be taken by the freezing
    6082          160 :         let flush_id = self
    6083          160 :             .tl
    6084          160 :             .freeze_inmem_layer_at(freeze_at, &mut self.write_guard)
    6085          160 :             .await?;
    6086              : 
    6087          160 :         assert!(self.write_guard.is_none());
    6088              : 
    6089          160 :         if let Some(wait_threshold) = wait_threshold {
    6090            0 :             if l0_count >= wait_threshold {
    6091            0 :                 info!("layer roll waiting for flush due to compaction backpressure at {l0_count} L0 layers");
    6092            0 :                 self.tl.wait_flush_completion(flush_id).await?;
    6093            0 :             }
    6094          160 :         }
    6095              : 
    6096          160 :         if current_size >= self.get_checkpoint_distance() * 2 {
    6097            0 :             warn!("Flushed oversized open layer with size {}", current_size)
    6098          160 :         }
    6099              : 
    6100          160 :         Ok(())
    6101          160 :     }
    6102              : 
    6103      9608464 :     fn get_open_layer_action(&self, lsn: Lsn, new_value_size: u64) -> OpenLayerAction {
    6104      9608464 :         let state = &*self.write_guard;
    6105      9608464 :         let Some(state) = &state else {
    6106         2432 :             return OpenLayerAction::Open;
    6107              :         };
    6108              : 
    6109              :         #[cfg(feature = "testing")]
    6110      9606032 :         if state.cached_last_freeze_at < self.tl.last_freeze_at.load() {
    6111              :             // this check and assertion are not really needed because
    6112              :             // LayerManager::try_freeze_in_memory_layer will always clear out the
    6113              :             // TimelineWriterState if something is frozen. however, we can advance last_freeze_at when there
    6114              :             // is no TimelineWriterState.
    6115            0 :             assert!(
    6116            0 :                 state.open_layer.end_lsn.get().is_some(),
    6117            0 :                 "our open_layer must be outdated"
    6118              :             );
    6119              : 
    6120              :             // this would be a memory leak waiting to happen because the in-memory layer always has
    6121              :             // an index
    6122            0 :             panic!("BUG: TimelineWriterState held on to frozen in-memory layer.");
    6123      9606032 :         }
    6124      9606032 : 
    6125      9606032 :         if state.prev_lsn == Some(lsn) {
    6126              :             // Rolling mid LSN is not supported by [downstream code].
    6127              :             // Hence, only roll at LSN boundaries.
    6128              :             //
    6129              :             // [downstream code]: https://github.com/neondatabase/neon/pull/7993#discussion_r1633345422
    6130           12 :             return OpenLayerAction::None;
    6131      9606020 :         }
    6132      9606020 : 
    6133      9606020 :         if state.current_size == 0 {
    6134              :             // Don't roll empty layers
    6135            0 :             return OpenLayerAction::None;
    6136      9606020 :         }
    6137      9606020 : 
    6138      9606020 :         if self.tl.should_roll(
    6139      9606020 :             state.current_size,
    6140      9606020 :             state.current_size + new_value_size,
    6141      9606020 :             self.get_checkpoint_distance(),
    6142      9606020 :             lsn,
    6143      9606020 :             state.cached_last_freeze_at,
    6144      9606020 :             state.open_layer.get_opened_at(),
    6145      9606020 :         ) {
    6146          160 :             OpenLayerAction::Roll
    6147              :         } else {
    6148      9605860 :             OpenLayerAction::None
    6149              :         }
    6150      9608464 :     }
    6151              : 
    6152              :     /// Put a batch of keys at the specified Lsns.
    6153      9608460 :     pub(crate) async fn put_batch(
    6154      9608460 :         &mut self,
    6155      9608460 :         batch: SerializedValueBatch,
    6156      9608460 :         ctx: &RequestContext,
    6157      9608460 :     ) -> anyhow::Result<()> {
    6158      9608460 :         if !batch.has_data() {
    6159            0 :             return Ok(());
    6160      9608460 :         }
    6161      9608460 : 
    6162      9608460 :         // In debug builds, assert that we don't write any keys that don't belong to this shard.
    6163      9608460 :         // We don't assert this in release builds, since key ownership policies may change over
    6164      9608460 :         // time. Stray keys will be removed during compaction.
    6165      9608460 :         if cfg!(debug_assertions) {
    6166     19789792 :             for metadata in &batch.metadata {
    6167     10181332 :                 if let ValueMeta::Serialized(metadata) = metadata {
    6168     10181332 :                     let key = Key::from_compact(metadata.key);
    6169     10181332 :                     assert!(
    6170     10181332 :                         self.shard_identity.is_key_local(&key)
    6171            0 :                             || self.shard_identity.is_key_global(&key),
    6172            0 :                         "key {key} does not belong on shard {}",
    6173            0 :                         self.shard_identity.shard_index()
    6174              :                     );
    6175            0 :                 }
    6176              :             }
    6177            0 :         }
    6178              : 
    6179      9608460 :         let batch_max_lsn = batch.max_lsn;
    6180      9608460 :         let buf_size: u64 = batch.buffer_size() as u64;
    6181      9608460 : 
    6182      9608460 :         let action = self.get_open_layer_action(batch_max_lsn, buf_size);
    6183      9608460 :         let layer = self
    6184      9608460 :             .handle_open_layer_action(batch_max_lsn, action, ctx)
    6185      9608460 :             .await?;
    6186              : 
    6187      9608460 :         let res = layer.put_batch(batch, ctx).await;
    6188              : 
    6189      9608460 :         if res.is_ok() {
    6190      9608460 :             // Update the current size only when the entire write was ok.
    6191      9608460 :             // In case of failures, we may have had partial writes which
    6192      9608460 :             // render the size tracking out of sync. That's ok because
    6193      9608460 :             // the checkpoint distance should be significantly smaller
    6194      9608460 :             // than the S3 single shot upload limit of 5GiB.
    6195      9608460 :             let state = self.write_guard.as_mut().unwrap();
    6196      9608460 : 
    6197      9608460 :             state.current_size += buf_size;
    6198      9608460 :             state.prev_lsn = Some(batch_max_lsn);
    6199      9608460 :             state.max_lsn = std::cmp::max(state.max_lsn, Some(batch_max_lsn));
    6200      9608460 :         }
    6201              : 
    6202      9608460 :         res
    6203      9608460 :     }
    6204              : 
    6205              :     #[cfg(test)]
    6206              :     /// Test helper, for tests that would like to poke individual values without composing a batch
    6207      8780308 :     pub(crate) async fn put(
    6208      8780308 :         &mut self,
    6209      8780308 :         key: Key,
    6210      8780308 :         lsn: Lsn,
    6211      8780308 :         value: &Value,
    6212      8780308 :         ctx: &RequestContext,
    6213      8780308 :     ) -> anyhow::Result<()> {
    6214              :         use utils::bin_ser::BeSer;
    6215      8780308 :         if !key.is_valid_key_on_write_path() {
    6216            0 :             bail!(
    6217            0 :                 "the request contains data not supported by pageserver at TimelineWriter::put: {}",
    6218            0 :                 key
    6219            0 :             );
    6220      8780308 :         }
    6221      8780308 :         let val_ser_size = value.serialized_size().unwrap() as usize;
    6222      8780308 :         let batch = SerializedValueBatch::from_values(vec![(
    6223      8780308 :             key.to_compact(),
    6224      8780308 :             lsn,
    6225      8780308 :             val_ser_size,
    6226      8780308 :             value.clone(),
    6227      8780308 :         )]);
    6228      8780308 : 
    6229      8780308 :         self.put_batch(batch, ctx).await
    6230      8780308 :     }
    6231              : 
    6232            4 :     pub(crate) async fn delete_batch(
    6233            4 :         &mut self,
    6234            4 :         batch: &[(Range<Key>, Lsn)],
    6235            4 :         ctx: &RequestContext,
    6236            4 :     ) -> anyhow::Result<()> {
    6237            4 :         if let Some((_, lsn)) = batch.first() {
    6238            4 :             let action = self.get_open_layer_action(*lsn, 0);
    6239            4 :             let layer = self.handle_open_layer_action(*lsn, action, ctx).await?;
    6240            4 :             layer.put_tombstones(batch).await?;
    6241            0 :         }
    6242              : 
    6243            4 :         Ok(())
    6244            4 :     }
    6245              : 
    6246              :     /// Track the end of the latest digested WAL record.
    6247              :     /// Remember the (end of) last valid WAL record remembered in the timeline.
    6248              :     ///
    6249              :     /// Call this after you have finished writing all the WAL up to 'lsn'.
    6250              :     ///
    6251              :     /// 'lsn' must be aligned. This wakes up any wait_lsn() callers waiting for
    6252              :     /// the 'lsn' or anything older. The previous last record LSN is stored alongside
    6253              :     /// the latest and can be read.
    6254     10558184 :     pub(crate) fn finish_write(&self, new_lsn: Lsn) {
    6255     10558184 :         self.tl.finish_write(new_lsn);
    6256     10558184 :     }
    6257              : 
    6258       541140 :     pub(crate) fn update_current_logical_size(&self, delta: i64) {
    6259       541140 :         self.tl.update_current_logical_size(delta)
    6260       541140 :     }
    6261              : }
    6262              : 
    6263              : // We need TimelineWriter to be send in upcoming conversion of
    6264              : // Timeline::layers to tokio::sync::RwLock.
    6265              : #[test]
    6266            4 : fn is_send() {
    6267            4 :     fn _assert_send<T: Send>() {}
    6268            4 :     _assert_send::<TimelineWriter<'_>>();
    6269            4 : }
    6270              : 
    6271              : #[cfg(test)]
    6272              : mod tests {
    6273              :     use pageserver_api::key::Key;
    6274              :     use pageserver_api::value::Value;
    6275              :     use utils::{id::TimelineId, lsn::Lsn};
    6276              : 
    6277              :     use crate::tenant::{
    6278              :         harness::{test_img, TenantHarness},
    6279              :         layer_map::LayerMap,
    6280              :         storage_layer::{Layer, LayerName},
    6281              :         timeline::{DeltaLayerTestDesc, EvictionError},
    6282              :         Timeline,
    6283              :     };
    6284              : 
    6285              :     #[tokio::test]
    6286            4 :     async fn test_heatmap_generation() {
    6287            4 :         let harness = TenantHarness::create("heatmap_generation").await.unwrap();
    6288            4 : 
    6289            4 :         let covered_delta = DeltaLayerTestDesc::new_with_inferred_key_range(
    6290            4 :             Lsn(0x10)..Lsn(0x20),
    6291            4 :             vec![(
    6292            4 :                 Key::from_hex("620000000033333333444444445500000000").unwrap(),
    6293            4 :                 Lsn(0x11),
    6294            4 :                 Value::Image(test_img("foo")),
    6295            4 :             )],
    6296            4 :         );
    6297            4 :         let visible_delta = DeltaLayerTestDesc::new_with_inferred_key_range(
    6298            4 :             Lsn(0x10)..Lsn(0x20),
    6299            4 :             vec![(
    6300            4 :                 Key::from_hex("720000000033333333444444445500000000").unwrap(),
    6301            4 :                 Lsn(0x11),
    6302            4 :                 Value::Image(test_img("foo")),
    6303            4 :             )],
    6304            4 :         );
    6305            4 :         let l0_delta = DeltaLayerTestDesc::new(
    6306            4 :             Lsn(0x20)..Lsn(0x30),
    6307            4 :             Key::from_hex("000000000000000000000000000000000000").unwrap()
    6308            4 :                 ..Key::from_hex("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF").unwrap(),
    6309            4 :             vec![(
    6310            4 :                 Key::from_hex("720000000033333333444444445500000000").unwrap(),
    6311            4 :                 Lsn(0x25),
    6312            4 :                 Value::Image(test_img("foo")),
    6313            4 :             )],
    6314            4 :         );
    6315            4 :         let delta_layers = vec![
    6316            4 :             covered_delta.clone(),
    6317            4 :             visible_delta.clone(),
    6318            4 :             l0_delta.clone(),
    6319            4 :         ];
    6320            4 : 
    6321            4 :         let image_layer = (
    6322            4 :             Lsn(0x40),
    6323            4 :             vec![(
    6324            4 :                 Key::from_hex("620000000033333333444444445500000000").unwrap(),
    6325            4 :                 test_img("bar"),
    6326            4 :             )],
    6327            4 :         );
    6328            4 :         let image_layers = vec![image_layer];
    6329            4 : 
    6330            4 :         let (tenant, ctx) = harness.load().await;
    6331            4 :         let timeline = tenant
    6332            4 :             .create_test_timeline_with_layers(
    6333            4 :                 TimelineId::generate(),
    6334            4 :                 Lsn(0x10),
    6335            4 :                 14,
    6336            4 :                 &ctx,
    6337            4 :                 delta_layers,
    6338            4 :                 image_layers,
    6339            4 :                 Lsn(0x100),
    6340            4 :             )
    6341            4 :             .await
    6342            4 :             .unwrap();
    6343            4 : 
    6344            4 :         // Layer visibility is an input to heatmap generation, so refresh it first
    6345            4 :         timeline.update_layer_visibility().await.unwrap();
    6346            4 : 
    6347            4 :         let heatmap = timeline
    6348            4 :             .generate_heatmap()
    6349            4 :             .await
    6350            4 :             .expect("Infallible while timeline is not shut down");
    6351            4 : 
    6352            4 :         assert_eq!(heatmap.timeline_id, timeline.timeline_id);
    6353            4 : 
    6354            4 :         // L0 should come last
    6355            4 :         assert_eq!(heatmap.layers.last().unwrap().name, l0_delta.layer_name());
    6356            4 : 
    6357            4 :         let mut last_lsn = Lsn::MAX;
    6358           20 :         for layer in heatmap.layers {
    6359            4 :             // Covered layer should be omitted
    6360           16 :             assert!(layer.name != covered_delta.layer_name());
    6361            4 : 
    6362           16 :             let layer_lsn = match &layer.name {
    6363            8 :                 LayerName::Delta(d) => d.lsn_range.end,
    6364            8 :                 LayerName::Image(i) => i.lsn,
    6365            4 :             };
    6366            4 : 
    6367            4 :             // Apart from L0s, newest Layers should come first
    6368           16 :             if !LayerMap::is_l0(layer.name.key_range(), layer.name.is_delta()) {
    6369           12 :                 assert!(layer_lsn <= last_lsn);
    6370           12 :                 last_lsn = layer_lsn;
    6371            4 :             }
    6372            4 :         }
    6373            4 :     }
    6374              : 
    6375              :     #[tokio::test]
    6376            4 :     async fn two_layer_eviction_attempts_at_the_same_time() {
    6377            4 :         let harness = TenantHarness::create("two_layer_eviction_attempts_at_the_same_time")
    6378            4 :             .await
    6379            4 :             .unwrap();
    6380            4 : 
    6381            4 :         let (tenant, ctx) = harness.load().await;
    6382            4 :         let timeline = tenant
    6383            4 :             .create_test_timeline(TimelineId::generate(), Lsn(0x10), 14, &ctx)
    6384            4 :             .await
    6385            4 :             .unwrap();
    6386            4 : 
    6387            4 :         let layer = find_some_layer(&timeline).await;
    6388            4 :         let layer = layer
    6389            4 :             .keep_resident()
    6390            4 :             .await
    6391            4 :             .expect("no download => no downloading errors")
    6392            4 :             .drop_eviction_guard();
    6393            4 : 
    6394            4 :         let forever = std::time::Duration::from_secs(120);
    6395            4 : 
    6396            4 :         let first = layer.evict_and_wait(forever);
    6397            4 :         let second = layer.evict_and_wait(forever);
    6398            4 : 
    6399            4 :         let (first, second) = tokio::join!(first, second);
    6400            4 : 
    6401            4 :         let res = layer.keep_resident().await;
    6402            4 :         assert!(res.is_none(), "{res:?}");
    6403            4 : 
    6404            4 :         match (first, second) {
    6405            4 :             (Ok(()), Ok(())) => {
    6406            4 :                 // because there are no more timeline locks being taken on eviction path, we can
    6407            4 :                 // witness all three outcomes here.
    6408            4 :             }
    6409            4 :             (Ok(()), Err(EvictionError::NotFound)) | (Err(EvictionError::NotFound), Ok(())) => {
    6410            0 :                 // if one completes before the other, this is fine just as well.
    6411            0 :             }
    6412            4 :             other => unreachable!("unexpected {:?}", other),
    6413            4 :         }
    6414            4 :     }
    6415              : 
    6416            4 :     async fn find_some_layer(timeline: &Timeline) -> Layer {
    6417            4 :         let layers = timeline.layers.read().await;
    6418            4 :         let desc = layers
    6419            4 :             .layer_map()
    6420            4 :             .unwrap()
    6421            4 :             .iter_historic_layers()
    6422            4 :             .next()
    6423            4 :             .expect("must find one layer to evict");
    6424            4 : 
    6425            4 :         layers.get_from_desc(&desc)
    6426            4 :     }
    6427              : }
        

Generated by: LCOV version 2.1-beta