LCOV - code coverage report
Current view: top level - pageserver/src - disk_usage_eviction_task.rs (source / functions) Coverage Total Hit
Test: 1b0a6a0c05cee5a7de360813c8034804e105ce1c.info Lines: 16.6 % 657 109
Test Date: 2025-03-12 00:01:28 Functions: 23.1 % 65 15

            Line data    Source code
       1              : //! This module implements the pageserver-global disk-usage-based layer eviction task.
       2              : //!
       3              : //! # Mechanics
       4              : //!
       5              : //! Function `launch_disk_usage_global_eviction_task` starts a pageserver-global background
       6              : //! loop that evicts layers in response to a shortage of available bytes
       7              : //! in the $repo/tenants directory's filesystem.
       8              : //!
       9              : //! The loop runs periodically at a configurable `period`.
      10              : //!
      11              : //! Each loop iteration uses `statvfs` to determine filesystem-level space usage.
      12              : //! It compares the returned usage data against two different types of thresholds.
      13              : //! The iteration tries to evict layers until app-internal accounting says we should be below the thresholds.
      14              : //! We cross-check this internal accounting with the real world by making another `statvfs` at the end of the iteration.
      15              : //! We're good if that second statvfs shows that we're _actually_ below the configured thresholds.
      16              : //! If we're still above one or more thresholds, we emit a warning log message, leaving it to the operator to investigate further.
      17              : //!
      18              : //! # Eviction Policy
      19              : //!
      20              : //! There are two thresholds:
      21              : //! `max_usage_pct` is the relative available space, expressed in percent of the total filesystem space.
      22              : //! If the actual usage is higher, the threshold is exceeded.
      23              : //! `min_avail_bytes` is the absolute available space in bytes.
      24              : //! If the actual usage is lower, the threshold is exceeded.
      25              : //! If either of these thresholds is exceeded, the system is considered to have "disk pressure", and eviction
      26              : //! is performed on the next iteration, to release disk space and bring the usage below the thresholds again.
      27              : //! The iteration evicts layers in LRU fashion, but, with a weak reservation per tenant.
      28              : //! The reservation is to keep the most recently accessed X bytes per tenant resident.
      29              : //! If we cannot relieve pressure by evicting layers outside of the reservation, we
      30              : //! start evicting layers that are part of the reservation, LRU first.
      31              : //!
      32              : //! The value for the per-tenant reservation is referred to as `tenant_min_resident_size`
      33              : //! throughout the code, but, no actual variable carries that name.
      34              : //! The per-tenant default value is the `max(tenant's layer file sizes, regardless of local or remote)`.
      35              : //! The idea is to allow at least one layer to be resident per tenant, to ensure it can make forward progress
      36              : //! during page reconstruction.
      37              : //! An alternative default for all tenants can be specified in the `tenant_config` section of the config.
      38              : //! Lastly, each tenant can have an override in their respective tenant config (`min_resident_size_override`).
      39              : 
      40              : // Implementation notes:
      41              : // - The `#[allow(dead_code)]` above various structs are to suppress warnings about only the Debug impl
      42              : //   reading these fields. We use the Debug impl for semi-structured logging, though.
      43              : 
      44              : use std::sync::Arc;
      45              : use std::time::SystemTime;
      46              : 
      47              : use anyhow::Context;
      48              : use pageserver_api::config::DiskUsageEvictionTaskConfig;
      49              : use pageserver_api::shard::TenantShardId;
      50              : use remote_storage::GenericRemoteStorage;
      51              : use serde::Serialize;
      52              : use tokio::time::Instant;
      53              : use tokio_util::sync::CancellationToken;
      54              : use tracing::{Instrument, debug, error, info, instrument, warn};
      55              : use utils::completion;
      56              : use utils::id::TimelineId;
      57              : 
      58              : use crate::config::PageServerConf;
      59              : use crate::metrics::disk_usage_based_eviction::METRICS;
      60              : use crate::task_mgr::{self, BACKGROUND_RUNTIME};
      61              : use crate::tenant::mgr::TenantManager;
      62              : use crate::tenant::remote_timeline_client::LayerFileMetadata;
      63              : use crate::tenant::secondary::SecondaryTenant;
      64              : use crate::tenant::storage_layer::{
      65              :     AsLayerDesc, EvictionError, Layer, LayerName, LayerVisibilityHint,
      66              : };
      67              : use crate::tenant::tasks::sleep_random;
      68              : use crate::{CancellableTask, DiskUsageEvictionTask};
      69              : 
      70              : /// Selects the sort order for eviction candidates *after* per tenant `min_resident_size`
      71              : /// partitioning.
      72              : #[derive(Debug, Clone, Copy, PartialEq, Eq)]
      73              : pub enum EvictionOrder {
      74              :     /// Order the layers to be evicted by how recently they have been accessed relatively within
      75              :     /// the set of resident layers of a tenant.
      76              :     RelativeAccessed {
      77              :         /// Determines if the tenant with most layers should lose first.
      78              :         ///
      79              :         /// Having this enabled is currently the only reasonable option, because the order in which
      80              :         /// we read tenants is deterministic. If we find the need to use this as `false`, we need
      81              :         /// to ensure nondeterminism by adding in a random number to break the
      82              :         /// `relative_last_activity==0.0` ties.
      83              :         highest_layer_count_loses_first: bool,
      84              :     },
      85              : }
      86              : 
      87              : impl From<pageserver_api::config::EvictionOrder> for EvictionOrder {
      88            0 :     fn from(value: pageserver_api::config::EvictionOrder) -> Self {
      89            0 :         match value {
      90            0 :             pageserver_api::config::EvictionOrder::RelativeAccessed {
      91            0 :                 highest_layer_count_loses_first,
      92            0 :             } => Self::RelativeAccessed {
      93            0 :                 highest_layer_count_loses_first,
      94            0 :             },
      95            0 :         }
      96            0 :     }
      97              : }
      98              : 
      99              : impl EvictionOrder {
     100            0 :     fn sort(&self, candidates: &mut [(EvictionPartition, EvictionCandidate)]) {
     101              :         use EvictionOrder::*;
     102              : 
     103            0 :         match self {
     104            0 :             RelativeAccessed { .. } => candidates.sort_unstable_by_key(|(partition, candidate)| {
     105            0 :                 (*partition, candidate.relative_last_activity)
     106            0 :             }),
     107            0 :         }
     108            0 :     }
     109              : 
     110              :     /// Called to fill in the [`EvictionCandidate::relative_last_activity`] while iterating tenants
     111              :     /// layers in **most** recently used order.
     112           80 :     fn relative_last_activity(&self, total: usize, index: usize) -> finite_f32::FiniteF32 {
     113              :         use EvictionOrder::*;
     114              : 
     115           80 :         match self {
     116           80 :             RelativeAccessed {
     117           80 :                 highest_layer_count_loses_first,
     118              :             } => {
     119              :                 // keeping the -1 or not decides if every tenant should lose their least recently accessed
     120              :                 // layer OR if this should happen in the order of having highest layer count:
     121           80 :                 let fudge = if *highest_layer_count_loses_first {
     122              :                     // relative_last_activity vs. tenant layer count:
     123              :                     // - 0.1..=1.0 (10 layers)
     124              :                     // - 0.01..=1.0 (100 layers)
     125              :                     // - 0.001..=1.0 (1000 layers)
     126              :                     //
     127              :                     // leading to evicting less of the smallest tenants.
     128           40 :                     0
     129              :                 } else {
     130              :                     // use full 0.0..=1.0 range, which means even the smallest tenants could always lose a
     131              :                     // layer. the actual ordering is unspecified: for 10k tenants on a pageserver it could
     132              :                     // be that less than 10k layer evictions is enough, so we would not need to evict from
     133              :                     // all tenants.
     134              :                     //
     135              :                     // as the tenant ordering is now deterministic this could hit the same tenants
     136              :                     // disproportionetly on multiple invocations. alternative could be to remember how many
     137              :                     // layers did we evict last time from this tenant, and inject that as an additional
     138              :                     // fudge here.
     139           40 :                     1
     140              :                 };
     141              : 
     142           80 :                 let total = total.checked_sub(fudge).filter(|&x| x > 1).unwrap_or(1);
     143           80 :                 let divider = total as f32;
     144           80 : 
     145           80 :                 // most recently used is always (total - 0) / divider == 1.0
     146           80 :                 // least recently used depends on the fudge:
     147           80 :                 // -       (total - 1) - (total - 1) / total => 0 / total
     148           80 :                 // -             total - (total - 1) / total => 1 / total
     149           80 :                 let distance = (total - index) as f32;
     150           80 : 
     151           80 :                 finite_f32::FiniteF32::try_from_normalized(distance / divider)
     152           80 :                     .unwrap_or_else(|val| {
     153            0 :                         tracing::warn!(%fudge, "calculated invalid relative_last_activity for i={index}, total={total}: {val}");
     154            0 :                         finite_f32::FiniteF32::ZERO
     155           80 :                     })
     156           80 :             }
     157           80 :         }
     158           80 :     }
     159              : }
     160              : 
     161              : #[derive(Default)]
     162              : pub struct State {
     163              :     /// Exclude http requests and background task from running at the same time.
     164              :     mutex: tokio::sync::Mutex<()>,
     165              : }
     166              : 
     167            0 : pub fn launch_disk_usage_global_eviction_task(
     168            0 :     conf: &'static PageServerConf,
     169            0 :     storage: GenericRemoteStorage,
     170            0 :     state: Arc<State>,
     171            0 :     tenant_manager: Arc<TenantManager>,
     172            0 :     background_jobs_barrier: completion::Barrier,
     173            0 : ) -> Option<DiskUsageEvictionTask> {
     174            0 :     let Some(task_config) = &conf.disk_usage_based_eviction else {
     175            0 :         info!("disk usage based eviction task not configured");
     176            0 :         return None;
     177              :     };
     178              : 
     179            0 :     info!("launching disk usage based eviction task");
     180              : 
     181            0 :     let cancel = CancellationToken::new();
     182            0 :     let task = BACKGROUND_RUNTIME.spawn(task_mgr::exit_on_panic_or_error(
     183            0 :         "disk usage based eviction",
     184            0 :         {
     185            0 :             let cancel = cancel.clone();
     186            0 :             async move {
     187            0 :                 // wait until initial load is complete, because we cannot evict from loading tenants.
     188            0 :                 tokio::select! {
     189            0 :                     _ = cancel.cancelled() => { return anyhow::Ok(()); },
     190            0 :                     _ = background_jobs_barrier.wait() => { }
     191            0 :                 };
     192            0 : 
     193            0 :                 disk_usage_eviction_task(&state, task_config, &storage, tenant_manager, cancel)
     194            0 :                     .await;
     195            0 :                 anyhow::Ok(())
     196            0 :             }
     197            0 :         },
     198            0 :     ));
     199            0 : 
     200            0 :     Some(DiskUsageEvictionTask(CancellableTask { cancel, task }))
     201            0 : }
     202              : 
     203              : #[instrument(skip_all)]
     204              : async fn disk_usage_eviction_task(
     205              :     state: &State,
     206              :     task_config: &DiskUsageEvictionTaskConfig,
     207              :     storage: &GenericRemoteStorage,
     208              :     tenant_manager: Arc<TenantManager>,
     209              :     cancel: CancellationToken,
     210              : ) {
     211              :     scopeguard::defer! {
     212              :         info!("disk usage based eviction task finishing");
     213              :     };
     214              : 
     215              :     if sleep_random(task_config.period, &cancel).await.is_err() {
     216              :         return;
     217              :     }
     218              : 
     219              :     let mut iteration_no = 0;
     220              :     loop {
     221              :         iteration_no += 1;
     222              :         let start = Instant::now();
     223              : 
     224            0 :         async {
     225            0 :             let res = disk_usage_eviction_task_iteration(
     226            0 :                 state,
     227            0 :                 task_config,
     228            0 :                 storage,
     229            0 :                 &tenant_manager,
     230            0 :                 &cancel,
     231            0 :             )
     232            0 :             .await;
     233              : 
     234            0 :             match res {
     235            0 :                 Ok(()) => {}
     236            0 :                 Err(e) => {
     237            0 :                     // these stat failures are expected to be very rare
     238            0 :                     warn!("iteration failed, unexpected error: {e:#}");
     239              :                 }
     240              :             }
     241            0 :         }
     242              :         .instrument(tracing::info_span!("iteration", iteration_no))
     243              :         .await;
     244              : 
     245              :         let sleep_until = start + task_config.period;
     246              :         if tokio::time::timeout_at(sleep_until, cancel.cancelled())
     247              :             .await
     248              :             .is_ok()
     249              :         {
     250              :             break;
     251              :         }
     252              :     }
     253              : }
     254              : 
     255              : pub trait Usage: Clone + Copy + std::fmt::Debug {
     256              :     fn has_pressure(&self) -> bool;
     257              :     fn add_available_bytes(&mut self, bytes: u64);
     258              : }
     259              : 
     260            0 : async fn disk_usage_eviction_task_iteration(
     261            0 :     state: &State,
     262            0 :     task_config: &DiskUsageEvictionTaskConfig,
     263            0 :     storage: &GenericRemoteStorage,
     264            0 :     tenant_manager: &Arc<TenantManager>,
     265            0 :     cancel: &CancellationToken,
     266            0 : ) -> anyhow::Result<()> {
     267            0 :     let tenants_dir = tenant_manager.get_conf().tenants_path();
     268            0 :     let usage_pre = filesystem_level_usage::get(&tenants_dir, task_config)
     269            0 :         .context("get filesystem-level disk usage before evictions")?;
     270            0 :     let res = disk_usage_eviction_task_iteration_impl(
     271            0 :         state,
     272            0 :         storage,
     273            0 :         usage_pre,
     274            0 :         tenant_manager,
     275            0 :         task_config.eviction_order.into(),
     276            0 :         cancel,
     277            0 :     )
     278            0 :     .await;
     279            0 :     match res {
     280            0 :         Ok(outcome) => {
     281            0 :             debug!(?outcome, "disk_usage_eviction_iteration finished");
     282            0 :             match outcome {
     283            0 :                 IterationOutcome::NoPressure | IterationOutcome::Cancelled => {
     284            0 :                     // nothing to do, select statement below will handle things
     285            0 :                 }
     286            0 :                 IterationOutcome::Finished(outcome) => {
     287              :                     // Verify with statvfs whether we made any real progress
     288            0 :                     let after = filesystem_level_usage::get(&tenants_dir, task_config)
     289            0 :                         // It's quite unlikely to hit the error here. Keep the code simple and bail out.
     290            0 :                         .context("get filesystem-level disk usage after evictions")?;
     291              : 
     292            0 :                     debug!(?after, "disk usage");
     293              : 
     294            0 :                     if after.has_pressure() {
     295              :                         // Don't bother doing an out-of-order iteration here now.
     296              :                         // In practice, the task period is set to a value in the tens-of-seconds range,
     297              :                         // which will cause another iteration to happen soon enough.
     298              :                         // TODO: deltas between the three different usages would be helpful,
     299              :                         // consider MiB, GiB, TiB
     300            0 :                         warn!(?outcome, ?after, "disk usage still high");
     301              :                     } else {
     302            0 :                         info!(?outcome, ?after, "disk usage pressure relieved");
     303              :                     }
     304              :                 }
     305              :             }
     306              :         }
     307            0 :         Err(e) => {
     308            0 :             error!("disk_usage_eviction_iteration failed: {:#}", e);
     309              :         }
     310              :     }
     311              : 
     312            0 :     Ok(())
     313            0 : }
     314              : 
     315              : #[derive(Debug, Serialize)]
     316              : #[allow(clippy::large_enum_variant)]
     317              : pub enum IterationOutcome<U> {
     318              :     NoPressure,
     319              :     Cancelled,
     320              :     Finished(IterationOutcomeFinished<U>),
     321              : }
     322              : 
     323              : #[derive(Debug, Serialize)]
     324              : pub struct IterationOutcomeFinished<U> {
     325              :     /// The actual usage observed before we started the iteration.
     326              :     before: U,
     327              :     /// The expected value for `after`, according to internal accounting, after phase 1.
     328              :     planned: PlannedUsage<U>,
     329              :     /// The outcome of phase 2, where we actually do the evictions.
     330              :     ///
     331              :     /// If all layers that phase 1 planned to evict _can_ actually get evicted, this will
     332              :     /// be the same as `planned`.
     333              :     assumed: AssumedUsage<U>,
     334              : }
     335              : 
     336              : #[derive(Debug, Serialize)]
     337              : struct AssumedUsage<U> {
     338              :     /// The expected value for `after`, after phase 2.
     339              :     projected_after: U,
     340              :     /// The layers we failed to evict during phase 2.
     341              :     failed: LayerCount,
     342              : }
     343              : 
     344              : #[derive(Debug, Serialize)]
     345              : struct PlannedUsage<U> {
     346              :     respecting_tenant_min_resident_size: U,
     347              :     fallback_to_global_lru: Option<U>,
     348              : }
     349              : 
     350              : #[derive(Debug, Default, Serialize)]
     351              : struct LayerCount {
     352              :     file_sizes: u64,
     353              :     count: usize,
     354              : }
     355              : 
     356            0 : pub(crate) async fn disk_usage_eviction_task_iteration_impl<U: Usage>(
     357            0 :     state: &State,
     358            0 :     _storage: &GenericRemoteStorage,
     359            0 :     usage_pre: U,
     360            0 :     tenant_manager: &Arc<TenantManager>,
     361            0 :     eviction_order: EvictionOrder,
     362            0 :     cancel: &CancellationToken,
     363            0 : ) -> anyhow::Result<IterationOutcome<U>> {
     364              :     // use tokio's mutex to get a Sync guard (instead of std::sync::Mutex)
     365            0 :     let _g = state
     366            0 :         .mutex
     367            0 :         .try_lock()
     368            0 :         .map_err(|_| anyhow::anyhow!("iteration is already executing"))?;
     369              : 
     370            0 :     debug!(?usage_pre, "disk usage");
     371              : 
     372            0 :     if !usage_pre.has_pressure() {
     373            0 :         return Ok(IterationOutcome::NoPressure);
     374            0 :     }
     375            0 : 
     376            0 :     warn!(
     377              :         ?usage_pre,
     378            0 :         "running disk usage based eviction due to pressure"
     379              :     );
     380              : 
     381            0 :     let (candidates, collection_time) = {
     382            0 :         let started_at = std::time::Instant::now();
     383            0 :         match collect_eviction_candidates(tenant_manager, eviction_order, cancel).await? {
     384              :             EvictionCandidates::Cancelled => {
     385            0 :                 return Ok(IterationOutcome::Cancelled);
     386              :             }
     387            0 :             EvictionCandidates::Finished(partitioned) => (partitioned, started_at.elapsed()),
     388            0 :         }
     389            0 :     };
     390            0 : 
     391            0 :     METRICS.layers_collected.inc_by(candidates.len() as u64);
     392            0 : 
     393            0 :     tracing::info!(
     394            0 :         elapsed_ms = collection_time.as_millis(),
     395            0 :         total_layers = candidates.len(),
     396            0 :         "collection completed"
     397              :     );
     398              : 
     399              :     // Debug-log the list of candidates
     400            0 :     let now = SystemTime::now();
     401            0 :     for (i, (partition, candidate)) in candidates.iter().enumerate() {
     402            0 :         let nth = i + 1;
     403            0 :         let total_candidates = candidates.len();
     404            0 :         let size = candidate.layer.get_file_size();
     405            0 :         let rel = candidate.relative_last_activity;
     406            0 :         debug!(
     407            0 :             "cand {nth}/{total_candidates}: size={size}, rel_last_activity={rel}, no_access_for={}us, partition={partition:?}, {}/{}/{}",
     408            0 :             now.duration_since(candidate.last_activity_ts)
     409            0 :                 .unwrap()
     410            0 :                 .as_micros(),
     411            0 :             candidate.layer.get_tenant_shard_id(),
     412            0 :             candidate.layer.get_timeline_id(),
     413            0 :             candidate.layer.get_name(),
     414              :         );
     415              :     }
     416              : 
     417              :     // phase1: select victims to relieve pressure
     418              :     //
     419              :     // Walk through the list of candidates, until we have accumulated enough layers to get
     420              :     // us back under the pressure threshold. 'usage_planned' is updated so that it tracks
     421              :     // how much disk space would be used after evicting all the layers up to the current
     422              :     // point in the list.
     423              :     //
     424              :     // If we get far enough in the list that we start to evict layers that are below
     425              :     // the tenant's min-resident-size threshold, print a warning, and memorize the disk
     426              :     // usage at that point, in 'usage_planned_min_resident_size_respecting'.
     427              : 
     428            0 :     let (evicted_amount, usage_planned) =
     429            0 :         select_victims(&candidates, usage_pre).into_amount_and_planned();
     430            0 : 
     431            0 :     METRICS.layers_selected.inc_by(evicted_amount as u64);
     432            0 : 
     433            0 :     // phase2: evict layers
     434            0 : 
     435            0 :     let mut js = tokio::task::JoinSet::new();
     436            0 :     let limit = 1000;
     437            0 : 
     438            0 :     let mut evicted = candidates.into_iter().take(evicted_amount).fuse();
     439            0 :     let mut consumed_all = false;
     440            0 : 
     441            0 :     // After the evictions, `usage_assumed` is the post-eviction usage,
     442            0 :     // according to internal accounting.
     443            0 :     let mut usage_assumed = usage_pre;
     444            0 :     let mut evictions_failed = LayerCount::default();
     445            0 : 
     446            0 :     let evict_layers = async move {
     447              :         loop {
     448            0 :             let next = if js.len() >= limit || consumed_all {
     449            0 :                 js.join_next().await
     450            0 :             } else if !js.is_empty() {
     451              :                 // opportunistically consume ready result, one per each new evicted
     452            0 :                 futures::future::FutureExt::now_or_never(js.join_next()).and_then(|x| x)
     453              :             } else {
     454            0 :                 None
     455              :             };
     456              : 
     457            0 :             if let Some(next) = next {
     458            0 :                 match next {
     459            0 :                     Ok(Ok(file_size)) => {
     460            0 :                         METRICS.layers_evicted.inc();
     461            0 :                         usage_assumed.add_available_bytes(file_size);
     462            0 :                     }
     463              :                     Ok(Err((
     464            0 :                         file_size,
     465            0 :                         EvictionError::NotFound
     466            0 :                         | EvictionError::Downloaded
     467            0 :                         | EvictionError::Timeout,
     468            0 :                     ))) => {
     469            0 :                         evictions_failed.file_sizes += file_size;
     470            0 :                         evictions_failed.count += 1;
     471            0 :                     }
     472            0 :                     Err(je) if je.is_cancelled() => unreachable!("not used"),
     473            0 :                     Err(je) if je.is_panic() => { /* already logged */ }
     474            0 :                     Err(je) => tracing::error!("unknown JoinError: {je:?}"),
     475              :                 }
     476            0 :             }
     477              : 
     478            0 :             if consumed_all && js.is_empty() {
     479            0 :                 break;
     480            0 :             }
     481              : 
     482              :             // calling again when consumed_all is fine as evicted is fused.
     483            0 :             let Some((_partition, candidate)) = evicted.next() else {
     484            0 :                 if !consumed_all {
     485            0 :                     tracing::info!("all evictions started, waiting");
     486            0 :                     consumed_all = true;
     487            0 :                 }
     488            0 :                 continue;
     489              :             };
     490              : 
     491            0 :             match candidate.layer {
     492            0 :                 EvictionLayer::Attached(layer) => {
     493            0 :                     let file_size = layer.layer_desc().file_size;
     494            0 :                     js.spawn(async move {
     495            0 :                         // have a low eviction waiting timeout because our LRU calculations go stale fast;
     496            0 :                         // also individual layer evictions could hang because of bugs and we do not want to
     497            0 :                         // pause disk_usage_based_eviction for such.
     498            0 :                         let timeout = std::time::Duration::from_secs(5);
     499            0 : 
     500            0 :                         match layer.evict_and_wait(timeout).await {
     501            0 :                             Ok(()) => Ok(file_size),
     502            0 :                             Err(e) => Err((file_size, e)),
     503              :                         }
     504            0 :                     });
     505            0 :                 }
     506            0 :                 EvictionLayer::Secondary(layer) => {
     507            0 :                     let file_size = layer.metadata.file_size;
     508            0 : 
     509            0 :                     js.spawn(async move {
     510            0 :                         layer
     511            0 :                             .secondary_tenant
     512            0 :                             .evict_layer(layer.timeline_id, layer.name)
     513            0 :                             .await;
     514            0 :                         Ok(file_size)
     515            0 :                     });
     516            0 :                 }
     517              :             }
     518            0 :             tokio::task::yield_now().await;
     519              :         }
     520              : 
     521            0 :         (usage_assumed, evictions_failed)
     522            0 :     };
     523              : 
     524            0 :     let started_at = std::time::Instant::now();
     525            0 : 
     526            0 :     let evict_layers = async move {
     527            0 :         let mut evict_layers = std::pin::pin!(evict_layers);
     528            0 : 
     529            0 :         let maximum_expected = std::time::Duration::from_secs(10);
     530              : 
     531            0 :         let res = tokio::time::timeout(maximum_expected, &mut evict_layers).await;
     532            0 :         let tuple = if let Ok(tuple) = res {
     533            0 :             tuple
     534              :         } else {
     535            0 :             let elapsed = started_at.elapsed();
     536            0 :             tracing::info!(elapsed_ms = elapsed.as_millis(), "still ongoing");
     537            0 :             evict_layers.await
     538              :         };
     539              : 
     540            0 :         let elapsed = started_at.elapsed();
     541            0 :         tracing::info!(elapsed_ms = elapsed.as_millis(), "completed");
     542            0 :         tuple
     543            0 :     };
     544              : 
     545            0 :     let evict_layers =
     546            0 :         evict_layers.instrument(tracing::info_span!("evict_layers", layers=%evicted_amount));
     547              : 
     548            0 :     let (usage_assumed, evictions_failed) = tokio::select! {
     549            0 :         tuple = evict_layers => { tuple },
     550            0 :         _ = cancel.cancelled() => {
     551              :             // dropping joinset will abort all pending evict_and_waits and that is fine, our
     552              :             // requests will still stand
     553            0 :             return Ok(IterationOutcome::Cancelled);
     554              :         }
     555              :     };
     556              : 
     557            0 :     Ok(IterationOutcome::Finished(IterationOutcomeFinished {
     558            0 :         before: usage_pre,
     559            0 :         planned: usage_planned,
     560            0 :         assumed: AssumedUsage {
     561            0 :             projected_after: usage_assumed,
     562            0 :             failed: evictions_failed,
     563            0 :         },
     564            0 :     }))
     565            0 : }
     566              : 
     567              : #[derive(Clone)]
     568              : pub(crate) struct EvictionSecondaryLayer {
     569              :     pub(crate) secondary_tenant: Arc<SecondaryTenant>,
     570              :     pub(crate) timeline_id: TimelineId,
     571              :     pub(crate) name: LayerName,
     572              :     pub(crate) metadata: LayerFileMetadata,
     573              : }
     574              : 
     575              : /// Full [`Layer`] objects are specific to tenants in attached mode.  This type is a layer
     576              : /// of indirection to store either a `Layer`, or a reference to a secondary tenant and a layer name.
     577              : #[derive(Clone)]
     578              : pub(crate) enum EvictionLayer {
     579              :     Attached(Layer),
     580              :     Secondary(EvictionSecondaryLayer),
     581              : }
     582              : 
     583              : impl From<Layer> for EvictionLayer {
     584            0 :     fn from(value: Layer) -> Self {
     585            0 :         Self::Attached(value)
     586            0 :     }
     587              : }
     588              : 
     589              : impl EvictionLayer {
     590            0 :     pub(crate) fn get_tenant_shard_id(&self) -> &TenantShardId {
     591            0 :         match self {
     592            0 :             Self::Attached(l) => &l.layer_desc().tenant_shard_id,
     593            0 :             Self::Secondary(sl) => sl.secondary_tenant.get_tenant_shard_id(),
     594              :         }
     595            0 :     }
     596              : 
     597            0 :     pub(crate) fn get_timeline_id(&self) -> &TimelineId {
     598            0 :         match self {
     599            0 :             Self::Attached(l) => &l.layer_desc().timeline_id,
     600            0 :             Self::Secondary(sl) => &sl.timeline_id,
     601              :         }
     602            0 :     }
     603              : 
     604            0 :     pub(crate) fn get_name(&self) -> LayerName {
     605            0 :         match self {
     606            0 :             Self::Attached(l) => l.layer_desc().layer_name(),
     607            0 :             Self::Secondary(sl) => sl.name.clone(),
     608              :         }
     609            0 :     }
     610              : 
     611            0 :     pub(crate) fn get_file_size(&self) -> u64 {
     612            0 :         match self {
     613            0 :             Self::Attached(l) => l.layer_desc().file_size,
     614            0 :             Self::Secondary(sl) => sl.metadata.file_size,
     615              :         }
     616            0 :     }
     617              : }
     618              : 
     619              : #[derive(Clone)]
     620              : pub(crate) struct EvictionCandidate {
     621              :     pub(crate) layer: EvictionLayer,
     622              :     pub(crate) last_activity_ts: SystemTime,
     623              :     pub(crate) relative_last_activity: finite_f32::FiniteF32,
     624              :     pub(crate) visibility: LayerVisibilityHint,
     625              : }
     626              : 
     627              : impl std::fmt::Display for EvictionLayer {
     628            0 :     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
     629            0 :         match self {
     630            0 :             Self::Attached(l) => l.fmt(f),
     631            0 :             Self::Secondary(sl) => {
     632            0 :                 write!(f, "{}/{}", sl.timeline_id, sl.name)
     633              :             }
     634              :         }
     635            0 :     }
     636              : }
     637              : 
     638              : #[derive(Default)]
     639              : pub(crate) struct DiskUsageEvictionInfo {
     640              :     /// Timeline's largest layer (remote or resident)
     641              :     pub max_layer_size: Option<u64>,
     642              :     /// Timeline's resident layers
     643              :     pub resident_layers: Vec<EvictionCandidate>,
     644              : }
     645              : 
     646              : impl std::fmt::Debug for EvictionCandidate {
     647            0 :     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
     648            0 :         // format the tv_sec, tv_nsec into rfc3339 in case someone is looking at it
     649            0 :         // having to allocate a string to this is bad, but it will rarely be formatted
     650            0 :         let ts = chrono::DateTime::<chrono::Utc>::from(self.last_activity_ts);
     651            0 :         let ts = ts.to_rfc3339_opts(chrono::SecondsFormat::Nanos, true);
     652              :         struct DisplayIsDebug<'a, T>(&'a T);
     653              :         impl<T: std::fmt::Display> std::fmt::Debug for DisplayIsDebug<'_, T> {
     654            0 :             fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
     655            0 :                 write!(f, "{}", self.0)
     656            0 :             }
     657              :         }
     658            0 :         f.debug_struct("LocalLayerInfoForDiskUsageEviction")
     659            0 :             .field("layer", &DisplayIsDebug(&self.layer))
     660            0 :             .field("last_activity", &ts)
     661            0 :             .finish()
     662            0 :     }
     663              : }
     664              : 
     665              : #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
     666              : enum EvictionPartition {
     667              :     // A layer that is un-wanted by the tenant: evict all these first, before considering
     668              :     // any other layers
     669              :     EvictNow,
     670              : 
     671              :     // Above the minimum size threshold: this layer is a candidate for eviction.
     672              :     Above,
     673              : 
     674              :     // Below the minimum size threshold: this layer should only be evicted if all the
     675              :     // tenants' layers above the minimum size threshold have already been considered.
     676              :     Below,
     677              : }
     678              : 
     679              : enum EvictionCandidates {
     680              :     Cancelled,
     681              :     Finished(Vec<(EvictionPartition, EvictionCandidate)>),
     682              : }
     683              : 
     684              : /// Gather the eviction candidates.
     685              : ///
     686              : /// The returned `Ok(EvictionCandidates::Finished(candidates))` is sorted in eviction
     687              : /// order. A caller that evicts in that order, until pressure is relieved, implements
     688              : /// the eviction policy outlined in the module comment.
     689              : ///
     690              : /// # Example with EvictionOrder::AbsoluteAccessed
     691              : ///
     692              : /// Imagine that there are two tenants, A and B, with five layers each, a-e.
     693              : /// Each layer has size 100, and both tenant's min_resident_size is 150.
     694              : /// The eviction order would be
     695              : ///
     696              : /// ```text
     697              : /// partition last_activity_ts tenant/layer
     698              : /// Above     18:30            A/c
     699              : /// Above     19:00            A/b
     700              : /// Above     18:29            B/c
     701              : /// Above     19:05            B/b
     702              : /// Above     20:00            B/a
     703              : /// Above     20:03            A/a
     704              : /// Below     20:30            A/d
     705              : /// Below     20:40            B/d
     706              : /// Below     20:45            B/e
     707              : /// Below     20:58            A/e
     708              : /// ```
     709              : ///
     710              : /// Now, if we need to evict 300 bytes to relieve pressure, we'd evict `A/c, A/b, B/c`.
     711              : /// They are all in the `Above` partition, so, we respected each tenant's min_resident_size.
     712              : ///
     713              : /// But, if we need to evict 900 bytes to relieve pressure, we'd evict
     714              : /// `A/c, A/b, B/c, B/b, B/a, A/a, A/d, B/d, B/e`, reaching into the `Below` partition
     715              : /// after exhauting the `Above` partition.
     716              : /// So, we did not respect each tenant's min_resident_size.
     717              : ///
     718              : /// # Example with EvictionOrder::RelativeAccessed
     719              : ///
     720              : /// ```text
     721              : /// partition relative_age last_activity_ts tenant/layer
     722              : /// Above     0/4          18:30            A/c
     723              : /// Above     0/4          18:29            B/c
     724              : /// Above     1/4          19:00            A/b
     725              : /// Above     1/4          19:05            B/b
     726              : /// Above     2/4          20:00            B/a
     727              : /// Above     2/4          20:03            A/a
     728              : /// Below     3/4          20:30            A/d
     729              : /// Below     3/4          20:40            B/d
     730              : /// Below     4/4          20:45            B/e
     731              : /// Below     4/4          20:58            A/e
     732              : /// ```
     733              : ///
     734              : /// With tenants having the same number of layers the picture does not change much. The same with
     735              : /// A having many more layers **resident** (not all of them listed):
     736              : ///
     737              : /// ```text
     738              : /// Above       0/100      18:30            A/c
     739              : /// Above       0/4        18:29            B/c
     740              : /// Above       1/100      19:00            A/b
     741              : /// Above       2/100      20:03            A/a
     742              : /// Above       3/100      20:03            A/nth_3
     743              : /// Above       4/100      20:03            A/nth_4
     744              : ///             ...
     745              : /// Above       1/4        19:05            B/b
     746              : /// Above      25/100      20:04            A/nth_25
     747              : ///             ...
     748              : /// Above       2/4        20:00            B/a
     749              : /// Above      50/100      20:10            A/nth_50
     750              : ///             ...
     751              : /// Below       3/4        20:40            B/d
     752              : /// Below      99/100      20:30            A/nth_99
     753              : /// Below       4/4        20:45            B/e
     754              : /// Below     100/100      20:58            A/nth_100
     755              : /// ```
     756              : ///
     757              : /// Now it's easier to see that because A has grown fast it has more layers to get evicted. What is
     758              : /// difficult to see is what happens on the next round assuming the evicting 23 from the above list
     759              : /// relieves the pressure (22 A layers gone, 1 B layers gone) but a new fast growing tenant C has
     760              : /// appeared:
     761              : ///
     762              : /// ```text
     763              : /// Above       0/87       20:04            A/nth_23
     764              : /// Above       0/3        19:05            B/b
     765              : /// Above       0/50       20:59            C/nth_0
     766              : /// Above       1/87       20:04            A/nth_24
     767              : /// Above       1/50       21:00            C/nth_1
     768              : /// Above       2/87       20:04            A/nth_25
     769              : ///             ...
     770              : /// Above      16/50       21:02            C/nth_16
     771              : /// Above       1/3        20:00            B/a
     772              : /// Above      27/87       20:10            A/nth_50
     773              : ///             ...
     774              : /// Below       2/3        20:40            B/d
     775              : /// Below      49/50       21:05            C/nth_49
     776              : /// Below      86/87       20:30            A/nth_99
     777              : /// Below       3/3        20:45            B/e
     778              : /// Below      50/50       21:05            C/nth_50
     779              : /// Below      87/87       20:58            A/nth_100
     780              : /// ```
     781              : ///
     782              : /// Now relieving pressure with 23 layers would cost:
     783              : /// - tenant A 14 layers
     784              : /// - tenant B 1 layer
     785              : /// - tenant C 8 layers
     786            0 : async fn collect_eviction_candidates(
     787            0 :     tenant_manager: &Arc<TenantManager>,
     788            0 :     eviction_order: EvictionOrder,
     789            0 :     cancel: &CancellationToken,
     790            0 : ) -> anyhow::Result<EvictionCandidates> {
     791              :     const LOG_DURATION_THRESHOLD: std::time::Duration = std::time::Duration::from_secs(10);
     792              : 
     793              :     // get a snapshot of the list of tenants
     794            0 :     let tenants = tenant_manager
     795            0 :         .list_tenants()
     796            0 :         .context("get list of tenants")?;
     797              : 
     798              :     // TODO: avoid listing every layer in every tenant: this loop can block the executor,
     799              :     // and the resulting data structure can be huge.
     800              :     // (https://github.com/neondatabase/neon/issues/6224)
     801            0 :     let mut candidates = Vec::new();
     802              : 
     803            0 :     for (tenant_id, _state, _gen) in tenants {
     804            0 :         if cancel.is_cancelled() {
     805            0 :             return Ok(EvictionCandidates::Cancelled);
     806            0 :         }
     807            0 :         let tenant = match tenant_manager.get_attached_tenant_shard(tenant_id) {
     808            0 :             Ok(tenant) if tenant.is_active() => tenant,
     809              :             Ok(_) => {
     810            0 :                 debug!(tenant_id=%tenant_id.tenant_id, shard_id=%tenant_id.shard_slug(), "Tenant shard is not active");
     811            0 :                 continue;
     812              :             }
     813            0 :             Err(e) => {
     814            0 :                 // this can happen if tenant has lifecycle transition after we fetched it
     815            0 :                 debug!("failed to get tenant: {e:#}");
     816            0 :                 continue;
     817              :             }
     818              :         };
     819              : 
     820            0 :         if tenant.cancel.is_cancelled() {
     821            0 :             info!(%tenant_id, "Skipping tenant for eviction, it is shutting down");
     822            0 :             continue;
     823            0 :         }
     824            0 : 
     825            0 :         let started_at = std::time::Instant::now();
     826            0 : 
     827            0 :         // collect layers from all timelines in this tenant
     828            0 :         //
     829            0 :         // If one of the timelines becomes `!is_active()` during the iteration,
     830            0 :         // for example because we're shutting down, then `max_layer_size` can be too small.
     831            0 :         // That's OK. This code only runs under a disk pressure situation, and being
     832            0 :         // a little unfair to tenants during shutdown in such a situation is tolerable.
     833            0 :         let mut tenant_candidates = Vec::new();
     834            0 :         let mut max_layer_size = 0;
     835            0 :         for tl in tenant.list_timelines() {
     836            0 :             if !tl.is_active() {
     837            0 :                 continue;
     838            0 :             }
     839            0 :             let info = tl.get_local_layers_for_disk_usage_eviction().await;
     840            0 :             debug!(tenant_id=%tl.tenant_shard_id.tenant_id, shard_id=%tl.tenant_shard_id.shard_slug(), timeline_id=%tl.timeline_id, "timeline resident layers count: {}", info.resident_layers.len());
     841              : 
     842            0 :             tenant_candidates.extend(info.resident_layers.into_iter());
     843            0 :             max_layer_size = max_layer_size.max(info.max_layer_size.unwrap_or(0));
     844            0 : 
     845            0 :             if cancel.is_cancelled() {
     846            0 :                 return Ok(EvictionCandidates::Cancelled);
     847            0 :             }
     848              :         }
     849              : 
     850              :         // `min_resident_size` defaults to maximum layer file size of the tenant.
     851              :         // This ensures that each tenant can have at least one layer resident at a given time,
     852              :         // ensuring forward progress for a single Timeline::get in that tenant.
     853              :         // It's a questionable heuristic since, usually, there are many Timeline::get
     854              :         // requests going on for a tenant, and, at least in Neon prod, the median
     855              :         // layer file size is much smaller than the compaction target size.
     856              :         // We could be better here, e.g., sum of all L0 layers + most recent L1 layer.
     857              :         // That's what's typically used by the various background loops.
     858              :         //
     859              :         // The default can be overridden with a fixed value in the tenant conf.
     860              :         // A default override can be put in the default tenant conf in the pageserver.toml.
     861            0 :         let min_resident_size = if let Some(s) = tenant.get_min_resident_size_override() {
     862            0 :             debug!(
     863            0 :                 tenant_id=%tenant.tenant_shard_id().tenant_id,
     864            0 :                 shard_id=%tenant.tenant_shard_id().shard_slug(),
     865            0 :                 overridden_size=s,
     866            0 :                 "using overridden min resident size for tenant"
     867              :             );
     868            0 :             s
     869              :         } else {
     870            0 :             debug!(
     871            0 :                 tenant_id=%tenant.tenant_shard_id().tenant_id,
     872            0 :                 shard_id=%tenant.tenant_shard_id().shard_slug(),
     873            0 :                 max_layer_size,
     874            0 :                 "using max layer size as min_resident_size for tenant",
     875              :             );
     876            0 :             max_layer_size
     877              :         };
     878              : 
     879              :         // Sort layers most-recently-used first, then calculate [`EvictionPartition`] for each layer,
     880              :         // where the inputs are:
     881              :         //  - whether the layer is visible
     882              :         //  - whether the layer is above/below the min_resident_size cutline
     883            0 :         tenant_candidates
     884            0 :             .sort_unstable_by_key(|layer_info| std::cmp::Reverse(layer_info.last_activity_ts));
     885            0 :         let mut cumsum: i128 = 0;
     886            0 : 
     887            0 :         let total = tenant_candidates.len();
     888            0 : 
     889            0 :         let tenant_candidates =
     890            0 :             tenant_candidates
     891            0 :                 .into_iter()
     892            0 :                 .enumerate()
     893            0 :                 .map(|(i, mut candidate)| {
     894            0 :                     // as we iterate this reverse sorted list, the most recently accessed layer will always
     895            0 :                     // be 1.0; this is for us to evict it last.
     896            0 :                     candidate.relative_last_activity =
     897            0 :                         eviction_order.relative_last_activity(total, i);
     898              : 
     899            0 :                     let partition = match candidate.visibility {
     900              :                         LayerVisibilityHint::Covered => {
     901              :                             // Covered layers are evicted first
     902            0 :                             EvictionPartition::EvictNow
     903              :                         }
     904              :                         LayerVisibilityHint::Visible => {
     905            0 :                             cumsum += i128::from(candidate.layer.get_file_size());
     906            0 : 
     907            0 :                             if cumsum > min_resident_size as i128 {
     908            0 :                                 EvictionPartition::Above
     909              :                             } else {
     910              :                                 // The most recent layers below the min_resident_size threshold
     911              :                                 // are the last to be evicted.
     912            0 :                                 EvictionPartition::Below
     913              :                             }
     914              :                         }
     915              :                     };
     916              : 
     917            0 :                     (partition, candidate)
     918            0 :                 });
     919            0 : 
     920            0 :         METRICS
     921            0 :             .tenant_layer_count
     922            0 :             .observe(tenant_candidates.len() as f64);
     923            0 : 
     924            0 :         candidates.extend(tenant_candidates);
     925            0 : 
     926            0 :         let elapsed = started_at.elapsed();
     927            0 :         METRICS
     928            0 :             .tenant_collection_time
     929            0 :             .observe(elapsed.as_secs_f64());
     930            0 : 
     931            0 :         if elapsed > LOG_DURATION_THRESHOLD {
     932            0 :             tracing::info!(
     933            0 :                 tenant_id=%tenant.tenant_shard_id().tenant_id,
     934            0 :                 shard_id=%tenant.tenant_shard_id().shard_slug(),
     935            0 :                 elapsed_ms = elapsed.as_millis(),
     936            0 :                 "collection took longer than threshold"
     937              :             );
     938            0 :         }
     939              :     }
     940              : 
     941              :     // Note: the same tenant ID might be hit twice, if it transitions from attached to
     942              :     // secondary while we run.  That is okay: when we eventually try and run the eviction,
     943              :     // the `Gate` on the object will ensure that whichever one has already been shut down
     944              :     // will not delete anything.
     945              : 
     946            0 :     let mut secondary_tenants = Vec::new();
     947            0 :     tenant_manager.foreach_secondary_tenants(
     948            0 :         |_tenant_shard_id: &TenantShardId, state: &Arc<SecondaryTenant>| {
     949            0 :             secondary_tenants.push(state.clone());
     950            0 :         },
     951            0 :     );
     952              : 
     953            0 :     for tenant in secondary_tenants {
     954              :         // for secondary tenants we use a sum of on_disk layers and already evicted layers. this is
     955              :         // to prevent repeated disk usage based evictions from completely draining less often
     956              :         // updating secondaries.
     957            0 :         let (mut layer_info, total_layers) = tenant.get_layers_for_eviction();
     958            0 : 
     959            0 :         debug_assert!(
     960            0 :             total_layers >= layer_info.resident_layers.len(),
     961            0 :             "total_layers ({total_layers}) must be at least the resident_layers.len() ({})",
     962            0 :             layer_info.resident_layers.len()
     963              :         );
     964              : 
     965            0 :         let started_at = std::time::Instant::now();
     966            0 : 
     967            0 :         layer_info
     968            0 :             .resident_layers
     969            0 :             .sort_unstable_by_key(|layer_info| std::cmp::Reverse(layer_info.last_activity_ts));
     970            0 : 
     971            0 :         let tenant_candidates =
     972            0 :             layer_info
     973            0 :                 .resident_layers
     974            0 :                 .into_iter()
     975            0 :                 .enumerate()
     976            0 :                 .map(|(i, mut candidate)| {
     977            0 :                     candidate.relative_last_activity =
     978            0 :                         eviction_order.relative_last_activity(total_layers, i);
     979            0 :                     (
     980            0 :                         // Secondary locations' layers are always considered above the min resident size,
     981            0 :                         // i.e. secondary locations are permitted to be trimmed to zero layers if all
     982            0 :                         // the layers have sufficiently old access times.
     983            0 :                         EvictionPartition::Above,
     984            0 :                         candidate,
     985            0 :                     )
     986            0 :                 });
     987            0 : 
     988            0 :         METRICS
     989            0 :             .tenant_layer_count
     990            0 :             .observe(tenant_candidates.len() as f64);
     991            0 :         candidates.extend(tenant_candidates);
     992            0 : 
     993            0 :         tokio::task::yield_now().await;
     994              : 
     995            0 :         let elapsed = started_at.elapsed();
     996            0 : 
     997            0 :         METRICS
     998            0 :             .tenant_collection_time
     999            0 :             .observe(elapsed.as_secs_f64());
    1000            0 : 
    1001            0 :         if elapsed > LOG_DURATION_THRESHOLD {
    1002            0 :             tracing::info!(
    1003            0 :                 tenant_id=%tenant.tenant_shard_id().tenant_id,
    1004            0 :                 shard_id=%tenant.tenant_shard_id().shard_slug(),
    1005            0 :                 elapsed_ms = elapsed.as_millis(),
    1006            0 :                 "collection took longer than threshold"
    1007              :             );
    1008            0 :         }
    1009              :     }
    1010              : 
    1011            0 :     debug_assert!(
    1012            0 :         EvictionPartition::Above < EvictionPartition::Below,
    1013            0 :         "as explained in the function's doc comment, layers that aren't in the tenant's min_resident_size are evicted first"
    1014              :     );
    1015            0 :     debug_assert!(
    1016            0 :         EvictionPartition::EvictNow < EvictionPartition::Above,
    1017            0 :         "as explained in the function's doc comment, layers that aren't in the tenant's min_resident_size are evicted first"
    1018              :     );
    1019              : 
    1020            0 :     eviction_order.sort(&mut candidates);
    1021            0 : 
    1022            0 :     Ok(EvictionCandidates::Finished(candidates))
    1023            0 : }
    1024              : 
    1025              : /// Given a pre-sorted vec of all layers in the system, select the first N which are enough to
    1026              : /// relieve pressure.
    1027              : ///
    1028              : /// Returns the amount of candidates selected, with the planned usage.
    1029            0 : fn select_victims<U: Usage>(
    1030            0 :     candidates: &[(EvictionPartition, EvictionCandidate)],
    1031            0 :     usage_pre: U,
    1032            0 : ) -> VictimSelection<U> {
    1033            0 :     let mut usage_when_switched = None;
    1034            0 :     let mut usage_planned = usage_pre;
    1035            0 :     let mut evicted_amount = 0;
    1036              : 
    1037            0 :     for (i, (partition, candidate)) in candidates.iter().enumerate() {
    1038            0 :         if !usage_planned.has_pressure() {
    1039            0 :             break;
    1040            0 :         }
    1041            0 : 
    1042            0 :         if partition == &EvictionPartition::Below && usage_when_switched.is_none() {
    1043            0 :             usage_when_switched = Some((usage_planned, i));
    1044            0 :         }
    1045              : 
    1046            0 :         usage_planned.add_available_bytes(candidate.layer.get_file_size());
    1047            0 :         evicted_amount += 1;
    1048              :     }
    1049              : 
    1050            0 :     VictimSelection {
    1051            0 :         amount: evicted_amount,
    1052            0 :         usage_pre,
    1053            0 :         usage_when_switched,
    1054            0 :         usage_planned,
    1055            0 :     }
    1056            0 : }
    1057              : 
    1058              : struct VictimSelection<U> {
    1059              :     amount: usize,
    1060              :     usage_pre: U,
    1061              :     usage_when_switched: Option<(U, usize)>,
    1062              :     usage_planned: U,
    1063              : }
    1064              : 
    1065              : impl<U: Usage> VictimSelection<U> {
    1066            0 :     fn into_amount_and_planned(self) -> (usize, PlannedUsage<U>) {
    1067            0 :         debug!(
    1068              :             evicted_amount=%self.amount,
    1069            0 :             "took enough candidates for pressure to be relieved"
    1070              :         );
    1071              : 
    1072            0 :         if let Some((usage_planned, candidate_no)) = self.usage_when_switched.as_ref() {
    1073            0 :             warn!(usage_pre=?self.usage_pre, ?usage_planned, candidate_no, "tenant_min_resident_size-respecting LRU would not relieve pressure, evicting more following global LRU policy");
    1074            0 :         }
    1075              : 
    1076            0 :         let planned = match self.usage_when_switched {
    1077            0 :             Some((respecting_tenant_min_resident_size, _)) => PlannedUsage {
    1078            0 :                 respecting_tenant_min_resident_size,
    1079            0 :                 fallback_to_global_lru: Some(self.usage_planned),
    1080            0 :             },
    1081            0 :             None => PlannedUsage {
    1082            0 :                 respecting_tenant_min_resident_size: self.usage_planned,
    1083            0 :                 fallback_to_global_lru: None,
    1084            0 :             },
    1085              :         };
    1086              : 
    1087            0 :         (self.amount, planned)
    1088            0 :     }
    1089              : }
    1090              : 
    1091              : /// A totally ordered f32 subset we can use with sorting functions.
    1092              : pub(crate) mod finite_f32 {
    1093              : 
    1094              :     /// A totally ordered f32 subset we can use with sorting functions.
    1095              :     #[derive(Clone, Copy, PartialEq)]
    1096              :     pub struct FiniteF32(f32);
    1097              : 
    1098              :     impl std::fmt::Debug for FiniteF32 {
    1099            0 :         fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
    1100            0 :             std::fmt::Debug::fmt(&self.0, f)
    1101            0 :         }
    1102              :     }
    1103              : 
    1104              :     impl std::fmt::Display for FiniteF32 {
    1105            0 :         fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
    1106            0 :             std::fmt::Display::fmt(&self.0, f)
    1107            0 :         }
    1108              :     }
    1109              : 
    1110              :     impl std::cmp::Eq for FiniteF32 {}
    1111              : 
    1112              :     impl std::cmp::PartialOrd for FiniteF32 {
    1113            0 :         fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
    1114            0 :             Some(self.cmp(other))
    1115            0 :         }
    1116              :     }
    1117              : 
    1118              :     impl std::cmp::Ord for FiniteF32 {
    1119            0 :         fn cmp(&self, other: &Self) -> std::cmp::Ordering {
    1120            0 :             self.0.total_cmp(&other.0)
    1121            0 :         }
    1122              :     }
    1123              : 
    1124              :     impl TryFrom<f32> for FiniteF32 {
    1125              :         type Error = f32;
    1126              : 
    1127            0 :         fn try_from(value: f32) -> Result<Self, Self::Error> {
    1128            0 :             if value.is_finite() {
    1129            0 :                 Ok(FiniteF32(value))
    1130              :             } else {
    1131            0 :                 Err(value)
    1132              :             }
    1133            0 :         }
    1134              :     }
    1135              : 
    1136              :     impl From<FiniteF32> for f32 {
    1137           80 :         fn from(value: FiniteF32) -> f32 {
    1138           80 :             value.0
    1139           80 :         }
    1140              :     }
    1141              : 
    1142              :     impl FiniteF32 {
    1143              :         pub const ZERO: FiniteF32 = FiniteF32(0.0);
    1144              : 
    1145           80 :         pub fn try_from_normalized(value: f32) -> Result<Self, f32> {
    1146           80 :             if (0.0..=1.0).contains(&value) {
    1147              :                 // -0.0 is within the range, make sure it is assumed 0.0..=1.0
    1148           80 :                 let value = value.abs();
    1149           80 :                 Ok(FiniteF32(value))
    1150              :             } else {
    1151            0 :                 Err(value)
    1152              :             }
    1153           80 :         }
    1154              : 
    1155           80 :         pub fn into_inner(self) -> f32 {
    1156           80 :             self.into()
    1157           80 :         }
    1158              :     }
    1159              : }
    1160              : 
    1161              : mod filesystem_level_usage {
    1162              :     use anyhow::Context;
    1163              :     use camino::Utf8Path;
    1164              : 
    1165              :     use super::DiskUsageEvictionTaskConfig;
    1166              :     use crate::statvfs::Statvfs;
    1167              : 
    1168              :     #[derive(Debug, Clone, Copy)]
    1169              :     pub struct Usage<'a> {
    1170              :         config: &'a DiskUsageEvictionTaskConfig,
    1171              : 
    1172              :         /// Filesystem capacity
    1173              :         total_bytes: u64,
    1174              :         /// Free filesystem space
    1175              :         avail_bytes: u64,
    1176              :     }
    1177              : 
    1178              :     impl super::Usage for Usage<'_> {
    1179           28 :         fn has_pressure(&self) -> bool {
    1180           28 :             let usage_pct =
    1181           28 :                 (100.0 * (1.0 - ((self.avail_bytes as f64) / (self.total_bytes as f64)))) as u64;
    1182           28 : 
    1183           28 :             let pressures = [
    1184           28 :                 (
    1185           28 :                     "min_avail_bytes",
    1186           28 :                     self.avail_bytes < self.config.min_avail_bytes,
    1187           28 :                 ),
    1188           28 :                 (
    1189           28 :                     "max_usage_pct",
    1190           28 :                     usage_pct >= self.config.max_usage_pct.get() as u64,
    1191           28 :                 ),
    1192           28 :             ];
    1193           28 : 
    1194           56 :             pressures.into_iter().any(|(_, has_pressure)| has_pressure)
    1195           28 :         }
    1196              : 
    1197           24 :         fn add_available_bytes(&mut self, bytes: u64) {
    1198           24 :             self.avail_bytes += bytes;
    1199           24 :         }
    1200              :     }
    1201              : 
    1202            0 :     pub fn get<'a>(
    1203            0 :         tenants_dir: &Utf8Path,
    1204            0 :         config: &'a DiskUsageEvictionTaskConfig,
    1205            0 :     ) -> anyhow::Result<Usage<'a>> {
    1206            0 :         let mock_config = {
    1207            0 :             #[cfg(feature = "testing")]
    1208            0 :             {
    1209            0 :                 config.mock_statvfs.as_ref()
    1210              :             }
    1211              :             #[cfg(not(feature = "testing"))]
    1212              :             {
    1213              :                 None
    1214              :             }
    1215              :         };
    1216              : 
    1217            0 :         let stat = Statvfs::get(tenants_dir, mock_config)
    1218            0 :             .context("statvfs failed, presumably directory got unlinked")?;
    1219              : 
    1220            0 :         let (avail_bytes, total_bytes) = stat.get_avail_total_bytes();
    1221            0 : 
    1222            0 :         Ok(Usage {
    1223            0 :             config,
    1224            0 :             total_bytes,
    1225            0 :             avail_bytes,
    1226            0 :         })
    1227            0 :     }
    1228              : 
    1229              :     #[test]
    1230            4 :     fn max_usage_pct_pressure() {
    1231              :         use std::time::Duration;
    1232              : 
    1233              :         use utils::serde_percent::Percent;
    1234              : 
    1235              :         use super::Usage as _;
    1236              : 
    1237            4 :         let mut usage = Usage {
    1238            4 :             config: &DiskUsageEvictionTaskConfig {
    1239            4 :                 max_usage_pct: Percent::new(85).unwrap(),
    1240            4 :                 min_avail_bytes: 0,
    1241            4 :                 period: Duration::MAX,
    1242            4 :                 #[cfg(feature = "testing")]
    1243            4 :                 mock_statvfs: None,
    1244            4 :                 eviction_order: pageserver_api::config::EvictionOrder::default(),
    1245            4 :             },
    1246            4 :             total_bytes: 100_000,
    1247            4 :             avail_bytes: 0,
    1248            4 :         };
    1249            4 : 
    1250            4 :         assert!(usage.has_pressure(), "expected pressure at 100%");
    1251              : 
    1252            4 :         usage.add_available_bytes(14_000);
    1253            4 :         assert!(usage.has_pressure(), "expected pressure at 86%");
    1254              : 
    1255            4 :         usage.add_available_bytes(999);
    1256            4 :         assert!(usage.has_pressure(), "expected pressure at 85.001%");
    1257              : 
    1258            4 :         usage.add_available_bytes(1);
    1259            4 :         assert!(usage.has_pressure(), "expected pressure at precisely 85%");
    1260              : 
    1261            4 :         usage.add_available_bytes(1);
    1262            4 :         assert!(!usage.has_pressure(), "no pressure at 84.999%");
    1263              : 
    1264            4 :         usage.add_available_bytes(999);
    1265            4 :         assert!(!usage.has_pressure(), "no pressure at 84%");
    1266              : 
    1267            4 :         usage.add_available_bytes(16_000);
    1268            4 :         assert!(!usage.has_pressure());
    1269            4 :     }
    1270              : }
    1271              : 
    1272              : #[cfg(test)]
    1273              : mod tests {
    1274              :     use super::*;
    1275              : 
    1276              :     #[test]
    1277            4 :     fn relative_equal_bounds() {
    1278            4 :         let order = EvictionOrder::RelativeAccessed {
    1279            4 :             highest_layer_count_loses_first: false,
    1280            4 :         };
    1281            4 : 
    1282            4 :         let len = 10;
    1283            4 :         let v = (0..len)
    1284           40 :             .map(|i| order.relative_last_activity(len, i).into_inner())
    1285            4 :             .collect::<Vec<_>>();
    1286            4 : 
    1287            4 :         assert_eq!(v.first(), Some(&1.0));
    1288            4 :         assert_eq!(v.last(), Some(&0.0));
    1289           36 :         assert!(v.windows(2).all(|slice| slice[0] > slice[1]));
    1290            4 :     }
    1291              : 
    1292              :     #[test]
    1293            4 :     fn relative_spare_bounds() {
    1294            4 :         let order = EvictionOrder::RelativeAccessed {
    1295            4 :             highest_layer_count_loses_first: true,
    1296            4 :         };
    1297            4 : 
    1298            4 :         let len = 10;
    1299            4 :         let v = (0..len)
    1300           40 :             .map(|i| order.relative_last_activity(len, i).into_inner())
    1301            4 :             .collect::<Vec<_>>();
    1302            4 : 
    1303            4 :         assert_eq!(v.first(), Some(&1.0));
    1304            4 :         assert_eq!(v.last(), Some(&0.1));
    1305           36 :         assert!(v.windows(2).all(|slice| slice[0] > slice[1]));
    1306            4 :     }
    1307              : }
        

Generated by: LCOV version 2.1-beta