LCOV - code coverage report
Current view: top level - pageserver/src/http - routes.rs (source / functions) Coverage Total Hit
Test: 5fe7fa8d483b39476409aee736d6d5e32728bfac.info Lines: 0.0 % 2765 0
Test Date: 2025-03-12 16:10:49 Functions: 0.0 % 846 0

            Line data    Source code
       1              : //!
       2              : //! Management HTTP API
       3              : //!
       4              : use std::cmp::Reverse;
       5              : use std::collections::{BinaryHeap, HashMap};
       6              : use std::str::FromStr;
       7              : use std::sync::Arc;
       8              : use std::time::Duration;
       9              : 
      10              : use anyhow::{Context, Result, anyhow};
      11              : use enumset::EnumSet;
      12              : use futures::future::join_all;
      13              : use futures::{StreamExt, TryFutureExt};
      14              : use http_utils::endpoint::{
      15              :     self, attach_openapi_ui, auth_middleware, check_permission_with, profile_cpu_handler,
      16              :     profile_heap_handler, prometheus_metrics_handler, request_span,
      17              : };
      18              : use http_utils::error::{ApiError, HttpErrorBody};
      19              : use http_utils::failpoints::failpoints_handler;
      20              : use http_utils::json::{json_request, json_request_maybe, json_response};
      21              : use http_utils::request::{
      22              :     get_request_param, must_get_query_param, must_parse_query_param, parse_query_param,
      23              :     parse_request_param,
      24              : };
      25              : use http_utils::{RequestExt, RouterBuilder};
      26              : use humantime::format_rfc3339;
      27              : use hyper::{Body, Request, Response, StatusCode, Uri, header};
      28              : use metrics::launch_timestamp::LaunchTimestamp;
      29              : use pageserver_api::models::virtual_file::IoMode;
      30              : use pageserver_api::models::{
      31              :     DownloadRemoteLayersTaskSpawnRequest, IngestAuxFilesRequest, ListAuxFilesRequest,
      32              :     LocationConfig, LocationConfigListResponse, LocationConfigMode, LsnLease, LsnLeaseRequest,
      33              :     OffloadedTimelineInfo, PageTraceEvent, ShardParameters, StatusResponse,
      34              :     TenantConfigPatchRequest, TenantConfigRequest, TenantDetails, TenantInfo,
      35              :     TenantLocationConfigRequest, TenantLocationConfigResponse, TenantScanRemoteStorageResponse,
      36              :     TenantScanRemoteStorageShard, TenantShardLocation, TenantShardSplitRequest,
      37              :     TenantShardSplitResponse, TenantSorting, TenantState, TenantWaitLsnRequest,
      38              :     TimelineArchivalConfigRequest, TimelineCreateRequest, TimelineCreateRequestMode,
      39              :     TimelineCreateRequestModeImportPgdata, TimelineGcRequest, TimelineInfo,
      40              :     TimelinePatchIndexPartRequest, TimelinesInfoAndOffloaded, TopTenantShardItem,
      41              :     TopTenantShardsRequest, TopTenantShardsResponse,
      42              : };
      43              : use pageserver_api::shard::{ShardCount, TenantShardId};
      44              : use remote_storage::{DownloadError, GenericRemoteStorage, TimeTravelError};
      45              : use scopeguard::defer;
      46              : use tenant_size_model::svg::SvgBranchKind;
      47              : use tenant_size_model::{SizeResult, StorageModel};
      48              : use tokio::time::Instant;
      49              : use tokio_util::io::StreamReader;
      50              : use tokio_util::sync::CancellationToken;
      51              : use tracing::*;
      52              : use utils::auth::SwappableJwtAuth;
      53              : use utils::generation::Generation;
      54              : use utils::id::{TenantId, TimelineId};
      55              : use utils::lsn::Lsn;
      56              : 
      57              : use crate::config::PageServerConf;
      58              : use crate::context;
      59              : use crate::context::{DownloadBehavior, RequestContext, RequestContextBuilder};
      60              : use crate::deletion_queue::DeletionQueueClient;
      61              : use crate::pgdatadir_mapping::LsnForTimestamp;
      62              : use crate::task_mgr::TaskKind;
      63              : use crate::tenant::config::{LocationConf, TenantConfOpt};
      64              : use crate::tenant::mgr::{
      65              :     GetActiveTenantError, GetTenantError, TenantManager, TenantMapError, TenantMapInsertError,
      66              :     TenantSlot, TenantSlotError, TenantSlotUpsertError, TenantStateError, UpsertLocationError,
      67              : };
      68              : use crate::tenant::remote_timeline_client::index::GcCompactionState;
      69              : use crate::tenant::remote_timeline_client::{
      70              :     download_index_part, list_remote_tenant_shards, list_remote_timelines,
      71              : };
      72              : use crate::tenant::secondary::SecondaryController;
      73              : use crate::tenant::size::ModelInputs;
      74              : use crate::tenant::storage_layer::{IoConcurrency, LayerAccessStatsReset, LayerName};
      75              : use crate::tenant::timeline::offload::{OffloadError, offload_timeline};
      76              : use crate::tenant::timeline::{
      77              :     CompactFlags, CompactOptions, CompactRequest, CompactionError, Timeline, WaitLsnTimeout,
      78              :     WaitLsnWaiter, import_pgdata,
      79              : };
      80              : use crate::tenant::{
      81              :     GetTimelineError, LogicalSizeCalculationCause, OffloadedTimeline, PageReconstructError,
      82              :     remote_timeline_client,
      83              : };
      84              : use crate::{DEFAULT_PG_VERSION, disk_usage_eviction_task, tenant};
      85              : 
      86              : // For APIs that require an Active tenant, how long should we block waiting for that state?
      87              : // This is not functionally necessary (clients will retry), but avoids generating a lot of
      88              : // failed API calls while tenants are activating.
      89              : #[cfg(not(feature = "testing"))]
      90              : pub(crate) const ACTIVE_TENANT_TIMEOUT: Duration = Duration::from_millis(5000);
      91              : 
      92              : // Tests run on slow/oversubscribed nodes, and may need to wait much longer for tenants to
      93              : // finish attaching, if calls to remote storage are slow.
      94              : #[cfg(feature = "testing")]
      95              : pub(crate) const ACTIVE_TENANT_TIMEOUT: Duration = Duration::from_millis(30000);
      96              : 
      97              : pub struct State {
      98              :     conf: &'static PageServerConf,
      99              :     tenant_manager: Arc<TenantManager>,
     100              :     auth: Option<Arc<SwappableJwtAuth>>,
     101              :     allowlist_routes: &'static [&'static str],
     102              :     remote_storage: GenericRemoteStorage,
     103              :     broker_client: storage_broker::BrokerClientChannel,
     104              :     disk_usage_eviction_state: Arc<disk_usage_eviction_task::State>,
     105              :     deletion_queue_client: DeletionQueueClient,
     106              :     secondary_controller: SecondaryController,
     107              :     latest_utilization: tokio::sync::Mutex<Option<(std::time::Instant, bytes::Bytes)>>,
     108              : }
     109              : 
     110              : impl State {
     111              :     #[allow(clippy::too_many_arguments)]
     112            0 :     pub fn new(
     113            0 :         conf: &'static PageServerConf,
     114            0 :         tenant_manager: Arc<TenantManager>,
     115            0 :         auth: Option<Arc<SwappableJwtAuth>>,
     116            0 :         remote_storage: GenericRemoteStorage,
     117            0 :         broker_client: storage_broker::BrokerClientChannel,
     118            0 :         disk_usage_eviction_state: Arc<disk_usage_eviction_task::State>,
     119            0 :         deletion_queue_client: DeletionQueueClient,
     120            0 :         secondary_controller: SecondaryController,
     121            0 :     ) -> anyhow::Result<Self> {
     122            0 :         let allowlist_routes = &[
     123            0 :             "/v1/status",
     124            0 :             "/v1/doc",
     125            0 :             "/swagger.yml",
     126            0 :             "/metrics",
     127            0 :             "/profile/cpu",
     128            0 :             "/profile/heap",
     129            0 :         ];
     130            0 :         Ok(Self {
     131            0 :             conf,
     132            0 :             tenant_manager,
     133            0 :             auth,
     134            0 :             allowlist_routes,
     135            0 :             remote_storage,
     136            0 :             broker_client,
     137            0 :             disk_usage_eviction_state,
     138            0 :             deletion_queue_client,
     139            0 :             secondary_controller,
     140            0 :             latest_utilization: Default::default(),
     141            0 :         })
     142            0 :     }
     143              : }
     144              : 
     145              : #[inline(always)]
     146            0 : fn get_state(request: &Request<Body>) -> &State {
     147            0 :     request
     148            0 :         .data::<Arc<State>>()
     149            0 :         .expect("unknown state type")
     150            0 :         .as_ref()
     151            0 : }
     152              : 
     153              : #[inline(always)]
     154            0 : fn get_config(request: &Request<Body>) -> &'static PageServerConf {
     155            0 :     get_state(request).conf
     156            0 : }
     157              : 
     158              : /// Check that the requester is authorized to operate on given tenant
     159            0 : fn check_permission(request: &Request<Body>, tenant_id: Option<TenantId>) -> Result<(), ApiError> {
     160            0 :     check_permission_with(request, |claims| {
     161            0 :         crate::auth::check_permission(claims, tenant_id)
     162            0 :     })
     163            0 : }
     164              : 
     165              : impl From<PageReconstructError> for ApiError {
     166            0 :     fn from(pre: PageReconstructError) -> ApiError {
     167            0 :         match pre {
     168            0 :             PageReconstructError::Other(other) => ApiError::InternalServerError(other),
     169            0 :             PageReconstructError::MissingKey(e) => ApiError::InternalServerError(e.into()),
     170            0 :             PageReconstructError::Cancelled => ApiError::Cancelled,
     171            0 :             PageReconstructError::AncestorLsnTimeout(e) => ApiError::Timeout(format!("{e}").into()),
     172            0 :             PageReconstructError::WalRedo(pre) => ApiError::InternalServerError(pre),
     173              :         }
     174            0 :     }
     175              : }
     176              : 
     177              : impl From<TenantMapInsertError> for ApiError {
     178            0 :     fn from(tmie: TenantMapInsertError) -> ApiError {
     179            0 :         match tmie {
     180            0 :             TenantMapInsertError::SlotError(e) => e.into(),
     181            0 :             TenantMapInsertError::SlotUpsertError(e) => e.into(),
     182            0 :             TenantMapInsertError::Other(e) => ApiError::InternalServerError(e),
     183              :         }
     184            0 :     }
     185              : }
     186              : 
     187              : impl From<TenantSlotError> for ApiError {
     188            0 :     fn from(e: TenantSlotError) -> ApiError {
     189              :         use TenantSlotError::*;
     190            0 :         match e {
     191            0 :             NotFound(tenant_id) => {
     192            0 :                 ApiError::NotFound(anyhow::anyhow!("NotFound: tenant {tenant_id}").into())
     193              :             }
     194              :             InProgress => {
     195            0 :                 ApiError::ResourceUnavailable("Tenant is being modified concurrently".into())
     196              :             }
     197            0 :             MapState(e) => e.into(),
     198              :         }
     199            0 :     }
     200              : }
     201              : 
     202              : impl From<TenantSlotUpsertError> for ApiError {
     203            0 :     fn from(e: TenantSlotUpsertError) -> ApiError {
     204              :         use TenantSlotUpsertError::*;
     205            0 :         match e {
     206            0 :             InternalError(e) => ApiError::InternalServerError(anyhow::anyhow!("{e}")),
     207            0 :             MapState(e) => e.into(),
     208            0 :             ShuttingDown(_) => ApiError::ShuttingDown,
     209              :         }
     210            0 :     }
     211              : }
     212              : 
     213              : impl From<UpsertLocationError> for ApiError {
     214            0 :     fn from(e: UpsertLocationError) -> ApiError {
     215              :         use UpsertLocationError::*;
     216            0 :         match e {
     217            0 :             BadRequest(e) => ApiError::BadRequest(e),
     218            0 :             Unavailable(_) => ApiError::ShuttingDown,
     219            0 :             e @ InProgress => ApiError::Conflict(format!("{e}")),
     220            0 :             Flush(e) | InternalError(e) => ApiError::InternalServerError(e),
     221              :         }
     222            0 :     }
     223              : }
     224              : 
     225              : impl From<TenantMapError> for ApiError {
     226            0 :     fn from(e: TenantMapError) -> ApiError {
     227              :         use TenantMapError::*;
     228            0 :         match e {
     229              :             StillInitializing | ShuttingDown => {
     230            0 :                 ApiError::ResourceUnavailable(format!("{e}").into())
     231            0 :             }
     232            0 :         }
     233            0 :     }
     234              : }
     235              : 
     236              : impl From<TenantStateError> for ApiError {
     237            0 :     fn from(tse: TenantStateError) -> ApiError {
     238            0 :         match tse {
     239              :             TenantStateError::IsStopping(_) => {
     240            0 :                 ApiError::ResourceUnavailable("Tenant is stopping".into())
     241              :             }
     242            0 :             TenantStateError::SlotError(e) => e.into(),
     243            0 :             TenantStateError::SlotUpsertError(e) => e.into(),
     244            0 :             TenantStateError::Other(e) => ApiError::InternalServerError(anyhow!(e)),
     245              :         }
     246            0 :     }
     247              : }
     248              : 
     249              : impl From<GetTenantError> for ApiError {
     250            0 :     fn from(tse: GetTenantError) -> ApiError {
     251            0 :         match tse {
     252            0 :             GetTenantError::NotFound(tid) => ApiError::NotFound(anyhow!("tenant {tid}").into()),
     253            0 :             GetTenantError::ShardNotFound(tid) => {
     254            0 :                 ApiError::NotFound(anyhow!("tenant {tid}").into())
     255              :             }
     256              :             GetTenantError::NotActive(_) => {
     257              :                 // Why is this not `ApiError::NotFound`?
     258              :                 // Because we must be careful to never return 404 for a tenant if it does
     259              :                 // in fact exist locally. If we did, the caller could draw the conclusion
     260              :                 // that it can attach the tenant to another PS and we'd be in split-brain.
     261            0 :                 ApiError::ResourceUnavailable("Tenant not yet active".into())
     262              :             }
     263            0 :             GetTenantError::MapState(e) => ApiError::ResourceUnavailable(format!("{e}").into()),
     264              :         }
     265            0 :     }
     266              : }
     267              : 
     268              : impl From<GetTimelineError> for ApiError {
     269            0 :     fn from(gte: GetTimelineError) -> Self {
     270            0 :         // Rationale: tenant is activated only after eligble timelines activate
     271            0 :         ApiError::NotFound(gte.into())
     272            0 :     }
     273              : }
     274              : 
     275              : impl From<GetActiveTenantError> for ApiError {
     276            0 :     fn from(e: GetActiveTenantError) -> ApiError {
     277            0 :         match e {
     278            0 :             GetActiveTenantError::Broken(reason) => {
     279            0 :                 ApiError::InternalServerError(anyhow!("tenant is broken: {}", reason))
     280              :             }
     281              :             GetActiveTenantError::WillNotBecomeActive(TenantState::Stopping { .. }) => {
     282            0 :                 ApiError::ShuttingDown
     283              :             }
     284            0 :             GetActiveTenantError::WillNotBecomeActive(_) => ApiError::Conflict(format!("{}", e)),
     285            0 :             GetActiveTenantError::Cancelled => ApiError::ShuttingDown,
     286            0 :             GetActiveTenantError::NotFound(gte) => gte.into(),
     287              :             GetActiveTenantError::WaitForActiveTimeout { .. } => {
     288            0 :                 ApiError::ResourceUnavailable(format!("{}", e).into())
     289              :             }
     290              :             GetActiveTenantError::SwitchedTenant => {
     291              :                 // in our HTTP handlers, this error doesn't happen
     292              :                 // TODO: separate error types
     293            0 :                 ApiError::ResourceUnavailable("switched tenant".into())
     294              :             }
     295              :         }
     296            0 :     }
     297              : }
     298              : 
     299              : impl From<crate::tenant::DeleteTimelineError> for ApiError {
     300            0 :     fn from(value: crate::tenant::DeleteTimelineError) -> Self {
     301              :         use crate::tenant::DeleteTimelineError::*;
     302            0 :         match value {
     303            0 :             NotFound => ApiError::NotFound(anyhow::anyhow!("timeline not found").into()),
     304            0 :             HasChildren(children) => ApiError::PreconditionFailed(
     305            0 :                 format!("Cannot delete timeline which has child timelines: {children:?}")
     306            0 :                     .into_boxed_str(),
     307            0 :             ),
     308            0 :             a @ AlreadyInProgress(_) => ApiError::Conflict(a.to_string()),
     309            0 :             Cancelled => ApiError::ResourceUnavailable("shutting down".into()),
     310            0 :             Other(e) => ApiError::InternalServerError(e),
     311              :         }
     312            0 :     }
     313              : }
     314              : 
     315              : impl From<crate::tenant::TimelineArchivalError> for ApiError {
     316            0 :     fn from(value: crate::tenant::TimelineArchivalError) -> Self {
     317              :         use crate::tenant::TimelineArchivalError::*;
     318            0 :         match value {
     319            0 :             NotFound => ApiError::NotFound(anyhow::anyhow!("timeline not found").into()),
     320            0 :             Timeout => ApiError::Timeout("hit pageserver internal timeout".into()),
     321            0 :             Cancelled => ApiError::ShuttingDown,
     322            0 :             e @ HasArchivedParent(_) => {
     323            0 :                 ApiError::PreconditionFailed(e.to_string().into_boxed_str())
     324              :             }
     325            0 :             HasUnarchivedChildren(children) => ApiError::PreconditionFailed(
     326            0 :                 format!(
     327            0 :                     "Cannot archive timeline which has non-archived child timelines: {children:?}"
     328            0 :                 )
     329            0 :                 .into_boxed_str(),
     330            0 :             ),
     331            0 :             a @ AlreadyInProgress => ApiError::Conflict(a.to_string()),
     332            0 :             Other(e) => ApiError::InternalServerError(e),
     333              :         }
     334            0 :     }
     335              : }
     336              : 
     337              : impl From<crate::tenant::mgr::DeleteTimelineError> for ApiError {
     338            0 :     fn from(value: crate::tenant::mgr::DeleteTimelineError) -> Self {
     339              :         use crate::tenant::mgr::DeleteTimelineError::*;
     340            0 :         match value {
     341              :             // Report Precondition failed so client can distinguish between
     342              :             // "tenant is missing" case from "timeline is missing"
     343            0 :             Tenant(GetTenantError::NotFound(..)) => ApiError::PreconditionFailed(
     344            0 :                 "Requested tenant is missing".to_owned().into_boxed_str(),
     345            0 :             ),
     346            0 :             Tenant(t) => ApiError::from(t),
     347            0 :             Timeline(t) => ApiError::from(t),
     348              :         }
     349            0 :     }
     350              : }
     351              : 
     352              : impl From<crate::tenant::mgr::DeleteTenantError> for ApiError {
     353            0 :     fn from(value: crate::tenant::mgr::DeleteTenantError) -> Self {
     354              :         use crate::tenant::mgr::DeleteTenantError::*;
     355            0 :         match value {
     356            0 :             SlotError(e) => e.into(),
     357            0 :             Other(o) => ApiError::InternalServerError(o),
     358            0 :             Cancelled => ApiError::ShuttingDown,
     359              :         }
     360            0 :     }
     361              : }
     362              : 
     363              : impl From<crate::tenant::secondary::SecondaryTenantError> for ApiError {
     364            0 :     fn from(ste: crate::tenant::secondary::SecondaryTenantError) -> ApiError {
     365              :         use crate::tenant::secondary::SecondaryTenantError;
     366            0 :         match ste {
     367            0 :             SecondaryTenantError::GetTenant(gte) => gte.into(),
     368            0 :             SecondaryTenantError::ShuttingDown => ApiError::ShuttingDown,
     369              :         }
     370            0 :     }
     371              : }
     372              : 
     373              : // Helper function to construct a TimelineInfo struct for a timeline
     374            0 : async fn build_timeline_info(
     375            0 :     timeline: &Arc<Timeline>,
     376            0 :     include_non_incremental_logical_size: bool,
     377            0 :     force_await_initial_logical_size: bool,
     378            0 :     ctx: &RequestContext,
     379            0 : ) -> anyhow::Result<TimelineInfo> {
     380            0 :     crate::tenant::debug_assert_current_span_has_tenant_and_timeline_id();
     381            0 : 
     382            0 :     if force_await_initial_logical_size {
     383            0 :         timeline.clone().await_initial_logical_size().await
     384            0 :     }
     385              : 
     386            0 :     let mut info = build_timeline_info_common(
     387            0 :         timeline,
     388            0 :         ctx,
     389            0 :         tenant::timeline::GetLogicalSizePriority::Background,
     390            0 :     )
     391            0 :     .await?;
     392            0 :     if include_non_incremental_logical_size {
     393              :         // XXX we should be using spawn_ondemand_logical_size_calculation here.
     394              :         // Otherwise, if someone deletes the timeline / detaches the tenant while
     395              :         // we're executing this function, we will outlive the timeline on-disk state.
     396              :         info.current_logical_size_non_incremental = Some(
     397            0 :             timeline
     398            0 :                 .get_current_logical_size_non_incremental(info.last_record_lsn, ctx)
     399            0 :                 .await?,
     400              :         );
     401            0 :     }
     402            0 :     Ok(info)
     403            0 : }
     404              : 
     405            0 : async fn build_timeline_info_common(
     406            0 :     timeline: &Arc<Timeline>,
     407            0 :     ctx: &RequestContext,
     408            0 :     logical_size_task_priority: tenant::timeline::GetLogicalSizePriority,
     409            0 : ) -> anyhow::Result<TimelineInfo> {
     410            0 :     crate::tenant::debug_assert_current_span_has_tenant_and_timeline_id();
     411            0 :     let initdb_lsn = timeline.initdb_lsn;
     412            0 :     let last_record_lsn = timeline.get_last_record_lsn();
     413            0 :     let (wal_source_connstr, last_received_msg_lsn, last_received_msg_ts) = {
     414            0 :         let guard = timeline.last_received_wal.lock().unwrap();
     415            0 :         if let Some(info) = guard.as_ref() {
     416            0 :             (
     417            0 :                 Some(format!("{}", info.wal_source_connconf)), // Password is hidden, but it's for statistics only.
     418            0 :                 Some(info.last_received_msg_lsn),
     419            0 :                 Some(info.last_received_msg_ts),
     420            0 :             )
     421              :         } else {
     422            0 :             (None, None, None)
     423              :         }
     424              :     };
     425              : 
     426            0 :     let ancestor_timeline_id = timeline.get_ancestor_timeline_id();
     427            0 :     let ancestor_lsn = match timeline.get_ancestor_lsn() {
     428            0 :         Lsn(0) => None,
     429            0 :         lsn @ Lsn(_) => Some(lsn),
     430              :     };
     431            0 :     let current_logical_size = timeline.get_current_logical_size(logical_size_task_priority, ctx);
     432            0 :     let current_physical_size = Some(timeline.layer_size_sum().await);
     433            0 :     let state = timeline.current_state();
     434            0 :     // Report is_archived = false if the timeline is still loading
     435            0 :     let is_archived = timeline.is_archived().unwrap_or(false);
     436            0 :     let remote_consistent_lsn_projected = timeline
     437            0 :         .get_remote_consistent_lsn_projected()
     438            0 :         .unwrap_or(Lsn(0));
     439            0 :     let remote_consistent_lsn_visible = timeline
     440            0 :         .get_remote_consistent_lsn_visible()
     441            0 :         .unwrap_or(Lsn(0));
     442            0 : 
     443            0 :     let walreceiver_status = timeline.walreceiver_status();
     444            0 : 
     445            0 :     let (pitr_history_size, within_ancestor_pitr) = timeline.get_pitr_history_stats();
     446            0 : 
     447            0 :     let min_readable_lsn = std::cmp::max(
     448            0 :         timeline.get_gc_cutoff_lsn(),
     449            0 :         *timeline.get_applied_gc_cutoff_lsn(),
     450            0 :     );
     451              : 
     452            0 :     let info = TimelineInfo {
     453            0 :         tenant_id: timeline.tenant_shard_id,
     454            0 :         timeline_id: timeline.timeline_id,
     455            0 :         ancestor_timeline_id,
     456            0 :         ancestor_lsn,
     457            0 :         disk_consistent_lsn: timeline.get_disk_consistent_lsn(),
     458            0 :         remote_consistent_lsn: remote_consistent_lsn_projected,
     459            0 :         remote_consistent_lsn_visible,
     460            0 :         initdb_lsn,
     461            0 :         last_record_lsn,
     462            0 :         prev_record_lsn: Some(timeline.get_prev_record_lsn()),
     463            0 :         _unused: Default::default(), // Unused, for legacy decode only
     464            0 :         min_readable_lsn,
     465            0 :         applied_gc_cutoff_lsn: *timeline.get_applied_gc_cutoff_lsn(),
     466            0 :         current_logical_size: current_logical_size.size_dont_care_about_accuracy(),
     467            0 :         current_logical_size_is_accurate: match current_logical_size.accuracy() {
     468            0 :             tenant::timeline::logical_size::Accuracy::Approximate => false,
     469            0 :             tenant::timeline::logical_size::Accuracy::Exact => true,
     470              :         },
     471            0 :         directory_entries_counts: timeline.get_directory_metrics().to_vec(),
     472            0 :         current_physical_size,
     473            0 :         current_logical_size_non_incremental: None,
     474            0 :         pitr_history_size,
     475            0 :         within_ancestor_pitr,
     476            0 :         timeline_dir_layer_file_size_sum: None,
     477            0 :         wal_source_connstr,
     478            0 :         last_received_msg_lsn,
     479            0 :         last_received_msg_ts,
     480            0 :         pg_version: timeline.pg_version,
     481            0 : 
     482            0 :         state,
     483            0 :         is_archived: Some(is_archived),
     484            0 :         rel_size_migration: Some(timeline.get_rel_size_v2_status()),
     485            0 : 
     486            0 :         walreceiver_status,
     487            0 :     };
     488            0 :     Ok(info)
     489            0 : }
     490              : 
     491            0 : fn build_timeline_offloaded_info(offloaded: &Arc<OffloadedTimeline>) -> OffloadedTimelineInfo {
     492            0 :     let &OffloadedTimeline {
     493            0 :         tenant_shard_id,
     494            0 :         timeline_id,
     495            0 :         ancestor_retain_lsn,
     496            0 :         ancestor_timeline_id,
     497            0 :         archived_at,
     498            0 :         ..
     499            0 :     } = offloaded.as_ref();
     500            0 :     OffloadedTimelineInfo {
     501            0 :         tenant_id: tenant_shard_id,
     502            0 :         timeline_id,
     503            0 :         ancestor_retain_lsn,
     504            0 :         ancestor_timeline_id,
     505            0 :         archived_at: archived_at.and_utc(),
     506            0 :     }
     507            0 : }
     508              : 
     509              : // healthcheck handler
     510            0 : async fn status_handler(
     511            0 :     request: Request<Body>,
     512            0 :     _cancel: CancellationToken,
     513            0 : ) -> Result<Response<Body>, ApiError> {
     514            0 :     check_permission(&request, None)?;
     515            0 :     let config = get_config(&request);
     516            0 :     json_response(StatusCode::OK, StatusResponse { id: config.id })
     517            0 : }
     518              : 
     519            0 : async fn reload_auth_validation_keys_handler(
     520            0 :     request: Request<Body>,
     521            0 :     _cancel: CancellationToken,
     522            0 : ) -> Result<Response<Body>, ApiError> {
     523            0 :     check_permission(&request, None)?;
     524            0 :     let config = get_config(&request);
     525            0 :     let state = get_state(&request);
     526            0 :     let Some(shared_auth) = &state.auth else {
     527            0 :         return json_response(StatusCode::BAD_REQUEST, ());
     528              :     };
     529              :     // unwrap is ok because check is performed when creating config, so path is set and exists
     530            0 :     let key_path = config.auth_validation_public_key_path.as_ref().unwrap();
     531            0 :     info!("Reloading public key(s) for verifying JWT tokens from {key_path:?}");
     532              : 
     533            0 :     match utils::auth::JwtAuth::from_key_path(key_path) {
     534            0 :         Ok(new_auth) => {
     535            0 :             shared_auth.swap(new_auth);
     536            0 :             json_response(StatusCode::OK, ())
     537              :         }
     538            0 :         Err(e) => {
     539            0 :             let err_msg = "Error reloading public keys";
     540            0 :             warn!("Error reloading public keys from {key_path:?}: {e:}");
     541            0 :             json_response(
     542            0 :                 StatusCode::INTERNAL_SERVER_ERROR,
     543            0 :                 HttpErrorBody::from_msg(err_msg.to_string()),
     544            0 :             )
     545              :         }
     546              :     }
     547            0 : }
     548              : 
     549            0 : async fn timeline_create_handler(
     550            0 :     mut request: Request<Body>,
     551            0 :     _cancel: CancellationToken,
     552            0 : ) -> Result<Response<Body>, ApiError> {
     553            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
     554            0 :     let request_data: TimelineCreateRequest = json_request(&mut request).await?;
     555            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
     556              : 
     557            0 :     let new_timeline_id = request_data.new_timeline_id;
     558              :     // fill in the default pg_version if not provided & convert request into domain model
     559            0 :     let params: tenant::CreateTimelineParams = match request_data.mode {
     560              :         TimelineCreateRequestMode::Bootstrap {
     561            0 :             existing_initdb_timeline_id,
     562            0 :             pg_version,
     563            0 :         } => tenant::CreateTimelineParams::Bootstrap(tenant::CreateTimelineParamsBootstrap {
     564            0 :             new_timeline_id,
     565            0 :             existing_initdb_timeline_id,
     566            0 :             pg_version: pg_version.unwrap_or(DEFAULT_PG_VERSION),
     567            0 :         }),
     568              :         TimelineCreateRequestMode::Branch {
     569            0 :             ancestor_timeline_id,
     570            0 :             ancestor_start_lsn,
     571            0 :             pg_version: _,
     572            0 :         } => tenant::CreateTimelineParams::Branch(tenant::CreateTimelineParamsBranch {
     573            0 :             new_timeline_id,
     574            0 :             ancestor_timeline_id,
     575            0 :             ancestor_start_lsn,
     576            0 :         }),
     577              :         TimelineCreateRequestMode::ImportPgdata {
     578              :             import_pgdata:
     579              :                 TimelineCreateRequestModeImportPgdata {
     580            0 :                     location,
     581            0 :                     idempotency_key,
     582            0 :                 },
     583            0 :         } => tenant::CreateTimelineParams::ImportPgdata(tenant::CreateTimelineParamsImportPgdata {
     584            0 :             idempotency_key: import_pgdata::index_part_format::IdempotencyKey::new(
     585            0 :                 idempotency_key.0,
     586            0 :             ),
     587            0 :             new_timeline_id,
     588              :             location: {
     589            0 :                 use import_pgdata::index_part_format::Location;
     590            0 :                 use pageserver_api::models::ImportPgdataLocation;
     591            0 :                 match location {
     592              :                     #[cfg(feature = "testing")]
     593            0 :                     ImportPgdataLocation::LocalFs { path } => Location::LocalFs { path },
     594              :                     ImportPgdataLocation::AwsS3 {
     595            0 :                         region,
     596            0 :                         bucket,
     597            0 :                         key,
     598            0 :                     } => Location::AwsS3 {
     599            0 :                         region,
     600            0 :                         bucket,
     601            0 :                         key,
     602            0 :                     },
     603              :                 }
     604              :             },
     605              :         }),
     606              :     };
     607              : 
     608            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Error);
     609            0 : 
     610            0 :     let state = get_state(&request);
     611              : 
     612            0 :     async {
     613            0 :         let tenant = state
     614            0 :             .tenant_manager
     615            0 :             .get_attached_tenant_shard(tenant_shard_id)?;
     616              : 
     617            0 :         tenant.wait_to_become_active(ACTIVE_TENANT_TIMEOUT).await?;
     618              : 
     619              :         // earlier versions of the code had pg_version and ancestor_lsn in the span
     620              :         // => continue to provide that information, but, through a log message that doesn't require us to destructure
     621            0 :         tracing::info!(?params, "creating timeline");
     622              : 
     623            0 :         match tenant
     624            0 :             .create_timeline(params, state.broker_client.clone(), &ctx)
     625            0 :             .await
     626              :         {
     627            0 :             Ok(new_timeline) => {
     628              :                 // Created. Construct a TimelineInfo for it.
     629            0 :                 let timeline_info = build_timeline_info_common(
     630            0 :                     &new_timeline,
     631            0 :                     &ctx,
     632            0 :                     tenant::timeline::GetLogicalSizePriority::User,
     633            0 :                 )
     634            0 :                 .await
     635            0 :                 .map_err(ApiError::InternalServerError)?;
     636            0 :                 json_response(StatusCode::CREATED, timeline_info)
     637              :             }
     638            0 :             Err(_) if tenant.cancel.is_cancelled() => {
     639            0 :                 // In case we get some ugly error type during shutdown, cast it into a clean 503.
     640            0 :                 json_response(
     641            0 :                     StatusCode::SERVICE_UNAVAILABLE,
     642            0 :                     HttpErrorBody::from_msg("Tenant shutting down".to_string()),
     643            0 :                 )
     644              :             }
     645            0 :             Err(e @ tenant::CreateTimelineError::Conflict) => {
     646            0 :                 json_response(StatusCode::CONFLICT, HttpErrorBody::from_msg(e.to_string()))
     647              :             }
     648            0 :             Err(e @ tenant::CreateTimelineError::AlreadyCreating) => json_response(
     649            0 :                 StatusCode::TOO_MANY_REQUESTS,
     650            0 :                 HttpErrorBody::from_msg(e.to_string()),
     651            0 :             ),
     652            0 :             Err(tenant::CreateTimelineError::AncestorLsn(err)) => json_response(
     653            0 :                 StatusCode::NOT_ACCEPTABLE,
     654            0 :                 HttpErrorBody::from_msg(format!("{err:#}")),
     655            0 :             ),
     656            0 :             Err(e @ tenant::CreateTimelineError::AncestorNotActive) => json_response(
     657            0 :                 StatusCode::SERVICE_UNAVAILABLE,
     658            0 :                 HttpErrorBody::from_msg(e.to_string()),
     659            0 :             ),
     660            0 :             Err(e @ tenant::CreateTimelineError::AncestorArchived) => json_response(
     661            0 :                 StatusCode::NOT_ACCEPTABLE,
     662            0 :                 HttpErrorBody::from_msg(e.to_string()),
     663            0 :             ),
     664            0 :             Err(tenant::CreateTimelineError::ShuttingDown) => json_response(
     665            0 :                 StatusCode::SERVICE_UNAVAILABLE,
     666            0 :                 HttpErrorBody::from_msg("tenant shutting down".to_string()),
     667            0 :             ),
     668            0 :             Err(tenant::CreateTimelineError::Other(err)) => Err(ApiError::InternalServerError(err)),
     669              :         }
     670            0 :     }
     671            0 :     .instrument(info_span!("timeline_create",
     672              :         tenant_id = %tenant_shard_id.tenant_id,
     673            0 :         shard_id = %tenant_shard_id.shard_slug(),
     674              :         timeline_id = %new_timeline_id,
     675              :     ))
     676            0 :     .await
     677            0 : }
     678              : 
     679            0 : async fn timeline_list_handler(
     680            0 :     request: Request<Body>,
     681            0 :     _cancel: CancellationToken,
     682            0 : ) -> Result<Response<Body>, ApiError> {
     683            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
     684            0 :     let include_non_incremental_logical_size: Option<bool> =
     685            0 :         parse_query_param(&request, "include-non-incremental-logical-size")?;
     686            0 :     let force_await_initial_logical_size: Option<bool> =
     687            0 :         parse_query_param(&request, "force-await-initial-logical-size")?;
     688            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
     689              : 
     690            0 :     let state = get_state(&request);
     691            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download);
     692              : 
     693            0 :     let response_data = async {
     694            0 :         let tenant = state
     695            0 :             .tenant_manager
     696            0 :             .get_attached_tenant_shard(tenant_shard_id)?;
     697              : 
     698            0 :         tenant.wait_to_become_active(ACTIVE_TENANT_TIMEOUT).await?;
     699              : 
     700            0 :         let timelines = tenant.list_timelines();
     701            0 : 
     702            0 :         let mut response_data = Vec::with_capacity(timelines.len());
     703            0 :         for timeline in timelines {
     704            0 :             let timeline_info = build_timeline_info(
     705            0 :                 &timeline,
     706            0 :                 include_non_incremental_logical_size.unwrap_or(false),
     707            0 :                 force_await_initial_logical_size.unwrap_or(false),
     708            0 :                 &ctx,
     709            0 :             )
     710            0 :             .instrument(info_span!("build_timeline_info", timeline_id = %timeline.timeline_id))
     711            0 :             .await
     712            0 :             .context("Failed to build timeline info")
     713            0 :             .map_err(ApiError::InternalServerError)?;
     714              : 
     715            0 :             response_data.push(timeline_info);
     716              :         }
     717            0 :         Ok::<Vec<TimelineInfo>, ApiError>(response_data)
     718            0 :     }
     719            0 :     .instrument(info_span!("timeline_list",
     720              :                 tenant_id = %tenant_shard_id.tenant_id,
     721            0 :                 shard_id = %tenant_shard_id.shard_slug()))
     722            0 :     .await?;
     723              : 
     724            0 :     json_response(StatusCode::OK, response_data)
     725            0 : }
     726              : 
     727            0 : async fn timeline_and_offloaded_list_handler(
     728            0 :     request: Request<Body>,
     729            0 :     _cancel: CancellationToken,
     730            0 : ) -> Result<Response<Body>, ApiError> {
     731            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
     732            0 :     let include_non_incremental_logical_size: Option<bool> =
     733            0 :         parse_query_param(&request, "include-non-incremental-logical-size")?;
     734            0 :     let force_await_initial_logical_size: Option<bool> =
     735            0 :         parse_query_param(&request, "force-await-initial-logical-size")?;
     736            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
     737              : 
     738            0 :     let state = get_state(&request);
     739            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download);
     740              : 
     741            0 :     let response_data = async {
     742            0 :         let tenant = state
     743            0 :             .tenant_manager
     744            0 :             .get_attached_tenant_shard(tenant_shard_id)?;
     745              : 
     746            0 :         tenant.wait_to_become_active(ACTIVE_TENANT_TIMEOUT).await?;
     747              : 
     748            0 :         let (timelines, offloadeds) = tenant.list_timelines_and_offloaded();
     749            0 : 
     750            0 :         let mut timeline_infos = Vec::with_capacity(timelines.len());
     751            0 :         for timeline in timelines {
     752            0 :             let timeline_info = build_timeline_info(
     753            0 :                 &timeline,
     754            0 :                 include_non_incremental_logical_size.unwrap_or(false),
     755            0 :                 force_await_initial_logical_size.unwrap_or(false),
     756            0 :                 &ctx,
     757            0 :             )
     758            0 :             .instrument(info_span!("build_timeline_info", timeline_id = %timeline.timeline_id))
     759            0 :             .await
     760            0 :             .context("Failed to build timeline info")
     761            0 :             .map_err(ApiError::InternalServerError)?;
     762              : 
     763            0 :             timeline_infos.push(timeline_info);
     764              :         }
     765            0 :         let offloaded_infos = offloadeds
     766            0 :             .into_iter()
     767            0 :             .map(|offloaded| build_timeline_offloaded_info(&offloaded))
     768            0 :             .collect::<Vec<_>>();
     769            0 :         let res = TimelinesInfoAndOffloaded {
     770            0 :             timelines: timeline_infos,
     771            0 :             offloaded: offloaded_infos,
     772            0 :         };
     773            0 :         Ok::<TimelinesInfoAndOffloaded, ApiError>(res)
     774            0 :     }
     775            0 :     .instrument(info_span!("timeline_and_offloaded_list",
     776              :                 tenant_id = %tenant_shard_id.tenant_id,
     777            0 :                 shard_id = %tenant_shard_id.shard_slug()))
     778            0 :     .await?;
     779              : 
     780            0 :     json_response(StatusCode::OK, response_data)
     781            0 : }
     782              : 
     783            0 : async fn timeline_preserve_initdb_handler(
     784            0 :     request: Request<Body>,
     785            0 :     _cancel: CancellationToken,
     786            0 : ) -> Result<Response<Body>, ApiError> {
     787            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
     788            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
     789            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
     790            0 :     let state = get_state(&request);
     791              : 
     792              :     // Part of the process for disaster recovery from safekeeper-stored WAL:
     793              :     // If we don't recover into a new timeline but want to keep the timeline ID,
     794              :     // then the initdb archive is deleted. This endpoint copies it to a different
     795              :     // location where timeline recreation cand find it.
     796              : 
     797            0 :     async {
     798            0 :         let tenant = state
     799            0 :             .tenant_manager
     800            0 :             .get_attached_tenant_shard(tenant_shard_id)?;
     801              : 
     802            0 :         let timeline = tenant.get_timeline(timeline_id, false)?;
     803              : 
     804            0 :         timeline
     805            0 :             .preserve_initdb_archive()
     806            0 :             .await
     807            0 :             .context("preserving initdb archive")
     808            0 :             .map_err(ApiError::InternalServerError)?;
     809              : 
     810            0 :         Ok::<_, ApiError>(())
     811            0 :     }
     812            0 :     .instrument(info_span!("timeline_preserve_initdb_archive",
     813              :                 tenant_id = %tenant_shard_id.tenant_id,
     814            0 :                 shard_id = %tenant_shard_id.shard_slug(),
     815              :                 %timeline_id))
     816            0 :     .await?;
     817              : 
     818            0 :     json_response(StatusCode::OK, ())
     819            0 : }
     820              : 
     821            0 : async fn timeline_archival_config_handler(
     822            0 :     mut request: Request<Body>,
     823            0 :     _cancel: CancellationToken,
     824            0 : ) -> Result<Response<Body>, ApiError> {
     825            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
     826            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
     827              : 
     828            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Warn);
     829              : 
     830            0 :     let request_data: TimelineArchivalConfigRequest = json_request(&mut request).await?;
     831            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
     832            0 :     let state = get_state(&request);
     833              : 
     834            0 :     async {
     835            0 :         let tenant = state
     836            0 :             .tenant_manager
     837            0 :             .get_attached_tenant_shard(tenant_shard_id)?;
     838              : 
     839            0 :         tenant.wait_to_become_active(ACTIVE_TENANT_TIMEOUT).await?;
     840              : 
     841            0 :         tenant
     842            0 :             .apply_timeline_archival_config(
     843            0 :                 timeline_id,
     844            0 :                 request_data.state,
     845            0 :                 state.broker_client.clone(),
     846            0 :                 ctx,
     847            0 :             )
     848            0 :             .await?;
     849            0 :         Ok::<_, ApiError>(())
     850            0 :     }
     851            0 :     .instrument(info_span!("timeline_archival_config",
     852              :                 tenant_id = %tenant_shard_id.tenant_id,
     853            0 :                 shard_id = %tenant_shard_id.shard_slug(),
     854              :                 state = ?request_data.state,
     855              :                 %timeline_id))
     856            0 :     .await?;
     857              : 
     858            0 :     json_response(StatusCode::OK, ())
     859            0 : }
     860              : 
     861              : /// This API is used to patch the index part of a timeline. You must ensure such patches are safe to apply. Use this API as an emergency
     862              : /// measure only.
     863              : ///
     864              : /// Some examples of safe patches:
     865              : /// - Increase the gc_cutoff and gc_compaction_cutoff to a larger value in case of a bug that didn't bump the cutoff and cause read errors.
     866              : /// - Force set the index part to use reldir v2 (migrating/migrated).
     867              : ///
     868              : /// Some examples of unsafe patches:
     869              : /// - Force set the index part from v2 to v1 (legacy). This will cause the code path to ignore anything written to the new keyspace and cause
     870              : ///   errors.
     871              : /// - Decrease the gc_cutoff without validating the data really exists. It will cause read errors in the background.
     872            0 : async fn timeline_patch_index_part_handler(
     873            0 :     mut request: Request<Body>,
     874            0 :     _cancel: CancellationToken,
     875            0 : ) -> Result<Response<Body>, ApiError> {
     876            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
     877            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
     878              : 
     879            0 :     let request_data: TimelinePatchIndexPartRequest = json_request(&mut request).await?;
     880            0 :     check_permission(&request, None)?; // require global permission for this request
     881            0 :     let state = get_state(&request);
     882              : 
     883            0 :     async {
     884            0 :         let timeline =
     885            0 :             active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
     886            0 :                 .await?;
     887              : 
     888            0 :         if let Some(rel_size_migration) = request_data.rel_size_migration {
     889            0 :             timeline
     890            0 :                 .update_rel_size_v2_status(rel_size_migration)
     891            0 :                 .map_err(ApiError::InternalServerError)?;
     892            0 :         }
     893              : 
     894            0 :         if let Some(gc_compaction_last_completed_lsn) =
     895            0 :             request_data.gc_compaction_last_completed_lsn
     896              :         {
     897            0 :             timeline
     898            0 :                 .update_gc_compaction_state(GcCompactionState {
     899            0 :                     last_completed_lsn: gc_compaction_last_completed_lsn,
     900            0 :                 })
     901            0 :                 .map_err(ApiError::InternalServerError)?;
     902            0 :         }
     903              : 
     904            0 :         if let Some(applied_gc_cutoff_lsn) = request_data.applied_gc_cutoff_lsn {
     905            0 :             {
     906            0 :                 let guard = timeline.applied_gc_cutoff_lsn.lock_for_write();
     907            0 :                 guard.store_and_unlock(applied_gc_cutoff_lsn);
     908            0 :             }
     909            0 :         }
     910              : 
     911            0 :         if request_data.force_index_update {
     912            0 :             timeline
     913            0 :                 .remote_client
     914            0 :                 .force_schedule_index_upload()
     915            0 :                 .context("force schedule index upload")
     916            0 :                 .map_err(ApiError::InternalServerError)?;
     917            0 :         }
     918              : 
     919            0 :         Ok::<_, ApiError>(())
     920            0 :     }
     921            0 :     .instrument(info_span!("timeline_patch_index_part",
     922              :                 tenant_id = %tenant_shard_id.tenant_id,
     923            0 :                 shard_id = %tenant_shard_id.shard_slug(),
     924              :                 %timeline_id))
     925            0 :     .await?;
     926              : 
     927            0 :     json_response(StatusCode::OK, ())
     928            0 : }
     929              : 
     930            0 : async fn timeline_detail_handler(
     931            0 :     request: Request<Body>,
     932            0 :     _cancel: CancellationToken,
     933            0 : ) -> Result<Response<Body>, ApiError> {
     934            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
     935            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
     936            0 :     let include_non_incremental_logical_size: Option<bool> =
     937            0 :         parse_query_param(&request, "include-non-incremental-logical-size")?;
     938            0 :     let force_await_initial_logical_size: Option<bool> =
     939            0 :         parse_query_param(&request, "force-await-initial-logical-size")?;
     940            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
     941              : 
     942              :     // Logical size calculation needs downloading.
     943            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download);
     944            0 :     let state = get_state(&request);
     945              : 
     946            0 :     let timeline_info = async {
     947            0 :         let tenant = state
     948            0 :             .tenant_manager
     949            0 :             .get_attached_tenant_shard(tenant_shard_id)?;
     950              : 
     951            0 :         tenant.wait_to_become_active(ACTIVE_TENANT_TIMEOUT).await?;
     952              : 
     953            0 :         let timeline = tenant.get_timeline(timeline_id, false)?;
     954            0 :         let ctx = &ctx.with_scope_timeline(&timeline);
     955              : 
     956            0 :         let timeline_info = build_timeline_info(
     957            0 :             &timeline,
     958            0 :             include_non_incremental_logical_size.unwrap_or(false),
     959            0 :             force_await_initial_logical_size.unwrap_or(false),
     960            0 :             ctx,
     961            0 :         )
     962            0 :         .await
     963            0 :         .context("get local timeline info")
     964            0 :         .map_err(ApiError::InternalServerError)?;
     965              : 
     966            0 :         Ok::<_, ApiError>(timeline_info)
     967            0 :     }
     968            0 :     .instrument(info_span!("timeline_detail",
     969              :                 tenant_id = %tenant_shard_id.tenant_id,
     970            0 :                 shard_id = %tenant_shard_id.shard_slug(),
     971              :                 %timeline_id))
     972            0 :     .await?;
     973              : 
     974            0 :     json_response(StatusCode::OK, timeline_info)
     975            0 : }
     976              : 
     977            0 : async fn get_lsn_by_timestamp_handler(
     978            0 :     request: Request<Body>,
     979            0 :     cancel: CancellationToken,
     980            0 : ) -> Result<Response<Body>, ApiError> {
     981            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
     982            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
     983            0 :     let state = get_state(&request);
     984            0 : 
     985            0 :     if !tenant_shard_id.is_shard_zero() {
     986              :         // Requires SLRU contents, which are only stored on shard zero
     987            0 :         return Err(ApiError::BadRequest(anyhow!(
     988            0 :             "Size calculations are only available on shard zero"
     989            0 :         )));
     990            0 :     }
     991              : 
     992            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
     993            0 :     let timestamp_raw = must_get_query_param(&request, "timestamp")?;
     994            0 :     let timestamp = humantime::parse_rfc3339(&timestamp_raw)
     995            0 :         .with_context(|| format!("Invalid time: {:?}", timestamp_raw))
     996            0 :         .map_err(ApiError::BadRequest)?;
     997            0 :     let timestamp_pg = postgres_ffi::to_pg_timestamp(timestamp);
     998              : 
     999            0 :     let with_lease = parse_query_param(&request, "with_lease")?.unwrap_or(false);
    1000              : 
    1001            0 :     let timeline =
    1002            0 :         active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
    1003            0 :             .await?;
    1004            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download)
    1005            0 :         .with_scope_timeline(&timeline);
    1006            0 :     let result = timeline
    1007            0 :         .find_lsn_for_timestamp(timestamp_pg, &cancel, &ctx)
    1008            0 :         .await?;
    1009              : 
    1010              :     #[derive(serde::Serialize, Debug)]
    1011              :     struct Result {
    1012              :         lsn: Lsn,
    1013              :         kind: &'static str,
    1014              :         #[serde(default)]
    1015              :         #[serde(skip_serializing_if = "Option::is_none")]
    1016              :         #[serde(flatten)]
    1017              :         lease: Option<LsnLease>,
    1018              :     }
    1019            0 :     let (lsn, kind) = match result {
    1020            0 :         LsnForTimestamp::Present(lsn) => (lsn, "present"),
    1021            0 :         LsnForTimestamp::Future(lsn) => (lsn, "future"),
    1022            0 :         LsnForTimestamp::Past(lsn) => (lsn, "past"),
    1023            0 :         LsnForTimestamp::NoData(lsn) => (lsn, "nodata"),
    1024              :     };
    1025              : 
    1026            0 :     let lease = if with_lease {
    1027            0 :         timeline
    1028            0 :             .init_lsn_lease(lsn, timeline.get_lsn_lease_length_for_ts(), &ctx)
    1029            0 :             .inspect_err(|_| {
    1030            0 :                 warn!("fail to grant a lease to {}", lsn);
    1031            0 :             })
    1032            0 :             .ok()
    1033              :     } else {
    1034            0 :         None
    1035              :     };
    1036              : 
    1037            0 :     let result = Result { lsn, kind, lease };
    1038            0 :     let valid_until = result
    1039            0 :         .lease
    1040            0 :         .as_ref()
    1041            0 :         .map(|l| humantime::format_rfc3339_millis(l.valid_until).to_string());
    1042            0 :     tracing::info!(
    1043              :         lsn=?result.lsn,
    1044              :         kind=%result.kind,
    1045              :         timestamp=%timestamp_raw,
    1046              :         valid_until=?valid_until,
    1047            0 :         "lsn_by_timestamp finished"
    1048              :     );
    1049            0 :     json_response(StatusCode::OK, result)
    1050            0 : }
    1051              : 
    1052            0 : async fn get_timestamp_of_lsn_handler(
    1053            0 :     request: Request<Body>,
    1054            0 :     _cancel: CancellationToken,
    1055            0 : ) -> Result<Response<Body>, ApiError> {
    1056            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    1057            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    1058            0 :     let state = get_state(&request);
    1059            0 : 
    1060            0 :     if !tenant_shard_id.is_shard_zero() {
    1061              :         // Requires SLRU contents, which are only stored on shard zero
    1062            0 :         return Err(ApiError::BadRequest(anyhow!(
    1063            0 :             "Size calculations are only available on shard zero"
    1064            0 :         )));
    1065            0 :     }
    1066              : 
    1067            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    1068              : 
    1069            0 :     let lsn_str = must_get_query_param(&request, "lsn")?;
    1070            0 :     let lsn = Lsn::from_str(&lsn_str)
    1071            0 :         .with_context(|| format!("Invalid LSN: {lsn_str:?}"))
    1072            0 :         .map_err(ApiError::BadRequest)?;
    1073              : 
    1074            0 :     let timeline =
    1075            0 :         active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
    1076            0 :             .await?;
    1077            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download)
    1078            0 :         .with_scope_timeline(&timeline);
    1079            0 :     let result = timeline.get_timestamp_for_lsn(lsn, &ctx).await?;
    1080              : 
    1081            0 :     match result {
    1082            0 :         Some(time) => {
    1083            0 :             let time = format_rfc3339(
    1084            0 :                 postgres_ffi::try_from_pg_timestamp(time).map_err(ApiError::InternalServerError)?,
    1085              :             )
    1086            0 :             .to_string();
    1087            0 :             json_response(StatusCode::OK, time)
    1088              :         }
    1089            0 :         None => Err(ApiError::NotFound(
    1090            0 :             anyhow::anyhow!("Timestamp for lsn {} not found", lsn).into(),
    1091            0 :         )),
    1092              :     }
    1093            0 : }
    1094              : 
    1095            0 : async fn timeline_delete_handler(
    1096            0 :     request: Request<Body>,
    1097            0 :     _cancel: CancellationToken,
    1098            0 : ) -> Result<Response<Body>, ApiError> {
    1099            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    1100            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    1101            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    1102              : 
    1103            0 :     let state = get_state(&request);
    1104              : 
    1105            0 :     let tenant = state
    1106            0 :         .tenant_manager
    1107            0 :         .get_attached_tenant_shard(tenant_shard_id)
    1108            0 :         .map_err(|e| {
    1109            0 :             match e {
    1110              :                 // GetTenantError has a built-in conversion to ApiError, but in this context we don't
    1111              :                 // want to treat missing tenants as 404, to avoid ambiguity with successful deletions.
    1112              :                 GetTenantError::NotFound(_) | GetTenantError::ShardNotFound(_) => {
    1113            0 :                     ApiError::PreconditionFailed(
    1114            0 :                         "Requested tenant is missing".to_string().into_boxed_str(),
    1115            0 :                     )
    1116              :                 }
    1117            0 :                 e => e.into(),
    1118              :             }
    1119            0 :         })?;
    1120            0 :     tenant.wait_to_become_active(ACTIVE_TENANT_TIMEOUT).await?;
    1121            0 :     tenant.delete_timeline(timeline_id).instrument(info_span!("timeline_delete", tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug(), %timeline_id))
    1122            0 :         .await?;
    1123              : 
    1124            0 :     json_response(StatusCode::ACCEPTED, ())
    1125            0 : }
    1126              : 
    1127            0 : async fn tenant_reset_handler(
    1128            0 :     request: Request<Body>,
    1129            0 :     _cancel: CancellationToken,
    1130            0 : ) -> Result<Response<Body>, ApiError> {
    1131            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    1132            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    1133              : 
    1134            0 :     let drop_cache: Option<bool> = parse_query_param(&request, "drop_cache")?;
    1135              : 
    1136            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Warn);
    1137            0 :     let state = get_state(&request);
    1138            0 :     state
    1139            0 :         .tenant_manager
    1140            0 :         .reset_tenant(tenant_shard_id, drop_cache.unwrap_or(false), &ctx)
    1141            0 :         .await
    1142            0 :         .map_err(ApiError::InternalServerError)?;
    1143              : 
    1144            0 :     json_response(StatusCode::OK, ())
    1145            0 : }
    1146              : 
    1147            0 : async fn tenant_list_handler(
    1148            0 :     request: Request<Body>,
    1149            0 :     _cancel: CancellationToken,
    1150            0 : ) -> Result<Response<Body>, ApiError> {
    1151            0 :     check_permission(&request, None)?;
    1152            0 :     let state = get_state(&request);
    1153              : 
    1154            0 :     let response_data = state
    1155            0 :         .tenant_manager
    1156            0 :         .list_tenants()
    1157            0 :         .map_err(|_| {
    1158            0 :             ApiError::ResourceUnavailable("Tenant map is initializing or shutting down".into())
    1159            0 :         })?
    1160            0 :         .iter()
    1161            0 :         .map(|(id, state, gen_)| TenantInfo {
    1162            0 :             id: *id,
    1163            0 :             state: state.clone(),
    1164            0 :             current_physical_size: None,
    1165            0 :             attachment_status: state.attachment_status(),
    1166            0 :             generation: (*gen_)
    1167            0 :                 .into()
    1168            0 :                 .expect("Tenants are always attached with a generation"),
    1169            0 :             gc_blocking: None,
    1170            0 :         })
    1171            0 :         .collect::<Vec<TenantInfo>>();
    1172            0 : 
    1173            0 :     json_response(StatusCode::OK, response_data)
    1174            0 : }
    1175              : 
    1176            0 : async fn tenant_status(
    1177            0 :     request: Request<Body>,
    1178            0 :     _cancel: CancellationToken,
    1179            0 : ) -> Result<Response<Body>, ApiError> {
    1180            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    1181            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    1182            0 :     let state = get_state(&request);
    1183            0 : 
    1184            0 :     // In tests, sometimes we want to query the state of a tenant without auto-activating it if it's currently waiting.
    1185            0 :     let activate = true;
    1186              :     #[cfg(feature = "testing")]
    1187            0 :     let activate = parse_query_param(&request, "activate")?.unwrap_or(activate);
    1188              : 
    1189            0 :     let tenant_info = async {
    1190            0 :         let tenant = state
    1191            0 :             .tenant_manager
    1192            0 :             .get_attached_tenant_shard(tenant_shard_id)?;
    1193              : 
    1194            0 :         if activate {
    1195              :             // This is advisory: we prefer to let the tenant activate on-demand when this function is
    1196              :             // called, but it is still valid to return 200 and describe the current state of the tenant
    1197              :             // if it doesn't make it into an active state.
    1198            0 :             tenant
    1199            0 :                 .wait_to_become_active(ACTIVE_TENANT_TIMEOUT)
    1200            0 :                 .await
    1201            0 :                 .ok();
    1202            0 :         }
    1203              : 
    1204              :         // Calculate total physical size of all timelines
    1205            0 :         let mut current_physical_size = 0;
    1206            0 :         for timeline in tenant.list_timelines().iter() {
    1207            0 :             current_physical_size += timeline.layer_size_sum().await;
    1208              :         }
    1209              : 
    1210            0 :         let state = tenant.current_state();
    1211            0 :         Result::<_, ApiError>::Ok(TenantDetails {
    1212            0 :             tenant_info: TenantInfo {
    1213            0 :                 id: tenant_shard_id,
    1214            0 :                 state: state.clone(),
    1215            0 :                 current_physical_size: Some(current_physical_size),
    1216            0 :                 attachment_status: state.attachment_status(),
    1217            0 :                 generation: tenant
    1218            0 :                     .generation()
    1219            0 :                     .into()
    1220            0 :                     .expect("Tenants are always attached with a generation"),
    1221            0 :                 gc_blocking: tenant.gc_block.summary().map(|x| format!("{x:?}")),
    1222            0 :             },
    1223            0 :             walredo: tenant.wal_redo_manager_status(),
    1224            0 :             timelines: tenant.list_timeline_ids(),
    1225            0 :         })
    1226            0 :     }
    1227            0 :     .instrument(info_span!("tenant_status_handler",
    1228              :                 tenant_id = %tenant_shard_id.tenant_id,
    1229            0 :                 shard_id = %tenant_shard_id.shard_slug()))
    1230            0 :     .await?;
    1231              : 
    1232            0 :     json_response(StatusCode::OK, tenant_info)
    1233            0 : }
    1234              : 
    1235            0 : async fn tenant_delete_handler(
    1236            0 :     request: Request<Body>,
    1237            0 :     _cancel: CancellationToken,
    1238            0 : ) -> Result<Response<Body>, ApiError> {
    1239              :     // TODO openapi spec
    1240            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    1241            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    1242              : 
    1243            0 :     let state = get_state(&request);
    1244            0 : 
    1245            0 :     state
    1246            0 :         .tenant_manager
    1247            0 :         .delete_tenant(tenant_shard_id)
    1248            0 :         .instrument(info_span!("tenant_delete_handler",
    1249              :             tenant_id = %tenant_shard_id.tenant_id,
    1250            0 :             shard_id = %tenant_shard_id.shard_slug()
    1251              :         ))
    1252            0 :         .await?;
    1253              : 
    1254            0 :     json_response(StatusCode::OK, ())
    1255            0 : }
    1256              : 
    1257              : /// HTTP endpoint to query the current tenant_size of a tenant.
    1258              : ///
    1259              : /// This is not used by consumption metrics under [`crate::consumption_metrics`], but can be used
    1260              : /// to debug any of the calculations. Requires `tenant_id` request parameter, supports
    1261              : /// `inputs_only=true|false` (default false) which supports debugging failure to calculate model
    1262              : /// values.
    1263              : ///
    1264              : /// 'retention_period' query parameter overrides the cutoff that is used to calculate the size
    1265              : /// (only if it is shorter than the real cutoff).
    1266              : ///
    1267              : /// Note: we don't update the cached size and prometheus metric here.
    1268              : /// The retention period might be different, and it's nice to have a method to just calculate it
    1269              : /// without modifying anything anyway.
    1270            0 : async fn tenant_size_handler(
    1271            0 :     request: Request<Body>,
    1272            0 :     cancel: CancellationToken,
    1273            0 : ) -> Result<Response<Body>, ApiError> {
    1274            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    1275            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    1276            0 :     let inputs_only: Option<bool> = parse_query_param(&request, "inputs_only")?;
    1277            0 :     let retention_period: Option<u64> = parse_query_param(&request, "retention_period")?;
    1278            0 :     let headers = request.headers();
    1279            0 :     let state = get_state(&request);
    1280            0 : 
    1281            0 :     if !tenant_shard_id.is_shard_zero() {
    1282            0 :         return Err(ApiError::BadRequest(anyhow!(
    1283            0 :             "Size calculations are only available on shard zero"
    1284            0 :         )));
    1285            0 :     }
    1286            0 : 
    1287            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download);
    1288            0 :     let tenant = state
    1289            0 :         .tenant_manager
    1290            0 :         .get_attached_tenant_shard(tenant_shard_id)?;
    1291            0 :     tenant.wait_to_become_active(ACTIVE_TENANT_TIMEOUT).await?;
    1292              : 
    1293              :     // this can be long operation
    1294            0 :     let inputs = tenant
    1295            0 :         .gather_size_inputs(
    1296            0 :             retention_period,
    1297            0 :             LogicalSizeCalculationCause::TenantSizeHandler,
    1298            0 :             &cancel,
    1299            0 :             &ctx,
    1300            0 :         )
    1301            0 :         .await
    1302            0 :         .map_err(|e| match e {
    1303            0 :             crate::tenant::size::CalculateSyntheticSizeError::Cancelled => ApiError::ShuttingDown,
    1304            0 :             other => ApiError::InternalServerError(anyhow::anyhow!(other)),
    1305            0 :         })?;
    1306              : 
    1307            0 :     let mut sizes = None;
    1308            0 :     let accepts_html = headers
    1309            0 :         .get(header::ACCEPT)
    1310            0 :         .map(|v| v == "text/html")
    1311            0 :         .unwrap_or_default();
    1312            0 :     if !inputs_only.unwrap_or(false) {
    1313            0 :         let storage_model = inputs.calculate_model();
    1314            0 :         let size = storage_model.calculate();
    1315            0 : 
    1316            0 :         // If request header expects html, return html
    1317            0 :         if accepts_html {
    1318            0 :             return synthetic_size_html_response(inputs, storage_model, size);
    1319            0 :         }
    1320            0 :         sizes = Some(size);
    1321            0 :     } else if accepts_html {
    1322            0 :         return Err(ApiError::BadRequest(anyhow!(
    1323            0 :             "inputs_only parameter is incompatible with html output request"
    1324            0 :         )));
    1325            0 :     }
    1326              : 
    1327              :     /// The type resides in the pageserver not to expose `ModelInputs`.
    1328              :     #[derive(serde::Serialize)]
    1329              :     struct TenantHistorySize {
    1330              :         id: TenantId,
    1331              :         /// Size is a mixture of WAL and logical size, so the unit is bytes.
    1332              :         ///
    1333              :         /// Will be none if `?inputs_only=true` was given.
    1334              :         size: Option<u64>,
    1335              :         /// Size of each segment used in the model.
    1336              :         /// Will be null if `?inputs_only=true` was given.
    1337              :         segment_sizes: Option<Vec<tenant_size_model::SegmentSizeResult>>,
    1338              :         inputs: crate::tenant::size::ModelInputs,
    1339              :     }
    1340              : 
    1341            0 :     json_response(
    1342            0 :         StatusCode::OK,
    1343            0 :         TenantHistorySize {
    1344            0 :             id: tenant_shard_id.tenant_id,
    1345            0 :             size: sizes.as_ref().map(|x| x.total_size),
    1346            0 :             segment_sizes: sizes.map(|x| x.segments),
    1347            0 :             inputs,
    1348            0 :         },
    1349            0 :     )
    1350            0 : }
    1351              : 
    1352            0 : async fn tenant_shard_split_handler(
    1353            0 :     mut request: Request<Body>,
    1354            0 :     _cancel: CancellationToken,
    1355            0 : ) -> Result<Response<Body>, ApiError> {
    1356            0 :     let req: TenantShardSplitRequest = json_request(&mut request).await?;
    1357              : 
    1358            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    1359            0 :     let state = get_state(&request);
    1360            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Warn);
    1361              : 
    1362            0 :     let tenant = state
    1363            0 :         .tenant_manager
    1364            0 :         .get_attached_tenant_shard(tenant_shard_id)?;
    1365            0 :     tenant.wait_to_become_active(ACTIVE_TENANT_TIMEOUT).await?;
    1366              : 
    1367            0 :     let new_shards = state
    1368            0 :         .tenant_manager
    1369            0 :         .shard_split(
    1370            0 :             tenant,
    1371            0 :             ShardCount::new(req.new_shard_count),
    1372            0 :             req.new_stripe_size,
    1373            0 :             &ctx,
    1374            0 :         )
    1375            0 :         .await
    1376            0 :         .map_err(ApiError::InternalServerError)?;
    1377              : 
    1378            0 :     json_response(StatusCode::OK, TenantShardSplitResponse { new_shards })
    1379            0 : }
    1380              : 
    1381            0 : async fn layer_map_info_handler(
    1382            0 :     request: Request<Body>,
    1383            0 :     _cancel: CancellationToken,
    1384            0 : ) -> Result<Response<Body>, ApiError> {
    1385            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    1386            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    1387            0 :     let reset: LayerAccessStatsReset =
    1388            0 :         parse_query_param(&request, "reset")?.unwrap_or(LayerAccessStatsReset::NoReset);
    1389            0 :     let state = get_state(&request);
    1390            0 : 
    1391            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    1392              : 
    1393            0 :     let timeline =
    1394            0 :         active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
    1395            0 :             .await?;
    1396            0 :     let layer_map_info = timeline
    1397            0 :         .layer_map_info(reset)
    1398            0 :         .await
    1399            0 :         .map_err(|_shutdown| ApiError::ShuttingDown)?;
    1400              : 
    1401            0 :     json_response(StatusCode::OK, layer_map_info)
    1402            0 : }
    1403              : 
    1404              : #[instrument(skip_all, fields(tenant_id, shard_id, timeline_id, layer_name))]
    1405              : async fn timeline_layer_scan_disposable_keys(
    1406              :     request: Request<Body>,
    1407              :     cancel: CancellationToken,
    1408              : ) -> Result<Response<Body>, ApiError> {
    1409              :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    1410              :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    1411              :     let layer_name: LayerName = parse_request_param(&request, "layer_name")?;
    1412              : 
    1413              :     tracing::Span::current().record(
    1414              :         "tenant_id",
    1415              :         tracing::field::display(&tenant_shard_id.tenant_id),
    1416              :     );
    1417              :     tracing::Span::current().record(
    1418              :         "shard_id",
    1419              :         tracing::field::display(tenant_shard_id.shard_slug()),
    1420              :     );
    1421              :     tracing::Span::current().record("timeline_id", tracing::field::display(&timeline_id));
    1422              :     tracing::Span::current().record("layer_name", tracing::field::display(&layer_name));
    1423              : 
    1424              :     let state = get_state(&request);
    1425              : 
    1426              :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    1427              : 
    1428              :     // technically the timeline need not be active for this scan to complete
    1429              :     let timeline =
    1430              :         active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
    1431              :             .await?;
    1432              : 
    1433              :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download)
    1434              :         .with_scope_timeline(&timeline);
    1435              : 
    1436              :     let guard = timeline.layers.read().await;
    1437              :     let Some(layer) = guard.try_get_from_key(&layer_name.clone().into()) else {
    1438              :         return Err(ApiError::NotFound(
    1439              :             anyhow::anyhow!("Layer {tenant_shard_id}/{timeline_id}/{layer_name} not found").into(),
    1440              :         ));
    1441              :     };
    1442              : 
    1443              :     let resident_layer = layer
    1444              :         .download_and_keep_resident(&ctx)
    1445              :         .await
    1446            0 :         .map_err(|err| match err {
    1447              :             tenant::storage_layer::layer::DownloadError::TimelineShutdown
    1448              :             | tenant::storage_layer::layer::DownloadError::DownloadCancelled => {
    1449            0 :                 ApiError::ShuttingDown
    1450              :             }
    1451              :             tenant::storage_layer::layer::DownloadError::ContextAndConfigReallyDeniesDownloads
    1452              :             | tenant::storage_layer::layer::DownloadError::DownloadRequired
    1453              :             | tenant::storage_layer::layer::DownloadError::NotFile(_)
    1454              :             | tenant::storage_layer::layer::DownloadError::DownloadFailed
    1455              :             | tenant::storage_layer::layer::DownloadError::PreStatFailed(_) => {
    1456            0 :                 ApiError::InternalServerError(err.into())
    1457              :             }
    1458              :             #[cfg(test)]
    1459              :             tenant::storage_layer::layer::DownloadError::Failpoint(_) => {
    1460            0 :                 ApiError::InternalServerError(err.into())
    1461              :             }
    1462            0 :         })?;
    1463              : 
    1464              :     let keys = resident_layer
    1465              :         .load_keys(&ctx)
    1466              :         .await
    1467              :         .map_err(ApiError::InternalServerError)?;
    1468              : 
    1469              :     let shard_identity = timeline.get_shard_identity();
    1470              : 
    1471              :     let mut disposable_count = 0;
    1472              :     let mut not_disposable_count = 0;
    1473              :     let cancel = cancel.clone();
    1474              :     for (i, key) in keys.into_iter().enumerate() {
    1475              :         if shard_identity.is_key_disposable(&key) {
    1476              :             disposable_count += 1;
    1477              :             tracing::debug!(key = %key, key.dbg=?key, "disposable key");
    1478              :         } else {
    1479              :             not_disposable_count += 1;
    1480              :         }
    1481              :         #[allow(clippy::collapsible_if)]
    1482              :         if i % 10000 == 0 {
    1483              :             if cancel.is_cancelled() || timeline.cancel.is_cancelled() || timeline.is_stopping() {
    1484              :                 return Err(ApiError::ShuttingDown);
    1485              :             }
    1486              :         }
    1487              :     }
    1488              : 
    1489              :     json_response(
    1490              :         StatusCode::OK,
    1491              :         pageserver_api::models::ScanDisposableKeysResponse {
    1492              :             disposable_count,
    1493              :             not_disposable_count,
    1494              :         },
    1495              :     )
    1496              : }
    1497              : 
    1498            0 : async fn timeline_download_heatmap_layers_handler(
    1499            0 :     request: Request<Body>,
    1500            0 :     _cancel: CancellationToken,
    1501            0 : ) -> Result<Response<Body>, ApiError> {
    1502              :     // Only used in the case where remote storage is not configured.
    1503              :     const DEFAULT_MAX_CONCURRENCY: usize = 100;
    1504              :     // A conservative default.
    1505              :     const DEFAULT_CONCURRENCY: usize = 16;
    1506              : 
    1507            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    1508            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    1509              : 
    1510            0 :     let desired_concurrency =
    1511            0 :         parse_query_param(&request, "concurrency")?.unwrap_or(DEFAULT_CONCURRENCY);
    1512            0 :     let recurse = parse_query_param(&request, "recurse")?.unwrap_or(false);
    1513            0 : 
    1514            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    1515              : 
    1516            0 :     let state = get_state(&request);
    1517            0 :     let timeline =
    1518            0 :         active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
    1519            0 :             .await?;
    1520            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download)
    1521            0 :         .with_scope_timeline(&timeline);
    1522            0 : 
    1523            0 :     let max_concurrency = get_config(&request)
    1524            0 :         .remote_storage_config
    1525            0 :         .as_ref()
    1526            0 :         .map(|c| c.concurrency_limit())
    1527            0 :         .unwrap_or(DEFAULT_MAX_CONCURRENCY);
    1528            0 :     let concurrency = std::cmp::min(max_concurrency, desired_concurrency);
    1529            0 : 
    1530            0 :     timeline.start_heatmap_layers_download(concurrency, recurse, &ctx)?;
    1531              : 
    1532            0 :     json_response(StatusCode::ACCEPTED, ())
    1533            0 : }
    1534              : 
    1535            0 : async fn timeline_shutdown_download_heatmap_layers_handler(
    1536            0 :     request: Request<Body>,
    1537            0 :     _cancel: CancellationToken,
    1538            0 : ) -> Result<Response<Body>, ApiError> {
    1539            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    1540            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    1541              : 
    1542            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    1543              : 
    1544            0 :     let state = get_state(&request);
    1545            0 :     let timeline =
    1546            0 :         active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
    1547            0 :             .await?;
    1548              : 
    1549            0 :     timeline.stop_and_drain_heatmap_layers_download().await;
    1550              : 
    1551            0 :     json_response(StatusCode::OK, ())
    1552            0 : }
    1553              : 
    1554            0 : async fn layer_download_handler(
    1555            0 :     request: Request<Body>,
    1556            0 :     _cancel: CancellationToken,
    1557            0 : ) -> Result<Response<Body>, ApiError> {
    1558            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    1559            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    1560            0 :     let layer_file_name = get_request_param(&request, "layer_file_name")?;
    1561            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    1562            0 :     let layer_name = LayerName::from_str(layer_file_name)
    1563            0 :         .map_err(|s| ApiError::BadRequest(anyhow::anyhow!(s)))?;
    1564            0 :     let state = get_state(&request);
    1565              : 
    1566            0 :     let timeline =
    1567            0 :         active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
    1568            0 :             .await?;
    1569            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download)
    1570            0 :         .with_scope_timeline(&timeline);
    1571            0 :     let downloaded = timeline
    1572            0 :         .download_layer(&layer_name, &ctx)
    1573            0 :         .await
    1574            0 :         .map_err(|e| match e {
    1575              :             tenant::storage_layer::layer::DownloadError::TimelineShutdown
    1576              :             | tenant::storage_layer::layer::DownloadError::DownloadCancelled => {
    1577            0 :                 ApiError::ShuttingDown
    1578              :             }
    1579            0 :             other => ApiError::InternalServerError(other.into()),
    1580            0 :         })?;
    1581              : 
    1582            0 :     match downloaded {
    1583            0 :         Some(true) => json_response(StatusCode::OK, ()),
    1584            0 :         Some(false) => json_response(StatusCode::NOT_MODIFIED, ()),
    1585            0 :         None => json_response(
    1586            0 :             StatusCode::BAD_REQUEST,
    1587            0 :             format!("Layer {tenant_shard_id}/{timeline_id}/{layer_file_name} not found"),
    1588            0 :         ),
    1589              :     }
    1590            0 : }
    1591              : 
    1592            0 : async fn evict_timeline_layer_handler(
    1593            0 :     request: Request<Body>,
    1594            0 :     _cancel: CancellationToken,
    1595            0 : ) -> Result<Response<Body>, ApiError> {
    1596            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    1597            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    1598            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    1599            0 :     let layer_file_name = get_request_param(&request, "layer_file_name")?;
    1600            0 :     let state = get_state(&request);
    1601              : 
    1602            0 :     let layer_name = LayerName::from_str(layer_file_name)
    1603            0 :         .map_err(|s| ApiError::BadRequest(anyhow::anyhow!(s)))?;
    1604              : 
    1605            0 :     let timeline =
    1606            0 :         active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
    1607            0 :             .await?;
    1608            0 :     let evicted = timeline
    1609            0 :         .evict_layer(&layer_name)
    1610            0 :         .await
    1611            0 :         .map_err(ApiError::InternalServerError)?;
    1612              : 
    1613            0 :     match evicted {
    1614            0 :         Some(true) => json_response(StatusCode::OK, ()),
    1615            0 :         Some(false) => json_response(StatusCode::NOT_MODIFIED, ()),
    1616            0 :         None => json_response(
    1617            0 :             StatusCode::BAD_REQUEST,
    1618            0 :             format!("Layer {tenant_shard_id}/{timeline_id}/{layer_file_name} not found"),
    1619            0 :         ),
    1620              :     }
    1621            0 : }
    1622              : 
    1623            0 : async fn timeline_gc_blocking_handler(
    1624            0 :     request: Request<Body>,
    1625            0 :     _cancel: CancellationToken,
    1626            0 : ) -> Result<Response<Body>, ApiError> {
    1627            0 :     block_or_unblock_gc(request, true).await
    1628            0 : }
    1629              : 
    1630            0 : async fn timeline_gc_unblocking_handler(
    1631            0 :     request: Request<Body>,
    1632            0 :     _cancel: CancellationToken,
    1633            0 : ) -> Result<Response<Body>, ApiError> {
    1634            0 :     block_or_unblock_gc(request, false).await
    1635            0 : }
    1636              : 
    1637              : /// Traces GetPage@LSN requests for a timeline, and emits metadata in an efficient binary encoding.
    1638              : /// Use the `pagectl page-trace` command to decode and analyze the output.
    1639            0 : async fn timeline_page_trace_handler(
    1640            0 :     request: Request<Body>,
    1641            0 :     cancel: CancellationToken,
    1642            0 : ) -> Result<Response<Body>, ApiError> {
    1643            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    1644            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    1645            0 :     let state = get_state(&request);
    1646            0 :     check_permission(&request, None)?;
    1647              : 
    1648            0 :     let size_limit: usize = parse_query_param(&request, "size_limit_bytes")?.unwrap_or(1024 * 1024);
    1649            0 :     let time_limit_secs: u64 = parse_query_param(&request, "time_limit_secs")?.unwrap_or(5);
    1650              : 
    1651              :     // Convert size limit to event limit based on the serialized size of an event. The event size is
    1652              :     // fixed, as the default bincode serializer uses fixed-width integer encoding.
    1653            0 :     let event_size = bincode::serialize(&PageTraceEvent::default())
    1654            0 :         .map_err(|err| ApiError::InternalServerError(err.into()))?
    1655            0 :         .len();
    1656            0 :     let event_limit = size_limit / event_size;
    1657              : 
    1658            0 :     let timeline =
    1659            0 :         active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
    1660            0 :             .await?;
    1661              : 
    1662              :     // Install a page trace, unless one is already in progress. We just use a buffered channel,
    1663              :     // which may 2x the memory usage in the worst case, but it's still bounded.
    1664            0 :     let (trace_tx, mut trace_rx) = tokio::sync::mpsc::channel(event_limit);
    1665            0 :     let cur = timeline.page_trace.load();
    1666            0 :     let installed = cur.is_none()
    1667            0 :         && timeline
    1668            0 :             .page_trace
    1669            0 :             .compare_and_swap(cur, Some(Arc::new(trace_tx)))
    1670            0 :             .is_none();
    1671            0 :     if !installed {
    1672            0 :         return Err(ApiError::Conflict("page trace already active".to_string()));
    1673            0 :     }
    1674            0 :     defer!(timeline.page_trace.store(None)); // uninstall on return
    1675            0 : 
    1676            0 :     // Collect the trace and return it to the client. We could stream the response, but this is
    1677            0 :     // simple and fine.
    1678            0 :     let mut body = Vec::with_capacity(size_limit);
    1679            0 :     let deadline = Instant::now() + Duration::from_secs(time_limit_secs);
    1680              : 
    1681            0 :     while body.len() < size_limit {
    1682            0 :         tokio::select! {
    1683            0 :             event = trace_rx.recv() => {
    1684            0 :                 let Some(event) = event else {
    1685            0 :                     break; // shouldn't happen (sender doesn't close, unless timeline dropped)
    1686              :                 };
    1687            0 :                 bincode::serialize_into(&mut body, &event)
    1688            0 :                     .map_err(|err| ApiError::InternalServerError(err.into()))?;
    1689              :             }
    1690            0 :             _ = tokio::time::sleep_until(deadline) => break, // time limit reached
    1691            0 :             _ = cancel.cancelled() => return Err(ApiError::Cancelled),
    1692              :         }
    1693              :     }
    1694              : 
    1695            0 :     Ok(Response::builder()
    1696            0 :         .status(StatusCode::OK)
    1697            0 :         .header(header::CONTENT_TYPE, "application/octet-stream")
    1698            0 :         .body(hyper::Body::from(body))
    1699            0 :         .unwrap())
    1700            0 : }
    1701              : 
    1702              : /// Adding a block is `POST ../block_gc`, removing a block is `POST ../unblock_gc`.
    1703              : ///
    1704              : /// Both are technically unsafe because they might fire off index uploads, thus they are POST.
    1705            0 : async fn block_or_unblock_gc(
    1706            0 :     request: Request<Body>,
    1707            0 :     block: bool,
    1708            0 : ) -> Result<Response<Body>, ApiError> {
    1709              :     use crate::tenant::remote_timeline_client::WaitCompletionError;
    1710              :     use crate::tenant::upload_queue::NotInitialized;
    1711            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    1712            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    1713            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    1714            0 :     let state = get_state(&request);
    1715              : 
    1716            0 :     let tenant = state
    1717            0 :         .tenant_manager
    1718            0 :         .get_attached_tenant_shard(tenant_shard_id)?;
    1719              : 
    1720            0 :     tenant.wait_to_become_active(ACTIVE_TENANT_TIMEOUT).await?;
    1721              : 
    1722            0 :     let timeline = tenant.get_timeline(timeline_id, true)?;
    1723              : 
    1724            0 :     let fut = async {
    1725            0 :         if block {
    1726            0 :             timeline.block_gc(&tenant).await.map(|_| ())
    1727              :         } else {
    1728            0 :             timeline.unblock_gc(&tenant).await
    1729              :         }
    1730            0 :     };
    1731              : 
    1732            0 :     let span = tracing::info_span!(
    1733              :         "block_or_unblock_gc",
    1734              :         tenant_id = %tenant_shard_id.tenant_id,
    1735            0 :         shard_id = %tenant_shard_id.shard_slug(),
    1736              :         timeline_id = %timeline_id,
    1737              :         block = block,
    1738              :     );
    1739              : 
    1740            0 :     let res = fut.instrument(span).await;
    1741              : 
    1742            0 :     res.map_err(|e| {
    1743            0 :         if e.is::<NotInitialized>() || e.is::<WaitCompletionError>() {
    1744            0 :             ApiError::ShuttingDown
    1745              :         } else {
    1746            0 :             ApiError::InternalServerError(e)
    1747              :         }
    1748            0 :     })?;
    1749              : 
    1750            0 :     json_response(StatusCode::OK, ())
    1751            0 : }
    1752              : 
    1753              : /// Get tenant_size SVG graph along with the JSON data.
    1754            0 : fn synthetic_size_html_response(
    1755            0 :     inputs: ModelInputs,
    1756            0 :     storage_model: StorageModel,
    1757            0 :     sizes: SizeResult,
    1758            0 : ) -> Result<Response<Body>, ApiError> {
    1759            0 :     let mut timeline_ids: Vec<String> = Vec::new();
    1760            0 :     let mut timeline_map: HashMap<TimelineId, usize> = HashMap::new();
    1761            0 :     for (index, ti) in inputs.timeline_inputs.iter().enumerate() {
    1762            0 :         timeline_map.insert(ti.timeline_id, index);
    1763            0 :         timeline_ids.push(ti.timeline_id.to_string());
    1764            0 :     }
    1765            0 :     let seg_to_branch: Vec<(usize, SvgBranchKind)> = inputs
    1766            0 :         .segments
    1767            0 :         .iter()
    1768            0 :         .map(|seg| {
    1769            0 :             (
    1770            0 :                 *timeline_map.get(&seg.timeline_id).unwrap(),
    1771            0 :                 seg.kind.into(),
    1772            0 :             )
    1773            0 :         })
    1774            0 :         .collect();
    1775              : 
    1776            0 :     let svg =
    1777            0 :         tenant_size_model::svg::draw_svg(&storage_model, &timeline_ids, &seg_to_branch, &sizes)
    1778            0 :             .map_err(ApiError::InternalServerError)?;
    1779              : 
    1780            0 :     let mut response = String::new();
    1781              : 
    1782              :     use std::fmt::Write;
    1783            0 :     write!(response, "<html>\n<body>\n").unwrap();
    1784            0 :     write!(response, "<div>\n{svg}\n</div>").unwrap();
    1785            0 :     writeln!(response, "Project size: {}", sizes.total_size).unwrap();
    1786            0 :     writeln!(response, "<pre>").unwrap();
    1787            0 :     writeln!(
    1788            0 :         response,
    1789            0 :         "{}",
    1790            0 :         serde_json::to_string_pretty(&inputs).unwrap()
    1791            0 :     )
    1792            0 :     .unwrap();
    1793            0 :     writeln!(
    1794            0 :         response,
    1795            0 :         "{}",
    1796            0 :         serde_json::to_string_pretty(&sizes.segments).unwrap()
    1797            0 :     )
    1798            0 :     .unwrap();
    1799            0 :     writeln!(response, "</pre>").unwrap();
    1800            0 :     write!(response, "</body>\n</html>\n").unwrap();
    1801            0 : 
    1802            0 :     html_response(StatusCode::OK, response)
    1803            0 : }
    1804              : 
    1805            0 : pub fn html_response(status: StatusCode, data: String) -> Result<Response<Body>, ApiError> {
    1806            0 :     let response = Response::builder()
    1807            0 :         .status(status)
    1808            0 :         .header(header::CONTENT_TYPE, "text/html")
    1809            0 :         .body(Body::from(data.as_bytes().to_vec()))
    1810            0 :         .map_err(|e| ApiError::InternalServerError(e.into()))?;
    1811            0 :     Ok(response)
    1812            0 : }
    1813              : 
    1814            0 : async fn get_tenant_config_handler(
    1815            0 :     request: Request<Body>,
    1816            0 :     _cancel: CancellationToken,
    1817            0 : ) -> Result<Response<Body>, ApiError> {
    1818            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    1819            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    1820            0 :     let state = get_state(&request);
    1821              : 
    1822            0 :     let tenant = state
    1823            0 :         .tenant_manager
    1824            0 :         .get_attached_tenant_shard(tenant_shard_id)?;
    1825              : 
    1826            0 :     let response = HashMap::from([
    1827              :         (
    1828              :             "tenant_specific_overrides",
    1829            0 :             serde_json::to_value(tenant.tenant_specific_overrides())
    1830            0 :                 .context("serializing tenant specific overrides")
    1831            0 :                 .map_err(ApiError::InternalServerError)?,
    1832              :         ),
    1833              :         (
    1834            0 :             "effective_config",
    1835            0 :             serde_json::to_value(tenant.effective_config())
    1836            0 :                 .context("serializing effective config")
    1837            0 :                 .map_err(ApiError::InternalServerError)?,
    1838              :         ),
    1839              :     ]);
    1840              : 
    1841            0 :     json_response(StatusCode::OK, response)
    1842            0 : }
    1843              : 
    1844            0 : async fn update_tenant_config_handler(
    1845            0 :     mut request: Request<Body>,
    1846            0 :     _cancel: CancellationToken,
    1847            0 : ) -> Result<Response<Body>, ApiError> {
    1848            0 :     let request_data: TenantConfigRequest = json_request(&mut request).await?;
    1849            0 :     let tenant_id = request_data.tenant_id;
    1850            0 :     check_permission(&request, Some(tenant_id))?;
    1851              : 
    1852            0 :     let new_tenant_conf =
    1853            0 :         TenantConfOpt::try_from(&request_data.config).map_err(ApiError::BadRequest)?;
    1854              : 
    1855            0 :     let state = get_state(&request);
    1856            0 : 
    1857            0 :     let tenant_shard_id = TenantShardId::unsharded(tenant_id);
    1858              : 
    1859            0 :     let tenant = state
    1860            0 :         .tenant_manager
    1861            0 :         .get_attached_tenant_shard(tenant_shard_id)?;
    1862            0 :     tenant.wait_to_become_active(ACTIVE_TENANT_TIMEOUT).await?;
    1863              : 
    1864              :     // This is a legacy API that only operates on attached tenants: the preferred
    1865              :     // API to use is the location_config/ endpoint, which lets the caller provide
    1866              :     // the full LocationConf.
    1867            0 :     let location_conf = LocationConf::attached_single(
    1868            0 :         new_tenant_conf.clone(),
    1869            0 :         tenant.get_generation(),
    1870            0 :         &ShardParameters::default(),
    1871            0 :     );
    1872            0 : 
    1873            0 :     crate::tenant::Tenant::persist_tenant_config(state.conf, &tenant_shard_id, &location_conf)
    1874            0 :         .await
    1875            0 :         .map_err(|e| ApiError::InternalServerError(anyhow::anyhow!(e)))?;
    1876              : 
    1877            0 :     let _ = tenant
    1878            0 :         .update_tenant_config(|_crnt| Ok(new_tenant_conf.clone()))
    1879            0 :         .expect("Closure returns Ok()");
    1880            0 : 
    1881            0 :     json_response(StatusCode::OK, ())
    1882            0 : }
    1883              : 
    1884            0 : async fn patch_tenant_config_handler(
    1885            0 :     mut request: Request<Body>,
    1886            0 :     _cancel: CancellationToken,
    1887            0 : ) -> Result<Response<Body>, ApiError> {
    1888            0 :     let request_data: TenantConfigPatchRequest = json_request(&mut request).await?;
    1889            0 :     let tenant_id = request_data.tenant_id;
    1890            0 :     check_permission(&request, Some(tenant_id))?;
    1891              : 
    1892            0 :     let state = get_state(&request);
    1893            0 : 
    1894            0 :     let tenant_shard_id = TenantShardId::unsharded(tenant_id);
    1895              : 
    1896            0 :     let tenant = state
    1897            0 :         .tenant_manager
    1898            0 :         .get_attached_tenant_shard(tenant_shard_id)?;
    1899            0 :     tenant.wait_to_become_active(ACTIVE_TENANT_TIMEOUT).await?;
    1900              : 
    1901            0 :     let updated = tenant
    1902            0 :         .update_tenant_config(|crnt| crnt.apply_patch(request_data.config.clone()))
    1903            0 :         .map_err(ApiError::BadRequest)?;
    1904              : 
    1905              :     // This is a legacy API that only operates on attached tenants: the preferred
    1906              :     // API to use is the location_config/ endpoint, which lets the caller provide
    1907              :     // the full LocationConf.
    1908            0 :     let location_conf = LocationConf::attached_single(
    1909            0 :         updated,
    1910            0 :         tenant.get_generation(),
    1911            0 :         &ShardParameters::default(),
    1912            0 :     );
    1913            0 : 
    1914            0 :     crate::tenant::Tenant::persist_tenant_config(state.conf, &tenant_shard_id, &location_conf)
    1915            0 :         .await
    1916            0 :         .map_err(|e| ApiError::InternalServerError(anyhow::anyhow!(e)))?;
    1917              : 
    1918            0 :     json_response(StatusCode::OK, ())
    1919            0 : }
    1920              : 
    1921            0 : async fn put_tenant_location_config_handler(
    1922            0 :     mut request: Request<Body>,
    1923            0 :     _cancel: CancellationToken,
    1924            0 : ) -> Result<Response<Body>, ApiError> {
    1925            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    1926              : 
    1927            0 :     let request_data: TenantLocationConfigRequest = json_request(&mut request).await?;
    1928            0 :     let flush = parse_query_param(&request, "flush_ms")?.map(Duration::from_millis);
    1929            0 :     let lazy = parse_query_param(&request, "lazy")?.unwrap_or(false);
    1930            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    1931              : 
    1932            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Warn);
    1933            0 :     let state = get_state(&request);
    1934            0 :     let conf = state.conf;
    1935            0 : 
    1936            0 :     // The `Detached` state is special, it doesn't upsert a tenant, it removes
    1937            0 :     // its local disk content and drops it from memory.
    1938            0 :     if let LocationConfigMode::Detached = request_data.config.mode {
    1939            0 :         if let Err(e) = state
    1940            0 :             .tenant_manager
    1941            0 :             .detach_tenant(conf, tenant_shard_id, &state.deletion_queue_client)
    1942            0 :             .instrument(info_span!("tenant_detach",
    1943              :                 tenant_id = %tenant_shard_id.tenant_id,
    1944            0 :                 shard_id = %tenant_shard_id.shard_slug()
    1945              :             ))
    1946            0 :             .await
    1947              :         {
    1948            0 :             match e {
    1949            0 :                 TenantStateError::SlotError(TenantSlotError::NotFound(_)) => {
    1950            0 :                     // This API is idempotent: a NotFound on a detach is fine.
    1951            0 :                 }
    1952            0 :                 _ => return Err(e.into()),
    1953              :             }
    1954            0 :         }
    1955            0 :         return json_response(StatusCode::OK, ());
    1956            0 :     }
    1957              : 
    1958            0 :     let location_conf =
    1959            0 :         LocationConf::try_from(&request_data.config).map_err(ApiError::BadRequest)?;
    1960              : 
    1961              :     // lazy==true queues up for activation or jumps the queue like normal when a compute connects,
    1962              :     // similar to at startup ordering.
    1963            0 :     let spawn_mode = if lazy {
    1964            0 :         tenant::SpawnMode::Lazy
    1965              :     } else {
    1966            0 :         tenant::SpawnMode::Eager
    1967              :     };
    1968              : 
    1969            0 :     let tenant = state
    1970            0 :         .tenant_manager
    1971            0 :         .upsert_location(tenant_shard_id, location_conf, flush, spawn_mode, &ctx)
    1972            0 :         .await?;
    1973            0 :     let stripe_size = tenant.as_ref().map(|t| t.get_shard_stripe_size());
    1974            0 :     let attached = tenant.is_some();
    1975              : 
    1976            0 :     if let Some(_flush_ms) = flush {
    1977            0 :         match state
    1978            0 :             .secondary_controller
    1979            0 :             .upload_tenant(tenant_shard_id)
    1980            0 :             .await
    1981              :         {
    1982              :             Ok(()) => {
    1983            0 :                 tracing::info!("Uploaded heatmap during flush");
    1984              :             }
    1985            0 :             Err(e) => {
    1986            0 :                 tracing::warn!("Failed to flush heatmap: {e}");
    1987              :             }
    1988              :         }
    1989              :     } else {
    1990            0 :         tracing::info!("No flush requested when configuring");
    1991              :     }
    1992              : 
    1993              :     // This API returns a vector of pageservers where the tenant is attached: this is
    1994              :     // primarily for use in the sharding service.  For compatibilty, we also return this
    1995              :     // when called directly on a pageserver, but the payload is always zero or one shards.
    1996            0 :     let mut response = TenantLocationConfigResponse {
    1997            0 :         shards: Vec::new(),
    1998            0 :         stripe_size: None,
    1999            0 :     };
    2000            0 :     if attached {
    2001            0 :         response.shards.push(TenantShardLocation {
    2002            0 :             shard_id: tenant_shard_id,
    2003            0 :             node_id: state.conf.id,
    2004            0 :         });
    2005            0 :         if tenant_shard_id.shard_count.count() > 1 {
    2006              :             // Stripe size should be set if we are attached
    2007            0 :             debug_assert!(stripe_size.is_some());
    2008            0 :             response.stripe_size = stripe_size;
    2009            0 :         }
    2010            0 :     }
    2011              : 
    2012            0 :     json_response(StatusCode::OK, response)
    2013            0 : }
    2014              : 
    2015            0 : async fn list_location_config_handler(
    2016            0 :     request: Request<Body>,
    2017            0 :     _cancel: CancellationToken,
    2018            0 : ) -> Result<Response<Body>, ApiError> {
    2019            0 :     let state = get_state(&request);
    2020            0 :     let slots = state.tenant_manager.list();
    2021            0 :     let result = LocationConfigListResponse {
    2022            0 :         tenant_shards: slots
    2023            0 :             .into_iter()
    2024            0 :             .map(|(tenant_shard_id, slot)| {
    2025            0 :                 let v = match slot {
    2026            0 :                     TenantSlot::Attached(t) => Some(t.get_location_conf()),
    2027            0 :                     TenantSlot::Secondary(s) => Some(s.get_location_conf()),
    2028            0 :                     TenantSlot::InProgress(_) => None,
    2029              :                 };
    2030            0 :                 (tenant_shard_id, v)
    2031            0 :             })
    2032            0 :             .collect(),
    2033            0 :     };
    2034            0 :     json_response(StatusCode::OK, result)
    2035            0 : }
    2036              : 
    2037            0 : async fn get_location_config_handler(
    2038            0 :     request: Request<Body>,
    2039            0 :     _cancel: CancellationToken,
    2040            0 : ) -> Result<Response<Body>, ApiError> {
    2041            0 :     let state = get_state(&request);
    2042            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    2043            0 :     let slot = state.tenant_manager.get(tenant_shard_id);
    2044              : 
    2045            0 :     let Some(slot) = slot else {
    2046            0 :         return Err(ApiError::NotFound(
    2047            0 :             anyhow::anyhow!("Tenant shard not found").into(),
    2048            0 :         ));
    2049              :     };
    2050              : 
    2051            0 :     let result: Option<LocationConfig> = match slot {
    2052            0 :         TenantSlot::Attached(t) => Some(t.get_location_conf()),
    2053            0 :         TenantSlot::Secondary(s) => Some(s.get_location_conf()),
    2054            0 :         TenantSlot::InProgress(_) => None,
    2055              :     };
    2056              : 
    2057            0 :     json_response(StatusCode::OK, result)
    2058            0 : }
    2059              : 
    2060              : // Do a time travel recovery on the given tenant/tenant shard. Tenant needs to be detached
    2061              : // (from all pageservers) as it invalidates consistency assumptions.
    2062            0 : async fn tenant_time_travel_remote_storage_handler(
    2063            0 :     request: Request<Body>,
    2064            0 :     cancel: CancellationToken,
    2065            0 : ) -> Result<Response<Body>, ApiError> {
    2066            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    2067              : 
    2068            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    2069              : 
    2070            0 :     let timestamp_raw = must_get_query_param(&request, "travel_to")?;
    2071            0 :     let timestamp = humantime::parse_rfc3339(&timestamp_raw)
    2072            0 :         .with_context(|| format!("Invalid time for travel_to: {timestamp_raw:?}"))
    2073            0 :         .map_err(ApiError::BadRequest)?;
    2074              : 
    2075            0 :     let done_if_after_raw = must_get_query_param(&request, "done_if_after")?;
    2076            0 :     let done_if_after = humantime::parse_rfc3339(&done_if_after_raw)
    2077            0 :         .with_context(|| format!("Invalid time for done_if_after: {done_if_after_raw:?}"))
    2078            0 :         .map_err(ApiError::BadRequest)?;
    2079              : 
    2080              :     // This is just a sanity check to fend off naive wrong usages of the API:
    2081              :     // the tenant needs to be detached *everywhere*
    2082            0 :     let state = get_state(&request);
    2083            0 :     let we_manage_tenant = state.tenant_manager.manages_tenant_shard(tenant_shard_id);
    2084            0 :     if we_manage_tenant {
    2085            0 :         return Err(ApiError::BadRequest(anyhow!(
    2086            0 :             "Tenant {tenant_shard_id} is already attached at this pageserver"
    2087            0 :         )));
    2088            0 :     }
    2089            0 : 
    2090            0 :     if timestamp > done_if_after {
    2091            0 :         return Err(ApiError::BadRequest(anyhow!(
    2092            0 :             "The done_if_after timestamp comes before the timestamp to recover to"
    2093            0 :         )));
    2094            0 :     }
    2095            0 : 
    2096            0 :     tracing::info!(
    2097            0 :         "Issuing time travel request internally. timestamp={timestamp_raw}, done_if_after={done_if_after_raw}"
    2098              :     );
    2099              : 
    2100            0 :     remote_timeline_client::upload::time_travel_recover_tenant(
    2101            0 :         &state.remote_storage,
    2102            0 :         &tenant_shard_id,
    2103            0 :         timestamp,
    2104            0 :         done_if_after,
    2105            0 :         &cancel,
    2106            0 :     )
    2107            0 :     .await
    2108            0 :     .map_err(|e| match e {
    2109            0 :         TimeTravelError::BadInput(e) => {
    2110            0 :             warn!("bad input error: {e}");
    2111            0 :             ApiError::BadRequest(anyhow!("bad input error"))
    2112              :         }
    2113              :         TimeTravelError::Unimplemented => {
    2114            0 :             ApiError::BadRequest(anyhow!("unimplemented for the configured remote storage"))
    2115              :         }
    2116            0 :         TimeTravelError::Cancelled => ApiError::InternalServerError(anyhow!("cancelled")),
    2117              :         TimeTravelError::TooManyVersions => {
    2118            0 :             ApiError::InternalServerError(anyhow!("too many versions in remote storage"))
    2119              :         }
    2120            0 :         TimeTravelError::Other(e) => {
    2121            0 :             warn!("internal error: {e}");
    2122            0 :             ApiError::InternalServerError(anyhow!("internal error"))
    2123              :         }
    2124            0 :     })?;
    2125              : 
    2126            0 :     json_response(StatusCode::OK, ())
    2127            0 : }
    2128              : 
    2129              : /// Testing helper to transition a tenant to [`crate::tenant::TenantState::Broken`].
    2130            0 : async fn handle_tenant_break(
    2131            0 :     r: Request<Body>,
    2132            0 :     _cancel: CancellationToken,
    2133            0 : ) -> Result<Response<Body>, ApiError> {
    2134            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&r, "tenant_shard_id")?;
    2135              : 
    2136            0 :     let state = get_state(&r);
    2137            0 :     state
    2138            0 :         .tenant_manager
    2139            0 :         .get_attached_tenant_shard(tenant_shard_id)?
    2140            0 :         .set_broken("broken from test".to_owned())
    2141            0 :         .await;
    2142              : 
    2143            0 :     json_response(StatusCode::OK, ())
    2144            0 : }
    2145              : 
    2146              : // Obtains an lsn lease on the given timeline.
    2147            0 : async fn lsn_lease_handler(
    2148            0 :     mut request: Request<Body>,
    2149            0 :     _cancel: CancellationToken,
    2150            0 : ) -> Result<Response<Body>, ApiError> {
    2151            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    2152            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    2153            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    2154            0 :     let lsn = json_request::<LsnLeaseRequest>(&mut request).await?.lsn;
    2155              : 
    2156            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download);
    2157            0 : 
    2158            0 :     let state = get_state(&request);
    2159              : 
    2160            0 :     let timeline =
    2161            0 :         active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
    2162            0 :             .await?;
    2163              : 
    2164            0 :     let result = async {
    2165            0 :         timeline
    2166            0 :             .init_lsn_lease(lsn, timeline.get_lsn_lease_length(), &ctx)
    2167            0 :             .map_err(|e| {
    2168            0 :                 ApiError::InternalServerError(
    2169            0 :                     e.context(format!("invalid lsn lease request at {lsn}")),
    2170            0 :                 )
    2171            0 :             })
    2172            0 :     }
    2173            0 :     .instrument(info_span!("init_lsn_lease", tenant_id = %tenant_shard_id.tenant_id, shard_id = %tenant_shard_id.shard_slug(), %timeline_id))
    2174            0 :     .await?;
    2175              : 
    2176            0 :     json_response(StatusCode::OK, result)
    2177            0 : }
    2178              : 
    2179              : // Run GC immediately on given timeline.
    2180            0 : async fn timeline_gc_handler(
    2181            0 :     mut request: Request<Body>,
    2182            0 :     cancel: CancellationToken,
    2183            0 : ) -> Result<Response<Body>, ApiError> {
    2184            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    2185            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    2186            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    2187              : 
    2188            0 :     let gc_req: TimelineGcRequest = json_request(&mut request).await?;
    2189              : 
    2190            0 :     let state = get_state(&request);
    2191            0 : 
    2192            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download);
    2193            0 :     let gc_result = state
    2194            0 :         .tenant_manager
    2195            0 :         .immediate_gc(tenant_shard_id, timeline_id, gc_req, cancel, &ctx)
    2196            0 :         .await?;
    2197              : 
    2198            0 :     json_response(StatusCode::OK, gc_result)
    2199            0 : }
    2200              : 
    2201              : // Cancel scheduled compaction tasks
    2202            0 : async fn timeline_cancel_compact_handler(
    2203            0 :     request: Request<Body>,
    2204            0 :     _cancel: CancellationToken,
    2205            0 : ) -> Result<Response<Body>, ApiError> {
    2206            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    2207            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    2208            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    2209            0 :     let state = get_state(&request);
    2210            0 :     async {
    2211            0 :         let tenant = state
    2212            0 :             .tenant_manager
    2213            0 :             .get_attached_tenant_shard(tenant_shard_id)?;
    2214            0 :         tenant.cancel_scheduled_compaction(timeline_id);
    2215            0 :         json_response(StatusCode::OK, ())
    2216            0 :     }
    2217            0 :     .instrument(info_span!("timeline_cancel_compact", tenant_id = %tenant_shard_id.tenant_id, shard_id = %tenant_shard_id.shard_slug(), %timeline_id))
    2218            0 :     .await
    2219            0 : }
    2220              : 
    2221              : // Get compact info of a timeline
    2222            0 : async fn timeline_compact_info_handler(
    2223            0 :     request: Request<Body>,
    2224            0 :     _cancel: CancellationToken,
    2225            0 : ) -> Result<Response<Body>, ApiError> {
    2226            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    2227            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    2228            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    2229            0 :     let state = get_state(&request);
    2230            0 :     async {
    2231            0 :         let tenant = state
    2232            0 :             .tenant_manager
    2233            0 :             .get_attached_tenant_shard(tenant_shard_id)?;
    2234            0 :         let resp = tenant.get_scheduled_compaction_tasks(timeline_id);
    2235            0 :         json_response(StatusCode::OK, resp)
    2236            0 :     }
    2237            0 :     .instrument(info_span!("timeline_compact_info", tenant_id = %tenant_shard_id.tenant_id, shard_id = %tenant_shard_id.shard_slug(), %timeline_id))
    2238            0 :     .await
    2239            0 : }
    2240              : 
    2241              : // Run compaction immediately on given timeline.
    2242            0 : async fn timeline_compact_handler(
    2243            0 :     mut request: Request<Body>,
    2244            0 :     cancel: CancellationToken,
    2245            0 : ) -> Result<Response<Body>, ApiError> {
    2246            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    2247            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    2248            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    2249              : 
    2250            0 :     let compact_request = json_request_maybe::<Option<CompactRequest>>(&mut request).await?;
    2251              : 
    2252            0 :     let state = get_state(&request);
    2253            0 : 
    2254            0 :     let mut flags = EnumSet::empty();
    2255            0 :     flags |= CompactFlags::NoYield; // run compaction to completion
    2256            0 : 
    2257            0 :     if Some(true) == parse_query_param::<_, bool>(&request, "force_l0_compaction")? {
    2258            0 :         flags |= CompactFlags::ForceL0Compaction;
    2259            0 :     }
    2260            0 :     if Some(true) == parse_query_param::<_, bool>(&request, "force_repartition")? {
    2261            0 :         flags |= CompactFlags::ForceRepartition;
    2262            0 :     }
    2263            0 :     if Some(true) == parse_query_param::<_, bool>(&request, "force_image_layer_creation")? {
    2264            0 :         flags |= CompactFlags::ForceImageLayerCreation;
    2265            0 :     }
    2266            0 :     if Some(true) == parse_query_param::<_, bool>(&request, "enhanced_gc_bottom_most_compaction")? {
    2267            0 :         flags |= CompactFlags::EnhancedGcBottomMostCompaction;
    2268            0 :     }
    2269            0 :     if Some(true) == parse_query_param::<_, bool>(&request, "dry_run")? {
    2270            0 :         flags |= CompactFlags::DryRun;
    2271            0 :     }
    2272              : 
    2273            0 :     let wait_until_uploaded =
    2274            0 :         parse_query_param::<_, bool>(&request, "wait_until_uploaded")?.unwrap_or(false);
    2275              : 
    2276            0 :     let wait_until_scheduled_compaction_done =
    2277            0 :         parse_query_param::<_, bool>(&request, "wait_until_scheduled_compaction_done")?
    2278            0 :             .unwrap_or(false);
    2279            0 : 
    2280            0 :     let sub_compaction = compact_request
    2281            0 :         .as_ref()
    2282            0 :         .map(|r| r.sub_compaction)
    2283            0 :         .unwrap_or(false);
    2284            0 :     let sub_compaction_max_job_size_mb = compact_request
    2285            0 :         .as_ref()
    2286            0 :         .and_then(|r| r.sub_compaction_max_job_size_mb);
    2287            0 : 
    2288            0 :     let options = CompactOptions {
    2289            0 :         compact_key_range: compact_request
    2290            0 :             .as_ref()
    2291            0 :             .and_then(|r| r.compact_key_range.clone()),
    2292            0 :         compact_lsn_range: compact_request
    2293            0 :             .as_ref()
    2294            0 :             .and_then(|r| r.compact_lsn_range.clone()),
    2295            0 :         flags,
    2296            0 :         sub_compaction,
    2297            0 :         sub_compaction_max_job_size_mb,
    2298            0 :     };
    2299            0 : 
    2300            0 :     let scheduled = compact_request
    2301            0 :         .as_ref()
    2302            0 :         .map(|r| r.scheduled)
    2303            0 :         .unwrap_or(false);
    2304              : 
    2305            0 :     async {
    2306            0 :         let timeline = active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id).await?;
    2307            0 :         let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download).with_scope_timeline(&timeline);
    2308            0 :         if scheduled {
    2309            0 :             let tenant = state
    2310            0 :                 .tenant_manager
    2311            0 :                 .get_attached_tenant_shard(tenant_shard_id)?;
    2312            0 :             let rx = tenant.schedule_compaction(timeline_id, options).await.map_err(ApiError::InternalServerError)?;
    2313            0 :             if wait_until_scheduled_compaction_done {
    2314              :                 // It is possible that this will take a long time, dropping the HTTP request will not cancel the compaction.
    2315            0 :                 rx.await.ok();
    2316            0 :             }
    2317              :         } else {
    2318            0 :             timeline
    2319            0 :                 .compact_with_options(&cancel, options, &ctx)
    2320            0 :                 .await
    2321            0 :                 .map_err(|e| ApiError::InternalServerError(e.into()))?;
    2322            0 :             if wait_until_uploaded {
    2323            0 :                 timeline.remote_client.wait_completion().await
    2324              :                 // XXX map to correct ApiError for the cases where it's due to shutdown
    2325            0 :                 .context("wait completion").map_err(ApiError::InternalServerError)?;
    2326            0 :             }
    2327              :         }
    2328            0 :         json_response(StatusCode::OK, ())
    2329            0 :     }
    2330            0 :     .instrument(info_span!("manual_compaction", tenant_id = %tenant_shard_id.tenant_id, shard_id = %tenant_shard_id.shard_slug(), %timeline_id))
    2331            0 :     .await
    2332            0 : }
    2333              : 
    2334              : // Run offload immediately on given timeline.
    2335            0 : async fn timeline_offload_handler(
    2336            0 :     request: Request<Body>,
    2337            0 :     _cancel: CancellationToken,
    2338            0 : ) -> Result<Response<Body>, ApiError> {
    2339            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    2340            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    2341            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    2342              : 
    2343            0 :     let state = get_state(&request);
    2344              : 
    2345            0 :     async {
    2346            0 :         let tenant = state
    2347            0 :             .tenant_manager
    2348            0 :             .get_attached_tenant_shard(tenant_shard_id)?;
    2349              : 
    2350            0 :         if tenant.get_offloaded_timeline(timeline_id).is_ok() {
    2351            0 :             return json_response(StatusCode::OK, ());
    2352            0 :         }
    2353            0 :         let timeline =
    2354            0 :             active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
    2355            0 :                 .await?;
    2356              : 
    2357            0 :         if !tenant.timeline_has_no_attached_children(timeline_id) {
    2358            0 :             return Err(ApiError::PreconditionFailed(
    2359            0 :                 "timeline has attached children".into(),
    2360            0 :             ));
    2361            0 :         }
    2362            0 :         if let (false, reason) = timeline.can_offload() {
    2363            0 :             return Err(ApiError::PreconditionFailed(
    2364            0 :                 format!("Timeline::can_offload() check failed: {}", reason) .into(),
    2365            0 :             ));
    2366            0 :         }
    2367            0 :         offload_timeline(&tenant, &timeline)
    2368            0 :             .await
    2369            0 :             .map_err(|e| {
    2370            0 :                 match e {
    2371            0 :                     OffloadError::Cancelled => ApiError::ResourceUnavailable("Timeline shutting down".into()),
    2372            0 :                     _ => ApiError::InternalServerError(anyhow!(e))
    2373              :                 }
    2374            0 :             })?;
    2375              : 
    2376            0 :         json_response(StatusCode::OK, ())
    2377            0 :     }
    2378            0 :     .instrument(info_span!("manual_timeline_offload", tenant_id = %tenant_shard_id.tenant_id, shard_id = %tenant_shard_id.shard_slug(), %timeline_id))
    2379            0 :     .await
    2380            0 : }
    2381              : 
    2382              : // Run checkpoint immediately on given timeline.
    2383            0 : async fn timeline_checkpoint_handler(
    2384            0 :     request: Request<Body>,
    2385            0 :     cancel: CancellationToken,
    2386            0 : ) -> Result<Response<Body>, ApiError> {
    2387            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    2388            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    2389            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    2390              : 
    2391            0 :     let state = get_state(&request);
    2392            0 : 
    2393            0 :     let mut flags = EnumSet::empty();
    2394            0 :     if Some(true) == parse_query_param::<_, bool>(&request, "force_l0_compaction")? {
    2395            0 :         flags |= CompactFlags::ForceL0Compaction;
    2396            0 :     }
    2397            0 :     if Some(true) == parse_query_param::<_, bool>(&request, "force_repartition")? {
    2398            0 :         flags |= CompactFlags::ForceRepartition;
    2399            0 :     }
    2400            0 :     if Some(true) == parse_query_param::<_, bool>(&request, "force_image_layer_creation")? {
    2401            0 :         flags |= CompactFlags::ForceImageLayerCreation;
    2402            0 :     }
    2403              : 
    2404              :     // By default, checkpoints come with a compaction, but this may be optionally disabled by tests that just want to flush + upload.
    2405            0 :     let compact = parse_query_param::<_, bool>(&request, "compact")?.unwrap_or(true);
    2406              : 
    2407            0 :     let wait_until_flushed: bool =
    2408            0 :         parse_query_param(&request, "wait_until_flushed")?.unwrap_or(true);
    2409              : 
    2410            0 :     let wait_until_uploaded =
    2411            0 :         parse_query_param::<_, bool>(&request, "wait_until_uploaded")?.unwrap_or(false);
    2412              : 
    2413            0 :     async {
    2414            0 :         let timeline = active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id).await?;
    2415            0 :         let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download).with_scope_timeline(&timeline);
    2416            0 :         if wait_until_flushed {
    2417            0 :             timeline.freeze_and_flush().await
    2418              :         } else {
    2419            0 :             timeline.freeze().await.and(Ok(()))
    2420            0 :         }.map_err(|e| {
    2421            0 :                 match e {
    2422            0 :                     tenant::timeline::FlushLayerError::Cancelled => ApiError::ShuttingDown,
    2423            0 :                     other => ApiError::InternalServerError(other.into()),
    2424              : 
    2425              :                 }
    2426            0 :             })?;
    2427            0 :         if compact {
    2428            0 :             timeline
    2429            0 :                 .compact(&cancel, flags, &ctx)
    2430            0 :                 .await
    2431            0 :                 .map_err(|e|
    2432            0 :                     match e {
    2433            0 :                         CompactionError::ShuttingDown => ApiError::ShuttingDown,
    2434            0 :                         CompactionError::Offload(e) => ApiError::InternalServerError(anyhow::anyhow!(e)),
    2435            0 :                         CompactionError::CollectKeySpaceError(e) => ApiError::InternalServerError(anyhow::anyhow!(e)),
    2436            0 :                         CompactionError::Other(e) => ApiError::InternalServerError(e),
    2437            0 :                         CompactionError::AlreadyRunning(_) => ApiError::InternalServerError(anyhow::anyhow!(e)),
    2438            0 :                     }
    2439            0 :                 )?;
    2440            0 :         }
    2441              : 
    2442            0 :         if wait_until_uploaded {
    2443            0 :             tracing::info!("Waiting for uploads to complete...");
    2444            0 :             timeline.remote_client.wait_completion().await
    2445              :             // XXX map to correct ApiError for the cases where it's due to shutdown
    2446            0 :             .context("wait completion").map_err(ApiError::InternalServerError)?;
    2447            0 :             tracing::info!("Uploads completed up to {}", timeline.get_remote_consistent_lsn_projected().unwrap_or(Lsn(0)));
    2448            0 :         }
    2449              : 
    2450            0 :         json_response(StatusCode::OK, ())
    2451            0 :     }
    2452            0 :     .instrument(info_span!("manual_checkpoint", tenant_id = %tenant_shard_id.tenant_id, shard_id = %tenant_shard_id.shard_slug(), %timeline_id))
    2453            0 :     .await
    2454            0 : }
    2455              : 
    2456            0 : async fn timeline_download_remote_layers_handler_post(
    2457            0 :     mut request: Request<Body>,
    2458            0 :     _cancel: CancellationToken,
    2459            0 : ) -> Result<Response<Body>, ApiError> {
    2460            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    2461            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    2462            0 :     let body: DownloadRemoteLayersTaskSpawnRequest = json_request(&mut request).await?;
    2463            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    2464              : 
    2465            0 :     let state = get_state(&request);
    2466              : 
    2467            0 :     let timeline =
    2468            0 :         active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
    2469            0 :             .await?;
    2470            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download)
    2471            0 :         .with_scope_timeline(&timeline);
    2472            0 :     match timeline.spawn_download_all_remote_layers(body, &ctx).await {
    2473            0 :         Ok(st) => json_response(StatusCode::ACCEPTED, st),
    2474            0 :         Err(st) => json_response(StatusCode::CONFLICT, st),
    2475              :     }
    2476            0 : }
    2477              : 
    2478            0 : async fn timeline_download_remote_layers_handler_get(
    2479            0 :     request: Request<Body>,
    2480            0 :     _cancel: CancellationToken,
    2481            0 : ) -> Result<Response<Body>, ApiError> {
    2482            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    2483            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    2484            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    2485            0 :     let state = get_state(&request);
    2486              : 
    2487            0 :     let timeline =
    2488            0 :         active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
    2489            0 :             .await?;
    2490            0 :     let info = timeline
    2491            0 :         .get_download_all_remote_layers_task_info()
    2492            0 :         .context("task never started since last pageserver process start")
    2493            0 :         .map_err(|e| ApiError::NotFound(e.into()))?;
    2494            0 :     json_response(StatusCode::OK, info)
    2495            0 : }
    2496              : 
    2497            0 : async fn timeline_detach_ancestor_handler(
    2498            0 :     request: Request<Body>,
    2499            0 :     _cancel: CancellationToken,
    2500            0 : ) -> Result<Response<Body>, ApiError> {
    2501              :     use pageserver_api::models::detach_ancestor::AncestorDetached;
    2502              : 
    2503              :     use crate::tenant::timeline::detach_ancestor;
    2504              : 
    2505            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    2506            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    2507            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    2508              : 
    2509            0 :     let span = tracing::info_span!("detach_ancestor", tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug(), %timeline_id);
    2510              : 
    2511            0 :     async move {
    2512            0 :         let mut options = detach_ancestor::Options::default();
    2513              : 
    2514            0 :         let rewrite_concurrency =
    2515            0 :             parse_query_param::<_, std::num::NonZeroUsize>(&request, "rewrite_concurrency")?;
    2516            0 :         let copy_concurrency =
    2517            0 :             parse_query_param::<_, std::num::NonZeroUsize>(&request, "copy_concurrency")?;
    2518              : 
    2519            0 :         [
    2520            0 :             (&mut options.rewrite_concurrency, rewrite_concurrency),
    2521            0 :             (&mut options.copy_concurrency, copy_concurrency),
    2522            0 :         ]
    2523            0 :         .into_iter()
    2524            0 :         .filter_map(|(target, val)| val.map(|val| (target, val)))
    2525            0 :         .for_each(|(target, val)| *target = val);
    2526            0 : 
    2527            0 :         let state = get_state(&request);
    2528              : 
    2529            0 :         let tenant = state
    2530            0 :             .tenant_manager
    2531            0 :             .get_attached_tenant_shard(tenant_shard_id)?;
    2532              : 
    2533            0 :         tenant.wait_to_become_active(ACTIVE_TENANT_TIMEOUT).await?;
    2534              : 
    2535            0 :         let ctx = RequestContext::new(TaskKind::DetachAncestor, DownloadBehavior::Download);
    2536            0 :         let ctx = &ctx;
    2537              : 
    2538              :         // Flush the upload queues of all timelines before detaching ancestor. We do the same thing again
    2539              :         // during shutdown. This early upload ensures the pageserver does not need to upload too many
    2540              :         // things and creates downtime during timeline reloads.
    2541            0 :         for timeline in tenant.list_timelines() {
    2542            0 :             timeline
    2543            0 :                 .remote_client
    2544            0 :                 .wait_completion()
    2545            0 :                 .await
    2546            0 :                 .map_err(|e| {
    2547            0 :                     ApiError::PreconditionFailed(format!("cannot drain upload queue: {e}").into())
    2548            0 :                 })?;
    2549              :         }
    2550              : 
    2551            0 :         tracing::info!("all timeline upload queues are drained");
    2552              : 
    2553            0 :         let timeline = tenant.get_timeline(timeline_id, true)?;
    2554            0 :         let ctx = &ctx.with_scope_timeline(&timeline);
    2555              : 
    2556            0 :         let progress = timeline
    2557            0 :             .prepare_to_detach_from_ancestor(&tenant, options, ctx)
    2558            0 :             .await?;
    2559              : 
    2560              :         // uncomment to allow early as possible Tenant::drop
    2561              :         // drop(tenant);
    2562              : 
    2563            0 :         let resp = match progress {
    2564            0 :             detach_ancestor::Progress::Prepared(attempt, prepared) => {
    2565              :                 // it would be great to tag the guard on to the tenant activation future
    2566            0 :                 let reparented_timelines = state
    2567            0 :                     .tenant_manager
    2568            0 :                     .complete_detaching_timeline_ancestor(
    2569            0 :                         tenant_shard_id,
    2570            0 :                         timeline_id,
    2571            0 :                         prepared,
    2572            0 :                         attempt,
    2573            0 :                         ctx,
    2574            0 :                     )
    2575            0 :                     .await?;
    2576              : 
    2577            0 :                 AncestorDetached {
    2578            0 :                     reparented_timelines,
    2579            0 :                 }
    2580              :             }
    2581            0 :             detach_ancestor::Progress::Done(resp) => resp,
    2582              :         };
    2583              : 
    2584            0 :         json_response(StatusCode::OK, resp)
    2585            0 :     }
    2586            0 :     .instrument(span)
    2587            0 :     .await
    2588            0 : }
    2589              : 
    2590            0 : async fn deletion_queue_flush(
    2591            0 :     r: Request<Body>,
    2592            0 :     cancel: CancellationToken,
    2593            0 : ) -> Result<Response<Body>, ApiError> {
    2594            0 :     let state = get_state(&r);
    2595              : 
    2596            0 :     let execute = parse_query_param(&r, "execute")?.unwrap_or(false);
    2597            0 : 
    2598            0 :     let flush = async {
    2599            0 :         if execute {
    2600            0 :             state.deletion_queue_client.flush_execute().await
    2601              :         } else {
    2602            0 :             state.deletion_queue_client.flush().await
    2603              :         }
    2604            0 :     }
    2605              :     // DeletionQueueError's only case is shutting down.
    2606            0 :     .map_err(|_| ApiError::ShuttingDown);
    2607            0 : 
    2608            0 :     tokio::select! {
    2609            0 :         res = flush => {
    2610            0 :             res.map(|()| json_response(StatusCode::OK, ()))?
    2611              :         }
    2612            0 :         _ = cancel.cancelled() => {
    2613            0 :             Err(ApiError::ShuttingDown)
    2614              :         }
    2615              :     }
    2616            0 : }
    2617              : 
    2618            0 : async fn getpage_at_lsn_handler(
    2619            0 :     request: Request<Body>,
    2620            0 :     cancel: CancellationToken,
    2621            0 : ) -> Result<Response<Body>, ApiError> {
    2622            0 :     getpage_at_lsn_handler_inner(false, request, cancel).await
    2623            0 : }
    2624              : 
    2625            0 : async fn touchpage_at_lsn_handler(
    2626            0 :     request: Request<Body>,
    2627            0 :     cancel: CancellationToken,
    2628            0 : ) -> Result<Response<Body>, ApiError> {
    2629            0 :     getpage_at_lsn_handler_inner(true, request, cancel).await
    2630            0 : }
    2631              : 
    2632              : /// Try if `GetPage@Lsn` is successful, useful for manual debugging.
    2633            0 : async fn getpage_at_lsn_handler_inner(
    2634            0 :     touch: bool,
    2635            0 :     request: Request<Body>,
    2636            0 :     _cancel: CancellationToken,
    2637            0 : ) -> Result<Response<Body>, ApiError> {
    2638            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    2639            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    2640              :     // Require pageserver admin permission for this API instead of only tenant-level token.
    2641            0 :     check_permission(&request, None)?;
    2642            0 :     let state = get_state(&request);
    2643              : 
    2644              :     struct Key(pageserver_api::key::Key);
    2645              : 
    2646              :     impl std::str::FromStr for Key {
    2647              :         type Err = anyhow::Error;
    2648              : 
    2649            0 :         fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
    2650            0 :             pageserver_api::key::Key::from_hex(s).map(Key)
    2651            0 :         }
    2652              :     }
    2653              : 
    2654            0 :     let key: Key = parse_query_param(&request, "key")?
    2655            0 :         .ok_or_else(|| ApiError::BadRequest(anyhow!("missing 'key' query parameter")))?;
    2656            0 :     let lsn: Option<Lsn> = parse_query_param(&request, "lsn")?;
    2657              : 
    2658            0 :     async {
    2659            0 :         let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download);
    2660              :         // Enable read path debugging
    2661            0 :         let timeline = active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id).await?;
    2662            0 :         let ctx = RequestContextBuilder::extend(&ctx).read_path_debug(true)
    2663            0 :         .scope(context::Scope::new_timeline(&timeline)).build();
    2664            0 : 
    2665            0 :         // Use last_record_lsn if no lsn is provided
    2666            0 :         let lsn = lsn.unwrap_or_else(|| timeline.get_last_record_lsn());
    2667            0 :         let page = timeline.get(key.0, lsn, &ctx).await?;
    2668              : 
    2669            0 :         if touch {
    2670            0 :             json_response(StatusCode::OK, ())
    2671              :         } else {
    2672            0 :             Result::<_, ApiError>::Ok(
    2673            0 :                 Response::builder()
    2674            0 :                     .status(StatusCode::OK)
    2675            0 :                     .header(header::CONTENT_TYPE, "application/octet-stream")
    2676            0 :                     .body(hyper::Body::from(page))
    2677            0 :                     .unwrap(),
    2678            0 :             )
    2679              :         }
    2680            0 :     }
    2681            0 :     .instrument(info_span!("timeline_get", tenant_id = %tenant_shard_id.tenant_id, shard_id = %tenant_shard_id.shard_slug(), %timeline_id))
    2682            0 :     .await
    2683            0 : }
    2684              : 
    2685            0 : async fn timeline_collect_keyspace(
    2686            0 :     request: Request<Body>,
    2687            0 :     _cancel: CancellationToken,
    2688            0 : ) -> Result<Response<Body>, ApiError> {
    2689            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    2690            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    2691            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    2692            0 :     let state = get_state(&request);
    2693              : 
    2694            0 :     let at_lsn: Option<Lsn> = parse_query_param(&request, "at_lsn")?;
    2695              : 
    2696            0 :     async {
    2697            0 :         let timeline = active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id).await?;
    2698            0 :         let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download).with_scope_timeline(&timeline);
    2699            0 :         let at_lsn = at_lsn.unwrap_or_else(|| timeline.get_last_record_lsn());
    2700            0 :         let (dense_ks, sparse_ks) = timeline
    2701            0 :             .collect_keyspace(at_lsn, &ctx)
    2702            0 :             .await
    2703            0 :             .map_err(|e| ApiError::InternalServerError(e.into()))?;
    2704              : 
    2705              :         // This API is currently used by pagebench. Pagebench will iterate all keys within the keyspace.
    2706              :         // Therefore, we split dense/sparse keys in this API.
    2707            0 :         let res = pageserver_api::models::partitioning::Partitioning { keys: dense_ks, sparse_keys: sparse_ks, at_lsn };
    2708            0 : 
    2709            0 :         json_response(StatusCode::OK, res)
    2710            0 :     }
    2711            0 :     .instrument(info_span!("timeline_collect_keyspace", tenant_id = %tenant_shard_id.tenant_id, shard_id = %tenant_shard_id.shard_slug(), %timeline_id))
    2712            0 :     .await
    2713            0 : }
    2714              : 
    2715            0 : async fn active_timeline_of_active_tenant(
    2716            0 :     tenant_manager: &TenantManager,
    2717            0 :     tenant_shard_id: TenantShardId,
    2718            0 :     timeline_id: TimelineId,
    2719            0 : ) -> Result<Arc<Timeline>, ApiError> {
    2720            0 :     let tenant = tenant_manager.get_attached_tenant_shard(tenant_shard_id)?;
    2721              : 
    2722            0 :     tenant.wait_to_become_active(ACTIVE_TENANT_TIMEOUT).await?;
    2723              : 
    2724            0 :     Ok(tenant.get_timeline(timeline_id, true)?)
    2725            0 : }
    2726              : 
    2727            0 : async fn always_panic_handler(
    2728            0 :     req: Request<Body>,
    2729            0 :     _cancel: CancellationToken,
    2730            0 : ) -> Result<Response<Body>, ApiError> {
    2731            0 :     // Deliberately cause a panic to exercise the panic hook registered via std::panic::set_hook().
    2732            0 :     // For pageserver, the relevant panic hook is `tracing_panic_hook` , and the `sentry` crate's wrapper around it.
    2733            0 :     // Use catch_unwind to ensure that tokio nor hyper are distracted by our panic.
    2734            0 :     let query = req.uri().query();
    2735            0 :     let _ = std::panic::catch_unwind(|| {
    2736            0 :         panic!("unconditional panic for testing panic hook integration; request query: {query:?}")
    2737            0 :     });
    2738            0 :     json_response(StatusCode::NO_CONTENT, ())
    2739            0 : }
    2740              : 
    2741            0 : async fn disk_usage_eviction_run(
    2742            0 :     mut r: Request<Body>,
    2743            0 :     cancel: CancellationToken,
    2744            0 : ) -> Result<Response<Body>, ApiError> {
    2745            0 :     check_permission(&r, None)?;
    2746              : 
    2747            0 :     #[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize)]
    2748              :     struct Config {
    2749              :         /// How many bytes to evict before reporting that pressure is relieved.
    2750              :         evict_bytes: u64,
    2751              : 
    2752              :         #[serde(default)]
    2753              :         eviction_order: pageserver_api::config::EvictionOrder,
    2754              :     }
    2755              : 
    2756              :     #[derive(Debug, Clone, Copy, serde::Serialize)]
    2757              :     struct Usage {
    2758              :         // remains unchanged after instantiation of the struct
    2759              :         evict_bytes: u64,
    2760              :         // updated by `add_available_bytes`
    2761              :         freed_bytes: u64,
    2762              :     }
    2763              : 
    2764              :     impl crate::disk_usage_eviction_task::Usage for Usage {
    2765            0 :         fn has_pressure(&self) -> bool {
    2766            0 :             self.evict_bytes > self.freed_bytes
    2767            0 :         }
    2768              : 
    2769            0 :         fn add_available_bytes(&mut self, bytes: u64) {
    2770            0 :             self.freed_bytes += bytes;
    2771            0 :         }
    2772              :     }
    2773              : 
    2774            0 :     let config = json_request::<Config>(&mut r).await?;
    2775              : 
    2776            0 :     let usage = Usage {
    2777            0 :         evict_bytes: config.evict_bytes,
    2778            0 :         freed_bytes: 0,
    2779            0 :     };
    2780            0 : 
    2781            0 :     let state = get_state(&r);
    2782            0 :     let eviction_state = state.disk_usage_eviction_state.clone();
    2783              : 
    2784            0 :     let res = crate::disk_usage_eviction_task::disk_usage_eviction_task_iteration_impl(
    2785            0 :         &eviction_state,
    2786            0 :         &state.remote_storage,
    2787            0 :         usage,
    2788            0 :         &state.tenant_manager,
    2789            0 :         config.eviction_order.into(),
    2790            0 :         &cancel,
    2791            0 :     )
    2792            0 :     .await;
    2793              : 
    2794            0 :     info!(?res, "disk_usage_eviction_task_iteration_impl finished");
    2795              : 
    2796            0 :     let res = res.map_err(ApiError::InternalServerError)?;
    2797              : 
    2798            0 :     json_response(StatusCode::OK, res)
    2799            0 : }
    2800              : 
    2801            0 : async fn secondary_upload_handler(
    2802            0 :     request: Request<Body>,
    2803            0 :     _cancel: CancellationToken,
    2804            0 : ) -> Result<Response<Body>, ApiError> {
    2805            0 :     let state = get_state(&request);
    2806            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    2807            0 :     state
    2808            0 :         .secondary_controller
    2809            0 :         .upload_tenant(tenant_shard_id)
    2810            0 :         .await?;
    2811              : 
    2812            0 :     json_response(StatusCode::OK, ())
    2813            0 : }
    2814              : 
    2815            0 : async fn tenant_scan_remote_handler(
    2816            0 :     request: Request<Body>,
    2817            0 :     cancel: CancellationToken,
    2818            0 : ) -> Result<Response<Body>, ApiError> {
    2819            0 :     let state = get_state(&request);
    2820            0 :     let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
    2821              : 
    2822            0 :     let mut response = TenantScanRemoteStorageResponse::default();
    2823              : 
    2824            0 :     let (shards, _other_keys) =
    2825            0 :         list_remote_tenant_shards(&state.remote_storage, tenant_id, cancel.clone())
    2826            0 :             .await
    2827            0 :             .map_err(|e| ApiError::InternalServerError(anyhow::anyhow!(e)))?;
    2828              : 
    2829            0 :     for tenant_shard_id in shards {
    2830            0 :         let (timeline_ids, _other_keys) =
    2831            0 :             list_remote_timelines(&state.remote_storage, tenant_shard_id, cancel.clone())
    2832            0 :                 .await
    2833            0 :                 .map_err(|e| ApiError::InternalServerError(anyhow::anyhow!(e)))?;
    2834              : 
    2835            0 :         let mut generation = Generation::none();
    2836            0 :         for timeline_id in timeline_ids {
    2837            0 :             match download_index_part(
    2838            0 :                 &state.remote_storage,
    2839            0 :                 &tenant_shard_id,
    2840            0 :                 &timeline_id,
    2841            0 :                 Generation::MAX,
    2842            0 :                 &cancel,
    2843            0 :             )
    2844            0 :             .instrument(info_span!("download_index_part",
    2845              :                          tenant_id=%tenant_shard_id.tenant_id,
    2846            0 :                          shard_id=%tenant_shard_id.shard_slug(),
    2847              :                          %timeline_id))
    2848            0 :             .await
    2849              :             {
    2850            0 :                 Ok((index_part, index_generation, _index_mtime)) => {
    2851            0 :                     tracing::info!(
    2852            0 :                         "Found timeline {tenant_shard_id}/{timeline_id} metadata (gen {index_generation:?}, {} layers, {} consistent LSN)",
    2853            0 :                         index_part.layer_metadata.len(),
    2854            0 :                         index_part.metadata.disk_consistent_lsn()
    2855              :                     );
    2856            0 :                     generation = std::cmp::max(generation, index_generation);
    2857              :                 }
    2858              :                 Err(DownloadError::NotFound) => {
    2859              :                     // This is normal for tenants that were created with multiple shards: they have an unsharded path
    2860              :                     // containing the timeline's initdb tarball but no index.  Otherwise it is a bit strange.
    2861            0 :                     tracing::info!(
    2862            0 :                         "Timeline path {tenant_shard_id}/{timeline_id} exists in remote storage but has no index, skipping"
    2863              :                     );
    2864            0 :                     continue;
    2865              :                 }
    2866            0 :                 Err(e) => {
    2867            0 :                     return Err(ApiError::InternalServerError(anyhow::anyhow!(e)));
    2868              :                 }
    2869              :             };
    2870              :         }
    2871              : 
    2872            0 :         response.shards.push(TenantScanRemoteStorageShard {
    2873            0 :             tenant_shard_id,
    2874            0 :             generation: generation.into(),
    2875            0 :         });
    2876              :     }
    2877              : 
    2878            0 :     if response.shards.is_empty() {
    2879            0 :         return Err(ApiError::NotFound(
    2880            0 :             anyhow::anyhow!("No shards found for tenant ID {tenant_id}").into(),
    2881            0 :         ));
    2882            0 :     }
    2883            0 : 
    2884            0 :     json_response(StatusCode::OK, response)
    2885            0 : }
    2886              : 
    2887            0 : async fn secondary_download_handler(
    2888            0 :     request: Request<Body>,
    2889            0 :     _cancel: CancellationToken,
    2890            0 : ) -> Result<Response<Body>, ApiError> {
    2891            0 :     let state = get_state(&request);
    2892            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    2893            0 :     let wait = parse_query_param(&request, "wait_ms")?.map(Duration::from_millis);
    2894              : 
    2895              :     // We don't need this to issue the download request, but:
    2896              :     // - it enables us to cleanly return 404 if we get a request for an absent shard
    2897              :     // - we will use this to provide status feedback in the response
    2898            0 :     let Some(secondary_tenant) = state
    2899            0 :         .tenant_manager
    2900            0 :         .get_secondary_tenant_shard(tenant_shard_id)
    2901              :     else {
    2902            0 :         return Err(ApiError::NotFound(
    2903            0 :             anyhow::anyhow!("Shard {} not found", tenant_shard_id).into(),
    2904            0 :         ));
    2905              :     };
    2906              : 
    2907            0 :     let timeout = wait.unwrap_or(Duration::MAX);
    2908              : 
    2909            0 :     let result = tokio::time::timeout(
    2910            0 :         timeout,
    2911            0 :         state.secondary_controller.download_tenant(tenant_shard_id),
    2912            0 :     )
    2913            0 :     .await;
    2914              : 
    2915            0 :     let progress = secondary_tenant.progress.lock().unwrap().clone();
    2916              : 
    2917            0 :     let status = match result {
    2918              :         Ok(Ok(())) => {
    2919            0 :             if progress.layers_downloaded >= progress.layers_total {
    2920              :                 // Download job ran to completion
    2921            0 :                 StatusCode::OK
    2922              :             } else {
    2923              :                 // Download dropped out without errors because it ran out of time budget
    2924            0 :                 StatusCode::ACCEPTED
    2925              :             }
    2926              :         }
    2927              :         // Edge case: downloads aren't usually fallible: things like a missing heatmap are considered
    2928              :         // okay.  We could get an error here in the unlikely edge case that the tenant
    2929              :         // was detached between our check above and executing the download job.
    2930            0 :         Ok(Err(e)) => return Err(e.into()),
    2931              :         // A timeout is not an error: we have started the download, we're just not done
    2932              :         // yet.  The caller will get a response body indicating status.
    2933            0 :         Err(_) => StatusCode::ACCEPTED,
    2934              :     };
    2935              : 
    2936            0 :     json_response(status, progress)
    2937            0 : }
    2938              : 
    2939            0 : async fn wait_lsn_handler(
    2940            0 :     mut request: Request<Body>,
    2941            0 :     cancel: CancellationToken,
    2942            0 : ) -> Result<Response<Body>, ApiError> {
    2943            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    2944            0 :     let wait_lsn_request: TenantWaitLsnRequest = json_request(&mut request).await?;
    2945              : 
    2946            0 :     let state = get_state(&request);
    2947            0 :     let tenant = state
    2948            0 :         .tenant_manager
    2949            0 :         .get_attached_tenant_shard(tenant_shard_id)?;
    2950              : 
    2951            0 :     let mut wait_futures = Vec::default();
    2952            0 :     for timeline in tenant.list_timelines() {
    2953            0 :         let Some(lsn) = wait_lsn_request.timelines.get(&timeline.timeline_id) else {
    2954            0 :             continue;
    2955              :         };
    2956              : 
    2957            0 :         let fut = {
    2958            0 :             let timeline = timeline.clone();
    2959            0 :             let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Error);
    2960            0 :             async move {
    2961            0 :                 timeline
    2962            0 :                     .wait_lsn(
    2963            0 :                         *lsn,
    2964            0 :                         WaitLsnWaiter::HttpEndpoint,
    2965            0 :                         WaitLsnTimeout::Custom(wait_lsn_request.timeout),
    2966            0 :                         &ctx,
    2967            0 :                     )
    2968            0 :                     .await
    2969            0 :             }
    2970            0 :         };
    2971            0 :         wait_futures.push(fut);
    2972            0 :     }
    2973              : 
    2974            0 :     if wait_futures.is_empty() {
    2975            0 :         return json_response(StatusCode::NOT_FOUND, ());
    2976            0 :     }
    2977              : 
    2978            0 :     let all_done = tokio::select! {
    2979            0 :         results = join_all(wait_futures) => {
    2980            0 :             results.iter().all(|res| res.is_ok())
    2981              :         },
    2982            0 :         _ = cancel.cancelled() => {
    2983            0 :             return Err(ApiError::Cancelled);
    2984              :         }
    2985              :     };
    2986              : 
    2987            0 :     let status = if all_done {
    2988            0 :         StatusCode::OK
    2989              :     } else {
    2990            0 :         StatusCode::ACCEPTED
    2991              :     };
    2992              : 
    2993            0 :     json_response(status, ())
    2994            0 : }
    2995              : 
    2996            0 : async fn secondary_status_handler(
    2997            0 :     request: Request<Body>,
    2998            0 :     _cancel: CancellationToken,
    2999            0 : ) -> Result<Response<Body>, ApiError> {
    3000            0 :     let state = get_state(&request);
    3001            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    3002              : 
    3003            0 :     let Some(secondary_tenant) = state
    3004            0 :         .tenant_manager
    3005            0 :         .get_secondary_tenant_shard(tenant_shard_id)
    3006              :     else {
    3007            0 :         return Err(ApiError::NotFound(
    3008            0 :             anyhow::anyhow!("Shard {} not found", tenant_shard_id).into(),
    3009            0 :         ));
    3010              :     };
    3011              : 
    3012            0 :     let progress = secondary_tenant.progress.lock().unwrap().clone();
    3013            0 : 
    3014            0 :     json_response(StatusCode::OK, progress)
    3015            0 : }
    3016              : 
    3017            0 : async fn handler_404(_: Request<Body>) -> Result<Response<Body>, ApiError> {
    3018            0 :     json_response(
    3019            0 :         StatusCode::NOT_FOUND,
    3020            0 :         HttpErrorBody::from_msg("page not found".to_owned()),
    3021            0 :     )
    3022            0 : }
    3023              : 
    3024            0 : async fn post_tracing_event_handler(
    3025            0 :     mut r: Request<Body>,
    3026            0 :     _cancel: CancellationToken,
    3027            0 : ) -> Result<Response<Body>, ApiError> {
    3028            0 :     #[derive(Debug, serde::Deserialize)]
    3029              :     #[serde(rename_all = "lowercase")]
    3030              :     enum Level {
    3031              :         Error,
    3032              :         Warn,
    3033              :         Info,
    3034              :         Debug,
    3035              :         Trace,
    3036              :     }
    3037            0 :     #[derive(Debug, serde::Deserialize)]
    3038              :     struct Request {
    3039              :         level: Level,
    3040              :         message: String,
    3041              :     }
    3042            0 :     let body: Request = json_request(&mut r)
    3043            0 :         .await
    3044            0 :         .map_err(|_| ApiError::BadRequest(anyhow::anyhow!("invalid JSON body")))?;
    3045              : 
    3046            0 :     match body.level {
    3047            0 :         Level::Error => tracing::error!(?body.message),
    3048            0 :         Level::Warn => tracing::warn!(?body.message),
    3049            0 :         Level::Info => tracing::info!(?body.message),
    3050            0 :         Level::Debug => tracing::debug!(?body.message),
    3051            0 :         Level::Trace => tracing::trace!(?body.message),
    3052              :     }
    3053              : 
    3054            0 :     json_response(StatusCode::OK, ())
    3055            0 : }
    3056              : 
    3057            0 : async fn put_io_engine_handler(
    3058            0 :     mut r: Request<Body>,
    3059            0 :     _cancel: CancellationToken,
    3060            0 : ) -> Result<Response<Body>, ApiError> {
    3061            0 :     check_permission(&r, None)?;
    3062            0 :     let kind: crate::virtual_file::IoEngineKind = json_request(&mut r).await?;
    3063            0 :     crate::virtual_file::io_engine::set(kind);
    3064            0 :     json_response(StatusCode::OK, ())
    3065            0 : }
    3066              : 
    3067            0 : async fn put_io_mode_handler(
    3068            0 :     mut r: Request<Body>,
    3069            0 :     _cancel: CancellationToken,
    3070            0 : ) -> Result<Response<Body>, ApiError> {
    3071            0 :     check_permission(&r, None)?;
    3072            0 :     let mode: IoMode = json_request(&mut r).await?;
    3073            0 :     crate::virtual_file::set_io_mode(mode);
    3074            0 :     json_response(StatusCode::OK, ())
    3075            0 : }
    3076              : 
    3077              : /// Polled by control plane.
    3078              : ///
    3079              : /// See [`crate::utilization`].
    3080            0 : async fn get_utilization(
    3081            0 :     r: Request<Body>,
    3082            0 :     _cancel: CancellationToken,
    3083            0 : ) -> Result<Response<Body>, ApiError> {
    3084            0 :     fail::fail_point!("get-utilization-http-handler", |_| {
    3085            0 :         Err(ApiError::ResourceUnavailable("failpoint".into()))
    3086            0 :     });
    3087              : 
    3088              :     // this probably could be completely public, but lets make that change later.
    3089            0 :     check_permission(&r, None)?;
    3090              : 
    3091            0 :     let state = get_state(&r);
    3092            0 :     let mut g = state.latest_utilization.lock().await;
    3093              : 
    3094            0 :     let regenerate_every = Duration::from_secs(1);
    3095            0 :     let still_valid = g
    3096            0 :         .as_ref()
    3097            0 :         .is_some_and(|(captured_at, _)| captured_at.elapsed() < regenerate_every);
    3098            0 : 
    3099            0 :     // avoid needless statvfs calls even though those should be non-blocking fast.
    3100            0 :     // regenerate at most 1Hz to allow polling at any rate.
    3101            0 :     if !still_valid {
    3102            0 :         let path = state.conf.tenants_path();
    3103            0 :         let doc =
    3104            0 :             crate::utilization::regenerate(state.conf, path.as_std_path(), &state.tenant_manager)
    3105            0 :                 .map_err(ApiError::InternalServerError)?;
    3106              : 
    3107            0 :         let mut buf = Vec::new();
    3108            0 :         serde_json::to_writer(&mut buf, &doc)
    3109            0 :             .context("serialize")
    3110            0 :             .map_err(ApiError::InternalServerError)?;
    3111              : 
    3112            0 :         let body = bytes::Bytes::from(buf);
    3113            0 : 
    3114            0 :         *g = Some((std::time::Instant::now(), body));
    3115            0 :     }
    3116              : 
    3117              :     // hyper 0.14 doesn't yet have Response::clone so this is a bit of extra legwork
    3118            0 :     let cached = g.as_ref().expect("just set").1.clone();
    3119            0 : 
    3120            0 :     Response::builder()
    3121            0 :         .header(hyper::http::header::CONTENT_TYPE, "application/json")
    3122            0 :         // thought of using http date header, but that is second precision which does not give any
    3123            0 :         // debugging aid
    3124            0 :         .status(StatusCode::OK)
    3125            0 :         .body(hyper::Body::from(cached))
    3126            0 :         .context("build response")
    3127            0 :         .map_err(ApiError::InternalServerError)
    3128            0 : }
    3129              : 
    3130            0 : async fn list_aux_files(
    3131            0 :     mut request: Request<Body>,
    3132            0 :     _cancel: CancellationToken,
    3133            0 : ) -> Result<Response<Body>, ApiError> {
    3134            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    3135            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    3136            0 :     let body: ListAuxFilesRequest = json_request(&mut request).await?;
    3137            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    3138              : 
    3139            0 :     let state = get_state(&request);
    3140              : 
    3141            0 :     let timeline =
    3142            0 :         active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
    3143            0 :             .await?;
    3144              : 
    3145            0 :     let io_concurrency = IoConcurrency::spawn_from_conf(
    3146            0 :         state.conf,
    3147            0 :         timeline.gate.enter().map_err(|_| ApiError::Cancelled)?,
    3148              :     );
    3149              : 
    3150            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download);
    3151            0 :     let files = timeline
    3152            0 :         .list_aux_files(body.lsn, &ctx, io_concurrency)
    3153            0 :         .await?;
    3154            0 :     json_response(StatusCode::OK, files)
    3155            0 : }
    3156              : 
    3157            0 : async fn perf_info(
    3158            0 :     request: Request<Body>,
    3159            0 :     _cancel: CancellationToken,
    3160            0 : ) -> Result<Response<Body>, ApiError> {
    3161            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    3162            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    3163            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    3164              : 
    3165            0 :     let state = get_state(&request);
    3166              : 
    3167            0 :     let timeline =
    3168            0 :         active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
    3169            0 :             .await?;
    3170              : 
    3171            0 :     let result = timeline.perf_info().await;
    3172              : 
    3173            0 :     json_response(StatusCode::OK, result)
    3174            0 : }
    3175              : 
    3176            0 : async fn ingest_aux_files(
    3177            0 :     mut request: Request<Body>,
    3178            0 :     _cancel: CancellationToken,
    3179            0 : ) -> Result<Response<Body>, ApiError> {
    3180            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    3181            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    3182            0 :     let body: IngestAuxFilesRequest = json_request(&mut request).await?;
    3183            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    3184              : 
    3185            0 :     let state = get_state(&request);
    3186              : 
    3187            0 :     let timeline =
    3188            0 :         active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
    3189            0 :             .await?;
    3190              : 
    3191            0 :     let mut modification = timeline.begin_modification(
    3192            0 :         Lsn(timeline.get_last_record_lsn().0 + 8), /* advance LSN by 8 */
    3193            0 :     );
    3194            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download);
    3195            0 :     for (fname, content) in body.aux_files {
    3196            0 :         modification
    3197            0 :             .put_file(&fname, content.as_bytes(), &ctx)
    3198            0 :             .await
    3199            0 :             .map_err(ApiError::InternalServerError)?;
    3200              :     }
    3201            0 :     modification
    3202            0 :         .commit(&ctx)
    3203            0 :         .await
    3204            0 :         .map_err(ApiError::InternalServerError)?;
    3205              : 
    3206            0 :     json_response(StatusCode::OK, ())
    3207            0 : }
    3208              : 
    3209              : /// Report on the largest tenants on this pageserver, for the storage controller to identify
    3210              : /// candidates for splitting
    3211            0 : async fn post_top_tenants(
    3212            0 :     mut r: Request<Body>,
    3213            0 :     _cancel: CancellationToken,
    3214            0 : ) -> Result<Response<Body>, ApiError> {
    3215            0 :     check_permission(&r, None)?;
    3216            0 :     let request: TopTenantShardsRequest = json_request(&mut r).await?;
    3217            0 :     let state = get_state(&r);
    3218              : 
    3219            0 :     fn get_size_metric(sizes: &TopTenantShardItem, order_by: &TenantSorting) -> u64 {
    3220            0 :         match order_by {
    3221            0 :             TenantSorting::ResidentSize => sizes.resident_size,
    3222            0 :             TenantSorting::MaxLogicalSize => sizes.max_logical_size,
    3223            0 :             TenantSorting::MaxLogicalSizePerShard => sizes.max_logical_size_per_shard,
    3224              :         }
    3225            0 :     }
    3226              : 
    3227              :     #[derive(Eq, PartialEq)]
    3228              :     struct HeapItem {
    3229              :         metric: u64,
    3230              :         sizes: TopTenantShardItem,
    3231              :     }
    3232              : 
    3233              :     impl PartialOrd for HeapItem {
    3234            0 :         fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
    3235            0 :             Some(self.cmp(other))
    3236            0 :         }
    3237              :     }
    3238              : 
    3239              :     /// Heap items have reverse ordering on their metric: this enables using BinaryHeap, which
    3240              :     /// supports popping the greatest item but not the smallest.
    3241              :     impl Ord for HeapItem {
    3242            0 :         fn cmp(&self, other: &Self) -> std::cmp::Ordering {
    3243            0 :             Reverse(self.metric).cmp(&Reverse(other.metric))
    3244            0 :         }
    3245              :     }
    3246              : 
    3247            0 :     let mut top_n: BinaryHeap<HeapItem> = BinaryHeap::with_capacity(request.limit);
    3248              : 
    3249              :     // FIXME: this is a lot of clones to take this tenant list
    3250            0 :     for (tenant_shard_id, tenant_slot) in state.tenant_manager.list() {
    3251            0 :         if let Some(shards_lt) = request.where_shards_lt {
    3252              :             // Ignore tenants which already have >= this many shards
    3253            0 :             if tenant_shard_id.shard_count >= shards_lt {
    3254            0 :                 continue;
    3255            0 :             }
    3256            0 :         }
    3257              : 
    3258            0 :         let sizes = match tenant_slot {
    3259            0 :             TenantSlot::Attached(tenant) => tenant.get_sizes(),
    3260              :             TenantSlot::Secondary(_) | TenantSlot::InProgress(_) => {
    3261            0 :                 continue;
    3262              :             }
    3263              :         };
    3264            0 :         let metric = get_size_metric(&sizes, &request.order_by);
    3265              : 
    3266            0 :         if let Some(gt) = request.where_gt {
    3267              :             // Ignore tenants whose metric is <= the lower size threshold, to do less sorting work
    3268            0 :             if metric <= gt {
    3269            0 :                 continue;
    3270            0 :             }
    3271            0 :         };
    3272              : 
    3273            0 :         match top_n.peek() {
    3274            0 :             None => {
    3275            0 :                 // Top N list is empty: candidate becomes first member
    3276            0 :                 top_n.push(HeapItem { metric, sizes });
    3277            0 :             }
    3278            0 :             Some(i) if i.metric > metric && top_n.len() < request.limit => {
    3279            0 :                 // Lowest item in list is greater than our candidate, but we aren't at limit yet: push to end
    3280            0 :                 top_n.push(HeapItem { metric, sizes });
    3281            0 :             }
    3282            0 :             Some(i) if i.metric > metric => {
    3283            0 :                 // List is at limit and lowest value is greater than our candidate, drop it.
    3284            0 :             }
    3285            0 :             Some(_) => top_n.push(HeapItem { metric, sizes }),
    3286              :         }
    3287              : 
    3288            0 :         while top_n.len() > request.limit {
    3289            0 :             top_n.pop();
    3290            0 :         }
    3291              :     }
    3292              : 
    3293            0 :     json_response(
    3294            0 :         StatusCode::OK,
    3295            0 :         TopTenantShardsResponse {
    3296            0 :             shards: top_n.into_iter().map(|i| i.sizes).collect(),
    3297            0 :         },
    3298            0 :     )
    3299            0 : }
    3300              : 
    3301            0 : async fn put_tenant_timeline_import_basebackup(
    3302            0 :     request: Request<Body>,
    3303            0 :     _cancel: CancellationToken,
    3304            0 : ) -> Result<Response<Body>, ApiError> {
    3305            0 :     let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
    3306            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    3307            0 :     let base_lsn: Lsn = must_parse_query_param(&request, "base_lsn")?;
    3308            0 :     let end_lsn: Lsn = must_parse_query_param(&request, "end_lsn")?;
    3309            0 :     let pg_version: u32 = must_parse_query_param(&request, "pg_version")?;
    3310              : 
    3311            0 :     check_permission(&request, Some(tenant_id))?;
    3312              : 
    3313            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Warn);
    3314            0 : 
    3315            0 :     let tenant_shard_id = TenantShardId::unsharded(tenant_id);
    3316              : 
    3317            0 :     let span = info_span!("import_basebackup",
    3318            0 :         tenant_id=%tenant_id, timeline_id=%timeline_id, shard_id=%tenant_shard_id.shard_slug(),
    3319              :         base_lsn=%base_lsn, end_lsn=%end_lsn, pg_version=%pg_version);
    3320            0 :     async move {
    3321            0 :         let state = get_state(&request);
    3322            0 :         let tenant = state
    3323            0 :             .tenant_manager
    3324            0 :             .get_attached_tenant_shard(tenant_shard_id)?;
    3325              : 
    3326            0 :         let broker_client = state.broker_client.clone();
    3327            0 : 
    3328            0 :         let mut body = StreamReader::new(request.into_body().map(|res| {
    3329            0 :             res.map_err(|error| {
    3330            0 :                 std::io::Error::new(std::io::ErrorKind::Other, anyhow::anyhow!(error))
    3331            0 :             })
    3332            0 :         }));
    3333            0 : 
    3334            0 :         tenant.wait_to_become_active(ACTIVE_TENANT_TIMEOUT).await?;
    3335              : 
    3336            0 :         let (timeline, timeline_ctx) = tenant
    3337            0 :             .create_empty_timeline(timeline_id, base_lsn, pg_version, &ctx)
    3338            0 :             .map_err(ApiError::InternalServerError)
    3339            0 :             .await?;
    3340              : 
    3341              :         // TODO mark timeline as not ready until it reaches end_lsn.
    3342              :         // We might have some wal to import as well, and we should prevent compute
    3343              :         // from connecting before that and writing conflicting wal.
    3344              :         //
    3345              :         // This is not relevant for pageserver->pageserver migrations, since there's
    3346              :         // no wal to import. But should be fixed if we want to import from postgres.
    3347              : 
    3348              :         // TODO leave clean state on error. For now you can use detach to clean
    3349              :         // up broken state from a failed import.
    3350              : 
    3351              :         // Import basebackup provided via CopyData
    3352            0 :         info!("importing basebackup");
    3353              : 
    3354            0 :         timeline
    3355            0 :             .import_basebackup_from_tar(
    3356            0 :                 tenant.clone(),
    3357            0 :                 &mut body,
    3358            0 :                 base_lsn,
    3359            0 :                 broker_client,
    3360            0 :                 &timeline_ctx,
    3361            0 :             )
    3362            0 :             .await
    3363            0 :             .map_err(ApiError::InternalServerError)?;
    3364              : 
    3365              :         // Read the end of the tar archive.
    3366            0 :         read_tar_eof(body)
    3367            0 :             .await
    3368            0 :             .map_err(ApiError::InternalServerError)?;
    3369              : 
    3370              :         // TODO check checksum
    3371              :         // Meanwhile you can verify client-side by taking fullbackup
    3372              :         // and checking that it matches in size with what was imported.
    3373              :         // It wouldn't work if base came from vanilla postgres though,
    3374              :         // since we discard some log files.
    3375              : 
    3376            0 :         info!("done");
    3377            0 :         json_response(StatusCode::OK, ())
    3378            0 :     }
    3379            0 :     .instrument(span)
    3380            0 :     .await
    3381            0 : }
    3382              : 
    3383            0 : async fn put_tenant_timeline_import_wal(
    3384            0 :     request: Request<Body>,
    3385            0 :     _cancel: CancellationToken,
    3386            0 : ) -> Result<Response<Body>, ApiError> {
    3387            0 :     let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
    3388            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    3389            0 :     let start_lsn: Lsn = must_parse_query_param(&request, "start_lsn")?;
    3390            0 :     let end_lsn: Lsn = must_parse_query_param(&request, "end_lsn")?;
    3391              : 
    3392            0 :     check_permission(&request, Some(tenant_id))?;
    3393              : 
    3394            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Warn);
    3395              : 
    3396            0 :     let span = info_span!("import_wal", tenant_id=%tenant_id, timeline_id=%timeline_id, start_lsn=%start_lsn, end_lsn=%end_lsn);
    3397            0 :     async move {
    3398            0 :         let state = get_state(&request);
    3399              : 
    3400            0 :         let timeline = active_timeline_of_active_tenant(&state.tenant_manager, TenantShardId::unsharded(tenant_id), timeline_id).await?;
    3401            0 :         let ctx = RequestContextBuilder::extend(&ctx).scope(context::Scope::new_timeline(&timeline)).build();
    3402            0 : 
    3403            0 :         let mut body = StreamReader::new(request.into_body().map(|res| {
    3404            0 :             res.map_err(|error| {
    3405            0 :                 std::io::Error::new(std::io::ErrorKind::Other, anyhow::anyhow!(error))
    3406            0 :             })
    3407            0 :         }));
    3408            0 : 
    3409            0 :         let last_record_lsn = timeline.get_last_record_lsn();
    3410            0 :         if last_record_lsn != start_lsn {
    3411            0 :             return Err(ApiError::InternalServerError(anyhow::anyhow!("Cannot import WAL from Lsn {start_lsn} because timeline does not start from the same lsn: {last_record_lsn}")));
    3412            0 :         }
    3413            0 : 
    3414            0 :         // TODO leave clean state on error. For now you can use detach to clean
    3415            0 :         // up broken state from a failed import.
    3416            0 : 
    3417            0 :         // Import wal provided via CopyData
    3418            0 :         info!("importing wal");
    3419            0 :         crate::import_datadir::import_wal_from_tar(&timeline, &mut body, start_lsn, end_lsn, &ctx).await.map_err(ApiError::InternalServerError)?;
    3420            0 :         info!("wal import complete");
    3421              : 
    3422              :         // Read the end of the tar archive.
    3423            0 :         read_tar_eof(body).await.map_err(ApiError::InternalServerError)?;
    3424              : 
    3425              :         // TODO Does it make sense to overshoot?
    3426            0 :         if timeline.get_last_record_lsn() < end_lsn {
    3427            0 :             return Err(ApiError::InternalServerError(anyhow::anyhow!("Cannot import WAL from Lsn {start_lsn} because timeline does not start from the same lsn: {last_record_lsn}")));
    3428            0 :         }
    3429            0 : 
    3430            0 :         // Flush data to disk, then upload to s3. No need for a forced checkpoint.
    3431            0 :         // We only want to persist the data, and it doesn't matter if it's in the
    3432            0 :         // shape of deltas or images.
    3433            0 :         info!("flushing layers");
    3434            0 :         timeline.freeze_and_flush().await.map_err(|e| match e {
    3435            0 :             tenant::timeline::FlushLayerError::Cancelled => ApiError::ShuttingDown,
    3436            0 :             other => ApiError::InternalServerError(anyhow::anyhow!(other)),
    3437            0 :         })?;
    3438              : 
    3439            0 :         info!("done");
    3440              : 
    3441            0 :         json_response(StatusCode::OK, ())
    3442            0 :     }.instrument(span).await
    3443            0 : }
    3444              : 
    3445              : /// Read the end of a tar archive.
    3446              : ///
    3447              : /// A tar archive normally ends with two consecutive blocks of zeros, 512 bytes each.
    3448              : /// `tokio_tar` already read the first such block. Read the second all-zeros block,
    3449              : /// and check that there is no more data after the EOF marker.
    3450              : ///
    3451              : /// 'tar' command can also write extra blocks of zeros, up to a record
    3452              : /// size, controlled by the --record-size argument. Ignore them too.
    3453            0 : async fn read_tar_eof(mut reader: (impl tokio::io::AsyncRead + Unpin)) -> anyhow::Result<()> {
    3454              :     use tokio::io::AsyncReadExt;
    3455            0 :     let mut buf = [0u8; 512];
    3456            0 : 
    3457            0 :     // Read the all-zeros block, and verify it
    3458            0 :     let mut total_bytes = 0;
    3459            0 :     while total_bytes < 512 {
    3460            0 :         let nbytes = reader.read(&mut buf[total_bytes..]).await?;
    3461            0 :         total_bytes += nbytes;
    3462            0 :         if nbytes == 0 {
    3463            0 :             break;
    3464            0 :         }
    3465              :     }
    3466            0 :     if total_bytes < 512 {
    3467            0 :         anyhow::bail!("incomplete or invalid tar EOF marker");
    3468            0 :     }
    3469            0 :     if !buf.iter().all(|&x| x == 0) {
    3470            0 :         anyhow::bail!("invalid tar EOF marker");
    3471            0 :     }
    3472            0 : 
    3473            0 :     // Drain any extra zero-blocks after the EOF marker
    3474            0 :     let mut trailing_bytes = 0;
    3475            0 :     let mut seen_nonzero_bytes = false;
    3476              :     loop {
    3477            0 :         let nbytes = reader.read(&mut buf).await?;
    3478            0 :         trailing_bytes += nbytes;
    3479            0 :         if !buf.iter().all(|&x| x == 0) {
    3480            0 :             seen_nonzero_bytes = true;
    3481            0 :         }
    3482            0 :         if nbytes == 0 {
    3483            0 :             break;
    3484            0 :         }
    3485              :     }
    3486            0 :     if seen_nonzero_bytes {
    3487            0 :         anyhow::bail!("unexpected non-zero bytes after the tar archive");
    3488            0 :     }
    3489            0 :     if trailing_bytes % 512 != 0 {
    3490            0 :         anyhow::bail!(
    3491            0 :             "unexpected number of zeros ({trailing_bytes}), not divisible by tar block size (512 bytes), after the tar archive"
    3492            0 :         );
    3493            0 :     }
    3494            0 :     Ok(())
    3495            0 : }
    3496              : 
    3497              : /// Common functionality of all the HTTP API handlers.
    3498              : ///
    3499              : /// - Adds a tracing span to each request (by `request_span`)
    3500              : /// - Logs the request depending on the request method (by `request_span`)
    3501              : /// - Logs the response if it was not successful (by `request_span`
    3502              : /// - Shields the handler function from async cancellations. Hyper can drop the handler
    3503              : ///   Future if the connection to the client is lost, but most of the pageserver code is
    3504              : ///   not async cancellation safe. This converts the dropped future into a graceful cancellation
    3505              : ///   request with a CancellationToken.
    3506            0 : async fn api_handler<R, H>(request: Request<Body>, handler: H) -> Result<Response<Body>, ApiError>
    3507            0 : where
    3508            0 :     R: std::future::Future<Output = Result<Response<Body>, ApiError>> + Send + 'static,
    3509            0 :     H: FnOnce(Request<Body>, CancellationToken) -> R + Send + Sync + 'static,
    3510            0 : {
    3511            0 :     if request.uri() != &"/v1/failpoints".parse::<Uri>().unwrap() {
    3512            0 :         fail::fail_point!("api-503", |_| Err(ApiError::ResourceUnavailable(
    3513            0 :             "failpoint".into()
    3514            0 :         )));
    3515              : 
    3516            0 :         fail::fail_point!("api-500", |_| Err(ApiError::InternalServerError(
    3517            0 :             anyhow::anyhow!("failpoint")
    3518            0 :         )));
    3519            0 :     }
    3520              : 
    3521              :     // Spawn a new task to handle the request, to protect the handler from unexpected
    3522              :     // async cancellations. Most pageserver functions are not async cancellation safe.
    3523              :     // We arm a drop-guard, so that if Hyper drops the Future, we signal the task
    3524              :     // with the cancellation token.
    3525            0 :     let token = CancellationToken::new();
    3526            0 :     let cancel_guard = token.clone().drop_guard();
    3527            0 :     let result = request_span(request, move |r| async {
    3528            0 :         let handle = tokio::spawn(
    3529            0 :             async {
    3530            0 :                 let token_cloned = token.clone();
    3531            0 :                 let result = handler(r, token).await;
    3532            0 :                 if token_cloned.is_cancelled() {
    3533              :                     // dropguard has executed: we will never turn this result into response.
    3534              :                     //
    3535              :                     // at least temporarily do {:?} logging; these failures are rare enough but
    3536              :                     // could hide difficult errors.
    3537            0 :                     match &result {
    3538            0 :                         Ok(response) => {
    3539            0 :                             let status = response.status();
    3540            0 :                             info!(%status, "Cancelled request finished successfully")
    3541              :                         }
    3542            0 :                         Err(e) => match e {
    3543              :                             ApiError::ShuttingDown | ApiError::ResourceUnavailable(_) => {
    3544              :                                 // Don't log this at error severity: they are normal during lifecycle of tenants/process
    3545            0 :                                 info!("Cancelled request aborted for shutdown")
    3546              :                             }
    3547              :                             _ => {
    3548              :                                 // Log these in a highly visible way, because we have no client to send the response to, but
    3549              :                                 // would like to know that something went wrong.
    3550            0 :                                 error!("Cancelled request finished with an error: {e:?}")
    3551              :                             }
    3552              :                         },
    3553              :                     }
    3554            0 :                 }
    3555              :                 // only logging for cancelled panicked request handlers is the tracing_panic_hook,
    3556              :                 // which should suffice.
    3557              :                 //
    3558              :                 // there is still a chance to lose the result due to race between
    3559              :                 // returning from here and the actual connection closing happening
    3560              :                 // before outer task gets to execute. leaving that up for #5815.
    3561            0 :                 result
    3562            0 :             }
    3563            0 :             .in_current_span(),
    3564            0 :         );
    3565            0 : 
    3566            0 :         match handle.await {
    3567              :             // TODO: never actually return Err from here, always Ok(...) so that we can log
    3568              :             // spanned errors. Call api_error_handler instead and return appropriate Body.
    3569            0 :             Ok(result) => result,
    3570            0 :             Err(e) => {
    3571            0 :                 // The handler task panicked. We have a global panic handler that logs the
    3572            0 :                 // panic with its backtrace, so no need to log that here. Only log a brief
    3573            0 :                 // message to make it clear that we returned the error to the client.
    3574            0 :                 error!("HTTP request handler task panicked: {e:#}");
    3575              : 
    3576              :                 // Don't return an Error here, because then fallback error handler that was
    3577              :                 // installed in make_router() will print the error. Instead, construct the
    3578              :                 // HTTP error response and return that.
    3579            0 :                 Ok(
    3580            0 :                     ApiError::InternalServerError(anyhow!("HTTP request handler task panicked"))
    3581            0 :                         .into_response(),
    3582            0 :                 )
    3583              :             }
    3584              :         }
    3585            0 :     })
    3586            0 :     .await;
    3587              : 
    3588            0 :     cancel_guard.disarm();
    3589            0 : 
    3590            0 :     result
    3591            0 : }
    3592              : 
    3593              : /// Like api_handler, but returns an error response if the server is built without
    3594              : /// the 'testing' feature.
    3595            0 : async fn testing_api_handler<R, H>(
    3596            0 :     desc: &str,
    3597            0 :     request: Request<Body>,
    3598            0 :     handler: H,
    3599            0 : ) -> Result<Response<Body>, ApiError>
    3600            0 : where
    3601            0 :     R: std::future::Future<Output = Result<Response<Body>, ApiError>> + Send + 'static,
    3602            0 :     H: FnOnce(Request<Body>, CancellationToken) -> R + Send + Sync + 'static,
    3603            0 : {
    3604            0 :     if cfg!(feature = "testing") {
    3605            0 :         api_handler(request, handler).await
    3606              :     } else {
    3607            0 :         std::future::ready(Err(ApiError::BadRequest(anyhow!(
    3608            0 :             "Cannot {desc} because pageserver was compiled without testing APIs",
    3609            0 :         ))))
    3610            0 :         .await
    3611              :     }
    3612            0 : }
    3613              : 
    3614            0 : pub fn make_router(
    3615            0 :     state: Arc<State>,
    3616            0 :     launch_ts: &'static LaunchTimestamp,
    3617            0 :     auth: Option<Arc<SwappableJwtAuth>>,
    3618            0 : ) -> anyhow::Result<RouterBuilder<hyper::Body, ApiError>> {
    3619            0 :     let spec = include_bytes!("openapi_spec.yml");
    3620            0 :     let mut router = attach_openapi_ui(endpoint::make_router(), spec, "/swagger.yml", "/v1/doc");
    3621            0 :     if auth.is_some() {
    3622            0 :         router = router.middleware(auth_middleware(|request| {
    3623            0 :             let state = get_state(request);
    3624            0 :             if state.allowlist_routes.contains(&request.uri().path()) {
    3625            0 :                 None
    3626              :             } else {
    3627            0 :                 state.auth.as_deref()
    3628              :             }
    3629            0 :         }))
    3630            0 :     }
    3631              : 
    3632            0 :     router = router.middleware(
    3633            0 :         endpoint::add_response_header_middleware(
    3634            0 :             "PAGESERVER_LAUNCH_TIMESTAMP",
    3635            0 :             &launch_ts.to_string(),
    3636            0 :         )
    3637            0 :         .expect("construct launch timestamp header middleware"),
    3638            0 :     );
    3639            0 : 
    3640            0 :     Ok(router
    3641            0 :         .data(state)
    3642            0 :         .get("/metrics", |r| request_span(r, prometheus_metrics_handler))
    3643            0 :         .get("/profile/cpu", |r| request_span(r, profile_cpu_handler))
    3644            0 :         .get("/profile/heap", |r| request_span(r, profile_heap_handler))
    3645            0 :         .get("/v1/status", |r| api_handler(r, status_handler))
    3646            0 :         .put("/v1/failpoints", |r| {
    3647            0 :             testing_api_handler("manage failpoints", r, failpoints_handler)
    3648            0 :         })
    3649            0 :         .post("/v1/reload_auth_validation_keys", |r| {
    3650            0 :             api_handler(r, reload_auth_validation_keys_handler)
    3651            0 :         })
    3652            0 :         .get("/v1/tenant", |r| api_handler(r, tenant_list_handler))
    3653            0 :         .get("/v1/tenant/:tenant_shard_id", |r| {
    3654            0 :             api_handler(r, tenant_status)
    3655            0 :         })
    3656            0 :         .delete("/v1/tenant/:tenant_shard_id", |r| {
    3657            0 :             api_handler(r, tenant_delete_handler)
    3658            0 :         })
    3659            0 :         .get("/v1/tenant/:tenant_shard_id/synthetic_size", |r| {
    3660            0 :             api_handler(r, tenant_size_handler)
    3661            0 :         })
    3662            0 :         .patch("/v1/tenant/config", |r| {
    3663            0 :             api_handler(r, patch_tenant_config_handler)
    3664            0 :         })
    3665            0 :         .put("/v1/tenant/config", |r| {
    3666            0 :             api_handler(r, update_tenant_config_handler)
    3667            0 :         })
    3668            0 :         .put("/v1/tenant/:tenant_shard_id/shard_split", |r| {
    3669            0 :             api_handler(r, tenant_shard_split_handler)
    3670            0 :         })
    3671            0 :         .get("/v1/tenant/:tenant_shard_id/config", |r| {
    3672            0 :             api_handler(r, get_tenant_config_handler)
    3673            0 :         })
    3674            0 :         .put("/v1/tenant/:tenant_shard_id/location_config", |r| {
    3675            0 :             api_handler(r, put_tenant_location_config_handler)
    3676            0 :         })
    3677            0 :         .get("/v1/location_config", |r| {
    3678            0 :             api_handler(r, list_location_config_handler)
    3679            0 :         })
    3680            0 :         .get("/v1/location_config/:tenant_shard_id", |r| {
    3681            0 :             api_handler(r, get_location_config_handler)
    3682            0 :         })
    3683            0 :         .put(
    3684            0 :             "/v1/tenant/:tenant_shard_id/time_travel_remote_storage",
    3685            0 :             |r| api_handler(r, tenant_time_travel_remote_storage_handler),
    3686            0 :         )
    3687            0 :         .get("/v1/tenant/:tenant_shard_id/timeline", |r| {
    3688            0 :             api_handler(r, timeline_list_handler)
    3689            0 :         })
    3690            0 :         .get("/v1/tenant/:tenant_shard_id/timeline_and_offloaded", |r| {
    3691            0 :             api_handler(r, timeline_and_offloaded_list_handler)
    3692            0 :         })
    3693            0 :         .post("/v1/tenant/:tenant_shard_id/timeline", |r| {
    3694            0 :             api_handler(r, timeline_create_handler)
    3695            0 :         })
    3696            0 :         .post("/v1/tenant/:tenant_shard_id/reset", |r| {
    3697            0 :             api_handler(r, tenant_reset_handler)
    3698            0 :         })
    3699            0 :         .post(
    3700            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/preserve_initdb_archive",
    3701            0 :             |r| api_handler(r, timeline_preserve_initdb_handler),
    3702            0 :         )
    3703            0 :         .put(
    3704            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/archival_config",
    3705            0 :             |r| api_handler(r, timeline_archival_config_handler),
    3706            0 :         )
    3707            0 :         .get("/v1/tenant/:tenant_shard_id/timeline/:timeline_id", |r| {
    3708            0 :             api_handler(r, timeline_detail_handler)
    3709            0 :         })
    3710            0 :         .get(
    3711            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/get_lsn_by_timestamp",
    3712            0 :             |r| api_handler(r, get_lsn_by_timestamp_handler),
    3713            0 :         )
    3714            0 :         .get(
    3715            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/get_timestamp_of_lsn",
    3716            0 :             |r| api_handler(r, get_timestamp_of_lsn_handler),
    3717            0 :         )
    3718            0 :         .post(
    3719            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/patch_index_part",
    3720            0 :             |r| api_handler(r, timeline_patch_index_part_handler),
    3721            0 :         )
    3722            0 :         .post(
    3723            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/lsn_lease",
    3724            0 :             |r| api_handler(r, lsn_lease_handler),
    3725            0 :         )
    3726            0 :         .put(
    3727            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/do_gc",
    3728            0 :             |r| api_handler(r, timeline_gc_handler),
    3729            0 :         )
    3730            0 :         .get(
    3731            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/compact",
    3732            0 :             |r| api_handler(r, timeline_compact_info_handler),
    3733            0 :         )
    3734            0 :         .put(
    3735            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/compact",
    3736            0 :             |r| api_handler(r, timeline_compact_handler),
    3737            0 :         )
    3738            0 :         .delete(
    3739            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/compact",
    3740            0 :             |r| api_handler(r, timeline_cancel_compact_handler),
    3741            0 :         )
    3742            0 :         .put(
    3743            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/offload",
    3744            0 :             |r| testing_api_handler("attempt timeline offload", r, timeline_offload_handler),
    3745            0 :         )
    3746            0 :         .put(
    3747            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/checkpoint",
    3748            0 :             |r| testing_api_handler("run timeline checkpoint", r, timeline_checkpoint_handler),
    3749            0 :         )
    3750            0 :         .post(
    3751            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/download_remote_layers",
    3752            0 :             |r| api_handler(r, timeline_download_remote_layers_handler_post),
    3753            0 :         )
    3754            0 :         .get(
    3755            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/download_remote_layers",
    3756            0 :             |r| api_handler(r, timeline_download_remote_layers_handler_get),
    3757            0 :         )
    3758            0 :         .put(
    3759            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/detach_ancestor",
    3760            0 :             |r| api_handler(r, timeline_detach_ancestor_handler),
    3761            0 :         )
    3762            0 :         .delete("/v1/tenant/:tenant_shard_id/timeline/:timeline_id", |r| {
    3763            0 :             api_handler(r, timeline_delete_handler)
    3764            0 :         })
    3765            0 :         .get(
    3766            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/layer",
    3767            0 :             |r| api_handler(r, layer_map_info_handler),
    3768            0 :         )
    3769            0 :         .post(
    3770            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/download_heatmap_layers",
    3771            0 :             |r| api_handler(r, timeline_download_heatmap_layers_handler),
    3772            0 :         )
    3773            0 :         .delete(
    3774            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/download_heatmap_layers",
    3775            0 :             |r| api_handler(r, timeline_shutdown_download_heatmap_layers_handler),
    3776            0 :         )
    3777            0 :         .get(
    3778            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/layer/:layer_file_name",
    3779            0 :             |r| api_handler(r, layer_download_handler),
    3780            0 :         )
    3781            0 :         .delete(
    3782            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/layer/:layer_file_name",
    3783            0 :             |r| api_handler(r, evict_timeline_layer_handler),
    3784            0 :         )
    3785            0 :         .post(
    3786            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/layer/:layer_name/scan_disposable_keys",
    3787            0 :             |r| testing_api_handler("timeline_layer_scan_disposable_keys", r, timeline_layer_scan_disposable_keys),
    3788            0 :         )
    3789            0 :         .post(
    3790            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/block_gc",
    3791            0 :             |r| api_handler(r, timeline_gc_blocking_handler),
    3792            0 :         )
    3793            0 :         .post(
    3794            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/unblock_gc",
    3795            0 :             |r| api_handler(r, timeline_gc_unblocking_handler),
    3796            0 :         )
    3797            0 :         .get(
    3798            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/page_trace",
    3799            0 :             |r| api_handler(r, timeline_page_trace_handler),
    3800            0 :         )
    3801            0 :         .post("/v1/tenant/:tenant_shard_id/heatmap_upload", |r| {
    3802            0 :             api_handler(r, secondary_upload_handler)
    3803            0 :         })
    3804            0 :         .get("/v1/tenant/:tenant_id/scan_remote_storage", |r| {
    3805            0 :             api_handler(r, tenant_scan_remote_handler)
    3806            0 :         })
    3807            0 :         .put("/v1/disk_usage_eviction/run", |r| {
    3808            0 :             api_handler(r, disk_usage_eviction_run)
    3809            0 :         })
    3810            0 :         .put("/v1/deletion_queue/flush", |r| {
    3811            0 :             api_handler(r, deletion_queue_flush)
    3812            0 :         })
    3813            0 :         .get("/v1/tenant/:tenant_shard_id/secondary/status", |r| {
    3814            0 :             api_handler(r, secondary_status_handler)
    3815            0 :         })
    3816            0 :         .post("/v1/tenant/:tenant_shard_id/secondary/download", |r| {
    3817            0 :             api_handler(r, secondary_download_handler)
    3818            0 :         })
    3819            0 :         .post("/v1/tenant/:tenant_shard_id/wait_lsn", |r| {
    3820            0 :             api_handler(r, wait_lsn_handler)
    3821            0 :         })
    3822            0 :         .put("/v1/tenant/:tenant_shard_id/break", |r| {
    3823            0 :             testing_api_handler("set tenant state to broken", r, handle_tenant_break)
    3824            0 :         })
    3825            0 :         .get("/v1/panic", |r| api_handler(r, always_panic_handler))
    3826            0 :         .post("/v1/tracing/event", |r| {
    3827            0 :             testing_api_handler("emit a tracing event", r, post_tracing_event_handler)
    3828            0 :         })
    3829            0 :         .get(
    3830            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/getpage",
    3831            0 :             |r| testing_api_handler("getpage@lsn", r, getpage_at_lsn_handler),
    3832            0 :         )
    3833            0 :         .get(
    3834            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/touchpage",
    3835            0 :             |r| api_handler(r, touchpage_at_lsn_handler),
    3836            0 :         )
    3837            0 :         .get(
    3838            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/keyspace",
    3839            0 :             |r| api_handler(r, timeline_collect_keyspace),
    3840            0 :         )
    3841            0 :         .put("/v1/io_engine", |r| api_handler(r, put_io_engine_handler))
    3842            0 :         .put("/v1/io_mode", |r| api_handler(r, put_io_mode_handler))
    3843            0 :         .get("/v1/utilization", |r| api_handler(r, get_utilization))
    3844            0 :         .post(
    3845            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/ingest_aux_files",
    3846            0 :             |r| testing_api_handler("ingest_aux_files", r, ingest_aux_files),
    3847            0 :         )
    3848            0 :         .post(
    3849            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/list_aux_files",
    3850            0 :             |r| testing_api_handler("list_aux_files", r, list_aux_files),
    3851            0 :         )
    3852            0 :         .post("/v1/top_tenants", |r| api_handler(r, post_top_tenants))
    3853            0 :         .post(
    3854            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/perf_info",
    3855            0 :             |r| testing_api_handler("perf_info", r, perf_info),
    3856            0 :         )
    3857            0 :         .put(
    3858            0 :             "/v1/tenant/:tenant_id/timeline/:timeline_id/import_basebackup",
    3859            0 :             |r| api_handler(r, put_tenant_timeline_import_basebackup),
    3860            0 :         )
    3861            0 :         .put(
    3862            0 :             "/v1/tenant/:tenant_id/timeline/:timeline_id/import_wal",
    3863            0 :             |r| api_handler(r, put_tenant_timeline_import_wal),
    3864            0 :         )
    3865            0 :         .any(handler_404))
    3866            0 : }
        

Generated by: LCOV version 2.1-beta