LCOV - code coverage report
Current view: top level - pageserver/src/http - routes.rs (source / functions) Coverage Total Hit
Test: 1b0a6a0c05cee5a7de360813c8034804e105ce1c.info Lines: 0.0 % 2768 0
Test Date: 2025-03-12 00:01:28 Functions: 0.0 % 846 0

            Line data    Source code
       1              : //!
       2              : //! Management HTTP API
       3              : //!
       4              : use std::cmp::Reverse;
       5              : use std::collections::{BinaryHeap, HashMap};
       6              : use std::str::FromStr;
       7              : use std::sync::Arc;
       8              : use std::time::Duration;
       9              : 
      10              : use anyhow::{Context, Result, anyhow};
      11              : use enumset::EnumSet;
      12              : use futures::future::join_all;
      13              : use futures::{StreamExt, TryFutureExt};
      14              : use http_utils::endpoint::{
      15              :     self, attach_openapi_ui, auth_middleware, check_permission_with, profile_cpu_handler,
      16              :     profile_heap_handler, prometheus_metrics_handler, request_span,
      17              : };
      18              : use http_utils::error::{ApiError, HttpErrorBody};
      19              : use http_utils::failpoints::failpoints_handler;
      20              : use http_utils::json::{json_request, json_request_maybe, json_response};
      21              : use http_utils::request::{
      22              :     get_request_param, must_get_query_param, must_parse_query_param, parse_query_param,
      23              :     parse_request_param,
      24              : };
      25              : use http_utils::{RequestExt, RouterBuilder};
      26              : use humantime::format_rfc3339;
      27              : use hyper::{Body, Request, Response, StatusCode, Uri, header};
      28              : use metrics::launch_timestamp::LaunchTimestamp;
      29              : use pageserver_api::models::virtual_file::IoMode;
      30              : use pageserver_api::models::{
      31              :     DownloadRemoteLayersTaskSpawnRequest, IngestAuxFilesRequest, ListAuxFilesRequest,
      32              :     LocationConfig, LocationConfigListResponse, LocationConfigMode, LsnLease, LsnLeaseRequest,
      33              :     OffloadedTimelineInfo, PageTraceEvent, ShardParameters, StatusResponse,
      34              :     TenantConfigPatchRequest, TenantConfigRequest, TenantDetails, TenantInfo,
      35              :     TenantLocationConfigRequest, TenantLocationConfigResponse, TenantScanRemoteStorageResponse,
      36              :     TenantScanRemoteStorageShard, TenantShardLocation, TenantShardSplitRequest,
      37              :     TenantShardSplitResponse, TenantSorting, TenantState, TenantWaitLsnRequest,
      38              :     TimelineArchivalConfigRequest, TimelineCreateRequest, TimelineCreateRequestMode,
      39              :     TimelineCreateRequestModeImportPgdata, TimelineGcRequest, TimelineInfo,
      40              :     TimelinePatchIndexPartRequest, TimelinesInfoAndOffloaded, TopTenantShardItem,
      41              :     TopTenantShardsRequest, TopTenantShardsResponse,
      42              : };
      43              : use pageserver_api::shard::{ShardCount, TenantShardId};
      44              : use remote_storage::{DownloadError, GenericRemoteStorage, TimeTravelError};
      45              : use scopeguard::defer;
      46              : use tenant_size_model::svg::SvgBranchKind;
      47              : use tenant_size_model::{SizeResult, StorageModel};
      48              : use tokio::time::Instant;
      49              : use tokio_util::io::StreamReader;
      50              : use tokio_util::sync::CancellationToken;
      51              : use tracing::*;
      52              : use utils::auth::SwappableJwtAuth;
      53              : use utils::generation::Generation;
      54              : use utils::id::{TenantId, TimelineId};
      55              : use utils::lsn::Lsn;
      56              : 
      57              : use crate::config::PageServerConf;
      58              : use crate::context;
      59              : use crate::context::{DownloadBehavior, RequestContext, RequestContextBuilder};
      60              : use crate::deletion_queue::DeletionQueueClient;
      61              : use crate::pgdatadir_mapping::LsnForTimestamp;
      62              : use crate::task_mgr::TaskKind;
      63              : use crate::tenant::config::{LocationConf, TenantConfOpt};
      64              : use crate::tenant::mgr::{
      65              :     GetActiveTenantError, GetTenantError, TenantManager, TenantMapError, TenantMapInsertError,
      66              :     TenantSlot, TenantSlotError, TenantSlotUpsertError, TenantStateError, UpsertLocationError,
      67              : };
      68              : use crate::tenant::remote_timeline_client::index::GcCompactionState;
      69              : use crate::tenant::remote_timeline_client::{
      70              :     download_index_part, list_remote_tenant_shards, list_remote_timelines,
      71              : };
      72              : use crate::tenant::secondary::SecondaryController;
      73              : use crate::tenant::size::ModelInputs;
      74              : use crate::tenant::storage_layer::{IoConcurrency, LayerAccessStatsReset, LayerName};
      75              : use crate::tenant::timeline::offload::{OffloadError, offload_timeline};
      76              : use crate::tenant::timeline::{
      77              :     CompactFlags, CompactOptions, CompactRequest, CompactionError, Timeline, WaitLsnTimeout,
      78              :     WaitLsnWaiter, import_pgdata,
      79              : };
      80              : use crate::tenant::{
      81              :     GetTimelineError, LogicalSizeCalculationCause, OffloadedTimeline, PageReconstructError,
      82              :     remote_timeline_client,
      83              : };
      84              : use crate::{DEFAULT_PG_VERSION, disk_usage_eviction_task, tenant};
      85              : 
      86              : // For APIs that require an Active tenant, how long should we block waiting for that state?
      87              : // This is not functionally necessary (clients will retry), but avoids generating a lot of
      88              : // failed API calls while tenants are activating.
      89              : #[cfg(not(feature = "testing"))]
      90              : pub(crate) const ACTIVE_TENANT_TIMEOUT: Duration = Duration::from_millis(5000);
      91              : 
      92              : // Tests run on slow/oversubscribed nodes, and may need to wait much longer for tenants to
      93              : // finish attaching, if calls to remote storage are slow.
      94              : #[cfg(feature = "testing")]
      95              : pub(crate) const ACTIVE_TENANT_TIMEOUT: Duration = Duration::from_millis(30000);
      96              : 
      97              : pub struct State {
      98              :     conf: &'static PageServerConf,
      99              :     tenant_manager: Arc<TenantManager>,
     100              :     auth: Option<Arc<SwappableJwtAuth>>,
     101              :     allowlist_routes: &'static [&'static str],
     102              :     remote_storage: GenericRemoteStorage,
     103              :     broker_client: storage_broker::BrokerClientChannel,
     104              :     disk_usage_eviction_state: Arc<disk_usage_eviction_task::State>,
     105              :     deletion_queue_client: DeletionQueueClient,
     106              :     secondary_controller: SecondaryController,
     107              :     latest_utilization: tokio::sync::Mutex<Option<(std::time::Instant, bytes::Bytes)>>,
     108              : }
     109              : 
     110              : impl State {
     111              :     #[allow(clippy::too_many_arguments)]
     112            0 :     pub fn new(
     113            0 :         conf: &'static PageServerConf,
     114            0 :         tenant_manager: Arc<TenantManager>,
     115            0 :         auth: Option<Arc<SwappableJwtAuth>>,
     116            0 :         remote_storage: GenericRemoteStorage,
     117            0 :         broker_client: storage_broker::BrokerClientChannel,
     118            0 :         disk_usage_eviction_state: Arc<disk_usage_eviction_task::State>,
     119            0 :         deletion_queue_client: DeletionQueueClient,
     120            0 :         secondary_controller: SecondaryController,
     121            0 :     ) -> anyhow::Result<Self> {
     122            0 :         let allowlist_routes = &[
     123            0 :             "/v1/status",
     124            0 :             "/v1/doc",
     125            0 :             "/swagger.yml",
     126            0 :             "/metrics",
     127            0 :             "/profile/cpu",
     128            0 :             "/profile/heap",
     129            0 :         ];
     130            0 :         Ok(Self {
     131            0 :             conf,
     132            0 :             tenant_manager,
     133            0 :             auth,
     134            0 :             allowlist_routes,
     135            0 :             remote_storage,
     136            0 :             broker_client,
     137            0 :             disk_usage_eviction_state,
     138            0 :             deletion_queue_client,
     139            0 :             secondary_controller,
     140            0 :             latest_utilization: Default::default(),
     141            0 :         })
     142            0 :     }
     143              : }
     144              : 
     145              : #[inline(always)]
     146            0 : fn get_state(request: &Request<Body>) -> &State {
     147            0 :     request
     148            0 :         .data::<Arc<State>>()
     149            0 :         .expect("unknown state type")
     150            0 :         .as_ref()
     151            0 : }
     152              : 
     153              : #[inline(always)]
     154            0 : fn get_config(request: &Request<Body>) -> &'static PageServerConf {
     155            0 :     get_state(request).conf
     156            0 : }
     157              : 
     158              : /// Check that the requester is authorized to operate on given tenant
     159            0 : fn check_permission(request: &Request<Body>, tenant_id: Option<TenantId>) -> Result<(), ApiError> {
     160            0 :     check_permission_with(request, |claims| {
     161            0 :         crate::auth::check_permission(claims, tenant_id)
     162            0 :     })
     163            0 : }
     164              : 
     165              : impl From<PageReconstructError> for ApiError {
     166            0 :     fn from(pre: PageReconstructError) -> ApiError {
     167            0 :         match pre {
     168            0 :             PageReconstructError::Other(other) => ApiError::InternalServerError(other),
     169            0 :             PageReconstructError::MissingKey(e) => ApiError::InternalServerError(e.into()),
     170            0 :             PageReconstructError::Cancelled => ApiError::Cancelled,
     171            0 :             PageReconstructError::AncestorLsnTimeout(e) => ApiError::Timeout(format!("{e}").into()),
     172            0 :             PageReconstructError::WalRedo(pre) => ApiError::InternalServerError(pre),
     173              :         }
     174            0 :     }
     175              : }
     176              : 
     177              : impl From<TenantMapInsertError> for ApiError {
     178            0 :     fn from(tmie: TenantMapInsertError) -> ApiError {
     179            0 :         match tmie {
     180            0 :             TenantMapInsertError::SlotError(e) => e.into(),
     181            0 :             TenantMapInsertError::SlotUpsertError(e) => e.into(),
     182            0 :             TenantMapInsertError::Other(e) => ApiError::InternalServerError(e),
     183              :         }
     184            0 :     }
     185              : }
     186              : 
     187              : impl From<TenantSlotError> for ApiError {
     188            0 :     fn from(e: TenantSlotError) -> ApiError {
     189              :         use TenantSlotError::*;
     190            0 :         match e {
     191            0 :             NotFound(tenant_id) => {
     192            0 :                 ApiError::NotFound(anyhow::anyhow!("NotFound: tenant {tenant_id}").into())
     193              :             }
     194              :             InProgress => {
     195            0 :                 ApiError::ResourceUnavailable("Tenant is being modified concurrently".into())
     196              :             }
     197            0 :             MapState(e) => e.into(),
     198              :         }
     199            0 :     }
     200              : }
     201              : 
     202              : impl From<TenantSlotUpsertError> for ApiError {
     203            0 :     fn from(e: TenantSlotUpsertError) -> ApiError {
     204              :         use TenantSlotUpsertError::*;
     205            0 :         match e {
     206            0 :             InternalError(e) => ApiError::InternalServerError(anyhow::anyhow!("{e}")),
     207            0 :             MapState(e) => e.into(),
     208            0 :             ShuttingDown(_) => ApiError::ShuttingDown,
     209              :         }
     210            0 :     }
     211              : }
     212              : 
     213              : impl From<UpsertLocationError> for ApiError {
     214            0 :     fn from(e: UpsertLocationError) -> ApiError {
     215              :         use UpsertLocationError::*;
     216            0 :         match e {
     217            0 :             BadRequest(e) => ApiError::BadRequest(e),
     218            0 :             Unavailable(_) => ApiError::ShuttingDown,
     219            0 :             e @ InProgress => ApiError::Conflict(format!("{e}")),
     220            0 :             Flush(e) | InternalError(e) => ApiError::InternalServerError(e),
     221              :         }
     222            0 :     }
     223              : }
     224              : 
     225              : impl From<TenantMapError> for ApiError {
     226            0 :     fn from(e: TenantMapError) -> ApiError {
     227              :         use TenantMapError::*;
     228            0 :         match e {
     229              :             StillInitializing | ShuttingDown => {
     230            0 :                 ApiError::ResourceUnavailable(format!("{e}").into())
     231            0 :             }
     232            0 :         }
     233            0 :     }
     234              : }
     235              : 
     236              : impl From<TenantStateError> for ApiError {
     237            0 :     fn from(tse: TenantStateError) -> ApiError {
     238            0 :         match tse {
     239              :             TenantStateError::IsStopping(_) => {
     240            0 :                 ApiError::ResourceUnavailable("Tenant is stopping".into())
     241              :             }
     242            0 :             TenantStateError::SlotError(e) => e.into(),
     243            0 :             TenantStateError::SlotUpsertError(e) => e.into(),
     244            0 :             TenantStateError::Other(e) => ApiError::InternalServerError(anyhow!(e)),
     245              :         }
     246            0 :     }
     247              : }
     248              : 
     249              : impl From<GetTenantError> for ApiError {
     250            0 :     fn from(tse: GetTenantError) -> ApiError {
     251            0 :         match tse {
     252            0 :             GetTenantError::NotFound(tid) => ApiError::NotFound(anyhow!("tenant {tid}").into()),
     253            0 :             GetTenantError::ShardNotFound(tid) => {
     254            0 :                 ApiError::NotFound(anyhow!("tenant {tid}").into())
     255              :             }
     256              :             GetTenantError::NotActive(_) => {
     257              :                 // Why is this not `ApiError::NotFound`?
     258              :                 // Because we must be careful to never return 404 for a tenant if it does
     259              :                 // in fact exist locally. If we did, the caller could draw the conclusion
     260              :                 // that it can attach the tenant to another PS and we'd be in split-brain.
     261            0 :                 ApiError::ResourceUnavailable("Tenant not yet active".into())
     262              :             }
     263            0 :             GetTenantError::MapState(e) => ApiError::ResourceUnavailable(format!("{e}").into()),
     264              :         }
     265            0 :     }
     266              : }
     267              : 
     268              : impl From<GetTimelineError> for ApiError {
     269            0 :     fn from(gte: GetTimelineError) -> Self {
     270            0 :         // Rationale: tenant is activated only after eligble timelines activate
     271            0 :         ApiError::NotFound(gte.into())
     272            0 :     }
     273              : }
     274              : 
     275              : impl From<GetActiveTenantError> for ApiError {
     276            0 :     fn from(e: GetActiveTenantError) -> ApiError {
     277            0 :         match e {
     278            0 :             GetActiveTenantError::Broken(reason) => {
     279            0 :                 ApiError::InternalServerError(anyhow!("tenant is broken: {}", reason))
     280              :             }
     281              :             GetActiveTenantError::WillNotBecomeActive(TenantState::Stopping { .. }) => {
     282            0 :                 ApiError::ShuttingDown
     283              :             }
     284            0 :             GetActiveTenantError::WillNotBecomeActive(_) => ApiError::Conflict(format!("{}", e)),
     285            0 :             GetActiveTenantError::Cancelled => ApiError::ShuttingDown,
     286            0 :             GetActiveTenantError::NotFound(gte) => gte.into(),
     287              :             GetActiveTenantError::WaitForActiveTimeout { .. } => {
     288            0 :                 ApiError::ResourceUnavailable(format!("{}", e).into())
     289              :             }
     290              :             GetActiveTenantError::SwitchedTenant => {
     291              :                 // in our HTTP handlers, this error doesn't happen
     292              :                 // TODO: separate error types
     293            0 :                 ApiError::ResourceUnavailable("switched tenant".into())
     294              :             }
     295              :         }
     296            0 :     }
     297              : }
     298              : 
     299              : impl From<crate::tenant::DeleteTimelineError> for ApiError {
     300            0 :     fn from(value: crate::tenant::DeleteTimelineError) -> Self {
     301              :         use crate::tenant::DeleteTimelineError::*;
     302            0 :         match value {
     303            0 :             NotFound => ApiError::NotFound(anyhow::anyhow!("timeline not found").into()),
     304            0 :             HasChildren(children) => ApiError::PreconditionFailed(
     305            0 :                 format!("Cannot delete timeline which has child timelines: {children:?}")
     306            0 :                     .into_boxed_str(),
     307            0 :             ),
     308            0 :             a @ AlreadyInProgress(_) => ApiError::Conflict(a.to_string()),
     309            0 :             Cancelled => ApiError::ResourceUnavailable("shutting down".into()),
     310            0 :             Other(e) => ApiError::InternalServerError(e),
     311              :         }
     312            0 :     }
     313              : }
     314              : 
     315              : impl From<crate::tenant::TimelineArchivalError> for ApiError {
     316            0 :     fn from(value: crate::tenant::TimelineArchivalError) -> Self {
     317              :         use crate::tenant::TimelineArchivalError::*;
     318            0 :         match value {
     319            0 :             NotFound => ApiError::NotFound(anyhow::anyhow!("timeline not found").into()),
     320            0 :             Timeout => ApiError::Timeout("hit pageserver internal timeout".into()),
     321            0 :             Cancelled => ApiError::ShuttingDown,
     322            0 :             e @ HasArchivedParent(_) => {
     323            0 :                 ApiError::PreconditionFailed(e.to_string().into_boxed_str())
     324              :             }
     325            0 :             HasUnarchivedChildren(children) => ApiError::PreconditionFailed(
     326            0 :                 format!(
     327            0 :                     "Cannot archive timeline which has non-archived child timelines: {children:?}"
     328            0 :                 )
     329            0 :                 .into_boxed_str(),
     330            0 :             ),
     331            0 :             a @ AlreadyInProgress => ApiError::Conflict(a.to_string()),
     332            0 :             Other(e) => ApiError::InternalServerError(e),
     333              :         }
     334            0 :     }
     335              : }
     336              : 
     337              : impl From<crate::tenant::mgr::DeleteTimelineError> for ApiError {
     338            0 :     fn from(value: crate::tenant::mgr::DeleteTimelineError) -> Self {
     339              :         use crate::tenant::mgr::DeleteTimelineError::*;
     340            0 :         match value {
     341              :             // Report Precondition failed so client can distinguish between
     342              :             // "tenant is missing" case from "timeline is missing"
     343            0 :             Tenant(GetTenantError::NotFound(..)) => ApiError::PreconditionFailed(
     344            0 :                 "Requested tenant is missing".to_owned().into_boxed_str(),
     345            0 :             ),
     346            0 :             Tenant(t) => ApiError::from(t),
     347            0 :             Timeline(t) => ApiError::from(t),
     348              :         }
     349            0 :     }
     350              : }
     351              : 
     352              : impl From<crate::tenant::mgr::DeleteTenantError> for ApiError {
     353            0 :     fn from(value: crate::tenant::mgr::DeleteTenantError) -> Self {
     354              :         use crate::tenant::mgr::DeleteTenantError::*;
     355            0 :         match value {
     356            0 :             SlotError(e) => e.into(),
     357            0 :             Other(o) => ApiError::InternalServerError(o),
     358            0 :             Cancelled => ApiError::ShuttingDown,
     359              :         }
     360            0 :     }
     361              : }
     362              : 
     363              : impl From<crate::tenant::secondary::SecondaryTenantError> for ApiError {
     364            0 :     fn from(ste: crate::tenant::secondary::SecondaryTenantError) -> ApiError {
     365              :         use crate::tenant::secondary::SecondaryTenantError;
     366            0 :         match ste {
     367            0 :             SecondaryTenantError::GetTenant(gte) => gte.into(),
     368            0 :             SecondaryTenantError::ShuttingDown => ApiError::ShuttingDown,
     369              :         }
     370            0 :     }
     371              : }
     372              : 
     373              : // Helper function to construct a TimelineInfo struct for a timeline
     374            0 : async fn build_timeline_info(
     375            0 :     timeline: &Arc<Timeline>,
     376            0 :     include_non_incremental_logical_size: bool,
     377            0 :     force_await_initial_logical_size: bool,
     378            0 :     ctx: &RequestContext,
     379            0 : ) -> anyhow::Result<TimelineInfo> {
     380            0 :     crate::tenant::debug_assert_current_span_has_tenant_and_timeline_id();
     381            0 : 
     382            0 :     if force_await_initial_logical_size {
     383            0 :         timeline.clone().await_initial_logical_size().await
     384            0 :     }
     385              : 
     386            0 :     let mut info = build_timeline_info_common(
     387            0 :         timeline,
     388            0 :         ctx,
     389            0 :         tenant::timeline::GetLogicalSizePriority::Background,
     390            0 :     )
     391            0 :     .await?;
     392            0 :     if include_non_incremental_logical_size {
     393              :         // XXX we should be using spawn_ondemand_logical_size_calculation here.
     394              :         // Otherwise, if someone deletes the timeline / detaches the tenant while
     395              :         // we're executing this function, we will outlive the timeline on-disk state.
     396              :         info.current_logical_size_non_incremental = Some(
     397            0 :             timeline
     398            0 :                 .get_current_logical_size_non_incremental(info.last_record_lsn, ctx)
     399            0 :                 .await?,
     400              :         );
     401            0 :     }
     402            0 :     Ok(info)
     403            0 : }
     404              : 
     405            0 : async fn build_timeline_info_common(
     406            0 :     timeline: &Arc<Timeline>,
     407            0 :     ctx: &RequestContext,
     408            0 :     logical_size_task_priority: tenant::timeline::GetLogicalSizePriority,
     409            0 : ) -> anyhow::Result<TimelineInfo> {
     410            0 :     crate::tenant::debug_assert_current_span_has_tenant_and_timeline_id();
     411            0 :     let initdb_lsn = timeline.initdb_lsn;
     412            0 :     let last_record_lsn = timeline.get_last_record_lsn();
     413            0 :     let (wal_source_connstr, last_received_msg_lsn, last_received_msg_ts) = {
     414            0 :         let guard = timeline.last_received_wal.lock().unwrap();
     415            0 :         if let Some(info) = guard.as_ref() {
     416            0 :             (
     417            0 :                 Some(format!("{}", info.wal_source_connconf)), // Password is hidden, but it's for statistics only.
     418            0 :                 Some(info.last_received_msg_lsn),
     419            0 :                 Some(info.last_received_msg_ts),
     420            0 :             )
     421              :         } else {
     422            0 :             (None, None, None)
     423              :         }
     424              :     };
     425              : 
     426            0 :     let ancestor_timeline_id = timeline.get_ancestor_timeline_id();
     427            0 :     let ancestor_lsn = match timeline.get_ancestor_lsn() {
     428            0 :         Lsn(0) => None,
     429            0 :         lsn @ Lsn(_) => Some(lsn),
     430              :     };
     431            0 :     let current_logical_size = timeline.get_current_logical_size(logical_size_task_priority, ctx);
     432            0 :     let current_physical_size = Some(timeline.layer_size_sum().await);
     433            0 :     let state = timeline.current_state();
     434            0 :     // Report is_archived = false if the timeline is still loading
     435            0 :     let is_archived = timeline.is_archived().unwrap_or(false);
     436            0 :     let remote_consistent_lsn_projected = timeline
     437            0 :         .get_remote_consistent_lsn_projected()
     438            0 :         .unwrap_or(Lsn(0));
     439            0 :     let remote_consistent_lsn_visible = timeline
     440            0 :         .get_remote_consistent_lsn_visible()
     441            0 :         .unwrap_or(Lsn(0));
     442            0 : 
     443            0 :     let walreceiver_status = timeline.walreceiver_status();
     444            0 : 
     445            0 :     let (pitr_history_size, within_ancestor_pitr) = timeline.get_pitr_history_stats();
     446            0 : 
     447            0 :     let min_readable_lsn = std::cmp::max(
     448            0 :         timeline.get_gc_cutoff_lsn(),
     449            0 :         *timeline.get_applied_gc_cutoff_lsn(),
     450            0 :     );
     451              : 
     452            0 :     let info = TimelineInfo {
     453            0 :         tenant_id: timeline.tenant_shard_id,
     454            0 :         timeline_id: timeline.timeline_id,
     455            0 :         ancestor_timeline_id,
     456            0 :         ancestor_lsn,
     457            0 :         disk_consistent_lsn: timeline.get_disk_consistent_lsn(),
     458            0 :         remote_consistent_lsn: remote_consistent_lsn_projected,
     459            0 :         remote_consistent_lsn_visible,
     460            0 :         initdb_lsn,
     461            0 :         last_record_lsn,
     462            0 :         prev_record_lsn: Some(timeline.get_prev_record_lsn()),
     463            0 :         // Externally, expose the lowest LSN that can be used to create a branch as the "GC cutoff", although internally
     464            0 :         // we distinguish between the "planned" GC cutoff (PITR point) and the "latest" GC cutoff (where we
     465            0 :         // actually trimmed data to), which can pass each other when PITR is changed.
     466            0 :         latest_gc_cutoff_lsn: min_readable_lsn,
     467            0 :         min_readable_lsn,
     468            0 :         applied_gc_cutoff_lsn: *timeline.get_applied_gc_cutoff_lsn(),
     469            0 :         current_logical_size: current_logical_size.size_dont_care_about_accuracy(),
     470            0 :         current_logical_size_is_accurate: match current_logical_size.accuracy() {
     471            0 :             tenant::timeline::logical_size::Accuracy::Approximate => false,
     472            0 :             tenant::timeline::logical_size::Accuracy::Exact => true,
     473              :         },
     474            0 :         directory_entries_counts: timeline.get_directory_metrics().to_vec(),
     475            0 :         current_physical_size,
     476            0 :         current_logical_size_non_incremental: None,
     477            0 :         pitr_history_size,
     478            0 :         within_ancestor_pitr,
     479            0 :         timeline_dir_layer_file_size_sum: None,
     480            0 :         wal_source_connstr,
     481            0 :         last_received_msg_lsn,
     482            0 :         last_received_msg_ts,
     483            0 :         pg_version: timeline.pg_version,
     484            0 : 
     485            0 :         state,
     486            0 :         is_archived: Some(is_archived),
     487            0 :         rel_size_migration: Some(timeline.get_rel_size_v2_status()),
     488            0 : 
     489            0 :         walreceiver_status,
     490            0 :     };
     491            0 :     Ok(info)
     492            0 : }
     493              : 
     494            0 : fn build_timeline_offloaded_info(offloaded: &Arc<OffloadedTimeline>) -> OffloadedTimelineInfo {
     495            0 :     let &OffloadedTimeline {
     496            0 :         tenant_shard_id,
     497            0 :         timeline_id,
     498            0 :         ancestor_retain_lsn,
     499            0 :         ancestor_timeline_id,
     500            0 :         archived_at,
     501            0 :         ..
     502            0 :     } = offloaded.as_ref();
     503            0 :     OffloadedTimelineInfo {
     504            0 :         tenant_id: tenant_shard_id,
     505            0 :         timeline_id,
     506            0 :         ancestor_retain_lsn,
     507            0 :         ancestor_timeline_id,
     508            0 :         archived_at: archived_at.and_utc(),
     509            0 :     }
     510            0 : }
     511              : 
     512              : // healthcheck handler
     513            0 : async fn status_handler(
     514            0 :     request: Request<Body>,
     515            0 :     _cancel: CancellationToken,
     516            0 : ) -> Result<Response<Body>, ApiError> {
     517            0 :     check_permission(&request, None)?;
     518            0 :     let config = get_config(&request);
     519            0 :     json_response(StatusCode::OK, StatusResponse { id: config.id })
     520            0 : }
     521              : 
     522            0 : async fn reload_auth_validation_keys_handler(
     523            0 :     request: Request<Body>,
     524            0 :     _cancel: CancellationToken,
     525            0 : ) -> Result<Response<Body>, ApiError> {
     526            0 :     check_permission(&request, None)?;
     527            0 :     let config = get_config(&request);
     528            0 :     let state = get_state(&request);
     529            0 :     let Some(shared_auth) = &state.auth else {
     530            0 :         return json_response(StatusCode::BAD_REQUEST, ());
     531              :     };
     532              :     // unwrap is ok because check is performed when creating config, so path is set and exists
     533            0 :     let key_path = config.auth_validation_public_key_path.as_ref().unwrap();
     534            0 :     info!("Reloading public key(s) for verifying JWT tokens from {key_path:?}");
     535              : 
     536            0 :     match utils::auth::JwtAuth::from_key_path(key_path) {
     537            0 :         Ok(new_auth) => {
     538            0 :             shared_auth.swap(new_auth);
     539            0 :             json_response(StatusCode::OK, ())
     540              :         }
     541            0 :         Err(e) => {
     542            0 :             let err_msg = "Error reloading public keys";
     543            0 :             warn!("Error reloading public keys from {key_path:?}: {e:}");
     544            0 :             json_response(
     545            0 :                 StatusCode::INTERNAL_SERVER_ERROR,
     546            0 :                 HttpErrorBody::from_msg(err_msg.to_string()),
     547            0 :             )
     548              :         }
     549              :     }
     550            0 : }
     551              : 
     552            0 : async fn timeline_create_handler(
     553            0 :     mut request: Request<Body>,
     554            0 :     _cancel: CancellationToken,
     555            0 : ) -> Result<Response<Body>, ApiError> {
     556            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
     557            0 :     let request_data: TimelineCreateRequest = json_request(&mut request).await?;
     558            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
     559              : 
     560            0 :     let new_timeline_id = request_data.new_timeline_id;
     561              :     // fill in the default pg_version if not provided & convert request into domain model
     562            0 :     let params: tenant::CreateTimelineParams = match request_data.mode {
     563              :         TimelineCreateRequestMode::Bootstrap {
     564            0 :             existing_initdb_timeline_id,
     565            0 :             pg_version,
     566            0 :         } => tenant::CreateTimelineParams::Bootstrap(tenant::CreateTimelineParamsBootstrap {
     567            0 :             new_timeline_id,
     568            0 :             existing_initdb_timeline_id,
     569            0 :             pg_version: pg_version.unwrap_or(DEFAULT_PG_VERSION),
     570            0 :         }),
     571              :         TimelineCreateRequestMode::Branch {
     572            0 :             ancestor_timeline_id,
     573            0 :             ancestor_start_lsn,
     574            0 :             pg_version: _,
     575            0 :         } => tenant::CreateTimelineParams::Branch(tenant::CreateTimelineParamsBranch {
     576            0 :             new_timeline_id,
     577            0 :             ancestor_timeline_id,
     578            0 :             ancestor_start_lsn,
     579            0 :         }),
     580              :         TimelineCreateRequestMode::ImportPgdata {
     581              :             import_pgdata:
     582              :                 TimelineCreateRequestModeImportPgdata {
     583            0 :                     location,
     584            0 :                     idempotency_key,
     585            0 :                 },
     586            0 :         } => tenant::CreateTimelineParams::ImportPgdata(tenant::CreateTimelineParamsImportPgdata {
     587            0 :             idempotency_key: import_pgdata::index_part_format::IdempotencyKey::new(
     588            0 :                 idempotency_key.0,
     589            0 :             ),
     590            0 :             new_timeline_id,
     591              :             location: {
     592            0 :                 use import_pgdata::index_part_format::Location;
     593            0 :                 use pageserver_api::models::ImportPgdataLocation;
     594            0 :                 match location {
     595              :                     #[cfg(feature = "testing")]
     596            0 :                     ImportPgdataLocation::LocalFs { path } => Location::LocalFs { path },
     597              :                     ImportPgdataLocation::AwsS3 {
     598            0 :                         region,
     599            0 :                         bucket,
     600            0 :                         key,
     601            0 :                     } => Location::AwsS3 {
     602            0 :                         region,
     603            0 :                         bucket,
     604            0 :                         key,
     605            0 :                     },
     606              :                 }
     607              :             },
     608              :         }),
     609              :     };
     610              : 
     611            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Error);
     612            0 : 
     613            0 :     let state = get_state(&request);
     614              : 
     615            0 :     async {
     616            0 :         let tenant = state
     617            0 :             .tenant_manager
     618            0 :             .get_attached_tenant_shard(tenant_shard_id)?;
     619              : 
     620            0 :         tenant.wait_to_become_active(ACTIVE_TENANT_TIMEOUT).await?;
     621              : 
     622              :         // earlier versions of the code had pg_version and ancestor_lsn in the span
     623              :         // => continue to provide that information, but, through a log message that doesn't require us to destructure
     624            0 :         tracing::info!(?params, "creating timeline");
     625              : 
     626            0 :         match tenant
     627            0 :             .create_timeline(params, state.broker_client.clone(), &ctx)
     628            0 :             .await
     629              :         {
     630            0 :             Ok(new_timeline) => {
     631              :                 // Created. Construct a TimelineInfo for it.
     632            0 :                 let timeline_info = build_timeline_info_common(
     633            0 :                     &new_timeline,
     634            0 :                     &ctx,
     635            0 :                     tenant::timeline::GetLogicalSizePriority::User,
     636            0 :                 )
     637            0 :                 .await
     638            0 :                 .map_err(ApiError::InternalServerError)?;
     639            0 :                 json_response(StatusCode::CREATED, timeline_info)
     640              :             }
     641            0 :             Err(_) if tenant.cancel.is_cancelled() => {
     642            0 :                 // In case we get some ugly error type during shutdown, cast it into a clean 503.
     643            0 :                 json_response(
     644            0 :                     StatusCode::SERVICE_UNAVAILABLE,
     645            0 :                     HttpErrorBody::from_msg("Tenant shutting down".to_string()),
     646            0 :                 )
     647              :             }
     648            0 :             Err(e @ tenant::CreateTimelineError::Conflict) => {
     649            0 :                 json_response(StatusCode::CONFLICT, HttpErrorBody::from_msg(e.to_string()))
     650              :             }
     651            0 :             Err(e @ tenant::CreateTimelineError::AlreadyCreating) => json_response(
     652            0 :                 StatusCode::TOO_MANY_REQUESTS,
     653            0 :                 HttpErrorBody::from_msg(e.to_string()),
     654            0 :             ),
     655            0 :             Err(tenant::CreateTimelineError::AncestorLsn(err)) => json_response(
     656            0 :                 StatusCode::NOT_ACCEPTABLE,
     657            0 :                 HttpErrorBody::from_msg(format!("{err:#}")),
     658            0 :             ),
     659            0 :             Err(e @ tenant::CreateTimelineError::AncestorNotActive) => json_response(
     660            0 :                 StatusCode::SERVICE_UNAVAILABLE,
     661            0 :                 HttpErrorBody::from_msg(e.to_string()),
     662            0 :             ),
     663            0 :             Err(e @ tenant::CreateTimelineError::AncestorArchived) => json_response(
     664            0 :                 StatusCode::NOT_ACCEPTABLE,
     665            0 :                 HttpErrorBody::from_msg(e.to_string()),
     666            0 :             ),
     667            0 :             Err(tenant::CreateTimelineError::ShuttingDown) => json_response(
     668            0 :                 StatusCode::SERVICE_UNAVAILABLE,
     669            0 :                 HttpErrorBody::from_msg("tenant shutting down".to_string()),
     670            0 :             ),
     671            0 :             Err(tenant::CreateTimelineError::Other(err)) => Err(ApiError::InternalServerError(err)),
     672              :         }
     673            0 :     }
     674            0 :     .instrument(info_span!("timeline_create",
     675              :         tenant_id = %tenant_shard_id.tenant_id,
     676            0 :         shard_id = %tenant_shard_id.shard_slug(),
     677              :         timeline_id = %new_timeline_id,
     678              :     ))
     679            0 :     .await
     680            0 : }
     681              : 
     682            0 : async fn timeline_list_handler(
     683            0 :     request: Request<Body>,
     684            0 :     _cancel: CancellationToken,
     685            0 : ) -> Result<Response<Body>, ApiError> {
     686            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
     687            0 :     let include_non_incremental_logical_size: Option<bool> =
     688            0 :         parse_query_param(&request, "include-non-incremental-logical-size")?;
     689            0 :     let force_await_initial_logical_size: Option<bool> =
     690            0 :         parse_query_param(&request, "force-await-initial-logical-size")?;
     691            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
     692              : 
     693            0 :     let state = get_state(&request);
     694            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download);
     695              : 
     696            0 :     let response_data = async {
     697            0 :         let tenant = state
     698            0 :             .tenant_manager
     699            0 :             .get_attached_tenant_shard(tenant_shard_id)?;
     700              : 
     701            0 :         tenant.wait_to_become_active(ACTIVE_TENANT_TIMEOUT).await?;
     702              : 
     703            0 :         let timelines = tenant.list_timelines();
     704            0 : 
     705            0 :         let mut response_data = Vec::with_capacity(timelines.len());
     706            0 :         for timeline in timelines {
     707            0 :             let timeline_info = build_timeline_info(
     708            0 :                 &timeline,
     709            0 :                 include_non_incremental_logical_size.unwrap_or(false),
     710            0 :                 force_await_initial_logical_size.unwrap_or(false),
     711            0 :                 &ctx,
     712            0 :             )
     713            0 :             .instrument(info_span!("build_timeline_info", timeline_id = %timeline.timeline_id))
     714            0 :             .await
     715            0 :             .context("Failed to build timeline info")
     716            0 :             .map_err(ApiError::InternalServerError)?;
     717              : 
     718            0 :             response_data.push(timeline_info);
     719              :         }
     720            0 :         Ok::<Vec<TimelineInfo>, ApiError>(response_data)
     721            0 :     }
     722            0 :     .instrument(info_span!("timeline_list",
     723              :                 tenant_id = %tenant_shard_id.tenant_id,
     724            0 :                 shard_id = %tenant_shard_id.shard_slug()))
     725            0 :     .await?;
     726              : 
     727            0 :     json_response(StatusCode::OK, response_data)
     728            0 : }
     729              : 
     730            0 : async fn timeline_and_offloaded_list_handler(
     731            0 :     request: Request<Body>,
     732            0 :     _cancel: CancellationToken,
     733            0 : ) -> Result<Response<Body>, ApiError> {
     734            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
     735            0 :     let include_non_incremental_logical_size: Option<bool> =
     736            0 :         parse_query_param(&request, "include-non-incremental-logical-size")?;
     737            0 :     let force_await_initial_logical_size: Option<bool> =
     738            0 :         parse_query_param(&request, "force-await-initial-logical-size")?;
     739            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
     740              : 
     741            0 :     let state = get_state(&request);
     742            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download);
     743              : 
     744            0 :     let response_data = async {
     745            0 :         let tenant = state
     746            0 :             .tenant_manager
     747            0 :             .get_attached_tenant_shard(tenant_shard_id)?;
     748              : 
     749            0 :         tenant.wait_to_become_active(ACTIVE_TENANT_TIMEOUT).await?;
     750              : 
     751            0 :         let (timelines, offloadeds) = tenant.list_timelines_and_offloaded();
     752            0 : 
     753            0 :         let mut timeline_infos = Vec::with_capacity(timelines.len());
     754            0 :         for timeline in timelines {
     755            0 :             let timeline_info = build_timeline_info(
     756            0 :                 &timeline,
     757            0 :                 include_non_incremental_logical_size.unwrap_or(false),
     758            0 :                 force_await_initial_logical_size.unwrap_or(false),
     759            0 :                 &ctx,
     760            0 :             )
     761            0 :             .instrument(info_span!("build_timeline_info", timeline_id = %timeline.timeline_id))
     762            0 :             .await
     763            0 :             .context("Failed to build timeline info")
     764            0 :             .map_err(ApiError::InternalServerError)?;
     765              : 
     766            0 :             timeline_infos.push(timeline_info);
     767              :         }
     768            0 :         let offloaded_infos = offloadeds
     769            0 :             .into_iter()
     770            0 :             .map(|offloaded| build_timeline_offloaded_info(&offloaded))
     771            0 :             .collect::<Vec<_>>();
     772            0 :         let res = TimelinesInfoAndOffloaded {
     773            0 :             timelines: timeline_infos,
     774            0 :             offloaded: offloaded_infos,
     775            0 :         };
     776            0 :         Ok::<TimelinesInfoAndOffloaded, ApiError>(res)
     777            0 :     }
     778            0 :     .instrument(info_span!("timeline_and_offloaded_list",
     779              :                 tenant_id = %tenant_shard_id.tenant_id,
     780            0 :                 shard_id = %tenant_shard_id.shard_slug()))
     781            0 :     .await?;
     782              : 
     783            0 :     json_response(StatusCode::OK, response_data)
     784            0 : }
     785              : 
     786            0 : async fn timeline_preserve_initdb_handler(
     787            0 :     request: Request<Body>,
     788            0 :     _cancel: CancellationToken,
     789            0 : ) -> Result<Response<Body>, ApiError> {
     790            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
     791            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
     792            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
     793            0 :     let state = get_state(&request);
     794              : 
     795              :     // Part of the process for disaster recovery from safekeeper-stored WAL:
     796              :     // If we don't recover into a new timeline but want to keep the timeline ID,
     797              :     // then the initdb archive is deleted. This endpoint copies it to a different
     798              :     // location where timeline recreation cand find it.
     799              : 
     800            0 :     async {
     801            0 :         let tenant = state
     802            0 :             .tenant_manager
     803            0 :             .get_attached_tenant_shard(tenant_shard_id)?;
     804              : 
     805            0 :         let timeline = tenant.get_timeline(timeline_id, false)?;
     806              : 
     807            0 :         timeline
     808            0 :             .preserve_initdb_archive()
     809            0 :             .await
     810            0 :             .context("preserving initdb archive")
     811            0 :             .map_err(ApiError::InternalServerError)?;
     812              : 
     813            0 :         Ok::<_, ApiError>(())
     814            0 :     }
     815            0 :     .instrument(info_span!("timeline_preserve_initdb_archive",
     816              :                 tenant_id = %tenant_shard_id.tenant_id,
     817            0 :                 shard_id = %tenant_shard_id.shard_slug(),
     818              :                 %timeline_id))
     819            0 :     .await?;
     820              : 
     821            0 :     json_response(StatusCode::OK, ())
     822            0 : }
     823              : 
     824            0 : async fn timeline_archival_config_handler(
     825            0 :     mut request: Request<Body>,
     826            0 :     _cancel: CancellationToken,
     827            0 : ) -> Result<Response<Body>, ApiError> {
     828            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
     829            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
     830              : 
     831            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Warn);
     832              : 
     833            0 :     let request_data: TimelineArchivalConfigRequest = json_request(&mut request).await?;
     834            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
     835            0 :     let state = get_state(&request);
     836              : 
     837            0 :     async {
     838            0 :         let tenant = state
     839            0 :             .tenant_manager
     840            0 :             .get_attached_tenant_shard(tenant_shard_id)?;
     841              : 
     842            0 :         tenant.wait_to_become_active(ACTIVE_TENANT_TIMEOUT).await?;
     843              : 
     844            0 :         tenant
     845            0 :             .apply_timeline_archival_config(
     846            0 :                 timeline_id,
     847            0 :                 request_data.state,
     848            0 :                 state.broker_client.clone(),
     849            0 :                 ctx,
     850            0 :             )
     851            0 :             .await?;
     852            0 :         Ok::<_, ApiError>(())
     853            0 :     }
     854            0 :     .instrument(info_span!("timeline_archival_config",
     855              :                 tenant_id = %tenant_shard_id.tenant_id,
     856            0 :                 shard_id = %tenant_shard_id.shard_slug(),
     857              :                 state = ?request_data.state,
     858              :                 %timeline_id))
     859            0 :     .await?;
     860              : 
     861            0 :     json_response(StatusCode::OK, ())
     862            0 : }
     863              : 
     864              : /// This API is used to patch the index part of a timeline. You must ensure such patches are safe to apply. Use this API as an emergency
     865              : /// measure only.
     866              : ///
     867              : /// Some examples of safe patches:
     868              : /// - Increase the gc_cutoff and gc_compaction_cutoff to a larger value in case of a bug that didn't bump the cutoff and cause read errors.
     869              : /// - Force set the index part to use reldir v2 (migrating/migrated).
     870              : ///
     871              : /// Some examples of unsafe patches:
     872              : /// - Force set the index part from v2 to v1 (legacy). This will cause the code path to ignore anything written to the new keyspace and cause
     873              : ///   errors.
     874              : /// - Decrease the gc_cutoff without validating the data really exists. It will cause read errors in the background.
     875            0 : async fn timeline_patch_index_part_handler(
     876            0 :     mut request: Request<Body>,
     877            0 :     _cancel: CancellationToken,
     878            0 : ) -> Result<Response<Body>, ApiError> {
     879            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
     880            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
     881              : 
     882            0 :     let request_data: TimelinePatchIndexPartRequest = json_request(&mut request).await?;
     883            0 :     check_permission(&request, None)?; // require global permission for this request
     884            0 :     let state = get_state(&request);
     885              : 
     886            0 :     async {
     887            0 :         let timeline =
     888            0 :             active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
     889            0 :                 .await?;
     890              : 
     891            0 :         if let Some(rel_size_migration) = request_data.rel_size_migration {
     892            0 :             timeline
     893            0 :                 .update_rel_size_v2_status(rel_size_migration)
     894            0 :                 .map_err(ApiError::InternalServerError)?;
     895            0 :         }
     896              : 
     897            0 :         if let Some(gc_compaction_last_completed_lsn) =
     898            0 :             request_data.gc_compaction_last_completed_lsn
     899              :         {
     900            0 :             timeline
     901            0 :                 .update_gc_compaction_state(GcCompactionState {
     902            0 :                     last_completed_lsn: gc_compaction_last_completed_lsn,
     903            0 :                 })
     904            0 :                 .map_err(ApiError::InternalServerError)?;
     905            0 :         }
     906              : 
     907            0 :         if let Some(applied_gc_cutoff_lsn) = request_data.applied_gc_cutoff_lsn {
     908            0 :             {
     909            0 :                 let guard = timeline.applied_gc_cutoff_lsn.lock_for_write();
     910            0 :                 guard.store_and_unlock(applied_gc_cutoff_lsn);
     911            0 :             }
     912            0 :         }
     913              : 
     914            0 :         if request_data.force_index_update {
     915            0 :             timeline
     916            0 :                 .remote_client
     917            0 :                 .force_schedule_index_upload()
     918            0 :                 .context("force schedule index upload")
     919            0 :                 .map_err(ApiError::InternalServerError)?;
     920            0 :         }
     921              : 
     922            0 :         Ok::<_, ApiError>(())
     923            0 :     }
     924            0 :     .instrument(info_span!("timeline_patch_index_part",
     925              :                 tenant_id = %tenant_shard_id.tenant_id,
     926            0 :                 shard_id = %tenant_shard_id.shard_slug(),
     927              :                 %timeline_id))
     928            0 :     .await?;
     929              : 
     930            0 :     json_response(StatusCode::OK, ())
     931            0 : }
     932              : 
     933            0 : async fn timeline_detail_handler(
     934            0 :     request: Request<Body>,
     935            0 :     _cancel: CancellationToken,
     936            0 : ) -> Result<Response<Body>, ApiError> {
     937            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
     938            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
     939            0 :     let include_non_incremental_logical_size: Option<bool> =
     940            0 :         parse_query_param(&request, "include-non-incremental-logical-size")?;
     941            0 :     let force_await_initial_logical_size: Option<bool> =
     942            0 :         parse_query_param(&request, "force-await-initial-logical-size")?;
     943            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
     944              : 
     945              :     // Logical size calculation needs downloading.
     946            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download);
     947            0 :     let state = get_state(&request);
     948              : 
     949            0 :     let timeline_info = async {
     950            0 :         let tenant = state
     951            0 :             .tenant_manager
     952            0 :             .get_attached_tenant_shard(tenant_shard_id)?;
     953              : 
     954            0 :         tenant.wait_to_become_active(ACTIVE_TENANT_TIMEOUT).await?;
     955              : 
     956            0 :         let timeline = tenant.get_timeline(timeline_id, false)?;
     957            0 :         let ctx = &ctx.with_scope_timeline(&timeline);
     958              : 
     959            0 :         let timeline_info = build_timeline_info(
     960            0 :             &timeline,
     961            0 :             include_non_incremental_logical_size.unwrap_or(false),
     962            0 :             force_await_initial_logical_size.unwrap_or(false),
     963            0 :             ctx,
     964            0 :         )
     965            0 :         .await
     966            0 :         .context("get local timeline info")
     967            0 :         .map_err(ApiError::InternalServerError)?;
     968              : 
     969            0 :         Ok::<_, ApiError>(timeline_info)
     970            0 :     }
     971            0 :     .instrument(info_span!("timeline_detail",
     972              :                 tenant_id = %tenant_shard_id.tenant_id,
     973            0 :                 shard_id = %tenant_shard_id.shard_slug(),
     974              :                 %timeline_id))
     975            0 :     .await?;
     976              : 
     977            0 :     json_response(StatusCode::OK, timeline_info)
     978            0 : }
     979              : 
     980            0 : async fn get_lsn_by_timestamp_handler(
     981            0 :     request: Request<Body>,
     982            0 :     cancel: CancellationToken,
     983            0 : ) -> Result<Response<Body>, ApiError> {
     984            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
     985            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
     986            0 :     let state = get_state(&request);
     987            0 : 
     988            0 :     if !tenant_shard_id.is_shard_zero() {
     989              :         // Requires SLRU contents, which are only stored on shard zero
     990            0 :         return Err(ApiError::BadRequest(anyhow!(
     991            0 :             "Size calculations are only available on shard zero"
     992            0 :         )));
     993            0 :     }
     994              : 
     995            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
     996            0 :     let timestamp_raw = must_get_query_param(&request, "timestamp")?;
     997            0 :     let timestamp = humantime::parse_rfc3339(&timestamp_raw)
     998            0 :         .with_context(|| format!("Invalid time: {:?}", timestamp_raw))
     999            0 :         .map_err(ApiError::BadRequest)?;
    1000            0 :     let timestamp_pg = postgres_ffi::to_pg_timestamp(timestamp);
    1001              : 
    1002            0 :     let with_lease = parse_query_param(&request, "with_lease")?.unwrap_or(false);
    1003              : 
    1004            0 :     let timeline =
    1005            0 :         active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
    1006            0 :             .await?;
    1007            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download)
    1008            0 :         .with_scope_timeline(&timeline);
    1009            0 :     let result = timeline
    1010            0 :         .find_lsn_for_timestamp(timestamp_pg, &cancel, &ctx)
    1011            0 :         .await?;
    1012              : 
    1013              :     #[derive(serde::Serialize, Debug)]
    1014              :     struct Result {
    1015              :         lsn: Lsn,
    1016              :         kind: &'static str,
    1017              :         #[serde(default)]
    1018              :         #[serde(skip_serializing_if = "Option::is_none")]
    1019              :         #[serde(flatten)]
    1020              :         lease: Option<LsnLease>,
    1021              :     }
    1022            0 :     let (lsn, kind) = match result {
    1023            0 :         LsnForTimestamp::Present(lsn) => (lsn, "present"),
    1024            0 :         LsnForTimestamp::Future(lsn) => (lsn, "future"),
    1025            0 :         LsnForTimestamp::Past(lsn) => (lsn, "past"),
    1026            0 :         LsnForTimestamp::NoData(lsn) => (lsn, "nodata"),
    1027              :     };
    1028              : 
    1029            0 :     let lease = if with_lease {
    1030            0 :         timeline
    1031            0 :             .init_lsn_lease(lsn, timeline.get_lsn_lease_length_for_ts(), &ctx)
    1032            0 :             .inspect_err(|_| {
    1033            0 :                 warn!("fail to grant a lease to {}", lsn);
    1034            0 :             })
    1035            0 :             .ok()
    1036              :     } else {
    1037            0 :         None
    1038              :     };
    1039              : 
    1040            0 :     let result = Result { lsn, kind, lease };
    1041            0 :     let valid_until = result
    1042            0 :         .lease
    1043            0 :         .as_ref()
    1044            0 :         .map(|l| humantime::format_rfc3339_millis(l.valid_until).to_string());
    1045            0 :     tracing::info!(
    1046              :         lsn=?result.lsn,
    1047              :         kind=%result.kind,
    1048              :         timestamp=%timestamp_raw,
    1049              :         valid_until=?valid_until,
    1050            0 :         "lsn_by_timestamp finished"
    1051              :     );
    1052            0 :     json_response(StatusCode::OK, result)
    1053            0 : }
    1054              : 
    1055            0 : async fn get_timestamp_of_lsn_handler(
    1056            0 :     request: Request<Body>,
    1057            0 :     _cancel: CancellationToken,
    1058            0 : ) -> Result<Response<Body>, ApiError> {
    1059            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    1060            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    1061            0 :     let state = get_state(&request);
    1062            0 : 
    1063            0 :     if !tenant_shard_id.is_shard_zero() {
    1064              :         // Requires SLRU contents, which are only stored on shard zero
    1065            0 :         return Err(ApiError::BadRequest(anyhow!(
    1066            0 :             "Size calculations are only available on shard zero"
    1067            0 :         )));
    1068            0 :     }
    1069              : 
    1070            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    1071              : 
    1072            0 :     let lsn_str = must_get_query_param(&request, "lsn")?;
    1073            0 :     let lsn = Lsn::from_str(&lsn_str)
    1074            0 :         .with_context(|| format!("Invalid LSN: {lsn_str:?}"))
    1075            0 :         .map_err(ApiError::BadRequest)?;
    1076              : 
    1077            0 :     let timeline =
    1078            0 :         active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
    1079            0 :             .await?;
    1080            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download)
    1081            0 :         .with_scope_timeline(&timeline);
    1082            0 :     let result = timeline.get_timestamp_for_lsn(lsn, &ctx).await?;
    1083              : 
    1084            0 :     match result {
    1085            0 :         Some(time) => {
    1086            0 :             let time = format_rfc3339(
    1087            0 :                 postgres_ffi::try_from_pg_timestamp(time).map_err(ApiError::InternalServerError)?,
    1088              :             )
    1089            0 :             .to_string();
    1090            0 :             json_response(StatusCode::OK, time)
    1091              :         }
    1092            0 :         None => Err(ApiError::NotFound(
    1093            0 :             anyhow::anyhow!("Timestamp for lsn {} not found", lsn).into(),
    1094            0 :         )),
    1095              :     }
    1096            0 : }
    1097              : 
    1098            0 : async fn timeline_delete_handler(
    1099            0 :     request: Request<Body>,
    1100            0 :     _cancel: CancellationToken,
    1101            0 : ) -> Result<Response<Body>, ApiError> {
    1102            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    1103            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    1104            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    1105              : 
    1106            0 :     let state = get_state(&request);
    1107              : 
    1108            0 :     let tenant = state
    1109            0 :         .tenant_manager
    1110            0 :         .get_attached_tenant_shard(tenant_shard_id)
    1111            0 :         .map_err(|e| {
    1112            0 :             match e {
    1113              :                 // GetTenantError has a built-in conversion to ApiError, but in this context we don't
    1114              :                 // want to treat missing tenants as 404, to avoid ambiguity with successful deletions.
    1115              :                 GetTenantError::NotFound(_) | GetTenantError::ShardNotFound(_) => {
    1116            0 :                     ApiError::PreconditionFailed(
    1117            0 :                         "Requested tenant is missing".to_string().into_boxed_str(),
    1118            0 :                     )
    1119              :                 }
    1120            0 :                 e => e.into(),
    1121              :             }
    1122            0 :         })?;
    1123            0 :     tenant.wait_to_become_active(ACTIVE_TENANT_TIMEOUT).await?;
    1124            0 :     tenant.delete_timeline(timeline_id).instrument(info_span!("timeline_delete", tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug(), %timeline_id))
    1125            0 :         .await?;
    1126              : 
    1127            0 :     json_response(StatusCode::ACCEPTED, ())
    1128            0 : }
    1129              : 
    1130            0 : async fn tenant_reset_handler(
    1131            0 :     request: Request<Body>,
    1132            0 :     _cancel: CancellationToken,
    1133            0 : ) -> Result<Response<Body>, ApiError> {
    1134            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    1135            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    1136              : 
    1137            0 :     let drop_cache: Option<bool> = parse_query_param(&request, "drop_cache")?;
    1138              : 
    1139            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Warn);
    1140            0 :     let state = get_state(&request);
    1141            0 :     state
    1142            0 :         .tenant_manager
    1143            0 :         .reset_tenant(tenant_shard_id, drop_cache.unwrap_or(false), &ctx)
    1144            0 :         .await
    1145            0 :         .map_err(ApiError::InternalServerError)?;
    1146              : 
    1147            0 :     json_response(StatusCode::OK, ())
    1148            0 : }
    1149              : 
    1150            0 : async fn tenant_list_handler(
    1151            0 :     request: Request<Body>,
    1152            0 :     _cancel: CancellationToken,
    1153            0 : ) -> Result<Response<Body>, ApiError> {
    1154            0 :     check_permission(&request, None)?;
    1155            0 :     let state = get_state(&request);
    1156              : 
    1157            0 :     let response_data = state
    1158            0 :         .tenant_manager
    1159            0 :         .list_tenants()
    1160            0 :         .map_err(|_| {
    1161            0 :             ApiError::ResourceUnavailable("Tenant map is initializing or shutting down".into())
    1162            0 :         })?
    1163            0 :         .iter()
    1164            0 :         .map(|(id, state, gen_)| TenantInfo {
    1165            0 :             id: *id,
    1166            0 :             state: state.clone(),
    1167            0 :             current_physical_size: None,
    1168            0 :             attachment_status: state.attachment_status(),
    1169            0 :             generation: (*gen_)
    1170            0 :                 .into()
    1171            0 :                 .expect("Tenants are always attached with a generation"),
    1172            0 :             gc_blocking: None,
    1173            0 :         })
    1174            0 :         .collect::<Vec<TenantInfo>>();
    1175            0 : 
    1176            0 :     json_response(StatusCode::OK, response_data)
    1177            0 : }
    1178              : 
    1179            0 : async fn tenant_status(
    1180            0 :     request: Request<Body>,
    1181            0 :     _cancel: CancellationToken,
    1182            0 : ) -> Result<Response<Body>, ApiError> {
    1183            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    1184            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    1185            0 :     let state = get_state(&request);
    1186            0 : 
    1187            0 :     // In tests, sometimes we want to query the state of a tenant without auto-activating it if it's currently waiting.
    1188            0 :     let activate = true;
    1189              :     #[cfg(feature = "testing")]
    1190            0 :     let activate = parse_query_param(&request, "activate")?.unwrap_or(activate);
    1191              : 
    1192            0 :     let tenant_info = async {
    1193            0 :         let tenant = state
    1194            0 :             .tenant_manager
    1195            0 :             .get_attached_tenant_shard(tenant_shard_id)?;
    1196              : 
    1197            0 :         if activate {
    1198              :             // This is advisory: we prefer to let the tenant activate on-demand when this function is
    1199              :             // called, but it is still valid to return 200 and describe the current state of the tenant
    1200              :             // if it doesn't make it into an active state.
    1201            0 :             tenant
    1202            0 :                 .wait_to_become_active(ACTIVE_TENANT_TIMEOUT)
    1203            0 :                 .await
    1204            0 :                 .ok();
    1205            0 :         }
    1206              : 
    1207              :         // Calculate total physical size of all timelines
    1208            0 :         let mut current_physical_size = 0;
    1209            0 :         for timeline in tenant.list_timelines().iter() {
    1210            0 :             current_physical_size += timeline.layer_size_sum().await;
    1211              :         }
    1212              : 
    1213            0 :         let state = tenant.current_state();
    1214            0 :         Result::<_, ApiError>::Ok(TenantDetails {
    1215            0 :             tenant_info: TenantInfo {
    1216            0 :                 id: tenant_shard_id,
    1217            0 :                 state: state.clone(),
    1218            0 :                 current_physical_size: Some(current_physical_size),
    1219            0 :                 attachment_status: state.attachment_status(),
    1220            0 :                 generation: tenant
    1221            0 :                     .generation()
    1222            0 :                     .into()
    1223            0 :                     .expect("Tenants are always attached with a generation"),
    1224            0 :                 gc_blocking: tenant.gc_block.summary().map(|x| format!("{x:?}")),
    1225            0 :             },
    1226            0 :             walredo: tenant.wal_redo_manager_status(),
    1227            0 :             timelines: tenant.list_timeline_ids(),
    1228            0 :         })
    1229            0 :     }
    1230            0 :     .instrument(info_span!("tenant_status_handler",
    1231              :                 tenant_id = %tenant_shard_id.tenant_id,
    1232            0 :                 shard_id = %tenant_shard_id.shard_slug()))
    1233            0 :     .await?;
    1234              : 
    1235            0 :     json_response(StatusCode::OK, tenant_info)
    1236            0 : }
    1237              : 
    1238            0 : async fn tenant_delete_handler(
    1239            0 :     request: Request<Body>,
    1240            0 :     _cancel: CancellationToken,
    1241            0 : ) -> Result<Response<Body>, ApiError> {
    1242              :     // TODO openapi spec
    1243            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    1244            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    1245              : 
    1246            0 :     let state = get_state(&request);
    1247            0 : 
    1248            0 :     state
    1249            0 :         .tenant_manager
    1250            0 :         .delete_tenant(tenant_shard_id)
    1251            0 :         .instrument(info_span!("tenant_delete_handler",
    1252              :             tenant_id = %tenant_shard_id.tenant_id,
    1253            0 :             shard_id = %tenant_shard_id.shard_slug()
    1254              :         ))
    1255            0 :         .await?;
    1256              : 
    1257            0 :     json_response(StatusCode::OK, ())
    1258            0 : }
    1259              : 
    1260              : /// HTTP endpoint to query the current tenant_size of a tenant.
    1261              : ///
    1262              : /// This is not used by consumption metrics under [`crate::consumption_metrics`], but can be used
    1263              : /// to debug any of the calculations. Requires `tenant_id` request parameter, supports
    1264              : /// `inputs_only=true|false` (default false) which supports debugging failure to calculate model
    1265              : /// values.
    1266              : ///
    1267              : /// 'retention_period' query parameter overrides the cutoff that is used to calculate the size
    1268              : /// (only if it is shorter than the real cutoff).
    1269              : ///
    1270              : /// Note: we don't update the cached size and prometheus metric here.
    1271              : /// The retention period might be different, and it's nice to have a method to just calculate it
    1272              : /// without modifying anything anyway.
    1273            0 : async fn tenant_size_handler(
    1274            0 :     request: Request<Body>,
    1275            0 :     cancel: CancellationToken,
    1276            0 : ) -> Result<Response<Body>, ApiError> {
    1277            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    1278            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    1279            0 :     let inputs_only: Option<bool> = parse_query_param(&request, "inputs_only")?;
    1280            0 :     let retention_period: Option<u64> = parse_query_param(&request, "retention_period")?;
    1281            0 :     let headers = request.headers();
    1282            0 :     let state = get_state(&request);
    1283            0 : 
    1284            0 :     if !tenant_shard_id.is_shard_zero() {
    1285            0 :         return Err(ApiError::BadRequest(anyhow!(
    1286            0 :             "Size calculations are only available on shard zero"
    1287            0 :         )));
    1288            0 :     }
    1289            0 : 
    1290            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download);
    1291            0 :     let tenant = state
    1292            0 :         .tenant_manager
    1293            0 :         .get_attached_tenant_shard(tenant_shard_id)?;
    1294            0 :     tenant.wait_to_become_active(ACTIVE_TENANT_TIMEOUT).await?;
    1295              : 
    1296              :     // this can be long operation
    1297            0 :     let inputs = tenant
    1298            0 :         .gather_size_inputs(
    1299            0 :             retention_period,
    1300            0 :             LogicalSizeCalculationCause::TenantSizeHandler,
    1301            0 :             &cancel,
    1302            0 :             &ctx,
    1303            0 :         )
    1304            0 :         .await
    1305            0 :         .map_err(|e| match e {
    1306            0 :             crate::tenant::size::CalculateSyntheticSizeError::Cancelled => ApiError::ShuttingDown,
    1307            0 :             other => ApiError::InternalServerError(anyhow::anyhow!(other)),
    1308            0 :         })?;
    1309              : 
    1310            0 :     let mut sizes = None;
    1311            0 :     let accepts_html = headers
    1312            0 :         .get(header::ACCEPT)
    1313            0 :         .map(|v| v == "text/html")
    1314            0 :         .unwrap_or_default();
    1315            0 :     if !inputs_only.unwrap_or(false) {
    1316            0 :         let storage_model = inputs.calculate_model();
    1317            0 :         let size = storage_model.calculate();
    1318            0 : 
    1319            0 :         // If request header expects html, return html
    1320            0 :         if accepts_html {
    1321            0 :             return synthetic_size_html_response(inputs, storage_model, size);
    1322            0 :         }
    1323            0 :         sizes = Some(size);
    1324            0 :     } else if accepts_html {
    1325            0 :         return Err(ApiError::BadRequest(anyhow!(
    1326            0 :             "inputs_only parameter is incompatible with html output request"
    1327            0 :         )));
    1328            0 :     }
    1329              : 
    1330              :     /// The type resides in the pageserver not to expose `ModelInputs`.
    1331              :     #[derive(serde::Serialize)]
    1332              :     struct TenantHistorySize {
    1333              :         id: TenantId,
    1334              :         /// Size is a mixture of WAL and logical size, so the unit is bytes.
    1335              :         ///
    1336              :         /// Will be none if `?inputs_only=true` was given.
    1337              :         size: Option<u64>,
    1338              :         /// Size of each segment used in the model.
    1339              :         /// Will be null if `?inputs_only=true` was given.
    1340              :         segment_sizes: Option<Vec<tenant_size_model::SegmentSizeResult>>,
    1341              :         inputs: crate::tenant::size::ModelInputs,
    1342              :     }
    1343              : 
    1344            0 :     json_response(
    1345            0 :         StatusCode::OK,
    1346            0 :         TenantHistorySize {
    1347            0 :             id: tenant_shard_id.tenant_id,
    1348            0 :             size: sizes.as_ref().map(|x| x.total_size),
    1349            0 :             segment_sizes: sizes.map(|x| x.segments),
    1350            0 :             inputs,
    1351            0 :         },
    1352            0 :     )
    1353            0 : }
    1354              : 
    1355            0 : async fn tenant_shard_split_handler(
    1356            0 :     mut request: Request<Body>,
    1357            0 :     _cancel: CancellationToken,
    1358            0 : ) -> Result<Response<Body>, ApiError> {
    1359            0 :     let req: TenantShardSplitRequest = json_request(&mut request).await?;
    1360              : 
    1361            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    1362            0 :     let state = get_state(&request);
    1363            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Warn);
    1364              : 
    1365            0 :     let tenant = state
    1366            0 :         .tenant_manager
    1367            0 :         .get_attached_tenant_shard(tenant_shard_id)?;
    1368            0 :     tenant.wait_to_become_active(ACTIVE_TENANT_TIMEOUT).await?;
    1369              : 
    1370            0 :     let new_shards = state
    1371            0 :         .tenant_manager
    1372            0 :         .shard_split(
    1373            0 :             tenant,
    1374            0 :             ShardCount::new(req.new_shard_count),
    1375            0 :             req.new_stripe_size,
    1376            0 :             &ctx,
    1377            0 :         )
    1378            0 :         .await
    1379            0 :         .map_err(ApiError::InternalServerError)?;
    1380              : 
    1381            0 :     json_response(StatusCode::OK, TenantShardSplitResponse { new_shards })
    1382            0 : }
    1383              : 
    1384            0 : async fn layer_map_info_handler(
    1385            0 :     request: Request<Body>,
    1386            0 :     _cancel: CancellationToken,
    1387            0 : ) -> Result<Response<Body>, ApiError> {
    1388            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    1389            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    1390            0 :     let reset: LayerAccessStatsReset =
    1391            0 :         parse_query_param(&request, "reset")?.unwrap_or(LayerAccessStatsReset::NoReset);
    1392            0 :     let state = get_state(&request);
    1393            0 : 
    1394            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    1395              : 
    1396            0 :     let timeline =
    1397            0 :         active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
    1398            0 :             .await?;
    1399            0 :     let layer_map_info = timeline
    1400            0 :         .layer_map_info(reset)
    1401            0 :         .await
    1402            0 :         .map_err(|_shutdown| ApiError::ShuttingDown)?;
    1403              : 
    1404            0 :     json_response(StatusCode::OK, layer_map_info)
    1405            0 : }
    1406              : 
    1407              : #[instrument(skip_all, fields(tenant_id, shard_id, timeline_id, layer_name))]
    1408              : async fn timeline_layer_scan_disposable_keys(
    1409              :     request: Request<Body>,
    1410              :     cancel: CancellationToken,
    1411              : ) -> Result<Response<Body>, ApiError> {
    1412              :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    1413              :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    1414              :     let layer_name: LayerName = parse_request_param(&request, "layer_name")?;
    1415              : 
    1416              :     tracing::Span::current().record(
    1417              :         "tenant_id",
    1418              :         tracing::field::display(&tenant_shard_id.tenant_id),
    1419              :     );
    1420              :     tracing::Span::current().record(
    1421              :         "shard_id",
    1422              :         tracing::field::display(tenant_shard_id.shard_slug()),
    1423              :     );
    1424              :     tracing::Span::current().record("timeline_id", tracing::field::display(&timeline_id));
    1425              :     tracing::Span::current().record("layer_name", tracing::field::display(&layer_name));
    1426              : 
    1427              :     let state = get_state(&request);
    1428              : 
    1429              :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    1430              : 
    1431              :     // technically the timeline need not be active for this scan to complete
    1432              :     let timeline =
    1433              :         active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
    1434              :             .await?;
    1435              : 
    1436              :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download)
    1437              :         .with_scope_timeline(&timeline);
    1438              : 
    1439              :     let guard = timeline.layers.read().await;
    1440              :     let Some(layer) = guard.try_get_from_key(&layer_name.clone().into()) else {
    1441              :         return Err(ApiError::NotFound(
    1442              :             anyhow::anyhow!("Layer {tenant_shard_id}/{timeline_id}/{layer_name} not found").into(),
    1443              :         ));
    1444              :     };
    1445              : 
    1446              :     let resident_layer = layer
    1447              :         .download_and_keep_resident(&ctx)
    1448              :         .await
    1449            0 :         .map_err(|err| match err {
    1450              :             tenant::storage_layer::layer::DownloadError::TimelineShutdown
    1451              :             | tenant::storage_layer::layer::DownloadError::DownloadCancelled => {
    1452            0 :                 ApiError::ShuttingDown
    1453              :             }
    1454              :             tenant::storage_layer::layer::DownloadError::ContextAndConfigReallyDeniesDownloads
    1455              :             | tenant::storage_layer::layer::DownloadError::DownloadRequired
    1456              :             | tenant::storage_layer::layer::DownloadError::NotFile(_)
    1457              :             | tenant::storage_layer::layer::DownloadError::DownloadFailed
    1458              :             | tenant::storage_layer::layer::DownloadError::PreStatFailed(_) => {
    1459            0 :                 ApiError::InternalServerError(err.into())
    1460              :             }
    1461              :             #[cfg(test)]
    1462              :             tenant::storage_layer::layer::DownloadError::Failpoint(_) => {
    1463            0 :                 ApiError::InternalServerError(err.into())
    1464              :             }
    1465            0 :         })?;
    1466              : 
    1467              :     let keys = resident_layer
    1468              :         .load_keys(&ctx)
    1469              :         .await
    1470              :         .map_err(ApiError::InternalServerError)?;
    1471              : 
    1472              :     let shard_identity = timeline.get_shard_identity();
    1473              : 
    1474              :     let mut disposable_count = 0;
    1475              :     let mut not_disposable_count = 0;
    1476              :     let cancel = cancel.clone();
    1477              :     for (i, key) in keys.into_iter().enumerate() {
    1478              :         if shard_identity.is_key_disposable(&key) {
    1479              :             disposable_count += 1;
    1480              :             tracing::debug!(key = %key, key.dbg=?key, "disposable key");
    1481              :         } else {
    1482              :             not_disposable_count += 1;
    1483              :         }
    1484              :         #[allow(clippy::collapsible_if)]
    1485              :         if i % 10000 == 0 {
    1486              :             if cancel.is_cancelled() || timeline.cancel.is_cancelled() || timeline.is_stopping() {
    1487              :                 return Err(ApiError::ShuttingDown);
    1488              :             }
    1489              :         }
    1490              :     }
    1491              : 
    1492              :     json_response(
    1493              :         StatusCode::OK,
    1494              :         pageserver_api::models::ScanDisposableKeysResponse {
    1495              :             disposable_count,
    1496              :             not_disposable_count,
    1497              :         },
    1498              :     )
    1499              : }
    1500              : 
    1501            0 : async fn timeline_download_heatmap_layers_handler(
    1502            0 :     request: Request<Body>,
    1503            0 :     _cancel: CancellationToken,
    1504            0 : ) -> Result<Response<Body>, ApiError> {
    1505              :     // Only used in the case where remote storage is not configured.
    1506              :     const DEFAULT_MAX_CONCURRENCY: usize = 100;
    1507              :     // A conservative default.
    1508              :     const DEFAULT_CONCURRENCY: usize = 16;
    1509              : 
    1510            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    1511            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    1512              : 
    1513            0 :     let desired_concurrency =
    1514            0 :         parse_query_param(&request, "concurrency")?.unwrap_or(DEFAULT_CONCURRENCY);
    1515            0 :     let recurse = parse_query_param(&request, "recurse")?.unwrap_or(false);
    1516            0 : 
    1517            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    1518              : 
    1519            0 :     let state = get_state(&request);
    1520            0 :     let timeline =
    1521            0 :         active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
    1522            0 :             .await?;
    1523            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download)
    1524            0 :         .with_scope_timeline(&timeline);
    1525            0 : 
    1526            0 :     let max_concurrency = get_config(&request)
    1527            0 :         .remote_storage_config
    1528            0 :         .as_ref()
    1529            0 :         .map(|c| c.concurrency_limit())
    1530            0 :         .unwrap_or(DEFAULT_MAX_CONCURRENCY);
    1531            0 :     let concurrency = std::cmp::min(max_concurrency, desired_concurrency);
    1532            0 : 
    1533            0 :     timeline.start_heatmap_layers_download(concurrency, recurse, &ctx)?;
    1534              : 
    1535            0 :     json_response(StatusCode::ACCEPTED, ())
    1536            0 : }
    1537              : 
    1538            0 : async fn timeline_shutdown_download_heatmap_layers_handler(
    1539            0 :     request: Request<Body>,
    1540            0 :     _cancel: CancellationToken,
    1541            0 : ) -> Result<Response<Body>, ApiError> {
    1542            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    1543            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    1544              : 
    1545            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    1546              : 
    1547            0 :     let state = get_state(&request);
    1548            0 :     let timeline =
    1549            0 :         active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
    1550            0 :             .await?;
    1551              : 
    1552            0 :     timeline.stop_and_drain_heatmap_layers_download().await;
    1553              : 
    1554            0 :     json_response(StatusCode::OK, ())
    1555            0 : }
    1556              : 
    1557            0 : async fn layer_download_handler(
    1558            0 :     request: Request<Body>,
    1559            0 :     _cancel: CancellationToken,
    1560            0 : ) -> Result<Response<Body>, ApiError> {
    1561            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    1562            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    1563            0 :     let layer_file_name = get_request_param(&request, "layer_file_name")?;
    1564            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    1565            0 :     let layer_name = LayerName::from_str(layer_file_name)
    1566            0 :         .map_err(|s| ApiError::BadRequest(anyhow::anyhow!(s)))?;
    1567            0 :     let state = get_state(&request);
    1568              : 
    1569            0 :     let timeline =
    1570            0 :         active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
    1571            0 :             .await?;
    1572            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download)
    1573            0 :         .with_scope_timeline(&timeline);
    1574            0 :     let downloaded = timeline
    1575            0 :         .download_layer(&layer_name, &ctx)
    1576            0 :         .await
    1577            0 :         .map_err(|e| match e {
    1578              :             tenant::storage_layer::layer::DownloadError::TimelineShutdown
    1579              :             | tenant::storage_layer::layer::DownloadError::DownloadCancelled => {
    1580            0 :                 ApiError::ShuttingDown
    1581              :             }
    1582            0 :             other => ApiError::InternalServerError(other.into()),
    1583            0 :         })?;
    1584              : 
    1585            0 :     match downloaded {
    1586            0 :         Some(true) => json_response(StatusCode::OK, ()),
    1587            0 :         Some(false) => json_response(StatusCode::NOT_MODIFIED, ()),
    1588            0 :         None => json_response(
    1589            0 :             StatusCode::BAD_REQUEST,
    1590            0 :             format!("Layer {tenant_shard_id}/{timeline_id}/{layer_file_name} not found"),
    1591            0 :         ),
    1592              :     }
    1593            0 : }
    1594              : 
    1595            0 : async fn evict_timeline_layer_handler(
    1596            0 :     request: Request<Body>,
    1597            0 :     _cancel: CancellationToken,
    1598            0 : ) -> Result<Response<Body>, ApiError> {
    1599            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    1600            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    1601            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    1602            0 :     let layer_file_name = get_request_param(&request, "layer_file_name")?;
    1603            0 :     let state = get_state(&request);
    1604              : 
    1605            0 :     let layer_name = LayerName::from_str(layer_file_name)
    1606            0 :         .map_err(|s| ApiError::BadRequest(anyhow::anyhow!(s)))?;
    1607              : 
    1608            0 :     let timeline =
    1609            0 :         active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
    1610            0 :             .await?;
    1611            0 :     let evicted = timeline
    1612            0 :         .evict_layer(&layer_name)
    1613            0 :         .await
    1614            0 :         .map_err(ApiError::InternalServerError)?;
    1615              : 
    1616            0 :     match evicted {
    1617            0 :         Some(true) => json_response(StatusCode::OK, ()),
    1618            0 :         Some(false) => json_response(StatusCode::NOT_MODIFIED, ()),
    1619            0 :         None => json_response(
    1620            0 :             StatusCode::BAD_REQUEST,
    1621            0 :             format!("Layer {tenant_shard_id}/{timeline_id}/{layer_file_name} not found"),
    1622            0 :         ),
    1623              :     }
    1624            0 : }
    1625              : 
    1626            0 : async fn timeline_gc_blocking_handler(
    1627            0 :     request: Request<Body>,
    1628            0 :     _cancel: CancellationToken,
    1629            0 : ) -> Result<Response<Body>, ApiError> {
    1630            0 :     block_or_unblock_gc(request, true).await
    1631            0 : }
    1632              : 
    1633            0 : async fn timeline_gc_unblocking_handler(
    1634            0 :     request: Request<Body>,
    1635            0 :     _cancel: CancellationToken,
    1636            0 : ) -> Result<Response<Body>, ApiError> {
    1637            0 :     block_or_unblock_gc(request, false).await
    1638            0 : }
    1639              : 
    1640              : /// Traces GetPage@LSN requests for a timeline, and emits metadata in an efficient binary encoding.
    1641              : /// Use the `pagectl page-trace` command to decode and analyze the output.
    1642            0 : async fn timeline_page_trace_handler(
    1643            0 :     request: Request<Body>,
    1644            0 :     cancel: CancellationToken,
    1645            0 : ) -> Result<Response<Body>, ApiError> {
    1646            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    1647            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    1648            0 :     let state = get_state(&request);
    1649            0 :     check_permission(&request, None)?;
    1650              : 
    1651            0 :     let size_limit: usize = parse_query_param(&request, "size_limit_bytes")?.unwrap_or(1024 * 1024);
    1652            0 :     let time_limit_secs: u64 = parse_query_param(&request, "time_limit_secs")?.unwrap_or(5);
    1653              : 
    1654              :     // Convert size limit to event limit based on the serialized size of an event. The event size is
    1655              :     // fixed, as the default bincode serializer uses fixed-width integer encoding.
    1656            0 :     let event_size = bincode::serialize(&PageTraceEvent::default())
    1657            0 :         .map_err(|err| ApiError::InternalServerError(err.into()))?
    1658            0 :         .len();
    1659            0 :     let event_limit = size_limit / event_size;
    1660              : 
    1661            0 :     let timeline =
    1662            0 :         active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
    1663            0 :             .await?;
    1664              : 
    1665              :     // Install a page trace, unless one is already in progress. We just use a buffered channel,
    1666              :     // which may 2x the memory usage in the worst case, but it's still bounded.
    1667            0 :     let (trace_tx, mut trace_rx) = tokio::sync::mpsc::channel(event_limit);
    1668            0 :     let cur = timeline.page_trace.load();
    1669            0 :     let installed = cur.is_none()
    1670            0 :         && timeline
    1671            0 :             .page_trace
    1672            0 :             .compare_and_swap(cur, Some(Arc::new(trace_tx)))
    1673            0 :             .is_none();
    1674            0 :     if !installed {
    1675            0 :         return Err(ApiError::Conflict("page trace already active".to_string()));
    1676            0 :     }
    1677            0 :     defer!(timeline.page_trace.store(None)); // uninstall on return
    1678            0 : 
    1679            0 :     // Collect the trace and return it to the client. We could stream the response, but this is
    1680            0 :     // simple and fine.
    1681            0 :     let mut body = Vec::with_capacity(size_limit);
    1682            0 :     let deadline = Instant::now() + Duration::from_secs(time_limit_secs);
    1683              : 
    1684            0 :     while body.len() < size_limit {
    1685            0 :         tokio::select! {
    1686            0 :             event = trace_rx.recv() => {
    1687            0 :                 let Some(event) = event else {
    1688            0 :                     break; // shouldn't happen (sender doesn't close, unless timeline dropped)
    1689              :                 };
    1690            0 :                 bincode::serialize_into(&mut body, &event)
    1691            0 :                     .map_err(|err| ApiError::InternalServerError(err.into()))?;
    1692              :             }
    1693            0 :             _ = tokio::time::sleep_until(deadline) => break, // time limit reached
    1694            0 :             _ = cancel.cancelled() => return Err(ApiError::Cancelled),
    1695              :         }
    1696              :     }
    1697              : 
    1698            0 :     Ok(Response::builder()
    1699            0 :         .status(StatusCode::OK)
    1700            0 :         .header(header::CONTENT_TYPE, "application/octet-stream")
    1701            0 :         .body(hyper::Body::from(body))
    1702            0 :         .unwrap())
    1703            0 : }
    1704              : 
    1705              : /// Adding a block is `POST ../block_gc`, removing a block is `POST ../unblock_gc`.
    1706              : ///
    1707              : /// Both are technically unsafe because they might fire off index uploads, thus they are POST.
    1708            0 : async fn block_or_unblock_gc(
    1709            0 :     request: Request<Body>,
    1710            0 :     block: bool,
    1711            0 : ) -> Result<Response<Body>, ApiError> {
    1712              :     use crate::tenant::remote_timeline_client::WaitCompletionError;
    1713              :     use crate::tenant::upload_queue::NotInitialized;
    1714            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    1715            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    1716            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    1717            0 :     let state = get_state(&request);
    1718              : 
    1719            0 :     let tenant = state
    1720            0 :         .tenant_manager
    1721            0 :         .get_attached_tenant_shard(tenant_shard_id)?;
    1722              : 
    1723            0 :     tenant.wait_to_become_active(ACTIVE_TENANT_TIMEOUT).await?;
    1724              : 
    1725            0 :     let timeline = tenant.get_timeline(timeline_id, true)?;
    1726              : 
    1727            0 :     let fut = async {
    1728            0 :         if block {
    1729            0 :             timeline.block_gc(&tenant).await.map(|_| ())
    1730              :         } else {
    1731            0 :             timeline.unblock_gc(&tenant).await
    1732              :         }
    1733            0 :     };
    1734              : 
    1735            0 :     let span = tracing::info_span!(
    1736              :         "block_or_unblock_gc",
    1737              :         tenant_id = %tenant_shard_id.tenant_id,
    1738            0 :         shard_id = %tenant_shard_id.shard_slug(),
    1739              :         timeline_id = %timeline_id,
    1740              :         block = block,
    1741              :     );
    1742              : 
    1743            0 :     let res = fut.instrument(span).await;
    1744              : 
    1745            0 :     res.map_err(|e| {
    1746            0 :         if e.is::<NotInitialized>() || e.is::<WaitCompletionError>() {
    1747            0 :             ApiError::ShuttingDown
    1748              :         } else {
    1749            0 :             ApiError::InternalServerError(e)
    1750              :         }
    1751            0 :     })?;
    1752              : 
    1753            0 :     json_response(StatusCode::OK, ())
    1754            0 : }
    1755              : 
    1756              : /// Get tenant_size SVG graph along with the JSON data.
    1757            0 : fn synthetic_size_html_response(
    1758            0 :     inputs: ModelInputs,
    1759            0 :     storage_model: StorageModel,
    1760            0 :     sizes: SizeResult,
    1761            0 : ) -> Result<Response<Body>, ApiError> {
    1762            0 :     let mut timeline_ids: Vec<String> = Vec::new();
    1763            0 :     let mut timeline_map: HashMap<TimelineId, usize> = HashMap::new();
    1764            0 :     for (index, ti) in inputs.timeline_inputs.iter().enumerate() {
    1765            0 :         timeline_map.insert(ti.timeline_id, index);
    1766            0 :         timeline_ids.push(ti.timeline_id.to_string());
    1767            0 :     }
    1768            0 :     let seg_to_branch: Vec<(usize, SvgBranchKind)> = inputs
    1769            0 :         .segments
    1770            0 :         .iter()
    1771            0 :         .map(|seg| {
    1772            0 :             (
    1773            0 :                 *timeline_map.get(&seg.timeline_id).unwrap(),
    1774            0 :                 seg.kind.into(),
    1775            0 :             )
    1776            0 :         })
    1777            0 :         .collect();
    1778              : 
    1779            0 :     let svg =
    1780            0 :         tenant_size_model::svg::draw_svg(&storage_model, &timeline_ids, &seg_to_branch, &sizes)
    1781            0 :             .map_err(ApiError::InternalServerError)?;
    1782              : 
    1783            0 :     let mut response = String::new();
    1784              : 
    1785              :     use std::fmt::Write;
    1786            0 :     write!(response, "<html>\n<body>\n").unwrap();
    1787            0 :     write!(response, "<div>\n{svg}\n</div>").unwrap();
    1788            0 :     writeln!(response, "Project size: {}", sizes.total_size).unwrap();
    1789            0 :     writeln!(response, "<pre>").unwrap();
    1790            0 :     writeln!(
    1791            0 :         response,
    1792            0 :         "{}",
    1793            0 :         serde_json::to_string_pretty(&inputs).unwrap()
    1794            0 :     )
    1795            0 :     .unwrap();
    1796            0 :     writeln!(
    1797            0 :         response,
    1798            0 :         "{}",
    1799            0 :         serde_json::to_string_pretty(&sizes.segments).unwrap()
    1800            0 :     )
    1801            0 :     .unwrap();
    1802            0 :     writeln!(response, "</pre>").unwrap();
    1803            0 :     write!(response, "</body>\n</html>\n").unwrap();
    1804            0 : 
    1805            0 :     html_response(StatusCode::OK, response)
    1806            0 : }
    1807              : 
    1808            0 : pub fn html_response(status: StatusCode, data: String) -> Result<Response<Body>, ApiError> {
    1809            0 :     let response = Response::builder()
    1810            0 :         .status(status)
    1811            0 :         .header(header::CONTENT_TYPE, "text/html")
    1812            0 :         .body(Body::from(data.as_bytes().to_vec()))
    1813            0 :         .map_err(|e| ApiError::InternalServerError(e.into()))?;
    1814            0 :     Ok(response)
    1815            0 : }
    1816              : 
    1817            0 : async fn get_tenant_config_handler(
    1818            0 :     request: Request<Body>,
    1819            0 :     _cancel: CancellationToken,
    1820            0 : ) -> Result<Response<Body>, ApiError> {
    1821            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    1822            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    1823            0 :     let state = get_state(&request);
    1824              : 
    1825            0 :     let tenant = state
    1826            0 :         .tenant_manager
    1827            0 :         .get_attached_tenant_shard(tenant_shard_id)?;
    1828              : 
    1829            0 :     let response = HashMap::from([
    1830              :         (
    1831              :             "tenant_specific_overrides",
    1832            0 :             serde_json::to_value(tenant.tenant_specific_overrides())
    1833            0 :                 .context("serializing tenant specific overrides")
    1834            0 :                 .map_err(ApiError::InternalServerError)?,
    1835              :         ),
    1836              :         (
    1837            0 :             "effective_config",
    1838            0 :             serde_json::to_value(tenant.effective_config())
    1839            0 :                 .context("serializing effective config")
    1840            0 :                 .map_err(ApiError::InternalServerError)?,
    1841              :         ),
    1842              :     ]);
    1843              : 
    1844            0 :     json_response(StatusCode::OK, response)
    1845            0 : }
    1846              : 
    1847            0 : async fn update_tenant_config_handler(
    1848            0 :     mut request: Request<Body>,
    1849            0 :     _cancel: CancellationToken,
    1850            0 : ) -> Result<Response<Body>, ApiError> {
    1851            0 :     let request_data: TenantConfigRequest = json_request(&mut request).await?;
    1852            0 :     let tenant_id = request_data.tenant_id;
    1853            0 :     check_permission(&request, Some(tenant_id))?;
    1854              : 
    1855            0 :     let new_tenant_conf =
    1856            0 :         TenantConfOpt::try_from(&request_data.config).map_err(ApiError::BadRequest)?;
    1857              : 
    1858            0 :     let state = get_state(&request);
    1859            0 : 
    1860            0 :     let tenant_shard_id = TenantShardId::unsharded(tenant_id);
    1861              : 
    1862            0 :     let tenant = state
    1863            0 :         .tenant_manager
    1864            0 :         .get_attached_tenant_shard(tenant_shard_id)?;
    1865            0 :     tenant.wait_to_become_active(ACTIVE_TENANT_TIMEOUT).await?;
    1866              : 
    1867              :     // This is a legacy API that only operates on attached tenants: the preferred
    1868              :     // API to use is the location_config/ endpoint, which lets the caller provide
    1869              :     // the full LocationConf.
    1870            0 :     let location_conf = LocationConf::attached_single(
    1871            0 :         new_tenant_conf.clone(),
    1872            0 :         tenant.get_generation(),
    1873            0 :         &ShardParameters::default(),
    1874            0 :     );
    1875            0 : 
    1876            0 :     crate::tenant::Tenant::persist_tenant_config(state.conf, &tenant_shard_id, &location_conf)
    1877            0 :         .await
    1878            0 :         .map_err(|e| ApiError::InternalServerError(anyhow::anyhow!(e)))?;
    1879              : 
    1880            0 :     let _ = tenant
    1881            0 :         .update_tenant_config(|_crnt| Ok(new_tenant_conf.clone()))
    1882            0 :         .expect("Closure returns Ok()");
    1883            0 : 
    1884            0 :     json_response(StatusCode::OK, ())
    1885            0 : }
    1886              : 
    1887            0 : async fn patch_tenant_config_handler(
    1888            0 :     mut request: Request<Body>,
    1889            0 :     _cancel: CancellationToken,
    1890            0 : ) -> Result<Response<Body>, ApiError> {
    1891            0 :     let request_data: TenantConfigPatchRequest = json_request(&mut request).await?;
    1892            0 :     let tenant_id = request_data.tenant_id;
    1893            0 :     check_permission(&request, Some(tenant_id))?;
    1894              : 
    1895            0 :     let state = get_state(&request);
    1896            0 : 
    1897            0 :     let tenant_shard_id = TenantShardId::unsharded(tenant_id);
    1898              : 
    1899            0 :     let tenant = state
    1900            0 :         .tenant_manager
    1901            0 :         .get_attached_tenant_shard(tenant_shard_id)?;
    1902            0 :     tenant.wait_to_become_active(ACTIVE_TENANT_TIMEOUT).await?;
    1903              : 
    1904            0 :     let updated = tenant
    1905            0 :         .update_tenant_config(|crnt| crnt.apply_patch(request_data.config.clone()))
    1906            0 :         .map_err(ApiError::BadRequest)?;
    1907              : 
    1908              :     // This is a legacy API that only operates on attached tenants: the preferred
    1909              :     // API to use is the location_config/ endpoint, which lets the caller provide
    1910              :     // the full LocationConf.
    1911            0 :     let location_conf = LocationConf::attached_single(
    1912            0 :         updated,
    1913            0 :         tenant.get_generation(),
    1914            0 :         &ShardParameters::default(),
    1915            0 :     );
    1916            0 : 
    1917            0 :     crate::tenant::Tenant::persist_tenant_config(state.conf, &tenant_shard_id, &location_conf)
    1918            0 :         .await
    1919            0 :         .map_err(|e| ApiError::InternalServerError(anyhow::anyhow!(e)))?;
    1920              : 
    1921            0 :     json_response(StatusCode::OK, ())
    1922            0 : }
    1923              : 
    1924            0 : async fn put_tenant_location_config_handler(
    1925            0 :     mut request: Request<Body>,
    1926            0 :     _cancel: CancellationToken,
    1927            0 : ) -> Result<Response<Body>, ApiError> {
    1928            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    1929              : 
    1930            0 :     let request_data: TenantLocationConfigRequest = json_request(&mut request).await?;
    1931            0 :     let flush = parse_query_param(&request, "flush_ms")?.map(Duration::from_millis);
    1932            0 :     let lazy = parse_query_param(&request, "lazy")?.unwrap_or(false);
    1933            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    1934              : 
    1935            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Warn);
    1936            0 :     let state = get_state(&request);
    1937            0 :     let conf = state.conf;
    1938            0 : 
    1939            0 :     // The `Detached` state is special, it doesn't upsert a tenant, it removes
    1940            0 :     // its local disk content and drops it from memory.
    1941            0 :     if let LocationConfigMode::Detached = request_data.config.mode {
    1942            0 :         if let Err(e) = state
    1943            0 :             .tenant_manager
    1944            0 :             .detach_tenant(conf, tenant_shard_id, &state.deletion_queue_client)
    1945            0 :             .instrument(info_span!("tenant_detach",
    1946              :                 tenant_id = %tenant_shard_id.tenant_id,
    1947            0 :                 shard_id = %tenant_shard_id.shard_slug()
    1948              :             ))
    1949            0 :             .await
    1950              :         {
    1951            0 :             match e {
    1952            0 :                 TenantStateError::SlotError(TenantSlotError::NotFound(_)) => {
    1953            0 :                     // This API is idempotent: a NotFound on a detach is fine.
    1954            0 :                 }
    1955            0 :                 _ => return Err(e.into()),
    1956              :             }
    1957            0 :         }
    1958            0 :         return json_response(StatusCode::OK, ());
    1959            0 :     }
    1960              : 
    1961            0 :     let location_conf =
    1962            0 :         LocationConf::try_from(&request_data.config).map_err(ApiError::BadRequest)?;
    1963              : 
    1964              :     // lazy==true queues up for activation or jumps the queue like normal when a compute connects,
    1965              :     // similar to at startup ordering.
    1966            0 :     let spawn_mode = if lazy {
    1967            0 :         tenant::SpawnMode::Lazy
    1968              :     } else {
    1969            0 :         tenant::SpawnMode::Eager
    1970              :     };
    1971              : 
    1972            0 :     let tenant = state
    1973            0 :         .tenant_manager
    1974            0 :         .upsert_location(tenant_shard_id, location_conf, flush, spawn_mode, &ctx)
    1975            0 :         .await?;
    1976            0 :     let stripe_size = tenant.as_ref().map(|t| t.get_shard_stripe_size());
    1977            0 :     let attached = tenant.is_some();
    1978              : 
    1979            0 :     if let Some(_flush_ms) = flush {
    1980            0 :         match state
    1981            0 :             .secondary_controller
    1982            0 :             .upload_tenant(tenant_shard_id)
    1983            0 :             .await
    1984              :         {
    1985              :             Ok(()) => {
    1986            0 :                 tracing::info!("Uploaded heatmap during flush");
    1987              :             }
    1988            0 :             Err(e) => {
    1989            0 :                 tracing::warn!("Failed to flush heatmap: {e}");
    1990              :             }
    1991              :         }
    1992              :     } else {
    1993            0 :         tracing::info!("No flush requested when configuring");
    1994              :     }
    1995              : 
    1996              :     // This API returns a vector of pageservers where the tenant is attached: this is
    1997              :     // primarily for use in the sharding service.  For compatibilty, we also return this
    1998              :     // when called directly on a pageserver, but the payload is always zero or one shards.
    1999            0 :     let mut response = TenantLocationConfigResponse {
    2000            0 :         shards: Vec::new(),
    2001            0 :         stripe_size: None,
    2002            0 :     };
    2003            0 :     if attached {
    2004            0 :         response.shards.push(TenantShardLocation {
    2005            0 :             shard_id: tenant_shard_id,
    2006            0 :             node_id: state.conf.id,
    2007            0 :         });
    2008            0 :         if tenant_shard_id.shard_count.count() > 1 {
    2009              :             // Stripe size should be set if we are attached
    2010            0 :             debug_assert!(stripe_size.is_some());
    2011            0 :             response.stripe_size = stripe_size;
    2012            0 :         }
    2013            0 :     }
    2014              : 
    2015            0 :     json_response(StatusCode::OK, response)
    2016            0 : }
    2017              : 
    2018            0 : async fn list_location_config_handler(
    2019            0 :     request: Request<Body>,
    2020            0 :     _cancel: CancellationToken,
    2021            0 : ) -> Result<Response<Body>, ApiError> {
    2022            0 :     let state = get_state(&request);
    2023            0 :     let slots = state.tenant_manager.list();
    2024            0 :     let result = LocationConfigListResponse {
    2025            0 :         tenant_shards: slots
    2026            0 :             .into_iter()
    2027            0 :             .map(|(tenant_shard_id, slot)| {
    2028            0 :                 let v = match slot {
    2029            0 :                     TenantSlot::Attached(t) => Some(t.get_location_conf()),
    2030            0 :                     TenantSlot::Secondary(s) => Some(s.get_location_conf()),
    2031            0 :                     TenantSlot::InProgress(_) => None,
    2032              :                 };
    2033            0 :                 (tenant_shard_id, v)
    2034            0 :             })
    2035            0 :             .collect(),
    2036            0 :     };
    2037            0 :     json_response(StatusCode::OK, result)
    2038            0 : }
    2039              : 
    2040            0 : async fn get_location_config_handler(
    2041            0 :     request: Request<Body>,
    2042            0 :     _cancel: CancellationToken,
    2043            0 : ) -> Result<Response<Body>, ApiError> {
    2044            0 :     let state = get_state(&request);
    2045            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    2046            0 :     let slot = state.tenant_manager.get(tenant_shard_id);
    2047              : 
    2048            0 :     let Some(slot) = slot else {
    2049            0 :         return Err(ApiError::NotFound(
    2050            0 :             anyhow::anyhow!("Tenant shard not found").into(),
    2051            0 :         ));
    2052              :     };
    2053              : 
    2054            0 :     let result: Option<LocationConfig> = match slot {
    2055            0 :         TenantSlot::Attached(t) => Some(t.get_location_conf()),
    2056            0 :         TenantSlot::Secondary(s) => Some(s.get_location_conf()),
    2057            0 :         TenantSlot::InProgress(_) => None,
    2058              :     };
    2059              : 
    2060            0 :     json_response(StatusCode::OK, result)
    2061            0 : }
    2062              : 
    2063              : // Do a time travel recovery on the given tenant/tenant shard. Tenant needs to be detached
    2064              : // (from all pageservers) as it invalidates consistency assumptions.
    2065            0 : async fn tenant_time_travel_remote_storage_handler(
    2066            0 :     request: Request<Body>,
    2067            0 :     cancel: CancellationToken,
    2068            0 : ) -> Result<Response<Body>, ApiError> {
    2069            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    2070              : 
    2071            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    2072              : 
    2073            0 :     let timestamp_raw = must_get_query_param(&request, "travel_to")?;
    2074            0 :     let timestamp = humantime::parse_rfc3339(&timestamp_raw)
    2075            0 :         .with_context(|| format!("Invalid time for travel_to: {timestamp_raw:?}"))
    2076            0 :         .map_err(ApiError::BadRequest)?;
    2077              : 
    2078            0 :     let done_if_after_raw = must_get_query_param(&request, "done_if_after")?;
    2079            0 :     let done_if_after = humantime::parse_rfc3339(&done_if_after_raw)
    2080            0 :         .with_context(|| format!("Invalid time for done_if_after: {done_if_after_raw:?}"))
    2081            0 :         .map_err(ApiError::BadRequest)?;
    2082              : 
    2083              :     // This is just a sanity check to fend off naive wrong usages of the API:
    2084              :     // the tenant needs to be detached *everywhere*
    2085            0 :     let state = get_state(&request);
    2086            0 :     let we_manage_tenant = state.tenant_manager.manages_tenant_shard(tenant_shard_id);
    2087            0 :     if we_manage_tenant {
    2088            0 :         return Err(ApiError::BadRequest(anyhow!(
    2089            0 :             "Tenant {tenant_shard_id} is already attached at this pageserver"
    2090            0 :         )));
    2091            0 :     }
    2092            0 : 
    2093            0 :     if timestamp > done_if_after {
    2094            0 :         return Err(ApiError::BadRequest(anyhow!(
    2095            0 :             "The done_if_after timestamp comes before the timestamp to recover to"
    2096            0 :         )));
    2097            0 :     }
    2098            0 : 
    2099            0 :     tracing::info!(
    2100            0 :         "Issuing time travel request internally. timestamp={timestamp_raw}, done_if_after={done_if_after_raw}"
    2101              :     );
    2102              : 
    2103            0 :     remote_timeline_client::upload::time_travel_recover_tenant(
    2104            0 :         &state.remote_storage,
    2105            0 :         &tenant_shard_id,
    2106            0 :         timestamp,
    2107            0 :         done_if_after,
    2108            0 :         &cancel,
    2109            0 :     )
    2110            0 :     .await
    2111            0 :     .map_err(|e| match e {
    2112            0 :         TimeTravelError::BadInput(e) => {
    2113            0 :             warn!("bad input error: {e}");
    2114            0 :             ApiError::BadRequest(anyhow!("bad input error"))
    2115              :         }
    2116              :         TimeTravelError::Unimplemented => {
    2117            0 :             ApiError::BadRequest(anyhow!("unimplemented for the configured remote storage"))
    2118              :         }
    2119            0 :         TimeTravelError::Cancelled => ApiError::InternalServerError(anyhow!("cancelled")),
    2120              :         TimeTravelError::TooManyVersions => {
    2121            0 :             ApiError::InternalServerError(anyhow!("too many versions in remote storage"))
    2122              :         }
    2123            0 :         TimeTravelError::Other(e) => {
    2124            0 :             warn!("internal error: {e}");
    2125            0 :             ApiError::InternalServerError(anyhow!("internal error"))
    2126              :         }
    2127            0 :     })?;
    2128              : 
    2129            0 :     json_response(StatusCode::OK, ())
    2130            0 : }
    2131              : 
    2132              : /// Testing helper to transition a tenant to [`crate::tenant::TenantState::Broken`].
    2133            0 : async fn handle_tenant_break(
    2134            0 :     r: Request<Body>,
    2135            0 :     _cancel: CancellationToken,
    2136            0 : ) -> Result<Response<Body>, ApiError> {
    2137            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&r, "tenant_shard_id")?;
    2138              : 
    2139            0 :     let state = get_state(&r);
    2140            0 :     state
    2141            0 :         .tenant_manager
    2142            0 :         .get_attached_tenant_shard(tenant_shard_id)?
    2143            0 :         .set_broken("broken from test".to_owned())
    2144            0 :         .await;
    2145              : 
    2146            0 :     json_response(StatusCode::OK, ())
    2147            0 : }
    2148              : 
    2149              : // Obtains an lsn lease on the given timeline.
    2150            0 : async fn lsn_lease_handler(
    2151            0 :     mut request: Request<Body>,
    2152            0 :     _cancel: CancellationToken,
    2153            0 : ) -> Result<Response<Body>, ApiError> {
    2154            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    2155            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    2156            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    2157            0 :     let lsn = json_request::<LsnLeaseRequest>(&mut request).await?.lsn;
    2158              : 
    2159            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download);
    2160            0 : 
    2161            0 :     let state = get_state(&request);
    2162              : 
    2163            0 :     let timeline =
    2164            0 :         active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
    2165            0 :             .await?;
    2166              : 
    2167            0 :     let result = async {
    2168            0 :         timeline
    2169            0 :             .init_lsn_lease(lsn, timeline.get_lsn_lease_length(), &ctx)
    2170            0 :             .map_err(|e| {
    2171            0 :                 ApiError::InternalServerError(
    2172            0 :                     e.context(format!("invalid lsn lease request at {lsn}")),
    2173            0 :                 )
    2174            0 :             })
    2175            0 :     }
    2176            0 :     .instrument(info_span!("init_lsn_lease", tenant_id = %tenant_shard_id.tenant_id, shard_id = %tenant_shard_id.shard_slug(), %timeline_id))
    2177            0 :     .await?;
    2178              : 
    2179            0 :     json_response(StatusCode::OK, result)
    2180            0 : }
    2181              : 
    2182              : // Run GC immediately on given timeline.
    2183            0 : async fn timeline_gc_handler(
    2184            0 :     mut request: Request<Body>,
    2185            0 :     cancel: CancellationToken,
    2186            0 : ) -> Result<Response<Body>, ApiError> {
    2187            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    2188            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    2189            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    2190              : 
    2191            0 :     let gc_req: TimelineGcRequest = json_request(&mut request).await?;
    2192              : 
    2193            0 :     let state = get_state(&request);
    2194            0 : 
    2195            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download);
    2196            0 :     let gc_result = state
    2197            0 :         .tenant_manager
    2198            0 :         .immediate_gc(tenant_shard_id, timeline_id, gc_req, cancel, &ctx)
    2199            0 :         .await?;
    2200              : 
    2201            0 :     json_response(StatusCode::OK, gc_result)
    2202            0 : }
    2203              : 
    2204              : // Cancel scheduled compaction tasks
    2205            0 : async fn timeline_cancel_compact_handler(
    2206            0 :     request: Request<Body>,
    2207            0 :     _cancel: CancellationToken,
    2208            0 : ) -> Result<Response<Body>, ApiError> {
    2209            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    2210            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    2211            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    2212            0 :     let state = get_state(&request);
    2213            0 :     async {
    2214            0 :         let tenant = state
    2215            0 :             .tenant_manager
    2216            0 :             .get_attached_tenant_shard(tenant_shard_id)?;
    2217            0 :         tenant.cancel_scheduled_compaction(timeline_id);
    2218            0 :         json_response(StatusCode::OK, ())
    2219            0 :     }
    2220            0 :     .instrument(info_span!("timeline_cancel_compact", tenant_id = %tenant_shard_id.tenant_id, shard_id = %tenant_shard_id.shard_slug(), %timeline_id))
    2221            0 :     .await
    2222            0 : }
    2223              : 
    2224              : // Get compact info of a timeline
    2225            0 : async fn timeline_compact_info_handler(
    2226            0 :     request: Request<Body>,
    2227            0 :     _cancel: CancellationToken,
    2228            0 : ) -> Result<Response<Body>, ApiError> {
    2229            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    2230            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    2231            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    2232            0 :     let state = get_state(&request);
    2233            0 :     async {
    2234            0 :         let tenant = state
    2235            0 :             .tenant_manager
    2236            0 :             .get_attached_tenant_shard(tenant_shard_id)?;
    2237            0 :         let resp = tenant.get_scheduled_compaction_tasks(timeline_id);
    2238            0 :         json_response(StatusCode::OK, resp)
    2239            0 :     }
    2240            0 :     .instrument(info_span!("timeline_compact_info", tenant_id = %tenant_shard_id.tenant_id, shard_id = %tenant_shard_id.shard_slug(), %timeline_id))
    2241            0 :     .await
    2242            0 : }
    2243              : 
    2244              : // Run compaction immediately on given timeline.
    2245            0 : async fn timeline_compact_handler(
    2246            0 :     mut request: Request<Body>,
    2247            0 :     cancel: CancellationToken,
    2248            0 : ) -> Result<Response<Body>, ApiError> {
    2249            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    2250            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    2251            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    2252              : 
    2253            0 :     let compact_request = json_request_maybe::<Option<CompactRequest>>(&mut request).await?;
    2254              : 
    2255            0 :     let state = get_state(&request);
    2256            0 : 
    2257            0 :     let mut flags = EnumSet::empty();
    2258            0 :     flags |= CompactFlags::NoYield; // run compaction to completion
    2259            0 : 
    2260            0 :     if Some(true) == parse_query_param::<_, bool>(&request, "force_l0_compaction")? {
    2261            0 :         flags |= CompactFlags::ForceL0Compaction;
    2262            0 :     }
    2263            0 :     if Some(true) == parse_query_param::<_, bool>(&request, "force_repartition")? {
    2264            0 :         flags |= CompactFlags::ForceRepartition;
    2265            0 :     }
    2266            0 :     if Some(true) == parse_query_param::<_, bool>(&request, "force_image_layer_creation")? {
    2267            0 :         flags |= CompactFlags::ForceImageLayerCreation;
    2268            0 :     }
    2269            0 :     if Some(true) == parse_query_param::<_, bool>(&request, "enhanced_gc_bottom_most_compaction")? {
    2270            0 :         flags |= CompactFlags::EnhancedGcBottomMostCompaction;
    2271            0 :     }
    2272            0 :     if Some(true) == parse_query_param::<_, bool>(&request, "dry_run")? {
    2273            0 :         flags |= CompactFlags::DryRun;
    2274            0 :     }
    2275              : 
    2276            0 :     let wait_until_uploaded =
    2277            0 :         parse_query_param::<_, bool>(&request, "wait_until_uploaded")?.unwrap_or(false);
    2278              : 
    2279            0 :     let wait_until_scheduled_compaction_done =
    2280            0 :         parse_query_param::<_, bool>(&request, "wait_until_scheduled_compaction_done")?
    2281            0 :             .unwrap_or(false);
    2282            0 : 
    2283            0 :     let sub_compaction = compact_request
    2284            0 :         .as_ref()
    2285            0 :         .map(|r| r.sub_compaction)
    2286            0 :         .unwrap_or(false);
    2287            0 :     let sub_compaction_max_job_size_mb = compact_request
    2288            0 :         .as_ref()
    2289            0 :         .and_then(|r| r.sub_compaction_max_job_size_mb);
    2290            0 : 
    2291            0 :     let options = CompactOptions {
    2292            0 :         compact_key_range: compact_request
    2293            0 :             .as_ref()
    2294            0 :             .and_then(|r| r.compact_key_range.clone()),
    2295            0 :         compact_lsn_range: compact_request
    2296            0 :             .as_ref()
    2297            0 :             .and_then(|r| r.compact_lsn_range.clone()),
    2298            0 :         flags,
    2299            0 :         sub_compaction,
    2300            0 :         sub_compaction_max_job_size_mb,
    2301            0 :     };
    2302            0 : 
    2303            0 :     let scheduled = compact_request
    2304            0 :         .as_ref()
    2305            0 :         .map(|r| r.scheduled)
    2306            0 :         .unwrap_or(false);
    2307              : 
    2308            0 :     async {
    2309            0 :         let timeline = active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id).await?;
    2310            0 :         let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download).with_scope_timeline(&timeline);
    2311            0 :         if scheduled {
    2312            0 :             let tenant = state
    2313            0 :                 .tenant_manager
    2314            0 :                 .get_attached_tenant_shard(tenant_shard_id)?;
    2315            0 :             let rx = tenant.schedule_compaction(timeline_id, options).await.map_err(ApiError::InternalServerError)?;
    2316            0 :             if wait_until_scheduled_compaction_done {
    2317              :                 // It is possible that this will take a long time, dropping the HTTP request will not cancel the compaction.
    2318            0 :                 rx.await.ok();
    2319            0 :             }
    2320              :         } else {
    2321            0 :             timeline
    2322            0 :                 .compact_with_options(&cancel, options, &ctx)
    2323            0 :                 .await
    2324            0 :                 .map_err(|e| ApiError::InternalServerError(e.into()))?;
    2325            0 :             if wait_until_uploaded {
    2326            0 :                 timeline.remote_client.wait_completion().await
    2327              :                 // XXX map to correct ApiError for the cases where it's due to shutdown
    2328            0 :                 .context("wait completion").map_err(ApiError::InternalServerError)?;
    2329            0 :             }
    2330              :         }
    2331            0 :         json_response(StatusCode::OK, ())
    2332            0 :     }
    2333            0 :     .instrument(info_span!("manual_compaction", tenant_id = %tenant_shard_id.tenant_id, shard_id = %tenant_shard_id.shard_slug(), %timeline_id))
    2334            0 :     .await
    2335            0 : }
    2336              : 
    2337              : // Run offload immediately on given timeline.
    2338            0 : async fn timeline_offload_handler(
    2339            0 :     request: Request<Body>,
    2340            0 :     _cancel: CancellationToken,
    2341            0 : ) -> Result<Response<Body>, ApiError> {
    2342            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    2343            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    2344            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    2345              : 
    2346            0 :     let state = get_state(&request);
    2347              : 
    2348            0 :     async {
    2349            0 :         let tenant = state
    2350            0 :             .tenant_manager
    2351            0 :             .get_attached_tenant_shard(tenant_shard_id)?;
    2352              : 
    2353            0 :         if tenant.get_offloaded_timeline(timeline_id).is_ok() {
    2354            0 :             return json_response(StatusCode::OK, ());
    2355            0 :         }
    2356            0 :         let timeline =
    2357            0 :             active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
    2358            0 :                 .await?;
    2359              : 
    2360            0 :         if !tenant.timeline_has_no_attached_children(timeline_id) {
    2361            0 :             return Err(ApiError::PreconditionFailed(
    2362            0 :                 "timeline has attached children".into(),
    2363            0 :             ));
    2364            0 :         }
    2365            0 :         if let (false, reason) = timeline.can_offload() {
    2366            0 :             return Err(ApiError::PreconditionFailed(
    2367            0 :                 format!("Timeline::can_offload() check failed: {}", reason) .into(),
    2368            0 :             ));
    2369            0 :         }
    2370            0 :         offload_timeline(&tenant, &timeline)
    2371            0 :             .await
    2372            0 :             .map_err(|e| {
    2373            0 :                 match e {
    2374            0 :                     OffloadError::Cancelled => ApiError::ResourceUnavailable("Timeline shutting down".into()),
    2375            0 :                     _ => ApiError::InternalServerError(anyhow!(e))
    2376              :                 }
    2377            0 :             })?;
    2378              : 
    2379            0 :         json_response(StatusCode::OK, ())
    2380            0 :     }
    2381            0 :     .instrument(info_span!("manual_timeline_offload", tenant_id = %tenant_shard_id.tenant_id, shard_id = %tenant_shard_id.shard_slug(), %timeline_id))
    2382            0 :     .await
    2383            0 : }
    2384              : 
    2385              : // Run checkpoint immediately on given timeline.
    2386            0 : async fn timeline_checkpoint_handler(
    2387            0 :     request: Request<Body>,
    2388            0 :     cancel: CancellationToken,
    2389            0 : ) -> Result<Response<Body>, ApiError> {
    2390            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    2391            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    2392            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    2393              : 
    2394            0 :     let state = get_state(&request);
    2395            0 : 
    2396            0 :     let mut flags = EnumSet::empty();
    2397            0 :     if Some(true) == parse_query_param::<_, bool>(&request, "force_l0_compaction")? {
    2398            0 :         flags |= CompactFlags::ForceL0Compaction;
    2399            0 :     }
    2400            0 :     if Some(true) == parse_query_param::<_, bool>(&request, "force_repartition")? {
    2401            0 :         flags |= CompactFlags::ForceRepartition;
    2402            0 :     }
    2403            0 :     if Some(true) == parse_query_param::<_, bool>(&request, "force_image_layer_creation")? {
    2404            0 :         flags |= CompactFlags::ForceImageLayerCreation;
    2405            0 :     }
    2406              : 
    2407              :     // By default, checkpoints come with a compaction, but this may be optionally disabled by tests that just want to flush + upload.
    2408            0 :     let compact = parse_query_param::<_, bool>(&request, "compact")?.unwrap_or(true);
    2409              : 
    2410            0 :     let wait_until_flushed: bool =
    2411            0 :         parse_query_param(&request, "wait_until_flushed")?.unwrap_or(true);
    2412              : 
    2413            0 :     let wait_until_uploaded =
    2414            0 :         parse_query_param::<_, bool>(&request, "wait_until_uploaded")?.unwrap_or(false);
    2415              : 
    2416            0 :     async {
    2417            0 :         let timeline = active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id).await?;
    2418            0 :         let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download).with_scope_timeline(&timeline);
    2419            0 :         if wait_until_flushed {
    2420            0 :             timeline.freeze_and_flush().await
    2421              :         } else {
    2422            0 :             timeline.freeze().await.and(Ok(()))
    2423            0 :         }.map_err(|e| {
    2424            0 :                 match e {
    2425            0 :                     tenant::timeline::FlushLayerError::Cancelled => ApiError::ShuttingDown,
    2426            0 :                     other => ApiError::InternalServerError(other.into()),
    2427              : 
    2428              :                 }
    2429            0 :             })?;
    2430            0 :         if compact {
    2431            0 :             timeline
    2432            0 :                 .compact(&cancel, flags, &ctx)
    2433            0 :                 .await
    2434            0 :                 .map_err(|e|
    2435            0 :                     match e {
    2436            0 :                         CompactionError::ShuttingDown => ApiError::ShuttingDown,
    2437            0 :                         CompactionError::Offload(e) => ApiError::InternalServerError(anyhow::anyhow!(e)),
    2438            0 :                         CompactionError::CollectKeySpaceError(e) => ApiError::InternalServerError(anyhow::anyhow!(e)),
    2439            0 :                         CompactionError::Other(e) => ApiError::InternalServerError(e),
    2440            0 :                         CompactionError::AlreadyRunning(_) => ApiError::InternalServerError(anyhow::anyhow!(e)),
    2441            0 :                     }
    2442            0 :                 )?;
    2443            0 :         }
    2444              : 
    2445            0 :         if wait_until_uploaded {
    2446            0 :             tracing::info!("Waiting for uploads to complete...");
    2447            0 :             timeline.remote_client.wait_completion().await
    2448              :             // XXX map to correct ApiError for the cases where it's due to shutdown
    2449            0 :             .context("wait completion").map_err(ApiError::InternalServerError)?;
    2450            0 :             tracing::info!("Uploads completed up to {}", timeline.get_remote_consistent_lsn_projected().unwrap_or(Lsn(0)));
    2451            0 :         }
    2452              : 
    2453            0 :         json_response(StatusCode::OK, ())
    2454            0 :     }
    2455            0 :     .instrument(info_span!("manual_checkpoint", tenant_id = %tenant_shard_id.tenant_id, shard_id = %tenant_shard_id.shard_slug(), %timeline_id))
    2456            0 :     .await
    2457            0 : }
    2458              : 
    2459            0 : async fn timeline_download_remote_layers_handler_post(
    2460            0 :     mut request: Request<Body>,
    2461            0 :     _cancel: CancellationToken,
    2462            0 : ) -> Result<Response<Body>, ApiError> {
    2463            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    2464            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    2465            0 :     let body: DownloadRemoteLayersTaskSpawnRequest = json_request(&mut request).await?;
    2466            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    2467              : 
    2468            0 :     let state = get_state(&request);
    2469              : 
    2470            0 :     let timeline =
    2471            0 :         active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
    2472            0 :             .await?;
    2473            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download)
    2474            0 :         .with_scope_timeline(&timeline);
    2475            0 :     match timeline.spawn_download_all_remote_layers(body, &ctx).await {
    2476            0 :         Ok(st) => json_response(StatusCode::ACCEPTED, st),
    2477            0 :         Err(st) => json_response(StatusCode::CONFLICT, st),
    2478              :     }
    2479            0 : }
    2480              : 
    2481            0 : async fn timeline_download_remote_layers_handler_get(
    2482            0 :     request: Request<Body>,
    2483            0 :     _cancel: CancellationToken,
    2484            0 : ) -> Result<Response<Body>, ApiError> {
    2485            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    2486            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    2487            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    2488            0 :     let state = get_state(&request);
    2489              : 
    2490            0 :     let timeline =
    2491            0 :         active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
    2492            0 :             .await?;
    2493            0 :     let info = timeline
    2494            0 :         .get_download_all_remote_layers_task_info()
    2495            0 :         .context("task never started since last pageserver process start")
    2496            0 :         .map_err(|e| ApiError::NotFound(e.into()))?;
    2497            0 :     json_response(StatusCode::OK, info)
    2498            0 : }
    2499              : 
    2500            0 : async fn timeline_detach_ancestor_handler(
    2501            0 :     request: Request<Body>,
    2502            0 :     _cancel: CancellationToken,
    2503            0 : ) -> Result<Response<Body>, ApiError> {
    2504              :     use pageserver_api::models::detach_ancestor::AncestorDetached;
    2505              : 
    2506              :     use crate::tenant::timeline::detach_ancestor;
    2507              : 
    2508            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    2509            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    2510            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    2511              : 
    2512            0 :     let span = tracing::info_span!("detach_ancestor", tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug(), %timeline_id);
    2513              : 
    2514            0 :     async move {
    2515            0 :         let mut options = detach_ancestor::Options::default();
    2516              : 
    2517            0 :         let rewrite_concurrency =
    2518            0 :             parse_query_param::<_, std::num::NonZeroUsize>(&request, "rewrite_concurrency")?;
    2519            0 :         let copy_concurrency =
    2520            0 :             parse_query_param::<_, std::num::NonZeroUsize>(&request, "copy_concurrency")?;
    2521              : 
    2522            0 :         [
    2523            0 :             (&mut options.rewrite_concurrency, rewrite_concurrency),
    2524            0 :             (&mut options.copy_concurrency, copy_concurrency),
    2525            0 :         ]
    2526            0 :         .into_iter()
    2527            0 :         .filter_map(|(target, val)| val.map(|val| (target, val)))
    2528            0 :         .for_each(|(target, val)| *target = val);
    2529            0 : 
    2530            0 :         let state = get_state(&request);
    2531              : 
    2532            0 :         let tenant = state
    2533            0 :             .tenant_manager
    2534            0 :             .get_attached_tenant_shard(tenant_shard_id)?;
    2535              : 
    2536            0 :         tenant.wait_to_become_active(ACTIVE_TENANT_TIMEOUT).await?;
    2537              : 
    2538            0 :         let ctx = RequestContext::new(TaskKind::DetachAncestor, DownloadBehavior::Download);
    2539            0 :         let ctx = &ctx;
    2540              : 
    2541              :         // Flush the upload queues of all timelines before detaching ancestor. We do the same thing again
    2542              :         // during shutdown. This early upload ensures the pageserver does not need to upload too many
    2543              :         // things and creates downtime during timeline reloads.
    2544            0 :         for timeline in tenant.list_timelines() {
    2545            0 :             timeline
    2546            0 :                 .remote_client
    2547            0 :                 .wait_completion()
    2548            0 :                 .await
    2549            0 :                 .map_err(|e| {
    2550            0 :                     ApiError::PreconditionFailed(format!("cannot drain upload queue: {e}").into())
    2551            0 :                 })?;
    2552              :         }
    2553              : 
    2554            0 :         tracing::info!("all timeline upload queues are drained");
    2555              : 
    2556            0 :         let timeline = tenant.get_timeline(timeline_id, true)?;
    2557            0 :         let ctx = &ctx.with_scope_timeline(&timeline);
    2558              : 
    2559            0 :         let progress = timeline
    2560            0 :             .prepare_to_detach_from_ancestor(&tenant, options, ctx)
    2561            0 :             .await?;
    2562              : 
    2563              :         // uncomment to allow early as possible Tenant::drop
    2564              :         // drop(tenant);
    2565              : 
    2566            0 :         let resp = match progress {
    2567            0 :             detach_ancestor::Progress::Prepared(attempt, prepared) => {
    2568              :                 // it would be great to tag the guard on to the tenant activation future
    2569            0 :                 let reparented_timelines = state
    2570            0 :                     .tenant_manager
    2571            0 :                     .complete_detaching_timeline_ancestor(
    2572            0 :                         tenant_shard_id,
    2573            0 :                         timeline_id,
    2574            0 :                         prepared,
    2575            0 :                         attempt,
    2576            0 :                         ctx,
    2577            0 :                     )
    2578            0 :                     .await?;
    2579              : 
    2580            0 :                 AncestorDetached {
    2581            0 :                     reparented_timelines,
    2582            0 :                 }
    2583              :             }
    2584            0 :             detach_ancestor::Progress::Done(resp) => resp,
    2585              :         };
    2586              : 
    2587            0 :         json_response(StatusCode::OK, resp)
    2588            0 :     }
    2589            0 :     .instrument(span)
    2590            0 :     .await
    2591            0 : }
    2592              : 
    2593            0 : async fn deletion_queue_flush(
    2594            0 :     r: Request<Body>,
    2595            0 :     cancel: CancellationToken,
    2596            0 : ) -> Result<Response<Body>, ApiError> {
    2597            0 :     let state = get_state(&r);
    2598              : 
    2599            0 :     let execute = parse_query_param(&r, "execute")?.unwrap_or(false);
    2600            0 : 
    2601            0 :     let flush = async {
    2602            0 :         if execute {
    2603            0 :             state.deletion_queue_client.flush_execute().await
    2604              :         } else {
    2605            0 :             state.deletion_queue_client.flush().await
    2606              :         }
    2607            0 :     }
    2608              :     // DeletionQueueError's only case is shutting down.
    2609            0 :     .map_err(|_| ApiError::ShuttingDown);
    2610            0 : 
    2611            0 :     tokio::select! {
    2612            0 :         res = flush => {
    2613            0 :             res.map(|()| json_response(StatusCode::OK, ()))?
    2614              :         }
    2615            0 :         _ = cancel.cancelled() => {
    2616            0 :             Err(ApiError::ShuttingDown)
    2617              :         }
    2618              :     }
    2619            0 : }
    2620              : 
    2621            0 : async fn getpage_at_lsn_handler(
    2622            0 :     request: Request<Body>,
    2623            0 :     cancel: CancellationToken,
    2624            0 : ) -> Result<Response<Body>, ApiError> {
    2625            0 :     getpage_at_lsn_handler_inner(false, request, cancel).await
    2626            0 : }
    2627              : 
    2628            0 : async fn touchpage_at_lsn_handler(
    2629            0 :     request: Request<Body>,
    2630            0 :     cancel: CancellationToken,
    2631            0 : ) -> Result<Response<Body>, ApiError> {
    2632            0 :     getpage_at_lsn_handler_inner(true, request, cancel).await
    2633            0 : }
    2634              : 
    2635              : /// Try if `GetPage@Lsn` is successful, useful for manual debugging.
    2636            0 : async fn getpage_at_lsn_handler_inner(
    2637            0 :     touch: bool,
    2638            0 :     request: Request<Body>,
    2639            0 :     _cancel: CancellationToken,
    2640            0 : ) -> Result<Response<Body>, ApiError> {
    2641            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    2642            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    2643              :     // Require pageserver admin permission for this API instead of only tenant-level token.
    2644            0 :     check_permission(&request, None)?;
    2645            0 :     let state = get_state(&request);
    2646              : 
    2647              :     struct Key(pageserver_api::key::Key);
    2648              : 
    2649              :     impl std::str::FromStr for Key {
    2650              :         type Err = anyhow::Error;
    2651              : 
    2652            0 :         fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
    2653            0 :             pageserver_api::key::Key::from_hex(s).map(Key)
    2654            0 :         }
    2655              :     }
    2656              : 
    2657            0 :     let key: Key = parse_query_param(&request, "key")?
    2658            0 :         .ok_or_else(|| ApiError::BadRequest(anyhow!("missing 'key' query parameter")))?;
    2659            0 :     let lsn: Option<Lsn> = parse_query_param(&request, "lsn")?;
    2660              : 
    2661            0 :     async {
    2662            0 :         let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download);
    2663              :         // Enable read path debugging
    2664            0 :         let timeline = active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id).await?;
    2665            0 :         let ctx = RequestContextBuilder::extend(&ctx).read_path_debug(true)
    2666            0 :         .scope(context::Scope::new_timeline(&timeline)).build();
    2667            0 : 
    2668            0 :         // Use last_record_lsn if no lsn is provided
    2669            0 :         let lsn = lsn.unwrap_or_else(|| timeline.get_last_record_lsn());
    2670            0 :         let page = timeline.get(key.0, lsn, &ctx).await?;
    2671              : 
    2672            0 :         if touch {
    2673            0 :             json_response(StatusCode::OK, ())
    2674              :         } else {
    2675            0 :             Result::<_, ApiError>::Ok(
    2676            0 :                 Response::builder()
    2677            0 :                     .status(StatusCode::OK)
    2678            0 :                     .header(header::CONTENT_TYPE, "application/octet-stream")
    2679            0 :                     .body(hyper::Body::from(page))
    2680            0 :                     .unwrap(),
    2681            0 :             )
    2682              :         }
    2683            0 :     }
    2684            0 :     .instrument(info_span!("timeline_get", tenant_id = %tenant_shard_id.tenant_id, shard_id = %tenant_shard_id.shard_slug(), %timeline_id))
    2685            0 :     .await
    2686            0 : }
    2687              : 
    2688            0 : async fn timeline_collect_keyspace(
    2689            0 :     request: Request<Body>,
    2690            0 :     _cancel: CancellationToken,
    2691            0 : ) -> Result<Response<Body>, ApiError> {
    2692            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    2693            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    2694            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    2695            0 :     let state = get_state(&request);
    2696              : 
    2697            0 :     let at_lsn: Option<Lsn> = parse_query_param(&request, "at_lsn")?;
    2698              : 
    2699            0 :     async {
    2700            0 :         let timeline = active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id).await?;
    2701            0 :         let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download).with_scope_timeline(&timeline);
    2702            0 :         let at_lsn = at_lsn.unwrap_or_else(|| timeline.get_last_record_lsn());
    2703            0 :         let (dense_ks, sparse_ks) = timeline
    2704            0 :             .collect_keyspace(at_lsn, &ctx)
    2705            0 :             .await
    2706            0 :             .map_err(|e| ApiError::InternalServerError(e.into()))?;
    2707              : 
    2708              :         // This API is currently used by pagebench. Pagebench will iterate all keys within the keyspace.
    2709              :         // Therefore, we split dense/sparse keys in this API.
    2710            0 :         let res = pageserver_api::models::partitioning::Partitioning { keys: dense_ks, sparse_keys: sparse_ks, at_lsn };
    2711            0 : 
    2712            0 :         json_response(StatusCode::OK, res)
    2713            0 :     }
    2714            0 :     .instrument(info_span!("timeline_collect_keyspace", tenant_id = %tenant_shard_id.tenant_id, shard_id = %tenant_shard_id.shard_slug(), %timeline_id))
    2715            0 :     .await
    2716            0 : }
    2717              : 
    2718            0 : async fn active_timeline_of_active_tenant(
    2719            0 :     tenant_manager: &TenantManager,
    2720            0 :     tenant_shard_id: TenantShardId,
    2721            0 :     timeline_id: TimelineId,
    2722            0 : ) -> Result<Arc<Timeline>, ApiError> {
    2723            0 :     let tenant = tenant_manager.get_attached_tenant_shard(tenant_shard_id)?;
    2724              : 
    2725            0 :     tenant.wait_to_become_active(ACTIVE_TENANT_TIMEOUT).await?;
    2726              : 
    2727            0 :     Ok(tenant.get_timeline(timeline_id, true)?)
    2728            0 : }
    2729              : 
    2730            0 : async fn always_panic_handler(
    2731            0 :     req: Request<Body>,
    2732            0 :     _cancel: CancellationToken,
    2733            0 : ) -> Result<Response<Body>, ApiError> {
    2734            0 :     // Deliberately cause a panic to exercise the panic hook registered via std::panic::set_hook().
    2735            0 :     // For pageserver, the relevant panic hook is `tracing_panic_hook` , and the `sentry` crate's wrapper around it.
    2736            0 :     // Use catch_unwind to ensure that tokio nor hyper are distracted by our panic.
    2737            0 :     let query = req.uri().query();
    2738            0 :     let _ = std::panic::catch_unwind(|| {
    2739            0 :         panic!("unconditional panic for testing panic hook integration; request query: {query:?}")
    2740            0 :     });
    2741            0 :     json_response(StatusCode::NO_CONTENT, ())
    2742            0 : }
    2743              : 
    2744            0 : async fn disk_usage_eviction_run(
    2745            0 :     mut r: Request<Body>,
    2746            0 :     cancel: CancellationToken,
    2747            0 : ) -> Result<Response<Body>, ApiError> {
    2748            0 :     check_permission(&r, None)?;
    2749              : 
    2750            0 :     #[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize)]
    2751              :     struct Config {
    2752              :         /// How many bytes to evict before reporting that pressure is relieved.
    2753              :         evict_bytes: u64,
    2754              : 
    2755              :         #[serde(default)]
    2756              :         eviction_order: pageserver_api::config::EvictionOrder,
    2757              :     }
    2758              : 
    2759              :     #[derive(Debug, Clone, Copy, serde::Serialize)]
    2760              :     struct Usage {
    2761              :         // remains unchanged after instantiation of the struct
    2762              :         evict_bytes: u64,
    2763              :         // updated by `add_available_bytes`
    2764              :         freed_bytes: u64,
    2765              :     }
    2766              : 
    2767              :     impl crate::disk_usage_eviction_task::Usage for Usage {
    2768            0 :         fn has_pressure(&self) -> bool {
    2769            0 :             self.evict_bytes > self.freed_bytes
    2770            0 :         }
    2771              : 
    2772            0 :         fn add_available_bytes(&mut self, bytes: u64) {
    2773            0 :             self.freed_bytes += bytes;
    2774            0 :         }
    2775              :     }
    2776              : 
    2777            0 :     let config = json_request::<Config>(&mut r).await?;
    2778              : 
    2779            0 :     let usage = Usage {
    2780            0 :         evict_bytes: config.evict_bytes,
    2781            0 :         freed_bytes: 0,
    2782            0 :     };
    2783            0 : 
    2784            0 :     let state = get_state(&r);
    2785            0 :     let eviction_state = state.disk_usage_eviction_state.clone();
    2786              : 
    2787            0 :     let res = crate::disk_usage_eviction_task::disk_usage_eviction_task_iteration_impl(
    2788            0 :         &eviction_state,
    2789            0 :         &state.remote_storage,
    2790            0 :         usage,
    2791            0 :         &state.tenant_manager,
    2792            0 :         config.eviction_order.into(),
    2793            0 :         &cancel,
    2794            0 :     )
    2795            0 :     .await;
    2796              : 
    2797            0 :     info!(?res, "disk_usage_eviction_task_iteration_impl finished");
    2798              : 
    2799            0 :     let res = res.map_err(ApiError::InternalServerError)?;
    2800              : 
    2801            0 :     json_response(StatusCode::OK, res)
    2802            0 : }
    2803              : 
    2804            0 : async fn secondary_upload_handler(
    2805            0 :     request: Request<Body>,
    2806            0 :     _cancel: CancellationToken,
    2807            0 : ) -> Result<Response<Body>, ApiError> {
    2808            0 :     let state = get_state(&request);
    2809            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    2810            0 :     state
    2811            0 :         .secondary_controller
    2812            0 :         .upload_tenant(tenant_shard_id)
    2813            0 :         .await?;
    2814              : 
    2815            0 :     json_response(StatusCode::OK, ())
    2816            0 : }
    2817              : 
    2818            0 : async fn tenant_scan_remote_handler(
    2819            0 :     request: Request<Body>,
    2820            0 :     cancel: CancellationToken,
    2821            0 : ) -> Result<Response<Body>, ApiError> {
    2822            0 :     let state = get_state(&request);
    2823            0 :     let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
    2824              : 
    2825            0 :     let mut response = TenantScanRemoteStorageResponse::default();
    2826              : 
    2827            0 :     let (shards, _other_keys) =
    2828            0 :         list_remote_tenant_shards(&state.remote_storage, tenant_id, cancel.clone())
    2829            0 :             .await
    2830            0 :             .map_err(|e| ApiError::InternalServerError(anyhow::anyhow!(e)))?;
    2831              : 
    2832            0 :     for tenant_shard_id in shards {
    2833            0 :         let (timeline_ids, _other_keys) =
    2834            0 :             list_remote_timelines(&state.remote_storage, tenant_shard_id, cancel.clone())
    2835            0 :                 .await
    2836            0 :                 .map_err(|e| ApiError::InternalServerError(anyhow::anyhow!(e)))?;
    2837              : 
    2838            0 :         let mut generation = Generation::none();
    2839            0 :         for timeline_id in timeline_ids {
    2840            0 :             match download_index_part(
    2841            0 :                 &state.remote_storage,
    2842            0 :                 &tenant_shard_id,
    2843            0 :                 &timeline_id,
    2844            0 :                 Generation::MAX,
    2845            0 :                 &cancel,
    2846            0 :             )
    2847            0 :             .instrument(info_span!("download_index_part",
    2848              :                          tenant_id=%tenant_shard_id.tenant_id,
    2849            0 :                          shard_id=%tenant_shard_id.shard_slug(),
    2850              :                          %timeline_id))
    2851            0 :             .await
    2852              :             {
    2853            0 :                 Ok((index_part, index_generation, _index_mtime)) => {
    2854            0 :                     tracing::info!(
    2855            0 :                         "Found timeline {tenant_shard_id}/{timeline_id} metadata (gen {index_generation:?}, {} layers, {} consistent LSN)",
    2856            0 :                         index_part.layer_metadata.len(),
    2857            0 :                         index_part.metadata.disk_consistent_lsn()
    2858              :                     );
    2859            0 :                     generation = std::cmp::max(generation, index_generation);
    2860              :                 }
    2861              :                 Err(DownloadError::NotFound) => {
    2862              :                     // This is normal for tenants that were created with multiple shards: they have an unsharded path
    2863              :                     // containing the timeline's initdb tarball but no index.  Otherwise it is a bit strange.
    2864            0 :                     tracing::info!(
    2865            0 :                         "Timeline path {tenant_shard_id}/{timeline_id} exists in remote storage but has no index, skipping"
    2866              :                     );
    2867            0 :                     continue;
    2868              :                 }
    2869            0 :                 Err(e) => {
    2870            0 :                     return Err(ApiError::InternalServerError(anyhow::anyhow!(e)));
    2871              :                 }
    2872              :             };
    2873              :         }
    2874              : 
    2875            0 :         response.shards.push(TenantScanRemoteStorageShard {
    2876            0 :             tenant_shard_id,
    2877            0 :             generation: generation.into(),
    2878            0 :         });
    2879              :     }
    2880              : 
    2881            0 :     if response.shards.is_empty() {
    2882            0 :         return Err(ApiError::NotFound(
    2883            0 :             anyhow::anyhow!("No shards found for tenant ID {tenant_id}").into(),
    2884            0 :         ));
    2885            0 :     }
    2886            0 : 
    2887            0 :     json_response(StatusCode::OK, response)
    2888            0 : }
    2889              : 
    2890            0 : async fn secondary_download_handler(
    2891            0 :     request: Request<Body>,
    2892            0 :     _cancel: CancellationToken,
    2893            0 : ) -> Result<Response<Body>, ApiError> {
    2894            0 :     let state = get_state(&request);
    2895            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    2896            0 :     let wait = parse_query_param(&request, "wait_ms")?.map(Duration::from_millis);
    2897              : 
    2898              :     // We don't need this to issue the download request, but:
    2899              :     // - it enables us to cleanly return 404 if we get a request for an absent shard
    2900              :     // - we will use this to provide status feedback in the response
    2901            0 :     let Some(secondary_tenant) = state
    2902            0 :         .tenant_manager
    2903            0 :         .get_secondary_tenant_shard(tenant_shard_id)
    2904              :     else {
    2905            0 :         return Err(ApiError::NotFound(
    2906            0 :             anyhow::anyhow!("Shard {} not found", tenant_shard_id).into(),
    2907            0 :         ));
    2908              :     };
    2909              : 
    2910            0 :     let timeout = wait.unwrap_or(Duration::MAX);
    2911              : 
    2912            0 :     let result = tokio::time::timeout(
    2913            0 :         timeout,
    2914            0 :         state.secondary_controller.download_tenant(tenant_shard_id),
    2915            0 :     )
    2916            0 :     .await;
    2917              : 
    2918            0 :     let progress = secondary_tenant.progress.lock().unwrap().clone();
    2919              : 
    2920            0 :     let status = match result {
    2921              :         Ok(Ok(())) => {
    2922            0 :             if progress.layers_downloaded >= progress.layers_total {
    2923              :                 // Download job ran to completion
    2924            0 :                 StatusCode::OK
    2925              :             } else {
    2926              :                 // Download dropped out without errors because it ran out of time budget
    2927            0 :                 StatusCode::ACCEPTED
    2928              :             }
    2929              :         }
    2930              :         // Edge case: downloads aren't usually fallible: things like a missing heatmap are considered
    2931              :         // okay.  We could get an error here in the unlikely edge case that the tenant
    2932              :         // was detached between our check above and executing the download job.
    2933            0 :         Ok(Err(e)) => return Err(e.into()),
    2934              :         // A timeout is not an error: we have started the download, we're just not done
    2935              :         // yet.  The caller will get a response body indicating status.
    2936            0 :         Err(_) => StatusCode::ACCEPTED,
    2937              :     };
    2938              : 
    2939            0 :     json_response(status, progress)
    2940            0 : }
    2941              : 
    2942            0 : async fn wait_lsn_handler(
    2943            0 :     mut request: Request<Body>,
    2944            0 :     cancel: CancellationToken,
    2945            0 : ) -> Result<Response<Body>, ApiError> {
    2946            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    2947            0 :     let wait_lsn_request: TenantWaitLsnRequest = json_request(&mut request).await?;
    2948              : 
    2949            0 :     let state = get_state(&request);
    2950            0 :     let tenant = state
    2951            0 :         .tenant_manager
    2952            0 :         .get_attached_tenant_shard(tenant_shard_id)?;
    2953              : 
    2954            0 :     let mut wait_futures = Vec::default();
    2955            0 :     for timeline in tenant.list_timelines() {
    2956            0 :         let Some(lsn) = wait_lsn_request.timelines.get(&timeline.timeline_id) else {
    2957            0 :             continue;
    2958              :         };
    2959              : 
    2960            0 :         let fut = {
    2961            0 :             let timeline = timeline.clone();
    2962            0 :             let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Error);
    2963            0 :             async move {
    2964            0 :                 timeline
    2965            0 :                     .wait_lsn(
    2966            0 :                         *lsn,
    2967            0 :                         WaitLsnWaiter::HttpEndpoint,
    2968            0 :                         WaitLsnTimeout::Custom(wait_lsn_request.timeout),
    2969            0 :                         &ctx,
    2970            0 :                     )
    2971            0 :                     .await
    2972            0 :             }
    2973            0 :         };
    2974            0 :         wait_futures.push(fut);
    2975            0 :     }
    2976              : 
    2977            0 :     if wait_futures.is_empty() {
    2978            0 :         return json_response(StatusCode::NOT_FOUND, ());
    2979            0 :     }
    2980              : 
    2981            0 :     let all_done = tokio::select! {
    2982            0 :         results = join_all(wait_futures) => {
    2983            0 :             results.iter().all(|res| res.is_ok())
    2984              :         },
    2985            0 :         _ = cancel.cancelled() => {
    2986            0 :             return Err(ApiError::Cancelled);
    2987              :         }
    2988              :     };
    2989              : 
    2990            0 :     let status = if all_done {
    2991            0 :         StatusCode::OK
    2992              :     } else {
    2993            0 :         StatusCode::ACCEPTED
    2994              :     };
    2995              : 
    2996            0 :     json_response(status, ())
    2997            0 : }
    2998              : 
    2999            0 : async fn secondary_status_handler(
    3000            0 :     request: Request<Body>,
    3001            0 :     _cancel: CancellationToken,
    3002            0 : ) -> Result<Response<Body>, ApiError> {
    3003            0 :     let state = get_state(&request);
    3004            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    3005              : 
    3006            0 :     let Some(secondary_tenant) = state
    3007            0 :         .tenant_manager
    3008            0 :         .get_secondary_tenant_shard(tenant_shard_id)
    3009              :     else {
    3010            0 :         return Err(ApiError::NotFound(
    3011            0 :             anyhow::anyhow!("Shard {} not found", tenant_shard_id).into(),
    3012            0 :         ));
    3013              :     };
    3014              : 
    3015            0 :     let progress = secondary_tenant.progress.lock().unwrap().clone();
    3016            0 : 
    3017            0 :     json_response(StatusCode::OK, progress)
    3018            0 : }
    3019              : 
    3020            0 : async fn handler_404(_: Request<Body>) -> Result<Response<Body>, ApiError> {
    3021            0 :     json_response(
    3022            0 :         StatusCode::NOT_FOUND,
    3023            0 :         HttpErrorBody::from_msg("page not found".to_owned()),
    3024            0 :     )
    3025            0 : }
    3026              : 
    3027            0 : async fn post_tracing_event_handler(
    3028            0 :     mut r: Request<Body>,
    3029            0 :     _cancel: CancellationToken,
    3030            0 : ) -> Result<Response<Body>, ApiError> {
    3031            0 :     #[derive(Debug, serde::Deserialize)]
    3032              :     #[serde(rename_all = "lowercase")]
    3033              :     enum Level {
    3034              :         Error,
    3035              :         Warn,
    3036              :         Info,
    3037              :         Debug,
    3038              :         Trace,
    3039              :     }
    3040            0 :     #[derive(Debug, serde::Deserialize)]
    3041              :     struct Request {
    3042              :         level: Level,
    3043              :         message: String,
    3044              :     }
    3045            0 :     let body: Request = json_request(&mut r)
    3046            0 :         .await
    3047            0 :         .map_err(|_| ApiError::BadRequest(anyhow::anyhow!("invalid JSON body")))?;
    3048              : 
    3049            0 :     match body.level {
    3050            0 :         Level::Error => tracing::error!(?body.message),
    3051            0 :         Level::Warn => tracing::warn!(?body.message),
    3052            0 :         Level::Info => tracing::info!(?body.message),
    3053            0 :         Level::Debug => tracing::debug!(?body.message),
    3054            0 :         Level::Trace => tracing::trace!(?body.message),
    3055              :     }
    3056              : 
    3057            0 :     json_response(StatusCode::OK, ())
    3058            0 : }
    3059              : 
    3060            0 : async fn put_io_engine_handler(
    3061            0 :     mut r: Request<Body>,
    3062            0 :     _cancel: CancellationToken,
    3063            0 : ) -> Result<Response<Body>, ApiError> {
    3064            0 :     check_permission(&r, None)?;
    3065            0 :     let kind: crate::virtual_file::IoEngineKind = json_request(&mut r).await?;
    3066            0 :     crate::virtual_file::io_engine::set(kind);
    3067            0 :     json_response(StatusCode::OK, ())
    3068            0 : }
    3069              : 
    3070            0 : async fn put_io_mode_handler(
    3071            0 :     mut r: Request<Body>,
    3072            0 :     _cancel: CancellationToken,
    3073            0 : ) -> Result<Response<Body>, ApiError> {
    3074            0 :     check_permission(&r, None)?;
    3075            0 :     let mode: IoMode = json_request(&mut r).await?;
    3076            0 :     crate::virtual_file::set_io_mode(mode);
    3077            0 :     json_response(StatusCode::OK, ())
    3078            0 : }
    3079              : 
    3080              : /// Polled by control plane.
    3081              : ///
    3082              : /// See [`crate::utilization`].
    3083            0 : async fn get_utilization(
    3084            0 :     r: Request<Body>,
    3085            0 :     _cancel: CancellationToken,
    3086            0 : ) -> Result<Response<Body>, ApiError> {
    3087            0 :     fail::fail_point!("get-utilization-http-handler", |_| {
    3088            0 :         Err(ApiError::ResourceUnavailable("failpoint".into()))
    3089            0 :     });
    3090              : 
    3091              :     // this probably could be completely public, but lets make that change later.
    3092            0 :     check_permission(&r, None)?;
    3093              : 
    3094            0 :     let state = get_state(&r);
    3095            0 :     let mut g = state.latest_utilization.lock().await;
    3096              : 
    3097            0 :     let regenerate_every = Duration::from_secs(1);
    3098            0 :     let still_valid = g
    3099            0 :         .as_ref()
    3100            0 :         .is_some_and(|(captured_at, _)| captured_at.elapsed() < regenerate_every);
    3101            0 : 
    3102            0 :     // avoid needless statvfs calls even though those should be non-blocking fast.
    3103            0 :     // regenerate at most 1Hz to allow polling at any rate.
    3104            0 :     if !still_valid {
    3105            0 :         let path = state.conf.tenants_path();
    3106            0 :         let doc =
    3107            0 :             crate::utilization::regenerate(state.conf, path.as_std_path(), &state.tenant_manager)
    3108            0 :                 .map_err(ApiError::InternalServerError)?;
    3109              : 
    3110            0 :         let mut buf = Vec::new();
    3111            0 :         serde_json::to_writer(&mut buf, &doc)
    3112            0 :             .context("serialize")
    3113            0 :             .map_err(ApiError::InternalServerError)?;
    3114              : 
    3115            0 :         let body = bytes::Bytes::from(buf);
    3116            0 : 
    3117            0 :         *g = Some((std::time::Instant::now(), body));
    3118            0 :     }
    3119              : 
    3120              :     // hyper 0.14 doesn't yet have Response::clone so this is a bit of extra legwork
    3121            0 :     let cached = g.as_ref().expect("just set").1.clone();
    3122            0 : 
    3123            0 :     Response::builder()
    3124            0 :         .header(hyper::http::header::CONTENT_TYPE, "application/json")
    3125            0 :         // thought of using http date header, but that is second precision which does not give any
    3126            0 :         // debugging aid
    3127            0 :         .status(StatusCode::OK)
    3128            0 :         .body(hyper::Body::from(cached))
    3129            0 :         .context("build response")
    3130            0 :         .map_err(ApiError::InternalServerError)
    3131            0 : }
    3132              : 
    3133            0 : async fn list_aux_files(
    3134            0 :     mut request: Request<Body>,
    3135            0 :     _cancel: CancellationToken,
    3136            0 : ) -> Result<Response<Body>, ApiError> {
    3137            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    3138            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    3139            0 :     let body: ListAuxFilesRequest = json_request(&mut request).await?;
    3140            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    3141              : 
    3142            0 :     let state = get_state(&request);
    3143              : 
    3144            0 :     let timeline =
    3145            0 :         active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
    3146            0 :             .await?;
    3147              : 
    3148            0 :     let io_concurrency = IoConcurrency::spawn_from_conf(
    3149            0 :         state.conf,
    3150            0 :         timeline.gate.enter().map_err(|_| ApiError::Cancelled)?,
    3151              :     );
    3152              : 
    3153            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download);
    3154            0 :     let files = timeline
    3155            0 :         .list_aux_files(body.lsn, &ctx, io_concurrency)
    3156            0 :         .await?;
    3157            0 :     json_response(StatusCode::OK, files)
    3158            0 : }
    3159              : 
    3160            0 : async fn perf_info(
    3161            0 :     request: Request<Body>,
    3162            0 :     _cancel: CancellationToken,
    3163            0 : ) -> Result<Response<Body>, ApiError> {
    3164            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    3165            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    3166            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    3167              : 
    3168            0 :     let state = get_state(&request);
    3169              : 
    3170            0 :     let timeline =
    3171            0 :         active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
    3172            0 :             .await?;
    3173              : 
    3174            0 :     let result = timeline.perf_info().await;
    3175              : 
    3176            0 :     json_response(StatusCode::OK, result)
    3177            0 : }
    3178              : 
    3179            0 : async fn ingest_aux_files(
    3180            0 :     mut request: Request<Body>,
    3181            0 :     _cancel: CancellationToken,
    3182            0 : ) -> Result<Response<Body>, ApiError> {
    3183            0 :     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
    3184            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    3185            0 :     let body: IngestAuxFilesRequest = json_request(&mut request).await?;
    3186            0 :     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
    3187              : 
    3188            0 :     let state = get_state(&request);
    3189              : 
    3190            0 :     let timeline =
    3191            0 :         active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
    3192            0 :             .await?;
    3193              : 
    3194            0 :     let mut modification = timeline.begin_modification(
    3195            0 :         Lsn(timeline.get_last_record_lsn().0 + 8), /* advance LSN by 8 */
    3196            0 :     );
    3197            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download);
    3198            0 :     for (fname, content) in body.aux_files {
    3199            0 :         modification
    3200            0 :             .put_file(&fname, content.as_bytes(), &ctx)
    3201            0 :             .await
    3202            0 :             .map_err(ApiError::InternalServerError)?;
    3203              :     }
    3204            0 :     modification
    3205            0 :         .commit(&ctx)
    3206            0 :         .await
    3207            0 :         .map_err(ApiError::InternalServerError)?;
    3208              : 
    3209            0 :     json_response(StatusCode::OK, ())
    3210            0 : }
    3211              : 
    3212              : /// Report on the largest tenants on this pageserver, for the storage controller to identify
    3213              : /// candidates for splitting
    3214            0 : async fn post_top_tenants(
    3215            0 :     mut r: Request<Body>,
    3216            0 :     _cancel: CancellationToken,
    3217            0 : ) -> Result<Response<Body>, ApiError> {
    3218            0 :     check_permission(&r, None)?;
    3219            0 :     let request: TopTenantShardsRequest = json_request(&mut r).await?;
    3220            0 :     let state = get_state(&r);
    3221              : 
    3222            0 :     fn get_size_metric(sizes: &TopTenantShardItem, order_by: &TenantSorting) -> u64 {
    3223            0 :         match order_by {
    3224            0 :             TenantSorting::ResidentSize => sizes.resident_size,
    3225            0 :             TenantSorting::MaxLogicalSize => sizes.max_logical_size,
    3226            0 :             TenantSorting::MaxLogicalSizePerShard => sizes.max_logical_size_per_shard,
    3227              :         }
    3228            0 :     }
    3229              : 
    3230              :     #[derive(Eq, PartialEq)]
    3231              :     struct HeapItem {
    3232              :         metric: u64,
    3233              :         sizes: TopTenantShardItem,
    3234              :     }
    3235              : 
    3236              :     impl PartialOrd for HeapItem {
    3237            0 :         fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
    3238            0 :             Some(self.cmp(other))
    3239            0 :         }
    3240              :     }
    3241              : 
    3242              :     /// Heap items have reverse ordering on their metric: this enables using BinaryHeap, which
    3243              :     /// supports popping the greatest item but not the smallest.
    3244              :     impl Ord for HeapItem {
    3245            0 :         fn cmp(&self, other: &Self) -> std::cmp::Ordering {
    3246            0 :             Reverse(self.metric).cmp(&Reverse(other.metric))
    3247            0 :         }
    3248              :     }
    3249              : 
    3250            0 :     let mut top_n: BinaryHeap<HeapItem> = BinaryHeap::with_capacity(request.limit);
    3251              : 
    3252              :     // FIXME: this is a lot of clones to take this tenant list
    3253            0 :     for (tenant_shard_id, tenant_slot) in state.tenant_manager.list() {
    3254            0 :         if let Some(shards_lt) = request.where_shards_lt {
    3255              :             // Ignore tenants which already have >= this many shards
    3256            0 :             if tenant_shard_id.shard_count >= shards_lt {
    3257            0 :                 continue;
    3258            0 :             }
    3259            0 :         }
    3260              : 
    3261            0 :         let sizes = match tenant_slot {
    3262            0 :             TenantSlot::Attached(tenant) => tenant.get_sizes(),
    3263              :             TenantSlot::Secondary(_) | TenantSlot::InProgress(_) => {
    3264            0 :                 continue;
    3265              :             }
    3266              :         };
    3267            0 :         let metric = get_size_metric(&sizes, &request.order_by);
    3268              : 
    3269            0 :         if let Some(gt) = request.where_gt {
    3270              :             // Ignore tenants whose metric is <= the lower size threshold, to do less sorting work
    3271            0 :             if metric <= gt {
    3272            0 :                 continue;
    3273            0 :             }
    3274            0 :         };
    3275              : 
    3276            0 :         match top_n.peek() {
    3277            0 :             None => {
    3278            0 :                 // Top N list is empty: candidate becomes first member
    3279            0 :                 top_n.push(HeapItem { metric, sizes });
    3280            0 :             }
    3281            0 :             Some(i) if i.metric > metric && top_n.len() < request.limit => {
    3282            0 :                 // Lowest item in list is greater than our candidate, but we aren't at limit yet: push to end
    3283            0 :                 top_n.push(HeapItem { metric, sizes });
    3284            0 :             }
    3285            0 :             Some(i) if i.metric > metric => {
    3286            0 :                 // List is at limit and lowest value is greater than our candidate, drop it.
    3287            0 :             }
    3288            0 :             Some(_) => top_n.push(HeapItem { metric, sizes }),
    3289              :         }
    3290              : 
    3291            0 :         while top_n.len() > request.limit {
    3292            0 :             top_n.pop();
    3293            0 :         }
    3294              :     }
    3295              : 
    3296            0 :     json_response(
    3297            0 :         StatusCode::OK,
    3298            0 :         TopTenantShardsResponse {
    3299            0 :             shards: top_n.into_iter().map(|i| i.sizes).collect(),
    3300            0 :         },
    3301            0 :     )
    3302            0 : }
    3303              : 
    3304            0 : async fn put_tenant_timeline_import_basebackup(
    3305            0 :     request: Request<Body>,
    3306            0 :     _cancel: CancellationToken,
    3307            0 : ) -> Result<Response<Body>, ApiError> {
    3308            0 :     let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
    3309            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    3310            0 :     let base_lsn: Lsn = must_parse_query_param(&request, "base_lsn")?;
    3311            0 :     let end_lsn: Lsn = must_parse_query_param(&request, "end_lsn")?;
    3312            0 :     let pg_version: u32 = must_parse_query_param(&request, "pg_version")?;
    3313              : 
    3314            0 :     check_permission(&request, Some(tenant_id))?;
    3315              : 
    3316            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Warn);
    3317            0 : 
    3318            0 :     let tenant_shard_id = TenantShardId::unsharded(tenant_id);
    3319              : 
    3320            0 :     let span = info_span!("import_basebackup",
    3321            0 :         tenant_id=%tenant_id, timeline_id=%timeline_id, shard_id=%tenant_shard_id.shard_slug(),
    3322              :         base_lsn=%base_lsn, end_lsn=%end_lsn, pg_version=%pg_version);
    3323            0 :     async move {
    3324            0 :         let state = get_state(&request);
    3325            0 :         let tenant = state
    3326            0 :             .tenant_manager
    3327            0 :             .get_attached_tenant_shard(tenant_shard_id)?;
    3328              : 
    3329            0 :         let broker_client = state.broker_client.clone();
    3330            0 : 
    3331            0 :         let mut body = StreamReader::new(request.into_body().map(|res| {
    3332            0 :             res.map_err(|error| {
    3333            0 :                 std::io::Error::new(std::io::ErrorKind::Other, anyhow::anyhow!(error))
    3334            0 :             })
    3335            0 :         }));
    3336            0 : 
    3337            0 :         tenant.wait_to_become_active(ACTIVE_TENANT_TIMEOUT).await?;
    3338              : 
    3339            0 :         let (timeline, timeline_ctx) = tenant
    3340            0 :             .create_empty_timeline(timeline_id, base_lsn, pg_version, &ctx)
    3341            0 :             .map_err(ApiError::InternalServerError)
    3342            0 :             .await?;
    3343              : 
    3344              :         // TODO mark timeline as not ready until it reaches end_lsn.
    3345              :         // We might have some wal to import as well, and we should prevent compute
    3346              :         // from connecting before that and writing conflicting wal.
    3347              :         //
    3348              :         // This is not relevant for pageserver->pageserver migrations, since there's
    3349              :         // no wal to import. But should be fixed if we want to import from postgres.
    3350              : 
    3351              :         // TODO leave clean state on error. For now you can use detach to clean
    3352              :         // up broken state from a failed import.
    3353              : 
    3354              :         // Import basebackup provided via CopyData
    3355            0 :         info!("importing basebackup");
    3356              : 
    3357            0 :         timeline
    3358            0 :             .import_basebackup_from_tar(
    3359            0 :                 tenant.clone(),
    3360            0 :                 &mut body,
    3361            0 :                 base_lsn,
    3362            0 :                 broker_client,
    3363            0 :                 &timeline_ctx,
    3364            0 :             )
    3365            0 :             .await
    3366            0 :             .map_err(ApiError::InternalServerError)?;
    3367              : 
    3368              :         // Read the end of the tar archive.
    3369            0 :         read_tar_eof(body)
    3370            0 :             .await
    3371            0 :             .map_err(ApiError::InternalServerError)?;
    3372              : 
    3373              :         // TODO check checksum
    3374              :         // Meanwhile you can verify client-side by taking fullbackup
    3375              :         // and checking that it matches in size with what was imported.
    3376              :         // It wouldn't work if base came from vanilla postgres though,
    3377              :         // since we discard some log files.
    3378              : 
    3379            0 :         info!("done");
    3380            0 :         json_response(StatusCode::OK, ())
    3381            0 :     }
    3382            0 :     .instrument(span)
    3383            0 :     .await
    3384            0 : }
    3385              : 
    3386            0 : async fn put_tenant_timeline_import_wal(
    3387            0 :     request: Request<Body>,
    3388            0 :     _cancel: CancellationToken,
    3389            0 : ) -> Result<Response<Body>, ApiError> {
    3390            0 :     let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
    3391            0 :     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    3392            0 :     let start_lsn: Lsn = must_parse_query_param(&request, "start_lsn")?;
    3393            0 :     let end_lsn: Lsn = must_parse_query_param(&request, "end_lsn")?;
    3394              : 
    3395            0 :     check_permission(&request, Some(tenant_id))?;
    3396              : 
    3397            0 :     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Warn);
    3398              : 
    3399            0 :     let span = info_span!("import_wal", tenant_id=%tenant_id, timeline_id=%timeline_id, start_lsn=%start_lsn, end_lsn=%end_lsn);
    3400            0 :     async move {
    3401            0 :         let state = get_state(&request);
    3402              : 
    3403            0 :         let timeline = active_timeline_of_active_tenant(&state.tenant_manager, TenantShardId::unsharded(tenant_id), timeline_id).await?;
    3404            0 :         let ctx = RequestContextBuilder::extend(&ctx).scope(context::Scope::new_timeline(&timeline)).build();
    3405            0 : 
    3406            0 :         let mut body = StreamReader::new(request.into_body().map(|res| {
    3407            0 :             res.map_err(|error| {
    3408            0 :                 std::io::Error::new(std::io::ErrorKind::Other, anyhow::anyhow!(error))
    3409            0 :             })
    3410            0 :         }));
    3411            0 : 
    3412            0 :         let last_record_lsn = timeline.get_last_record_lsn();
    3413            0 :         if last_record_lsn != start_lsn {
    3414            0 :             return Err(ApiError::InternalServerError(anyhow::anyhow!("Cannot import WAL from Lsn {start_lsn} because timeline does not start from the same lsn: {last_record_lsn}")));
    3415            0 :         }
    3416            0 : 
    3417            0 :         // TODO leave clean state on error. For now you can use detach to clean
    3418            0 :         // up broken state from a failed import.
    3419            0 : 
    3420            0 :         // Import wal provided via CopyData
    3421            0 :         info!("importing wal");
    3422            0 :         crate::import_datadir::import_wal_from_tar(&timeline, &mut body, start_lsn, end_lsn, &ctx).await.map_err(ApiError::InternalServerError)?;
    3423            0 :         info!("wal import complete");
    3424              : 
    3425              :         // Read the end of the tar archive.
    3426            0 :         read_tar_eof(body).await.map_err(ApiError::InternalServerError)?;
    3427              : 
    3428              :         // TODO Does it make sense to overshoot?
    3429            0 :         if timeline.get_last_record_lsn() < end_lsn {
    3430            0 :             return Err(ApiError::InternalServerError(anyhow::anyhow!("Cannot import WAL from Lsn {start_lsn} because timeline does not start from the same lsn: {last_record_lsn}")));
    3431            0 :         }
    3432            0 : 
    3433            0 :         // Flush data to disk, then upload to s3. No need for a forced checkpoint.
    3434            0 :         // We only want to persist the data, and it doesn't matter if it's in the
    3435            0 :         // shape of deltas or images.
    3436            0 :         info!("flushing layers");
    3437            0 :         timeline.freeze_and_flush().await.map_err(|e| match e {
    3438            0 :             tenant::timeline::FlushLayerError::Cancelled => ApiError::ShuttingDown,
    3439            0 :             other => ApiError::InternalServerError(anyhow::anyhow!(other)),
    3440            0 :         })?;
    3441              : 
    3442            0 :         info!("done");
    3443              : 
    3444            0 :         json_response(StatusCode::OK, ())
    3445            0 :     }.instrument(span).await
    3446            0 : }
    3447              : 
    3448              : /// Read the end of a tar archive.
    3449              : ///
    3450              : /// A tar archive normally ends with two consecutive blocks of zeros, 512 bytes each.
    3451              : /// `tokio_tar` already read the first such block. Read the second all-zeros block,
    3452              : /// and check that there is no more data after the EOF marker.
    3453              : ///
    3454              : /// 'tar' command can also write extra blocks of zeros, up to a record
    3455              : /// size, controlled by the --record-size argument. Ignore them too.
    3456            0 : async fn read_tar_eof(mut reader: (impl tokio::io::AsyncRead + Unpin)) -> anyhow::Result<()> {
    3457              :     use tokio::io::AsyncReadExt;
    3458            0 :     let mut buf = [0u8; 512];
    3459            0 : 
    3460            0 :     // Read the all-zeros block, and verify it
    3461            0 :     let mut total_bytes = 0;
    3462            0 :     while total_bytes < 512 {
    3463            0 :         let nbytes = reader.read(&mut buf[total_bytes..]).await?;
    3464            0 :         total_bytes += nbytes;
    3465            0 :         if nbytes == 0 {
    3466            0 :             break;
    3467            0 :         }
    3468              :     }
    3469            0 :     if total_bytes < 512 {
    3470            0 :         anyhow::bail!("incomplete or invalid tar EOF marker");
    3471            0 :     }
    3472            0 :     if !buf.iter().all(|&x| x == 0) {
    3473            0 :         anyhow::bail!("invalid tar EOF marker");
    3474            0 :     }
    3475            0 : 
    3476            0 :     // Drain any extra zero-blocks after the EOF marker
    3477            0 :     let mut trailing_bytes = 0;
    3478            0 :     let mut seen_nonzero_bytes = false;
    3479              :     loop {
    3480            0 :         let nbytes = reader.read(&mut buf).await?;
    3481            0 :         trailing_bytes += nbytes;
    3482            0 :         if !buf.iter().all(|&x| x == 0) {
    3483            0 :             seen_nonzero_bytes = true;
    3484            0 :         }
    3485            0 :         if nbytes == 0 {
    3486            0 :             break;
    3487            0 :         }
    3488              :     }
    3489            0 :     if seen_nonzero_bytes {
    3490            0 :         anyhow::bail!("unexpected non-zero bytes after the tar archive");
    3491            0 :     }
    3492            0 :     if trailing_bytes % 512 != 0 {
    3493            0 :         anyhow::bail!(
    3494            0 :             "unexpected number of zeros ({trailing_bytes}), not divisible by tar block size (512 bytes), after the tar archive"
    3495            0 :         );
    3496            0 :     }
    3497            0 :     Ok(())
    3498            0 : }
    3499              : 
    3500              : /// Common functionality of all the HTTP API handlers.
    3501              : ///
    3502              : /// - Adds a tracing span to each request (by `request_span`)
    3503              : /// - Logs the request depending on the request method (by `request_span`)
    3504              : /// - Logs the response if it was not successful (by `request_span`
    3505              : /// - Shields the handler function from async cancellations. Hyper can drop the handler
    3506              : ///   Future if the connection to the client is lost, but most of the pageserver code is
    3507              : ///   not async cancellation safe. This converts the dropped future into a graceful cancellation
    3508              : ///   request with a CancellationToken.
    3509            0 : async fn api_handler<R, H>(request: Request<Body>, handler: H) -> Result<Response<Body>, ApiError>
    3510            0 : where
    3511            0 :     R: std::future::Future<Output = Result<Response<Body>, ApiError>> + Send + 'static,
    3512            0 :     H: FnOnce(Request<Body>, CancellationToken) -> R + Send + Sync + 'static,
    3513            0 : {
    3514            0 :     if request.uri() != &"/v1/failpoints".parse::<Uri>().unwrap() {
    3515            0 :         fail::fail_point!("api-503", |_| Err(ApiError::ResourceUnavailable(
    3516            0 :             "failpoint".into()
    3517            0 :         )));
    3518              : 
    3519            0 :         fail::fail_point!("api-500", |_| Err(ApiError::InternalServerError(
    3520            0 :             anyhow::anyhow!("failpoint")
    3521            0 :         )));
    3522            0 :     }
    3523              : 
    3524              :     // Spawn a new task to handle the request, to protect the handler from unexpected
    3525              :     // async cancellations. Most pageserver functions are not async cancellation safe.
    3526              :     // We arm a drop-guard, so that if Hyper drops the Future, we signal the task
    3527              :     // with the cancellation token.
    3528            0 :     let token = CancellationToken::new();
    3529            0 :     let cancel_guard = token.clone().drop_guard();
    3530            0 :     let result = request_span(request, move |r| async {
    3531            0 :         let handle = tokio::spawn(
    3532            0 :             async {
    3533            0 :                 let token_cloned = token.clone();
    3534            0 :                 let result = handler(r, token).await;
    3535            0 :                 if token_cloned.is_cancelled() {
    3536              :                     // dropguard has executed: we will never turn this result into response.
    3537              :                     //
    3538              :                     // at least temporarily do {:?} logging; these failures are rare enough but
    3539              :                     // could hide difficult errors.
    3540            0 :                     match &result {
    3541            0 :                         Ok(response) => {
    3542            0 :                             let status = response.status();
    3543            0 :                             info!(%status, "Cancelled request finished successfully")
    3544              :                         }
    3545            0 :                         Err(e) => match e {
    3546              :                             ApiError::ShuttingDown | ApiError::ResourceUnavailable(_) => {
    3547              :                                 // Don't log this at error severity: they are normal during lifecycle of tenants/process
    3548            0 :                                 info!("Cancelled request aborted for shutdown")
    3549              :                             }
    3550              :                             _ => {
    3551              :                                 // Log these in a highly visible way, because we have no client to send the response to, but
    3552              :                                 // would like to know that something went wrong.
    3553            0 :                                 error!("Cancelled request finished with an error: {e:?}")
    3554              :                             }
    3555              :                         },
    3556              :                     }
    3557            0 :                 }
    3558              :                 // only logging for cancelled panicked request handlers is the tracing_panic_hook,
    3559              :                 // which should suffice.
    3560              :                 //
    3561              :                 // there is still a chance to lose the result due to race between
    3562              :                 // returning from here and the actual connection closing happening
    3563              :                 // before outer task gets to execute. leaving that up for #5815.
    3564            0 :                 result
    3565            0 :             }
    3566            0 :             .in_current_span(),
    3567            0 :         );
    3568            0 : 
    3569            0 :         match handle.await {
    3570              :             // TODO: never actually return Err from here, always Ok(...) so that we can log
    3571              :             // spanned errors. Call api_error_handler instead and return appropriate Body.
    3572            0 :             Ok(result) => result,
    3573            0 :             Err(e) => {
    3574            0 :                 // The handler task panicked. We have a global panic handler that logs the
    3575            0 :                 // panic with its backtrace, so no need to log that here. Only log a brief
    3576            0 :                 // message to make it clear that we returned the error to the client.
    3577            0 :                 error!("HTTP request handler task panicked: {e:#}");
    3578              : 
    3579              :                 // Don't return an Error here, because then fallback error handler that was
    3580              :                 // installed in make_router() will print the error. Instead, construct the
    3581              :                 // HTTP error response and return that.
    3582            0 :                 Ok(
    3583            0 :                     ApiError::InternalServerError(anyhow!("HTTP request handler task panicked"))
    3584            0 :                         .into_response(),
    3585            0 :                 )
    3586              :             }
    3587              :         }
    3588            0 :     })
    3589            0 :     .await;
    3590              : 
    3591            0 :     cancel_guard.disarm();
    3592            0 : 
    3593            0 :     result
    3594            0 : }
    3595              : 
    3596              : /// Like api_handler, but returns an error response if the server is built without
    3597              : /// the 'testing' feature.
    3598            0 : async fn testing_api_handler<R, H>(
    3599            0 :     desc: &str,
    3600            0 :     request: Request<Body>,
    3601            0 :     handler: H,
    3602            0 : ) -> Result<Response<Body>, ApiError>
    3603            0 : where
    3604            0 :     R: std::future::Future<Output = Result<Response<Body>, ApiError>> + Send + 'static,
    3605            0 :     H: FnOnce(Request<Body>, CancellationToken) -> R + Send + Sync + 'static,
    3606            0 : {
    3607            0 :     if cfg!(feature = "testing") {
    3608            0 :         api_handler(request, handler).await
    3609              :     } else {
    3610            0 :         std::future::ready(Err(ApiError::BadRequest(anyhow!(
    3611            0 :             "Cannot {desc} because pageserver was compiled without testing APIs",
    3612            0 :         ))))
    3613            0 :         .await
    3614              :     }
    3615            0 : }
    3616              : 
    3617            0 : pub fn make_router(
    3618            0 :     state: Arc<State>,
    3619            0 :     launch_ts: &'static LaunchTimestamp,
    3620            0 :     auth: Option<Arc<SwappableJwtAuth>>,
    3621            0 : ) -> anyhow::Result<RouterBuilder<hyper::Body, ApiError>> {
    3622            0 :     let spec = include_bytes!("openapi_spec.yml");
    3623            0 :     let mut router = attach_openapi_ui(endpoint::make_router(), spec, "/swagger.yml", "/v1/doc");
    3624            0 :     if auth.is_some() {
    3625            0 :         router = router.middleware(auth_middleware(|request| {
    3626            0 :             let state = get_state(request);
    3627            0 :             if state.allowlist_routes.contains(&request.uri().path()) {
    3628            0 :                 None
    3629              :             } else {
    3630            0 :                 state.auth.as_deref()
    3631              :             }
    3632            0 :         }))
    3633            0 :     }
    3634              : 
    3635            0 :     router = router.middleware(
    3636            0 :         endpoint::add_response_header_middleware(
    3637            0 :             "PAGESERVER_LAUNCH_TIMESTAMP",
    3638            0 :             &launch_ts.to_string(),
    3639            0 :         )
    3640            0 :         .expect("construct launch timestamp header middleware"),
    3641            0 :     );
    3642            0 : 
    3643            0 :     Ok(router
    3644            0 :         .data(state)
    3645            0 :         .get("/metrics", |r| request_span(r, prometheus_metrics_handler))
    3646            0 :         .get("/profile/cpu", |r| request_span(r, profile_cpu_handler))
    3647            0 :         .get("/profile/heap", |r| request_span(r, profile_heap_handler))
    3648            0 :         .get("/v1/status", |r| api_handler(r, status_handler))
    3649            0 :         .put("/v1/failpoints", |r| {
    3650            0 :             testing_api_handler("manage failpoints", r, failpoints_handler)
    3651            0 :         })
    3652            0 :         .post("/v1/reload_auth_validation_keys", |r| {
    3653            0 :             api_handler(r, reload_auth_validation_keys_handler)
    3654            0 :         })
    3655            0 :         .get("/v1/tenant", |r| api_handler(r, tenant_list_handler))
    3656            0 :         .get("/v1/tenant/:tenant_shard_id", |r| {
    3657            0 :             api_handler(r, tenant_status)
    3658            0 :         })
    3659            0 :         .delete("/v1/tenant/:tenant_shard_id", |r| {
    3660            0 :             api_handler(r, tenant_delete_handler)
    3661            0 :         })
    3662            0 :         .get("/v1/tenant/:tenant_shard_id/synthetic_size", |r| {
    3663            0 :             api_handler(r, tenant_size_handler)
    3664            0 :         })
    3665            0 :         .patch("/v1/tenant/config", |r| {
    3666            0 :             api_handler(r, patch_tenant_config_handler)
    3667            0 :         })
    3668            0 :         .put("/v1/tenant/config", |r| {
    3669            0 :             api_handler(r, update_tenant_config_handler)
    3670            0 :         })
    3671            0 :         .put("/v1/tenant/:tenant_shard_id/shard_split", |r| {
    3672            0 :             api_handler(r, tenant_shard_split_handler)
    3673            0 :         })
    3674            0 :         .get("/v1/tenant/:tenant_shard_id/config", |r| {
    3675            0 :             api_handler(r, get_tenant_config_handler)
    3676            0 :         })
    3677            0 :         .put("/v1/tenant/:tenant_shard_id/location_config", |r| {
    3678            0 :             api_handler(r, put_tenant_location_config_handler)
    3679            0 :         })
    3680            0 :         .get("/v1/location_config", |r| {
    3681            0 :             api_handler(r, list_location_config_handler)
    3682            0 :         })
    3683            0 :         .get("/v1/location_config/:tenant_shard_id", |r| {
    3684            0 :             api_handler(r, get_location_config_handler)
    3685            0 :         })
    3686            0 :         .put(
    3687            0 :             "/v1/tenant/:tenant_shard_id/time_travel_remote_storage",
    3688            0 :             |r| api_handler(r, tenant_time_travel_remote_storage_handler),
    3689            0 :         )
    3690            0 :         .get("/v1/tenant/:tenant_shard_id/timeline", |r| {
    3691            0 :             api_handler(r, timeline_list_handler)
    3692            0 :         })
    3693            0 :         .get("/v1/tenant/:tenant_shard_id/timeline_and_offloaded", |r| {
    3694            0 :             api_handler(r, timeline_and_offloaded_list_handler)
    3695            0 :         })
    3696            0 :         .post("/v1/tenant/:tenant_shard_id/timeline", |r| {
    3697            0 :             api_handler(r, timeline_create_handler)
    3698            0 :         })
    3699            0 :         .post("/v1/tenant/:tenant_shard_id/reset", |r| {
    3700            0 :             api_handler(r, tenant_reset_handler)
    3701            0 :         })
    3702            0 :         .post(
    3703            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/preserve_initdb_archive",
    3704            0 :             |r| api_handler(r, timeline_preserve_initdb_handler),
    3705            0 :         )
    3706            0 :         .put(
    3707            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/archival_config",
    3708            0 :             |r| api_handler(r, timeline_archival_config_handler),
    3709            0 :         )
    3710            0 :         .get("/v1/tenant/:tenant_shard_id/timeline/:timeline_id", |r| {
    3711            0 :             api_handler(r, timeline_detail_handler)
    3712            0 :         })
    3713            0 :         .get(
    3714            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/get_lsn_by_timestamp",
    3715            0 :             |r| api_handler(r, get_lsn_by_timestamp_handler),
    3716            0 :         )
    3717            0 :         .get(
    3718            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/get_timestamp_of_lsn",
    3719            0 :             |r| api_handler(r, get_timestamp_of_lsn_handler),
    3720            0 :         )
    3721            0 :         .post(
    3722            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/patch_index_part",
    3723            0 :             |r| api_handler(r, timeline_patch_index_part_handler),
    3724            0 :         )
    3725            0 :         .post(
    3726            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/lsn_lease",
    3727            0 :             |r| api_handler(r, lsn_lease_handler),
    3728            0 :         )
    3729            0 :         .put(
    3730            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/do_gc",
    3731            0 :             |r| api_handler(r, timeline_gc_handler),
    3732            0 :         )
    3733            0 :         .get(
    3734            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/compact",
    3735            0 :             |r| api_handler(r, timeline_compact_info_handler),
    3736            0 :         )
    3737            0 :         .put(
    3738            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/compact",
    3739            0 :             |r| api_handler(r, timeline_compact_handler),
    3740            0 :         )
    3741            0 :         .delete(
    3742            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/compact",
    3743            0 :             |r| api_handler(r, timeline_cancel_compact_handler),
    3744            0 :         )
    3745            0 :         .put(
    3746            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/offload",
    3747            0 :             |r| testing_api_handler("attempt timeline offload", r, timeline_offload_handler),
    3748            0 :         )
    3749            0 :         .put(
    3750            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/checkpoint",
    3751            0 :             |r| testing_api_handler("run timeline checkpoint", r, timeline_checkpoint_handler),
    3752            0 :         )
    3753            0 :         .post(
    3754            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/download_remote_layers",
    3755            0 :             |r| api_handler(r, timeline_download_remote_layers_handler_post),
    3756            0 :         )
    3757            0 :         .get(
    3758            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/download_remote_layers",
    3759            0 :             |r| api_handler(r, timeline_download_remote_layers_handler_get),
    3760            0 :         )
    3761            0 :         .put(
    3762            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/detach_ancestor",
    3763            0 :             |r| api_handler(r, timeline_detach_ancestor_handler),
    3764            0 :         )
    3765            0 :         .delete("/v1/tenant/:tenant_shard_id/timeline/:timeline_id", |r| {
    3766            0 :             api_handler(r, timeline_delete_handler)
    3767            0 :         })
    3768            0 :         .get(
    3769            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/layer",
    3770            0 :             |r| api_handler(r, layer_map_info_handler),
    3771            0 :         )
    3772            0 :         .post(
    3773            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/download_heatmap_layers",
    3774            0 :             |r| api_handler(r, timeline_download_heatmap_layers_handler),
    3775            0 :         )
    3776            0 :         .delete(
    3777            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/download_heatmap_layers",
    3778            0 :             |r| api_handler(r, timeline_shutdown_download_heatmap_layers_handler),
    3779            0 :         )
    3780            0 :         .get(
    3781            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/layer/:layer_file_name",
    3782            0 :             |r| api_handler(r, layer_download_handler),
    3783            0 :         )
    3784            0 :         .delete(
    3785            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/layer/:layer_file_name",
    3786            0 :             |r| api_handler(r, evict_timeline_layer_handler),
    3787            0 :         )
    3788            0 :         .post(
    3789            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/layer/:layer_name/scan_disposable_keys",
    3790            0 :             |r| testing_api_handler("timeline_layer_scan_disposable_keys", r, timeline_layer_scan_disposable_keys),
    3791            0 :         )
    3792            0 :         .post(
    3793            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/block_gc",
    3794            0 :             |r| api_handler(r, timeline_gc_blocking_handler),
    3795            0 :         )
    3796            0 :         .post(
    3797            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/unblock_gc",
    3798            0 :             |r| api_handler(r, timeline_gc_unblocking_handler),
    3799            0 :         )
    3800            0 :         .get(
    3801            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/page_trace",
    3802            0 :             |r| api_handler(r, timeline_page_trace_handler),
    3803            0 :         )
    3804            0 :         .post("/v1/tenant/:tenant_shard_id/heatmap_upload", |r| {
    3805            0 :             api_handler(r, secondary_upload_handler)
    3806            0 :         })
    3807            0 :         .get("/v1/tenant/:tenant_id/scan_remote_storage", |r| {
    3808            0 :             api_handler(r, tenant_scan_remote_handler)
    3809            0 :         })
    3810            0 :         .put("/v1/disk_usage_eviction/run", |r| {
    3811            0 :             api_handler(r, disk_usage_eviction_run)
    3812            0 :         })
    3813            0 :         .put("/v1/deletion_queue/flush", |r| {
    3814            0 :             api_handler(r, deletion_queue_flush)
    3815            0 :         })
    3816            0 :         .get("/v1/tenant/:tenant_shard_id/secondary/status", |r| {
    3817            0 :             api_handler(r, secondary_status_handler)
    3818            0 :         })
    3819            0 :         .post("/v1/tenant/:tenant_shard_id/secondary/download", |r| {
    3820            0 :             api_handler(r, secondary_download_handler)
    3821            0 :         })
    3822            0 :         .post("/v1/tenant/:tenant_shard_id/wait_lsn", |r| {
    3823            0 :             api_handler(r, wait_lsn_handler)
    3824            0 :         })
    3825            0 :         .put("/v1/tenant/:tenant_shard_id/break", |r| {
    3826            0 :             testing_api_handler("set tenant state to broken", r, handle_tenant_break)
    3827            0 :         })
    3828            0 :         .get("/v1/panic", |r| api_handler(r, always_panic_handler))
    3829            0 :         .post("/v1/tracing/event", |r| {
    3830            0 :             testing_api_handler("emit a tracing event", r, post_tracing_event_handler)
    3831            0 :         })
    3832            0 :         .get(
    3833            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/getpage",
    3834            0 :             |r| testing_api_handler("getpage@lsn", r, getpage_at_lsn_handler),
    3835            0 :         )
    3836            0 :         .get(
    3837            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/touchpage",
    3838            0 :             |r| api_handler(r, touchpage_at_lsn_handler),
    3839            0 :         )
    3840            0 :         .get(
    3841            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/keyspace",
    3842            0 :             |r| api_handler(r, timeline_collect_keyspace),
    3843            0 :         )
    3844            0 :         .put("/v1/io_engine", |r| api_handler(r, put_io_engine_handler))
    3845            0 :         .put("/v1/io_mode", |r| api_handler(r, put_io_mode_handler))
    3846            0 :         .get("/v1/utilization", |r| api_handler(r, get_utilization))
    3847            0 :         .post(
    3848            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/ingest_aux_files",
    3849            0 :             |r| testing_api_handler("ingest_aux_files", r, ingest_aux_files),
    3850            0 :         )
    3851            0 :         .post(
    3852            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/list_aux_files",
    3853            0 :             |r| testing_api_handler("list_aux_files", r, list_aux_files),
    3854            0 :         )
    3855            0 :         .post("/v1/top_tenants", |r| api_handler(r, post_top_tenants))
    3856            0 :         .post(
    3857            0 :             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/perf_info",
    3858            0 :             |r| testing_api_handler("perf_info", r, perf_info),
    3859            0 :         )
    3860            0 :         .put(
    3861            0 :             "/v1/tenant/:tenant_id/timeline/:timeline_id/import_basebackup",
    3862            0 :             |r| api_handler(r, put_tenant_timeline_import_basebackup),
    3863            0 :         )
    3864            0 :         .put(
    3865            0 :             "/v1/tenant/:tenant_id/timeline/:timeline_id/import_wal",
    3866            0 :             |r| api_handler(r, put_tenant_timeline_import_wal),
    3867            0 :         )
    3868            0 :         .any(handler_404))
    3869            0 : }
        

Generated by: LCOV version 2.1-beta