LCOV - code coverage report
Current view: top level - storage_controller/src - service.rs (source / functions) Coverage Total Hit
Test: 1b0a6a0c05cee5a7de360813c8034804e105ce1c.info Lines: 0.0 % 5505 0
Test Date: 2025-03-12 00:01:28 Functions: 0.0 % 483 0

            Line data    Source code
       1              : pub mod chaos_injector;
       2              : mod context_iterator;
       3              : pub(crate) mod safekeeper_reconciler;
       4              : 
       5              : use std::borrow::Cow;
       6              : use std::cmp::Ordering;
       7              : use std::collections::{BTreeMap, HashMap, HashSet};
       8              : use std::error::Error;
       9              : use std::num::NonZeroU32;
      10              : use std::ops::{Deref, DerefMut};
      11              : use std::path::PathBuf;
      12              : use std::str::FromStr;
      13              : use std::sync::Arc;
      14              : use std::time::{Duration, Instant};
      15              : 
      16              : use anyhow::Context;
      17              : use context_iterator::TenantShardContextIterator;
      18              : use control_plane::storage_controller::{
      19              :     AttachHookRequest, AttachHookResponse, InspectRequest, InspectResponse,
      20              : };
      21              : use diesel::result::DatabaseErrorKind;
      22              : use futures::StreamExt;
      23              : use futures::stream::FuturesUnordered;
      24              : use http_utils::error::ApiError;
      25              : use hyper::Uri;
      26              : use itertools::Itertools;
      27              : use pageserver_api::controller_api::{
      28              :     AvailabilityZone, MetadataHealthRecord, MetadataHealthUpdateRequest, NodeAvailability,
      29              :     NodeRegisterRequest, NodeSchedulingPolicy, NodeShard, NodeShardResponse, PlacementPolicy,
      30              :     SafekeeperDescribeResponse, ShardSchedulingPolicy, ShardsPreferredAzsRequest,
      31              :     ShardsPreferredAzsResponse, SkSchedulingPolicy, TenantCreateRequest, TenantCreateResponse,
      32              :     TenantCreateResponseShard, TenantDescribeResponse, TenantDescribeResponseShard,
      33              :     TenantLocateResponse, TenantPolicyRequest, TenantShardMigrateRequest,
      34              :     TenantShardMigrateResponse,
      35              : };
      36              : use pageserver_api::models::{
      37              :     self, LocationConfig, LocationConfigListResponse, LocationConfigMode, PageserverUtilization,
      38              :     SafekeeperInfo, SafekeepersInfo, SecondaryProgress, ShardParameters, TenantConfig,
      39              :     TenantConfigPatchRequest, TenantConfigRequest, TenantLocationConfigRequest,
      40              :     TenantLocationConfigResponse, TenantShardLocation, TenantShardSplitRequest,
      41              :     TenantShardSplitResponse, TenantSorting, TenantTimeTravelRequest,
      42              :     TimelineArchivalConfigRequest, TimelineCreateRequest, TimelineCreateResponseStorcon,
      43              :     TimelineInfo, TopTenantShardItem, TopTenantShardsRequest,
      44              : };
      45              : use pageserver_api::shard::{
      46              :     ShardCount, ShardIdentity, ShardNumber, ShardStripeSize, TenantShardId,
      47              : };
      48              : use pageserver_api::upcall_api::{
      49              :     ReAttachRequest, ReAttachResponse, ReAttachResponseTenant, ValidateRequest, ValidateResponse,
      50              :     ValidateResponseTenant,
      51              : };
      52              : use pageserver_client::{BlockUnblock, mgmt_api};
      53              : use reqwest::{Certificate, StatusCode};
      54              : use safekeeper_api::membership::{MemberSet, SafekeeperId};
      55              : use safekeeper_api::models::SafekeeperUtilization;
      56              : use safekeeper_reconciler::{SafekeeperReconcilers, ScheduleRequest};
      57              : use tokio::sync::TryAcquireError;
      58              : use tokio::sync::mpsc::error::TrySendError;
      59              : use tokio::task::JoinSet;
      60              : use tokio_util::sync::CancellationToken;
      61              : use tracing::{Instrument, debug, error, info, info_span, instrument, warn};
      62              : use utils::completion::Barrier;
      63              : use utils::generation::Generation;
      64              : use utils::id::{NodeId, TenantId, TimelineId};
      65              : use utils::logging::SecretString;
      66              : use utils::sync::gate::Gate;
      67              : use utils::{failpoint_support, pausable_failpoint};
      68              : 
      69              : use crate::background_node_operations::{
      70              :     Drain, Fill, MAX_RECONCILES_PER_OPERATION, Operation, OperationError, OperationHandler,
      71              : };
      72              : use crate::compute_hook::{self, ComputeHook, NotifyError};
      73              : use crate::drain_utils::{self, TenantShardDrain, TenantShardIterator};
      74              : use crate::heartbeater::{Heartbeater, PageserverState, SafekeeperState};
      75              : use crate::id_lock_map::{
      76              :     IdLockMap, TracingExclusiveGuard, trace_exclusive_lock, trace_shared_lock,
      77              : };
      78              : use crate::leadership::Leadership;
      79              : use crate::metrics;
      80              : use crate::node::{AvailabilityTransition, Node};
      81              : use crate::pageserver_client::PageserverClient;
      82              : use crate::peer_client::GlobalObservedState;
      83              : use crate::persistence::split_state::SplitState;
      84              : use crate::persistence::{
      85              :     AbortShardSplitStatus, ControllerPersistence, DatabaseError, DatabaseResult,
      86              :     MetadataHealthPersistence, Persistence, SafekeeperTimelineOpKind, ShardGenerationState,
      87              :     TenantFilter, TenantShardPersistence, TimelinePendingOpPersistence, TimelinePersistence,
      88              : };
      89              : use crate::reconciler::{
      90              :     ReconcileError, ReconcileUnits, ReconcilerConfig, ReconcilerConfigBuilder, ReconcilerPriority,
      91              :     attached_location_conf,
      92              : };
      93              : use crate::safekeeper::Safekeeper;
      94              : use crate::scheduler::{
      95              :     AttachedShardTag, MaySchedule, ScheduleContext, ScheduleError, ScheduleMode, Scheduler,
      96              : };
      97              : use crate::tenant_shard::{
      98              :     IntentState, MigrateAttachment, ObservedState, ObservedStateDelta, ObservedStateLocation,
      99              :     ReconcileNeeded, ReconcileResult, ReconcileWaitError, ReconcilerStatus, ReconcilerWaiter,
     100              :     ScheduleOptimization, ScheduleOptimizationAction, TenantShard,
     101              : };
     102              : 
     103              : const WAITER_FILL_DRAIN_POLL_TIMEOUT: Duration = Duration::from_millis(500);
     104              : 
     105              : // For operations that should be quick, like attaching a new tenant
     106              : const SHORT_RECONCILE_TIMEOUT: Duration = Duration::from_secs(5);
     107              : 
     108              : // For operations that might be slow, like migrating a tenant with
     109              : // some data in it.
     110              : pub const RECONCILE_TIMEOUT: Duration = Duration::from_secs(30);
     111              : 
     112              : // If we receive a call using Secondary mode initially, it will omit generation.  We will initialize
     113              : // tenant shards into this generation, and as long as it remains in this generation, we will accept
     114              : // input generation from future requests as authoritative.
     115              : const INITIAL_GENERATION: Generation = Generation::new(0);
     116              : 
     117              : /// How long [`Service::startup_reconcile`] is allowed to take before it should give
     118              : /// up on unresponsive pageservers and proceed.
     119              : pub(crate) const STARTUP_RECONCILE_TIMEOUT: Duration = Duration::from_secs(30);
     120              : 
     121              : /// How long a node may be unresponsive to heartbeats before we declare it offline.
     122              : /// This must be long enough to cover node restarts as well as normal operations: in future
     123              : pub const MAX_OFFLINE_INTERVAL_DEFAULT: Duration = Duration::from_secs(30);
     124              : 
     125              : /// How long a node may be unresponsive to heartbeats during start up before we declare it
     126              : /// offline.
     127              : ///
     128              : /// This is much more lenient than [`MAX_OFFLINE_INTERVAL_DEFAULT`] since the pageserver's
     129              : /// handling of the re-attach response may take a long time and blocks heartbeats from
     130              : /// being handled on the pageserver side.
     131              : pub const MAX_WARMING_UP_INTERVAL_DEFAULT: Duration = Duration::from_secs(300);
     132              : 
     133              : /// How often to send heartbeats to registered nodes?
     134              : pub const HEARTBEAT_INTERVAL_DEFAULT: Duration = Duration::from_secs(5);
     135              : 
     136              : /// How long is too long for a reconciliation?
     137              : pub const LONG_RECONCILE_THRESHOLD_DEFAULT: Duration = Duration::from_secs(120);
     138              : 
     139              : #[derive(Clone, strum_macros::Display)]
     140              : enum TenantOperations {
     141              :     Create,
     142              :     LocationConfig,
     143              :     ConfigSet,
     144              :     ConfigPatch,
     145              :     TimeTravelRemoteStorage,
     146              :     Delete,
     147              :     UpdatePolicy,
     148              :     ShardSplit,
     149              :     SecondaryDownload,
     150              :     TimelineCreate,
     151              :     TimelineDelete,
     152              :     AttachHook,
     153              :     TimelineArchivalConfig,
     154              :     TimelineDetachAncestor,
     155              :     TimelineGcBlockUnblock,
     156              :     DropDetached,
     157              :     DownloadHeatmapLayers,
     158              : }
     159              : 
     160              : #[derive(Clone, strum_macros::Display)]
     161              : enum NodeOperations {
     162              :     Register,
     163              :     Configure,
     164              :     Delete,
     165              : }
     166              : 
     167              : /// The leadership status for the storage controller process.
     168              : /// Allowed transitions are:
     169              : /// 1. Leader -> SteppedDown
     170              : /// 2. Candidate -> Leader
     171              : #[derive(
     172              :     Eq,
     173              :     PartialEq,
     174              :     Copy,
     175              :     Clone,
     176              :     strum_macros::Display,
     177            0 :     strum_macros::EnumIter,
     178              :     measured::FixedCardinalityLabel,
     179              : )]
     180              : #[strum(serialize_all = "snake_case")]
     181              : pub(crate) enum LeadershipStatus {
     182              :     /// This is the steady state where the storage controller can produce
     183              :     /// side effects in the cluster.
     184              :     Leader,
     185              :     /// We've been notified to step down by another candidate. No reconciliations
     186              :     /// take place in this state.
     187              :     SteppedDown,
     188              :     /// Initial state for a new storage controller instance. Will attempt to assume leadership.
     189              :     #[allow(unused)]
     190              :     Candidate,
     191              : }
     192              : 
     193              : pub const RECONCILER_CONCURRENCY_DEFAULT: usize = 128;
     194              : pub const PRIORITY_RECONCILER_CONCURRENCY_DEFAULT: usize = 256;
     195              : 
     196              : // Depth of the channel used to enqueue shards for reconciliation when they can't do it immediately.
     197              : // This channel is finite-size to avoid using excessive memory if we get into a state where reconciles are finishing more slowly
     198              : // than they're being pushed onto the queue.
     199              : const MAX_DELAYED_RECONCILES: usize = 10000;
     200              : 
     201              : // Top level state available to all HTTP handlers
     202              : struct ServiceState {
     203              :     leadership_status: LeadershipStatus,
     204              : 
     205              :     tenants: BTreeMap<TenantShardId, TenantShard>,
     206              : 
     207              :     nodes: Arc<HashMap<NodeId, Node>>,
     208              : 
     209              :     safekeepers: Arc<HashMap<NodeId, Safekeeper>>,
     210              : 
     211              :     safekeeper_reconcilers: SafekeeperReconcilers,
     212              : 
     213              :     scheduler: Scheduler,
     214              : 
     215              :     /// Ongoing background operation on the cluster if any is running.
     216              :     /// Note that only one such operation may run at any given time,
     217              :     /// hence the type choice.
     218              :     ongoing_operation: Option<OperationHandler>,
     219              : 
     220              :     /// Queue of tenants who are waiting for concurrency limits to permit them to reconcile
     221              :     delayed_reconcile_rx: tokio::sync::mpsc::Receiver<TenantShardId>,
     222              : }
     223              : 
     224              : /// Transform an error from a pageserver into an error to return to callers of a storage
     225              : /// controller API.
     226            0 : fn passthrough_api_error(node: &Node, e: mgmt_api::Error) -> ApiError {
     227            0 :     match e {
     228            0 :         mgmt_api::Error::SendRequest(e) => {
     229            0 :             // Presume errors sending requests are connectivity/availability issues
     230            0 :             ApiError::ResourceUnavailable(format!("{node} error sending request: {e}").into())
     231              :         }
     232            0 :         mgmt_api::Error::ReceiveErrorBody(str) => {
     233            0 :             // Presume errors receiving body are connectivity/availability issues
     234            0 :             ApiError::ResourceUnavailable(
     235            0 :                 format!("{node} error receiving error body: {str}").into(),
     236            0 :             )
     237              :         }
     238            0 :         mgmt_api::Error::ReceiveBody(err) if err.is_decode() => {
     239            0 :             // Return 500 for decoding errors.
     240            0 :             ApiError::InternalServerError(anyhow::Error::from(err).context("error decoding body"))
     241              :         }
     242            0 :         mgmt_api::Error::ReceiveBody(err) => {
     243            0 :             // Presume errors receiving body are connectivity/availability issues except for decoding errors
     244            0 :             let src_str = err.source().map(|e| e.to_string()).unwrap_or_default();
     245            0 :             ApiError::ResourceUnavailable(
     246            0 :                 format!("{node} error receiving error body: {err} {}", src_str).into(),
     247            0 :             )
     248              :         }
     249            0 :         mgmt_api::Error::ApiError(StatusCode::NOT_FOUND, msg) => {
     250            0 :             ApiError::NotFound(anyhow::anyhow!(format!("{node}: {msg}")).into())
     251              :         }
     252            0 :         mgmt_api::Error::ApiError(StatusCode::SERVICE_UNAVAILABLE, msg) => {
     253            0 :             ApiError::ResourceUnavailable(format!("{node}: {msg}").into())
     254              :         }
     255            0 :         mgmt_api::Error::ApiError(status @ StatusCode::UNAUTHORIZED, msg)
     256            0 :         | mgmt_api::Error::ApiError(status @ StatusCode::FORBIDDEN, msg) => {
     257              :             // Auth errors talking to a pageserver are not auth errors for the caller: they are
     258              :             // internal server errors, showing that something is wrong with the pageserver or
     259              :             // storage controller's auth configuration.
     260            0 :             ApiError::InternalServerError(anyhow::anyhow!("{node} {status}: {msg}"))
     261              :         }
     262            0 :         mgmt_api::Error::ApiError(status @ StatusCode::TOO_MANY_REQUESTS, msg) => {
     263            0 :             // Pass through 429 errors: if pageserver is asking us to wait + retry, we in
     264            0 :             // turn ask our clients to wait + retry
     265            0 :             ApiError::Conflict(format!("{node} {status}: {status} {msg}"))
     266              :         }
     267            0 :         mgmt_api::Error::ApiError(status, msg) => {
     268            0 :             // Presume general case of pageserver API errors is that we tried to do something
     269            0 :             // that can't be done right now.
     270            0 :             ApiError::Conflict(format!("{node} {status}: {status} {msg}"))
     271              :         }
     272            0 :         mgmt_api::Error::Cancelled => ApiError::ShuttingDown,
     273            0 :         mgmt_api::Error::CreateClient(e) => ApiError::InternalServerError(anyhow::anyhow!(e)),
     274              :     }
     275            0 : }
     276              : 
     277              : impl ServiceState {
     278            0 :     fn new(
     279            0 :         nodes: HashMap<NodeId, Node>,
     280            0 :         safekeepers: HashMap<NodeId, Safekeeper>,
     281            0 :         tenants: BTreeMap<TenantShardId, TenantShard>,
     282            0 :         scheduler: Scheduler,
     283            0 :         delayed_reconcile_rx: tokio::sync::mpsc::Receiver<TenantShardId>,
     284            0 :         initial_leadership_status: LeadershipStatus,
     285            0 :         reconcilers_cancel: CancellationToken,
     286            0 :     ) -> Self {
     287            0 :         metrics::update_leadership_status(initial_leadership_status);
     288            0 : 
     289            0 :         Self {
     290            0 :             leadership_status: initial_leadership_status,
     291            0 :             tenants,
     292            0 :             nodes: Arc::new(nodes),
     293            0 :             safekeepers: Arc::new(safekeepers),
     294            0 :             safekeeper_reconcilers: SafekeeperReconcilers::new(reconcilers_cancel),
     295            0 :             scheduler,
     296            0 :             ongoing_operation: None,
     297            0 :             delayed_reconcile_rx,
     298            0 :         }
     299            0 :     }
     300              : 
     301            0 :     fn parts_mut(
     302            0 :         &mut self,
     303            0 :     ) -> (
     304            0 :         &mut Arc<HashMap<NodeId, Node>>,
     305            0 :         &mut BTreeMap<TenantShardId, TenantShard>,
     306            0 :         &mut Scheduler,
     307            0 :     ) {
     308            0 :         (&mut self.nodes, &mut self.tenants, &mut self.scheduler)
     309            0 :     }
     310              : 
     311              :     #[allow(clippy::type_complexity)]
     312            0 :     fn parts_mut_sk(
     313            0 :         &mut self,
     314            0 :     ) -> (
     315            0 :         &mut Arc<HashMap<NodeId, Node>>,
     316            0 :         &mut Arc<HashMap<NodeId, Safekeeper>>,
     317            0 :         &mut BTreeMap<TenantShardId, TenantShard>,
     318            0 :         &mut Scheduler,
     319            0 :     ) {
     320            0 :         (
     321            0 :             &mut self.nodes,
     322            0 :             &mut self.safekeepers,
     323            0 :             &mut self.tenants,
     324            0 :             &mut self.scheduler,
     325            0 :         )
     326            0 :     }
     327              : 
     328            0 :     fn get_leadership_status(&self) -> LeadershipStatus {
     329            0 :         self.leadership_status
     330            0 :     }
     331              : 
     332            0 :     fn step_down(&mut self) {
     333            0 :         self.leadership_status = LeadershipStatus::SteppedDown;
     334            0 :         metrics::update_leadership_status(self.leadership_status);
     335            0 :     }
     336              : 
     337            0 :     fn become_leader(&mut self) {
     338            0 :         self.leadership_status = LeadershipStatus::Leader;
     339            0 :         metrics::update_leadership_status(self.leadership_status);
     340            0 :     }
     341              : }
     342              : 
     343              : #[derive(Clone)]
     344              : pub struct Config {
     345              :     // All pageservers managed by one instance of this service must have
     346              :     // the same public key.  This JWT token will be used to authenticate
     347              :     // this service to the pageservers it manages.
     348              :     pub pageserver_jwt_token: Option<String>,
     349              : 
     350              :     // All safekeepers managed by one instance of this service must have
     351              :     // the same public key. This JWT token will be used to authenticate
     352              :     // this service to the safekeepers it manages.
     353              :     pub safekeeper_jwt_token: Option<String>,
     354              : 
     355              :     // This JWT token will be used to authenticate this service to the control plane.
     356              :     pub control_plane_jwt_token: Option<String>,
     357              : 
     358              :     // This JWT token will be used to authenticate with other storage controller instances
     359              :     pub peer_jwt_token: Option<String>,
     360              : 
     361              :     /// Where the compute hook should send notifications of pageserver attachment locations
     362              :     /// (this URL points to the control plane in prod). If this is None, the compute hook will
     363              :     /// assume it is running in a test environment and try to update neon_local.
     364              :     pub compute_hook_url: Option<String>,
     365              : 
     366              :     /// Prefix for storage API endpoints of the control plane. We use this prefix to compute
     367              :     /// URLs that we use to send pageserver and safekeeper attachment locations.
     368              :     /// If this is None, the compute hook will assume it is running in a test environment
     369              :     /// and try to invoke neon_local instead.
     370              :     ///
     371              :     /// For now, there is also `compute_hook_url` which allows configuration of the pageserver
     372              :     /// specific endpoint, but it is in the process of being phased out.
     373              :     pub control_plane_url: Option<String>,
     374              : 
     375              :     /// Grace period within which a pageserver does not respond to heartbeats, but is still
     376              :     /// considered active. Once the grace period elapses, the next heartbeat failure will
     377              :     /// mark the pagseserver offline.
     378              :     pub max_offline_interval: Duration,
     379              : 
     380              :     /// Extended grace period within which pageserver may not respond to heartbeats.
     381              :     /// This extended grace period kicks in after the node has been drained for restart
     382              :     /// and/or upon handling the re-attach request from a node.
     383              :     pub max_warming_up_interval: Duration,
     384              : 
     385              :     /// How many normal-priority Reconcilers may be spawned concurrently
     386              :     pub reconciler_concurrency: usize,
     387              : 
     388              :     /// How many high-priority Reconcilers may be spawned concurrently
     389              :     pub priority_reconciler_concurrency: usize,
     390              : 
     391              :     /// How many API requests per second to allow per tenant, across all
     392              :     /// tenant-scoped API endpoints. Further API requests queue until ready.
     393              :     pub tenant_rate_limit: NonZeroU32,
     394              : 
     395              :     /// The size at which an unsharded tenant should be split (into 8 shards). This uses the logical
     396              :     /// size of the largest timeline in the shard (i.e. max_logical_size).
     397              :     ///
     398              :     /// None or 0 disables auto-splitting.
     399              :     ///
     400              :     /// TODO: consider using total logical size of all timelines instead.
     401              :     pub split_threshold: Option<u64>,
     402              : 
     403              :     // TODO: make this cfg(feature  = "testing")
     404              :     pub neon_local_repo_dir: Option<PathBuf>,
     405              : 
     406              :     // Maximum acceptable download lag for the secondary location
     407              :     // while draining a node. If the secondary location is lagging
     408              :     // by more than the configured amount, then the secondary is not
     409              :     // upgraded to primary.
     410              :     pub max_secondary_lag_bytes: Option<u64>,
     411              : 
     412              :     pub heartbeat_interval: Duration,
     413              : 
     414              :     pub address_for_peers: Option<Uri>,
     415              : 
     416              :     pub start_as_candidate: bool,
     417              : 
     418              :     pub http_service_port: i32,
     419              : 
     420              :     pub long_reconcile_threshold: Duration,
     421              : 
     422              :     pub use_https_pageserver_api: bool,
     423              : 
     424              :     pub use_https_safekeeper_api: bool,
     425              : 
     426              :     pub ssl_ca_cert: Option<Certificate>,
     427              : 
     428              :     pub timelines_onto_safekeepers: bool,
     429              : }
     430              : 
     431              : impl From<DatabaseError> for ApiError {
     432            0 :     fn from(err: DatabaseError) -> ApiError {
     433            0 :         match err {
     434            0 :             DatabaseError::Query(e) => ApiError::InternalServerError(e.into()),
     435              :             // FIXME: ApiError doesn't have an Unavailable variant, but ShuttingDown maps to 503.
     436              :             DatabaseError::Connection(_) | DatabaseError::ConnectionPool(_) => {
     437            0 :                 ApiError::ShuttingDown
     438              :             }
     439            0 :             DatabaseError::Logical(reason) | DatabaseError::Migration(reason) => {
     440            0 :                 ApiError::InternalServerError(anyhow::anyhow!(reason))
     441              :             }
     442              :         }
     443            0 :     }
     444              : }
     445              : 
     446              : enum InitialShardScheduleOutcome {
     447              :     Scheduled(TenantCreateResponseShard),
     448              :     NotScheduled,
     449              :     ShardScheduleError(ScheduleError),
     450              : }
     451              : 
     452              : pub struct Service {
     453              :     inner: Arc<std::sync::RwLock<ServiceState>>,
     454              :     config: Config,
     455              :     persistence: Arc<Persistence>,
     456              :     compute_hook: Arc<ComputeHook>,
     457              :     result_tx: tokio::sync::mpsc::UnboundedSender<ReconcileResultRequest>,
     458              : 
     459              :     heartbeater_ps: Heartbeater<Node, PageserverState>,
     460              :     heartbeater_sk: Heartbeater<Safekeeper, SafekeeperState>,
     461              : 
     462              :     // Channel for background cleanup from failed operations that require cleanup, such as shard split
     463              :     abort_tx: tokio::sync::mpsc::UnboundedSender<TenantShardSplitAbort>,
     464              : 
     465              :     // Locking on a tenant granularity (covers all shards in the tenant):
     466              :     // - Take exclusively for rare operations that mutate the tenant's persistent state (e.g. create/delete/split)
     467              :     // - Take in shared mode for operations that need the set of shards to stay the same to complete reliably (e.g. timeline CRUD)
     468              :     tenant_op_locks: IdLockMap<TenantId, TenantOperations>,
     469              : 
     470              :     // Locking for node-mutating operations: take exclusively for operations that modify the node's persistent state, or
     471              :     // that transition it to/from Active.
     472              :     node_op_locks: IdLockMap<NodeId, NodeOperations>,
     473              : 
     474              :     // Limit how many Reconcilers we will spawn concurrently for normal-priority tasks such as background reconciliations
     475              :     // and reconciliation on startup.
     476              :     reconciler_concurrency: Arc<tokio::sync::Semaphore>,
     477              : 
     478              :     // Limit how many Reconcilers we will spawn concurrently for high-priority tasks such as tenant/timeline CRUD, which
     479              :     // a human user might be waiting for.
     480              :     priority_reconciler_concurrency: Arc<tokio::sync::Semaphore>,
     481              : 
     482              :     /// Queue of tenants who are waiting for concurrency limits to permit them to reconcile
     483              :     /// Send into this queue to promptly attempt to reconcile this shard next time units are available.
     484              :     ///
     485              :     /// Note that this state logically lives inside ServiceState, but carrying Sender here makes the code simpler
     486              :     /// by avoiding needing a &mut ref to something inside the ServiceState.  This could be optimized to
     487              :     /// use a VecDeque instead of a channel to reduce synchronization overhead, at the cost of some code complexity.
     488              :     delayed_reconcile_tx: tokio::sync::mpsc::Sender<TenantShardId>,
     489              : 
     490              :     // Process shutdown will fire this token
     491              :     cancel: CancellationToken,
     492              : 
     493              :     // Child token of [`Service::cancel`] used by reconcilers
     494              :     reconcilers_cancel: CancellationToken,
     495              : 
     496              :     // Background tasks will hold this gate
     497              :     gate: Gate,
     498              : 
     499              :     // Reconcilers background tasks will hold this gate
     500              :     reconcilers_gate: Gate,
     501              : 
     502              :     /// This waits for initial reconciliation with pageservers to complete.  Until this barrier
     503              :     /// passes, it isn't safe to do any actions that mutate tenants.
     504              :     pub(crate) startup_complete: Barrier,
     505              : }
     506              : 
     507              : impl From<ReconcileWaitError> for ApiError {
     508            0 :     fn from(value: ReconcileWaitError) -> Self {
     509            0 :         match value {
     510            0 :             ReconcileWaitError::Shutdown => ApiError::ShuttingDown,
     511            0 :             e @ ReconcileWaitError::Timeout(_) => ApiError::Timeout(format!("{e}").into()),
     512            0 :             e @ ReconcileWaitError::Failed(..) => ApiError::InternalServerError(anyhow::anyhow!(e)),
     513              :         }
     514            0 :     }
     515              : }
     516              : 
     517              : impl From<OperationError> for ApiError {
     518            0 :     fn from(value: OperationError) -> Self {
     519            0 :         match value {
     520            0 :             OperationError::NodeStateChanged(err) | OperationError::FinalizeError(err) => {
     521            0 :                 ApiError::InternalServerError(anyhow::anyhow!(err))
     522              :             }
     523            0 :             OperationError::Cancelled => ApiError::Conflict("Operation was cancelled".into()),
     524              :         }
     525            0 :     }
     526              : }
     527              : 
     528              : #[allow(clippy::large_enum_variant)]
     529              : enum TenantCreateOrUpdate {
     530              :     Create(TenantCreateRequest),
     531              :     Update(Vec<ShardUpdate>),
     532              : }
     533              : 
     534              : struct ShardSplitParams {
     535              :     old_shard_count: ShardCount,
     536              :     new_shard_count: ShardCount,
     537              :     new_stripe_size: Option<ShardStripeSize>,
     538              :     targets: Vec<ShardSplitTarget>,
     539              :     policy: PlacementPolicy,
     540              :     config: TenantConfig,
     541              :     shard_ident: ShardIdentity,
     542              :     preferred_az_id: Option<AvailabilityZone>,
     543              : }
     544              : 
     545              : // When preparing for a shard split, we may either choose to proceed with the split,
     546              : // or find that the work is already done and return NoOp.
     547              : enum ShardSplitAction {
     548              :     Split(Box<ShardSplitParams>),
     549              :     NoOp(TenantShardSplitResponse),
     550              : }
     551              : 
     552              : // A parent shard which will be split
     553              : struct ShardSplitTarget {
     554              :     parent_id: TenantShardId,
     555              :     node: Node,
     556              :     child_ids: Vec<TenantShardId>,
     557              : }
     558              : 
     559              : /// When we tenant shard split operation fails, we may not be able to clean up immediately, because nodes
     560              : /// might not be available.  We therefore use a queue of abort operations processed in the background.
     561              : struct TenantShardSplitAbort {
     562              :     tenant_id: TenantId,
     563              :     /// The target values from the request that failed
     564              :     new_shard_count: ShardCount,
     565              :     new_stripe_size: Option<ShardStripeSize>,
     566              :     /// Until this abort op is complete, no other operations may be done on the tenant
     567              :     _tenant_lock: TracingExclusiveGuard<TenantOperations>,
     568              : }
     569              : 
     570              : #[derive(thiserror::Error, Debug)]
     571              : enum TenantShardSplitAbortError {
     572              :     #[error(transparent)]
     573              :     Database(#[from] DatabaseError),
     574              :     #[error(transparent)]
     575              :     Remote(#[from] mgmt_api::Error),
     576              :     #[error("Unavailable")]
     577              :     Unavailable,
     578              : }
     579              : 
     580              : struct ShardUpdate {
     581              :     tenant_shard_id: TenantShardId,
     582              :     placement_policy: PlacementPolicy,
     583              :     tenant_config: TenantConfig,
     584              : 
     585              :     /// If this is None, generation is not updated.
     586              :     generation: Option<Generation>,
     587              : 
     588              :     /// If this is None, scheduling policy is not updated.
     589              :     scheduling_policy: Option<ShardSchedulingPolicy>,
     590              : }
     591              : 
     592              : enum StopReconciliationsReason {
     593              :     ShuttingDown,
     594              :     SteppingDown,
     595              : }
     596              : 
     597              : impl std::fmt::Display for StopReconciliationsReason {
     598            0 :     fn fmt(&self, writer: &mut std::fmt::Formatter) -> std::fmt::Result {
     599            0 :         let s = match self {
     600            0 :             Self::ShuttingDown => "Shutting down",
     601            0 :             Self::SteppingDown => "Stepping down",
     602              :         };
     603            0 :         write!(writer, "{}", s)
     604            0 :     }
     605              : }
     606              : 
     607              : pub(crate) enum ReconcileResultRequest {
     608              :     ReconcileResult(ReconcileResult),
     609              :     Stop,
     610              : }
     611              : 
     612              : #[derive(Clone)]
     613              : struct MutationLocation {
     614              :     node: Node,
     615              :     generation: Generation,
     616              : }
     617              : 
     618              : #[derive(Clone)]
     619              : struct ShardMutationLocations {
     620              :     latest: MutationLocation,
     621              :     other: Vec<MutationLocation>,
     622              : }
     623              : 
     624              : #[derive(Default, Clone)]
     625              : struct TenantMutationLocations(BTreeMap<TenantShardId, ShardMutationLocations>);
     626              : 
     627              : impl Service {
     628            0 :     pub fn get_config(&self) -> &Config {
     629            0 :         &self.config
     630            0 :     }
     631              : 
     632              :     /// Called once on startup, this function attempts to contact all pageservers to build an up-to-date
     633              :     /// view of the world, and determine which pageservers are responsive.
     634              :     #[instrument(skip_all)]
     635              :     async fn startup_reconcile(
     636              :         self: &Arc<Service>,
     637              :         current_leader: Option<ControllerPersistence>,
     638              :         leader_step_down_state: Option<GlobalObservedState>,
     639              :         bg_compute_notify_result_tx: tokio::sync::mpsc::Sender<
     640              :             Result<(), (TenantShardId, NotifyError)>,
     641              :         >,
     642              :     ) {
     643              :         // Startup reconciliation does I/O to other services: whether they
     644              :         // are responsive or not, we should aim to finish within our deadline, because:
     645              :         // - If we don't, a k8s readiness hook watching /ready will kill us.
     646              :         // - While we're waiting for startup reconciliation, we are not fully
     647              :         //   available for end user operations like creating/deleting tenants and timelines.
     648              :         //
     649              :         // We set multiple deadlines to break up the time available between the phases of work: this is
     650              :         // arbitrary, but avoids a situation where the first phase could burn our entire timeout period.
     651              :         let start_at = Instant::now();
     652              :         let node_scan_deadline = start_at
     653              :             .checked_add(STARTUP_RECONCILE_TIMEOUT / 2)
     654              :             .expect("Reconcile timeout is a modest constant");
     655              : 
     656              :         let observed = if let Some(state) = leader_step_down_state {
     657              :             tracing::info!(
     658              :                 "Using observed state received from leader at {}",
     659              :                 current_leader.as_ref().unwrap().address
     660              :             );
     661              : 
     662              :             state
     663              :         } else {
     664              :             self.build_global_observed_state(node_scan_deadline).await
     665              :         };
     666              : 
     667              :         // Accumulate a list of any tenant locations that ought to be detached
     668              :         let mut cleanup = Vec::new();
     669              : 
     670              :         // Send initial heartbeat requests to all nodes loaded from the database
     671              :         let all_nodes = {
     672              :             let locked = self.inner.read().unwrap();
     673              :             locked.nodes.clone()
     674              :         };
     675              :         let (mut nodes_online, mut sks_online) =
     676              :             self.initial_heartbeat_round(all_nodes.keys()).await;
     677              : 
     678              :         // List of tenants for which we will attempt to notify compute of their location at startup
     679              :         let mut compute_notifications = Vec::new();
     680              : 
     681              :         // Populate intent and observed states for all tenants, based on reported state on pageservers
     682              :         tracing::info!("Populating tenant shards' states from initial pageserver scan...");
     683              :         let shard_count = {
     684              :             let mut locked = self.inner.write().unwrap();
     685              :             let (nodes, safekeepers, tenants, scheduler) = locked.parts_mut_sk();
     686              : 
     687              :             // Mark nodes online if they responded to us: nodes are offline by default after a restart.
     688              :             let mut new_nodes = (**nodes).clone();
     689              :             for (node_id, node) in new_nodes.iter_mut() {
     690              :                 if let Some(utilization) = nodes_online.remove(node_id) {
     691              :                     node.set_availability(NodeAvailability::Active(utilization));
     692              :                     scheduler.node_upsert(node);
     693              :                 }
     694              :             }
     695              :             *nodes = Arc::new(new_nodes);
     696              : 
     697              :             let mut new_sks = (**safekeepers).clone();
     698              :             for (node_id, node) in new_sks.iter_mut() {
     699              :                 if let Some((utilization, last_seen_at)) = sks_online.remove(node_id) {
     700              :                     node.set_availability(SafekeeperState::Available {
     701              :                         utilization,
     702              :                         last_seen_at,
     703              :                     });
     704              :                 }
     705              :             }
     706              :             *safekeepers = Arc::new(new_sks);
     707              : 
     708              :             for (tenant_shard_id, observed_state) in observed.0 {
     709              :                 let Some(tenant_shard) = tenants.get_mut(&tenant_shard_id) else {
     710              :                     for node_id in observed_state.locations.keys() {
     711              :                         cleanup.push((tenant_shard_id, *node_id));
     712              :                     }
     713              : 
     714              :                     continue;
     715              :                 };
     716              : 
     717              :                 tenant_shard.observed = observed_state;
     718              :             }
     719              : 
     720              :             // Populate each tenant's intent state
     721              :             let mut schedule_context = ScheduleContext::default();
     722              :             for (tenant_shard_id, tenant_shard) in tenants.iter_mut() {
     723              :                 if tenant_shard_id.shard_number == ShardNumber(0) {
     724              :                     // Reset scheduling context each time we advance to the next Tenant
     725              :                     schedule_context = ScheduleContext::default();
     726              :                 }
     727              : 
     728              :                 tenant_shard.intent_from_observed(scheduler);
     729              :                 if let Err(e) = tenant_shard.schedule(scheduler, &mut schedule_context) {
     730              :                     // Non-fatal error: we are unable to properly schedule the tenant, perhaps because
     731              :                     // not enough pageservers are available.  The tenant may well still be available
     732              :                     // to clients.
     733              :                     tracing::error!("Failed to schedule tenant {tenant_shard_id} at startup: {e}");
     734              :                 } else {
     735              :                     // If we're both intending and observed to be attached at a particular node, we will
     736              :                     // emit a compute notification for this. In the case where our observed state does not
     737              :                     // yet match our intent, we will eventually reconcile, and that will emit a compute notification.
     738              :                     if let Some(attached_at) = tenant_shard.stably_attached() {
     739              :                         compute_notifications.push(compute_hook::ShardUpdate {
     740              :                             tenant_shard_id: *tenant_shard_id,
     741              :                             node_id: attached_at,
     742              :                             stripe_size: tenant_shard.shard.stripe_size,
     743              :                             preferred_az: tenant_shard
     744              :                                 .preferred_az()
     745            0 :                                 .map(|az| Cow::Owned(az.clone())),
     746              :                         });
     747              :                     }
     748              :                 }
     749              :             }
     750              : 
     751              :             tenants.len()
     752              :         };
     753              : 
     754              :         // Before making any obeservable changes to the cluster, persist self
     755              :         // as leader in database and memory.
     756              :         let leadership = Leadership::new(
     757              :             self.persistence.clone(),
     758              :             self.config.clone(),
     759              :             self.cancel.child_token(),
     760              :         );
     761              : 
     762              :         if let Err(e) = leadership.become_leader(current_leader).await {
     763              :             tracing::error!("Failed to persist self as leader: {e}. Aborting start-up ...");
     764              :             std::process::exit(1);
     765              :         }
     766              : 
     767              :         let safekeepers = self.inner.read().unwrap().safekeepers.clone();
     768              :         let sk_schedule_requests =
     769              :             match safekeeper_reconciler::load_schedule_requests(self, &safekeepers).await {
     770              :                 Ok(v) => v,
     771              :                 Err(e) => {
     772              :                     tracing::warn!(
     773              :                         "Failed to load safekeeper pending ops at startup: {e}." // Don't abort for now: " Aborting start-up..."
     774              :                     );
     775              :                     // std::process::exit(1);
     776              :                     Vec::new()
     777              :                 }
     778              :             };
     779              : 
     780              :         {
     781              :             let mut locked = self.inner.write().unwrap();
     782              :             locked.become_leader();
     783              : 
     784              :             locked
     785              :                 .safekeeper_reconcilers
     786              :                 .schedule_request_vec(self, sk_schedule_requests);
     787              :         }
     788              : 
     789              :         // TODO: if any tenant's intent now differs from its loaded generation_pageserver, we should clear that
     790              :         // generation_pageserver in the database.
     791              : 
     792              :         // Emit compute hook notifications for all tenants which are already stably attached.  Other tenants
     793              :         // will emit compute hook notifications when they reconcile.
     794              :         //
     795              :         // Ordering: our calls to notify_background synchronously establish a relative order for these notifications vs. any later
     796              :         // calls into the ComputeHook for the same tenant: we can leave these to run to completion in the background and any later
     797              :         // calls will be correctly ordered wrt these.
     798              :         //
     799              :         // Concurrency: we call notify_background for all tenants, which will create O(N) tokio tasks, but almost all of them
     800              :         // will just wait on the ComputeHook::API_CONCURRENCY semaphore immediately, so very cheap until they get that semaphore
     801              :         // unit and start doing I/O.
     802              :         tracing::info!(
     803              :             "Sending {} compute notifications",
     804              :             compute_notifications.len()
     805              :         );
     806              :         self.compute_hook.notify_background(
     807              :             compute_notifications,
     808              :             bg_compute_notify_result_tx.clone(),
     809              :             &self.cancel,
     810              :         );
     811              : 
     812              :         // Finally, now that the service is up and running, launch reconcile operations for any tenants
     813              :         // which require it: under normal circumstances this should only include tenants that were in some
     814              :         // transient state before we restarted, or any tenants whose compute hooks failed above.
     815              :         tracing::info!("Checking for shards in need of reconciliation...");
     816              :         let reconcile_tasks = self.reconcile_all();
     817              :         // We will not wait for these reconciliation tasks to run here: we're now done with startup and
     818              :         // normal operations may proceed.
     819              : 
     820              :         // Clean up any tenants that were found on pageservers but are not known to us.  Do this in the
     821              :         // background because it does not need to complete in order to proceed with other work.
     822              :         if !cleanup.is_empty() {
     823              :             tracing::info!("Cleaning up {} locations in the background", cleanup.len());
     824              :             tokio::task::spawn({
     825              :                 let cleanup_self = self.clone();
     826            0 :                 async move { cleanup_self.cleanup_locations(cleanup).await }
     827              :             });
     828              :         }
     829              : 
     830              :         tracing::info!(
     831              :             "Startup complete, spawned {reconcile_tasks} reconciliation tasks ({shard_count} shards total)"
     832              :         );
     833              :     }
     834              : 
     835            0 :     async fn initial_heartbeat_round<'a>(
     836            0 :         &self,
     837            0 :         node_ids: impl Iterator<Item = &'a NodeId>,
     838            0 :     ) -> (
     839            0 :         HashMap<NodeId, PageserverUtilization>,
     840            0 :         HashMap<NodeId, (SafekeeperUtilization, Instant)>,
     841            0 :     ) {
     842            0 :         assert!(!self.startup_complete.is_ready());
     843              : 
     844            0 :         let all_nodes = {
     845            0 :             let locked = self.inner.read().unwrap();
     846            0 :             locked.nodes.clone()
     847            0 :         };
     848            0 : 
     849            0 :         let mut nodes_to_heartbeat = HashMap::new();
     850            0 :         for node_id in node_ids {
     851            0 :             match all_nodes.get(node_id) {
     852            0 :                 Some(node) => {
     853            0 :                     nodes_to_heartbeat.insert(*node_id, node.clone());
     854            0 :                 }
     855              :                 None => {
     856            0 :                     tracing::warn!("Node {node_id} was removed during start-up");
     857              :                 }
     858              :             }
     859              :         }
     860              : 
     861            0 :         let all_sks = {
     862            0 :             let locked = self.inner.read().unwrap();
     863            0 :             locked.safekeepers.clone()
     864            0 :         };
     865            0 : 
     866            0 :         tracing::info!("Sending initial heartbeats...");
     867              :         // Put a small, but reasonable timeout to get the initial heartbeats of the safekeepers to avoid a storage controller downtime
     868              :         const SK_TIMEOUT: Duration = Duration::from_secs(5);
     869            0 :         let (res_ps, res_sk) = tokio::join!(
     870            0 :             self.heartbeater_ps.heartbeat(Arc::new(nodes_to_heartbeat)),
     871            0 :             tokio::time::timeout(SK_TIMEOUT, self.heartbeater_sk.heartbeat(all_sks))
     872            0 :         );
     873              : 
     874            0 :         let mut online_nodes = HashMap::new();
     875            0 :         if let Ok(deltas) = res_ps {
     876            0 :             for (node_id, status) in deltas.0 {
     877            0 :                 match status {
     878            0 :                     PageserverState::Available { utilization, .. } => {
     879            0 :                         online_nodes.insert(node_id, utilization);
     880            0 :                     }
     881            0 :                     PageserverState::Offline => {}
     882              :                     PageserverState::WarmingUp { .. } => {
     883            0 :                         unreachable!("Nodes are never marked warming-up during startup reconcile")
     884              :                     }
     885              :                 }
     886              :             }
     887            0 :         }
     888              : 
     889            0 :         let mut online_sks = HashMap::new();
     890            0 :         if let Ok(Ok(deltas)) = res_sk {
     891            0 :             for (node_id, status) in deltas.0 {
     892            0 :                 match status {
     893              :                     SafekeeperState::Available {
     894            0 :                         utilization,
     895            0 :                         last_seen_at,
     896            0 :                     } => {
     897            0 :                         online_sks.insert(node_id, (utilization, last_seen_at));
     898            0 :                     }
     899            0 :                     SafekeeperState::Offline => {}
     900              :                 }
     901              :             }
     902            0 :         }
     903              : 
     904            0 :         (online_nodes, online_sks)
     905            0 :     }
     906              : 
     907              :     /// Used during [`Self::startup_reconcile`]: issue GETs to all nodes concurrently, with a deadline.
     908              :     ///
     909              :     /// The result includes only nodes which responded within the deadline
     910            0 :     async fn scan_node_locations(
     911            0 :         &self,
     912            0 :         deadline: Instant,
     913            0 :     ) -> HashMap<NodeId, LocationConfigListResponse> {
     914            0 :         let nodes = {
     915            0 :             let locked = self.inner.read().unwrap();
     916            0 :             locked.nodes.clone()
     917            0 :         };
     918            0 : 
     919            0 :         let mut node_results = HashMap::new();
     920            0 : 
     921            0 :         let mut node_list_futs = FuturesUnordered::new();
     922            0 : 
     923            0 :         tracing::info!("Scanning shards on {} nodes...", nodes.len());
     924            0 :         for node in nodes.values() {
     925            0 :             node_list_futs.push({
     926            0 :                 async move {
     927            0 :                     tracing::info!("Scanning shards on node {node}...");
     928            0 :                     let timeout = Duration::from_secs(5);
     929            0 :                     let response = node
     930            0 :                         .with_client_retries(
     931            0 :                             |client| async move { client.list_location_config().await },
     932            0 :                             &self.config.pageserver_jwt_token,
     933            0 :                             &self.config.ssl_ca_cert,
     934            0 :                             1,
     935            0 :                             5,
     936            0 :                             timeout,
     937            0 :                             &self.cancel,
     938            0 :                         )
     939            0 :                         .await;
     940            0 :                     (node.get_id(), response)
     941            0 :                 }
     942            0 :             });
     943            0 :         }
     944              : 
     945              :         loop {
     946            0 :             let (node_id, result) = tokio::select! {
     947            0 :                 next = node_list_futs.next() => {
     948            0 :                     match next {
     949            0 :                         Some(result) => result,
     950              :                         None =>{
     951              :                             // We got results for all our nodes
     952            0 :                             break;
     953              :                         }
     954              : 
     955              :                     }
     956              :                 },
     957            0 :                 _ = tokio::time::sleep(deadline.duration_since(Instant::now())) => {
     958              :                     // Give up waiting for anyone who hasn't responded: we will yield the results that we have
     959            0 :                     tracing::info!("Reached deadline while waiting for nodes to respond to location listing requests");
     960            0 :                     break;
     961              :                 }
     962              :             };
     963              : 
     964            0 :             let Some(list_response) = result else {
     965            0 :                 tracing::info!("Shutdown during startup_reconcile");
     966            0 :                 break;
     967              :             };
     968              : 
     969            0 :             match list_response {
     970            0 :                 Err(e) => {
     971            0 :                     tracing::warn!("Could not scan node {} ({e})", node_id);
     972              :                 }
     973            0 :                 Ok(listing) => {
     974            0 :                     node_results.insert(node_id, listing);
     975            0 :                 }
     976              :             }
     977              :         }
     978              : 
     979            0 :         node_results
     980            0 :     }
     981              : 
     982            0 :     async fn build_global_observed_state(&self, deadline: Instant) -> GlobalObservedState {
     983            0 :         let node_listings = self.scan_node_locations(deadline).await;
     984            0 :         let mut observed = GlobalObservedState::default();
     985              : 
     986            0 :         for (node_id, location_confs) in node_listings {
     987            0 :             tracing::info!(
     988            0 :                 "Received {} shard statuses from pageserver {}",
     989            0 :                 location_confs.tenant_shards.len(),
     990              :                 node_id
     991              :             );
     992              : 
     993            0 :             for (tid, location_conf) in location_confs.tenant_shards {
     994            0 :                 let entry = observed.0.entry(tid).or_default();
     995            0 :                 entry.locations.insert(
     996            0 :                     node_id,
     997            0 :                     ObservedStateLocation {
     998            0 :                         conf: location_conf,
     999            0 :                     },
    1000            0 :                 );
    1001            0 :             }
    1002              :         }
    1003              : 
    1004            0 :         observed
    1005            0 :     }
    1006              : 
    1007              :     /// Used during [`Self::startup_reconcile`]: detach a list of unknown-to-us tenants from pageservers.
    1008              :     ///
    1009              :     /// This is safe to run in the background, because if we don't have this TenantShardId in our map of
    1010              :     /// tenants, then it is probably something incompletely deleted before: we will not fight with any
    1011              :     /// other task trying to attach it.
    1012              :     #[instrument(skip_all)]
    1013              :     async fn cleanup_locations(&self, cleanup: Vec<(TenantShardId, NodeId)>) {
    1014              :         let nodes = self.inner.read().unwrap().nodes.clone();
    1015              : 
    1016              :         for (tenant_shard_id, node_id) in cleanup {
    1017              :             // A node reported a tenant_shard_id which is unknown to us: detach it.
    1018              :             let Some(node) = nodes.get(&node_id) else {
    1019              :                 // This is legitimate; we run in the background and [`Self::startup_reconcile`] might have identified
    1020              :                 // a location to clean up on a node that has since been removed.
    1021              :                 tracing::info!(
    1022              :                     "Not cleaning up location {node_id}/{tenant_shard_id}: node not found"
    1023              :                 );
    1024              :                 continue;
    1025              :             };
    1026              : 
    1027              :             if self.cancel.is_cancelled() {
    1028              :                 break;
    1029              :             }
    1030              : 
    1031              :             let client = match PageserverClient::new(
    1032              :                 node.get_id(),
    1033              :                 node.base_url(),
    1034              :                 self.config.pageserver_jwt_token.as_deref(),
    1035              :                 self.config.ssl_ca_cert.clone(),
    1036              :             ) {
    1037              :                 Ok(client) => client,
    1038              :                 Err(e) => {
    1039              :                     tracing::error!(
    1040              :                         "Failed to create client to detach unknown shard {tenant_shard_id} on pageserver {node_id}: {e}"
    1041              :                     );
    1042              :                     continue;
    1043              :                 }
    1044              :             };
    1045              :             match client
    1046              :                 .location_config(
    1047              :                     tenant_shard_id,
    1048              :                     LocationConfig {
    1049              :                         mode: LocationConfigMode::Detached,
    1050              :                         generation: None,
    1051              :                         secondary_conf: None,
    1052              :                         shard_number: tenant_shard_id.shard_number.0,
    1053              :                         shard_count: tenant_shard_id.shard_count.literal(),
    1054              :                         shard_stripe_size: 0,
    1055              :                         tenant_conf: models::TenantConfig::default(),
    1056              :                     },
    1057              :                     None,
    1058              :                     false,
    1059              :                 )
    1060              :                 .await
    1061              :             {
    1062              :                 Ok(()) => {
    1063              :                     tracing::info!(
    1064              :                         "Detached unknown shard {tenant_shard_id} on pageserver {node_id}"
    1065              :                     );
    1066              :                 }
    1067              :                 Err(e) => {
    1068              :                     // Non-fatal error: leaving a tenant shard behind that we are not managing shouldn't
    1069              :                     // break anything.
    1070              :                     tracing::error!(
    1071              :                         "Failed to detach unknown shard {tenant_shard_id} on pageserver {node_id}: {e}"
    1072              :                     );
    1073              :                 }
    1074              :             }
    1075              :         }
    1076              :     }
    1077              : 
    1078              :     /// Long running background task that periodically wakes up and looks for shards that need
    1079              :     /// reconciliation.  Reconciliation is fallible, so any reconciliation tasks that fail during
    1080              :     /// e.g. a tenant create/attach/migrate must eventually be retried: this task is responsible
    1081              :     /// for those retries.
    1082              :     #[instrument(skip_all)]
    1083              :     async fn background_reconcile(self: &Arc<Self>) {
    1084              :         self.startup_complete.clone().wait().await;
    1085              : 
    1086              :         const BACKGROUND_RECONCILE_PERIOD: Duration = Duration::from_secs(20);
    1087              :         let mut interval = tokio::time::interval(BACKGROUND_RECONCILE_PERIOD);
    1088              :         while !self.reconcilers_cancel.is_cancelled() {
    1089              :             tokio::select! {
    1090              :               _ = interval.tick() => {
    1091              :                 let reconciles_spawned = self.reconcile_all();
    1092              :                 if reconciles_spawned == 0 {
    1093              :                     // Run optimizer only when we didn't find any other work to do
    1094              :                     self.optimize_all().await;
    1095              :                 }
    1096              :                 // Always attempt autosplits. Sharding is crucial for bulk ingest performance, so we
    1097              :                 // must be responsive when new projects begin ingesting and reach the threshold.
    1098              :                 self.autosplit_tenants().await;
    1099              :             }
    1100              :               _ = self.reconcilers_cancel.cancelled() => return
    1101              :             }
    1102              :         }
    1103              :     }
    1104              :     /// Heartbeat all storage nodes once in a while.
    1105              :     #[instrument(skip_all)]
    1106              :     async fn spawn_heartbeat_driver(&self) {
    1107              :         self.startup_complete.clone().wait().await;
    1108              : 
    1109              :         let mut interval = tokio::time::interval(self.config.heartbeat_interval);
    1110              :         while !self.cancel.is_cancelled() {
    1111              :             tokio::select! {
    1112              :               _ = interval.tick() => { }
    1113              :               _ = self.cancel.cancelled() => return
    1114              :             };
    1115              : 
    1116              :             let nodes = {
    1117              :                 let locked = self.inner.read().unwrap();
    1118              :                 locked.nodes.clone()
    1119              :             };
    1120              : 
    1121              :             let safekeepers = {
    1122              :                 let locked = self.inner.read().unwrap();
    1123              :                 locked.safekeepers.clone()
    1124              :             };
    1125              : 
    1126              :             const SK_TIMEOUT: Duration = Duration::from_secs(3);
    1127              :             let (res_ps, res_sk) = tokio::join!(
    1128              :                 self.heartbeater_ps.heartbeat(nodes),
    1129              :                 tokio::time::timeout(SK_TIMEOUT, self.heartbeater_sk.heartbeat(safekeepers))
    1130              :             );
    1131              : 
    1132              :             if let Ok(deltas) = res_ps {
    1133              :                 let mut to_handle = Vec::default();
    1134              : 
    1135              :                 for (node_id, state) in deltas.0 {
    1136              :                     let new_availability = match state {
    1137              :                         PageserverState::Available { utilization, .. } => {
    1138              :                             NodeAvailability::Active(utilization)
    1139              :                         }
    1140              :                         PageserverState::WarmingUp { started_at } => {
    1141              :                             NodeAvailability::WarmingUp(started_at)
    1142              :                         }
    1143              :                         PageserverState::Offline => {
    1144              :                             // The node might have been placed in the WarmingUp state
    1145              :                             // while the heartbeat round was on-going. Hence, filter out
    1146              :                             // offline transitions for WarmingUp nodes that are still within
    1147              :                             // their grace period.
    1148              :                             if let Ok(NodeAvailability::WarmingUp(started_at)) = self
    1149              :                                 .get_node(node_id)
    1150              :                                 .await
    1151              :                                 .as_ref()
    1152            0 :                                 .map(|n| n.get_availability())
    1153              :                             {
    1154              :                                 let now = Instant::now();
    1155              :                                 if now - *started_at >= self.config.max_warming_up_interval {
    1156              :                                     NodeAvailability::Offline
    1157              :                                 } else {
    1158              :                                     NodeAvailability::WarmingUp(*started_at)
    1159              :                                 }
    1160              :                             } else {
    1161              :                                 NodeAvailability::Offline
    1162              :                             }
    1163              :                         }
    1164              :                     };
    1165              : 
    1166              :                     let node_lock = trace_exclusive_lock(
    1167              :                         &self.node_op_locks,
    1168              :                         node_id,
    1169              :                         NodeOperations::Configure,
    1170              :                     )
    1171              :                     .await;
    1172              : 
    1173              :                     pausable_failpoint!("heartbeat-pre-node-state-configure");
    1174              : 
    1175              :                     // This is the code path for geniune availability transitions (i.e node
    1176              :                     // goes unavailable and/or comes back online).
    1177              :                     let res = self
    1178              :                         .node_state_configure(node_id, Some(new_availability), None, &node_lock)
    1179              :                         .await;
    1180              : 
    1181              :                     match res {
    1182              :                         Ok(transition) => {
    1183              :                             // Keep hold of the lock until the availability transitions
    1184              :                             // have been handled in
    1185              :                             // [`Service::handle_node_availability_transitions`] in order avoid
    1186              :                             // racing with [`Service::external_node_configure`].
    1187              :                             to_handle.push((node_id, node_lock, transition));
    1188              :                         }
    1189              :                         Err(ApiError::NotFound(_)) => {
    1190              :                             // This should be rare, but legitimate since the heartbeats are done
    1191              :                             // on a snapshot of the nodes.
    1192              :                             tracing::info!("Node {} was not found after heartbeat round", node_id);
    1193              :                         }
    1194              :                         Err(ApiError::ShuttingDown) => {
    1195              :                             // No-op: we're shutting down, no need to try and update any nodes' statuses
    1196              :                         }
    1197              :                         Err(err) => {
    1198              :                             // Transition to active involves reconciling: if a node responds to a heartbeat then
    1199              :                             // becomes unavailable again, we may get an error here.
    1200              :                             tracing::error!(
    1201              :                                 "Failed to update node state {} after heartbeat round: {}",
    1202              :                                 node_id,
    1203              :                                 err
    1204              :                             );
    1205              :                         }
    1206              :                     }
    1207              :                 }
    1208              : 
    1209              :                 // We collected all the transitions above and now we handle them.
    1210              :                 let res = self.handle_node_availability_transitions(to_handle).await;
    1211              :                 if let Err(errs) = res {
    1212              :                     for (node_id, err) in errs {
    1213              :                         match err {
    1214              :                             ApiError::NotFound(_) => {
    1215              :                                 // This should be rare, but legitimate since the heartbeats are done
    1216              :                                 // on a snapshot of the nodes.
    1217              :                                 tracing::info!(
    1218              :                                     "Node {} was not found after heartbeat round",
    1219              :                                     node_id
    1220              :                                 );
    1221              :                             }
    1222              :                             err => {
    1223              :                                 tracing::error!(
    1224              :                                     "Failed to handle availability transition for {} after heartbeat round: {}",
    1225              :                                     node_id,
    1226              :                                     err
    1227              :                                 );
    1228              :                             }
    1229              :                         }
    1230              :                     }
    1231              :                 }
    1232              :             }
    1233              :             if let Ok(Ok(deltas)) = res_sk {
    1234              :                 let mut locked = self.inner.write().unwrap();
    1235              :                 let mut safekeepers = (*locked.safekeepers).clone();
    1236              :                 for (id, state) in deltas.0 {
    1237              :                     let Some(sk) = safekeepers.get_mut(&id) else {
    1238              :                         tracing::info!(
    1239              :                             "Couldn't update safekeeper safekeeper state for id {id} from heartbeat={state:?}"
    1240              :                         );
    1241              :                         continue;
    1242              :                     };
    1243              :                     sk.set_availability(state);
    1244              :                 }
    1245              :                 locked.safekeepers = Arc::new(safekeepers);
    1246              :             }
    1247              :         }
    1248              :     }
    1249              : 
    1250              :     /// Apply the contents of a [`ReconcileResult`] to our in-memory state: if the reconciliation
    1251              :     /// was successful and intent hasn't changed since the Reconciler was spawned, this will update
    1252              :     /// the observed state of the tenant such that subsequent calls to [`TenantShard::get_reconcile_needed`]
    1253              :     /// will indicate that reconciliation is not needed.
    1254              :     #[instrument(skip_all, fields(
    1255              :         seq=%result.sequence,
    1256              :         tenant_id=%result.tenant_shard_id.tenant_id,
    1257              :         shard_id=%result.tenant_shard_id.shard_slug(),
    1258              :     ))]
    1259              :     fn process_result(&self, result: ReconcileResult) {
    1260              :         let mut locked = self.inner.write().unwrap();
    1261              :         let (nodes, tenants, _scheduler) = locked.parts_mut();
    1262              :         let Some(tenant) = tenants.get_mut(&result.tenant_shard_id) else {
    1263              :             // A reconciliation result might race with removing a tenant: drop results for
    1264              :             // tenants that aren't in our map.
    1265              :             return;
    1266              :         };
    1267              : 
    1268              :         // Usually generation should only be updated via this path, so the max() isn't
    1269              :         // needed, but it is used to handle out-of-band updates via. e.g. test hook.
    1270              :         tenant.generation = std::cmp::max(tenant.generation, result.generation);
    1271              : 
    1272              :         // If the reconciler signals that it failed to notify compute, set this state on
    1273              :         // the shard so that a future [`TenantShard::maybe_reconcile`] will try again.
    1274              :         tenant.pending_compute_notification = result.pending_compute_notification;
    1275              : 
    1276              :         // Let the TenantShard know it is idle.
    1277              :         tenant.reconcile_complete(result.sequence);
    1278              : 
    1279              :         // In case a node was deleted while this reconcile is in flight, filter it out of the update we will
    1280              :         // make to the tenant
    1281            0 :         let deltas = result.observed_deltas.into_iter().flat_map(|delta| {
    1282              :             // In case a node was deleted while this reconcile is in flight, filter it out of the update we will
    1283              :             // make to the tenant
    1284            0 :             let node = nodes.get(delta.node_id())?;
    1285              : 
    1286            0 :             if node.is_available() {
    1287            0 :                 return Some(delta);
    1288            0 :             }
    1289            0 : 
    1290            0 :             // In case a node became unavailable concurrently with the reconcile, observed
    1291            0 :             // locations on it are now uncertain. By convention, set them to None in order
    1292            0 :             // for them to get refreshed when the node comes back online.
    1293            0 :             Some(ObservedStateDelta::Upsert(Box::new((
    1294            0 :                 node.get_id(),
    1295            0 :                 ObservedStateLocation { conf: None },
    1296            0 :             ))))
    1297            0 :         });
    1298              : 
    1299              :         match result.result {
    1300              :             Ok(()) => {
    1301              :                 tenant.apply_observed_deltas(deltas);
    1302              :                 tenant.waiter.advance(result.sequence);
    1303              :             }
    1304              :             Err(e) => {
    1305              :                 match e {
    1306              :                     ReconcileError::Cancel => {
    1307              :                         tracing::info!("Reconciler was cancelled");
    1308              :                     }
    1309              :                     ReconcileError::Remote(mgmt_api::Error::Cancelled) => {
    1310              :                         // This might be due to the reconciler getting cancelled, or it might
    1311              :                         // be due to the `Node` being marked offline.
    1312              :                         tracing::info!("Reconciler cancelled during pageserver API call");
    1313              :                     }
    1314              :                     _ => {
    1315              :                         tracing::warn!("Reconcile error: {}", e);
    1316              :                     }
    1317              :                 }
    1318              : 
    1319              :                 // Ordering: populate last_error before advancing error_seq,
    1320              :                 // so that waiters will see the correct error after waiting.
    1321              :                 tenant.set_last_error(result.sequence, e);
    1322              : 
    1323              :                 // Skip deletions on reconcile failures
    1324              :                 let upsert_deltas =
    1325            0 :                     deltas.filter(|delta| matches!(delta, ObservedStateDelta::Upsert(_)));
    1326              :                 tenant.apply_observed_deltas(upsert_deltas);
    1327              :             }
    1328              :         }
    1329              : 
    1330              :         // If we just finished detaching all shards for a tenant, it might be time to drop it from memory.
    1331              :         if tenant.policy == PlacementPolicy::Detached {
    1332              :             // We may only drop a tenant from memory while holding the exclusive lock on the tenant ID: this protects us
    1333              :             // from concurrent execution wrt a request handler that might expect the tenant to remain in memory for the
    1334              :             // duration of the request.
    1335              :             let guard = self.tenant_op_locks.try_exclusive(
    1336              :                 tenant.tenant_shard_id.tenant_id,
    1337              :                 TenantOperations::DropDetached,
    1338              :             );
    1339              :             if let Some(guard) = guard {
    1340              :                 self.maybe_drop_tenant(tenant.tenant_shard_id.tenant_id, &mut locked, &guard);
    1341              :             }
    1342              :         }
    1343              : 
    1344              :         // Maybe some other work can proceed now that this job finished.
    1345              :         //
    1346              :         // Only bother with this if we have some semaphore units available in the normal-priority semaphore (these
    1347              :         // reconciles are scheduled at `[ReconcilerPriority::Normal]`).
    1348              :         if self.reconciler_concurrency.available_permits() > 0 {
    1349              :             while let Ok(tenant_shard_id) = locked.delayed_reconcile_rx.try_recv() {
    1350              :                 let (nodes, tenants, _scheduler) = locked.parts_mut();
    1351              :                 if let Some(shard) = tenants.get_mut(&tenant_shard_id) {
    1352              :                     shard.delayed_reconcile = false;
    1353              :                     self.maybe_reconcile_shard(shard, nodes, ReconcilerPriority::Normal);
    1354              :                 }
    1355              : 
    1356              :                 if self.reconciler_concurrency.available_permits() == 0 {
    1357              :                     break;
    1358              :                 }
    1359              :             }
    1360              :         }
    1361              :     }
    1362              : 
    1363            0 :     async fn process_results(
    1364            0 :         &self,
    1365            0 :         mut result_rx: tokio::sync::mpsc::UnboundedReceiver<ReconcileResultRequest>,
    1366            0 :         mut bg_compute_hook_result_rx: tokio::sync::mpsc::Receiver<
    1367            0 :             Result<(), (TenantShardId, NotifyError)>,
    1368            0 :         >,
    1369            0 :     ) {
    1370              :         loop {
    1371              :             // Wait for the next result, or for cancellation
    1372            0 :             tokio::select! {
    1373            0 :                 r = result_rx.recv() => {
    1374            0 :                     match r {
    1375            0 :                         Some(ReconcileResultRequest::ReconcileResult(result)) => {self.process_result(result);},
    1376            0 :                         None | Some(ReconcileResultRequest::Stop) => {break;}
    1377              :                     }
    1378              :                 }
    1379            0 :                 _ = async{
    1380            0 :                     match bg_compute_hook_result_rx.recv().await {
    1381            0 :                         Some(result) => {
    1382            0 :                             if let Err((tenant_shard_id, notify_error)) = result {
    1383            0 :                                 tracing::warn!("Marking shard {tenant_shard_id} for notification retry, due to error {notify_error}");
    1384            0 :                                 let mut locked = self.inner.write().unwrap();
    1385            0 :                                 if let Some(shard) = locked.tenants.get_mut(&tenant_shard_id) {
    1386            0 :                                     shard.pending_compute_notification = true;
    1387            0 :                                 }
    1388              : 
    1389            0 :                             }
    1390              :                         },
    1391              :                         None => {
    1392              :                             // This channel is dead, but we don't want to terminate the outer loop{}: just wait for shutdown
    1393            0 :                             self.cancel.cancelled().await;
    1394              :                         }
    1395              :                     }
    1396            0 :                 } => {},
    1397            0 :                 _ = self.cancel.cancelled() => {
    1398            0 :                     break;
    1399              :                 }
    1400              :             };
    1401              :         }
    1402            0 :     }
    1403              : 
    1404            0 :     async fn process_aborts(
    1405            0 :         &self,
    1406            0 :         mut abort_rx: tokio::sync::mpsc::UnboundedReceiver<TenantShardSplitAbort>,
    1407            0 :     ) {
    1408              :         loop {
    1409              :             // Wait for the next result, or for cancellation
    1410            0 :             let op = tokio::select! {
    1411            0 :                 r = abort_rx.recv() => {
    1412            0 :                     match r {
    1413            0 :                         Some(op) => {op},
    1414            0 :                         None => {break;}
    1415              :                     }
    1416              :                 }
    1417            0 :                 _ = self.cancel.cancelled() => {
    1418            0 :                     break;
    1419              :                 }
    1420              :             };
    1421              : 
    1422              :             // Retry until shutdown: we must keep this request object alive until it is properly
    1423              :             // processed, as it holds a lock guard that prevents other operations trying to do things
    1424              :             // to the tenant while it is in a weird part-split state.
    1425            0 :             while !self.cancel.is_cancelled() {
    1426            0 :                 match self.abort_tenant_shard_split(&op).await {
    1427            0 :                     Ok(_) => break,
    1428            0 :                     Err(e) => {
    1429            0 :                         tracing::warn!(
    1430            0 :                             "Failed to abort shard split on {}, will retry: {e}",
    1431              :                             op.tenant_id
    1432              :                         );
    1433              : 
    1434              :                         // If a node is unavailable, we hope that it has been properly marked Offline
    1435              :                         // when we retry, so that the abort op will succeed.  If the abort op is failing
    1436              :                         // for some other reason, we will keep retrying forever, or until a human notices
    1437              :                         // and does something about it (either fixing a pageserver or restarting the controller).
    1438            0 :                         tokio::time::timeout(Duration::from_secs(5), self.cancel.cancelled())
    1439            0 :                             .await
    1440            0 :                             .ok();
    1441              :                     }
    1442              :                 }
    1443              :             }
    1444              :         }
    1445            0 :     }
    1446              : 
    1447            0 :     pub async fn spawn(config: Config, persistence: Arc<Persistence>) -> anyhow::Result<Arc<Self>> {
    1448            0 :         let (result_tx, result_rx) = tokio::sync::mpsc::unbounded_channel();
    1449            0 :         let (abort_tx, abort_rx) = tokio::sync::mpsc::unbounded_channel();
    1450            0 : 
    1451            0 :         let leadership_cancel = CancellationToken::new();
    1452            0 :         let leadership = Leadership::new(persistence.clone(), config.clone(), leadership_cancel);
    1453            0 :         let (leader, leader_step_down_state) = leadership.step_down_current_leader().await?;
    1454              : 
    1455              :         // Apply the migrations **after** the current leader has stepped down
    1456              :         // (or we've given up waiting for it), but **before** reading from the
    1457              :         // database. The only exception is reading the current leader before
    1458              :         // migrating.
    1459            0 :         persistence.migration_run().await?;
    1460              : 
    1461            0 :         tracing::info!("Loading nodes from database...");
    1462            0 :         let nodes = persistence
    1463            0 :             .list_nodes()
    1464            0 :             .await?
    1465            0 :             .into_iter()
    1466            0 :             .map(|x| Node::from_persistent(x, config.use_https_pageserver_api))
    1467            0 :             .collect::<anyhow::Result<Vec<Node>>>()?;
    1468            0 :         let nodes: HashMap<NodeId, Node> = nodes.into_iter().map(|n| (n.get_id(), n)).collect();
    1469            0 :         tracing::info!("Loaded {} nodes from database.", nodes.len());
    1470            0 :         metrics::METRICS_REGISTRY
    1471            0 :             .metrics_group
    1472            0 :             .storage_controller_pageserver_nodes
    1473            0 :             .set(nodes.len() as i64);
    1474            0 : 
    1475            0 :         tracing::info!("Loading safekeepers from database...");
    1476            0 :         let safekeepers = persistence
    1477            0 :             .list_safekeepers()
    1478            0 :             .await?
    1479            0 :             .into_iter()
    1480            0 :             .map(|skp| {
    1481            0 :                 Safekeeper::from_persistence(
    1482            0 :                     skp,
    1483            0 :                     CancellationToken::new(),
    1484            0 :                     config.use_https_safekeeper_api,
    1485            0 :                 )
    1486            0 :             })
    1487            0 :             .collect::<anyhow::Result<Vec<_>>>()?;
    1488            0 :         let safekeepers: HashMap<NodeId, Safekeeper> =
    1489            0 :             safekeepers.into_iter().map(|n| (n.get_id(), n)).collect();
    1490            0 :         tracing::info!("Loaded {} safekeepers from database.", safekeepers.len());
    1491              : 
    1492            0 :         tracing::info!("Loading shards from database...");
    1493            0 :         let mut tenant_shard_persistence = persistence.load_active_tenant_shards().await?;
    1494            0 :         tracing::info!(
    1495            0 :             "Loaded {} shards from database.",
    1496            0 :             tenant_shard_persistence.len()
    1497              :         );
    1498              : 
    1499              :         // If any shard splits were in progress, reset the database state to abort them
    1500            0 :         let mut tenant_shard_count_min_max: HashMap<TenantId, (ShardCount, ShardCount)> =
    1501            0 :             HashMap::new();
    1502            0 :         for tsp in &mut tenant_shard_persistence {
    1503            0 :             let shard = tsp.get_shard_identity()?;
    1504            0 :             let tenant_shard_id = tsp.get_tenant_shard_id()?;
    1505            0 :             let entry = tenant_shard_count_min_max
    1506            0 :                 .entry(tenant_shard_id.tenant_id)
    1507            0 :                 .or_insert_with(|| (shard.count, shard.count));
    1508            0 :             entry.0 = std::cmp::min(entry.0, shard.count);
    1509            0 :             entry.1 = std::cmp::max(entry.1, shard.count);
    1510            0 :         }
    1511              : 
    1512            0 :         for (tenant_id, (count_min, count_max)) in tenant_shard_count_min_max {
    1513            0 :             if count_min != count_max {
    1514              :                 // Aborting the split in the database and dropping the child shards is sufficient: the reconciliation in
    1515              :                 // [`Self::startup_reconcile`] will implicitly drop the child shards on remote pageservers, or they'll
    1516              :                 // be dropped later in [`Self::node_activate_reconcile`] if it isn't available right now.
    1517            0 :                 tracing::info!("Aborting shard split {tenant_id} {count_min:?} -> {count_max:?}");
    1518            0 :                 let abort_status = persistence.abort_shard_split(tenant_id, count_max).await?;
    1519              : 
    1520              :                 // We may never see the Complete status here: if the split was complete, we wouldn't have
    1521              :                 // identified this tenant has having mismatching min/max counts.
    1522            0 :                 assert!(matches!(abort_status, AbortShardSplitStatus::Aborted));
    1523              : 
    1524              :                 // Clear the splitting status in-memory, to reflect that we just aborted in the database
    1525            0 :                 tenant_shard_persistence.iter_mut().for_each(|tsp| {
    1526            0 :                     // Set idle split state on those shards that we will retain.
    1527            0 :                     let tsp_tenant_id = TenantId::from_str(tsp.tenant_id.as_str()).unwrap();
    1528            0 :                     if tsp_tenant_id == tenant_id
    1529            0 :                         && tsp.get_shard_identity().unwrap().count == count_min
    1530            0 :                     {
    1531            0 :                         tsp.splitting = SplitState::Idle;
    1532            0 :                     } else if tsp_tenant_id == tenant_id {
    1533              :                         // Leave the splitting state on the child shards: this will be used next to
    1534              :                         // drop them.
    1535            0 :                         tracing::info!(
    1536            0 :                             "Shard {tsp_tenant_id} will be dropped after shard split abort",
    1537              :                         );
    1538            0 :                     }
    1539            0 :                 });
    1540            0 : 
    1541            0 :                 // Drop shards for this tenant which we didn't just mark idle (i.e. child shards of the aborted split)
    1542            0 :                 tenant_shard_persistence.retain(|tsp| {
    1543            0 :                     TenantId::from_str(tsp.tenant_id.as_str()).unwrap() != tenant_id
    1544            0 :                         || tsp.splitting == SplitState::Idle
    1545            0 :                 });
    1546            0 :             }
    1547              :         }
    1548              : 
    1549            0 :         let mut tenants = BTreeMap::new();
    1550            0 : 
    1551            0 :         let mut scheduler = Scheduler::new(nodes.values());
    1552              : 
    1553              :         #[cfg(feature = "testing")]
    1554              :         {
    1555              :             use pageserver_api::controller_api::AvailabilityZone;
    1556              : 
    1557              :             // Hack: insert scheduler state for all nodes referenced by shards, as compatibility
    1558              :             // tests only store the shards, not the nodes.  The nodes will be loaded shortly
    1559              :             // after when pageservers start up and register.
    1560            0 :             let mut node_ids = HashSet::new();
    1561            0 :             for tsp in &tenant_shard_persistence {
    1562            0 :                 if let Some(node_id) = tsp.generation_pageserver {
    1563            0 :                     node_ids.insert(node_id);
    1564            0 :                 }
    1565              :             }
    1566            0 :             for node_id in node_ids {
    1567            0 :                 tracing::info!("Creating node {} in scheduler for tests", node_id);
    1568            0 :                 let node = Node::new(
    1569            0 :                     NodeId(node_id as u64),
    1570            0 :                     "".to_string(),
    1571            0 :                     123,
    1572            0 :                     None,
    1573            0 :                     "".to_string(),
    1574            0 :                     123,
    1575            0 :                     AvailabilityZone("test_az".to_string()),
    1576            0 :                     false,
    1577            0 :                 )
    1578            0 :                 .unwrap();
    1579            0 : 
    1580            0 :                 scheduler.node_upsert(&node);
    1581              :             }
    1582              :         }
    1583            0 :         for tsp in tenant_shard_persistence {
    1584            0 :             let tenant_shard_id = tsp.get_tenant_shard_id()?;
    1585              : 
    1586              :             // We will populate intent properly later in [`Self::startup_reconcile`], initially populate
    1587              :             // it with what we can infer: the node for which a generation was most recently issued.
    1588            0 :             let mut intent = IntentState::new(
    1589            0 :                 tsp.preferred_az_id
    1590            0 :                     .as_ref()
    1591            0 :                     .map(|az| AvailabilityZone(az.clone())),
    1592            0 :             );
    1593            0 :             if let Some(generation_pageserver) = tsp.generation_pageserver.map(|n| NodeId(n as u64))
    1594              :             {
    1595            0 :                 if nodes.contains_key(&generation_pageserver) {
    1596            0 :                     intent.set_attached(&mut scheduler, Some(generation_pageserver));
    1597            0 :                 } else {
    1598              :                     // If a node was removed before being completely drained, it is legal for it to leave behind a `generation_pageserver` referring
    1599              :                     // to a non-existent node, because node deletion doesn't block on completing the reconciliations that will issue new generations
    1600              :                     // on different pageservers.
    1601            0 :                     tracing::warn!(
    1602            0 :                         "Tenant shard {tenant_shard_id} references non-existent node {generation_pageserver} in database, will be rescheduled"
    1603              :                     );
    1604              :                 }
    1605            0 :             }
    1606            0 :             let new_tenant = TenantShard::from_persistent(tsp, intent)?;
    1607              : 
    1608            0 :             tenants.insert(tenant_shard_id, new_tenant);
    1609              :         }
    1610              : 
    1611            0 :         let (startup_completion, startup_complete) = utils::completion::channel();
    1612            0 : 
    1613            0 :         // This channel is continuously consumed by process_results, so doesn't need to be very large.
    1614            0 :         let (bg_compute_notify_result_tx, bg_compute_notify_result_rx) =
    1615            0 :             tokio::sync::mpsc::channel(512);
    1616            0 : 
    1617            0 :         let (delayed_reconcile_tx, delayed_reconcile_rx) =
    1618            0 :             tokio::sync::mpsc::channel(MAX_DELAYED_RECONCILES);
    1619            0 : 
    1620            0 :         let cancel = CancellationToken::new();
    1621            0 :         let reconcilers_cancel = cancel.child_token();
    1622            0 : 
    1623            0 :         let heartbeater_ps = Heartbeater::new(
    1624            0 :             config.pageserver_jwt_token.clone(),
    1625            0 :             config.ssl_ca_cert.clone(),
    1626            0 :             config.max_offline_interval,
    1627            0 :             config.max_warming_up_interval,
    1628            0 :             cancel.clone(),
    1629            0 :         );
    1630            0 : 
    1631            0 :         let heartbeater_sk = Heartbeater::new(
    1632            0 :             config.safekeeper_jwt_token.clone(),
    1633            0 :             config.ssl_ca_cert.clone(),
    1634            0 :             config.max_offline_interval,
    1635            0 :             config.max_warming_up_interval,
    1636            0 :             cancel.clone(),
    1637            0 :         );
    1638              : 
    1639            0 :         let initial_leadership_status = if config.start_as_candidate {
    1640            0 :             LeadershipStatus::Candidate
    1641              :         } else {
    1642            0 :             LeadershipStatus::Leader
    1643              :         };
    1644              : 
    1645            0 :         let this = Arc::new(Self {
    1646            0 :             inner: Arc::new(std::sync::RwLock::new(ServiceState::new(
    1647            0 :                 nodes,
    1648            0 :                 safekeepers,
    1649            0 :                 tenants,
    1650            0 :                 scheduler,
    1651            0 :                 delayed_reconcile_rx,
    1652            0 :                 initial_leadership_status,
    1653            0 :                 reconcilers_cancel.clone(),
    1654            0 :             ))),
    1655            0 :             config: config.clone(),
    1656            0 :             persistence,
    1657            0 :             compute_hook: Arc::new(ComputeHook::new(config.clone())),
    1658            0 :             result_tx,
    1659            0 :             heartbeater_ps,
    1660            0 :             heartbeater_sk,
    1661            0 :             reconciler_concurrency: Arc::new(tokio::sync::Semaphore::new(
    1662            0 :                 config.reconciler_concurrency,
    1663            0 :             )),
    1664            0 :             priority_reconciler_concurrency: Arc::new(tokio::sync::Semaphore::new(
    1665            0 :                 config.priority_reconciler_concurrency,
    1666            0 :             )),
    1667            0 :             delayed_reconcile_tx,
    1668            0 :             abort_tx,
    1669            0 :             startup_complete: startup_complete.clone(),
    1670            0 :             cancel,
    1671            0 :             reconcilers_cancel,
    1672            0 :             gate: Gate::default(),
    1673            0 :             reconcilers_gate: Gate::default(),
    1674            0 :             tenant_op_locks: Default::default(),
    1675            0 :             node_op_locks: Default::default(),
    1676            0 :         });
    1677            0 : 
    1678            0 :         let result_task_this = this.clone();
    1679            0 :         tokio::task::spawn(async move {
    1680              :             // Block shutdown until we're done (we must respect self.cancel)
    1681            0 :             if let Ok(_gate) = result_task_this.gate.enter() {
    1682            0 :                 result_task_this
    1683            0 :                     .process_results(result_rx, bg_compute_notify_result_rx)
    1684            0 :                     .await
    1685            0 :             }
    1686            0 :         });
    1687            0 : 
    1688            0 :         tokio::task::spawn({
    1689            0 :             let this = this.clone();
    1690            0 :             async move {
    1691              :                 // Block shutdown until we're done (we must respect self.cancel)
    1692            0 :                 if let Ok(_gate) = this.gate.enter() {
    1693            0 :                     this.process_aborts(abort_rx).await
    1694            0 :                 }
    1695            0 :             }
    1696            0 :         });
    1697            0 : 
    1698            0 :         tokio::task::spawn({
    1699            0 :             let this = this.clone();
    1700            0 :             async move {
    1701            0 :                 if let Ok(_gate) = this.gate.enter() {
    1702              :                     loop {
    1703            0 :                         tokio::select! {
    1704            0 :                             _ = this.cancel.cancelled() => {
    1705            0 :                                 break;
    1706              :                             },
    1707            0 :                             _ = tokio::time::sleep(Duration::from_secs(60)) => {}
    1708            0 :                         };
    1709            0 :                         this.tenant_op_locks.housekeeping();
    1710              :                     }
    1711            0 :                 }
    1712            0 :             }
    1713            0 :         });
    1714            0 : 
    1715            0 :         tokio::task::spawn({
    1716            0 :             let this = this.clone();
    1717            0 :             // We will block the [`Service::startup_complete`] barrier until [`Self::startup_reconcile`]
    1718            0 :             // is done.
    1719            0 :             let startup_completion = startup_completion.clone();
    1720            0 :             async move {
    1721              :                 // Block shutdown until we're done (we must respect self.cancel)
    1722            0 :                 let Ok(_gate) = this.gate.enter() else {
    1723            0 :                     return;
    1724              :                 };
    1725              : 
    1726            0 :                 this.startup_reconcile(leader, leader_step_down_state, bg_compute_notify_result_tx)
    1727            0 :                     .await;
    1728              : 
    1729            0 :                 drop(startup_completion);
    1730            0 :             }
    1731            0 :         });
    1732            0 : 
    1733            0 :         tokio::task::spawn({
    1734            0 :             let this = this.clone();
    1735            0 :             let startup_complete = startup_complete.clone();
    1736            0 :             async move {
    1737            0 :                 startup_complete.wait().await;
    1738            0 :                 this.background_reconcile().await;
    1739            0 :             }
    1740            0 :         });
    1741            0 : 
    1742            0 :         tokio::task::spawn({
    1743            0 :             let this = this.clone();
    1744            0 :             let startup_complete = startup_complete.clone();
    1745            0 :             async move {
    1746            0 :                 startup_complete.wait().await;
    1747            0 :                 this.spawn_heartbeat_driver().await;
    1748            0 :             }
    1749            0 :         });
    1750            0 : 
    1751            0 :         Ok(this)
    1752            0 :     }
    1753              : 
    1754            0 :     pub(crate) async fn attach_hook(
    1755            0 :         &self,
    1756            0 :         attach_req: AttachHookRequest,
    1757            0 :     ) -> anyhow::Result<AttachHookResponse> {
    1758            0 :         let _tenant_lock = trace_exclusive_lock(
    1759            0 :             &self.tenant_op_locks,
    1760            0 :             attach_req.tenant_shard_id.tenant_id,
    1761            0 :             TenantOperations::AttachHook,
    1762            0 :         )
    1763            0 :         .await;
    1764              : 
    1765              :         // This is a test hook.  To enable using it on tenants that were created directly with
    1766              :         // the pageserver API (not via this service), we will auto-create any missing tenant
    1767              :         // shards with default state.
    1768            0 :         let insert = {
    1769            0 :             match self
    1770            0 :                 .maybe_load_tenant(attach_req.tenant_shard_id.tenant_id, &_tenant_lock)
    1771            0 :                 .await
    1772              :             {
    1773            0 :                 Ok(_) => false,
    1774            0 :                 Err(ApiError::NotFound(_)) => true,
    1775            0 :                 Err(e) => return Err(e.into()),
    1776              :             }
    1777              :         };
    1778              : 
    1779            0 :         if insert {
    1780            0 :             let tsp = TenantShardPersistence {
    1781            0 :                 tenant_id: attach_req.tenant_shard_id.tenant_id.to_string(),
    1782            0 :                 shard_number: attach_req.tenant_shard_id.shard_number.0 as i32,
    1783            0 :                 shard_count: attach_req.tenant_shard_id.shard_count.literal() as i32,
    1784            0 :                 shard_stripe_size: 0,
    1785            0 :                 generation: attach_req.generation_override.or(Some(0)),
    1786            0 :                 generation_pageserver: None,
    1787            0 :                 placement_policy: serde_json::to_string(&PlacementPolicy::Attached(0)).unwrap(),
    1788            0 :                 config: serde_json::to_string(&TenantConfig::default()).unwrap(),
    1789            0 :                 splitting: SplitState::default(),
    1790            0 :                 scheduling_policy: serde_json::to_string(&ShardSchedulingPolicy::default())
    1791            0 :                     .unwrap(),
    1792            0 :                 preferred_az_id: None,
    1793            0 :             };
    1794            0 : 
    1795            0 :             match self.persistence.insert_tenant_shards(vec![tsp]).await {
    1796            0 :                 Err(e) => match e {
    1797              :                     DatabaseError::Query(diesel::result::Error::DatabaseError(
    1798              :                         DatabaseErrorKind::UniqueViolation,
    1799              :                         _,
    1800              :                     )) => {
    1801            0 :                         tracing::info!(
    1802            0 :                             "Raced with another request to insert tenant {}",
    1803              :                             attach_req.tenant_shard_id
    1804              :                         )
    1805              :                     }
    1806            0 :                     _ => return Err(e.into()),
    1807              :                 },
    1808              :                 Ok(()) => {
    1809            0 :                     tracing::info!("Inserted shard {} in database", attach_req.tenant_shard_id);
    1810              : 
    1811            0 :                     let mut locked = self.inner.write().unwrap();
    1812            0 :                     locked.tenants.insert(
    1813            0 :                         attach_req.tenant_shard_id,
    1814            0 :                         TenantShard::new(
    1815            0 :                             attach_req.tenant_shard_id,
    1816            0 :                             ShardIdentity::unsharded(),
    1817            0 :                             PlacementPolicy::Attached(0),
    1818            0 :                             None,
    1819            0 :                         ),
    1820            0 :                     );
    1821            0 :                     tracing::info!("Inserted shard {} in memory", attach_req.tenant_shard_id);
    1822              :                 }
    1823              :             }
    1824            0 :         }
    1825              : 
    1826            0 :         let new_generation = if let Some(req_node_id) = attach_req.node_id {
    1827            0 :             let maybe_tenant_conf = {
    1828            0 :                 let locked = self.inner.write().unwrap();
    1829            0 :                 locked
    1830            0 :                     .tenants
    1831            0 :                     .get(&attach_req.tenant_shard_id)
    1832            0 :                     .map(|t| t.config.clone())
    1833            0 :             };
    1834            0 : 
    1835            0 :             match maybe_tenant_conf {
    1836            0 :                 Some(conf) => {
    1837            0 :                     let new_generation = self
    1838            0 :                         .persistence
    1839            0 :                         .increment_generation(attach_req.tenant_shard_id, req_node_id)
    1840            0 :                         .await?;
    1841              : 
    1842              :                     // Persist the placement policy update. This is required
    1843              :                     // when we reattaching a detached tenant.
    1844            0 :                     self.persistence
    1845            0 :                         .update_tenant_shard(
    1846            0 :                             TenantFilter::Shard(attach_req.tenant_shard_id),
    1847            0 :                             Some(PlacementPolicy::Attached(0)),
    1848            0 :                             Some(conf),
    1849            0 :                             None,
    1850            0 :                             None,
    1851            0 :                         )
    1852            0 :                         .await?;
    1853            0 :                     Some(new_generation)
    1854              :                 }
    1855              :                 None => {
    1856            0 :                     anyhow::bail!("Attach hook handling raced with tenant removal")
    1857              :                 }
    1858              :             }
    1859              :         } else {
    1860            0 :             self.persistence.detach(attach_req.tenant_shard_id).await?;
    1861            0 :             None
    1862              :         };
    1863              : 
    1864            0 :         let mut locked = self.inner.write().unwrap();
    1865            0 :         let (_nodes, tenants, scheduler) = locked.parts_mut();
    1866            0 : 
    1867            0 :         let tenant_shard = tenants
    1868            0 :             .get_mut(&attach_req.tenant_shard_id)
    1869            0 :             .expect("Checked for existence above");
    1870              : 
    1871            0 :         if let Some(new_generation) = new_generation {
    1872            0 :             tenant_shard.generation = Some(new_generation);
    1873            0 :             tenant_shard.policy = PlacementPolicy::Attached(0);
    1874            0 :         } else {
    1875              :             // This is a detach notification.  We must update placement policy to avoid re-attaching
    1876              :             // during background scheduling/reconciliation, or during storage controller restart.
    1877            0 :             assert!(attach_req.node_id.is_none());
    1878            0 :             tenant_shard.policy = PlacementPolicy::Detached;
    1879              :         }
    1880              : 
    1881            0 :         if let Some(attaching_pageserver) = attach_req.node_id.as_ref() {
    1882            0 :             tracing::info!(
    1883              :                 tenant_id = %attach_req.tenant_shard_id,
    1884              :                 ps_id = %attaching_pageserver,
    1885              :                 generation = ?tenant_shard.generation,
    1886            0 :                 "issuing",
    1887              :             );
    1888            0 :         } else if let Some(ps_id) = tenant_shard.intent.get_attached() {
    1889            0 :             tracing::info!(
    1890              :                 tenant_id = %attach_req.tenant_shard_id,
    1891              :                 %ps_id,
    1892              :                 generation = ?tenant_shard.generation,
    1893            0 :                 "dropping",
    1894              :             );
    1895              :         } else {
    1896            0 :             tracing::info!(
    1897              :             tenant_id = %attach_req.tenant_shard_id,
    1898            0 :             "no-op: tenant already has no pageserver");
    1899              :         }
    1900            0 :         tenant_shard
    1901            0 :             .intent
    1902            0 :             .set_attached(scheduler, attach_req.node_id);
    1903            0 : 
    1904            0 :         tracing::info!(
    1905            0 :             "attach_hook: tenant {} set generation {:?}, pageserver {}",
    1906            0 :             attach_req.tenant_shard_id,
    1907            0 :             tenant_shard.generation,
    1908            0 :             // TODO: this is an odd number of 0xf's
    1909            0 :             attach_req.node_id.unwrap_or(utils::id::NodeId(0xfffffff))
    1910              :         );
    1911              : 
    1912              :         // Trick the reconciler into not doing anything for this tenant: this helps
    1913              :         // tests that manually configure a tenant on the pagesrever, and then call this
    1914              :         // attach hook: they don't want background reconciliation to modify what they
    1915              :         // did to the pageserver.
    1916              :         #[cfg(feature = "testing")]
    1917              :         {
    1918            0 :             if let Some(node_id) = attach_req.node_id {
    1919            0 :                 tenant_shard.observed.locations = HashMap::from([(
    1920            0 :                     node_id,
    1921            0 :                     ObservedStateLocation {
    1922            0 :                         conf: Some(attached_location_conf(
    1923            0 :                             tenant_shard.generation.unwrap(),
    1924            0 :                             &tenant_shard.shard,
    1925            0 :                             &tenant_shard.config,
    1926            0 :                             &PlacementPolicy::Attached(0),
    1927            0 :                         )),
    1928            0 :                     },
    1929            0 :                 )]);
    1930            0 :             } else {
    1931            0 :                 tenant_shard.observed.locations.clear();
    1932            0 :             }
    1933              :         }
    1934              : 
    1935            0 :         Ok(AttachHookResponse {
    1936            0 :             generation: attach_req
    1937            0 :                 .node_id
    1938            0 :                 .map(|_| tenant_shard.generation.expect("Test hook, not used on tenants that are mid-onboarding with a NULL generation").into().unwrap()),
    1939            0 :         })
    1940            0 :     }
    1941              : 
    1942            0 :     pub(crate) fn inspect(&self, inspect_req: InspectRequest) -> InspectResponse {
    1943            0 :         let locked = self.inner.read().unwrap();
    1944            0 : 
    1945            0 :         let tenant_shard = locked.tenants.get(&inspect_req.tenant_shard_id);
    1946            0 : 
    1947            0 :         InspectResponse {
    1948            0 :             attachment: tenant_shard.and_then(|s| {
    1949            0 :                 s.intent
    1950            0 :                     .get_attached()
    1951            0 :                     .map(|ps| (s.generation.expect("Test hook, not used on tenants that are mid-onboarding with a NULL generation").into().unwrap(), ps))
    1952            0 :             }),
    1953            0 :         }
    1954            0 :     }
    1955              : 
    1956              :     // When the availability state of a node transitions to active, we must do a full reconciliation
    1957              :     // of LocationConfigs on that node.  This is because while a node was offline:
    1958              :     // - we might have proceeded through startup_reconcile without checking for extraneous LocationConfigs on this node
    1959              :     // - aborting a tenant shard split might have left rogue child shards behind on this node.
    1960              :     //
    1961              :     // This function must complete _before_ setting a `Node` to Active: once it is set to Active, other
    1962              :     // Reconcilers might communicate with the node, and these must not overlap with the work we do in
    1963              :     // this function.
    1964              :     //
    1965              :     // The reconciliation logic in here is very similar to what [`Self::startup_reconcile`] does, but
    1966              :     // for written for a single node rather than as a batch job for all nodes.
    1967              :     #[tracing::instrument(skip_all, fields(node_id=%node.get_id()))]
    1968              :     async fn node_activate_reconcile(
    1969              :         &self,
    1970              :         mut node: Node,
    1971              :         _lock: &TracingExclusiveGuard<NodeOperations>,
    1972              :     ) -> Result<(), ApiError> {
    1973              :         // This Node is a mutable local copy: we will set it active so that we can use its
    1974              :         // API client to reconcile with the node.  The Node in [`Self::nodes`] will get updated
    1975              :         // later.
    1976              :         node.set_availability(NodeAvailability::Active(PageserverUtilization::full()));
    1977              : 
    1978              :         let configs = match node
    1979              :             .with_client_retries(
    1980            0 :                 |client| async move { client.list_location_config().await },
    1981              :                 &self.config.pageserver_jwt_token,
    1982              :                 &self.config.ssl_ca_cert,
    1983              :                 1,
    1984              :                 5,
    1985              :                 SHORT_RECONCILE_TIMEOUT,
    1986              :                 &self.cancel,
    1987              :             )
    1988              :             .await
    1989              :         {
    1990              :             None => {
    1991              :                 // We're shutting down (the Node's cancellation token can't have fired, because
    1992              :                 // we're the only scope that has a reference to it, and we didn't fire it).
    1993              :                 return Err(ApiError::ShuttingDown);
    1994              :             }
    1995              :             Some(Err(e)) => {
    1996              :                 // This node didn't succeed listing its locations: it may not proceed to active state
    1997              :                 // as it is apparently unavailable.
    1998              :                 return Err(ApiError::PreconditionFailed(
    1999              :                     format!("Failed to query node location configs, cannot activate ({e})").into(),
    2000              :                 ));
    2001              :             }
    2002              :             Some(Ok(configs)) => configs,
    2003              :         };
    2004              :         tracing::info!("Loaded {} LocationConfigs", configs.tenant_shards.len());
    2005              : 
    2006              :         let mut cleanup = Vec::new();
    2007              :         {
    2008              :             let mut locked = self.inner.write().unwrap();
    2009              : 
    2010              :             for (tenant_shard_id, observed_loc) in configs.tenant_shards {
    2011              :                 let Some(tenant_shard) = locked.tenants.get_mut(&tenant_shard_id) else {
    2012              :                     cleanup.push(tenant_shard_id);
    2013              :                     continue;
    2014              :                 };
    2015              :                 tenant_shard
    2016              :                     .observed
    2017              :                     .locations
    2018              :                     .insert(node.get_id(), ObservedStateLocation { conf: observed_loc });
    2019              :             }
    2020              :         }
    2021              : 
    2022              :         for tenant_shard_id in cleanup {
    2023              :             tracing::info!("Detaching {tenant_shard_id}");
    2024              :             match node
    2025              :                 .with_client_retries(
    2026            0 :                     |client| async move {
    2027            0 :                         let config = LocationConfig {
    2028            0 :                             mode: LocationConfigMode::Detached,
    2029            0 :                             generation: None,
    2030            0 :                             secondary_conf: None,
    2031            0 :                             shard_number: tenant_shard_id.shard_number.0,
    2032            0 :                             shard_count: tenant_shard_id.shard_count.literal(),
    2033            0 :                             shard_stripe_size: 0,
    2034            0 :                             tenant_conf: models::TenantConfig::default(),
    2035            0 :                         };
    2036            0 :                         client
    2037            0 :                             .location_config(tenant_shard_id, config, None, false)
    2038            0 :                             .await
    2039            0 :                     },
    2040              :                     &self.config.pageserver_jwt_token,
    2041              :                     &self.config.ssl_ca_cert,
    2042              :                     1,
    2043              :                     5,
    2044              :                     SHORT_RECONCILE_TIMEOUT,
    2045              :                     &self.cancel,
    2046              :                 )
    2047              :                 .await
    2048              :             {
    2049              :                 None => {
    2050              :                     // We're shutting down (the Node's cancellation token can't have fired, because
    2051              :                     // we're the only scope that has a reference to it, and we didn't fire it).
    2052              :                     return Err(ApiError::ShuttingDown);
    2053              :                 }
    2054              :                 Some(Err(e)) => {
    2055              :                     // Do not let the node proceed to Active state if it is not responsive to requests
    2056              :                     // to detach.  This could happen if e.g. a shutdown bug in the pageserver is preventing
    2057              :                     // detach completing: we should not let this node back into the set of nodes considered
    2058              :                     // okay for scheduling.
    2059              :                     return Err(ApiError::Conflict(format!(
    2060              :                         "Node {node} failed to detach {tenant_shard_id}: {e}"
    2061              :                     )));
    2062              :                 }
    2063              :                 Some(Ok(_)) => {}
    2064              :             };
    2065              :         }
    2066              : 
    2067              :         Ok(())
    2068              :     }
    2069              : 
    2070            0 :     pub(crate) async fn re_attach(
    2071            0 :         &self,
    2072            0 :         reattach_req: ReAttachRequest,
    2073            0 :     ) -> Result<ReAttachResponse, ApiError> {
    2074            0 :         if let Some(register_req) = reattach_req.register {
    2075            0 :             self.node_register(register_req).await?;
    2076            0 :         }
    2077              : 
    2078              :         // Ordering: we must persist generation number updates before making them visible in the in-memory state
    2079            0 :         let incremented_generations = self.persistence.re_attach(reattach_req.node_id).await?;
    2080              : 
    2081            0 :         tracing::info!(
    2082              :             node_id=%reattach_req.node_id,
    2083            0 :             "Incremented {} tenant shards' generations",
    2084            0 :             incremented_generations.len()
    2085              :         );
    2086              : 
    2087              :         // Apply the updated generation to our in-memory state, and
    2088              :         // gather discover secondary locations.
    2089            0 :         let mut locked = self.inner.write().unwrap();
    2090            0 :         let (nodes, tenants, scheduler) = locked.parts_mut();
    2091            0 : 
    2092            0 :         let mut response = ReAttachResponse {
    2093            0 :             tenants: Vec::new(),
    2094            0 :         };
    2095              : 
    2096              :         // TODO: cancel/restart any running reconciliation for this tenant, it might be trying
    2097              :         // to call location_conf API with an old generation.  Wait for cancellation to complete
    2098              :         // before responding to this request.  Requires well implemented CancellationToken logic
    2099              :         // all the way to where we call location_conf.  Even then, there can still be a location_conf
    2100              :         // request in flight over the network: TODO handle that by making location_conf API refuse
    2101              :         // to go backward in generations.
    2102              : 
    2103              :         // Scan through all shards, applying updates for ones where we updated generation
    2104              :         // and identifying shards that intend to have a secondary location on this node.
    2105            0 :         for (tenant_shard_id, shard) in tenants {
    2106            0 :             if let Some(new_gen) = incremented_generations.get(tenant_shard_id) {
    2107            0 :                 let new_gen = *new_gen;
    2108            0 :                 response.tenants.push(ReAttachResponseTenant {
    2109            0 :                     id: *tenant_shard_id,
    2110            0 :                     r#gen: Some(new_gen.into().unwrap()),
    2111            0 :                     // A tenant is only put into multi or stale modes in the middle of a [`Reconciler::live_migrate`]
    2112            0 :                     // execution.  If a pageserver is restarted during that process, then the reconcile pass will
    2113            0 :                     // fail, and start from scratch, so it doesn't make sense for us to try and preserve
    2114            0 :                     // the stale/multi states at this point.
    2115            0 :                     mode: LocationConfigMode::AttachedSingle,
    2116            0 :                 });
    2117            0 : 
    2118            0 :                 shard.generation = std::cmp::max(shard.generation, Some(new_gen));
    2119            0 :                 if let Some(observed) = shard.observed.locations.get_mut(&reattach_req.node_id) {
    2120              :                     // Why can we update `observed` even though we're not sure our response will be received
    2121              :                     // by the pageserver?  Because the pageserver will not proceed with startup until
    2122              :                     // it has processed response: if it loses it, we'll see another request and increment
    2123              :                     // generation again, avoiding any uncertainty about dirtiness of tenant's state.
    2124            0 :                     if let Some(conf) = observed.conf.as_mut() {
    2125            0 :                         conf.generation = new_gen.into();
    2126            0 :                     }
    2127            0 :                 } else {
    2128            0 :                     // This node has no observed state for the shard: perhaps it was offline
    2129            0 :                     // when the pageserver restarted.  Insert a None, so that the Reconciler
    2130            0 :                     // will be prompted to learn the location's state before it makes changes.
    2131            0 :                     shard
    2132            0 :                         .observed
    2133            0 :                         .locations
    2134            0 :                         .insert(reattach_req.node_id, ObservedStateLocation { conf: None });
    2135            0 :                 }
    2136            0 :             } else if shard.intent.get_secondary().contains(&reattach_req.node_id) {
    2137            0 :                 // Ordering: pageserver will not accept /location_config requests until it has
    2138            0 :                 // finished processing the response from re-attach.  So we can update our in-memory state
    2139            0 :                 // now, and be confident that we are not stamping on the result of some later location config.
    2140            0 :                 // TODO: however, we are not strictly ordered wrt ReconcileResults queue,
    2141            0 :                 // so we might update observed state here, and then get over-written by some racing
    2142            0 :                 // ReconcileResult.  The impact is low however, since we have set state on pageserver something
    2143            0 :                 // that matches intent, so worst case if we race then we end up doing a spurious reconcile.
    2144            0 : 
    2145            0 :                 response.tenants.push(ReAttachResponseTenant {
    2146            0 :                     id: *tenant_shard_id,
    2147            0 :                     r#gen: None,
    2148            0 :                     mode: LocationConfigMode::Secondary,
    2149            0 :                 });
    2150            0 : 
    2151            0 :                 // We must not update observed, because we have no guarantee that our
    2152            0 :                 // response will be received by the pageserver. This could leave it
    2153            0 :                 // falsely dirty, but the resulting reconcile should be idempotent.
    2154            0 :             }
    2155              :         }
    2156              : 
    2157              :         // We consider a node Active once we have composed a re-attach response, but we
    2158              :         // do not call [`Self::node_activate_reconcile`]: the handling of the re-attach response
    2159              :         // implicitly synchronizes the LocationConfigs on the node.
    2160              :         //
    2161              :         // Setting a node active unblocks any Reconcilers that might write to the location config API,
    2162              :         // but those requests will not be accepted by the node until it has finished processing
    2163              :         // the re-attach response.
    2164              :         //
    2165              :         // Additionally, reset the nodes scheduling policy to match the conditional update done
    2166              :         // in [`Persistence::re_attach`].
    2167            0 :         if let Some(node) = nodes.get(&reattach_req.node_id) {
    2168            0 :             let reset_scheduling = matches!(
    2169            0 :                 node.get_scheduling(),
    2170              :                 NodeSchedulingPolicy::PauseForRestart
    2171              :                     | NodeSchedulingPolicy::Draining
    2172              :                     | NodeSchedulingPolicy::Filling
    2173              :             );
    2174              : 
    2175            0 :             let mut new_nodes = (**nodes).clone();
    2176            0 :             if let Some(node) = new_nodes.get_mut(&reattach_req.node_id) {
    2177            0 :                 if reset_scheduling {
    2178            0 :                     node.set_scheduling(NodeSchedulingPolicy::Active);
    2179            0 :                 }
    2180              : 
    2181            0 :                 tracing::info!("Marking {} warming-up on reattach", reattach_req.node_id);
    2182            0 :                 node.set_availability(NodeAvailability::WarmingUp(std::time::Instant::now()));
    2183            0 : 
    2184            0 :                 scheduler.node_upsert(node);
    2185            0 :                 let new_nodes = Arc::new(new_nodes);
    2186            0 :                 *nodes = new_nodes;
    2187              :             } else {
    2188            0 :                 tracing::error!(
    2189            0 :                     "Reattaching node {} was removed while processing the request",
    2190              :                     reattach_req.node_id
    2191              :                 );
    2192              :             }
    2193            0 :         }
    2194              : 
    2195            0 :         Ok(response)
    2196            0 :     }
    2197              : 
    2198            0 :     pub(crate) async fn validate(
    2199            0 :         &self,
    2200            0 :         validate_req: ValidateRequest,
    2201            0 :     ) -> Result<ValidateResponse, DatabaseError> {
    2202              :         // Fast in-memory check: we may reject validation on anything that doesn't match our
    2203              :         // in-memory generation for a shard
    2204            0 :         let in_memory_result = {
    2205            0 :             let mut in_memory_result = Vec::new();
    2206            0 :             let locked = self.inner.read().unwrap();
    2207            0 :             for req_tenant in validate_req.tenants {
    2208            0 :                 if let Some(tenant_shard) = locked.tenants.get(&req_tenant.id) {
    2209            0 :                     let valid = tenant_shard.generation == Some(Generation::new(req_tenant.r#gen));
    2210            0 :                     tracing::info!(
    2211            0 :                         "handle_validate: {}(gen {}): valid={valid} (latest {:?})",
    2212              :                         req_tenant.id,
    2213              :                         req_tenant.r#gen,
    2214              :                         tenant_shard.generation
    2215              :                     );
    2216              : 
    2217            0 :                     in_memory_result.push((
    2218            0 :                         req_tenant.id,
    2219            0 :                         Generation::new(req_tenant.r#gen),
    2220            0 :                         valid,
    2221            0 :                     ));
    2222              :                 } else {
    2223              :                     // This is legal: for example during a shard split the pageserver may still
    2224              :                     // have deletions in its queue from the old pre-split shard, or after deletion
    2225              :                     // of a tenant that was busy with compaction/gc while being deleted.
    2226            0 :                     tracing::info!(
    2227            0 :                         "Refusing deletion validation for missing shard {}",
    2228              :                         req_tenant.id
    2229              :                     );
    2230              :                 }
    2231              :             }
    2232              : 
    2233            0 :             in_memory_result
    2234              :         };
    2235              : 
    2236              :         // Database calls to confirm validity for anything that passed the in-memory check.  We must do this
    2237              :         // in case of controller split-brain, where some other controller process might have incremented the generation.
    2238            0 :         let db_generations = self
    2239            0 :             .persistence
    2240            0 :             .shard_generations(
    2241            0 :                 in_memory_result
    2242            0 :                     .iter()
    2243            0 :                     .filter_map(|i| if i.2 { Some(&i.0) } else { None }),
    2244            0 :             )
    2245            0 :             .await?;
    2246            0 :         let db_generations = db_generations.into_iter().collect::<HashMap<_, _>>();
    2247            0 : 
    2248            0 :         let mut response = ValidateResponse {
    2249            0 :             tenants: Vec::new(),
    2250            0 :         };
    2251            0 :         for (tenant_shard_id, validate_generation, valid) in in_memory_result.into_iter() {
    2252            0 :             let valid = if valid {
    2253            0 :                 let db_generation = db_generations.get(&tenant_shard_id);
    2254            0 :                 db_generation == Some(&Some(validate_generation))
    2255              :             } else {
    2256              :                 // If in-memory state says it's invalid, trust that.  It's always safe to fail a validation, at worst
    2257              :                 // this prevents a pageserver from cleaning up an object in S3.
    2258            0 :                 false
    2259              :             };
    2260              : 
    2261            0 :             response.tenants.push(ValidateResponseTenant {
    2262            0 :                 id: tenant_shard_id,
    2263            0 :                 valid,
    2264            0 :             })
    2265              :         }
    2266              : 
    2267            0 :         Ok(response)
    2268            0 :     }
    2269              : 
    2270            0 :     pub(crate) async fn tenant_create(
    2271            0 :         &self,
    2272            0 :         create_req: TenantCreateRequest,
    2273            0 :     ) -> Result<TenantCreateResponse, ApiError> {
    2274            0 :         let tenant_id = create_req.new_tenant_id.tenant_id;
    2275              : 
    2276              :         // Exclude any concurrent attempts to create/access the same tenant ID
    2277            0 :         let _tenant_lock = trace_exclusive_lock(
    2278            0 :             &self.tenant_op_locks,
    2279            0 :             create_req.new_tenant_id.tenant_id,
    2280            0 :             TenantOperations::Create,
    2281            0 :         )
    2282            0 :         .await;
    2283            0 :         let (response, waiters) = self.do_tenant_create(create_req).await?;
    2284              : 
    2285            0 :         if let Err(e) = self.await_waiters(waiters, RECONCILE_TIMEOUT).await {
    2286              :             // Avoid deadlock: reconcile may fail while notifying compute, if the cloud control plane refuses to
    2287              :             // accept compute notifications while it is in the process of creating.  Reconciliation will
    2288              :             // be retried in the background.
    2289            0 :             tracing::warn!(%tenant_id, "Reconcile not done yet while creating tenant ({e})");
    2290            0 :         }
    2291            0 :         Ok(response)
    2292            0 :     }
    2293              : 
    2294            0 :     pub(crate) async fn do_tenant_create(
    2295            0 :         &self,
    2296            0 :         create_req: TenantCreateRequest,
    2297            0 :     ) -> Result<(TenantCreateResponse, Vec<ReconcilerWaiter>), ApiError> {
    2298            0 :         let placement_policy = create_req
    2299            0 :             .placement_policy
    2300            0 :             .clone()
    2301            0 :             // As a default, zero secondaries is convenient for tests that don't choose a policy.
    2302            0 :             .unwrap_or(PlacementPolicy::Attached(0));
    2303              : 
    2304              :         // This service expects to handle sharding itself: it is an error to try and directly create
    2305              :         // a particular shard here.
    2306            0 :         let tenant_id = if !create_req.new_tenant_id.is_unsharded() {
    2307            0 :             return Err(ApiError::BadRequest(anyhow::anyhow!(
    2308            0 :                 "Attempted to create a specific shard, this API is for creating the whole tenant"
    2309            0 :             )));
    2310              :         } else {
    2311            0 :             create_req.new_tenant_id.tenant_id
    2312            0 :         };
    2313            0 : 
    2314            0 :         tracing::info!(
    2315            0 :             "Creating tenant {}, shard_count={:?}",
    2316              :             create_req.new_tenant_id,
    2317              :             create_req.shard_parameters.count,
    2318              :         );
    2319              : 
    2320            0 :         let create_ids = (0..create_req.shard_parameters.count.count())
    2321            0 :             .map(|i| TenantShardId {
    2322            0 :                 tenant_id,
    2323            0 :                 shard_number: ShardNumber(i),
    2324            0 :                 shard_count: create_req.shard_parameters.count,
    2325            0 :             })
    2326            0 :             .collect::<Vec<_>>();
    2327              : 
    2328              :         // If the caller specifies a None generation, it means "start from default".  This is different
    2329              :         // to [`Self::tenant_location_config`], where a None generation is used to represent
    2330              :         // an incompletely-onboarded tenant.
    2331            0 :         let initial_generation = if matches!(placement_policy, PlacementPolicy::Secondary) {
    2332            0 :             tracing::info!(
    2333            0 :                 "tenant_create: secondary mode, generation is_some={}",
    2334            0 :                 create_req.generation.is_some()
    2335              :             );
    2336            0 :             create_req.generation.map(Generation::new)
    2337              :         } else {
    2338            0 :             tracing::info!(
    2339            0 :                 "tenant_create: not secondary mode, generation is_some={}",
    2340            0 :                 create_req.generation.is_some()
    2341              :             );
    2342            0 :             Some(
    2343            0 :                 create_req
    2344            0 :                     .generation
    2345            0 :                     .map(Generation::new)
    2346            0 :                     .unwrap_or(INITIAL_GENERATION),
    2347            0 :             )
    2348              :         };
    2349              : 
    2350            0 :         let preferred_az_id = {
    2351            0 :             let locked = self.inner.read().unwrap();
    2352              :             // Idempotency: take the existing value if the tenant already exists
    2353            0 :             if let Some(shard) = locked.tenants.get(create_ids.first().unwrap()) {
    2354            0 :                 shard.preferred_az().cloned()
    2355              :             } else {
    2356            0 :                 locked.scheduler.get_az_for_new_tenant()
    2357              :             }
    2358              :         };
    2359              : 
    2360              :         // Ordering: we persist tenant shards before creating them on the pageserver.  This enables a caller
    2361              :         // to clean up after themselves by issuing a tenant deletion if something goes wrong and we restart
    2362              :         // during the creation, rather than risking leaving orphan objects in S3.
    2363            0 :         let persist_tenant_shards = create_ids
    2364            0 :             .iter()
    2365            0 :             .map(|tenant_shard_id| TenantShardPersistence {
    2366            0 :                 tenant_id: tenant_shard_id.tenant_id.to_string(),
    2367            0 :                 shard_number: tenant_shard_id.shard_number.0 as i32,
    2368            0 :                 shard_count: tenant_shard_id.shard_count.literal() as i32,
    2369            0 :                 shard_stripe_size: create_req.shard_parameters.stripe_size.0 as i32,
    2370            0 :                 generation: initial_generation.map(|g| g.into().unwrap() as i32),
    2371            0 :                 // The pageserver is not known until scheduling happens: we will set this column when
    2372            0 :                 // incrementing the generation the first time we attach to a pageserver.
    2373            0 :                 generation_pageserver: None,
    2374            0 :                 placement_policy: serde_json::to_string(&placement_policy).unwrap(),
    2375            0 :                 config: serde_json::to_string(&create_req.config).unwrap(),
    2376            0 :                 splitting: SplitState::default(),
    2377            0 :                 scheduling_policy: serde_json::to_string(&ShardSchedulingPolicy::default())
    2378            0 :                     .unwrap(),
    2379            0 :                 preferred_az_id: preferred_az_id.as_ref().map(|az| az.to_string()),
    2380            0 :             })
    2381            0 :             .collect();
    2382            0 : 
    2383            0 :         match self
    2384            0 :             .persistence
    2385            0 :             .insert_tenant_shards(persist_tenant_shards)
    2386            0 :             .await
    2387              :         {
    2388            0 :             Ok(_) => {}
    2389              :             Err(DatabaseError::Query(diesel::result::Error::DatabaseError(
    2390              :                 DatabaseErrorKind::UniqueViolation,
    2391              :                 _,
    2392              :             ))) => {
    2393              :                 // Unique key violation: this is probably a retry.  Because the shard count is part of the unique key,
    2394              :                 // if we see a unique key violation it means that the creation request's shard count matches the previous
    2395              :                 // creation's shard count.
    2396            0 :                 tracing::info!(
    2397            0 :                     "Tenant shards already present in database, proceeding with idempotent creation..."
    2398              :                 );
    2399              :             }
    2400              :             // Any other database error is unexpected and a bug.
    2401            0 :             Err(e) => return Err(ApiError::InternalServerError(anyhow::anyhow!(e))),
    2402              :         };
    2403              : 
    2404            0 :         let mut schedule_context = ScheduleContext::default();
    2405            0 :         let mut schedule_error = None;
    2406            0 :         let mut response_shards = Vec::new();
    2407            0 :         for tenant_shard_id in create_ids {
    2408            0 :             tracing::info!("Creating shard {tenant_shard_id}...");
    2409              : 
    2410            0 :             let outcome = self
    2411            0 :                 .do_initial_shard_scheduling(
    2412            0 :                     tenant_shard_id,
    2413            0 :                     initial_generation,
    2414            0 :                     &create_req.shard_parameters,
    2415            0 :                     create_req.config.clone(),
    2416            0 :                     placement_policy.clone(),
    2417            0 :                     preferred_az_id.as_ref(),
    2418            0 :                     &mut schedule_context,
    2419            0 :                 )
    2420            0 :                 .await;
    2421              : 
    2422            0 :             match outcome {
    2423            0 :                 InitialShardScheduleOutcome::Scheduled(resp) => response_shards.push(resp),
    2424            0 :                 InitialShardScheduleOutcome::NotScheduled => {}
    2425            0 :                 InitialShardScheduleOutcome::ShardScheduleError(err) => {
    2426            0 :                     schedule_error = Some(err);
    2427            0 :                 }
    2428              :             }
    2429              :         }
    2430              : 
    2431              :         // If we failed to schedule shards, then they are still created in the controller,
    2432              :         // but we return an error to the requester to avoid a silent failure when someone
    2433              :         // tries to e.g. create a tenant whose placement policy requires more nodes than
    2434              :         // are present in the system.  We do this here rather than in the above loop, to
    2435              :         // avoid situations where we only create a subset of shards in the tenant.
    2436            0 :         if let Some(e) = schedule_error {
    2437            0 :             return Err(ApiError::Conflict(format!(
    2438            0 :                 "Failed to schedule shard(s): {e}"
    2439            0 :             )));
    2440            0 :         }
    2441            0 : 
    2442            0 :         let waiters = {
    2443            0 :             let mut locked = self.inner.write().unwrap();
    2444            0 :             let (nodes, tenants, _scheduler) = locked.parts_mut();
    2445            0 :             let config = ReconcilerConfigBuilder::new(ReconcilerPriority::High)
    2446            0 :                 .tenant_creation_hint(true)
    2447            0 :                 .build();
    2448            0 :             tenants
    2449            0 :                 .range_mut(TenantShardId::tenant_range(tenant_id))
    2450            0 :                 .filter_map(|(_shard_id, shard)| {
    2451            0 :                     self.maybe_configured_reconcile_shard(shard, nodes, config)
    2452            0 :                 })
    2453            0 :                 .collect::<Vec<_>>()
    2454            0 :         };
    2455            0 : 
    2456            0 :         Ok((
    2457            0 :             TenantCreateResponse {
    2458            0 :                 shards: response_shards,
    2459            0 :             },
    2460            0 :             waiters,
    2461            0 :         ))
    2462            0 :     }
    2463              : 
    2464              :     /// Helper for tenant creation that does the scheduling for an individual shard. Covers both the
    2465              :     /// case of a new tenant and a pre-existing one.
    2466              :     #[allow(clippy::too_many_arguments)]
    2467            0 :     async fn do_initial_shard_scheduling(
    2468            0 :         &self,
    2469            0 :         tenant_shard_id: TenantShardId,
    2470            0 :         initial_generation: Option<Generation>,
    2471            0 :         shard_params: &ShardParameters,
    2472            0 :         config: TenantConfig,
    2473            0 :         placement_policy: PlacementPolicy,
    2474            0 :         preferred_az_id: Option<&AvailabilityZone>,
    2475            0 :         schedule_context: &mut ScheduleContext,
    2476            0 :     ) -> InitialShardScheduleOutcome {
    2477            0 :         let mut locked = self.inner.write().unwrap();
    2478            0 :         let (_nodes, tenants, scheduler) = locked.parts_mut();
    2479              : 
    2480              :         use std::collections::btree_map::Entry;
    2481            0 :         match tenants.entry(tenant_shard_id) {
    2482            0 :             Entry::Occupied(mut entry) => {
    2483            0 :                 tracing::info!("Tenant shard {tenant_shard_id} already exists while creating");
    2484              : 
    2485            0 :                 if let Err(err) = entry.get_mut().schedule(scheduler, schedule_context) {
    2486            0 :                     return InitialShardScheduleOutcome::ShardScheduleError(err);
    2487            0 :                 }
    2488              : 
    2489            0 :                 if let Some(node_id) = entry.get().intent.get_attached() {
    2490            0 :                     let generation = entry
    2491            0 :                         .get()
    2492            0 :                         .generation
    2493            0 :                         .expect("Generation is set when in attached mode");
    2494            0 :                     InitialShardScheduleOutcome::Scheduled(TenantCreateResponseShard {
    2495            0 :                         shard_id: tenant_shard_id,
    2496            0 :                         node_id: *node_id,
    2497            0 :                         generation: generation.into().unwrap(),
    2498            0 :                     })
    2499              :                 } else {
    2500            0 :                     InitialShardScheduleOutcome::NotScheduled
    2501              :                 }
    2502              :             }
    2503            0 :             Entry::Vacant(entry) => {
    2504            0 :                 let state = entry.insert(TenantShard::new(
    2505            0 :                     tenant_shard_id,
    2506            0 :                     ShardIdentity::from_params(tenant_shard_id.shard_number, shard_params),
    2507            0 :                     placement_policy,
    2508            0 :                     preferred_az_id.cloned(),
    2509            0 :                 ));
    2510            0 : 
    2511            0 :                 state.generation = initial_generation;
    2512            0 :                 state.config = config;
    2513            0 :                 if let Err(e) = state.schedule(scheduler, schedule_context) {
    2514            0 :                     return InitialShardScheduleOutcome::ShardScheduleError(e);
    2515            0 :                 }
    2516              : 
    2517              :                 // Only include shards in result if we are attaching: the purpose
    2518              :                 // of the response is to tell the caller where the shards are attached.
    2519            0 :                 if let Some(node_id) = state.intent.get_attached() {
    2520            0 :                     let generation = state
    2521            0 :                         .generation
    2522            0 :                         .expect("Generation is set when in attached mode");
    2523            0 :                     InitialShardScheduleOutcome::Scheduled(TenantCreateResponseShard {
    2524            0 :                         shard_id: tenant_shard_id,
    2525            0 :                         node_id: *node_id,
    2526            0 :                         generation: generation.into().unwrap(),
    2527            0 :                     })
    2528              :                 } else {
    2529            0 :                     InitialShardScheduleOutcome::NotScheduled
    2530              :                 }
    2531              :             }
    2532              :         }
    2533            0 :     }
    2534              : 
    2535              :     /// Helper for functions that reconcile a number of shards, and would like to do a timeout-bounded
    2536              :     /// wait for reconciliation to complete before responding.
    2537            0 :     async fn await_waiters(
    2538            0 :         &self,
    2539            0 :         waiters: Vec<ReconcilerWaiter>,
    2540            0 :         timeout: Duration,
    2541            0 :     ) -> Result<(), ReconcileWaitError> {
    2542            0 :         let deadline = Instant::now().checked_add(timeout).unwrap();
    2543            0 :         for waiter in waiters {
    2544            0 :             let timeout = deadline.duration_since(Instant::now());
    2545            0 :             waiter.wait_timeout(timeout).await?;
    2546              :         }
    2547              : 
    2548            0 :         Ok(())
    2549            0 :     }
    2550              : 
    2551              :     /// Same as [`Service::await_waiters`], but returns the waiters which are still
    2552              :     /// in progress
    2553            0 :     async fn await_waiters_remainder(
    2554            0 :         &self,
    2555            0 :         waiters: Vec<ReconcilerWaiter>,
    2556            0 :         timeout: Duration,
    2557            0 :     ) -> Vec<ReconcilerWaiter> {
    2558            0 :         let deadline = Instant::now().checked_add(timeout).unwrap();
    2559            0 :         for waiter in waiters.iter() {
    2560            0 :             let timeout = deadline.duration_since(Instant::now());
    2561            0 :             let _ = waiter.wait_timeout(timeout).await;
    2562              :         }
    2563              : 
    2564            0 :         waiters
    2565            0 :             .into_iter()
    2566            0 :             .filter(|waiter| matches!(waiter.get_status(), ReconcilerStatus::InProgress))
    2567            0 :             .collect::<Vec<_>>()
    2568            0 :     }
    2569              : 
    2570              :     /// Part of [`Self::tenant_location_config`]: dissect an incoming location config request,
    2571              :     /// and transform it into either a tenant creation of a series of shard updates.
    2572              :     ///
    2573              :     /// If the incoming request makes no changes, a [`TenantCreateOrUpdate::Update`] result will
    2574              :     /// still be returned.
    2575            0 :     fn tenant_location_config_prepare(
    2576            0 :         &self,
    2577            0 :         tenant_id: TenantId,
    2578            0 :         req: TenantLocationConfigRequest,
    2579            0 :     ) -> TenantCreateOrUpdate {
    2580            0 :         let mut updates = Vec::new();
    2581            0 :         let mut locked = self.inner.write().unwrap();
    2582            0 :         let (nodes, tenants, _scheduler) = locked.parts_mut();
    2583            0 :         let tenant_shard_id = TenantShardId::unsharded(tenant_id);
    2584              : 
    2585              :         // Use location config mode as an indicator of policy.
    2586            0 :         let placement_policy = match req.config.mode {
    2587            0 :             LocationConfigMode::Detached => PlacementPolicy::Detached,
    2588            0 :             LocationConfigMode::Secondary => PlacementPolicy::Secondary,
    2589              :             LocationConfigMode::AttachedMulti
    2590              :             | LocationConfigMode::AttachedSingle
    2591              :             | LocationConfigMode::AttachedStale => {
    2592            0 :                 if nodes.len() > 1 {
    2593            0 :                     PlacementPolicy::Attached(1)
    2594              :                 } else {
    2595              :                     // Convenience for dev/test: if we just have one pageserver, import
    2596              :                     // tenants into non-HA mode so that scheduling will succeed.
    2597            0 :                     PlacementPolicy::Attached(0)
    2598              :                 }
    2599              :             }
    2600              :         };
    2601              : 
    2602              :         // Ordinarily we do not update scheduling policy, but when making major changes
    2603              :         // like detaching or demoting to secondary-only, we need to force the scheduling
    2604              :         // mode to Active, or the caller's expected outcome (detach it) will not happen.
    2605            0 :         let scheduling_policy = match req.config.mode {
    2606              :             LocationConfigMode::Detached | LocationConfigMode::Secondary => {
    2607              :                 // Special case: when making major changes like detaching or demoting to secondary-only,
    2608              :                 // we need to force the scheduling mode to Active, or nothing will happen.
    2609            0 :                 Some(ShardSchedulingPolicy::Active)
    2610              :             }
    2611              :             LocationConfigMode::AttachedMulti
    2612              :             | LocationConfigMode::AttachedSingle
    2613              :             | LocationConfigMode::AttachedStale => {
    2614              :                 // While attached, continue to respect whatever the existing scheduling mode is.
    2615            0 :                 None
    2616              :             }
    2617              :         };
    2618              : 
    2619            0 :         let mut create = true;
    2620            0 :         for (shard_id, shard) in tenants.range_mut(TenantShardId::tenant_range(tenant_id)) {
    2621              :             // Saw an existing shard: this is not a creation
    2622            0 :             create = false;
    2623              : 
    2624              :             // Shards may have initially been created by a Secondary request, where we
    2625              :             // would have left generation as None.
    2626              :             //
    2627              :             // We only update generation the first time we see an attached-mode request,
    2628              :             // and if there is no existing generation set. The caller is responsible for
    2629              :             // ensuring that no non-storage-controller pageserver ever uses a higher
    2630              :             // generation than they passed in here.
    2631              :             use LocationConfigMode::*;
    2632            0 :             let set_generation = match req.config.mode {
    2633            0 :                 AttachedMulti | AttachedSingle | AttachedStale if shard.generation.is_none() => {
    2634            0 :                     req.config.generation.map(Generation::new)
    2635              :                 }
    2636            0 :                 _ => None,
    2637              :             };
    2638              : 
    2639            0 :             updates.push(ShardUpdate {
    2640            0 :                 tenant_shard_id: *shard_id,
    2641            0 :                 placement_policy: placement_policy.clone(),
    2642            0 :                 tenant_config: req.config.tenant_conf.clone(),
    2643            0 :                 generation: set_generation,
    2644            0 :                 scheduling_policy,
    2645            0 :             });
    2646              :         }
    2647              : 
    2648            0 :         if create {
    2649              :             use LocationConfigMode::*;
    2650            0 :             let generation = match req.config.mode {
    2651            0 :                 AttachedMulti | AttachedSingle | AttachedStale => req.config.generation,
    2652              :                 // If a caller provided a generation in a non-attached request, ignore it
    2653              :                 // and leave our generation as None: this enables a subsequent update to set
    2654              :                 // the generation when setting an attached mode for the first time.
    2655            0 :                 _ => None,
    2656              :             };
    2657              : 
    2658            0 :             TenantCreateOrUpdate::Create(
    2659            0 :                 // Synthesize a creation request
    2660            0 :                 TenantCreateRequest {
    2661            0 :                     new_tenant_id: tenant_shard_id,
    2662            0 :                     generation,
    2663            0 :                     shard_parameters: ShardParameters {
    2664            0 :                         count: tenant_shard_id.shard_count,
    2665            0 :                         // We only import un-sharded or single-sharded tenants, so stripe
    2666            0 :                         // size can be made up arbitrarily here.
    2667            0 :                         stripe_size: ShardParameters::DEFAULT_STRIPE_SIZE,
    2668            0 :                     },
    2669            0 :                     placement_policy: Some(placement_policy),
    2670            0 :                     config: req.config.tenant_conf,
    2671            0 :                 },
    2672            0 :             )
    2673              :         } else {
    2674            0 :             assert!(!updates.is_empty());
    2675            0 :             TenantCreateOrUpdate::Update(updates)
    2676              :         }
    2677            0 :     }
    2678              : 
    2679              :     /// For APIs that might act on tenants with [`PlacementPolicy::Detached`], first check if
    2680              :     /// the tenant is present in memory. If not, load it from the database.  If it is found
    2681              :     /// in neither location, return a NotFound error.
    2682              :     ///
    2683              :     /// Caller must demonstrate they hold a lock guard, as otherwise two callers might try and load
    2684              :     /// it at the same time, or we might race with [`Self::maybe_drop_tenant`]
    2685            0 :     async fn maybe_load_tenant(
    2686            0 :         &self,
    2687            0 :         tenant_id: TenantId,
    2688            0 :         _guard: &TracingExclusiveGuard<TenantOperations>,
    2689            0 :     ) -> Result<(), ApiError> {
    2690              :         // Check if the tenant is present in memory, and select an AZ to use when loading
    2691              :         // if we will load it.
    2692            0 :         let load_in_az = {
    2693            0 :             let locked = self.inner.read().unwrap();
    2694            0 :             let existing = locked
    2695            0 :                 .tenants
    2696            0 :                 .range(TenantShardId::tenant_range(tenant_id))
    2697            0 :                 .next();
    2698            0 : 
    2699            0 :             // If the tenant is not present in memory, we expect to load it from database,
    2700            0 :             // so let's figure out what AZ to load it into while we have self.inner locked.
    2701            0 :             if existing.is_none() {
    2702            0 :                 locked
    2703            0 :                     .scheduler
    2704            0 :                     .get_az_for_new_tenant()
    2705            0 :                     .ok_or(ApiError::BadRequest(anyhow::anyhow!(
    2706            0 :                         "No AZ with nodes found to load tenant"
    2707            0 :                     )))?
    2708              :             } else {
    2709              :                 // We already have this tenant in memory
    2710            0 :                 return Ok(());
    2711              :             }
    2712              :         };
    2713              : 
    2714            0 :         let tenant_shards = self.persistence.load_tenant(tenant_id).await?;
    2715            0 :         if tenant_shards.is_empty() {
    2716            0 :             return Err(ApiError::NotFound(
    2717            0 :                 anyhow::anyhow!("Tenant {} not found", tenant_id).into(),
    2718            0 :             ));
    2719            0 :         }
    2720            0 : 
    2721            0 :         // Update the persistent shards with the AZ that we are about to apply to in-memory state
    2722            0 :         self.persistence
    2723            0 :             .set_tenant_shard_preferred_azs(
    2724            0 :                 tenant_shards
    2725            0 :                     .iter()
    2726            0 :                     .map(|t| {
    2727            0 :                         (
    2728            0 :                             t.get_tenant_shard_id().expect("Corrupt shard in database"),
    2729            0 :                             Some(load_in_az.clone()),
    2730            0 :                         )
    2731            0 :                     })
    2732            0 :                     .collect(),
    2733            0 :             )
    2734            0 :             .await?;
    2735              : 
    2736            0 :         let mut locked = self.inner.write().unwrap();
    2737            0 :         tracing::info!(
    2738            0 :             "Loaded {} shards for tenant {}",
    2739            0 :             tenant_shards.len(),
    2740              :             tenant_id
    2741              :         );
    2742              : 
    2743            0 :         locked.tenants.extend(tenant_shards.into_iter().map(|p| {
    2744            0 :             let intent = IntentState::new(Some(load_in_az.clone()));
    2745            0 :             let shard =
    2746            0 :                 TenantShard::from_persistent(p, intent).expect("Corrupt shard row in database");
    2747            0 : 
    2748            0 :             // Sanity check: when loading on-demand, we should always be loaded something Detached
    2749            0 :             debug_assert!(shard.policy == PlacementPolicy::Detached);
    2750            0 :             if shard.policy != PlacementPolicy::Detached {
    2751            0 :                 tracing::error!(
    2752            0 :                     "Tenant shard {} loaded on-demand, but has non-Detached policy {:?}",
    2753              :                     shard.tenant_shard_id,
    2754              :                     shard.policy
    2755              :                 );
    2756            0 :             }
    2757              : 
    2758            0 :             (shard.tenant_shard_id, shard)
    2759            0 :         }));
    2760            0 : 
    2761            0 :         Ok(())
    2762            0 :     }
    2763              : 
    2764              :     /// If all shards for a tenant are detached, and in a fully quiescent state (no observed locations on pageservers),
    2765              :     /// and have no reconciler running, then we can drop the tenant from memory.  It will be reloaded on-demand
    2766              :     /// if we are asked to attach it again (see [`Self::maybe_load_tenant`]).
    2767              :     ///
    2768              :     /// Caller must demonstrate they hold a lock guard, as otherwise it is unsafe to drop a tenant from
    2769              :     /// memory while some other function might assume it continues to exist while not holding the lock on Self::inner.
    2770            0 :     fn maybe_drop_tenant(
    2771            0 :         &self,
    2772            0 :         tenant_id: TenantId,
    2773            0 :         locked: &mut std::sync::RwLockWriteGuard<ServiceState>,
    2774            0 :         _guard: &TracingExclusiveGuard<TenantOperations>,
    2775            0 :     ) {
    2776            0 :         let mut tenant_shards = locked.tenants.range(TenantShardId::tenant_range(tenant_id));
    2777            0 :         if tenant_shards.all(|(_id, shard)| {
    2778            0 :             shard.policy == PlacementPolicy::Detached
    2779            0 :                 && shard.reconciler.is_none()
    2780            0 :                 && shard.observed.is_empty()
    2781            0 :         }) {
    2782            0 :             let keys = locked
    2783            0 :                 .tenants
    2784            0 :                 .range(TenantShardId::tenant_range(tenant_id))
    2785            0 :                 .map(|(id, _)| id)
    2786            0 :                 .copied()
    2787            0 :                 .collect::<Vec<_>>();
    2788            0 :             for key in keys {
    2789            0 :                 tracing::info!("Dropping detached tenant shard {} from memory", key);
    2790            0 :                 locked.tenants.remove(&key);
    2791              :             }
    2792            0 :         }
    2793            0 :     }
    2794              : 
    2795              :     /// This API is used by the cloud control plane to migrate unsharded tenants that it created
    2796              :     /// directly with pageservers into this service.
    2797              :     ///
    2798              :     /// Cloud control plane MUST NOT continue issuing GENERATION NUMBERS for this tenant once it
    2799              :     /// has attempted to call this API. Failure to oblige to this rule may lead to S3 corruption.
    2800              :     /// Think of the first attempt to call this API as a transfer of absolute authority over the
    2801              :     /// tenant's source of generation numbers.
    2802              :     ///
    2803              :     /// The mode in this request coarse-grained control of tenants:
    2804              :     /// - Call with mode Attached* to upsert the tenant.
    2805              :     /// - Call with mode Secondary to either onboard a tenant without attaching it, or
    2806              :     ///   to set an existing tenant to PolicyMode::Secondary
    2807              :     /// - Call with mode Detached to switch to PolicyMode::Detached
    2808            0 :     pub(crate) async fn tenant_location_config(
    2809            0 :         &self,
    2810            0 :         tenant_shard_id: TenantShardId,
    2811            0 :         req: TenantLocationConfigRequest,
    2812            0 :     ) -> Result<TenantLocationConfigResponse, ApiError> {
    2813              :         // We require an exclusive lock, because we are updating both persistent and in-memory state
    2814            0 :         let _tenant_lock = trace_exclusive_lock(
    2815            0 :             &self.tenant_op_locks,
    2816            0 :             tenant_shard_id.tenant_id,
    2817            0 :             TenantOperations::LocationConfig,
    2818            0 :         )
    2819            0 :         .await;
    2820              : 
    2821            0 :         let tenant_id = if !tenant_shard_id.is_unsharded() {
    2822            0 :             return Err(ApiError::BadRequest(anyhow::anyhow!(
    2823            0 :                 "This API is for importing single-sharded or unsharded tenants"
    2824            0 :             )));
    2825              :         } else {
    2826            0 :             tenant_shard_id.tenant_id
    2827            0 :         };
    2828            0 : 
    2829            0 :         // In case we are waking up a Detached tenant
    2830            0 :         match self.maybe_load_tenant(tenant_id, &_tenant_lock).await {
    2831            0 :             Ok(()) | Err(ApiError::NotFound(_)) => {
    2832            0 :                 // This is a creation or an update
    2833            0 :             }
    2834            0 :             Err(e) => {
    2835            0 :                 return Err(e);
    2836              :             }
    2837              :         };
    2838              : 
    2839              :         // First check if this is a creation or an update
    2840            0 :         let create_or_update = self.tenant_location_config_prepare(tenant_id, req);
    2841            0 : 
    2842            0 :         let mut result = TenantLocationConfigResponse {
    2843            0 :             shards: Vec::new(),
    2844            0 :             stripe_size: None,
    2845            0 :         };
    2846            0 :         let waiters = match create_or_update {
    2847            0 :             TenantCreateOrUpdate::Create(create_req) => {
    2848            0 :                 let (create_resp, waiters) = self.do_tenant_create(create_req).await?;
    2849            0 :                 result.shards = create_resp
    2850            0 :                     .shards
    2851            0 :                     .into_iter()
    2852            0 :                     .map(|s| TenantShardLocation {
    2853            0 :                         node_id: s.node_id,
    2854            0 :                         shard_id: s.shard_id,
    2855            0 :                     })
    2856            0 :                     .collect();
    2857            0 :                 waiters
    2858              :             }
    2859            0 :             TenantCreateOrUpdate::Update(updates) => {
    2860            0 :                 // Persist updates
    2861            0 :                 // Ordering: write to the database before applying changes in-memory, so that
    2862            0 :                 // we will not appear time-travel backwards on a restart.
    2863            0 : 
    2864            0 :                 let mut schedule_context = ScheduleContext::default();
    2865              :                 for ShardUpdate {
    2866            0 :                     tenant_shard_id,
    2867            0 :                     placement_policy,
    2868            0 :                     tenant_config,
    2869            0 :                     generation,
    2870            0 :                     scheduling_policy,
    2871            0 :                 } in &updates
    2872              :                 {
    2873            0 :                     self.persistence
    2874            0 :                         .update_tenant_shard(
    2875            0 :                             TenantFilter::Shard(*tenant_shard_id),
    2876            0 :                             Some(placement_policy.clone()),
    2877            0 :                             Some(tenant_config.clone()),
    2878            0 :                             *generation,
    2879            0 :                             *scheduling_policy,
    2880            0 :                         )
    2881            0 :                         .await?;
    2882              :                 }
    2883              : 
    2884              :                 // Apply updates in-memory
    2885            0 :                 let mut waiters = Vec::new();
    2886            0 :                 {
    2887            0 :                     let mut locked = self.inner.write().unwrap();
    2888            0 :                     let (nodes, tenants, scheduler) = locked.parts_mut();
    2889              : 
    2890              :                     for ShardUpdate {
    2891            0 :                         tenant_shard_id,
    2892            0 :                         placement_policy,
    2893            0 :                         tenant_config,
    2894            0 :                         generation: update_generation,
    2895            0 :                         scheduling_policy,
    2896            0 :                     } in updates
    2897              :                     {
    2898            0 :                         let Some(shard) = tenants.get_mut(&tenant_shard_id) else {
    2899            0 :                             tracing::warn!("Shard {tenant_shard_id} removed while updating");
    2900            0 :                             continue;
    2901              :                         };
    2902              : 
    2903              :                         // Update stripe size
    2904            0 :                         if result.stripe_size.is_none() && shard.shard.count.count() > 1 {
    2905            0 :                             result.stripe_size = Some(shard.shard.stripe_size);
    2906            0 :                         }
    2907              : 
    2908            0 :                         shard.policy = placement_policy;
    2909            0 :                         shard.config = tenant_config;
    2910            0 :                         if let Some(generation) = update_generation {
    2911            0 :                             shard.generation = Some(generation);
    2912            0 :                         }
    2913              : 
    2914            0 :                         if let Some(scheduling_policy) = scheduling_policy {
    2915            0 :                             shard.set_scheduling_policy(scheduling_policy);
    2916            0 :                         }
    2917              : 
    2918            0 :                         shard.schedule(scheduler, &mut schedule_context)?;
    2919              : 
    2920            0 :                         let maybe_waiter =
    2921            0 :                             self.maybe_reconcile_shard(shard, nodes, ReconcilerPriority::High);
    2922            0 :                         if let Some(waiter) = maybe_waiter {
    2923            0 :                             waiters.push(waiter);
    2924            0 :                         }
    2925              : 
    2926            0 :                         if let Some(node_id) = shard.intent.get_attached() {
    2927            0 :                             result.shards.push(TenantShardLocation {
    2928            0 :                                 shard_id: tenant_shard_id,
    2929            0 :                                 node_id: *node_id,
    2930            0 :                             })
    2931            0 :                         }
    2932              :                     }
    2933              :                 }
    2934            0 :                 waiters
    2935              :             }
    2936              :         };
    2937              : 
    2938            0 :         if let Err(e) = self.await_waiters(waiters, SHORT_RECONCILE_TIMEOUT).await {
    2939              :             // Do not treat a reconcile error as fatal: we have already applied any requested
    2940              :             // Intent changes, and the reconcile can fail for external reasons like unavailable
    2941              :             // compute notification API.  In these cases, it is important that we do not
    2942              :             // cause the cloud control plane to retry forever on this API.
    2943            0 :             tracing::warn!(
    2944            0 :                 "Failed to reconcile after /location_config: {e}, returning success anyway"
    2945              :             );
    2946            0 :         }
    2947              : 
    2948              :         // Logging the full result is useful because it lets us cross-check what the cloud control
    2949              :         // plane's tenant_shards table should contain.
    2950            0 :         tracing::info!("Complete, returning {result:?}");
    2951              : 
    2952            0 :         Ok(result)
    2953            0 :     }
    2954              : 
    2955            0 :     pub(crate) async fn tenant_config_patch(
    2956            0 :         &self,
    2957            0 :         req: TenantConfigPatchRequest,
    2958            0 :     ) -> Result<(), ApiError> {
    2959            0 :         let _tenant_lock = trace_exclusive_lock(
    2960            0 :             &self.tenant_op_locks,
    2961            0 :             req.tenant_id,
    2962            0 :             TenantOperations::ConfigPatch,
    2963            0 :         )
    2964            0 :         .await;
    2965              : 
    2966            0 :         let tenant_id = req.tenant_id;
    2967            0 :         let patch = req.config;
    2968            0 : 
    2969            0 :         self.maybe_load_tenant(tenant_id, &_tenant_lock).await?;
    2970              : 
    2971            0 :         let base = {
    2972            0 :             let locked = self.inner.read().unwrap();
    2973            0 :             let shards = locked
    2974            0 :                 .tenants
    2975            0 :                 .range(TenantShardId::tenant_range(req.tenant_id));
    2976            0 : 
    2977            0 :             let mut configs = shards.map(|(_sid, shard)| &shard.config).peekable();
    2978              : 
    2979            0 :             let first = match configs.peek() {
    2980            0 :                 Some(first) => (*first).clone(),
    2981              :                 None => {
    2982            0 :                     return Err(ApiError::NotFound(
    2983            0 :                         anyhow::anyhow!("Tenant {} not found", req.tenant_id).into(),
    2984            0 :                     ));
    2985              :                 }
    2986              :             };
    2987              : 
    2988            0 :             if !configs.all_equal() {
    2989            0 :                 tracing::error!("Tenant configs for {} are mismatched. ", req.tenant_id);
    2990              :                 // This can't happen because we atomically update the database records
    2991              :                 // of all shards to the new value in [`Self::set_tenant_config_and_reconcile`].
    2992            0 :                 return Err(ApiError::InternalServerError(anyhow::anyhow!(
    2993            0 :                     "Tenant configs for {} are mismatched",
    2994            0 :                     req.tenant_id
    2995            0 :                 )));
    2996            0 :             }
    2997            0 : 
    2998            0 :             first
    2999              :         };
    3000              : 
    3001            0 :         let updated_config = base
    3002            0 :             .apply_patch(patch)
    3003            0 :             .map_err(|err| ApiError::BadRequest(anyhow::anyhow!(err)))?;
    3004            0 :         self.set_tenant_config_and_reconcile(tenant_id, updated_config)
    3005            0 :             .await
    3006            0 :     }
    3007              : 
    3008            0 :     pub(crate) async fn tenant_config_set(&self, req: TenantConfigRequest) -> Result<(), ApiError> {
    3009              :         // We require an exclusive lock, because we are updating persistent and in-memory state
    3010            0 :         let _tenant_lock = trace_exclusive_lock(
    3011            0 :             &self.tenant_op_locks,
    3012            0 :             req.tenant_id,
    3013            0 :             TenantOperations::ConfigSet,
    3014            0 :         )
    3015            0 :         .await;
    3016              : 
    3017            0 :         self.maybe_load_tenant(req.tenant_id, &_tenant_lock).await?;
    3018              : 
    3019            0 :         self.set_tenant_config_and_reconcile(req.tenant_id, req.config)
    3020            0 :             .await
    3021            0 :     }
    3022              : 
    3023            0 :     async fn set_tenant_config_and_reconcile(
    3024            0 :         &self,
    3025            0 :         tenant_id: TenantId,
    3026            0 :         config: TenantConfig,
    3027            0 :     ) -> Result<(), ApiError> {
    3028            0 :         self.persistence
    3029            0 :             .update_tenant_shard(
    3030            0 :                 TenantFilter::Tenant(tenant_id),
    3031            0 :                 None,
    3032            0 :                 Some(config.clone()),
    3033            0 :                 None,
    3034            0 :                 None,
    3035            0 :             )
    3036            0 :             .await?;
    3037              : 
    3038            0 :         let waiters = {
    3039            0 :             let mut waiters = Vec::new();
    3040            0 :             let mut locked = self.inner.write().unwrap();
    3041            0 :             let (nodes, tenants, _scheduler) = locked.parts_mut();
    3042            0 :             for (_shard_id, shard) in tenants.range_mut(TenantShardId::tenant_range(tenant_id)) {
    3043            0 :                 shard.config = config.clone();
    3044            0 :                 if let Some(waiter) =
    3045            0 :                     self.maybe_reconcile_shard(shard, nodes, ReconcilerPriority::High)
    3046            0 :                 {
    3047            0 :                     waiters.push(waiter);
    3048            0 :                 }
    3049              :             }
    3050            0 :             waiters
    3051              :         };
    3052              : 
    3053            0 :         if let Err(e) = self.await_waiters(waiters, SHORT_RECONCILE_TIMEOUT).await {
    3054              :             // Treat this as success because we have stored the configuration.  If e.g.
    3055              :             // a node was unavailable at this time, it should not stop us accepting a
    3056              :             // configuration change.
    3057            0 :             tracing::warn!(%tenant_id, "Accepted configuration update but reconciliation failed: {e}");
    3058            0 :         }
    3059              : 
    3060            0 :         Ok(())
    3061            0 :     }
    3062              : 
    3063            0 :     pub(crate) fn tenant_config_get(
    3064            0 :         &self,
    3065            0 :         tenant_id: TenantId,
    3066            0 :     ) -> Result<HashMap<&str, serde_json::Value>, ApiError> {
    3067            0 :         let config = {
    3068            0 :             let locked = self.inner.read().unwrap();
    3069            0 : 
    3070            0 :             match locked
    3071            0 :                 .tenants
    3072            0 :                 .range(TenantShardId::tenant_range(tenant_id))
    3073            0 :                 .next()
    3074              :             {
    3075            0 :                 Some((_tenant_shard_id, shard)) => shard.config.clone(),
    3076              :                 None => {
    3077            0 :                     return Err(ApiError::NotFound(
    3078            0 :                         anyhow::anyhow!("Tenant not found").into(),
    3079            0 :                     ));
    3080              :                 }
    3081              :             }
    3082              :         };
    3083              : 
    3084              :         // Unlike the pageserver, we do not have a set of global defaults: the config is
    3085              :         // entirely per-tenant.  Therefore the distinction between `tenant_specific_overrides`
    3086              :         // and `effective_config` in the response is meaningless, but we retain that syntax
    3087              :         // in order to remain compatible with the pageserver API.
    3088              : 
    3089            0 :         let response = HashMap::from([
    3090              :             (
    3091              :                 "tenant_specific_overrides",
    3092            0 :                 serde_json::to_value(&config)
    3093            0 :                     .context("serializing tenant specific overrides")
    3094            0 :                     .map_err(ApiError::InternalServerError)?,
    3095              :             ),
    3096              :             (
    3097            0 :                 "effective_config",
    3098            0 :                 serde_json::to_value(&config)
    3099            0 :                     .context("serializing effective config")
    3100            0 :                     .map_err(ApiError::InternalServerError)?,
    3101              :             ),
    3102              :         ]);
    3103              : 
    3104            0 :         Ok(response)
    3105            0 :     }
    3106              : 
    3107            0 :     pub(crate) async fn tenant_time_travel_remote_storage(
    3108            0 :         &self,
    3109            0 :         time_travel_req: &TenantTimeTravelRequest,
    3110            0 :         tenant_id: TenantId,
    3111            0 :         timestamp: Cow<'_, str>,
    3112            0 :         done_if_after: Cow<'_, str>,
    3113            0 :     ) -> Result<(), ApiError> {
    3114            0 :         let _tenant_lock = trace_exclusive_lock(
    3115            0 :             &self.tenant_op_locks,
    3116            0 :             tenant_id,
    3117            0 :             TenantOperations::TimeTravelRemoteStorage,
    3118            0 :         )
    3119            0 :         .await;
    3120              : 
    3121            0 :         let node = {
    3122            0 :             let mut locked = self.inner.write().unwrap();
    3123              :             // Just a sanity check to prevent misuse: the API expects that the tenant is fully
    3124              :             // detached everywhere, and nothing writes to S3 storage. Here, we verify that,
    3125              :             // but only at the start of the process, so it's really just to prevent operator
    3126              :             // mistakes.
    3127            0 :             for (shard_id, shard) in locked.tenants.range(TenantShardId::tenant_range(tenant_id)) {
    3128            0 :                 if shard.intent.get_attached().is_some() || !shard.intent.get_secondary().is_empty()
    3129              :                 {
    3130            0 :                     return Err(ApiError::InternalServerError(anyhow::anyhow!(
    3131            0 :                         "We want tenant to be attached in shard with tenant_shard_id={shard_id}"
    3132            0 :                     )));
    3133            0 :                 }
    3134            0 :                 let maybe_attached = shard
    3135            0 :                     .observed
    3136            0 :                     .locations
    3137            0 :                     .iter()
    3138            0 :                     .filter_map(|(node_id, observed_location)| {
    3139            0 :                         observed_location
    3140            0 :                             .conf
    3141            0 :                             .as_ref()
    3142            0 :                             .map(|loc| (node_id, observed_location, loc.mode))
    3143            0 :                     })
    3144            0 :                     .find(|(_, _, mode)| *mode != LocationConfigMode::Detached);
    3145            0 :                 if let Some((node_id, _observed_location, mode)) = maybe_attached {
    3146            0 :                     return Err(ApiError::InternalServerError(anyhow::anyhow!(
    3147            0 :                         "We observed attached={mode:?} tenant in node_id={node_id} shard with tenant_shard_id={shard_id}"
    3148            0 :                     )));
    3149            0 :                 }
    3150              :             }
    3151            0 :             let scheduler = &mut locked.scheduler;
    3152              :             // Right now we only perform the operation on a single node without parallelization
    3153              :             // TODO fan out the operation to multiple nodes for better performance
    3154            0 :             let node_id = scheduler.any_available_node()?;
    3155            0 :             let node = locked
    3156            0 :                 .nodes
    3157            0 :                 .get(&node_id)
    3158            0 :                 .expect("Pageservers may not be deleted while lock is active");
    3159            0 :             node.clone()
    3160            0 :         };
    3161            0 : 
    3162            0 :         // The shard count is encoded in the remote storage's URL, so we need to handle all historically used shard counts
    3163            0 :         let mut counts = time_travel_req
    3164            0 :             .shard_counts
    3165            0 :             .iter()
    3166            0 :             .copied()
    3167            0 :             .collect::<HashSet<_>>()
    3168            0 :             .into_iter()
    3169            0 :             .collect::<Vec<_>>();
    3170            0 :         counts.sort_unstable();
    3171              : 
    3172            0 :         for count in counts {
    3173            0 :             let shard_ids = (0..count.count())
    3174            0 :                 .map(|i| TenantShardId {
    3175            0 :                     tenant_id,
    3176            0 :                     shard_number: ShardNumber(i),
    3177            0 :                     shard_count: count,
    3178            0 :                 })
    3179            0 :                 .collect::<Vec<_>>();
    3180            0 :             for tenant_shard_id in shard_ids {
    3181            0 :                 let client = PageserverClient::new(
    3182            0 :                     node.get_id(),
    3183            0 :                     node.base_url(),
    3184            0 :                     self.config.pageserver_jwt_token.as_deref(),
    3185            0 :                     self.config.ssl_ca_cert.clone(),
    3186            0 :                 )
    3187            0 :                 .map_err(|e| passthrough_api_error(&node, e))?;
    3188              : 
    3189            0 :                 tracing::info!("Doing time travel recovery for shard {tenant_shard_id}",);
    3190              : 
    3191            0 :                 client
    3192            0 :                     .tenant_time_travel_remote_storage(
    3193            0 :                         tenant_shard_id,
    3194            0 :                         &timestamp,
    3195            0 :                         &done_if_after,
    3196            0 :                     )
    3197            0 :                     .await
    3198            0 :                     .map_err(|e| {
    3199            0 :                         ApiError::InternalServerError(anyhow::anyhow!(
    3200            0 :                             "Error doing time travel recovery for shard {tenant_shard_id} on node {}: {e}",
    3201            0 :                             node
    3202            0 :                         ))
    3203            0 :                     })?;
    3204              :             }
    3205              :         }
    3206            0 :         Ok(())
    3207            0 :     }
    3208              : 
    3209            0 :     pub(crate) async fn tenant_secondary_download(
    3210            0 :         &self,
    3211            0 :         tenant_id: TenantId,
    3212            0 :         wait: Option<Duration>,
    3213            0 :     ) -> Result<(StatusCode, SecondaryProgress), ApiError> {
    3214            0 :         let _tenant_lock = trace_shared_lock(
    3215            0 :             &self.tenant_op_locks,
    3216            0 :             tenant_id,
    3217            0 :             TenantOperations::SecondaryDownload,
    3218            0 :         )
    3219            0 :         .await;
    3220              : 
    3221              :         // Acquire lock and yield the collection of shard-node tuples which we will send requests onward to
    3222            0 :         let targets = {
    3223            0 :             let locked = self.inner.read().unwrap();
    3224            0 :             let mut targets = Vec::new();
    3225              : 
    3226            0 :             for (tenant_shard_id, shard) in
    3227            0 :                 locked.tenants.range(TenantShardId::tenant_range(tenant_id))
    3228              :             {
    3229            0 :                 for node_id in shard.intent.get_secondary() {
    3230            0 :                     let node = locked
    3231            0 :                         .nodes
    3232            0 :                         .get(node_id)
    3233            0 :                         .expect("Pageservers may not be deleted while referenced");
    3234            0 : 
    3235            0 :                     targets.push((*tenant_shard_id, node.clone()));
    3236            0 :                 }
    3237              :             }
    3238            0 :             targets
    3239            0 :         };
    3240            0 : 
    3241            0 :         // Issue concurrent requests to all shards' locations
    3242            0 :         let mut futs = FuturesUnordered::new();
    3243            0 :         for (tenant_shard_id, node) in targets {
    3244            0 :             let client = PageserverClient::new(
    3245            0 :                 node.get_id(),
    3246            0 :                 node.base_url(),
    3247            0 :                 self.config.pageserver_jwt_token.as_deref(),
    3248            0 :                 self.config.ssl_ca_cert.clone(),
    3249            0 :             )
    3250            0 :             .map_err(|e| passthrough_api_error(&node, e))?;
    3251            0 :             futs.push(async move {
    3252            0 :                 let result = client
    3253            0 :                     .tenant_secondary_download(tenant_shard_id, wait)
    3254            0 :                     .await;
    3255            0 :                 (result, node, tenant_shard_id)
    3256            0 :             })
    3257              :         }
    3258              : 
    3259              :         // Handle any errors returned by pageservers.  This includes cases like this request racing with
    3260              :         // a scheduling operation, such that the tenant shard we're calling doesn't exist on that pageserver any more, as
    3261              :         // well as more general cases like 503s, 500s, or timeouts.
    3262            0 :         let mut aggregate_progress = SecondaryProgress::default();
    3263            0 :         let mut aggregate_status: Option<StatusCode> = None;
    3264            0 :         let mut error: Option<mgmt_api::Error> = None;
    3265            0 :         while let Some((result, node, tenant_shard_id)) = futs.next().await {
    3266            0 :             match result {
    3267            0 :                 Err(e) => {
    3268            0 :                     // Secondary downloads are always advisory: if something fails, we nevertheless report success, so that whoever
    3269            0 :                     // is calling us will proceed with whatever migration they're doing, albeit with a slightly less warm cache
    3270            0 :                     // than they had hoped for.
    3271            0 :                     tracing::warn!("Secondary download error from pageserver {node}: {e}",);
    3272            0 :                     error = Some(e)
    3273              :                 }
    3274            0 :                 Ok((status_code, progress)) => {
    3275            0 :                     tracing::info!(%tenant_shard_id, "Shard status={status_code} progress: {progress:?}");
    3276            0 :                     aggregate_progress.layers_downloaded += progress.layers_downloaded;
    3277            0 :                     aggregate_progress.layers_total += progress.layers_total;
    3278            0 :                     aggregate_progress.bytes_downloaded += progress.bytes_downloaded;
    3279            0 :                     aggregate_progress.bytes_total += progress.bytes_total;
    3280            0 :                     aggregate_progress.heatmap_mtime =
    3281            0 :                         std::cmp::max(aggregate_progress.heatmap_mtime, progress.heatmap_mtime);
    3282            0 :                     aggregate_status = match aggregate_status {
    3283            0 :                         None => Some(status_code),
    3284            0 :                         Some(StatusCode::OK) => Some(status_code),
    3285            0 :                         Some(cur) => {
    3286            0 :                             // Other status codes (e.g. 202) -- do not overwrite.
    3287            0 :                             Some(cur)
    3288              :                         }
    3289              :                     };
    3290              :                 }
    3291              :             }
    3292              :         }
    3293              : 
    3294              :         // If any of the shards return 202, indicate our result as 202.
    3295            0 :         match aggregate_status {
    3296              :             None => {
    3297            0 :                 match error {
    3298            0 :                     Some(e) => {
    3299            0 :                         // No successes, and an error: surface it
    3300            0 :                         Err(ApiError::Conflict(format!("Error from pageserver: {e}")))
    3301              :                     }
    3302              :                     None => {
    3303              :                         // No shards found
    3304            0 :                         Err(ApiError::NotFound(
    3305            0 :                             anyhow::anyhow!("Tenant {} not found", tenant_id).into(),
    3306            0 :                         ))
    3307              :                     }
    3308              :                 }
    3309              :             }
    3310            0 :             Some(aggregate_status) => Ok((aggregate_status, aggregate_progress)),
    3311              :         }
    3312            0 :     }
    3313              : 
    3314            0 :     pub(crate) async fn tenant_delete(&self, tenant_id: TenantId) -> Result<StatusCode, ApiError> {
    3315            0 :         let _tenant_lock =
    3316            0 :             trace_exclusive_lock(&self.tenant_op_locks, tenant_id, TenantOperations::Delete).await;
    3317              : 
    3318            0 :         self.maybe_load_tenant(tenant_id, &_tenant_lock).await?;
    3319              : 
    3320              :         // Detach all shards. This also deletes local pageserver shard data.
    3321            0 :         let (detach_waiters, node) = {
    3322            0 :             let mut detach_waiters = Vec::new();
    3323            0 :             let mut locked = self.inner.write().unwrap();
    3324            0 :             let (nodes, tenants, scheduler) = locked.parts_mut();
    3325            0 :             for (_, shard) in tenants.range_mut(TenantShardId::tenant_range(tenant_id)) {
    3326              :                 // Update the tenant's intent to remove all attachments
    3327            0 :                 shard.policy = PlacementPolicy::Detached;
    3328            0 :                 shard
    3329            0 :                     .schedule(scheduler, &mut ScheduleContext::default())
    3330            0 :                     .expect("De-scheduling is infallible");
    3331            0 :                 debug_assert!(shard.intent.get_attached().is_none());
    3332            0 :                 debug_assert!(shard.intent.get_secondary().is_empty());
    3333              : 
    3334            0 :                 if let Some(waiter) =
    3335            0 :                     self.maybe_reconcile_shard(shard, nodes, ReconcilerPriority::High)
    3336            0 :                 {
    3337            0 :                     detach_waiters.push(waiter);
    3338            0 :                 }
    3339              :             }
    3340              : 
    3341              :             // Pick an arbitrary node to use for remote deletions (does not have to be where the tenant
    3342              :             // was attached, just has to be able to see the S3 content)
    3343            0 :             let node_id = scheduler.any_available_node()?;
    3344            0 :             let node = nodes
    3345            0 :                 .get(&node_id)
    3346            0 :                 .expect("Pageservers may not be deleted while lock is active");
    3347            0 :             (detach_waiters, node.clone())
    3348            0 :         };
    3349            0 : 
    3350            0 :         // This reconcile wait can fail in a few ways:
    3351            0 :         //  A there is a very long queue for the reconciler semaphore
    3352            0 :         //  B some pageserver is failing to handle a detach promptly
    3353            0 :         //  C some pageserver goes offline right at the moment we send it a request.
    3354            0 :         //
    3355            0 :         // A and C are transient: the semaphore will eventually become available, and once a node is marked offline
    3356            0 :         // the next attempt to reconcile will silently skip detaches for an offline node and succeed.  If B happens,
    3357            0 :         // it's a bug, and needs resolving at the pageserver level (we shouldn't just leave attachments behind while
    3358            0 :         // deleting the underlying data).
    3359            0 :         self.await_waiters(detach_waiters, RECONCILE_TIMEOUT)
    3360            0 :             .await?;
    3361              : 
    3362              :         // Delete the entire tenant (all shards) from remote storage via a random pageserver.
    3363              :         // Passing an unsharded tenant ID will cause the pageserver to remove all remote paths with
    3364              :         // the tenant ID prefix, including all shards (even possibly stale ones).
    3365            0 :         match node
    3366            0 :             .with_client_retries(
    3367            0 :                 |client| async move {
    3368            0 :                     client
    3369            0 :                         .tenant_delete(TenantShardId::unsharded(tenant_id))
    3370            0 :                         .await
    3371            0 :                 },
    3372            0 :                 &self.config.pageserver_jwt_token,
    3373            0 :                 &self.config.ssl_ca_cert,
    3374            0 :                 1,
    3375            0 :                 3,
    3376            0 :                 RECONCILE_TIMEOUT,
    3377            0 :                 &self.cancel,
    3378            0 :             )
    3379            0 :             .await
    3380            0 :             .unwrap_or(Err(mgmt_api::Error::Cancelled))
    3381              :         {
    3382            0 :             Ok(_) => {}
    3383              :             Err(mgmt_api::Error::Cancelled) => {
    3384            0 :                 return Err(ApiError::ShuttingDown);
    3385              :             }
    3386            0 :             Err(e) => {
    3387            0 :                 // This is unexpected: remote deletion should be infallible, unless the object store
    3388            0 :                 // at large is unavailable.
    3389            0 :                 tracing::error!("Error deleting via node {node}: {e}");
    3390            0 :                 return Err(ApiError::InternalServerError(anyhow::anyhow!(e)));
    3391              :             }
    3392              :         }
    3393              : 
    3394              :         // Fall through: deletion of the tenant on pageservers is complete, we may proceed to drop
    3395              :         // our in-memory state and database state.
    3396              : 
    3397              :         // Ordering: we delete persistent state first: if we then
    3398              :         // crash, we will drop the in-memory state.
    3399              : 
    3400              :         // Drop persistent state.
    3401            0 :         self.persistence.delete_tenant(tenant_id).await?;
    3402              : 
    3403              :         // Drop in-memory state
    3404              :         {
    3405            0 :             let mut locked = self.inner.write().unwrap();
    3406            0 :             let (_nodes, tenants, scheduler) = locked.parts_mut();
    3407              : 
    3408              :             // Dereference Scheduler from shards before dropping them
    3409            0 :             for (_tenant_shard_id, shard) in
    3410            0 :                 tenants.range_mut(TenantShardId::tenant_range(tenant_id))
    3411            0 :             {
    3412            0 :                 shard.intent.clear(scheduler);
    3413            0 :             }
    3414              : 
    3415            0 :             tenants.retain(|tenant_shard_id, _shard| tenant_shard_id.tenant_id != tenant_id);
    3416            0 :             tracing::info!(
    3417            0 :                 "Deleted tenant {tenant_id}, now have {} tenants",
    3418            0 :                 locked.tenants.len()
    3419              :             );
    3420              :         };
    3421              : 
    3422              :         // Success is represented as 404, to imitate the existing pageserver deletion API
    3423            0 :         Ok(StatusCode::NOT_FOUND)
    3424            0 :     }
    3425              : 
    3426              :     /// Naming: this configures the storage controller's policies for a tenant, whereas [`Self::tenant_config_set`] is "set the TenantConfig"
    3427              :     /// for a tenant.  The TenantConfig is passed through to pageservers, whereas this function modifies
    3428              :     /// the tenant's policies (configuration) within the storage controller
    3429            0 :     pub(crate) async fn tenant_update_policy(
    3430            0 :         &self,
    3431            0 :         tenant_id: TenantId,
    3432            0 :         req: TenantPolicyRequest,
    3433            0 :     ) -> Result<(), ApiError> {
    3434              :         // We require an exclusive lock, because we are updating persistent and in-memory state
    3435            0 :         let _tenant_lock = trace_exclusive_lock(
    3436            0 :             &self.tenant_op_locks,
    3437            0 :             tenant_id,
    3438            0 :             TenantOperations::UpdatePolicy,
    3439            0 :         )
    3440            0 :         .await;
    3441              : 
    3442            0 :         self.maybe_load_tenant(tenant_id, &_tenant_lock).await?;
    3443              : 
    3444            0 :         failpoint_support::sleep_millis_async!("tenant-update-policy-exclusive-lock");
    3445              : 
    3446              :         let TenantPolicyRequest {
    3447            0 :             placement,
    3448            0 :             mut scheduling,
    3449            0 :         } = req;
    3450              : 
    3451            0 :         if let Some(PlacementPolicy::Detached | PlacementPolicy::Secondary) = placement {
    3452              :             // When someone configures a tenant to detach, we force the scheduling policy to enable
    3453              :             // this to take effect.
    3454            0 :             if scheduling.is_none() {
    3455            0 :                 scheduling = Some(ShardSchedulingPolicy::Active);
    3456            0 :             }
    3457            0 :         }
    3458              : 
    3459            0 :         self.persistence
    3460            0 :             .update_tenant_shard(
    3461            0 :                 TenantFilter::Tenant(tenant_id),
    3462            0 :                 placement.clone(),
    3463            0 :                 None,
    3464            0 :                 None,
    3465            0 :                 scheduling,
    3466            0 :             )
    3467            0 :             .await?;
    3468              : 
    3469            0 :         let mut schedule_context = ScheduleContext::default();
    3470            0 :         let mut locked = self.inner.write().unwrap();
    3471            0 :         let (nodes, tenants, scheduler) = locked.parts_mut();
    3472            0 :         for (shard_id, shard) in tenants.range_mut(TenantShardId::tenant_range(tenant_id)) {
    3473            0 :             if let Some(placement) = &placement {
    3474            0 :                 shard.policy = placement.clone();
    3475            0 : 
    3476            0 :                 tracing::info!(tenant_id=%shard_id.tenant_id, shard_id=%shard_id.shard_slug(),
    3477            0 :                                "Updated placement policy to {placement:?}");
    3478            0 :             }
    3479              : 
    3480            0 :             if let Some(scheduling) = &scheduling {
    3481            0 :                 shard.set_scheduling_policy(*scheduling);
    3482            0 : 
    3483            0 :                 tracing::info!(tenant_id=%shard_id.tenant_id, shard_id=%shard_id.shard_slug(),
    3484            0 :                                "Updated scheduling policy to {scheduling:?}");
    3485            0 :             }
    3486              : 
    3487              :             // In case scheduling is being switched back on, try it now.
    3488            0 :             shard.schedule(scheduler, &mut schedule_context).ok();
    3489            0 :             self.maybe_reconcile_shard(shard, nodes, ReconcilerPriority::High);
    3490              :         }
    3491              : 
    3492            0 :         Ok(())
    3493            0 :     }
    3494              : 
    3495            0 :     pub(crate) async fn tenant_timeline_create_pageservers(
    3496            0 :         &self,
    3497            0 :         tenant_id: TenantId,
    3498            0 :         mut create_req: TimelineCreateRequest,
    3499            0 :     ) -> Result<TimelineInfo, ApiError> {
    3500            0 :         tracing::info!(
    3501            0 :             "Creating timeline {}/{}",
    3502              :             tenant_id,
    3503              :             create_req.new_timeline_id,
    3504              :         );
    3505              : 
    3506            0 :         self.tenant_remote_mutation(tenant_id, move |mut targets| async move {
    3507            0 :             if targets.0.is_empty() {
    3508            0 :                 return Err(ApiError::NotFound(
    3509            0 :                     anyhow::anyhow!("Tenant not found").into(),
    3510            0 :                 ));
    3511            0 :             };
    3512            0 : 
    3513            0 :             let (shard_zero_tid, shard_zero_locations) =
    3514            0 :                 targets.0.pop_first().expect("Must have at least one shard");
    3515            0 :             assert!(shard_zero_tid.is_shard_zero());
    3516              : 
    3517            0 :             async fn create_one(
    3518            0 :                 tenant_shard_id: TenantShardId,
    3519            0 :                 locations: ShardMutationLocations,
    3520            0 :                 jwt: Option<String>,
    3521            0 :                 ssl_ca_cert: Option<Certificate>,
    3522            0 :                 create_req: TimelineCreateRequest,
    3523            0 :             ) -> Result<TimelineInfo, ApiError> {
    3524            0 :                 let latest = locations.latest.node;
    3525            0 : 
    3526            0 :                 tracing::info!(
    3527            0 :                     "Creating timeline on shard {}/{}, attached to node {latest} in generation {:?}",
    3528              :                     tenant_shard_id,
    3529              :                     create_req.new_timeline_id,
    3530              :                     locations.latest.generation
    3531              :                 );
    3532              : 
    3533            0 :                 let client =
    3534            0 :                     PageserverClient::new(latest.get_id(), latest.base_url(), jwt.as_deref(), ssl_ca_cert.clone())
    3535            0 :                     .map_err(|e| passthrough_api_error(&latest, e))?;
    3536              : 
    3537            0 :                 let timeline_info = client
    3538            0 :                     .timeline_create(tenant_shard_id, &create_req)
    3539            0 :                     .await
    3540            0 :                     .map_err(|e| passthrough_api_error(&latest, e))?;
    3541              : 
    3542              :                 // We propagate timeline creations to all attached locations such that a compute
    3543              :                 // for the new timeline is able to start regardless of the current state of the
    3544              :                 // tenant shard reconciliation.
    3545            0 :                 for location in locations.other {
    3546            0 :                     tracing::info!(
    3547            0 :                         "Creating timeline on shard {}/{}, stale attached to node {} in generation {:?}",
    3548              :                         tenant_shard_id,
    3549              :                         create_req.new_timeline_id,
    3550              :                         location.node,
    3551              :                         location.generation
    3552              :                     );
    3553              : 
    3554            0 :                     let client = PageserverClient::new(
    3555            0 :                         location.node.get_id(),
    3556            0 :                         location.node.base_url(),
    3557            0 :                         jwt.as_deref(),
    3558            0 :                         ssl_ca_cert.clone(),
    3559            0 :                     )
    3560            0 :                     .map_err(|e| passthrough_api_error(&location.node, e))?;
    3561              : 
    3562            0 :                     let res = client
    3563            0 :                         .timeline_create(tenant_shard_id, &create_req)
    3564            0 :                         .await;
    3565              : 
    3566            0 :                     if let Err(e) = res {
    3567            0 :                         match e {
    3568            0 :                             mgmt_api::Error::ApiError(StatusCode::NOT_FOUND, _) => {
    3569            0 :                                 // Tenant might have been detached from the stale location,
    3570            0 :                                 // so ignore 404s.
    3571            0 :                             },
    3572              :                             _ => {
    3573            0 :                                 return Err(passthrough_api_error(&location.node, e));
    3574              :                             }
    3575              :                         }
    3576            0 :                     }
    3577              :                 }
    3578              : 
    3579            0 :                 Ok(timeline_info)
    3580            0 :             }
    3581              : 
    3582              :             // Because the caller might not provide an explicit LSN, we must do the creation first on a single shard, and then
    3583              :             // use whatever LSN that shard picked when creating on subsequent shards.  We arbitrarily use shard zero as the shard
    3584              :             // that will get the first creation request, and propagate the LSN to all the >0 shards.
    3585            0 :             let timeline_info = create_one(
    3586            0 :                 shard_zero_tid,
    3587            0 :                 shard_zero_locations,
    3588            0 :                 self.config.pageserver_jwt_token.clone(),
    3589            0 :                 self.config.ssl_ca_cert.clone(),
    3590            0 :                 create_req.clone(),
    3591            0 :             )
    3592            0 :             .await?;
    3593              : 
    3594              :             // Propagate the LSN that shard zero picked, if caller didn't provide one
    3595            0 :             match &mut create_req.mode {
    3596            0 :                 models::TimelineCreateRequestMode::Branch { ancestor_start_lsn, .. } if ancestor_start_lsn.is_none() => {
    3597            0 :                     *ancestor_start_lsn = timeline_info.ancestor_lsn;
    3598            0 :                 },
    3599            0 :                 _ => {}
    3600              :             }
    3601              : 
    3602              :             // Create timeline on remaining shards with number >0
    3603            0 :             if !targets.0.is_empty() {
    3604              :                 // If we had multiple shards, issue requests for the remainder now.
    3605            0 :                 let jwt = &self.config.pageserver_jwt_token;
    3606            0 :                 self.tenant_for_shards(
    3607            0 :                     targets
    3608            0 :                         .0
    3609            0 :                         .iter()
    3610            0 :                         .map(|t| (*t.0, t.1.latest.node.clone()))
    3611            0 :                         .collect(),
    3612            0 :                     |tenant_shard_id: TenantShardId, _node: Node| {
    3613            0 :                         let create_req = create_req.clone();
    3614            0 :                         let mutation_locations = targets.0.remove(&tenant_shard_id).unwrap();
    3615            0 :                         Box::pin(create_one(
    3616            0 :                             tenant_shard_id,
    3617            0 :                             mutation_locations,
    3618            0 :                             jwt.clone(),
    3619            0 :                             self.config.ssl_ca_cert.clone(),
    3620            0 :                             create_req,
    3621            0 :                         ))
    3622            0 :                     },
    3623            0 :                 )
    3624            0 :                 .await?;
    3625            0 :             }
    3626              : 
    3627            0 :             Ok(timeline_info)
    3628            0 :         })
    3629            0 :         .await?
    3630            0 :     }
    3631              : 
    3632              :     /// Timeline creation on safekeepers
    3633              :     ///
    3634              :     /// Returns `Ok(left)` if the timeline has been created on a quorum of safekeepers,
    3635              :     /// where `left` contains the list of safekeepers that didn't have a successful response.
    3636              :     /// Assumes tenant lock is held while calling this function.
    3637            0 :     async fn tenant_timeline_create_safekeepers_quorum(
    3638            0 :         &self,
    3639            0 :         tenant_id: TenantId,
    3640            0 :         timeline_id: TimelineId,
    3641            0 :         pg_version: u32,
    3642            0 :         timeline_persistence: &TimelinePersistence,
    3643            0 :     ) -> Result<Vec<NodeId>, ApiError> {
    3644            0 :         // If quorum is reached, return if we are outside of a specified timeout
    3645            0 :         let jwt = self
    3646            0 :             .config
    3647            0 :             .safekeeper_jwt_token
    3648            0 :             .clone()
    3649            0 :             .map(SecretString::from);
    3650            0 :         let mut joinset = JoinSet::new();
    3651            0 : 
    3652            0 :         let safekeepers = {
    3653            0 :             let locked = self.inner.read().unwrap();
    3654            0 :             locked.safekeepers.clone()
    3655            0 :         };
    3656            0 : 
    3657            0 :         let mut members = Vec::new();
    3658            0 :         for sk_id in timeline_persistence.sk_set.iter() {
    3659            0 :             let sk_id = NodeId(*sk_id as u64);
    3660            0 :             let Some(safekeeper) = safekeepers.get(&sk_id) else {
    3661            0 :                 return Err(ApiError::InternalServerError(anyhow::anyhow!(
    3662            0 :                     "couldn't find entry for safekeeper with id {sk_id}"
    3663            0 :                 )))?;
    3664              :             };
    3665            0 :             members.push(SafekeeperId {
    3666            0 :                 id: sk_id,
    3667            0 :                 host: safekeeper.skp.host.clone(),
    3668            0 :                 pg_port: safekeeper.skp.port as u16,
    3669            0 :             });
    3670              :         }
    3671            0 :         let mset = MemberSet::new(members).map_err(ApiError::InternalServerError)?;
    3672            0 :         let mconf = safekeeper_api::membership::Configuration::new(mset);
    3673            0 : 
    3674            0 :         let req = safekeeper_api::models::TimelineCreateRequest {
    3675            0 :             commit_lsn: None,
    3676            0 :             mconf,
    3677            0 :             pg_version,
    3678            0 :             start_lsn: timeline_persistence.start_lsn.0,
    3679            0 :             system_id: None,
    3680            0 :             tenant_id,
    3681            0 :             timeline_id,
    3682            0 :             wal_seg_size: None,
    3683            0 :         };
    3684              :         const SK_CREATE_TIMELINE_RECONCILE_TIMEOUT: Duration = Duration::from_secs(30);
    3685            0 :         for sk in timeline_persistence.sk_set.iter() {
    3686            0 :             let sk_id = NodeId(*sk as u64);
    3687            0 :             let safekeepers = safekeepers.clone();
    3688            0 :             let jwt = jwt.clone();
    3689            0 :             let ssl_ca_cert = self.config.ssl_ca_cert.clone();
    3690            0 :             let req = req.clone();
    3691            0 :             joinset.spawn(async move {
    3692            0 :                 // Unwrap is fine as we already would have returned error above
    3693            0 :                 let sk_p = safekeepers.get(&sk_id).unwrap();
    3694            0 :                 let res = sk_p
    3695            0 :                     .with_client_retries(
    3696            0 :                         |client| {
    3697            0 :                             let req = req.clone();
    3698            0 :                             async move { client.create_timeline(&req).await }
    3699            0 :                         },
    3700            0 :                         &jwt,
    3701            0 :                         &ssl_ca_cert,
    3702            0 :                         3,
    3703            0 :                         3,
    3704            0 :                         SK_CREATE_TIMELINE_RECONCILE_TIMEOUT,
    3705            0 :                         &CancellationToken::new(),
    3706            0 :                     )
    3707            0 :                     .await;
    3708            0 :                 (sk_id, sk_p.skp.host.clone(), res)
    3709            0 :             });
    3710            0 :         }
    3711              :         // After we have built the joinset, we now wait for the tasks to complete,
    3712              :         // but with a specified timeout to make sure we return swiftly, either with
    3713              :         // a failure or success.
    3714            0 :         let reconcile_deadline = tokio::time::Instant::now() + SK_CREATE_TIMELINE_RECONCILE_TIMEOUT;
    3715            0 : 
    3716            0 :         // Wait until all tasks finish or timeout is hit, whichever occurs
    3717            0 :         // first.
    3718            0 :         let mut reconcile_results = Vec::new();
    3719              :         loop {
    3720            0 :             if let Ok(res) = tokio::time::timeout_at(reconcile_deadline, joinset.join_next()).await
    3721              :             {
    3722            0 :                 let Some(res) = res else { break };
    3723            0 :                 match res {
    3724            0 :                     Ok(res) => {
    3725            0 :                         tracing::info!(
    3726            0 :                             "response from safekeeper id:{} at {}: {:?}",
    3727              :                             res.0,
    3728              :                             res.1,
    3729              :                             res.2
    3730              :                         );
    3731            0 :                         reconcile_results.push(res);
    3732              :                     }
    3733            0 :                     Err(join_err) => {
    3734            0 :                         tracing::info!("join_err for task in joinset: {join_err}");
    3735              :                     }
    3736              :                 }
    3737              :             } else {
    3738            0 :                 tracing::info!(
    3739            0 :                     "timeout for creation call after {} responses",
    3740            0 :                     reconcile_results.len()
    3741              :                 );
    3742            0 :                 break;
    3743              :             }
    3744              :         }
    3745              : 
    3746              :         // Now check now if quorum was reached in reconcile_results.
    3747            0 :         let total_result_count = reconcile_results.len();
    3748            0 :         let remaining = reconcile_results
    3749            0 :             .into_iter()
    3750            0 :             .filter_map(|res| res.2.is_err().then_some(res.0))
    3751            0 :             .collect::<Vec<_>>();
    3752            0 :         tracing::info!(
    3753            0 :             "Got {} non-successful responses from initial creation request of total {total_result_count} responses",
    3754            0 :             remaining.len()
    3755              :         );
    3756            0 :         if remaining.len() >= 2 {
    3757              :             // Failure
    3758            0 :             return Err(ApiError::InternalServerError(anyhow::anyhow!(
    3759            0 :                 "not enough successful reconciliations to reach quorum, please retry: {} errored",
    3760            0 :                 remaining.len()
    3761            0 :             )));
    3762            0 :         }
    3763            0 : 
    3764            0 :         Ok(remaining)
    3765            0 :     }
    3766              : 
    3767              :     /// Create timeline in controller database and on safekeepers.
    3768              :     /// `timeline_info` is result of timeline creation on pageserver.
    3769              :     ///
    3770              :     /// All actions must be idempotent as the call is retried until success. It
    3771              :     /// tries to create timeline in the db and on at least majority of
    3772              :     /// safekeepers + queue creation for safekeepers which missed it in the db
    3773              :     /// for infinite retries; after that, call returns Ok.
    3774              :     ///
    3775              :     /// The idea is that once this is reached as long as we have alive majority
    3776              :     /// of safekeepers it is expected to get eventually operational as storcon
    3777              :     /// will be able to seed timeline on nodes which missed creation by making
    3778              :     /// pull_timeline from peers. On the other hand we don't want to fail
    3779              :     /// timeline creation if one safekeeper is down.
    3780            0 :     async fn tenant_timeline_create_safekeepers(
    3781            0 :         self: &Arc<Self>,
    3782            0 :         tenant_id: TenantId,
    3783            0 :         timeline_info: &TimelineInfo,
    3784            0 :         create_mode: models::TimelineCreateRequestMode,
    3785            0 :     ) -> Result<SafekeepersInfo, ApiError> {
    3786            0 :         let timeline_id = timeline_info.timeline_id;
    3787            0 :         let pg_version = timeline_info.pg_version;
    3788              :         // Initially start_lsn is determined by last_record_lsn in pageserver
    3789              :         // response as it does initdb. However, later we persist it and in sk
    3790              :         // creation calls replace with the value from the timeline row if it
    3791              :         // previously existed as on retries in theory endpoint might have
    3792              :         // already written some data and advanced last_record_lsn, while we want
    3793              :         // safekeepers to have consistent start_lsn.
    3794            0 :         let start_lsn = match create_mode {
    3795            0 :             models::TimelineCreateRequestMode::Bootstrap { .. } => timeline_info.last_record_lsn,
    3796            0 :             models::TimelineCreateRequestMode::Branch { .. } => timeline_info.last_record_lsn,
    3797              :             models::TimelineCreateRequestMode::ImportPgdata { .. } => {
    3798            0 :                 return Err(ApiError::InternalServerError(anyhow::anyhow!(
    3799            0 :                     "import pgdata doesn't specify the start lsn, aborting creation on safekeepers"
    3800            0 :                 )))?;
    3801              :             }
    3802              :         };
    3803              :         // Choose initial set of safekeepers respecting affinity
    3804            0 :         let sks = self.safekeepers_for_new_timeline().await?;
    3805            0 :         let sks_persistence = sks.iter().map(|sk| sk.id.0 as i64).collect::<Vec<_>>();
    3806            0 :         // Add timeline to db
    3807            0 :         let mut timeline_persist = TimelinePersistence {
    3808            0 :             tenant_id: tenant_id.to_string(),
    3809            0 :             timeline_id: timeline_id.to_string(),
    3810            0 :             start_lsn: start_lsn.into(),
    3811            0 :             generation: 0,
    3812            0 :             sk_set: sks_persistence.clone(),
    3813            0 :             new_sk_set: None,
    3814            0 :             cplane_notified_generation: 0,
    3815            0 :             deleted_at: None,
    3816            0 :         };
    3817            0 :         let inserted = self
    3818            0 :             .persistence
    3819            0 :             .insert_timeline(timeline_persist.clone())
    3820            0 :             .await?;
    3821            0 :         if !inserted {
    3822            0 :             if let Some(existent_persist) = self
    3823            0 :                 .persistence
    3824            0 :                 .get_timeline(tenant_id, timeline_id)
    3825            0 :                 .await?
    3826            0 :             {
    3827            0 :                 // Replace with what we have in the db, to get stuff like the generation right.
    3828            0 :                 // We do still repeat the http calls to the safekeepers. After all, we could have
    3829            0 :                 // crashed right after the wrote to the DB.
    3830            0 :                 timeline_persist = existent_persist;
    3831            0 :             } else {
    3832            0 :                 return Err(ApiError::InternalServerError(anyhow::anyhow!(
    3833            0 :                     "insertion said timeline already in db, but looking it up, it was gone"
    3834            0 :                 )));
    3835              :             }
    3836            0 :         }
    3837              :         // Create the timeline on a quorum of safekeepers
    3838            0 :         let remaining = self
    3839            0 :             .tenant_timeline_create_safekeepers_quorum(
    3840            0 :                 tenant_id,
    3841            0 :                 timeline_id,
    3842            0 :                 pg_version,
    3843            0 :                 &timeline_persist,
    3844            0 :             )
    3845            0 :             .await?;
    3846              : 
    3847              :         // For the remaining safekeepers, take care of their reconciliation asynchronously
    3848            0 :         for &remaining_id in remaining.iter() {
    3849            0 :             let pending_op = TimelinePendingOpPersistence {
    3850            0 :                 tenant_id: tenant_id.to_string(),
    3851            0 :                 timeline_id: timeline_id.to_string(),
    3852            0 :                 generation: timeline_persist.generation,
    3853            0 :                 op_kind: crate::persistence::SafekeeperTimelineOpKind::Pull,
    3854            0 :                 sk_id: remaining_id.0 as i64,
    3855            0 :             };
    3856            0 :             tracing::info!("writing pending op for sk id {remaining_id}");
    3857            0 :             self.persistence.insert_pending_op(pending_op).await?;
    3858              :         }
    3859            0 :         if !remaining.is_empty() {
    3860            0 :             let mut locked = self.inner.write().unwrap();
    3861            0 :             for remaining_id in remaining {
    3862            0 :                 let Some(sk) = locked.safekeepers.get(&remaining_id) else {
    3863            0 :                     return Err(ApiError::InternalServerError(anyhow::anyhow!(
    3864            0 :                         "Couldn't find safekeeper with id {remaining_id}"
    3865            0 :                     )));
    3866              :                 };
    3867            0 :                 let Ok(host_list) = sks
    3868            0 :                     .iter()
    3869            0 :                     .map(|sk| {
    3870            0 :                         Ok((
    3871            0 :                             sk.id,
    3872            0 :                             locked
    3873            0 :                                 .safekeepers
    3874            0 :                                 .get(&sk.id)
    3875            0 :                                 .ok_or_else(|| {
    3876            0 :                                     ApiError::InternalServerError(anyhow::anyhow!(
    3877            0 :                                         "Couldn't find safekeeper with id {remaining_id} to pull from"
    3878            0 :                                     ))
    3879            0 :                                 })?
    3880            0 :                                 .base_url(),
    3881              :                         ))
    3882            0 :                     })
    3883            0 :                     .collect::<Result<_, ApiError>>()
    3884              :                 else {
    3885            0 :                     continue;
    3886              :                 };
    3887            0 :                 let req = ScheduleRequest {
    3888            0 :                     safekeeper: Box::new(sk.clone()),
    3889            0 :                     host_list,
    3890            0 :                     tenant_id,
    3891            0 :                     timeline_id,
    3892            0 :                     generation: timeline_persist.generation as u32,
    3893            0 :                     kind: crate::persistence::SafekeeperTimelineOpKind::Pull,
    3894            0 :                 };
    3895            0 :                 locked.safekeeper_reconcilers.schedule_request(self, req);
    3896              :             }
    3897            0 :         }
    3898              : 
    3899            0 :         Ok(SafekeepersInfo {
    3900            0 :             generation: timeline_persist.generation as u32,
    3901            0 :             safekeepers: sks,
    3902            0 :             tenant_id,
    3903            0 :             timeline_id,
    3904            0 :         })
    3905            0 :     }
    3906              : 
    3907            0 :     pub(crate) async fn tenant_timeline_create(
    3908            0 :         self: &Arc<Self>,
    3909            0 :         tenant_id: TenantId,
    3910            0 :         create_req: TimelineCreateRequest,
    3911            0 :     ) -> Result<TimelineCreateResponseStorcon, ApiError> {
    3912            0 :         let safekeepers = self.config.timelines_onto_safekeepers;
    3913            0 :         tracing::info!(
    3914              :             %safekeepers,
    3915            0 :             "Creating timeline {}/{}",
    3916              :             tenant_id,
    3917              :             create_req.new_timeline_id,
    3918              :         );
    3919              : 
    3920            0 :         let _tenant_lock = trace_shared_lock(
    3921            0 :             &self.tenant_op_locks,
    3922            0 :             tenant_id,
    3923            0 :             TenantOperations::TimelineCreate,
    3924            0 :         )
    3925            0 :         .await;
    3926            0 :         failpoint_support::sleep_millis_async!("tenant-create-timeline-shared-lock");
    3927            0 :         let create_mode = create_req.mode.clone();
    3928              : 
    3929            0 :         let timeline_info = self
    3930            0 :             .tenant_timeline_create_pageservers(tenant_id, create_req)
    3931            0 :             .await?;
    3932              : 
    3933            0 :         let safekeepers = if safekeepers {
    3934            0 :             let res = self
    3935            0 :                 .tenant_timeline_create_safekeepers(tenant_id, &timeline_info, create_mode)
    3936            0 :                 .instrument(tracing::info_span!("timeline_create_safekeepers", %tenant_id, timeline_id=%timeline_info.timeline_id))
    3937            0 :                 .await?;
    3938            0 :             Some(res)
    3939              :         } else {
    3940            0 :             None
    3941              :         };
    3942              : 
    3943            0 :         Ok(TimelineCreateResponseStorcon {
    3944            0 :             timeline_info,
    3945            0 :             safekeepers,
    3946            0 :         })
    3947            0 :     }
    3948              : 
    3949            0 :     pub(crate) async fn tenant_timeline_archival_config(
    3950            0 :         &self,
    3951            0 :         tenant_id: TenantId,
    3952            0 :         timeline_id: TimelineId,
    3953            0 :         req: TimelineArchivalConfigRequest,
    3954            0 :     ) -> Result<(), ApiError> {
    3955            0 :         tracing::info!(
    3956            0 :             "Setting archival config of timeline {tenant_id}/{timeline_id} to '{:?}'",
    3957              :             req.state
    3958              :         );
    3959              : 
    3960            0 :         let _tenant_lock = trace_shared_lock(
    3961            0 :             &self.tenant_op_locks,
    3962            0 :             tenant_id,
    3963            0 :             TenantOperations::TimelineArchivalConfig,
    3964            0 :         )
    3965            0 :         .await;
    3966              : 
    3967            0 :         self.tenant_remote_mutation(tenant_id, move |targets| async move {
    3968            0 :             if targets.0.is_empty() {
    3969            0 :                 return Err(ApiError::NotFound(
    3970            0 :                     anyhow::anyhow!("Tenant not found").into(),
    3971            0 :                 ));
    3972            0 :             }
    3973            0 :             async fn config_one(
    3974            0 :                 tenant_shard_id: TenantShardId,
    3975            0 :                 timeline_id: TimelineId,
    3976            0 :                 node: Node,
    3977            0 :                 jwt: Option<String>,
    3978            0 :                 ssl_ca_cert: Option<Certificate>,
    3979            0 :                 req: TimelineArchivalConfigRequest,
    3980            0 :             ) -> Result<(), ApiError> {
    3981            0 :                 tracing::info!(
    3982            0 :                     "Setting archival config of timeline on shard {tenant_shard_id}/{timeline_id}, attached to node {node}",
    3983              :                 );
    3984              : 
    3985            0 :                 let client = PageserverClient::new(node.get_id(), node.base_url(), jwt.as_deref(), ssl_ca_cert)
    3986            0 :                     .map_err(|e| passthrough_api_error(&node, e))?;
    3987              : 
    3988            0 :                 client
    3989            0 :                     .timeline_archival_config(tenant_shard_id, timeline_id, &req)
    3990            0 :                     .await
    3991            0 :                     .map_err(|e| match e {
    3992            0 :                         mgmt_api::Error::ApiError(StatusCode::PRECONDITION_FAILED, msg) => {
    3993            0 :                             ApiError::PreconditionFailed(msg.into_boxed_str())
    3994              :                         }
    3995            0 :                         _ => passthrough_api_error(&node, e),
    3996            0 :                     })
    3997            0 :             }
    3998              : 
    3999              :             // no shard needs to go first/last; the operation should be idempotent
    4000              :             // TODO: it would be great to ensure that all shards return the same error
    4001            0 :             let locations = targets.0.iter().map(|t| (*t.0, t.1.latest.node.clone())).collect();
    4002            0 :             let results = self
    4003            0 :                 .tenant_for_shards(locations, |tenant_shard_id, node| {
    4004            0 :                     futures::FutureExt::boxed(config_one(
    4005            0 :                         tenant_shard_id,
    4006            0 :                         timeline_id,
    4007            0 :                         node,
    4008            0 :                         self.config.pageserver_jwt_token.clone(),
    4009            0 :                         self.config.ssl_ca_cert.clone(),
    4010            0 :                         req.clone(),
    4011            0 :                     ))
    4012            0 :                 })
    4013            0 :                 .await?;
    4014            0 :             assert!(!results.is_empty(), "must have at least one result");
    4015              : 
    4016            0 :             Ok(())
    4017            0 :         }).await?
    4018            0 :     }
    4019              : 
    4020            0 :     pub(crate) async fn tenant_timeline_detach_ancestor(
    4021            0 :         &self,
    4022            0 :         tenant_id: TenantId,
    4023            0 :         timeline_id: TimelineId,
    4024            0 :     ) -> Result<models::detach_ancestor::AncestorDetached, ApiError> {
    4025            0 :         tracing::info!("Detaching timeline {tenant_id}/{timeline_id}",);
    4026              : 
    4027            0 :         let _tenant_lock = trace_shared_lock(
    4028            0 :             &self.tenant_op_locks,
    4029            0 :             tenant_id,
    4030            0 :             TenantOperations::TimelineDetachAncestor,
    4031            0 :         )
    4032            0 :         .await;
    4033              : 
    4034            0 :         self.tenant_remote_mutation(tenant_id, move |targets| async move {
    4035            0 :             if targets.0.is_empty() {
    4036            0 :                 return Err(ApiError::NotFound(
    4037            0 :                     anyhow::anyhow!("Tenant not found").into(),
    4038            0 :                 ));
    4039            0 :             }
    4040              : 
    4041            0 :             async fn detach_one(
    4042            0 :                 tenant_shard_id: TenantShardId,
    4043            0 :                 timeline_id: TimelineId,
    4044            0 :                 node: Node,
    4045            0 :                 jwt: Option<String>,
    4046            0 :                 ssl_ca_cert: Option<Certificate>,
    4047            0 :             ) -> Result<(ShardNumber, models::detach_ancestor::AncestorDetached), ApiError> {
    4048            0 :                 tracing::info!(
    4049            0 :                     "Detaching timeline on shard {tenant_shard_id}/{timeline_id}, attached to node {node}",
    4050              :                 );
    4051              : 
    4052            0 :                 let client = PageserverClient::new(node.get_id(), node.base_url(), jwt.as_deref(), ssl_ca_cert)
    4053            0 :                     .map_err(|e| passthrough_api_error(&node, e))?;
    4054              : 
    4055            0 :                 client
    4056            0 :                     .timeline_detach_ancestor(tenant_shard_id, timeline_id)
    4057            0 :                     .await
    4058            0 :                     .map_err(|e| {
    4059              :                         use mgmt_api::Error;
    4060              : 
    4061            0 :                         match e {
    4062              :                             // no ancestor (ever)
    4063            0 :                             Error::ApiError(StatusCode::CONFLICT, msg) => ApiError::Conflict(format!(
    4064            0 :                                 "{node}: {}",
    4065            0 :                                 msg.strip_prefix("Conflict: ").unwrap_or(&msg)
    4066            0 :                             )),
    4067              :                             // too many ancestors
    4068            0 :                             Error::ApiError(StatusCode::BAD_REQUEST, msg) => {
    4069            0 :                                 ApiError::BadRequest(anyhow::anyhow!("{node}: {msg}"))
    4070              :                             }
    4071            0 :                             Error::ApiError(StatusCode::INTERNAL_SERVER_ERROR, msg) => {
    4072            0 :                                 // avoid turning these into conflicts to remain compatible with
    4073            0 :                                 // pageservers, 500 errors are sadly retryable with timeline ancestor
    4074            0 :                                 // detach
    4075            0 :                                 ApiError::InternalServerError(anyhow::anyhow!("{node}: {msg}"))
    4076              :                             }
    4077              :                             // rest can be mapped as usual
    4078            0 :                             other => passthrough_api_error(&node, other),
    4079              :                         }
    4080            0 :                     })
    4081            0 :                     .map(|res| (tenant_shard_id.shard_number, res))
    4082            0 :             }
    4083              : 
    4084              :             // no shard needs to go first/last; the operation should be idempotent
    4085            0 :             let locations = targets.0.iter().map(|t| (*t.0, t.1.latest.node.clone())).collect();
    4086            0 :             let mut results = self
    4087            0 :                 .tenant_for_shards(locations, |tenant_shard_id, node| {
    4088            0 :                     futures::FutureExt::boxed(detach_one(
    4089            0 :                         tenant_shard_id,
    4090            0 :                         timeline_id,
    4091            0 :                         node,
    4092            0 :                         self.config.pageserver_jwt_token.clone(),
    4093            0 :                         self.config.ssl_ca_cert.clone(),
    4094            0 :                     ))
    4095            0 :                 })
    4096            0 :                 .await?;
    4097              : 
    4098            0 :             let any = results.pop().expect("we must have at least one response");
    4099            0 : 
    4100            0 :             let mismatching = results
    4101            0 :                 .iter()
    4102            0 :                 .filter(|(_, res)| res != &any.1)
    4103            0 :                 .collect::<Vec<_>>();
    4104            0 :             if !mismatching.is_empty() {
    4105              :                 // this can be hit by races which should not happen because operation lock on cplane
    4106            0 :                 let matching = results.len() - mismatching.len();
    4107            0 :                 tracing::error!(
    4108              :                     matching,
    4109              :                     compared_against=?any,
    4110              :                     ?mismatching,
    4111            0 :                     "shards returned different results"
    4112              :                 );
    4113              : 
    4114            0 :                 return Err(ApiError::InternalServerError(anyhow::anyhow!("pageservers returned mixed results for ancestor detach; manual intervention is required.")));
    4115            0 :             }
    4116            0 : 
    4117            0 :             Ok(any.1)
    4118            0 :         }).await?
    4119            0 :     }
    4120              : 
    4121            0 :     pub(crate) async fn tenant_timeline_block_unblock_gc(
    4122            0 :         &self,
    4123            0 :         tenant_id: TenantId,
    4124            0 :         timeline_id: TimelineId,
    4125            0 :         dir: BlockUnblock,
    4126            0 :     ) -> Result<(), ApiError> {
    4127            0 :         let _tenant_lock = trace_shared_lock(
    4128            0 :             &self.tenant_op_locks,
    4129            0 :             tenant_id,
    4130            0 :             TenantOperations::TimelineGcBlockUnblock,
    4131            0 :         )
    4132            0 :         .await;
    4133              : 
    4134            0 :         self.tenant_remote_mutation(tenant_id, move |targets| async move {
    4135            0 :             if targets.0.is_empty() {
    4136            0 :                 return Err(ApiError::NotFound(
    4137            0 :                     anyhow::anyhow!("Tenant not found").into(),
    4138            0 :                 ));
    4139            0 :             }
    4140              : 
    4141            0 :             async fn do_one(
    4142            0 :                 tenant_shard_id: TenantShardId,
    4143            0 :                 timeline_id: TimelineId,
    4144            0 :                 node: Node,
    4145            0 :                 jwt: Option<String>,
    4146            0 :                 ssl_ca_cert: Option<Certificate>,
    4147            0 :                 dir: BlockUnblock,
    4148            0 :             ) -> Result<(), ApiError> {
    4149            0 :                 let client = PageserverClient::new(
    4150            0 :                     node.get_id(),
    4151            0 :                     node.base_url(),
    4152            0 :                     jwt.as_deref(),
    4153            0 :                     ssl_ca_cert,
    4154            0 :                 )
    4155            0 :                 .map_err(|e| passthrough_api_error(&node, e))?;
    4156              : 
    4157            0 :                 client
    4158            0 :                     .timeline_block_unblock_gc(tenant_shard_id, timeline_id, dir)
    4159            0 :                     .await
    4160            0 :                     .map_err(|e| passthrough_api_error(&node, e))
    4161            0 :             }
    4162              : 
    4163              :             // no shard needs to go first/last; the operation should be idempotent
    4164            0 :             let locations = targets
    4165            0 :                 .0
    4166            0 :                 .iter()
    4167            0 :                 .map(|t| (*t.0, t.1.latest.node.clone()))
    4168            0 :                 .collect();
    4169            0 :             self.tenant_for_shards(locations, |tenant_shard_id, node| {
    4170            0 :                 futures::FutureExt::boxed(do_one(
    4171            0 :                     tenant_shard_id,
    4172            0 :                     timeline_id,
    4173            0 :                     node,
    4174            0 :                     self.config.pageserver_jwt_token.clone(),
    4175            0 :                     self.config.ssl_ca_cert.clone(),
    4176            0 :                     dir,
    4177            0 :                 ))
    4178            0 :             })
    4179            0 :             .await
    4180            0 :         })
    4181            0 :         .await??;
    4182            0 :         Ok(())
    4183            0 :     }
    4184              : 
    4185            0 :     pub(crate) async fn tenant_timeline_download_heatmap_layers(
    4186            0 :         &self,
    4187            0 :         tenant_shard_id: TenantShardId,
    4188            0 :         timeline_id: TimelineId,
    4189            0 :         concurrency: Option<usize>,
    4190            0 :         recurse: bool,
    4191            0 :     ) -> Result<(), ApiError> {
    4192            0 :         let _tenant_lock = trace_shared_lock(
    4193            0 :             &self.tenant_op_locks,
    4194            0 :             tenant_shard_id.tenant_id,
    4195            0 :             TenantOperations::DownloadHeatmapLayers,
    4196            0 :         )
    4197            0 :         .await;
    4198              : 
    4199            0 :         let targets = {
    4200            0 :             let locked = self.inner.read().unwrap();
    4201            0 :             let mut targets = Vec::new();
    4202              : 
    4203              :             // If the request got an unsharded tenant id, then apply
    4204              :             // the operation to all shards. Otherwise, apply it to a specific shard.
    4205            0 :             let shards_range = if tenant_shard_id.is_unsharded() {
    4206            0 :                 TenantShardId::tenant_range(tenant_shard_id.tenant_id)
    4207              :             } else {
    4208            0 :                 tenant_shard_id.range()
    4209              :             };
    4210              : 
    4211            0 :             for (tenant_shard_id, shard) in locked.tenants.range(shards_range) {
    4212            0 :                 if let Some(node_id) = shard.intent.get_attached() {
    4213            0 :                     let node = locked
    4214            0 :                         .nodes
    4215            0 :                         .get(node_id)
    4216            0 :                         .expect("Pageservers may not be deleted while referenced");
    4217            0 : 
    4218            0 :                     targets.push((*tenant_shard_id, node.clone()));
    4219            0 :                 }
    4220              :             }
    4221            0 :             targets
    4222            0 :         };
    4223            0 : 
    4224            0 :         self.tenant_for_shards_api(
    4225            0 :             targets,
    4226            0 :             |tenant_shard_id, client| async move {
    4227            0 :                 client
    4228            0 :                     .timeline_download_heatmap_layers(
    4229            0 :                         tenant_shard_id,
    4230            0 :                         timeline_id,
    4231            0 :                         concurrency,
    4232            0 :                         recurse,
    4233            0 :                     )
    4234            0 :                     .await
    4235            0 :             },
    4236            0 :             1,
    4237            0 :             1,
    4238            0 :             SHORT_RECONCILE_TIMEOUT,
    4239            0 :             &self.cancel,
    4240            0 :         )
    4241            0 :         .await;
    4242              : 
    4243            0 :         Ok(())
    4244            0 :     }
    4245              : 
    4246              :     /// Helper for concurrently calling a pageserver API on a number of shards, such as timeline creation.
    4247              :     ///
    4248              :     /// On success, the returned vector contains exactly the same number of elements as the input `locations`.
    4249            0 :     async fn tenant_for_shards<F, R>(
    4250            0 :         &self,
    4251            0 :         locations: Vec<(TenantShardId, Node)>,
    4252            0 :         mut req_fn: F,
    4253            0 :     ) -> Result<Vec<R>, ApiError>
    4254            0 :     where
    4255            0 :         F: FnMut(
    4256            0 :             TenantShardId,
    4257            0 :             Node,
    4258            0 :         )
    4259            0 :             -> std::pin::Pin<Box<dyn futures::Future<Output = Result<R, ApiError>> + Send>>,
    4260            0 :     {
    4261            0 :         let mut futs = FuturesUnordered::new();
    4262            0 :         let mut results = Vec::with_capacity(locations.len());
    4263              : 
    4264            0 :         for (tenant_shard_id, node) in locations {
    4265            0 :             futs.push(req_fn(tenant_shard_id, node));
    4266            0 :         }
    4267              : 
    4268            0 :         while let Some(r) = futs.next().await {
    4269            0 :             results.push(r?);
    4270              :         }
    4271              : 
    4272            0 :         Ok(results)
    4273            0 :     }
    4274              : 
    4275              :     /// Concurrently invoke a pageserver API call on many shards at once
    4276            0 :     pub(crate) async fn tenant_for_shards_api<T, O, F>(
    4277            0 :         &self,
    4278            0 :         locations: Vec<(TenantShardId, Node)>,
    4279            0 :         op: O,
    4280            0 :         warn_threshold: u32,
    4281            0 :         max_retries: u32,
    4282            0 :         timeout: Duration,
    4283            0 :         cancel: &CancellationToken,
    4284            0 :     ) -> Vec<mgmt_api::Result<T>>
    4285            0 :     where
    4286            0 :         O: Fn(TenantShardId, PageserverClient) -> F + Copy,
    4287            0 :         F: std::future::Future<Output = mgmt_api::Result<T>>,
    4288            0 :     {
    4289            0 :         let mut futs = FuturesUnordered::new();
    4290            0 :         let mut results = Vec::with_capacity(locations.len());
    4291              : 
    4292            0 :         for (tenant_shard_id, node) in locations {
    4293            0 :             futs.push(async move {
    4294            0 :                 node.with_client_retries(
    4295            0 :                     |client| op(tenant_shard_id, client),
    4296            0 :                     &self.config.pageserver_jwt_token,
    4297            0 :                     &self.config.ssl_ca_cert,
    4298            0 :                     warn_threshold,
    4299            0 :                     max_retries,
    4300            0 :                     timeout,
    4301            0 :                     cancel,
    4302            0 :                 )
    4303            0 :                 .await
    4304            0 :             });
    4305            0 :         }
    4306              : 
    4307            0 :         while let Some(r) = futs.next().await {
    4308            0 :             let r = r.unwrap_or(Err(mgmt_api::Error::Cancelled));
    4309            0 :             results.push(r);
    4310            0 :         }
    4311              : 
    4312            0 :         results
    4313            0 :     }
    4314              : 
    4315              :     /// Helper for safely working with the shards in a tenant remotely on pageservers, for example
    4316              :     /// when creating and deleting timelines:
    4317              :     /// - Makes sure shards are attached somewhere if they weren't already
    4318              :     /// - Looks up the shards and the nodes where they were most recently attached
    4319              :     /// - Guarantees that after the inner function returns, the shards' generations haven't moved on: this
    4320              :     ///   ensures that the remote operation acted on the most recent generation, and is therefore durable.
    4321            0 :     async fn tenant_remote_mutation<R, O, F>(
    4322            0 :         &self,
    4323            0 :         tenant_id: TenantId,
    4324            0 :         op: O,
    4325            0 :     ) -> Result<R, ApiError>
    4326            0 :     where
    4327            0 :         O: FnOnce(TenantMutationLocations) -> F,
    4328            0 :         F: std::future::Future<Output = R>,
    4329            0 :     {
    4330            0 :         let mutation_locations = {
    4331            0 :             let mut locations = TenantMutationLocations::default();
    4332              : 
    4333              :             // Load the currently attached pageservers for the latest generation of each shard.  This can
    4334              :             // run concurrently with reconciliations, and it is not guaranteed that the node we find here
    4335              :             // will still be the latest when we're done: we will check generations again at the end of
    4336              :             // this function to handle that.
    4337            0 :             let generations = self.persistence.tenant_generations(tenant_id).await?;
    4338              : 
    4339            0 :             if generations
    4340            0 :                 .iter()
    4341            0 :                 .any(|i| i.generation.is_none() || i.generation_pageserver.is_none())
    4342              :             {
    4343            0 :                 let shard_generations = generations
    4344            0 :                     .into_iter()
    4345            0 :                     .map(|i| (i.tenant_shard_id, (i.generation, i.generation_pageserver)))
    4346            0 :                     .collect::<HashMap<_, _>>();
    4347            0 : 
    4348            0 :                 // One or more shards has not been attached to a pageserver.  Check if this is because it's configured
    4349            0 :                 // to be detached (409: caller should give up), or because it's meant to be attached but isn't yet (503: caller should retry)
    4350            0 :                 let locked = self.inner.read().unwrap();
    4351            0 :                 for (shard_id, shard) in
    4352            0 :                     locked.tenants.range(TenantShardId::tenant_range(tenant_id))
    4353              :                 {
    4354            0 :                     match shard.policy {
    4355              :                         PlacementPolicy::Attached(_) => {
    4356              :                             // This shard is meant to be attached: the caller is not wrong to try and
    4357              :                             // use this function, but we can't service the request right now.
    4358            0 :                             let Some(generation) = shard_generations.get(shard_id) else {
    4359              :                                 // This can only happen if there is a split brain controller modifying the database.  This should
    4360              :                                 // never happen when testing, and if it happens in production we can only log the issue.
    4361            0 :                                 debug_assert!(false);
    4362            0 :                                 tracing::error!(
    4363            0 :                                     "Shard {shard_id} not found in generation state!  Is another rogue controller running?"
    4364              :                                 );
    4365            0 :                                 continue;
    4366              :                             };
    4367            0 :                             let (generation, generation_pageserver) = generation;
    4368            0 :                             if let Some(generation) = generation {
    4369            0 :                                 if generation_pageserver.is_none() {
    4370              :                                     // This is legitimate only in a very narrow window where the shard was only just configured into
    4371              :                                     // Attached mode after being created in Secondary or Detached mode, and it has had its generation
    4372              :                                     // set but not yet had a Reconciler run (reconciler is the only thing that sets generation_pageserver).
    4373            0 :                                     tracing::warn!(
    4374            0 :                                         "Shard {shard_id} generation is set ({generation:?}) but generation_pageserver is None, reconciler not run yet?"
    4375              :                                     );
    4376            0 :                                 }
    4377              :                             } else {
    4378              :                                 // This should never happen: a shard with no generation is only permitted when it was created in some state
    4379              :                                 // other than PlacementPolicy::Attached (and generation is always written to DB before setting Attached in memory)
    4380            0 :                                 debug_assert!(false);
    4381            0 :                                 tracing::error!(
    4382            0 :                                     "Shard {shard_id} generation is None, but it is in PlacementPolicy::Attached mode!"
    4383              :                                 );
    4384            0 :                                 continue;
    4385              :                             }
    4386              :                         }
    4387              :                         PlacementPolicy::Secondary | PlacementPolicy::Detached => {
    4388            0 :                             return Err(ApiError::Conflict(format!(
    4389            0 :                                 "Shard {shard_id} tenant has policy {:?}",
    4390            0 :                                 shard.policy
    4391            0 :                             )));
    4392              :                         }
    4393              :                     }
    4394              :                 }
    4395              : 
    4396            0 :                 return Err(ApiError::ResourceUnavailable(
    4397            0 :                     "One or more shards in tenant is not yet attached".into(),
    4398            0 :                 ));
    4399            0 :             }
    4400            0 : 
    4401            0 :             let locked = self.inner.read().unwrap();
    4402              :             for ShardGenerationState {
    4403            0 :                 tenant_shard_id,
    4404            0 :                 generation,
    4405            0 :                 generation_pageserver,
    4406            0 :             } in generations
    4407              :             {
    4408            0 :                 let node_id = generation_pageserver.expect("We checked for None above");
    4409            0 :                 let node = locked
    4410            0 :                     .nodes
    4411            0 :                     .get(&node_id)
    4412            0 :                     .ok_or(ApiError::Conflict(format!(
    4413            0 :                         "Raced with removal of node {node_id}"
    4414            0 :                     )))?;
    4415            0 :                 let generation = generation.expect("Checked above");
    4416            0 : 
    4417            0 :                 let tenant = locked.tenants.get(&tenant_shard_id);
    4418              : 
    4419              :                 // TODO(vlad): Abstract the logic that finds stale attached locations
    4420              :                 // from observed state into a [`Service`] method.
    4421            0 :                 let other_locations = match tenant {
    4422            0 :                     Some(tenant) => {
    4423            0 :                         let mut other = tenant.attached_locations();
    4424            0 :                         let latest_location_index =
    4425            0 :                             other.iter().position(|&l| l == (node.get_id(), generation));
    4426            0 :                         if let Some(idx) = latest_location_index {
    4427            0 :                             other.remove(idx);
    4428            0 :                         }
    4429              : 
    4430            0 :                         other
    4431              :                     }
    4432            0 :                     None => Vec::default(),
    4433              :                 };
    4434              : 
    4435            0 :                 let location = ShardMutationLocations {
    4436            0 :                     latest: MutationLocation {
    4437            0 :                         node: node.clone(),
    4438            0 :                         generation,
    4439            0 :                     },
    4440            0 :                     other: other_locations
    4441            0 :                         .into_iter()
    4442            0 :                         .filter_map(|(node_id, generation)| {
    4443            0 :                             let node = locked.nodes.get(&node_id)?;
    4444              : 
    4445            0 :                             Some(MutationLocation {
    4446            0 :                                 node: node.clone(),
    4447            0 :                                 generation,
    4448            0 :                             })
    4449            0 :                         })
    4450            0 :                         .collect(),
    4451            0 :                 };
    4452            0 :                 locations.0.insert(tenant_shard_id, location);
    4453            0 :             }
    4454              : 
    4455            0 :             locations
    4456              :         };
    4457              : 
    4458            0 :         let result = op(mutation_locations.clone()).await;
    4459              : 
    4460              :         // Post-check: are all the generations of all the shards the same as they were initially?  This proves that
    4461              :         // our remote operation executed on the latest generation and is therefore persistent.
    4462              :         {
    4463            0 :             let latest_generations = self.persistence.tenant_generations(tenant_id).await?;
    4464            0 :             if latest_generations
    4465            0 :                 .into_iter()
    4466            0 :                 .map(
    4467            0 :                     |ShardGenerationState {
    4468              :                          tenant_shard_id,
    4469              :                          generation,
    4470              :                          generation_pageserver: _,
    4471            0 :                      }| (tenant_shard_id, generation),
    4472            0 :                 )
    4473            0 :                 .collect::<Vec<_>>()
    4474            0 :                 != mutation_locations
    4475            0 :                     .0
    4476            0 :                     .into_iter()
    4477            0 :                     .map(|i| (i.0, Some(i.1.latest.generation)))
    4478            0 :                     .collect::<Vec<_>>()
    4479              :             {
    4480              :                 // We raced with something that incremented the generation, and therefore cannot be
    4481              :                 // confident that our actions are persistent (they might have hit an old generation).
    4482              :                 //
    4483              :                 // This is safe but requires a retry: ask the client to do that by giving them a 503 response.
    4484            0 :                 return Err(ApiError::ResourceUnavailable(
    4485            0 :                     "Tenant attachment changed, please retry".into(),
    4486            0 :                 ));
    4487            0 :             }
    4488            0 :         }
    4489            0 : 
    4490            0 :         Ok(result)
    4491            0 :     }
    4492              : 
    4493            0 :     pub(crate) async fn tenant_timeline_delete(
    4494            0 :         self: &Arc<Self>,
    4495            0 :         tenant_id: TenantId,
    4496            0 :         timeline_id: TimelineId,
    4497            0 :     ) -> Result<StatusCode, ApiError> {
    4498            0 :         tracing::info!("Deleting timeline {}/{}", tenant_id, timeline_id,);
    4499            0 :         let _tenant_lock = trace_shared_lock(
    4500            0 :             &self.tenant_op_locks,
    4501            0 :             tenant_id,
    4502            0 :             TenantOperations::TimelineDelete,
    4503            0 :         )
    4504            0 :         .await;
    4505              : 
    4506            0 :         let status_code = self.tenant_remote_mutation(tenant_id, move |mut targets| async move {
    4507            0 :             if targets.0.is_empty() {
    4508            0 :                 return Err(ApiError::NotFound(
    4509            0 :                     anyhow::anyhow!("Tenant not found").into(),
    4510            0 :                 ));
    4511            0 :             }
    4512            0 : 
    4513            0 :             let (shard_zero_tid, shard_zero_locations) = targets.0.pop_first().expect("Must have at least one shard");
    4514            0 :             assert!(shard_zero_tid.is_shard_zero());
    4515              : 
    4516            0 :             async fn delete_one(
    4517            0 :                 tenant_shard_id: TenantShardId,
    4518            0 :                 timeline_id: TimelineId,
    4519            0 :                 node: Node,
    4520            0 :                 jwt: Option<String>,
    4521            0 :                 ssl_ca_cert: Option<Certificate>,
    4522            0 :             ) -> Result<StatusCode, ApiError> {
    4523            0 :                 tracing::info!(
    4524            0 :                     "Deleting timeline on shard {tenant_shard_id}/{timeline_id}, attached to node {node}",
    4525              :                 );
    4526              : 
    4527            0 :                 let client = PageserverClient::new(node.get_id(), node.base_url(), jwt.as_deref(), ssl_ca_cert)
    4528            0 :                     .map_err(|e| passthrough_api_error(&node, e))?;
    4529            0 :                 let res = client
    4530            0 :                     .timeline_delete(tenant_shard_id, timeline_id)
    4531            0 :                     .await;
    4532              : 
    4533            0 :                 match res {
    4534            0 :                     Ok(ok) => Ok(ok),
    4535            0 :                     Err(mgmt_api::Error::ApiError(StatusCode::CONFLICT, _)) => Ok(StatusCode::CONFLICT),
    4536            0 :                     Err(mgmt_api::Error::ApiError(StatusCode::SERVICE_UNAVAILABLE, msg)) => Err(ApiError::ResourceUnavailable(msg.into())),
    4537            0 :                     Err(e) => {
    4538            0 :                         Err(
    4539            0 :                             ApiError::InternalServerError(anyhow::anyhow!(
    4540            0 :                                 "Error deleting timeline {timeline_id} on {tenant_shard_id} on node {node}: {e}",
    4541            0 :                             ))
    4542            0 :                         )
    4543              :                     }
    4544              :                 }
    4545            0 :             }
    4546              : 
    4547            0 :             let locations = targets.0.iter().map(|t| (*t.0, t.1.latest.node.clone())).collect();
    4548            0 :             let statuses = self
    4549            0 :                 .tenant_for_shards(locations, |tenant_shard_id: TenantShardId, node: Node| {
    4550            0 :                     Box::pin(delete_one(
    4551            0 :                         tenant_shard_id,
    4552            0 :                         timeline_id,
    4553            0 :                         node,
    4554            0 :                         self.config.pageserver_jwt_token.clone(),
    4555            0 :                         self.config.ssl_ca_cert.clone(),
    4556            0 :                     ))
    4557            0 :                 })
    4558            0 :                 .await?;
    4559              : 
    4560              :             // If any shards >0 haven't finished deletion yet, don't start deletion on shard zero.
    4561              :             // We return 409 (Conflict) if deletion was already in progress on any of the shards
    4562              :             // and 202 (Accepted) if deletion was not already in progress on any of the shards.
    4563            0 :             if statuses.iter().any(|s| s == &StatusCode::CONFLICT) {
    4564            0 :                 return Ok(StatusCode::CONFLICT);
    4565            0 :             }
    4566            0 : 
    4567            0 :             if statuses.iter().any(|s| s != &StatusCode::NOT_FOUND) {
    4568            0 :                 return Ok(StatusCode::ACCEPTED);
    4569            0 :             }
    4570              : 
    4571              :             // Delete shard zero last: this is not strictly necessary, but since a caller's GET on a timeline will be routed
    4572              :             // to shard zero, it gives a more obvious behavior that a GET returns 404 once the deletion is done.
    4573            0 :             let shard_zero_status = delete_one(
    4574            0 :                 shard_zero_tid,
    4575            0 :                 timeline_id,
    4576            0 :                 shard_zero_locations.latest.node,
    4577            0 :                 self.config.pageserver_jwt_token.clone(),
    4578            0 :                 self.config.ssl_ca_cert.clone(),
    4579            0 :             )
    4580            0 :             .await?;
    4581            0 :             Ok(shard_zero_status)
    4582            0 :         }).await?;
    4583              : 
    4584            0 :         self.tenant_timeline_delete_safekeepers(tenant_id, timeline_id)
    4585            0 :             .await?;
    4586              : 
    4587            0 :         status_code
    4588            0 :     }
    4589              :     /// Perform timeline deletion on safekeepers. Will return success: we persist the deletion into the reconciler.
    4590            0 :     async fn tenant_timeline_delete_safekeepers(
    4591            0 :         self: &Arc<Self>,
    4592            0 :         tenant_id: TenantId,
    4593            0 :         timeline_id: TimelineId,
    4594            0 :     ) -> Result<(), ApiError> {
    4595            0 :         let tl = self
    4596            0 :             .persistence
    4597            0 :             .get_timeline(tenant_id, timeline_id)
    4598            0 :             .await?;
    4599            0 :         let Some(tl) = tl else {
    4600            0 :             tracing::info!(
    4601            0 :                 "timeline {tenant_id}/{timeline_id} doesn't exist in timelines table, no deletions on safekeepers needed"
    4602              :             );
    4603            0 :             return Ok(());
    4604              :         };
    4605            0 :         let all_sks = tl
    4606            0 :             .new_sk_set
    4607            0 :             .iter()
    4608            0 :             .flat_map(|sks| {
    4609            0 :                 sks.iter()
    4610            0 :                     .map(|sk| (*sk, SafekeeperTimelineOpKind::Exclude))
    4611            0 :             })
    4612            0 :             .chain(
    4613            0 :                 tl.sk_set
    4614            0 :                     .iter()
    4615            0 :                     .map(|v| (*v, SafekeeperTimelineOpKind::Delete)),
    4616            0 :             )
    4617            0 :             .collect::<HashMap<_, _>>();
    4618            0 : 
    4619            0 :         // Schedule reconciliations
    4620            0 :         {
    4621            0 :             let mut locked = self.inner.write().unwrap();
    4622            0 :             for (sk_id, kind) in all_sks {
    4623            0 :                 let sk_id = NodeId(sk_id as u64);
    4624            0 :                 let Some(sk) = locked.safekeepers.get(&sk_id) else {
    4625            0 :                     return Err(ApiError::InternalServerError(anyhow::anyhow!(
    4626            0 :                         "Couldn't find safekeeper with id {sk_id}"
    4627            0 :                     )));
    4628              :                 };
    4629              : 
    4630            0 :                 let req = ScheduleRequest {
    4631            0 :                     safekeeper: Box::new(sk.clone()),
    4632            0 :                     // we don't use this for this kind, put a dummy value
    4633            0 :                     host_list: Vec::new(),
    4634            0 :                     tenant_id,
    4635            0 :                     timeline_id,
    4636            0 :                     generation: tl.generation as u32,
    4637            0 :                     kind,
    4638            0 :                 };
    4639            0 :                 locked.safekeeper_reconcilers.schedule_request(self, req);
    4640              :             }
    4641              :         }
    4642            0 :         Ok(())
    4643            0 :     }
    4644              : 
    4645              :     /// When you know the TenantId but not a specific shard, and would like to get the node holding shard 0.
    4646            0 :     pub(crate) async fn tenant_shard0_node(
    4647            0 :         &self,
    4648            0 :         tenant_id: TenantId,
    4649            0 :     ) -> Result<(Node, TenantShardId), ApiError> {
    4650            0 :         let tenant_shard_id = {
    4651            0 :             let locked = self.inner.read().unwrap();
    4652            0 :             let Some((tenant_shard_id, _shard)) = locked
    4653            0 :                 .tenants
    4654            0 :                 .range(TenantShardId::tenant_range(tenant_id))
    4655            0 :                 .next()
    4656              :             else {
    4657            0 :                 return Err(ApiError::NotFound(
    4658            0 :                     anyhow::anyhow!("Tenant {tenant_id} not found").into(),
    4659            0 :                 ));
    4660              :             };
    4661              : 
    4662            0 :             *tenant_shard_id
    4663            0 :         };
    4664            0 : 
    4665            0 :         self.tenant_shard_node(tenant_shard_id)
    4666            0 :             .await
    4667            0 :             .map(|node| (node, tenant_shard_id))
    4668            0 :     }
    4669              : 
    4670              :     /// When you need to send an HTTP request to the pageserver that holds a shard of a tenant, this
    4671              :     /// function looks up and returns node. If the shard isn't found, returns Err(ApiError::NotFound)
    4672            0 :     pub(crate) async fn tenant_shard_node(
    4673            0 :         &self,
    4674            0 :         tenant_shard_id: TenantShardId,
    4675            0 :     ) -> Result<Node, ApiError> {
    4676            0 :         // Look up in-memory state and maybe use the node from there.
    4677            0 :         {
    4678            0 :             let locked = self.inner.read().unwrap();
    4679            0 :             let Some(shard) = locked.tenants.get(&tenant_shard_id) else {
    4680            0 :                 return Err(ApiError::NotFound(
    4681            0 :                     anyhow::anyhow!("Tenant shard {tenant_shard_id} not found").into(),
    4682            0 :                 ));
    4683              :             };
    4684              : 
    4685            0 :             let Some(intent_node_id) = shard.intent.get_attached() else {
    4686            0 :                 tracing::warn!(
    4687            0 :                     tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug(),
    4688            0 :                     "Shard not scheduled (policy {:?}), cannot generate pass-through URL",
    4689              :                     shard.policy
    4690              :                 );
    4691            0 :                 return Err(ApiError::Conflict(
    4692            0 :                     "Cannot call timeline API on non-attached tenant".to_string(),
    4693            0 :                 ));
    4694              :             };
    4695              : 
    4696            0 :             if shard.reconciler.is_none() {
    4697              :                 // Optimization: while no reconcile is in flight, we may trust our in-memory state
    4698              :                 // to tell us which pageserver to use. Otherwise we will fall through and hit the database
    4699            0 :                 let Some(node) = locked.nodes.get(intent_node_id) else {
    4700              :                     // This should never happen
    4701            0 :                     return Err(ApiError::InternalServerError(anyhow::anyhow!(
    4702            0 :                         "Shard refers to nonexistent node"
    4703            0 :                     )));
    4704              :                 };
    4705            0 :                 return Ok(node.clone());
    4706            0 :             }
    4707              :         };
    4708              : 
    4709              :         // Look up the latest attached pageserver location from the database
    4710              :         // generation state: this will reflect the progress of any ongoing migration.
    4711              :         // Note that it is not guaranteed to _stay_ here, our caller must still handle
    4712              :         // the case where they call through to the pageserver and get a 404.
    4713            0 :         let db_result = self
    4714            0 :             .persistence
    4715            0 :             .tenant_generations(tenant_shard_id.tenant_id)
    4716            0 :             .await?;
    4717              :         let Some(ShardGenerationState {
    4718              :             tenant_shard_id: _,
    4719              :             generation: _,
    4720            0 :             generation_pageserver: Some(node_id),
    4721            0 :         }) = db_result
    4722            0 :             .into_iter()
    4723            0 :             .find(|s| s.tenant_shard_id == tenant_shard_id)
    4724              :         else {
    4725              :             // This can happen if we raced with a tenant deletion or a shard split.  On a retry
    4726              :             // the caller will either succeed (shard split case), get a proper 404 (deletion case),
    4727              :             // or a conflict response (case where tenant was detached in background)
    4728            0 :             return Err(ApiError::ResourceUnavailable(
    4729            0 :                 format!("Shard {tenant_shard_id} not found in database, or is not attached").into(),
    4730            0 :             ));
    4731              :         };
    4732            0 :         let locked = self.inner.read().unwrap();
    4733            0 :         let Some(node) = locked.nodes.get(&node_id) else {
    4734              :             // This should never happen
    4735            0 :             return Err(ApiError::InternalServerError(anyhow::anyhow!(
    4736            0 :                 "Shard refers to nonexistent node"
    4737            0 :             )));
    4738              :         };
    4739              : 
    4740            0 :         Ok(node.clone())
    4741            0 :     }
    4742              : 
    4743            0 :     pub(crate) fn tenant_locate(
    4744            0 :         &self,
    4745            0 :         tenant_id: TenantId,
    4746            0 :     ) -> Result<TenantLocateResponse, ApiError> {
    4747            0 :         let locked = self.inner.read().unwrap();
    4748            0 :         tracing::info!("Locating shards for tenant {tenant_id}");
    4749              : 
    4750            0 :         let mut result = Vec::new();
    4751            0 :         let mut shard_params: Option<ShardParameters> = None;
    4752              : 
    4753            0 :         for (tenant_shard_id, shard) in locked.tenants.range(TenantShardId::tenant_range(tenant_id))
    4754              :         {
    4755            0 :             let node_id =
    4756            0 :                 shard
    4757            0 :                     .intent
    4758            0 :                     .get_attached()
    4759            0 :                     .ok_or(ApiError::BadRequest(anyhow::anyhow!(
    4760            0 :                         "Cannot locate a tenant that is not attached"
    4761            0 :                     )))?;
    4762              : 
    4763            0 :             let node = locked
    4764            0 :                 .nodes
    4765            0 :                 .get(&node_id)
    4766            0 :                 .expect("Pageservers may not be deleted while referenced");
    4767            0 : 
    4768            0 :             result.push(node.shard_location(*tenant_shard_id));
    4769            0 : 
    4770            0 :             match &shard_params {
    4771            0 :                 None => {
    4772            0 :                     shard_params = Some(ShardParameters {
    4773            0 :                         stripe_size: shard.shard.stripe_size,
    4774            0 :                         count: shard.shard.count,
    4775            0 :                     });
    4776            0 :                 }
    4777            0 :                 Some(params) => {
    4778            0 :                     if params.stripe_size != shard.shard.stripe_size {
    4779              :                         // This should never happen.  We enforce at runtime because it's simpler than
    4780              :                         // adding an extra per-tenant data structure to store the things that should be the same
    4781            0 :                         return Err(ApiError::InternalServerError(anyhow::anyhow!(
    4782            0 :                             "Inconsistent shard stripe size parameters!"
    4783            0 :                         )));
    4784            0 :                     }
    4785              :                 }
    4786              :             }
    4787              :         }
    4788              : 
    4789            0 :         if result.is_empty() {
    4790            0 :             return Err(ApiError::NotFound(
    4791            0 :                 anyhow::anyhow!("No shards for this tenant ID found").into(),
    4792            0 :             ));
    4793            0 :         }
    4794            0 :         let shard_params = shard_params.expect("result is non-empty, therefore this is set");
    4795            0 :         tracing::info!(
    4796            0 :             "Located tenant {} with params {:?} on shards {}",
    4797            0 :             tenant_id,
    4798            0 :             shard_params,
    4799            0 :             result
    4800            0 :                 .iter()
    4801            0 :                 .map(|s| format!("{:?}", s))
    4802            0 :                 .collect::<Vec<_>>()
    4803            0 :                 .join(",")
    4804              :         );
    4805              : 
    4806            0 :         Ok(TenantLocateResponse {
    4807            0 :             shards: result,
    4808            0 :             shard_params,
    4809            0 :         })
    4810            0 :     }
    4811              : 
    4812              :     /// Returns None if the input iterator of shards does not include a shard with number=0
    4813            0 :     fn tenant_describe_impl<'a>(
    4814            0 :         &self,
    4815            0 :         shards: impl Iterator<Item = &'a TenantShard>,
    4816            0 :     ) -> Option<TenantDescribeResponse> {
    4817            0 :         let mut shard_zero = None;
    4818            0 :         let mut describe_shards = Vec::new();
    4819              : 
    4820            0 :         for shard in shards {
    4821            0 :             if shard.tenant_shard_id.is_shard_zero() {
    4822            0 :                 shard_zero = Some(shard);
    4823            0 :             }
    4824              : 
    4825            0 :             describe_shards.push(TenantDescribeResponseShard {
    4826            0 :                 tenant_shard_id: shard.tenant_shard_id,
    4827            0 :                 node_attached: *shard.intent.get_attached(),
    4828            0 :                 node_secondary: shard.intent.get_secondary().to_vec(),
    4829            0 :                 last_error: shard
    4830            0 :                     .last_error
    4831            0 :                     .lock()
    4832            0 :                     .unwrap()
    4833            0 :                     .as_ref()
    4834            0 :                     .map(|e| format!("{e}"))
    4835            0 :                     .unwrap_or("".to_string())
    4836            0 :                     .clone(),
    4837            0 :                 is_reconciling: shard.reconciler.is_some(),
    4838            0 :                 is_pending_compute_notification: shard.pending_compute_notification,
    4839            0 :                 is_splitting: matches!(shard.splitting, SplitState::Splitting),
    4840            0 :                 scheduling_policy: shard.get_scheduling_policy(),
    4841            0 :                 preferred_az_id: shard.preferred_az().map(ToString::to_string),
    4842              :             })
    4843              :         }
    4844              : 
    4845            0 :         let shard_zero = shard_zero?;
    4846              : 
    4847            0 :         Some(TenantDescribeResponse {
    4848            0 :             tenant_id: shard_zero.tenant_shard_id.tenant_id,
    4849            0 :             shards: describe_shards,
    4850            0 :             stripe_size: shard_zero.shard.stripe_size,
    4851            0 :             policy: shard_zero.policy.clone(),
    4852            0 :             config: shard_zero.config.clone(),
    4853            0 :         })
    4854            0 :     }
    4855              : 
    4856            0 :     pub(crate) fn tenant_describe(
    4857            0 :         &self,
    4858            0 :         tenant_id: TenantId,
    4859            0 :     ) -> Result<TenantDescribeResponse, ApiError> {
    4860            0 :         let locked = self.inner.read().unwrap();
    4861            0 : 
    4862            0 :         self.tenant_describe_impl(
    4863            0 :             locked
    4864            0 :                 .tenants
    4865            0 :                 .range(TenantShardId::tenant_range(tenant_id))
    4866            0 :                 .map(|(_k, v)| v),
    4867            0 :         )
    4868            0 :         .ok_or_else(|| ApiError::NotFound(anyhow::anyhow!("Tenant {tenant_id} not found").into()))
    4869            0 :     }
    4870              : 
    4871              :     /// limit & offset are pagination parameters. Since we are walking an in-memory HashMap, `offset` does not
    4872              :     /// avoid traversing data, it just avoid returning it. This is suitable for our purposes, since our in memory
    4873              :     /// maps are small enough to traverse fast, our pagination is just to avoid serializing huge JSON responses
    4874              :     /// in our external API.
    4875            0 :     pub(crate) fn tenant_list(
    4876            0 :         &self,
    4877            0 :         limit: Option<usize>,
    4878            0 :         start_after: Option<TenantId>,
    4879            0 :     ) -> Vec<TenantDescribeResponse> {
    4880            0 :         let locked = self.inner.read().unwrap();
    4881              : 
    4882              :         // Apply start_from parameter
    4883            0 :         let shard_range = match start_after {
    4884            0 :             None => locked.tenants.range(..),
    4885            0 :             Some(tenant_id) => locked.tenants.range(
    4886            0 :                 TenantShardId {
    4887            0 :                     tenant_id,
    4888            0 :                     shard_number: ShardNumber(u8::MAX),
    4889            0 :                     shard_count: ShardCount(u8::MAX),
    4890            0 :                 }..,
    4891            0 :             ),
    4892              :         };
    4893              : 
    4894            0 :         let mut result = Vec::new();
    4895            0 :         for (_tenant_id, tenant_shards) in &shard_range.group_by(|(id, _shard)| id.tenant_id) {
    4896            0 :             result.push(
    4897            0 :                 self.tenant_describe_impl(tenant_shards.map(|(_k, v)| v))
    4898            0 :                     .expect("Groups are always non-empty"),
    4899            0 :             );
    4900              : 
    4901              :             // Enforce `limit` parameter
    4902            0 :             if let Some(limit) = limit {
    4903            0 :                 if result.len() >= limit {
    4904            0 :                     break;
    4905            0 :                 }
    4906            0 :             }
    4907              :         }
    4908              : 
    4909            0 :         result
    4910            0 :     }
    4911              : 
    4912              :     #[instrument(skip_all, fields(tenant_id=%op.tenant_id))]
    4913              :     async fn abort_tenant_shard_split(
    4914              :         &self,
    4915              :         op: &TenantShardSplitAbort,
    4916              :     ) -> Result<(), TenantShardSplitAbortError> {
    4917              :         // Cleaning up a split:
    4918              :         // - Parent shards are not destroyed during a split, just detached.
    4919              :         // - Failed pageserver split API calls can leave the remote node with just the parent attached,
    4920              :         //   just the children attached, or both.
    4921              :         //
    4922              :         // Therefore our work to do is to:
    4923              :         // 1. Clean up storage controller's internal state to just refer to parents, no children
    4924              :         // 2. Call out to pageservers to ensure that children are detached
    4925              :         // 3. Call out to pageservers to ensure that parents are attached.
    4926              :         //
    4927              :         // Crash safety:
    4928              :         // - If the storage controller stops running during this cleanup *after* clearing the splitting state
    4929              :         //   from our database, then [`Self::startup_reconcile`] will regard child attachments as garbage
    4930              :         //   and detach them.
    4931              :         // - TODO: If the storage controller stops running during this cleanup *before* clearing the splitting state
    4932              :         //   from our database, then we will re-enter this cleanup routine on startup.
    4933              : 
    4934              :         let TenantShardSplitAbort {
    4935              :             tenant_id,
    4936              :             new_shard_count,
    4937              :             new_stripe_size,
    4938              :             ..
    4939              :         } = op;
    4940              : 
    4941              :         // First abort persistent state, if any exists.
    4942              :         match self
    4943              :             .persistence
    4944              :             .abort_shard_split(*tenant_id, *new_shard_count)
    4945              :             .await?
    4946              :         {
    4947              :             AbortShardSplitStatus::Aborted => {
    4948              :                 // Proceed to roll back any child shards created on pageservers
    4949              :             }
    4950              :             AbortShardSplitStatus::Complete => {
    4951              :                 // The split completed (we might hit that path if e.g. our database transaction
    4952              :                 // to write the completion landed in the database, but we dropped connection
    4953              :                 // before seeing the result).
    4954              :                 //
    4955              :                 // We must update in-memory state to reflect the successful split.
    4956              :                 self.tenant_shard_split_commit_inmem(
    4957              :                     *tenant_id,
    4958              :                     *new_shard_count,
    4959              :                     *new_stripe_size,
    4960              :                 );
    4961              :                 return Ok(());
    4962              :             }
    4963              :         }
    4964              : 
    4965              :         // Clean up in-memory state, and accumulate the list of child locations that need detaching
    4966              :         let detach_locations: Vec<(Node, TenantShardId)> = {
    4967              :             let mut detach_locations = Vec::new();
    4968              :             let mut locked = self.inner.write().unwrap();
    4969              :             let (nodes, tenants, scheduler) = locked.parts_mut();
    4970              : 
    4971              :             for (tenant_shard_id, shard) in
    4972              :                 tenants.range_mut(TenantShardId::tenant_range(op.tenant_id))
    4973              :             {
    4974              :                 if shard.shard.count == op.new_shard_count {
    4975              :                     // Surprising: the phase of [`Self::do_tenant_shard_split`] which inserts child shards in-memory
    4976              :                     // is infallible, so if we got an error we shouldn't have got that far.
    4977              :                     tracing::warn!(
    4978              :                         "During split abort, child shard {tenant_shard_id} found in-memory"
    4979              :                     );
    4980              :                     continue;
    4981              :                 }
    4982              : 
    4983              :                 // Add the children of this shard to this list of things to detach
    4984              :                 if let Some(node_id) = shard.intent.get_attached() {
    4985              :                     for child_id in tenant_shard_id.split(*new_shard_count) {
    4986              :                         detach_locations.push((
    4987              :                             nodes
    4988              :                                 .get(node_id)
    4989              :                                 .expect("Intent references nonexistent node")
    4990              :                                 .clone(),
    4991              :                             child_id,
    4992              :                         ));
    4993              :                     }
    4994              :                 } else {
    4995              :                     tracing::warn!(
    4996              :                         "During split abort, shard {tenant_shard_id} has no attached location"
    4997              :                     );
    4998              :                 }
    4999              : 
    5000              :                 tracing::info!("Restoring parent shard {tenant_shard_id}");
    5001              : 
    5002              :                 // Drop any intents that refer to unavailable nodes, to enable this abort to proceed even
    5003              :                 // if the original attachment location is offline.
    5004              :                 if let Some(node_id) = shard.intent.get_attached() {
    5005              :                     if !nodes.get(node_id).unwrap().is_available() {
    5006              :                         tracing::info!(
    5007              :                             "Demoting attached intent for {tenant_shard_id} on unavailable node {node_id}"
    5008              :                         );
    5009              :                         shard.intent.demote_attached(scheduler, *node_id);
    5010              :                     }
    5011              :                 }
    5012              :                 for node_id in shard.intent.get_secondary().clone() {
    5013              :                     if !nodes.get(&node_id).unwrap().is_available() {
    5014              :                         tracing::info!(
    5015              :                             "Dropping secondary intent for {tenant_shard_id} on unavailable node {node_id}"
    5016              :                         );
    5017              :                         shard.intent.remove_secondary(scheduler, node_id);
    5018              :                     }
    5019              :                 }
    5020              : 
    5021              :                 shard.splitting = SplitState::Idle;
    5022              :                 if let Err(e) = shard.schedule(scheduler, &mut ScheduleContext::default()) {
    5023              :                     // If this shard can't be scheduled now (perhaps due to offline nodes or
    5024              :                     // capacity issues), that must not prevent us rolling back a split.  In this
    5025              :                     // case it should be eventually scheduled in the background.
    5026              :                     tracing::warn!("Failed to schedule {tenant_shard_id} during shard abort: {e}")
    5027              :                 }
    5028              : 
    5029              :                 self.maybe_reconcile_shard(shard, nodes, ReconcilerPriority::High);
    5030              :             }
    5031              : 
    5032              :             // We don't expect any new_shard_count shards to exist here, but drop them just in case
    5033            0 :             tenants.retain(|_id, s| s.shard.count != *new_shard_count);
    5034              : 
    5035              :             detach_locations
    5036              :         };
    5037              : 
    5038              :         for (node, child_id) in detach_locations {
    5039              :             if !node.is_available() {
    5040              :                 // An unavailable node cannot be cleaned up now: to avoid blocking forever, we will permit this, and
    5041              :                 // rely on the reconciliation that happens when a node transitions to Active to clean up. Since we have
    5042              :                 // removed child shards from our in-memory state and database, the reconciliation will implicitly remove
    5043              :                 // them from the node.
    5044              :                 tracing::warn!(
    5045              :                     "Node {node} unavailable, can't clean up during split abort. It will be cleaned up when it is reactivated."
    5046              :                 );
    5047              :                 continue;
    5048              :             }
    5049              : 
    5050              :             // Detach the remote child.  If the pageserver split API call is still in progress, this call will get
    5051              :             // a 503 and retry, up to our limit.
    5052              :             tracing::info!("Detaching {child_id} on {node}...");
    5053              :             match node
    5054              :                 .with_client_retries(
    5055            0 :                     |client| async move {
    5056            0 :                         let config = LocationConfig {
    5057            0 :                             mode: LocationConfigMode::Detached,
    5058            0 :                             generation: None,
    5059            0 :                             secondary_conf: None,
    5060            0 :                             shard_number: child_id.shard_number.0,
    5061            0 :                             shard_count: child_id.shard_count.literal(),
    5062            0 :                             // Stripe size and tenant config don't matter when detaching
    5063            0 :                             shard_stripe_size: 0,
    5064            0 :                             tenant_conf: TenantConfig::default(),
    5065            0 :                         };
    5066            0 : 
    5067            0 :                         client.location_config(child_id, config, None, false).await
    5068            0 :                     },
    5069              :                     &self.config.pageserver_jwt_token,
    5070              :                     &self.config.ssl_ca_cert,
    5071              :                     1,
    5072              :                     10,
    5073              :                     Duration::from_secs(5),
    5074              :                     &self.cancel,
    5075              :                 )
    5076              :                 .await
    5077              :             {
    5078              :                 Some(Ok(_)) => {}
    5079              :                 Some(Err(e)) => {
    5080              :                     // We failed to communicate with the remote node.  This is problematic: we may be
    5081              :                     // leaving it with a rogue child shard.
    5082              :                     tracing::warn!(
    5083              :                         "Failed to detach child {child_id} from node {node} during abort"
    5084              :                     );
    5085              :                     return Err(e.into());
    5086              :                 }
    5087              :                 None => {
    5088              :                     // Cancellation: we were shutdown or the node went offline. Shutdown is fine, we'll
    5089              :                     // clean up on restart. The node going offline requires a retry.
    5090              :                     return Err(TenantShardSplitAbortError::Unavailable);
    5091              :                 }
    5092              :             };
    5093              :         }
    5094              : 
    5095              :         tracing::info!("Successfully aborted split");
    5096              :         Ok(())
    5097              :     }
    5098              : 
    5099              :     /// Infallible final stage of [`Self::tenant_shard_split`]: update the contents
    5100              :     /// of the tenant map to reflect the child shards that exist after the split.
    5101            0 :     fn tenant_shard_split_commit_inmem(
    5102            0 :         &self,
    5103            0 :         tenant_id: TenantId,
    5104            0 :         new_shard_count: ShardCount,
    5105            0 :         new_stripe_size: Option<ShardStripeSize>,
    5106            0 :     ) -> (
    5107            0 :         TenantShardSplitResponse,
    5108            0 :         Vec<(TenantShardId, NodeId, ShardStripeSize)>,
    5109            0 :         Vec<ReconcilerWaiter>,
    5110            0 :     ) {
    5111            0 :         let mut response = TenantShardSplitResponse {
    5112            0 :             new_shards: Vec::new(),
    5113            0 :         };
    5114            0 :         let mut child_locations = Vec::new();
    5115            0 :         let mut waiters = Vec::new();
    5116            0 : 
    5117            0 :         {
    5118            0 :             let mut locked = self.inner.write().unwrap();
    5119            0 : 
    5120            0 :             let parent_ids = locked
    5121            0 :                 .tenants
    5122            0 :                 .range(TenantShardId::tenant_range(tenant_id))
    5123            0 :                 .map(|(shard_id, _)| *shard_id)
    5124            0 :                 .collect::<Vec<_>>();
    5125            0 : 
    5126            0 :             let (nodes, tenants, scheduler) = locked.parts_mut();
    5127            0 :             for parent_id in parent_ids {
    5128            0 :                 let child_ids = parent_id.split(new_shard_count);
    5129              : 
    5130            0 :                 let (pageserver, generation, policy, parent_ident, config, preferred_az) = {
    5131            0 :                     let mut old_state = tenants
    5132            0 :                         .remove(&parent_id)
    5133            0 :                         .expect("It was present, we just split it");
    5134            0 : 
    5135            0 :                     // A non-splitting state is impossible, because [`Self::tenant_shard_split`] holds
    5136            0 :                     // a TenantId lock and passes it through to [`TenantShardSplitAbort`] in case of cleanup:
    5137            0 :                     // nothing else can clear this.
    5138            0 :                     assert!(matches!(old_state.splitting, SplitState::Splitting));
    5139              : 
    5140            0 :                     let old_attached = old_state.intent.get_attached().unwrap();
    5141            0 :                     old_state.intent.clear(scheduler);
    5142            0 :                     let generation = old_state.generation.expect("Shard must have been attached");
    5143            0 :                     (
    5144            0 :                         old_attached,
    5145            0 :                         generation,
    5146            0 :                         old_state.policy.clone(),
    5147            0 :                         old_state.shard,
    5148            0 :                         old_state.config.clone(),
    5149            0 :                         old_state.preferred_az().cloned(),
    5150            0 :                     )
    5151            0 :                 };
    5152            0 : 
    5153            0 :                 let mut schedule_context = ScheduleContext::default();
    5154            0 :                 for child in child_ids {
    5155            0 :                     let mut child_shard = parent_ident;
    5156            0 :                     child_shard.number = child.shard_number;
    5157            0 :                     child_shard.count = child.shard_count;
    5158            0 :                     if let Some(stripe_size) = new_stripe_size {
    5159            0 :                         child_shard.stripe_size = stripe_size;
    5160            0 :                     }
    5161              : 
    5162            0 :                     let mut child_observed: HashMap<NodeId, ObservedStateLocation> = HashMap::new();
    5163            0 :                     child_observed.insert(
    5164            0 :                         pageserver,
    5165            0 :                         ObservedStateLocation {
    5166            0 :                             conf: Some(attached_location_conf(
    5167            0 :                                 generation,
    5168            0 :                                 &child_shard,
    5169            0 :                                 &config,
    5170            0 :                                 &policy,
    5171            0 :                             )),
    5172            0 :                         },
    5173            0 :                     );
    5174            0 : 
    5175            0 :                     let mut child_state =
    5176            0 :                         TenantShard::new(child, child_shard, policy.clone(), preferred_az.clone());
    5177            0 :                     child_state.intent =
    5178            0 :                         IntentState::single(scheduler, Some(pageserver), preferred_az.clone());
    5179            0 :                     child_state.observed = ObservedState {
    5180            0 :                         locations: child_observed,
    5181            0 :                     };
    5182            0 :                     child_state.generation = Some(generation);
    5183            0 :                     child_state.config = config.clone();
    5184            0 : 
    5185            0 :                     // The child's TenantShard::splitting is intentionally left at the default value of Idle,
    5186            0 :                     // as at this point in the split process we have succeeded and this part is infallible:
    5187            0 :                     // we will never need to do any special recovery from this state.
    5188            0 : 
    5189            0 :                     child_locations.push((child, pageserver, child_shard.stripe_size));
    5190              : 
    5191            0 :                     if let Err(e) = child_state.schedule(scheduler, &mut schedule_context) {
    5192              :                         // This is not fatal, because we've implicitly already got an attached
    5193              :                         // location for the child shard.  Failure here just means we couldn't
    5194              :                         // find a secondary (e.g. because cluster is overloaded).
    5195            0 :                         tracing::warn!("Failed to schedule child shard {child}: {e}");
    5196            0 :                     }
    5197              :                     // In the background, attach secondary locations for the new shards
    5198            0 :                     if let Some(waiter) = self.maybe_reconcile_shard(
    5199            0 :                         &mut child_state,
    5200            0 :                         nodes,
    5201            0 :                         ReconcilerPriority::High,
    5202            0 :                     ) {
    5203            0 :                         waiters.push(waiter);
    5204            0 :                     }
    5205              : 
    5206            0 :                     tenants.insert(child, child_state);
    5207            0 :                     response.new_shards.push(child);
    5208              :                 }
    5209              :             }
    5210            0 :             (response, child_locations, waiters)
    5211            0 :         }
    5212            0 :     }
    5213              : 
    5214            0 :     async fn tenant_shard_split_start_secondaries(
    5215            0 :         &self,
    5216            0 :         tenant_id: TenantId,
    5217            0 :         waiters: Vec<ReconcilerWaiter>,
    5218            0 :     ) {
    5219              :         // Wait for initial reconcile of child shards, this creates the secondary locations
    5220            0 :         if let Err(e) = self.await_waiters(waiters, RECONCILE_TIMEOUT).await {
    5221              :             // This is not a failure to split: it's some issue reconciling the new child shards, perhaps
    5222              :             // their secondaries couldn't be attached.
    5223            0 :             tracing::warn!("Failed to reconcile after split: {e}");
    5224            0 :             return;
    5225            0 :         }
    5226              : 
    5227              :         // Take the state lock to discover the attached & secondary intents for all shards
    5228            0 :         let (attached, secondary) = {
    5229            0 :             let locked = self.inner.read().unwrap();
    5230            0 :             let mut attached = Vec::new();
    5231            0 :             let mut secondary = Vec::new();
    5232              : 
    5233            0 :             for (tenant_shard_id, shard) in
    5234            0 :                 locked.tenants.range(TenantShardId::tenant_range(tenant_id))
    5235              :             {
    5236            0 :                 let Some(node_id) = shard.intent.get_attached() else {
    5237              :                     // Unexpected.  Race with a PlacementPolicy change?
    5238            0 :                     tracing::warn!(
    5239            0 :                         "No attached node on {tenant_shard_id} immediately after shard split!"
    5240              :                     );
    5241            0 :                     continue;
    5242              :                 };
    5243              : 
    5244            0 :                 let Some(secondary_node_id) = shard.intent.get_secondary().first() else {
    5245              :                     // No secondary location.  Nothing for us to do.
    5246            0 :                     continue;
    5247              :                 };
    5248              : 
    5249            0 :                 let attached_node = locked
    5250            0 :                     .nodes
    5251            0 :                     .get(node_id)
    5252            0 :                     .expect("Pageservers may not be deleted while referenced");
    5253            0 : 
    5254            0 :                 let secondary_node = locked
    5255            0 :                     .nodes
    5256            0 :                     .get(secondary_node_id)
    5257            0 :                     .expect("Pageservers may not be deleted while referenced");
    5258            0 : 
    5259            0 :                 attached.push((*tenant_shard_id, attached_node.clone()));
    5260            0 :                 secondary.push((*tenant_shard_id, secondary_node.clone()));
    5261              :             }
    5262            0 :             (attached, secondary)
    5263            0 :         };
    5264            0 : 
    5265            0 :         if secondary.is_empty() {
    5266              :             // No secondary locations; nothing for us to do
    5267            0 :             return;
    5268            0 :         }
    5269              : 
    5270            0 :         for result in self
    5271            0 :             .tenant_for_shards_api(
    5272            0 :                 attached,
    5273            0 :                 |tenant_shard_id, client| async move {
    5274            0 :                     client.tenant_heatmap_upload(tenant_shard_id).await
    5275            0 :                 },
    5276            0 :                 1,
    5277            0 :                 1,
    5278            0 :                 SHORT_RECONCILE_TIMEOUT,
    5279            0 :                 &self.cancel,
    5280            0 :             )
    5281            0 :             .await
    5282              :         {
    5283            0 :             if let Err(e) = result {
    5284            0 :                 tracing::warn!("Error calling heatmap upload after shard split: {e}");
    5285            0 :                 return;
    5286            0 :             }
    5287              :         }
    5288              : 
    5289            0 :         for result in self
    5290            0 :             .tenant_for_shards_api(
    5291            0 :                 secondary,
    5292            0 :                 |tenant_shard_id, client| async move {
    5293            0 :                     client
    5294            0 :                         .tenant_secondary_download(tenant_shard_id, Some(Duration::ZERO))
    5295            0 :                         .await
    5296            0 :                 },
    5297            0 :                 1,
    5298            0 :                 1,
    5299            0 :                 SHORT_RECONCILE_TIMEOUT,
    5300            0 :                 &self.cancel,
    5301            0 :             )
    5302            0 :             .await
    5303              :         {
    5304            0 :             if let Err(e) = result {
    5305            0 :                 tracing::warn!("Error calling secondary download after shard split: {e}");
    5306            0 :                 return;
    5307            0 :             }
    5308              :         }
    5309            0 :     }
    5310              : 
    5311            0 :     pub(crate) async fn tenant_shard_split(
    5312            0 :         &self,
    5313            0 :         tenant_id: TenantId,
    5314            0 :         split_req: TenantShardSplitRequest,
    5315            0 :     ) -> Result<TenantShardSplitResponse, ApiError> {
    5316              :         // TODO: return 503 if we get stuck waiting for this lock
    5317              :         // (issue https://github.com/neondatabase/neon/issues/7108)
    5318            0 :         let _tenant_lock = trace_exclusive_lock(
    5319            0 :             &self.tenant_op_locks,
    5320            0 :             tenant_id,
    5321            0 :             TenantOperations::ShardSplit,
    5322            0 :         )
    5323            0 :         .await;
    5324              : 
    5325            0 :         let new_shard_count = ShardCount::new(split_req.new_shard_count);
    5326            0 :         let new_stripe_size = split_req.new_stripe_size;
    5327              : 
    5328              :         // Validate the request and construct parameters.  This phase is fallible, but does not require
    5329              :         // rollback on errors, as it does no I/O and mutates no state.
    5330            0 :         let shard_split_params = match self.prepare_tenant_shard_split(tenant_id, split_req)? {
    5331            0 :             ShardSplitAction::NoOp(resp) => return Ok(resp),
    5332            0 :             ShardSplitAction::Split(params) => params,
    5333              :         };
    5334              : 
    5335              :         // Execute this split: this phase mutates state and does remote I/O on pageservers.  If it fails,
    5336              :         // we must roll back.
    5337            0 :         let r = self
    5338            0 :             .do_tenant_shard_split(tenant_id, shard_split_params)
    5339            0 :             .await;
    5340              : 
    5341            0 :         let (response, waiters) = match r {
    5342            0 :             Ok(r) => r,
    5343            0 :             Err(e) => {
    5344            0 :                 // Split might be part-done, we must do work to abort it.
    5345            0 :                 tracing::warn!("Enqueuing background abort of split on {tenant_id}");
    5346            0 :                 self.abort_tx
    5347            0 :                     .send(TenantShardSplitAbort {
    5348            0 :                         tenant_id,
    5349            0 :                         new_shard_count,
    5350            0 :                         new_stripe_size,
    5351            0 :                         _tenant_lock,
    5352            0 :                     })
    5353            0 :                     // Ignore error sending: that just means we're shutting down: aborts are ephemeral so it's fine to drop it.
    5354            0 :                     .ok();
    5355            0 :                 return Err(e);
    5356              :             }
    5357              :         };
    5358              : 
    5359              :         // The split is now complete.  As an optimization, we will trigger all the child shards to upload
    5360              :         // a heatmap immediately, and all their secondary locations to start downloading: this avoids waiting
    5361              :         // for the background heatmap/download interval before secondaries get warm enough to migrate shards
    5362              :         // in [`Self::optimize_all`]
    5363            0 :         self.tenant_shard_split_start_secondaries(tenant_id, waiters)
    5364            0 :             .await;
    5365            0 :         Ok(response)
    5366            0 :     }
    5367              : 
    5368            0 :     fn prepare_tenant_shard_split(
    5369            0 :         &self,
    5370            0 :         tenant_id: TenantId,
    5371            0 :         split_req: TenantShardSplitRequest,
    5372            0 :     ) -> Result<ShardSplitAction, ApiError> {
    5373            0 :         fail::fail_point!("shard-split-validation", |_| Err(ApiError::BadRequest(
    5374            0 :             anyhow::anyhow!("failpoint")
    5375            0 :         )));
    5376              : 
    5377            0 :         let mut policy = None;
    5378            0 :         let mut config = None;
    5379            0 :         let mut shard_ident = None;
    5380            0 :         let mut preferred_az_id = None;
    5381              :         // Validate input, and calculate which shards we will create
    5382            0 :         let (old_shard_count, targets) =
    5383              :             {
    5384            0 :                 let locked = self.inner.read().unwrap();
    5385            0 : 
    5386            0 :                 let pageservers = locked.nodes.clone();
    5387            0 : 
    5388            0 :                 let mut targets = Vec::new();
    5389            0 : 
    5390            0 :                 // In case this is a retry, count how many already-split shards we found
    5391            0 :                 let mut children_found = Vec::new();
    5392            0 :                 let mut old_shard_count = None;
    5393              : 
    5394            0 :                 for (tenant_shard_id, shard) in
    5395            0 :                     locked.tenants.range(TenantShardId::tenant_range(tenant_id))
    5396              :                 {
    5397            0 :                     match shard.shard.count.count().cmp(&split_req.new_shard_count) {
    5398              :                         Ordering::Equal => {
    5399              :                             //  Already split this
    5400            0 :                             children_found.push(*tenant_shard_id);
    5401            0 :                             continue;
    5402              :                         }
    5403              :                         Ordering::Greater => {
    5404            0 :                             return Err(ApiError::BadRequest(anyhow::anyhow!(
    5405            0 :                                 "Requested count {} but already have shards at count {}",
    5406            0 :                                 split_req.new_shard_count,
    5407            0 :                                 shard.shard.count.count()
    5408            0 :                             )));
    5409              :                         }
    5410            0 :                         Ordering::Less => {
    5411            0 :                             // Fall through: this shard has lower count than requested,
    5412            0 :                             // is a candidate for splitting.
    5413            0 :                         }
    5414            0 :                     }
    5415            0 : 
    5416            0 :                     match old_shard_count {
    5417            0 :                         None => old_shard_count = Some(shard.shard.count),
    5418            0 :                         Some(old_shard_count) => {
    5419            0 :                             if old_shard_count != shard.shard.count {
    5420              :                                 // We may hit this case if a caller asked for two splits to
    5421              :                                 // different sizes, before the first one is complete.
    5422              :                                 // e.g. 1->2, 2->4, where the 4 call comes while we have a mixture
    5423              :                                 // of shard_count=1 and shard_count=2 shards in the map.
    5424            0 :                                 return Err(ApiError::Conflict(
    5425            0 :                                     "Cannot split, currently mid-split".to_string(),
    5426            0 :                                 ));
    5427            0 :                             }
    5428              :                         }
    5429              :                     }
    5430            0 :                     if policy.is_none() {
    5431            0 :                         policy = Some(shard.policy.clone());
    5432            0 :                     }
    5433            0 :                     if shard_ident.is_none() {
    5434            0 :                         shard_ident = Some(shard.shard);
    5435            0 :                     }
    5436            0 :                     if config.is_none() {
    5437            0 :                         config = Some(shard.config.clone());
    5438            0 :                     }
    5439            0 :                     if preferred_az_id.is_none() {
    5440            0 :                         preferred_az_id = shard.preferred_az().cloned();
    5441            0 :                     }
    5442              : 
    5443            0 :                     if tenant_shard_id.shard_count.count() == split_req.new_shard_count {
    5444            0 :                         tracing::info!(
    5445            0 :                             "Tenant shard {} already has shard count {}",
    5446              :                             tenant_shard_id,
    5447              :                             split_req.new_shard_count
    5448              :                         );
    5449            0 :                         continue;
    5450            0 :                     }
    5451              : 
    5452            0 :                     let node_id = shard.intent.get_attached().ok_or(ApiError::BadRequest(
    5453            0 :                         anyhow::anyhow!("Cannot split a tenant that is not attached"),
    5454            0 :                     ))?;
    5455              : 
    5456            0 :                     let node = pageservers
    5457            0 :                         .get(&node_id)
    5458            0 :                         .expect("Pageservers may not be deleted while referenced");
    5459            0 : 
    5460            0 :                     targets.push(ShardSplitTarget {
    5461            0 :                         parent_id: *tenant_shard_id,
    5462            0 :                         node: node.clone(),
    5463            0 :                         child_ids: tenant_shard_id
    5464            0 :                             .split(ShardCount::new(split_req.new_shard_count)),
    5465            0 :                     });
    5466              :                 }
    5467              : 
    5468            0 :                 if targets.is_empty() {
    5469            0 :                     if children_found.len() == split_req.new_shard_count as usize {
    5470            0 :                         return Ok(ShardSplitAction::NoOp(TenantShardSplitResponse {
    5471            0 :                             new_shards: children_found,
    5472            0 :                         }));
    5473              :                     } else {
    5474              :                         // No shards found to split, and no existing children found: the
    5475              :                         // tenant doesn't exist at all.
    5476            0 :                         return Err(ApiError::NotFound(
    5477            0 :                             anyhow::anyhow!("Tenant {} not found", tenant_id).into(),
    5478            0 :                         ));
    5479              :                     }
    5480            0 :                 }
    5481            0 : 
    5482            0 :                 (old_shard_count, targets)
    5483            0 :             };
    5484            0 : 
    5485            0 :         // unwrap safety: we would have returned above if we didn't find at least one shard to split
    5486            0 :         let old_shard_count = old_shard_count.unwrap();
    5487            0 :         let shard_ident = if let Some(new_stripe_size) = split_req.new_stripe_size {
    5488              :             // This ShardIdentity will be used as the template for all children, so this implicitly
    5489              :             // applies the new stripe size to the children.
    5490            0 :             let mut shard_ident = shard_ident.unwrap();
    5491            0 :             if shard_ident.count.count() > 1 && shard_ident.stripe_size != new_stripe_size {
    5492            0 :                 return Err(ApiError::BadRequest(anyhow::anyhow!(
    5493            0 :                     "Attempted to change stripe size ({:?}->{new_stripe_size:?}) on a tenant with multiple shards",
    5494            0 :                     shard_ident.stripe_size
    5495            0 :                 )));
    5496            0 :             }
    5497            0 : 
    5498            0 :             shard_ident.stripe_size = new_stripe_size;
    5499            0 :             tracing::info!("applied  stripe size {}", shard_ident.stripe_size.0);
    5500            0 :             shard_ident
    5501              :         } else {
    5502            0 :             shard_ident.unwrap()
    5503              :         };
    5504            0 :         let policy = policy.unwrap();
    5505            0 :         let config = config.unwrap();
    5506            0 : 
    5507            0 :         Ok(ShardSplitAction::Split(Box::new(ShardSplitParams {
    5508            0 :             old_shard_count,
    5509            0 :             new_shard_count: ShardCount::new(split_req.new_shard_count),
    5510            0 :             new_stripe_size: split_req.new_stripe_size,
    5511            0 :             targets,
    5512            0 :             policy,
    5513            0 :             config,
    5514            0 :             shard_ident,
    5515            0 :             preferred_az_id,
    5516            0 :         })))
    5517            0 :     }
    5518              : 
    5519            0 :     async fn do_tenant_shard_split(
    5520            0 :         &self,
    5521            0 :         tenant_id: TenantId,
    5522            0 :         params: Box<ShardSplitParams>,
    5523            0 :     ) -> Result<(TenantShardSplitResponse, Vec<ReconcilerWaiter>), ApiError> {
    5524            0 :         // FIXME: we have dropped self.inner lock, and not yet written anything to the database: another
    5525            0 :         // request could occur here, deleting or mutating the tenant.  begin_shard_split checks that the
    5526            0 :         // parent shards exist as expected, but it would be neater to do the above pre-checks within the
    5527            0 :         // same database transaction rather than pre-check in-memory and then maybe-fail the database write.
    5528            0 :         // (https://github.com/neondatabase/neon/issues/6676)
    5529            0 : 
    5530            0 :         let ShardSplitParams {
    5531            0 :             old_shard_count,
    5532            0 :             new_shard_count,
    5533            0 :             new_stripe_size,
    5534            0 :             mut targets,
    5535            0 :             policy,
    5536            0 :             config,
    5537            0 :             shard_ident,
    5538            0 :             preferred_az_id,
    5539            0 :         } = *params;
    5540              : 
    5541              :         // Drop any secondary locations: pageservers do not support splitting these, and in any case the
    5542              :         // end-state for a split tenant will usually be to have secondary locations on different nodes.
    5543              :         // The reconciliation calls in this block also implicitly cancel+barrier wrt any ongoing reconciliation
    5544              :         // at the time of split.
    5545            0 :         let waiters = {
    5546            0 :             let mut locked = self.inner.write().unwrap();
    5547            0 :             let mut waiters = Vec::new();
    5548            0 :             let (nodes, tenants, scheduler) = locked.parts_mut();
    5549            0 :             for target in &mut targets {
    5550            0 :                 let Some(shard) = tenants.get_mut(&target.parent_id) else {
    5551              :                     // Paranoia check: this shouldn't happen: we have the oplock for this tenant ID.
    5552            0 :                     return Err(ApiError::InternalServerError(anyhow::anyhow!(
    5553            0 :                         "Shard {} not found",
    5554            0 :                         target.parent_id
    5555            0 :                     )));
    5556              :                 };
    5557              : 
    5558            0 :                 if shard.intent.get_attached() != &Some(target.node.get_id()) {
    5559              :                     // Paranoia check: this shouldn't happen: we have the oplock for this tenant ID.
    5560            0 :                     return Err(ApiError::Conflict(format!(
    5561            0 :                         "Shard {} unexpectedly rescheduled during split",
    5562            0 :                         target.parent_id
    5563            0 :                     )));
    5564            0 :                 }
    5565            0 : 
    5566            0 :                 // Irrespective of PlacementPolicy, clear secondary locations from intent
    5567            0 :                 shard.intent.clear_secondary(scheduler);
    5568              : 
    5569              :                 // Run Reconciler to execute detach fo secondary locations.
    5570            0 :                 if let Some(waiter) =
    5571            0 :                     self.maybe_reconcile_shard(shard, nodes, ReconcilerPriority::High)
    5572            0 :                 {
    5573            0 :                     waiters.push(waiter);
    5574            0 :                 }
    5575              :             }
    5576            0 :             waiters
    5577            0 :         };
    5578            0 :         self.await_waiters(waiters, RECONCILE_TIMEOUT).await?;
    5579              : 
    5580              :         // Before creating any new child shards in memory or on the pageservers, persist them: this
    5581              :         // enables us to ensure that we will always be able to clean up if something goes wrong.  This also
    5582              :         // acts as the protection against two concurrent attempts to split: one of them will get a database
    5583              :         // error trying to insert the child shards.
    5584            0 :         let mut child_tsps = Vec::new();
    5585            0 :         for target in &targets {
    5586            0 :             let mut this_child_tsps = Vec::new();
    5587            0 :             for child in &target.child_ids {
    5588            0 :                 let mut child_shard = shard_ident;
    5589            0 :                 child_shard.number = child.shard_number;
    5590            0 :                 child_shard.count = child.shard_count;
    5591            0 : 
    5592            0 :                 tracing::info!(
    5593            0 :                     "Create child shard persistence with stripe size {}",
    5594              :                     shard_ident.stripe_size.0
    5595              :                 );
    5596              : 
    5597            0 :                 this_child_tsps.push(TenantShardPersistence {
    5598            0 :                     tenant_id: child.tenant_id.to_string(),
    5599            0 :                     shard_number: child.shard_number.0 as i32,
    5600            0 :                     shard_count: child.shard_count.literal() as i32,
    5601            0 :                     shard_stripe_size: shard_ident.stripe_size.0 as i32,
    5602            0 :                     // Note: this generation is a placeholder, [`Persistence::begin_shard_split`] will
    5603            0 :                     // populate the correct generation as part of its transaction, to protect us
    5604            0 :                     // against racing with changes in the state of the parent.
    5605            0 :                     generation: None,
    5606            0 :                     generation_pageserver: Some(target.node.get_id().0 as i64),
    5607            0 :                     placement_policy: serde_json::to_string(&policy).unwrap(),
    5608            0 :                     config: serde_json::to_string(&config).unwrap(),
    5609            0 :                     splitting: SplitState::Splitting,
    5610            0 : 
    5611            0 :                     // Scheduling policies and preferred AZ do not carry through to children
    5612            0 :                     scheduling_policy: serde_json::to_string(&ShardSchedulingPolicy::default())
    5613            0 :                         .unwrap(),
    5614            0 :                     preferred_az_id: preferred_az_id.as_ref().map(|az| az.0.clone()),
    5615            0 :                 });
    5616            0 :             }
    5617              : 
    5618            0 :             child_tsps.push((target.parent_id, this_child_tsps));
    5619              :         }
    5620              : 
    5621            0 :         if let Err(e) = self
    5622            0 :             .persistence
    5623            0 :             .begin_shard_split(old_shard_count, tenant_id, child_tsps)
    5624            0 :             .await
    5625              :         {
    5626            0 :             match e {
    5627              :                 DatabaseError::Query(diesel::result::Error::DatabaseError(
    5628              :                     DatabaseErrorKind::UniqueViolation,
    5629              :                     _,
    5630              :                 )) => {
    5631              :                     // Inserting a child shard violated a unique constraint: we raced with another call to
    5632              :                     // this function
    5633            0 :                     tracing::warn!("Conflicting attempt to split {tenant_id}: {e}");
    5634            0 :                     return Err(ApiError::Conflict("Tenant is already splitting".into()));
    5635              :                 }
    5636            0 :                 _ => return Err(ApiError::InternalServerError(e.into())),
    5637              :             }
    5638            0 :         }
    5639            0 :         fail::fail_point!("shard-split-post-begin", |_| Err(
    5640            0 :             ApiError::InternalServerError(anyhow::anyhow!("failpoint"))
    5641            0 :         ));
    5642              : 
    5643              :         // Now that I have persisted the splitting state, apply it in-memory.  This is infallible, so
    5644              :         // callers may assume that if splitting is set in memory, then it was persisted, and if splitting
    5645              :         // is not set in memory, then it was not persisted.
    5646              :         {
    5647            0 :             let mut locked = self.inner.write().unwrap();
    5648            0 :             for target in &targets {
    5649            0 :                 if let Some(parent_shard) = locked.tenants.get_mut(&target.parent_id) {
    5650            0 :                     parent_shard.splitting = SplitState::Splitting;
    5651            0 :                     // Put the observed state to None, to reflect that it is indeterminate once we start the
    5652            0 :                     // split operation.
    5653            0 :                     parent_shard
    5654            0 :                         .observed
    5655            0 :                         .locations
    5656            0 :                         .insert(target.node.get_id(), ObservedStateLocation { conf: None });
    5657            0 :                 }
    5658              :             }
    5659              :         }
    5660              : 
    5661              :         // TODO: issue split calls concurrently (this only matters once we're splitting
    5662              :         // N>1 shards into M shards -- initially we're usually splitting 1 shard into N).
    5663              : 
    5664            0 :         for target in &targets {
    5665              :             let ShardSplitTarget {
    5666            0 :                 parent_id,
    5667            0 :                 node,
    5668            0 :                 child_ids,
    5669            0 :             } = target;
    5670            0 :             let client = PageserverClient::new(
    5671            0 :                 node.get_id(),
    5672            0 :                 node.base_url(),
    5673            0 :                 self.config.pageserver_jwt_token.as_deref(),
    5674            0 :                 self.config.ssl_ca_cert.clone(),
    5675            0 :             )
    5676            0 :             .map_err(|e| passthrough_api_error(node, e))?;
    5677            0 :             let response = client
    5678            0 :                 .tenant_shard_split(
    5679            0 :                     *parent_id,
    5680            0 :                     TenantShardSplitRequest {
    5681            0 :                         new_shard_count: new_shard_count.literal(),
    5682            0 :                         new_stripe_size,
    5683            0 :                     },
    5684            0 :                 )
    5685            0 :                 .await
    5686            0 :                 .map_err(|e| ApiError::Conflict(format!("Failed to split {}: {}", parent_id, e)))?;
    5687              : 
    5688            0 :             fail::fail_point!("shard-split-post-remote", |_| Err(ApiError::Conflict(
    5689            0 :                 "failpoint".to_string()
    5690            0 :             )));
    5691              : 
    5692            0 :             failpoint_support::sleep_millis_async!("shard-split-post-remote-sleep", &self.cancel);
    5693              : 
    5694            0 :             tracing::info!(
    5695            0 :                 "Split {} into {}",
    5696            0 :                 parent_id,
    5697            0 :                 response
    5698            0 :                     .new_shards
    5699            0 :                     .iter()
    5700            0 :                     .map(|s| format!("{:?}", s))
    5701            0 :                     .collect::<Vec<_>>()
    5702            0 :                     .join(",")
    5703              :             );
    5704              : 
    5705            0 :             if &response.new_shards != child_ids {
    5706              :                 // This should never happen: the pageserver should agree with us on how shard splits work.
    5707            0 :                 return Err(ApiError::InternalServerError(anyhow::anyhow!(
    5708            0 :                     "Splitting shard {} resulted in unexpected IDs: {:?} (expected {:?})",
    5709            0 :                     parent_id,
    5710            0 :                     response.new_shards,
    5711            0 :                     child_ids
    5712            0 :                 )));
    5713            0 :             }
    5714              :         }
    5715              : 
    5716              :         // TODO: if the pageserver restarted concurrently with our split API call,
    5717              :         // the actual generation of the child shard might differ from the generation
    5718              :         // we expect it to have.  In order for our in-database generation to end up
    5719              :         // correct, we should carry the child generation back in the response and apply it here
    5720              :         // in complete_shard_split (and apply the correct generation in memory)
    5721              :         // (or, we can carry generation in the request and reject the request if
    5722              :         //  it doesn't match, but that requires more retry logic on this side)
    5723              : 
    5724            0 :         self.persistence
    5725            0 :             .complete_shard_split(tenant_id, old_shard_count)
    5726            0 :             .await?;
    5727              : 
    5728            0 :         fail::fail_point!("shard-split-post-complete", |_| Err(
    5729            0 :             ApiError::InternalServerError(anyhow::anyhow!("failpoint"))
    5730            0 :         ));
    5731              : 
    5732              :         // Replace all the shards we just split with their children: this phase is infallible.
    5733            0 :         let (response, child_locations, waiters) =
    5734            0 :             self.tenant_shard_split_commit_inmem(tenant_id, new_shard_count, new_stripe_size);
    5735            0 : 
    5736            0 :         // Send compute notifications for all the new shards
    5737            0 :         let mut failed_notifications = Vec::new();
    5738            0 :         for (child_id, child_ps, stripe_size) in child_locations {
    5739            0 :             if let Err(e) = self
    5740            0 :                 .compute_hook
    5741            0 :                 .notify(
    5742            0 :                     compute_hook::ShardUpdate {
    5743            0 :                         tenant_shard_id: child_id,
    5744            0 :                         node_id: child_ps,
    5745            0 :                         stripe_size,
    5746            0 :                         preferred_az: preferred_az_id.as_ref().map(Cow::Borrowed),
    5747            0 :                     },
    5748            0 :                     &self.cancel,
    5749            0 :                 )
    5750            0 :                 .await
    5751              :             {
    5752            0 :                 tracing::warn!(
    5753            0 :                     "Failed to update compute of {}->{} during split, proceeding anyway to complete split ({e})",
    5754              :                     child_id,
    5755              :                     child_ps
    5756              :                 );
    5757            0 :                 failed_notifications.push(child_id);
    5758            0 :             }
    5759              :         }
    5760              : 
    5761              :         // If we failed any compute notifications, make a note to retry later.
    5762            0 :         if !failed_notifications.is_empty() {
    5763            0 :             let mut locked = self.inner.write().unwrap();
    5764            0 :             for failed in failed_notifications {
    5765            0 :                 if let Some(shard) = locked.tenants.get_mut(&failed) {
    5766            0 :                     shard.pending_compute_notification = true;
    5767            0 :                 }
    5768              :             }
    5769            0 :         }
    5770              : 
    5771            0 :         Ok((response, waiters))
    5772            0 :     }
    5773              : 
    5774              :     /// A graceful migration: update the preferred node and let optimisation handle the migration
    5775              :     /// in the background (may take a long time as it will fully warm up a location before cutting over)
    5776              :     ///
    5777              :     /// Our external API calls this a 'prewarm=true' migration, but internally it isn't a special prewarm step: it's
    5778              :     /// just a migration that uses the same graceful procedure as our background scheduling optimisations would use.
    5779            0 :     fn tenant_shard_migrate_with_prewarm(
    5780            0 :         &self,
    5781            0 :         migrate_req: &TenantShardMigrateRequest,
    5782            0 :         shard: &mut TenantShard,
    5783            0 :         scheduler: &mut Scheduler,
    5784            0 :         schedule_context: ScheduleContext,
    5785            0 :     ) -> Result<Option<ScheduleOptimization>, ApiError> {
    5786            0 :         shard.set_preferred_node(Some(migrate_req.node_id));
    5787            0 : 
    5788            0 :         // Generate whatever the initial change to the intent is: this could be creation of a secondary, or
    5789            0 :         // cutting over to an existing secondary.  Caller is responsible for validating this before applying it,
    5790            0 :         // e.g. by checking secondary is warm enough.
    5791            0 :         Ok(shard.optimize_attachment(scheduler, &schedule_context))
    5792            0 :     }
    5793              : 
    5794              :     /// Immediate migration: directly update the intent state and kick off a reconciler
    5795            0 :     fn tenant_shard_migrate_immediate(
    5796            0 :         &self,
    5797            0 :         migrate_req: &TenantShardMigrateRequest,
    5798            0 :         nodes: &Arc<HashMap<NodeId, Node>>,
    5799            0 :         shard: &mut TenantShard,
    5800            0 :         scheduler: &mut Scheduler,
    5801            0 :     ) -> Result<Option<ReconcilerWaiter>, ApiError> {
    5802            0 :         // Non-graceful migration: update the intent state immediately
    5803            0 :         let old_attached = *shard.intent.get_attached();
    5804            0 :         match shard.policy {
    5805            0 :             PlacementPolicy::Attached(n) => {
    5806            0 :                 // If our new attached node was a secondary, it no longer should be.
    5807            0 :                 shard
    5808            0 :                     .intent
    5809            0 :                     .remove_secondary(scheduler, migrate_req.node_id);
    5810            0 : 
    5811            0 :                 shard
    5812            0 :                     .intent
    5813            0 :                     .set_attached(scheduler, Some(migrate_req.node_id));
    5814              : 
    5815              :                 // If we were already attached to something, demote that to a secondary
    5816            0 :                 if let Some(old_attached) = old_attached {
    5817            0 :                     if n > 0 {
    5818              :                         // Remove other secondaries to make room for the location we'll demote
    5819            0 :                         while shard.intent.get_secondary().len() >= n {
    5820            0 :                             shard.intent.pop_secondary(scheduler);
    5821            0 :                         }
    5822              : 
    5823            0 :                         shard.intent.push_secondary(scheduler, old_attached);
    5824            0 :                     }
    5825            0 :                 }
    5826              :             }
    5827            0 :             PlacementPolicy::Secondary => {
    5828            0 :                 shard.intent.clear(scheduler);
    5829            0 :                 shard.intent.push_secondary(scheduler, migrate_req.node_id);
    5830            0 :             }
    5831              :             PlacementPolicy::Detached => {
    5832            0 :                 return Err(ApiError::BadRequest(anyhow::anyhow!(
    5833            0 :                     "Cannot migrate a tenant that is PlacementPolicy::Detached: configure it to an attached policy first"
    5834            0 :                 )));
    5835              :             }
    5836              :         }
    5837              : 
    5838            0 :         tracing::info!("Migrating: new intent {:?}", shard.intent);
    5839            0 :         shard.sequence = shard.sequence.next();
    5840            0 :         shard.set_preferred_node(None); // Abort any in-flight graceful migration
    5841            0 :         Ok(self.maybe_configured_reconcile_shard(
    5842            0 :             shard,
    5843            0 :             nodes,
    5844            0 :             (&migrate_req.migration_config).into(),
    5845            0 :         ))
    5846            0 :     }
    5847              : 
    5848            0 :     pub(crate) async fn tenant_shard_migrate(
    5849            0 :         &self,
    5850            0 :         tenant_shard_id: TenantShardId,
    5851            0 :         migrate_req: TenantShardMigrateRequest,
    5852            0 :     ) -> Result<TenantShardMigrateResponse, ApiError> {
    5853              :         // Depending on whether the migration is a change and whether it's graceful or immediate, we might
    5854              :         // get a different outcome to handle
    5855              :         enum MigrationOutcome {
    5856              :             Optimization(Option<ScheduleOptimization>),
    5857              :             Reconcile(Option<ReconcilerWaiter>),
    5858              :         }
    5859              : 
    5860            0 :         let outcome = {
    5861            0 :             let mut locked = self.inner.write().unwrap();
    5862            0 :             let (nodes, tenants, scheduler) = locked.parts_mut();
    5863              : 
    5864            0 :             let Some(node) = nodes.get(&migrate_req.node_id) else {
    5865            0 :                 return Err(ApiError::BadRequest(anyhow::anyhow!(
    5866            0 :                     "Node {} not found",
    5867            0 :                     migrate_req.node_id
    5868            0 :                 )));
    5869              :             };
    5870              : 
    5871              :             // Migration to unavavailable node requires force flag
    5872            0 :             if !node.is_available() {
    5873            0 :                 if migrate_req.migration_config.override_scheduler {
    5874              :                     // Warn but proceed: the caller may intend to manually adjust the placement of
    5875              :                     // a shard even if the node is down, e.g. if intervening during an incident.
    5876            0 :                     tracing::warn!("Forcibly migrating to unavailable node {node}");
    5877              :                 } else {
    5878            0 :                     tracing::warn!("Node {node} is unavailable, refusing migration");
    5879            0 :                     return Err(ApiError::PreconditionFailed(
    5880            0 :                         format!("Node {node} is unavailable").into_boxed_str(),
    5881            0 :                     ));
    5882              :                 }
    5883            0 :             }
    5884              : 
    5885              :             // Calculate the ScheduleContext for this tenant
    5886            0 :             let mut schedule_context = ScheduleContext::default();
    5887            0 :             for (_shard_id, shard) in
    5888            0 :                 tenants.range(TenantShardId::tenant_range(tenant_shard_id.tenant_id))
    5889            0 :             {
    5890            0 :                 schedule_context.avoid(&shard.intent.all_pageservers());
    5891            0 :             }
    5892              : 
    5893              :             // Look up the specific shard we will migrate
    5894            0 :             let Some(shard) = tenants.get_mut(&tenant_shard_id) else {
    5895            0 :                 return Err(ApiError::NotFound(
    5896            0 :                     anyhow::anyhow!("Tenant shard not found").into(),
    5897            0 :                 ));
    5898              :             };
    5899              : 
    5900              :             // Migration to a node with unfavorable scheduling score requires a force flag, because it might just
    5901              :             // be migrated back by the optimiser.
    5902            0 :             if let Some(better_node) = shard.find_better_location::<AttachedShardTag>(
    5903            0 :                 scheduler,
    5904            0 :                 &schedule_context,
    5905            0 :                 migrate_req.node_id,
    5906            0 :                 &[],
    5907            0 :             ) {
    5908            0 :                 if !migrate_req.migration_config.override_scheduler {
    5909            0 :                     return Err(ApiError::PreconditionFailed(
    5910            0 :                         "Migration to a worse-scoring node".into(),
    5911            0 :                     ));
    5912              :                 } else {
    5913            0 :                     tracing::info!(
    5914            0 :                         "Migrating to a worse-scoring node {} (optimiser would prefer {better_node})",
    5915              :                         migrate_req.node_id
    5916              :                     );
    5917              :                 }
    5918            0 :             }
    5919              : 
    5920            0 :             if let Some(origin_node_id) = migrate_req.origin_node_id {
    5921            0 :                 if shard.intent.get_attached() != &Some(origin_node_id) {
    5922            0 :                     return Err(ApiError::PreconditionFailed(
    5923            0 :                         format!(
    5924            0 :                             "Migration expected to originate from {} but shard is on {:?}",
    5925            0 :                             origin_node_id,
    5926            0 :                             shard.intent.get_attached()
    5927            0 :                         )
    5928            0 :                         .into(),
    5929            0 :                     ));
    5930            0 :                 }
    5931            0 :             }
    5932              : 
    5933            0 :             if shard.intent.get_attached() == &Some(migrate_req.node_id) {
    5934              :                 // No-op case: we will still proceed to wait for reconciliation in case it is
    5935              :                 // incomplete from an earlier update to the intent.
    5936            0 :                 tracing::info!("Migrating: intent is unchanged {:?}", shard.intent);
    5937              : 
    5938              :                 // An instruction to migrate to the currently attached node should
    5939              :                 // cancel any pending graceful migration
    5940            0 :                 shard.set_preferred_node(None);
    5941            0 : 
    5942            0 :                 MigrationOutcome::Reconcile(self.maybe_configured_reconcile_shard(
    5943            0 :                     shard,
    5944            0 :                     nodes,
    5945            0 :                     (&migrate_req.migration_config).into(),
    5946            0 :                 ))
    5947            0 :             } else if migrate_req.migration_config.prewarm {
    5948            0 :                 MigrationOutcome::Optimization(self.tenant_shard_migrate_with_prewarm(
    5949            0 :                     &migrate_req,
    5950            0 :                     shard,
    5951            0 :                     scheduler,
    5952            0 :                     schedule_context,
    5953            0 :                 )?)
    5954              :             } else {
    5955            0 :                 MigrationOutcome::Reconcile(self.tenant_shard_migrate_immediate(
    5956            0 :                     &migrate_req,
    5957            0 :                     nodes,
    5958            0 :                     shard,
    5959            0 :                     scheduler,
    5960            0 :                 )?)
    5961              :             }
    5962              :         };
    5963              : 
    5964              :         // We may need to validate + apply an optimisation, or we may need to just retrive a reconcile waiter
    5965            0 :         let waiter = match outcome {
    5966            0 :             MigrationOutcome::Optimization(Some(optimization)) => {
    5967              :                 // Validate and apply the optimization -- this would happen anyway in background reconcile loop, but
    5968              :                 // we might as well do it more promptly as this is a direct external request.
    5969            0 :                 let mut validated = self
    5970            0 :                     .optimize_all_validate(vec![(tenant_shard_id, optimization)])
    5971            0 :                     .await;
    5972            0 :                 if let Some((_shard_id, optimization)) = validated.pop() {
    5973            0 :                     let mut locked = self.inner.write().unwrap();
    5974            0 :                     let (nodes, tenants, scheduler) = locked.parts_mut();
    5975            0 :                     let Some(shard) = tenants.get_mut(&tenant_shard_id) else {
    5976              :                         // Rare but possible: tenant is removed between generating optimisation and validating it.
    5977            0 :                         return Err(ApiError::NotFound(
    5978            0 :                             anyhow::anyhow!("Tenant shard not found").into(),
    5979            0 :                         ));
    5980              :                     };
    5981              : 
    5982            0 :                     if !shard.apply_optimization(scheduler, optimization) {
    5983              :                         // This can happen but is unusual enough to warn on: something else changed in the shard that made the optimisation stale
    5984              :                         // and therefore not applied.
    5985            0 :                         tracing::warn!(
    5986            0 :                             "Schedule optimisation generated during graceful migration was not applied, shard changed?"
    5987              :                         );
    5988            0 :                     }
    5989            0 :                     self.maybe_configured_reconcile_shard(
    5990            0 :                         shard,
    5991            0 :                         nodes,
    5992            0 :                         (&migrate_req.migration_config).into(),
    5993            0 :                     )
    5994              :                 } else {
    5995            0 :                     None
    5996              :                 }
    5997              :             }
    5998            0 :             MigrationOutcome::Optimization(None) => None,
    5999            0 :             MigrationOutcome::Reconcile(waiter) => waiter,
    6000              :         };
    6001              : 
    6002              :         // Finally, wait for any reconcile we started to complete.  In the case of immediate-mode migrations to cold
    6003              :         // locations, this has a good chance of timing out.
    6004            0 :         if let Some(waiter) = waiter {
    6005            0 :             waiter.wait_timeout(RECONCILE_TIMEOUT).await?;
    6006              :         } else {
    6007            0 :             tracing::info!("Migration is a no-op");
    6008              :         }
    6009              : 
    6010            0 :         Ok(TenantShardMigrateResponse {})
    6011            0 :     }
    6012              : 
    6013            0 :     pub(crate) async fn tenant_shard_migrate_secondary(
    6014            0 :         &self,
    6015            0 :         tenant_shard_id: TenantShardId,
    6016            0 :         migrate_req: TenantShardMigrateRequest,
    6017            0 :     ) -> Result<TenantShardMigrateResponse, ApiError> {
    6018            0 :         let waiter = {
    6019            0 :             let mut locked = self.inner.write().unwrap();
    6020            0 :             let (nodes, tenants, scheduler) = locked.parts_mut();
    6021              : 
    6022            0 :             let Some(node) = nodes.get(&migrate_req.node_id) else {
    6023            0 :                 return Err(ApiError::BadRequest(anyhow::anyhow!(
    6024            0 :                     "Node {} not found",
    6025            0 :                     migrate_req.node_id
    6026            0 :                 )));
    6027              :             };
    6028              : 
    6029            0 :             if !node.is_available() {
    6030              :                 // Warn but proceed: the caller may intend to manually adjust the placement of
    6031              :                 // a shard even if the node is down, e.g. if intervening during an incident.
    6032            0 :                 tracing::warn!("Migrating to unavailable node {node}");
    6033            0 :             }
    6034              : 
    6035            0 :             let Some(shard) = tenants.get_mut(&tenant_shard_id) else {
    6036            0 :                 return Err(ApiError::NotFound(
    6037            0 :                     anyhow::anyhow!("Tenant shard not found").into(),
    6038            0 :                 ));
    6039              :             };
    6040              : 
    6041            0 :             if shard.intent.get_secondary().len() == 1
    6042            0 :                 && shard.intent.get_secondary()[0] == migrate_req.node_id
    6043              :             {
    6044            0 :                 tracing::info!(
    6045            0 :                     "Migrating secondary to {node}: intent is unchanged {:?}",
    6046              :                     shard.intent
    6047              :                 );
    6048            0 :             } else if shard.intent.get_attached() == &Some(migrate_req.node_id) {
    6049            0 :                 tracing::info!(
    6050            0 :                     "Migrating secondary to {node}: already attached where we were asked to create a secondary"
    6051              :                 );
    6052              :             } else {
    6053            0 :                 let old_secondaries = shard.intent.get_secondary().clone();
    6054            0 :                 for secondary in old_secondaries {
    6055            0 :                     shard.intent.remove_secondary(scheduler, secondary);
    6056            0 :                 }
    6057              : 
    6058            0 :                 shard.intent.push_secondary(scheduler, migrate_req.node_id);
    6059            0 :                 shard.sequence = shard.sequence.next();
    6060            0 :                 tracing::info!(
    6061            0 :                     "Migrating secondary to {node}: new intent {:?}",
    6062              :                     shard.intent
    6063              :                 );
    6064              :             }
    6065              : 
    6066            0 :             self.maybe_reconcile_shard(shard, nodes, ReconcilerPriority::High)
    6067              :         };
    6068              : 
    6069            0 :         if let Some(waiter) = waiter {
    6070            0 :             waiter.wait_timeout(RECONCILE_TIMEOUT).await?;
    6071              :         } else {
    6072            0 :             tracing::info!("Migration is a no-op");
    6073              :         }
    6074              : 
    6075            0 :         Ok(TenantShardMigrateResponse {})
    6076            0 :     }
    6077              : 
    6078              :     /// 'cancel' in this context means cancel any ongoing reconcile
    6079            0 :     pub(crate) async fn tenant_shard_cancel_reconcile(
    6080            0 :         &self,
    6081            0 :         tenant_shard_id: TenantShardId,
    6082            0 :     ) -> Result<(), ApiError> {
    6083              :         // Take state lock and fire the cancellation token, after which we drop lock and wait for any ongoing reconcile to complete
    6084            0 :         let waiter = {
    6085            0 :             let locked = self.inner.write().unwrap();
    6086            0 :             let Some(shard) = locked.tenants.get(&tenant_shard_id) else {
    6087            0 :                 return Err(ApiError::NotFound(
    6088            0 :                     anyhow::anyhow!("Tenant shard not found").into(),
    6089            0 :                 ));
    6090              :             };
    6091              : 
    6092            0 :             let waiter = shard.get_waiter();
    6093            0 :             match waiter {
    6094              :                 None => {
    6095            0 :                     tracing::info!("Shard does not have an ongoing Reconciler");
    6096            0 :                     return Ok(());
    6097              :                 }
    6098            0 :                 Some(waiter) => {
    6099            0 :                     tracing::info!("Cancelling Reconciler");
    6100            0 :                     shard.cancel_reconciler();
    6101            0 :                     waiter
    6102            0 :                 }
    6103            0 :             }
    6104            0 :         };
    6105            0 : 
    6106            0 :         // Cancellation should be prompt.  If this fails we have still done our job of firing the
    6107            0 :         // cancellation token, but by returning an ApiError we will indicate to the caller that
    6108            0 :         // the Reconciler is misbehaving and not respecting the cancellation token
    6109            0 :         self.await_waiters(vec![waiter], SHORT_RECONCILE_TIMEOUT)
    6110            0 :             .await?;
    6111              : 
    6112            0 :         Ok(())
    6113            0 :     }
    6114              : 
    6115              :     /// This is for debug/support only: we simply drop all state for a tenant, without
    6116              :     /// detaching or deleting it on pageservers.
    6117            0 :     pub(crate) async fn tenant_drop(&self, tenant_id: TenantId) -> Result<(), ApiError> {
    6118            0 :         self.persistence.delete_tenant(tenant_id).await?;
    6119              : 
    6120            0 :         let mut locked = self.inner.write().unwrap();
    6121            0 :         let (_nodes, tenants, scheduler) = locked.parts_mut();
    6122            0 :         let mut shards = Vec::new();
    6123            0 :         for (tenant_shard_id, _) in tenants.range(TenantShardId::tenant_range(tenant_id)) {
    6124            0 :             shards.push(*tenant_shard_id);
    6125            0 :         }
    6126              : 
    6127            0 :         for shard_id in shards {
    6128            0 :             if let Some(mut shard) = tenants.remove(&shard_id) {
    6129            0 :                 shard.intent.clear(scheduler);
    6130            0 :             }
    6131              :         }
    6132              : 
    6133            0 :         Ok(())
    6134            0 :     }
    6135              : 
    6136              :     /// This is for debug/support only: assuming tenant data is already present in S3, we "create" a
    6137              :     /// tenant with a very high generation number so that it will see the existing data.
    6138            0 :     pub(crate) async fn tenant_import(
    6139            0 :         &self,
    6140            0 :         tenant_id: TenantId,
    6141            0 :     ) -> Result<TenantCreateResponse, ApiError> {
    6142            0 :         // Pick an arbitrary available pageserver to use for scanning the tenant in remote storage
    6143            0 :         let maybe_node = {
    6144            0 :             self.inner
    6145            0 :                 .read()
    6146            0 :                 .unwrap()
    6147            0 :                 .nodes
    6148            0 :                 .values()
    6149            0 :                 .find(|n| n.is_available())
    6150            0 :                 .cloned()
    6151              :         };
    6152            0 :         let Some(node) = maybe_node else {
    6153            0 :             return Err(ApiError::BadRequest(anyhow::anyhow!("No nodes available")));
    6154              :         };
    6155              : 
    6156            0 :         let client = PageserverClient::new(
    6157            0 :             node.get_id(),
    6158            0 :             node.base_url(),
    6159            0 :             self.config.pageserver_jwt_token.as_deref(),
    6160            0 :             self.config.ssl_ca_cert.clone(),
    6161            0 :         )
    6162            0 :         .map_err(|e| passthrough_api_error(&node, e))?;
    6163              : 
    6164            0 :         let scan_result = client
    6165            0 :             .tenant_scan_remote_storage(tenant_id)
    6166            0 :             .await
    6167            0 :             .map_err(|e| passthrough_api_error(&node, e))?;
    6168              : 
    6169              :         // A post-split tenant may contain a mixture of shard counts in remote storage: pick the highest count.
    6170            0 :         let Some(shard_count) = scan_result
    6171            0 :             .shards
    6172            0 :             .iter()
    6173            0 :             .map(|s| s.tenant_shard_id.shard_count)
    6174            0 :             .max()
    6175              :         else {
    6176            0 :             return Err(ApiError::NotFound(
    6177            0 :                 anyhow::anyhow!("No shards found").into(),
    6178            0 :             ));
    6179              :         };
    6180              : 
    6181              :         // Ideally we would set each newly imported shard's generation independently, but for correctness it is sufficient
    6182              :         // to
    6183            0 :         let generation = scan_result
    6184            0 :             .shards
    6185            0 :             .iter()
    6186            0 :             .map(|s| s.generation)
    6187            0 :             .max()
    6188            0 :             .expect("We already validated >0 shards");
    6189            0 : 
    6190            0 :         // FIXME: we have no way to recover the shard stripe size from contents of remote storage: this will
    6191            0 :         // only work if they were using the default stripe size.
    6192            0 :         let stripe_size = ShardParameters::DEFAULT_STRIPE_SIZE;
    6193              : 
    6194            0 :         let (response, waiters) = self
    6195            0 :             .do_tenant_create(TenantCreateRequest {
    6196            0 :                 new_tenant_id: TenantShardId::unsharded(tenant_id),
    6197            0 :                 generation,
    6198            0 : 
    6199            0 :                 shard_parameters: ShardParameters {
    6200            0 :                     count: shard_count,
    6201            0 :                     stripe_size,
    6202            0 :                 },
    6203            0 :                 placement_policy: Some(PlacementPolicy::Attached(0)), // No secondaries, for convenient debug/hacking
    6204            0 :                 config: TenantConfig::default(),
    6205            0 :             })
    6206            0 :             .await?;
    6207              : 
    6208            0 :         if let Err(e) = self.await_waiters(waiters, SHORT_RECONCILE_TIMEOUT).await {
    6209              :             // Since this is a debug/support operation, all kinds of weird issues are possible (e.g. this
    6210              :             // tenant doesn't exist in the control plane), so don't fail the request if it can't fully
    6211              :             // reconcile, as reconciliation includes notifying compute.
    6212            0 :             tracing::warn!(%tenant_id, "Reconcile not done yet while importing tenant ({e})");
    6213            0 :         }
    6214              : 
    6215            0 :         Ok(response)
    6216            0 :     }
    6217              : 
    6218              :     /// For debug/support: a full JSON dump of TenantShards.  Returns a response so that
    6219              :     /// we don't have to make TenantShard clonable in the return path.
    6220            0 :     pub(crate) fn tenants_dump(&self) -> Result<hyper::Response<hyper::Body>, ApiError> {
    6221            0 :         let serialized = {
    6222            0 :             let locked = self.inner.read().unwrap();
    6223            0 :             let result = locked.tenants.values().collect::<Vec<_>>();
    6224            0 :             serde_json::to_string(&result).map_err(|e| ApiError::InternalServerError(e.into()))?
    6225              :         };
    6226              : 
    6227            0 :         hyper::Response::builder()
    6228            0 :             .status(hyper::StatusCode::OK)
    6229            0 :             .header(hyper::header::CONTENT_TYPE, "application/json")
    6230            0 :             .body(hyper::Body::from(serialized))
    6231            0 :             .map_err(|e| ApiError::InternalServerError(e.into()))
    6232            0 :     }
    6233              : 
    6234              :     /// Check the consistency of in-memory state vs. persistent state, and check that the
    6235              :     /// scheduler's statistics are up to date.
    6236              :     ///
    6237              :     /// These consistency checks expect an **idle** system.  If changes are going on while
    6238              :     /// we run, then we can falsely indicate a consistency issue.  This is sufficient for end-of-test
    6239              :     /// checks, but not suitable for running continuously in the background in the field.
    6240            0 :     pub(crate) async fn consistency_check(&self) -> Result<(), ApiError> {
    6241            0 :         let (mut expect_nodes, mut expect_shards) = {
    6242            0 :             let locked = self.inner.read().unwrap();
    6243            0 : 
    6244            0 :             locked
    6245            0 :                 .scheduler
    6246            0 :                 .consistency_check(locked.nodes.values(), locked.tenants.values())
    6247            0 :                 .context("Scheduler checks")
    6248            0 :                 .map_err(ApiError::InternalServerError)?;
    6249              : 
    6250            0 :             let expect_nodes = locked
    6251            0 :                 .nodes
    6252            0 :                 .values()
    6253            0 :                 .map(|n| n.to_persistent())
    6254            0 :                 .collect::<Vec<_>>();
    6255            0 : 
    6256            0 :             let expect_shards = locked
    6257            0 :                 .tenants
    6258            0 :                 .values()
    6259            0 :                 .map(|t| t.to_persistent())
    6260            0 :                 .collect::<Vec<_>>();
    6261              : 
    6262              :             // This method can only validate the state of an idle system: if a reconcile is in
    6263              :             // progress, fail out early to avoid giving false errors on state that won't match
    6264              :             // between database and memory under a ReconcileResult is processed.
    6265            0 :             for t in locked.tenants.values() {
    6266            0 :                 if t.reconciler.is_some() {
    6267            0 :                     return Err(ApiError::InternalServerError(anyhow::anyhow!(
    6268            0 :                         "Shard {} reconciliation in progress",
    6269            0 :                         t.tenant_shard_id
    6270            0 :                     )));
    6271            0 :                 }
    6272              :             }
    6273              : 
    6274            0 :             (expect_nodes, expect_shards)
    6275              :         };
    6276              : 
    6277            0 :         let mut nodes = self.persistence.list_nodes().await?;
    6278            0 :         expect_nodes.sort_by_key(|n| n.node_id);
    6279            0 :         nodes.sort_by_key(|n| n.node_id);
    6280              : 
    6281              :         // Errors relating to nodes are deferred so that we don't skip the shard checks below if we have a node error
    6282            0 :         let node_result = if nodes != expect_nodes {
    6283            0 :             tracing::error!("Consistency check failed on nodes.");
    6284            0 :             tracing::error!(
    6285            0 :                 "Nodes in memory: {}",
    6286            0 :                 serde_json::to_string(&expect_nodes)
    6287            0 :                     .map_err(|e| ApiError::InternalServerError(e.into()))?
    6288              :             );
    6289            0 :             tracing::error!(
    6290            0 :                 "Nodes in database: {}",
    6291            0 :                 serde_json::to_string(&nodes)
    6292            0 :                     .map_err(|e| ApiError::InternalServerError(e.into()))?
    6293              :             );
    6294            0 :             Err(ApiError::InternalServerError(anyhow::anyhow!(
    6295            0 :                 "Node consistency failure"
    6296            0 :             )))
    6297              :         } else {
    6298            0 :             Ok(())
    6299              :         };
    6300              : 
    6301            0 :         let mut persistent_shards = self.persistence.load_active_tenant_shards().await?;
    6302            0 :         persistent_shards
    6303            0 :             .sort_by_key(|tsp| (tsp.tenant_id.clone(), tsp.shard_number, tsp.shard_count));
    6304            0 : 
    6305            0 :         expect_shards.sort_by_key(|tsp| (tsp.tenant_id.clone(), tsp.shard_number, tsp.shard_count));
    6306              : 
    6307              :         // Because JSON contents of persistent tenants might disagree with the fields in current `TenantConfig`
    6308              :         // definition, we will do an encode/decode cycle to ensure any legacy fields are dropped and any new
    6309              :         // fields are added, before doing a comparison.
    6310            0 :         for tsp in &mut persistent_shards {
    6311            0 :             let config: TenantConfig = serde_json::from_str(&tsp.config)
    6312            0 :                 .map_err(|e| ApiError::InternalServerError(e.into()))?;
    6313            0 :             tsp.config = serde_json::to_string(&config).expect("Encoding config is infallible");
    6314              :         }
    6315              : 
    6316            0 :         if persistent_shards != expect_shards {
    6317            0 :             tracing::error!("Consistency check failed on shards.");
    6318              : 
    6319            0 :             tracing::error!(
    6320            0 :                 "Shards in memory: {}",
    6321            0 :                 serde_json::to_string(&expect_shards)
    6322            0 :                     .map_err(|e| ApiError::InternalServerError(e.into()))?
    6323              :             );
    6324            0 :             tracing::error!(
    6325            0 :                 "Shards in database: {}",
    6326            0 :                 serde_json::to_string(&persistent_shards)
    6327            0 :                     .map_err(|e| ApiError::InternalServerError(e.into()))?
    6328              :             );
    6329              : 
    6330              :             // The total dump log lines above are useful in testing but in the field grafana will
    6331              :             // usually just drop them because they're so large. So we also do some explicit logging
    6332              :             // of just the diffs.
    6333            0 :             let persistent_shards = persistent_shards
    6334            0 :                 .into_iter()
    6335            0 :                 .map(|tsp| (tsp.get_tenant_shard_id().unwrap(), tsp))
    6336            0 :                 .collect::<HashMap<_, _>>();
    6337            0 :             let expect_shards = expect_shards
    6338            0 :                 .into_iter()
    6339            0 :                 .map(|tsp| (tsp.get_tenant_shard_id().unwrap(), tsp))
    6340            0 :                 .collect::<HashMap<_, _>>();
    6341            0 :             for (tenant_shard_id, persistent_tsp) in &persistent_shards {
    6342            0 :                 match expect_shards.get(tenant_shard_id) {
    6343              :                     None => {
    6344            0 :                         tracing::error!(
    6345            0 :                             "Shard {} found in database but not in memory",
    6346              :                             tenant_shard_id
    6347              :                         );
    6348              :                     }
    6349            0 :                     Some(expect_tsp) => {
    6350            0 :                         if expect_tsp != persistent_tsp {
    6351            0 :                             tracing::error!(
    6352            0 :                                 "Shard {} is inconsistent.  In memory: {}, database has: {}",
    6353            0 :                                 tenant_shard_id,
    6354            0 :                                 serde_json::to_string(expect_tsp).unwrap(),
    6355            0 :                                 serde_json::to_string(&persistent_tsp).unwrap()
    6356              :                             );
    6357            0 :                         }
    6358              :                     }
    6359              :                 }
    6360              :             }
    6361              : 
    6362              :             // Having already logged any differences, log any shards that simply aren't present in the database
    6363            0 :             for (tenant_shard_id, memory_tsp) in &expect_shards {
    6364            0 :                 if !persistent_shards.contains_key(tenant_shard_id) {
    6365            0 :                     tracing::error!(
    6366            0 :                         "Shard {} found in memory but not in database: {}",
    6367            0 :                         tenant_shard_id,
    6368            0 :                         serde_json::to_string(memory_tsp)
    6369            0 :                             .map_err(|e| ApiError::InternalServerError(e.into()))?
    6370              :                     );
    6371            0 :                 }
    6372              :             }
    6373              : 
    6374            0 :             return Err(ApiError::InternalServerError(anyhow::anyhow!(
    6375            0 :                 "Shard consistency failure"
    6376            0 :             )));
    6377            0 :         }
    6378            0 : 
    6379            0 :         node_result
    6380            0 :     }
    6381              : 
    6382              :     /// For debug/support: a JSON dump of the [`Scheduler`].  Returns a response so that
    6383              :     /// we don't have to make TenantShard clonable in the return path.
    6384            0 :     pub(crate) fn scheduler_dump(&self) -> Result<hyper::Response<hyper::Body>, ApiError> {
    6385            0 :         let serialized = {
    6386            0 :             let locked = self.inner.read().unwrap();
    6387            0 :             serde_json::to_string(&locked.scheduler)
    6388            0 :                 .map_err(|e| ApiError::InternalServerError(e.into()))?
    6389              :         };
    6390              : 
    6391            0 :         hyper::Response::builder()
    6392            0 :             .status(hyper::StatusCode::OK)
    6393            0 :             .header(hyper::header::CONTENT_TYPE, "application/json")
    6394            0 :             .body(hyper::Body::from(serialized))
    6395            0 :             .map_err(|e| ApiError::InternalServerError(e.into()))
    6396            0 :     }
    6397              : 
    6398              :     /// This is for debug/support only: we simply drop all state for a tenant, without
    6399              :     /// detaching or deleting it on pageservers.  We do not try and re-schedule any
    6400              :     /// tenants that were on this node.
    6401            0 :     pub(crate) async fn node_drop(&self, node_id: NodeId) -> Result<(), ApiError> {
    6402            0 :         self.persistence.delete_node(node_id).await?;
    6403              : 
    6404            0 :         let mut locked = self.inner.write().unwrap();
    6405              : 
    6406            0 :         for shard in locked.tenants.values_mut() {
    6407            0 :             shard.deref_node(node_id);
    6408            0 :             shard.observed.locations.remove(&node_id);
    6409            0 :         }
    6410              : 
    6411            0 :         let mut nodes = (*locked.nodes).clone();
    6412            0 :         nodes.remove(&node_id);
    6413            0 :         locked.nodes = Arc::new(nodes);
    6414            0 :         metrics::METRICS_REGISTRY
    6415            0 :             .metrics_group
    6416            0 :             .storage_controller_pageserver_nodes
    6417            0 :             .set(locked.nodes.len() as i64);
    6418            0 : 
    6419            0 :         locked.scheduler.node_remove(node_id);
    6420            0 : 
    6421            0 :         Ok(())
    6422            0 :     }
    6423              : 
    6424              :     /// If a node has any work on it, it will be rescheduled: this is "clean" in the sense
    6425              :     /// that we don't leave any bad state behind in the storage controller, but unclean
    6426              :     /// in the sense that we are not carefully draining the node.
    6427            0 :     pub(crate) async fn node_delete(&self, node_id: NodeId) -> Result<(), ApiError> {
    6428            0 :         let _node_lock =
    6429            0 :             trace_exclusive_lock(&self.node_op_locks, node_id, NodeOperations::Delete).await;
    6430              : 
    6431              :         // 1. Atomically update in-memory state:
    6432              :         //    - set the scheduling state to Pause to make subsequent scheduling ops skip it
    6433              :         //    - update shards' intents to exclude the node, and reschedule any shards whose intents we modified.
    6434              :         //    - drop the node from the main nodes map, so that when running reconciles complete they do not
    6435              :         //      re-insert references to this node into the ObservedState of shards
    6436              :         //    - drop the node from the scheduler
    6437              :         {
    6438            0 :             let mut locked = self.inner.write().unwrap();
    6439            0 :             let (nodes, tenants, scheduler) = locked.parts_mut();
    6440            0 : 
    6441            0 :             {
    6442            0 :                 let mut nodes_mut = (*nodes).deref().clone();
    6443            0 :                 match nodes_mut.get_mut(&node_id) {
    6444            0 :                     Some(node) => {
    6445            0 :                         // We do not bother setting this in the database, because we're about to delete the row anyway, and
    6446            0 :                         // if we crash it would not be desirable to leave the node paused after a restart.
    6447            0 :                         node.set_scheduling(NodeSchedulingPolicy::Pause);
    6448            0 :                     }
    6449              :                     None => {
    6450            0 :                         tracing::info!(
    6451            0 :                             "Node not found: presuming this is a retry and returning success"
    6452              :                         );
    6453            0 :                         return Ok(());
    6454              :                     }
    6455              :                 }
    6456              : 
    6457            0 :                 *nodes = Arc::new(nodes_mut);
    6458              :             }
    6459              : 
    6460            0 :             for (_tenant_id, mut schedule_context, shards) in
    6461            0 :                 TenantShardContextIterator::new(tenants, ScheduleMode::Normal)
    6462              :             {
    6463            0 :                 for shard in shards {
    6464            0 :                     if shard.deref_node(node_id) {
    6465            0 :                         if let Err(e) = shard.schedule(scheduler, &mut schedule_context) {
    6466              :                             // TODO: implement force flag to remove a node even if we can't reschedule
    6467              :                             // a tenant
    6468            0 :                             tracing::error!(
    6469            0 :                                 "Refusing to delete node, shard {} can't be rescheduled: {e}",
    6470              :                                 shard.tenant_shard_id
    6471              :                             );
    6472            0 :                             return Err(e.into());
    6473              :                         } else {
    6474            0 :                             tracing::info!(
    6475            0 :                                 "Rescheduled shard {} away from node during deletion",
    6476              :                                 shard.tenant_shard_id
    6477              :                             )
    6478              :                         }
    6479              : 
    6480            0 :                         self.maybe_reconcile_shard(shard, nodes, ReconcilerPriority::Normal);
    6481            0 :                     }
    6482              : 
    6483              :                     // Here we remove an existing observed location for the node we're removing, and it will
    6484              :                     // not be re-added by a reconciler's completion because we filter out removed nodes in
    6485              :                     // process_result.
    6486              :                     //
    6487              :                     // Note that we update the shard's observed state _after_ calling maybe_reconcile_shard: that
    6488              :                     // means any reconciles we spawned will know about the node we're deleting, enabling them
    6489              :                     // to do live migrations if it's still online.
    6490            0 :                     shard.observed.locations.remove(&node_id);
    6491              :                 }
    6492              :             }
    6493              : 
    6494            0 :             scheduler.node_remove(node_id);
    6495            0 : 
    6496            0 :             {
    6497            0 :                 let mut nodes_mut = (**nodes).clone();
    6498            0 :                 if let Some(mut removed_node) = nodes_mut.remove(&node_id) {
    6499            0 :                     // Ensure that any reconciler holding an Arc<> to this node will
    6500            0 :                     // drop out when trying to RPC to it (setting Offline state sets the
    6501            0 :                     // cancellation token on the Node object).
    6502            0 :                     removed_node.set_availability(NodeAvailability::Offline);
    6503            0 :                 }
    6504            0 :                 *nodes = Arc::new(nodes_mut);
    6505            0 :                 metrics::METRICS_REGISTRY
    6506            0 :                     .metrics_group
    6507            0 :                     .storage_controller_pageserver_nodes
    6508            0 :                     .set(nodes.len() as i64);
    6509            0 :             }
    6510            0 :         }
    6511            0 : 
    6512            0 :         // Note: some `generation_pageserver` columns on tenant shards in the database may still refer to
    6513            0 :         // the removed node, as this column means "The pageserver to which this generation was issued", and
    6514            0 :         // their generations won't get updated until the reconcilers moving them away from this node complete.
    6515            0 :         // That is safe because in Service::spawn we only use generation_pageserver if it refers to a node
    6516            0 :         // that exists.
    6517            0 : 
    6518            0 :         // 2. Actually delete the node from the database and from in-memory state
    6519            0 :         tracing::info!("Deleting node from database");
    6520            0 :         self.persistence.delete_node(node_id).await?;
    6521              : 
    6522            0 :         Ok(())
    6523            0 :     }
    6524              : 
    6525            0 :     pub(crate) async fn node_list(&self) -> Result<Vec<Node>, ApiError> {
    6526            0 :         let nodes = {
    6527            0 :             self.inner
    6528            0 :                 .read()
    6529            0 :                 .unwrap()
    6530            0 :                 .nodes
    6531            0 :                 .values()
    6532            0 :                 .cloned()
    6533            0 :                 .collect::<Vec<_>>()
    6534            0 :         };
    6535            0 : 
    6536            0 :         Ok(nodes)
    6537            0 :     }
    6538              : 
    6539            0 :     pub(crate) async fn get_node(&self, node_id: NodeId) -> Result<Node, ApiError> {
    6540            0 :         self.inner
    6541            0 :             .read()
    6542            0 :             .unwrap()
    6543            0 :             .nodes
    6544            0 :             .get(&node_id)
    6545            0 :             .cloned()
    6546            0 :             .ok_or(ApiError::NotFound(
    6547            0 :                 format!("Node {node_id} not registered").into(),
    6548            0 :             ))
    6549            0 :     }
    6550              : 
    6551            0 :     pub(crate) async fn get_node_shards(
    6552            0 :         &self,
    6553            0 :         node_id: NodeId,
    6554            0 :     ) -> Result<NodeShardResponse, ApiError> {
    6555            0 :         let locked = self.inner.read().unwrap();
    6556            0 :         let mut shards = Vec::new();
    6557            0 :         for (tid, tenant) in locked.tenants.iter() {
    6558            0 :             let is_intended_secondary = match (
    6559            0 :                 tenant.intent.get_attached() == &Some(node_id),
    6560            0 :                 tenant.intent.get_secondary().contains(&node_id),
    6561            0 :             ) {
    6562              :                 (true, true) => {
    6563            0 :                     return Err(ApiError::InternalServerError(anyhow::anyhow!(
    6564            0 :                         "{} attached as primary+secondary on the same node",
    6565            0 :                         tid
    6566            0 :                     )));
    6567              :                 }
    6568            0 :                 (true, false) => Some(false),
    6569            0 :                 (false, true) => Some(true),
    6570            0 :                 (false, false) => None,
    6571              :             };
    6572            0 :             let is_observed_secondary = if let Some(ObservedStateLocation { conf: Some(conf) }) =
    6573            0 :                 tenant.observed.locations.get(&node_id)
    6574              :             {
    6575            0 :                 Some(conf.secondary_conf.is_some())
    6576              :             } else {
    6577            0 :                 None
    6578              :             };
    6579            0 :             if is_intended_secondary.is_some() || is_observed_secondary.is_some() {
    6580            0 :                 shards.push(NodeShard {
    6581            0 :                     tenant_shard_id: *tid,
    6582            0 :                     is_intended_secondary,
    6583            0 :                     is_observed_secondary,
    6584            0 :                 });
    6585            0 :             }
    6586              :         }
    6587            0 :         Ok(NodeShardResponse { node_id, shards })
    6588            0 :     }
    6589              : 
    6590            0 :     pub(crate) async fn get_leader(&self) -> DatabaseResult<Option<ControllerPersistence>> {
    6591            0 :         self.persistence.get_leader().await
    6592            0 :     }
    6593              : 
    6594            0 :     pub(crate) async fn node_register(
    6595            0 :         &self,
    6596            0 :         register_req: NodeRegisterRequest,
    6597            0 :     ) -> Result<(), ApiError> {
    6598            0 :         let _node_lock = trace_exclusive_lock(
    6599            0 :             &self.node_op_locks,
    6600            0 :             register_req.node_id,
    6601            0 :             NodeOperations::Register,
    6602            0 :         )
    6603            0 :         .await;
    6604              : 
    6605              :         #[derive(PartialEq)]
    6606              :         enum RegistrationStatus {
    6607              :             UpToDate,
    6608              :             NeedUpdate,
    6609              :             Mismatched,
    6610              :             New,
    6611              :         }
    6612              : 
    6613            0 :         let registration_status = {
    6614            0 :             let locked = self.inner.read().unwrap();
    6615            0 :             if let Some(node) = locked.nodes.get(&register_req.node_id) {
    6616            0 :                 if node.registration_match(&register_req) {
    6617            0 :                     if node.need_update(&register_req) {
    6618            0 :                         RegistrationStatus::NeedUpdate
    6619              :                     } else {
    6620            0 :                         RegistrationStatus::UpToDate
    6621              :                     }
    6622              :                 } else {
    6623            0 :                     RegistrationStatus::Mismatched
    6624              :                 }
    6625              :             } else {
    6626            0 :                 RegistrationStatus::New
    6627              :             }
    6628              :         };
    6629              : 
    6630            0 :         match registration_status {
    6631              :             RegistrationStatus::UpToDate => {
    6632            0 :                 tracing::info!(
    6633            0 :                     "Node {} re-registered with matching address and is up to date",
    6634              :                     register_req.node_id
    6635              :                 );
    6636              : 
    6637            0 :                 return Ok(());
    6638              :             }
    6639              :             RegistrationStatus::Mismatched => {
    6640              :                 // TODO: decide if we want to allow modifying node addresses without removing and re-adding
    6641              :                 // the node.  Safest/simplest thing is to refuse it, and usually we deploy with
    6642              :                 // a fixed address through the lifetime of a node.
    6643            0 :                 tracing::warn!(
    6644            0 :                     "Node {} tried to register with different address",
    6645              :                     register_req.node_id
    6646              :                 );
    6647            0 :                 return Err(ApiError::Conflict(
    6648            0 :                     "Node is already registered with different address".to_string(),
    6649            0 :                 ));
    6650              :             }
    6651            0 :             RegistrationStatus::New | RegistrationStatus::NeedUpdate => {
    6652            0 :                 // fallthrough
    6653            0 :             }
    6654            0 :         }
    6655            0 : 
    6656            0 :         // We do not require that a node is actually online when registered (it will start life
    6657            0 :         // with it's  availability set to Offline), but we _do_ require that its DNS record exists. We're
    6658            0 :         // therefore not immune to asymmetric L3 connectivity issues, but we are protected against nodes
    6659            0 :         // that register themselves with a broken DNS config.  We check only the HTTP hostname, because
    6660            0 :         // the postgres hostname might only be resolvable to clients (e.g. if we're on a different VPC than clients).
    6661            0 :         if tokio::net::lookup_host(format!(
    6662            0 :             "{}:{}",
    6663            0 :             register_req.listen_http_addr, register_req.listen_http_port
    6664            0 :         ))
    6665            0 :         .await
    6666            0 :         .is_err()
    6667              :         {
    6668              :             // If we have a transient DNS issue, it's up to the caller to retry their registration.  Because
    6669              :             // we can't robustly distinguish between an intermittent issue and a totally bogus DNS situation,
    6670              :             // we return a soft 503 error, to encourage callers to retry past transient issues.
    6671            0 :             return Err(ApiError::ResourceUnavailable(
    6672            0 :                 format!(
    6673            0 :                     "Node {} tried to register with unknown DNS name '{}'",
    6674            0 :                     register_req.node_id, register_req.listen_http_addr
    6675            0 :                 )
    6676            0 :                 .into(),
    6677            0 :             ));
    6678            0 :         }
    6679            0 : 
    6680            0 :         if self.config.use_https_pageserver_api && register_req.listen_https_port.is_none() {
    6681            0 :             return Err(ApiError::PreconditionFailed(
    6682            0 :                 format!(
    6683            0 :                     "Node {} has no https port, but use_https is enabled",
    6684            0 :                     register_req.node_id
    6685            0 :                 )
    6686            0 :                 .into(),
    6687            0 :             ));
    6688            0 :         }
    6689            0 : 
    6690            0 :         // Ordering: we must persist the new node _before_ adding it to in-memory state.
    6691            0 :         // This ensures that before we use it for anything or expose it via any external
    6692            0 :         // API, it is guaranteed to be available after a restart.
    6693            0 :         let new_node = Node::new(
    6694            0 :             register_req.node_id,
    6695            0 :             register_req.listen_http_addr,
    6696            0 :             register_req.listen_http_port,
    6697            0 :             register_req.listen_https_port,
    6698            0 :             register_req.listen_pg_addr,
    6699            0 :             register_req.listen_pg_port,
    6700            0 :             register_req.availability_zone_id.clone(),
    6701            0 :             self.config.use_https_pageserver_api,
    6702            0 :         );
    6703            0 :         let new_node = match new_node {
    6704            0 :             Ok(new_node) => new_node,
    6705            0 :             Err(error) => return Err(ApiError::InternalServerError(error)),
    6706              :         };
    6707              : 
    6708            0 :         match registration_status {
    6709            0 :             RegistrationStatus::New => self.persistence.insert_node(&new_node).await?,
    6710              :             RegistrationStatus::NeedUpdate => {
    6711            0 :                 self.persistence
    6712            0 :                     .update_node_on_registration(
    6713            0 :                         register_req.node_id,
    6714            0 :                         register_req.listen_https_port,
    6715            0 :                     )
    6716            0 :                     .await?
    6717              :             }
    6718            0 :             _ => unreachable!("Other statuses have been processed earlier"),
    6719              :         }
    6720              : 
    6721            0 :         let mut locked = self.inner.write().unwrap();
    6722            0 :         let mut new_nodes = (*locked.nodes).clone();
    6723            0 : 
    6724            0 :         locked.scheduler.node_upsert(&new_node);
    6725            0 :         new_nodes.insert(register_req.node_id, new_node);
    6726            0 : 
    6727            0 :         locked.nodes = Arc::new(new_nodes);
    6728            0 : 
    6729            0 :         metrics::METRICS_REGISTRY
    6730            0 :             .metrics_group
    6731            0 :             .storage_controller_pageserver_nodes
    6732            0 :             .set(locked.nodes.len() as i64);
    6733            0 : 
    6734            0 :         match registration_status {
    6735              :             RegistrationStatus::New => {
    6736            0 :                 tracing::info!(
    6737            0 :                     "Registered pageserver {} ({}), now have {} pageservers",
    6738            0 :                     register_req.node_id,
    6739            0 :                     register_req.availability_zone_id,
    6740            0 :                     locked.nodes.len()
    6741              :                 );
    6742              :             }
    6743              :             RegistrationStatus::NeedUpdate => {
    6744            0 :                 tracing::info!(
    6745            0 :                     "Re-registered and updated node {} ({})",
    6746              :                     register_req.node_id,
    6747              :                     register_req.availability_zone_id,
    6748              :                 );
    6749              :             }
    6750            0 :             _ => unreachable!("Other statuses have been processed earlier"),
    6751              :         }
    6752            0 :         Ok(())
    6753            0 :     }
    6754              : 
    6755              :     /// Configure in-memory and persistent state of a node as requested
    6756              :     ///
    6757              :     /// Note that this function does not trigger any immediate side effects in response
    6758              :     /// to the changes. That part is handled by [`Self::handle_node_availability_transition`].
    6759            0 :     async fn node_state_configure(
    6760            0 :         &self,
    6761            0 :         node_id: NodeId,
    6762            0 :         availability: Option<NodeAvailability>,
    6763            0 :         scheduling: Option<NodeSchedulingPolicy>,
    6764            0 :         node_lock: &TracingExclusiveGuard<NodeOperations>,
    6765            0 :     ) -> Result<AvailabilityTransition, ApiError> {
    6766            0 :         if let Some(scheduling) = scheduling {
    6767              :             // Scheduling is a persistent part of Node: we must write updates to the database before
    6768              :             // applying them in memory
    6769            0 :             self.persistence
    6770            0 :                 .update_node_scheduling_policy(node_id, scheduling)
    6771            0 :                 .await?;
    6772            0 :         }
    6773              : 
    6774              :         // If we're activating a node, then before setting it active we must reconcile any shard locations
    6775              :         // on that node, in case it is out of sync, e.g. due to being unavailable during controller startup,
    6776              :         // by calling [`Self::node_activate_reconcile`]
    6777              :         //
    6778              :         // The transition we calculate here remains valid later in the function because we hold the op lock on the node:
    6779              :         // nothing else can mutate its availability while we run.
    6780            0 :         let availability_transition = if let Some(input_availability) = availability.as_ref() {
    6781            0 :             let (activate_node, availability_transition) = {
    6782            0 :                 let locked = self.inner.read().unwrap();
    6783            0 :                 let Some(node) = locked.nodes.get(&node_id) else {
    6784            0 :                     return Err(ApiError::NotFound(
    6785            0 :                         anyhow::anyhow!("Node {} not registered", node_id).into(),
    6786            0 :                     ));
    6787              :                 };
    6788              : 
    6789            0 :                 (
    6790            0 :                     node.clone(),
    6791            0 :                     node.get_availability_transition(input_availability),
    6792            0 :                 )
    6793              :             };
    6794              : 
    6795            0 :             if matches!(availability_transition, AvailabilityTransition::ToActive) {
    6796            0 :                 self.node_activate_reconcile(activate_node, node_lock)
    6797            0 :                     .await?;
    6798            0 :             }
    6799            0 :             availability_transition
    6800              :         } else {
    6801            0 :             AvailabilityTransition::Unchanged
    6802              :         };
    6803              : 
    6804              :         // Apply changes from the request to our in-memory state for the Node
    6805            0 :         let mut locked = self.inner.write().unwrap();
    6806            0 :         let (nodes, _tenants, scheduler) = locked.parts_mut();
    6807            0 : 
    6808            0 :         let mut new_nodes = (**nodes).clone();
    6809              : 
    6810            0 :         let Some(node) = new_nodes.get_mut(&node_id) else {
    6811            0 :             return Err(ApiError::NotFound(
    6812            0 :                 anyhow::anyhow!("Node not registered").into(),
    6813            0 :             ));
    6814              :         };
    6815              : 
    6816            0 :         if let Some(availability) = availability {
    6817            0 :             node.set_availability(availability);
    6818            0 :         }
    6819              : 
    6820            0 :         if let Some(scheduling) = scheduling {
    6821            0 :             node.set_scheduling(scheduling);
    6822            0 :         }
    6823              : 
    6824              :         // Update the scheduler, in case the elegibility of the node for new shards has changed
    6825            0 :         scheduler.node_upsert(node);
    6826            0 : 
    6827            0 :         let new_nodes = Arc::new(new_nodes);
    6828            0 :         locked.nodes = new_nodes;
    6829            0 : 
    6830            0 :         Ok(availability_transition)
    6831            0 :     }
    6832              : 
    6833              :     /// Handle availability transition of one node
    6834              :     ///
    6835              :     /// Note that you should first call [`Self::node_state_configure`] to update
    6836              :     /// the in-memory state referencing that node. If you need to handle more than one transition
    6837              :     /// consider using [`Self::handle_node_availability_transitions`].
    6838            0 :     async fn handle_node_availability_transition(
    6839            0 :         &self,
    6840            0 :         node_id: NodeId,
    6841            0 :         transition: AvailabilityTransition,
    6842            0 :         _node_lock: &TracingExclusiveGuard<NodeOperations>,
    6843            0 :     ) -> Result<(), ApiError> {
    6844            0 :         // Modify scheduling state for any Tenants that are affected by a change in the node's availability state.
    6845            0 :         match transition {
    6846              :             AvailabilityTransition::ToOffline => {
    6847            0 :                 tracing::info!("Node {} transition to offline", node_id);
    6848              : 
    6849            0 :                 let mut locked = self.inner.write().unwrap();
    6850            0 :                 let (nodes, tenants, scheduler) = locked.parts_mut();
    6851            0 : 
    6852            0 :                 let mut tenants_affected: usize = 0;
    6853              : 
    6854            0 :                 for (_tenant_id, mut schedule_context, shards) in
    6855            0 :                     TenantShardContextIterator::new(tenants, ScheduleMode::Normal)
    6856              :                 {
    6857            0 :                     for tenant_shard in shards {
    6858            0 :                         let tenant_shard_id = tenant_shard.tenant_shard_id;
    6859            0 :                         if let Some(observed_loc) =
    6860            0 :                             tenant_shard.observed.locations.get_mut(&node_id)
    6861            0 :                         {
    6862            0 :                             // When a node goes offline, we set its observed configuration to None, indicating unknown: we will
    6863            0 :                             // not assume our knowledge of the node's configuration is accurate until it comes back online
    6864            0 :                             observed_loc.conf = None;
    6865            0 :                         }
    6866              : 
    6867            0 :                         if nodes.len() == 1 {
    6868              :                             // Special case for single-node cluster: there is no point trying to reschedule
    6869              :                             // any tenant shards: avoid doing so, in order to avoid spewing warnings about
    6870              :                             // failures to schedule them.
    6871            0 :                             continue;
    6872            0 :                         }
    6873            0 : 
    6874            0 :                         if !nodes
    6875            0 :                             .values()
    6876            0 :                             .any(|n| matches!(n.may_schedule(), MaySchedule::Yes(_)))
    6877              :                         {
    6878              :                             // Special case for when all nodes are unavailable and/or unschedulable: there is no point
    6879              :                             // trying to reschedule since there's nowhere else to go. Without this
    6880              :                             // branch we incorrectly detach tenants in response to node unavailability.
    6881            0 :                             continue;
    6882            0 :                         }
    6883            0 : 
    6884            0 :                         if tenant_shard.intent.demote_attached(scheduler, node_id) {
    6885            0 :                             tenant_shard.sequence = tenant_shard.sequence.next();
    6886            0 : 
    6887            0 :                             match tenant_shard.schedule(scheduler, &mut schedule_context) {
    6888            0 :                                 Err(e) => {
    6889            0 :                                     // It is possible that some tenants will become unschedulable when too many pageservers
    6890            0 :                                     // go offline: in this case there isn't much we can do other than make the issue observable.
    6891            0 :                                     // TODO: give TenantShard a scheduling error attribute to be queried later.
    6892            0 :                                     tracing::warn!(%tenant_shard_id, "Scheduling error when marking pageserver {} offline: {e}", node_id);
    6893              :                                 }
    6894              :                                 Ok(()) => {
    6895            0 :                                     if self
    6896            0 :                                         .maybe_reconcile_shard(
    6897            0 :                                             tenant_shard,
    6898            0 :                                             nodes,
    6899            0 :                                             ReconcilerPriority::Normal,
    6900            0 :                                         )
    6901            0 :                                         .is_some()
    6902            0 :                                     {
    6903            0 :                                         tenants_affected += 1;
    6904            0 :                                     };
    6905              :                                 }
    6906              :                             }
    6907            0 :                         }
    6908              :                     }
    6909              :                 }
    6910            0 :                 tracing::info!(
    6911            0 :                     "Launched {} reconciler tasks for tenants affected by node {} going offline",
    6912              :                     tenants_affected,
    6913              :                     node_id
    6914              :                 )
    6915              :             }
    6916              :             AvailabilityTransition::ToActive => {
    6917            0 :                 tracing::info!("Node {} transition to active", node_id);
    6918              : 
    6919            0 :                 let mut locked = self.inner.write().unwrap();
    6920            0 :                 let (nodes, tenants, _scheduler) = locked.parts_mut();
    6921              : 
    6922              :                 // When a node comes back online, we must reconcile any tenant that has a None observed
    6923              :                 // location on the node.
    6924            0 :                 for tenant_shard in tenants.values_mut() {
    6925              :                     // If a reconciliation is already in progress, rely on the previous scheduling
    6926              :                     // decision and skip triggering a new reconciliation.
    6927            0 :                     if tenant_shard.reconciler.is_some() {
    6928            0 :                         continue;
    6929            0 :                     }
    6930              : 
    6931            0 :                     if let Some(observed_loc) = tenant_shard.observed.locations.get_mut(&node_id) {
    6932            0 :                         if observed_loc.conf.is_none() {
    6933            0 :                             self.maybe_reconcile_shard(
    6934            0 :                                 tenant_shard,
    6935            0 :                                 nodes,
    6936            0 :                                 ReconcilerPriority::Normal,
    6937            0 :                             );
    6938            0 :                         }
    6939            0 :                     }
    6940              :                 }
    6941              : 
    6942              :                 // TODO: in the background, we should balance work back onto this pageserver
    6943              :             }
    6944              :             // No action required for the intermediate unavailable state.
    6945              :             // When we transition into active or offline from the unavailable state,
    6946              :             // the correct handling above will kick in.
    6947              :             AvailabilityTransition::ToWarmingUpFromActive => {
    6948            0 :                 tracing::info!("Node {} transition to unavailable from active", node_id);
    6949              :             }
    6950              :             AvailabilityTransition::ToWarmingUpFromOffline => {
    6951            0 :                 tracing::info!("Node {} transition to unavailable from offline", node_id);
    6952              :             }
    6953              :             AvailabilityTransition::Unchanged => {
    6954            0 :                 tracing::debug!("Node {} no availability change during config", node_id);
    6955              :             }
    6956              :         }
    6957              : 
    6958            0 :         Ok(())
    6959            0 :     }
    6960              : 
    6961              :     /// Handle availability transition for multiple nodes
    6962              :     ///
    6963              :     /// Note that you should first call [`Self::node_state_configure`] for
    6964              :     /// all nodes being handled here for the handling to use fresh in-memory state.
    6965            0 :     async fn handle_node_availability_transitions(
    6966            0 :         &self,
    6967            0 :         transitions: Vec<(
    6968            0 :             NodeId,
    6969            0 :             TracingExclusiveGuard<NodeOperations>,
    6970            0 :             AvailabilityTransition,
    6971            0 :         )>,
    6972            0 :     ) -> Result<(), Vec<(NodeId, ApiError)>> {
    6973            0 :         let mut errors = Vec::default();
    6974            0 :         for (node_id, node_lock, transition) in transitions {
    6975            0 :             let res = self
    6976            0 :                 .handle_node_availability_transition(node_id, transition, &node_lock)
    6977            0 :                 .await;
    6978            0 :             if let Err(err) = res {
    6979            0 :                 errors.push((node_id, err));
    6980            0 :             }
    6981              :         }
    6982              : 
    6983            0 :         if errors.is_empty() {
    6984            0 :             Ok(())
    6985              :         } else {
    6986            0 :             Err(errors)
    6987              :         }
    6988            0 :     }
    6989              : 
    6990            0 :     pub(crate) async fn node_configure(
    6991            0 :         &self,
    6992            0 :         node_id: NodeId,
    6993            0 :         availability: Option<NodeAvailability>,
    6994            0 :         scheduling: Option<NodeSchedulingPolicy>,
    6995            0 :     ) -> Result<(), ApiError> {
    6996            0 :         let node_lock =
    6997            0 :             trace_exclusive_lock(&self.node_op_locks, node_id, NodeOperations::Configure).await;
    6998              : 
    6999            0 :         let transition = self
    7000            0 :             .node_state_configure(node_id, availability, scheduling, &node_lock)
    7001            0 :             .await?;
    7002            0 :         self.handle_node_availability_transition(node_id, transition, &node_lock)
    7003            0 :             .await
    7004            0 :     }
    7005              : 
    7006              :     /// Wrapper around [`Self::node_configure`] which only allows changes while there is no ongoing
    7007              :     /// operation for HTTP api.
    7008            0 :     pub(crate) async fn external_node_configure(
    7009            0 :         &self,
    7010            0 :         node_id: NodeId,
    7011            0 :         availability: Option<NodeAvailability>,
    7012            0 :         scheduling: Option<NodeSchedulingPolicy>,
    7013            0 :     ) -> Result<(), ApiError> {
    7014            0 :         {
    7015            0 :             let locked = self.inner.read().unwrap();
    7016            0 :             if let Some(op) = locked.ongoing_operation.as_ref().map(|op| op.operation) {
    7017            0 :                 return Err(ApiError::PreconditionFailed(
    7018            0 :                     format!("Ongoing background operation forbids configuring: {op}").into(),
    7019            0 :                 ));
    7020            0 :             }
    7021            0 :         }
    7022            0 : 
    7023            0 :         self.node_configure(node_id, availability, scheduling).await
    7024            0 :     }
    7025              : 
    7026            0 :     pub(crate) async fn start_node_drain(
    7027            0 :         self: &Arc<Self>,
    7028            0 :         node_id: NodeId,
    7029            0 :     ) -> Result<(), ApiError> {
    7030            0 :         let (ongoing_op, node_available, node_policy, schedulable_nodes_count) = {
    7031            0 :             let locked = self.inner.read().unwrap();
    7032            0 :             let nodes = &locked.nodes;
    7033            0 :             let node = nodes.get(&node_id).ok_or(ApiError::NotFound(
    7034            0 :                 anyhow::anyhow!("Node {} not registered", node_id).into(),
    7035            0 :             ))?;
    7036            0 :             let schedulable_nodes_count = nodes
    7037            0 :                 .iter()
    7038            0 :                 .filter(|(_, n)| matches!(n.may_schedule(), MaySchedule::Yes(_)))
    7039            0 :                 .count();
    7040            0 : 
    7041            0 :             (
    7042            0 :                 locked
    7043            0 :                     .ongoing_operation
    7044            0 :                     .as_ref()
    7045            0 :                     .map(|ongoing| ongoing.operation),
    7046            0 :                 node.is_available(),
    7047            0 :                 node.get_scheduling(),
    7048            0 :                 schedulable_nodes_count,
    7049            0 :             )
    7050            0 :         };
    7051              : 
    7052            0 :         if let Some(ongoing) = ongoing_op {
    7053            0 :             return Err(ApiError::PreconditionFailed(
    7054            0 :                 format!("Background operation already ongoing for node: {}", ongoing).into(),
    7055            0 :             ));
    7056            0 :         }
    7057            0 : 
    7058            0 :         if !node_available {
    7059            0 :             return Err(ApiError::ResourceUnavailable(
    7060            0 :                 format!("Node {node_id} is currently unavailable").into(),
    7061            0 :             ));
    7062            0 :         }
    7063            0 : 
    7064            0 :         if schedulable_nodes_count == 0 {
    7065            0 :             return Err(ApiError::PreconditionFailed(
    7066            0 :                 "No other schedulable nodes to drain to".into(),
    7067            0 :             ));
    7068            0 :         }
    7069            0 : 
    7070            0 :         match node_policy {
    7071              :             NodeSchedulingPolicy::Active => {
    7072            0 :                 self.node_configure(node_id, None, Some(NodeSchedulingPolicy::Draining))
    7073            0 :                     .await?;
    7074              : 
    7075            0 :                 let cancel = self.cancel.child_token();
    7076            0 :                 let gate_guard = self.gate.enter().map_err(|_| ApiError::ShuttingDown)?;
    7077              : 
    7078            0 :                 self.inner.write().unwrap().ongoing_operation = Some(OperationHandler {
    7079            0 :                     operation: Operation::Drain(Drain { node_id }),
    7080            0 :                     cancel: cancel.clone(),
    7081            0 :                 });
    7082              : 
    7083            0 :                 let span = tracing::info_span!(parent: None, "drain_node", %node_id);
    7084              : 
    7085            0 :                 tokio::task::spawn({
    7086            0 :                     let service = self.clone();
    7087            0 :                     let cancel = cancel.clone();
    7088            0 :                     async move {
    7089            0 :                         let _gate_guard = gate_guard;
    7090            0 : 
    7091            0 :                         scopeguard::defer! {
    7092            0 :                             let prev = service.inner.write().unwrap().ongoing_operation.take();
    7093            0 : 
    7094            0 :                             if let Some(Operation::Drain(removed_drain)) = prev.map(|h| h.operation) {
    7095            0 :                                 assert_eq!(removed_drain.node_id, node_id, "We always take the same operation");
    7096            0 :                             } else {
    7097            0 :                                 panic!("We always remove the same operation")
    7098            0 :                             }
    7099            0 :                         }
    7100            0 : 
    7101            0 :                         tracing::info!("Drain background operation starting");
    7102            0 :                         let res = service.drain_node(node_id, cancel).await;
    7103            0 :                         match res {
    7104              :                             Ok(()) => {
    7105            0 :                                 tracing::info!("Drain background operation completed successfully");
    7106              :                             }
    7107              :                             Err(OperationError::Cancelled) => {
    7108            0 :                                 tracing::info!("Drain background operation was cancelled");
    7109              :                             }
    7110            0 :                             Err(err) => {
    7111            0 :                                 tracing::error!("Drain background operation encountered: {err}")
    7112              :                             }
    7113              :                         }
    7114            0 :                     }
    7115            0 :                 }.instrument(span));
    7116            0 :             }
    7117              :             NodeSchedulingPolicy::Draining => {
    7118            0 :                 return Err(ApiError::Conflict(format!(
    7119            0 :                     "Node {node_id} has drain in progress"
    7120            0 :                 )));
    7121              :             }
    7122            0 :             policy => {
    7123            0 :                 return Err(ApiError::PreconditionFailed(
    7124            0 :                     format!("Node {node_id} cannot be drained due to {policy:?} policy").into(),
    7125            0 :                 ));
    7126              :             }
    7127              :         }
    7128              : 
    7129            0 :         Ok(())
    7130            0 :     }
    7131              : 
    7132            0 :     pub(crate) async fn cancel_node_drain(&self, node_id: NodeId) -> Result<(), ApiError> {
    7133            0 :         let node_available = {
    7134            0 :             let locked = self.inner.read().unwrap();
    7135            0 :             let nodes = &locked.nodes;
    7136            0 :             let node = nodes.get(&node_id).ok_or(ApiError::NotFound(
    7137            0 :                 anyhow::anyhow!("Node {} not registered", node_id).into(),
    7138            0 :             ))?;
    7139              : 
    7140            0 :             node.is_available()
    7141            0 :         };
    7142            0 : 
    7143            0 :         if !node_available {
    7144            0 :             return Err(ApiError::ResourceUnavailable(
    7145            0 :                 format!("Node {node_id} is currently unavailable").into(),
    7146            0 :             ));
    7147            0 :         }
    7148              : 
    7149            0 :         if let Some(op_handler) = self.inner.read().unwrap().ongoing_operation.as_ref() {
    7150            0 :             if let Operation::Drain(drain) = op_handler.operation {
    7151            0 :                 if drain.node_id == node_id {
    7152            0 :                     tracing::info!("Cancelling background drain operation for node {node_id}");
    7153            0 :                     op_handler.cancel.cancel();
    7154            0 :                     return Ok(());
    7155            0 :                 }
    7156            0 :             }
    7157            0 :         }
    7158              : 
    7159            0 :         Err(ApiError::PreconditionFailed(
    7160            0 :             format!("Node {node_id} has no drain in progress").into(),
    7161            0 :         ))
    7162            0 :     }
    7163              : 
    7164            0 :     pub(crate) async fn start_node_fill(self: &Arc<Self>, node_id: NodeId) -> Result<(), ApiError> {
    7165            0 :         let (ongoing_op, node_available, node_policy, total_nodes_count) = {
    7166            0 :             let locked = self.inner.read().unwrap();
    7167            0 :             let nodes = &locked.nodes;
    7168            0 :             let node = nodes.get(&node_id).ok_or(ApiError::NotFound(
    7169            0 :                 anyhow::anyhow!("Node {} not registered", node_id).into(),
    7170            0 :             ))?;
    7171              : 
    7172            0 :             (
    7173            0 :                 locked
    7174            0 :                     .ongoing_operation
    7175            0 :                     .as_ref()
    7176            0 :                     .map(|ongoing| ongoing.operation),
    7177            0 :                 node.is_available(),
    7178            0 :                 node.get_scheduling(),
    7179            0 :                 nodes.len(),
    7180            0 :             )
    7181            0 :         };
    7182              : 
    7183            0 :         if let Some(ongoing) = ongoing_op {
    7184            0 :             return Err(ApiError::PreconditionFailed(
    7185            0 :                 format!("Background operation already ongoing for node: {}", ongoing).into(),
    7186            0 :             ));
    7187            0 :         }
    7188            0 : 
    7189            0 :         if !node_available {
    7190            0 :             return Err(ApiError::ResourceUnavailable(
    7191            0 :                 format!("Node {node_id} is currently unavailable").into(),
    7192            0 :             ));
    7193            0 :         }
    7194            0 : 
    7195            0 :         if total_nodes_count <= 1 {
    7196            0 :             return Err(ApiError::PreconditionFailed(
    7197            0 :                 "No other nodes to fill from".into(),
    7198            0 :             ));
    7199            0 :         }
    7200            0 : 
    7201            0 :         match node_policy {
    7202              :             NodeSchedulingPolicy::Active => {
    7203            0 :                 self.node_configure(node_id, None, Some(NodeSchedulingPolicy::Filling))
    7204            0 :                     .await?;
    7205              : 
    7206            0 :                 let cancel = self.cancel.child_token();
    7207            0 :                 let gate_guard = self.gate.enter().map_err(|_| ApiError::ShuttingDown)?;
    7208              : 
    7209            0 :                 self.inner.write().unwrap().ongoing_operation = Some(OperationHandler {
    7210            0 :                     operation: Operation::Fill(Fill { node_id }),
    7211            0 :                     cancel: cancel.clone(),
    7212            0 :                 });
    7213              : 
    7214            0 :                 let span = tracing::info_span!(parent: None, "fill_node", %node_id);
    7215              : 
    7216            0 :                 tokio::task::spawn({
    7217            0 :                     let service = self.clone();
    7218            0 :                     let cancel = cancel.clone();
    7219            0 :                     async move {
    7220            0 :                         let _gate_guard = gate_guard;
    7221            0 : 
    7222            0 :                         scopeguard::defer! {
    7223            0 :                             let prev = service.inner.write().unwrap().ongoing_operation.take();
    7224            0 : 
    7225            0 :                             if let Some(Operation::Fill(removed_fill)) = prev.map(|h| h.operation) {
    7226            0 :                                 assert_eq!(removed_fill.node_id, node_id, "We always take the same operation");
    7227            0 :                             } else {
    7228            0 :                                 panic!("We always remove the same operation")
    7229            0 :                             }
    7230            0 :                         }
    7231            0 : 
    7232            0 :                         tracing::info!("Fill background operation starting");
    7233            0 :                         let res = service.fill_node(node_id, cancel).await;
    7234            0 :                         match res {
    7235              :                             Ok(()) => {
    7236            0 :                                 tracing::info!("Fill background operation completed successfully");
    7237              :                             }
    7238              :                             Err(OperationError::Cancelled) => {
    7239            0 :                                 tracing::info!("Fill background operation was cancelled");
    7240              :                             }
    7241            0 :                             Err(err) => {
    7242            0 :                                 tracing::error!("Fill background operation encountered: {err}")
    7243              :                             }
    7244              :                         }
    7245            0 :                     }
    7246            0 :                 }.instrument(span));
    7247            0 :             }
    7248              :             NodeSchedulingPolicy::Filling => {
    7249            0 :                 return Err(ApiError::Conflict(format!(
    7250            0 :                     "Node {node_id} has fill in progress"
    7251            0 :                 )));
    7252              :             }
    7253            0 :             policy => {
    7254            0 :                 return Err(ApiError::PreconditionFailed(
    7255            0 :                     format!("Node {node_id} cannot be filled due to {policy:?} policy").into(),
    7256            0 :                 ));
    7257              :             }
    7258              :         }
    7259              : 
    7260            0 :         Ok(())
    7261            0 :     }
    7262              : 
    7263            0 :     pub(crate) async fn cancel_node_fill(&self, node_id: NodeId) -> Result<(), ApiError> {
    7264            0 :         let node_available = {
    7265            0 :             let locked = self.inner.read().unwrap();
    7266            0 :             let nodes = &locked.nodes;
    7267            0 :             let node = nodes.get(&node_id).ok_or(ApiError::NotFound(
    7268            0 :                 anyhow::anyhow!("Node {} not registered", node_id).into(),
    7269            0 :             ))?;
    7270              : 
    7271            0 :             node.is_available()
    7272            0 :         };
    7273            0 : 
    7274            0 :         if !node_available {
    7275            0 :             return Err(ApiError::ResourceUnavailable(
    7276            0 :                 format!("Node {node_id} is currently unavailable").into(),
    7277            0 :             ));
    7278            0 :         }
    7279              : 
    7280            0 :         if let Some(op_handler) = self.inner.read().unwrap().ongoing_operation.as_ref() {
    7281            0 :             if let Operation::Fill(fill) = op_handler.operation {
    7282            0 :                 if fill.node_id == node_id {
    7283            0 :                     tracing::info!("Cancelling background drain operation for node {node_id}");
    7284            0 :                     op_handler.cancel.cancel();
    7285            0 :                     return Ok(());
    7286            0 :                 }
    7287            0 :             }
    7288            0 :         }
    7289              : 
    7290            0 :         Err(ApiError::PreconditionFailed(
    7291            0 :             format!("Node {node_id} has no fill in progress").into(),
    7292            0 :         ))
    7293            0 :     }
    7294              : 
    7295              :     /// Like [`Self::maybe_configured_reconcile_shard`], but uses the default reconciler
    7296              :     /// configuration
    7297            0 :     fn maybe_reconcile_shard(
    7298            0 :         &self,
    7299            0 :         shard: &mut TenantShard,
    7300            0 :         nodes: &Arc<HashMap<NodeId, Node>>,
    7301            0 :         priority: ReconcilerPriority,
    7302            0 :     ) -> Option<ReconcilerWaiter> {
    7303            0 :         self.maybe_configured_reconcile_shard(shard, nodes, ReconcilerConfig::new(priority))
    7304            0 :     }
    7305              : 
    7306              :     /// Before constructing a Reconciler, acquire semaphore units from the appropriate concurrency limit (depends on priority)
    7307            0 :     fn get_reconciler_units(
    7308            0 :         &self,
    7309            0 :         priority: ReconcilerPriority,
    7310            0 :     ) -> Result<ReconcileUnits, TryAcquireError> {
    7311            0 :         let units = match priority {
    7312            0 :             ReconcilerPriority::Normal => self.reconciler_concurrency.clone().try_acquire_owned(),
    7313              :             ReconcilerPriority::High => {
    7314            0 :                 match self
    7315            0 :                     .priority_reconciler_concurrency
    7316            0 :                     .clone()
    7317            0 :                     .try_acquire_owned()
    7318              :                 {
    7319            0 :                     Ok(u) => Ok(u),
    7320              :                     Err(TryAcquireError::NoPermits) => {
    7321              :                         // If the high priority semaphore is exhausted, then high priority tasks may steal units from
    7322              :                         // the normal priority semaphore.
    7323            0 :                         self.reconciler_concurrency.clone().try_acquire_owned()
    7324              :                     }
    7325            0 :                     Err(e) => Err(e),
    7326              :                 }
    7327              :             }
    7328              :         };
    7329              : 
    7330            0 :         units.map(ReconcileUnits::new)
    7331            0 :     }
    7332              : 
    7333              :     /// Wrap [`TenantShard`] reconciliation methods with acquisition of [`Gate`] and [`ReconcileUnits`],
    7334            0 :     fn maybe_configured_reconcile_shard(
    7335            0 :         &self,
    7336            0 :         shard: &mut TenantShard,
    7337            0 :         nodes: &Arc<HashMap<NodeId, Node>>,
    7338            0 :         reconciler_config: ReconcilerConfig,
    7339            0 :     ) -> Option<ReconcilerWaiter> {
    7340            0 :         let reconcile_needed = shard.get_reconcile_needed(nodes);
    7341              : 
    7342            0 :         let reconcile_reason = match reconcile_needed {
    7343            0 :             ReconcileNeeded::No => return None,
    7344            0 :             ReconcileNeeded::WaitExisting(waiter) => return Some(waiter),
    7345            0 :             ReconcileNeeded::Yes(reason) => {
    7346            0 :                 // Fall through to try and acquire units for spawning reconciler
    7347            0 :                 reason
    7348              :             }
    7349              :         };
    7350              : 
    7351            0 :         let units = match self.get_reconciler_units(reconciler_config.priority) {
    7352            0 :             Ok(u) => u,
    7353              :             Err(_) => {
    7354            0 :                 tracing::info!(tenant_id=%shard.tenant_shard_id.tenant_id, shard_id=%shard.tenant_shard_id.shard_slug(),
    7355            0 :                     "Concurrency limited: enqueued for reconcile later");
    7356            0 :                 if !shard.delayed_reconcile {
    7357            0 :                     match self.delayed_reconcile_tx.try_send(shard.tenant_shard_id) {
    7358            0 :                         Err(TrySendError::Closed(_)) => {
    7359            0 :                             // Weird mid-shutdown case?
    7360            0 :                         }
    7361              :                         Err(TrySendError::Full(_)) => {
    7362              :                             // It is safe to skip sending our ID in the channel: we will eventually get retried by the background reconcile task.
    7363            0 :                             tracing::warn!(
    7364            0 :                                 "Many shards are waiting to reconcile: delayed_reconcile queue is full"
    7365              :                             );
    7366              :                         }
    7367            0 :                         Ok(()) => {
    7368            0 :                             shard.delayed_reconcile = true;
    7369            0 :                         }
    7370              :                     }
    7371            0 :                 }
    7372              : 
    7373              :                 // We won't spawn a reconciler, but we will construct a waiter that waits for the shard's sequence
    7374              :                 // number to advance.  When this function is eventually called again and succeeds in getting units,
    7375              :                 // it will spawn a reconciler that makes this waiter complete.
    7376            0 :                 return Some(shard.future_reconcile_waiter());
    7377              :             }
    7378              :         };
    7379              : 
    7380            0 :         let Ok(gate_guard) = self.reconcilers_gate.enter() else {
    7381              :             // Gate closed: we're shutting down, drop out.
    7382            0 :             return None;
    7383              :         };
    7384              : 
    7385            0 :         shard.spawn_reconciler(
    7386            0 :             reconcile_reason,
    7387            0 :             &self.result_tx,
    7388            0 :             nodes,
    7389            0 :             &self.compute_hook,
    7390            0 :             reconciler_config,
    7391            0 :             &self.config,
    7392            0 :             &self.persistence,
    7393            0 :             units,
    7394            0 :             gate_guard,
    7395            0 :             &self.reconcilers_cancel,
    7396            0 :         )
    7397            0 :     }
    7398              : 
    7399              :     /// Check all tenants for pending reconciliation work, and reconcile those in need.
    7400              :     /// Additionally, reschedule tenants that require it.
    7401              :     ///
    7402              :     /// Returns how many reconciliation tasks were started, or `1` if no reconciles were
    7403              :     /// spawned but some _would_ have been spawned if `reconciler_concurrency` units where
    7404              :     /// available.  A return value of 0 indicates that everything is fully reconciled already.
    7405            0 :     fn reconcile_all(&self) -> usize {
    7406            0 :         let mut locked = self.inner.write().unwrap();
    7407            0 :         let (nodes, tenants, scheduler) = locked.parts_mut();
    7408            0 :         let pageservers = nodes.clone();
    7409            0 : 
    7410            0 :         // This function is an efficient place to update lazy statistics, since we are walking
    7411            0 :         // all tenants.
    7412            0 :         let mut pending_reconciles = 0;
    7413            0 :         let mut az_violations = 0;
    7414            0 : 
    7415            0 :         // If we find any tenants to drop from memory, stash them to offload after
    7416            0 :         // we're done traversing the map of tenants.
    7417            0 :         let mut drop_detached_tenants = Vec::new();
    7418            0 : 
    7419            0 :         let mut reconciles_spawned = 0;
    7420            0 :         for shard in tenants.values_mut() {
    7421              :             // Accumulate scheduling statistics
    7422            0 :             if let (Some(attached), Some(preferred)) =
    7423            0 :                 (shard.intent.get_attached(), shard.preferred_az())
    7424              :             {
    7425            0 :                 let node_az = nodes
    7426            0 :                     .get(attached)
    7427            0 :                     .expect("Nodes exist if referenced")
    7428            0 :                     .get_availability_zone_id();
    7429            0 :                 if node_az != preferred {
    7430            0 :                     az_violations += 1;
    7431            0 :                 }
    7432            0 :             }
    7433              : 
    7434              :             // Skip checking if this shard is already enqueued for reconciliation
    7435            0 :             if shard.delayed_reconcile && self.reconciler_concurrency.available_permits() == 0 {
    7436              :                 // If there is something delayed, then return a nonzero count so that
    7437              :                 // callers like reconcile_all_now do not incorrectly get the impression
    7438              :                 // that the system is in a quiescent state.
    7439            0 :                 reconciles_spawned = std::cmp::max(1, reconciles_spawned);
    7440            0 :                 pending_reconciles += 1;
    7441            0 :                 continue;
    7442            0 :             }
    7443            0 : 
    7444            0 :             // Eventual consistency: if an earlier reconcile job failed, and the shard is still
    7445            0 :             // dirty, spawn another rone
    7446            0 :             if self
    7447            0 :                 .maybe_reconcile_shard(shard, &pageservers, ReconcilerPriority::Normal)
    7448            0 :                 .is_some()
    7449            0 :             {
    7450            0 :                 reconciles_spawned += 1;
    7451            0 :             } else if shard.delayed_reconcile {
    7452            0 :                 // Shard wanted to reconcile but for some reason couldn't.
    7453            0 :                 pending_reconciles += 1;
    7454            0 :             }
    7455              : 
    7456              :             // If this tenant is detached, try dropping it from memory. This is usually done
    7457              :             // proactively in [`Self::process_results`], but we do it here to handle the edge
    7458              :             // case where a reconcile completes while someone else is holding an op lock for the tenant.
    7459            0 :             if shard.tenant_shard_id.shard_number == ShardNumber(0)
    7460            0 :                 && shard.policy == PlacementPolicy::Detached
    7461              :             {
    7462            0 :                 if let Some(guard) = self.tenant_op_locks.try_exclusive(
    7463            0 :                     shard.tenant_shard_id.tenant_id,
    7464            0 :                     TenantOperations::DropDetached,
    7465            0 :                 ) {
    7466            0 :                     drop_detached_tenants.push((shard.tenant_shard_id.tenant_id, guard));
    7467            0 :                 }
    7468            0 :             }
    7469              :         }
    7470              : 
    7471              :         // Some metrics are calculated from SchedulerNode state, update these periodically
    7472            0 :         scheduler.update_metrics();
    7473              : 
    7474              :         // Process any deferred tenant drops
    7475            0 :         for (tenant_id, guard) in drop_detached_tenants {
    7476            0 :             self.maybe_drop_tenant(tenant_id, &mut locked, &guard);
    7477            0 :         }
    7478              : 
    7479            0 :         metrics::METRICS_REGISTRY
    7480            0 :             .metrics_group
    7481            0 :             .storage_controller_schedule_az_violation
    7482            0 :             .set(az_violations as i64);
    7483            0 : 
    7484            0 :         metrics::METRICS_REGISTRY
    7485            0 :             .metrics_group
    7486            0 :             .storage_controller_pending_reconciles
    7487            0 :             .set(pending_reconciles as i64);
    7488            0 : 
    7489            0 :         reconciles_spawned
    7490            0 :     }
    7491              : 
    7492              :     /// `optimize` in this context means identifying shards which have valid scheduled locations, but
    7493              :     /// could be scheduled somewhere better:
    7494              :     /// - Cutting over to a secondary if the node with the secondary is more lightly loaded
    7495              :     ///    * e.g. after a node fails then recovers, to move some work back to it
    7496              :     /// - Cutting over to a secondary if it improves the spread of shard attachments within a tenant
    7497              :     ///    * e.g. after a shard split, the initial attached locations will all be on the node where
    7498              :     ///      we did the split, but are probably better placed elsewhere.
    7499              :     /// - Creating new secondary locations if it improves the spreading of a sharded tenant
    7500              :     ///    * e.g. after a shard split, some locations will be on the same node (where the split
    7501              :     ///      happened), and will probably be better placed elsewhere.
    7502              :     ///
    7503              :     /// To put it more briefly: whereas the scheduler respects soft constraints in a ScheduleContext at
    7504              :     /// the time of scheduling, this function looks for cases where a better-scoring location is available
    7505              :     /// according to those same soft constraints.
    7506            0 :     async fn optimize_all(&self) -> usize {
    7507              :         // Limit on how many shards' optmizations each call to this function will execute.  Combined
    7508              :         // with the frequency of background calls, this acts as an implicit rate limit that runs a small
    7509              :         // trickle of optimizations in the background, rather than executing a large number in parallel
    7510              :         // when a change occurs.
    7511              :         const MAX_OPTIMIZATIONS_EXEC_PER_PASS: usize = 16;
    7512              : 
    7513              :         // Synchronous prepare: scan shards for possible scheduling optimizations
    7514            0 :         let candidate_work = self.optimize_all_plan();
    7515            0 :         let candidate_work_len = candidate_work.len();
    7516              : 
    7517              :         // Asynchronous validate: I/O to pageservers to make sure shards are in a good state to apply validation
    7518            0 :         let validated_work = self.optimize_all_validate(candidate_work).await;
    7519              : 
    7520            0 :         let was_work_filtered = validated_work.len() != candidate_work_len;
    7521            0 : 
    7522            0 :         // Synchronous apply: update the shards' intent states according to validated optimisations
    7523            0 :         let mut reconciles_spawned = 0;
    7524            0 :         let mut optimizations_applied = 0;
    7525            0 :         let mut locked = self.inner.write().unwrap();
    7526            0 :         let (nodes, tenants, scheduler) = locked.parts_mut();
    7527            0 :         for (tenant_shard_id, optimization) in validated_work {
    7528            0 :             let Some(shard) = tenants.get_mut(&tenant_shard_id) else {
    7529              :                 // Shard was dropped between planning and execution;
    7530            0 :                 continue;
    7531              :             };
    7532            0 :             tracing::info!(tenant_shard_id=%tenant_shard_id, "Applying optimization: {optimization:?}");
    7533            0 :             if shard.apply_optimization(scheduler, optimization) {
    7534            0 :                 optimizations_applied += 1;
    7535            0 :                 if self
    7536            0 :                     .maybe_reconcile_shard(shard, nodes, ReconcilerPriority::Normal)
    7537            0 :                     .is_some()
    7538            0 :                 {
    7539            0 :                     reconciles_spawned += 1;
    7540            0 :                 }
    7541            0 :             }
    7542              : 
    7543            0 :             if optimizations_applied >= MAX_OPTIMIZATIONS_EXEC_PER_PASS {
    7544            0 :                 break;
    7545            0 :             }
    7546              :         }
    7547              : 
    7548            0 :         if was_work_filtered {
    7549            0 :             // If we filtered any work out during validation, ensure we return a nonzero value to indicate
    7550            0 :             // to callers that the system is not in a truly quiet state, it's going to do some work as soon
    7551            0 :             // as these validations start passing.
    7552            0 :             reconciles_spawned = std::cmp::max(reconciles_spawned, 1);
    7553            0 :         }
    7554              : 
    7555            0 :         reconciles_spawned
    7556            0 :     }
    7557              : 
    7558            0 :     fn optimize_all_plan(&self) -> Vec<(TenantShardId, ScheduleOptimization)> {
    7559              :         // How many candidate optimizations we will generate, before evaluating them for readniess: setting
    7560              :         // this higher than the execution limit gives us a chance to execute some work even if the first
    7561              :         // few optimizations we find are not ready.
    7562              :         const MAX_OPTIMIZATIONS_PLAN_PER_PASS: usize = 64;
    7563              : 
    7564            0 :         let mut work = Vec::new();
    7565            0 :         let mut locked = self.inner.write().unwrap();
    7566            0 :         let (_nodes, tenants, scheduler) = locked.parts_mut();
    7567              : 
    7568              :         // We are going to plan a bunch of optimisations before applying any of them, so the
    7569              :         // utilisation stats on nodes will be effectively stale for the >1st optimisation we
    7570              :         // generate.  To avoid this causing unstable migrations/flapping, it's important that the
    7571              :         // code in TenantShard for finding optimisations uses [`NodeAttachmentSchedulingScore::disregard_utilization`]
    7572              :         // to ignore the utilisation component of the score.
    7573              : 
    7574            0 :         for (_tenant_id, schedule_context, shards) in
    7575            0 :             TenantShardContextIterator::new(tenants, ScheduleMode::Speculative)
    7576              :         {
    7577            0 :             for shard in shards {
    7578            0 :                 if work.len() >= MAX_OPTIMIZATIONS_PLAN_PER_PASS {
    7579            0 :                     break;
    7580            0 :                 }
    7581            0 :                 match shard.get_scheduling_policy() {
    7582            0 :                     ShardSchedulingPolicy::Active => {
    7583            0 :                         // Ok to do optimization
    7584            0 :                     }
    7585            0 :                     ShardSchedulingPolicy::Essential if shard.get_preferred_node().is_some() => {
    7586            0 :                         // Ok to do optimization: we are executing a graceful migration that
    7587            0 :                         // has set preferred_node
    7588            0 :                     }
    7589              :                     ShardSchedulingPolicy::Essential
    7590              :                     | ShardSchedulingPolicy::Pause
    7591              :                     | ShardSchedulingPolicy::Stop => {
    7592              :                         // Policy prevents optimizing this shard.
    7593            0 :                         continue;
    7594              :                     }
    7595              :                 }
    7596              : 
    7597            0 :                 if !matches!(shard.splitting, SplitState::Idle)
    7598            0 :                     || matches!(shard.policy, PlacementPolicy::Detached)
    7599            0 :                     || shard.reconciler.is_some()
    7600              :                 {
    7601              :                     // Do not start any optimizations while another change to the tenant is ongoing: this
    7602              :                     // is not necessary for correctness, but simplifies operations and implicitly throttles
    7603              :                     // optimization changes to happen in a "trickle" over time.
    7604            0 :                     continue;
    7605            0 :                 }
    7606            0 : 
    7607            0 :                 // Fast path: we may quickly identify shards that don't have any possible optimisations
    7608            0 :                 if !shard.maybe_optimizable(scheduler, &schedule_context) {
    7609            0 :                     if cfg!(feature = "testing") {
    7610              :                         // Check that maybe_optimizable doesn't disagree with the actual optimization functions.
    7611              :                         // Only do this in testing builds because it is not a correctness-critical check, so we shouldn't
    7612              :                         // panic in prod if we hit this, or spend cycles on it in prod.
    7613            0 :                         assert!(
    7614            0 :                             shard
    7615            0 :                                 .optimize_attachment(scheduler, &schedule_context)
    7616            0 :                                 .is_none()
    7617            0 :                         );
    7618            0 :                         assert!(
    7619            0 :                             shard
    7620            0 :                                 .optimize_secondary(scheduler, &schedule_context)
    7621            0 :                                 .is_none()
    7622            0 :                         );
    7623            0 :                     }
    7624            0 :                     continue;
    7625            0 :                 }
    7626              : 
    7627            0 :                 if let Some(optimization) =
    7628              :                     // If idle, maybe optimize attachments: if a shard has a secondary location that is preferable to
    7629              :                     // its primary location based on soft constraints, cut it over.
    7630            0 :                     shard.optimize_attachment(scheduler, &schedule_context)
    7631              :                 {
    7632            0 :                     tracing::info!(tenant_shard_id=%shard.tenant_shard_id, "Identified optimization for attachment: {optimization:?}");
    7633            0 :                     work.push((shard.tenant_shard_id, optimization));
    7634            0 :                     break;
    7635            0 :                 } else if let Some(optimization) =
    7636              :                     // If idle, maybe optimize secondary locations: if a shard has a secondary location that would be
    7637              :                     // better placed on another node, based on ScheduleContext, then adjust it.  This
    7638              :                     // covers cases like after a shard split, where we might have too many shards
    7639              :                     // in the same tenant with secondary locations on the node where they originally split.
    7640            0 :                     shard.optimize_secondary(scheduler, &schedule_context)
    7641              :                 {
    7642            0 :                     tracing::info!(tenant_shard_id=%shard.tenant_shard_id, "Identified optimization for secondary: {optimization:?}");
    7643            0 :                     work.push((shard.tenant_shard_id, optimization));
    7644            0 :                     break;
    7645            0 :                 }
    7646              :             }
    7647              :         }
    7648              : 
    7649            0 :         work
    7650            0 :     }
    7651              : 
    7652            0 :     async fn optimize_all_validate(
    7653            0 :         &self,
    7654            0 :         candidate_work: Vec<(TenantShardId, ScheduleOptimization)>,
    7655            0 :     ) -> Vec<(TenantShardId, ScheduleOptimization)> {
    7656            0 :         // Take a clone of the node map to use outside the lock in async validation phase
    7657            0 :         let validation_nodes = { self.inner.read().unwrap().nodes.clone() };
    7658            0 : 
    7659            0 :         let mut want_secondary_status = Vec::new();
    7660            0 : 
    7661            0 :         // Validate our plans: this is an async phase where we may do I/O to pageservers to
    7662            0 :         // check that the state of locations is acceptable to run the optimization, such as
    7663            0 :         // checking that a secondary location is sufficiently warmed-up to cleanly cut over
    7664            0 :         // in a live migration.
    7665            0 :         let mut validated_work = Vec::new();
    7666            0 :         for (tenant_shard_id, optimization) in candidate_work {
    7667            0 :             match optimization.action {
    7668              :                 ScheduleOptimizationAction::MigrateAttachment(MigrateAttachment {
    7669              :                     old_attached_node_id: _,
    7670            0 :                     new_attached_node_id,
    7671            0 :                 }) => {
    7672            0 :                     match validation_nodes.get(&new_attached_node_id) {
    7673            0 :                         None => {
    7674            0 :                             // Node was dropped between planning and validation
    7675            0 :                         }
    7676            0 :                         Some(node) => {
    7677            0 :                             if !node.is_available() {
    7678            0 :                                 tracing::info!(
    7679            0 :                                     "Skipping optimization migration of {tenant_shard_id} to {new_attached_node_id} because node unavailable"
    7680              :                                 );
    7681            0 :                             } else {
    7682            0 :                                 // Accumulate optimizations that require fetching secondary status, so that we can execute these
    7683            0 :                                 // remote API requests concurrently.
    7684            0 :                                 want_secondary_status.push((
    7685            0 :                                     tenant_shard_id,
    7686            0 :                                     node.clone(),
    7687            0 :                                     optimization,
    7688            0 :                                 ));
    7689            0 :                             }
    7690              :                         }
    7691              :                     }
    7692              :                 }
    7693              :                 ScheduleOptimizationAction::ReplaceSecondary(_)
    7694              :                 | ScheduleOptimizationAction::CreateSecondary(_)
    7695              :                 | ScheduleOptimizationAction::RemoveSecondary(_) => {
    7696              :                     // No extra checks needed to manage secondaries: this does not interrupt client access
    7697            0 :                     validated_work.push((tenant_shard_id, optimization))
    7698              :                 }
    7699              :             };
    7700              :         }
    7701              : 
    7702              :         // Call into pageserver API to find out if the destination secondary location is warm enough for a reasonably smooth migration: we
    7703              :         // do this so that we avoid spawning a Reconciler that would have to wait minutes/hours for a destination to warm up: that reconciler
    7704              :         // would hold a precious reconcile semaphore unit the whole time it was waiting for the destination to warm up.
    7705            0 :         let results = self
    7706            0 :             .tenant_for_shards_api(
    7707            0 :                 want_secondary_status
    7708            0 :                     .iter()
    7709            0 :                     .map(|i| (i.0, i.1.clone()))
    7710            0 :                     .collect(),
    7711            0 :                 |tenant_shard_id, client| async move {
    7712            0 :                     client.tenant_secondary_status(tenant_shard_id).await
    7713            0 :                 },
    7714            0 :                 1,
    7715            0 :                 1,
    7716            0 :                 SHORT_RECONCILE_TIMEOUT,
    7717            0 :                 &self.cancel,
    7718            0 :             )
    7719            0 :             .await;
    7720              : 
    7721            0 :         for ((tenant_shard_id, node, optimization), secondary_status) in
    7722            0 :             want_secondary_status.into_iter().zip(results.into_iter())
    7723              :         {
    7724            0 :             match secondary_status {
    7725            0 :                 Err(e) => {
    7726            0 :                     tracing::info!(
    7727            0 :                         "Skipping migration of {tenant_shard_id} to {node}, error querying secondary: {e}"
    7728              :                     );
    7729              :                 }
    7730            0 :                 Ok(progress) => {
    7731              :                     // We require secondary locations to have less than 10GiB of downloads pending before we will use
    7732              :                     // them in an optimization
    7733              :                     const DOWNLOAD_FRESHNESS_THRESHOLD: u64 = 10 * 1024 * 1024 * 1024;
    7734              : 
    7735            0 :                     if progress.heatmap_mtime.is_none()
    7736            0 :                         || progress.bytes_total < DOWNLOAD_FRESHNESS_THRESHOLD
    7737            0 :                             && progress.bytes_downloaded != progress.bytes_total
    7738            0 :                         || progress.bytes_total - progress.bytes_downloaded
    7739            0 :                             > DOWNLOAD_FRESHNESS_THRESHOLD
    7740              :                     {
    7741            0 :                         tracing::info!(
    7742            0 :                             "Skipping migration of {tenant_shard_id} to {node} because secondary isn't ready: {progress:?}"
    7743              :                         );
    7744              : 
    7745              :                         #[cfg(feature = "testing")]
    7746            0 :                         if progress.heatmap_mtime.is_none() {
    7747              :                             // No heatmap might mean the attached location has never uploaded one, or that
    7748              :                             // the secondary download hasn't happened yet.  This is relatively unusual in the field,
    7749              :                             // but fairly common in tests.
    7750            0 :                             self.kick_secondary_download(tenant_shard_id).await;
    7751            0 :                         }
    7752              :                     } else {
    7753              :                         // Location looks ready: proceed
    7754            0 :                         tracing::info!(
    7755            0 :                             "{tenant_shard_id} secondary on {node} is warm enough for migration: {progress:?}"
    7756              :                         );
    7757            0 :                         validated_work.push((tenant_shard_id, optimization))
    7758              :                     }
    7759              :                 }
    7760              :             }
    7761              :         }
    7762              : 
    7763            0 :         validated_work
    7764            0 :     }
    7765              : 
    7766              :     /// Some aspects of scheduling optimisation wait for secondary locations to be warm.  This
    7767              :     /// happens on multi-minute timescales in the field, which is fine because optimisation is meant
    7768              :     /// to be a lazy background thing. However, when testing, it is not practical to wait around, so
    7769              :     /// we have this helper to move things along faster.
    7770              :     #[cfg(feature = "testing")]
    7771            0 :     async fn kick_secondary_download(&self, tenant_shard_id: TenantShardId) {
    7772            0 :         let (attached_node, secondaries) = {
    7773            0 :             let locked = self.inner.read().unwrap();
    7774            0 :             let Some(shard) = locked.tenants.get(&tenant_shard_id) else {
    7775            0 :                 tracing::warn!(
    7776            0 :                     "Skipping kick of secondary download for {tenant_shard_id}: not found"
    7777              :                 );
    7778            0 :                 return;
    7779              :             };
    7780              : 
    7781            0 :             let Some(attached) = shard.intent.get_attached() else {
    7782            0 :                 tracing::warn!(
    7783            0 :                     "Skipping kick of secondary download for {tenant_shard_id}: no attached"
    7784              :                 );
    7785            0 :                 return;
    7786              :             };
    7787              : 
    7788            0 :             let secondaries = shard
    7789            0 :                 .intent
    7790            0 :                 .get_secondary()
    7791            0 :                 .iter()
    7792            0 :                 .map(|n| locked.nodes.get(n).unwrap().clone())
    7793            0 :                 .collect::<Vec<_>>();
    7794            0 : 
    7795            0 :             (locked.nodes.get(attached).unwrap().clone(), secondaries)
    7796            0 :         };
    7797            0 : 
    7798            0 :         // Make remote API calls to upload + download heatmaps: we ignore errors because this is just
    7799            0 :         // a 'kick' to let scheduling optimisation run more promptly.
    7800            0 :         match attached_node
    7801            0 :             .with_client_retries(
    7802            0 :                 |client| async move { client.tenant_heatmap_upload(tenant_shard_id).await },
    7803            0 :                 &self.config.pageserver_jwt_token,
    7804            0 :                 &self.config.ssl_ca_cert,
    7805            0 :                 3,
    7806            0 :                 10,
    7807            0 :                 SHORT_RECONCILE_TIMEOUT,
    7808            0 :                 &self.cancel,
    7809            0 :             )
    7810            0 :             .await
    7811              :         {
    7812            0 :             Some(Err(e)) => {
    7813            0 :                 tracing::info!(
    7814            0 :                     "Failed to upload heatmap from {attached_node} for {tenant_shard_id}: {e}"
    7815              :                 );
    7816              :             }
    7817              :             None => {
    7818            0 :                 tracing::info!(
    7819            0 :                     "Cancelled while uploading heatmap from {attached_node} for {tenant_shard_id}"
    7820              :                 );
    7821              :             }
    7822              :             Some(Ok(_)) => {
    7823            0 :                 tracing::info!(
    7824            0 :                     "Successfully uploaded heatmap from {attached_node} for {tenant_shard_id}"
    7825              :                 );
    7826              :             }
    7827              :         }
    7828              : 
    7829            0 :         for secondary_node in secondaries {
    7830            0 :             match secondary_node
    7831            0 :                 .with_client_retries(
    7832            0 :                     |client| async move {
    7833            0 :                         client
    7834            0 :                             .tenant_secondary_download(
    7835            0 :                                 tenant_shard_id,
    7836            0 :                                 Some(Duration::from_secs(1)),
    7837            0 :                             )
    7838            0 :                             .await
    7839            0 :                     },
    7840            0 :                     &self.config.pageserver_jwt_token,
    7841            0 :                     &self.config.ssl_ca_cert,
    7842            0 :                     3,
    7843            0 :                     10,
    7844            0 :                     SHORT_RECONCILE_TIMEOUT,
    7845            0 :                     &self.cancel,
    7846            0 :                 )
    7847            0 :                 .await
    7848              :             {
    7849            0 :                 Some(Err(e)) => {
    7850            0 :                     tracing::info!(
    7851            0 :                         "Failed to download heatmap from {secondary_node} for {tenant_shard_id}: {e}"
    7852              :                     );
    7853              :                 }
    7854              :                 None => {
    7855            0 :                     tracing::info!(
    7856            0 :                         "Cancelled while downloading heatmap from {secondary_node} for {tenant_shard_id}"
    7857              :                     );
    7858              :                 }
    7859            0 :                 Some(Ok(progress)) => {
    7860            0 :                     tracing::info!(
    7861            0 :                         "Successfully downloaded heatmap from {secondary_node} for {tenant_shard_id}: {progress:?}"
    7862              :                     );
    7863              :                 }
    7864              :             }
    7865              :         }
    7866            0 :     }
    7867              : 
    7868              :     /// Asynchronously split a tenant that's eligible for automatic splits:
    7869              :     ///
    7870              :     /// * The tenant is unsharded.
    7871              :     /// * The logical size of its largest timeline exceeds split_threshold.
    7872              :     /// * The tenant's scheduling policy is active.
    7873              :     ///
    7874              :     /// At most one tenant will be split per call: the one with the largest max logical size. It
    7875              :     /// will split 1 → 8 shards.
    7876              :     ///
    7877              :     /// TODO: consider splitting based on total logical size rather than max logical size.
    7878              :     ///
    7879              :     /// TODO: consider spawning multiple splits in parallel: this is only called once every 20
    7880              :     /// seconds, so a large backlog can take a long time, and if a tenant fails to split it will
    7881              :     /// block all other splits.
    7882            0 :     async fn autosplit_tenants(self: &Arc<Self>) {
    7883            0 :         let Some(split_threshold) = self.config.split_threshold else {
    7884            0 :             return; // auto-splits are disabled
    7885              :         };
    7886            0 :         if split_threshold == 0 {
    7887            0 :             return;
    7888            0 :         }
    7889              : 
    7890              :         // Fetch the largest eligible shards by logical size.
    7891              :         const MAX_SHARDS: ShardCount = ShardCount::new(8);
    7892              : 
    7893            0 :         let mut top_n = self
    7894            0 :             .get_top_tenant_shards(&TopTenantShardsRequest {
    7895            0 :                 order_by: TenantSorting::MaxLogicalSize,
    7896            0 :                 limit: 10,
    7897            0 :                 where_shards_lt: Some(MAX_SHARDS),
    7898            0 :                 where_gt: Some(split_threshold),
    7899            0 :             })
    7900            0 :             .await;
    7901              : 
    7902              :         // Filter out tenants in a prohibiting scheduling mode.
    7903            0 :         {
    7904            0 :             let state = self.inner.read().unwrap();
    7905            0 :             top_n.retain(|i| {
    7906            0 :                 let policy = state.tenants.get(&i.id).map(|s| s.get_scheduling_policy());
    7907            0 :                 policy == Some(ShardSchedulingPolicy::Active)
    7908            0 :             });
    7909            0 :         }
    7910              : 
    7911            0 :         let Some(split_candidate) = top_n.into_iter().next() else {
    7912            0 :             debug!("No split-elegible shards found");
    7913            0 :             return;
    7914              :         };
    7915              : 
    7916              :         // We spawn a task to run this, so it's exactly like some external API client requesting it.
    7917              :         // We don't want to block the background reconcile loop on this.
    7918            0 :         info!(
    7919            0 :             "Auto-splitting tenant for size threshold {split_threshold}: current size {split_candidate:?}"
    7920              :         );
    7921              : 
    7922            0 :         let this = self.clone();
    7923            0 :         tokio::spawn(
    7924            0 :             async move {
    7925            0 :                 match this
    7926            0 :                     .tenant_shard_split(
    7927            0 :                         split_candidate.id.tenant_id,
    7928            0 :                         TenantShardSplitRequest {
    7929            0 :                             // Always split to the max number of shards: this avoids stepping
    7930            0 :                             // through intervening shard counts and encountering the overhead of a
    7931            0 :                             // split+cleanup each time as a tenant grows, and is not too expensive
    7932            0 :                             // because our max shard count is relatively low anyway. This policy
    7933            0 :                             // will be adjusted in future once we support higher shard count.
    7934            0 :                             new_shard_count: MAX_SHARDS.literal(),
    7935            0 :                             new_stripe_size: Some(ShardParameters::DEFAULT_STRIPE_SIZE),
    7936            0 :                         },
    7937            0 :                     )
    7938            0 :                     .await
    7939              :                 {
    7940            0 :                     Ok(_) => info!("Successful auto-split"),
    7941            0 :                     Err(err) => error!("Auto-split failed: {err}"),
    7942              :                 }
    7943            0 :             }
    7944            0 :             .instrument(info_span!("auto_split", tenant_id=%split_candidate.id.tenant_id)),
    7945              :         );
    7946            0 :     }
    7947              : 
    7948              :     /// Fetches the top tenant shards from every node, in descending order of
    7949              :     /// max logical size. Any node errors will be logged and ignored.
    7950            0 :     async fn get_top_tenant_shards(
    7951            0 :         &self,
    7952            0 :         request: &TopTenantShardsRequest,
    7953            0 :     ) -> Vec<TopTenantShardItem> {
    7954            0 :         let nodes = self
    7955            0 :             .inner
    7956            0 :             .read()
    7957            0 :             .unwrap()
    7958            0 :             .nodes
    7959            0 :             .values()
    7960            0 :             .cloned()
    7961            0 :             .collect_vec();
    7962            0 : 
    7963            0 :         let mut futures = FuturesUnordered::new();
    7964            0 :         for node in nodes {
    7965            0 :             futures.push(async move {
    7966            0 :                 node.with_client_retries(
    7967            0 :                     |client| async move { client.top_tenant_shards(request.clone()).await },
    7968            0 :                     &self.config.pageserver_jwt_token,
    7969            0 :                     &self.config.ssl_ca_cert,
    7970            0 :                     3,
    7971            0 :                     3,
    7972            0 :                     Duration::from_secs(5),
    7973            0 :                     &self.cancel,
    7974            0 :                 )
    7975            0 :                 .await
    7976            0 :             });
    7977            0 :         }
    7978              : 
    7979            0 :         let mut top = Vec::new();
    7980            0 :         while let Some(output) = futures.next().await {
    7981            0 :             match output {
    7982            0 :                 Some(Ok(response)) => top.extend(response.shards),
    7983            0 :                 Some(Err(mgmt_api::Error::Cancelled)) => {}
    7984            0 :                 Some(Err(err)) => warn!("failed to fetch top tenants: {err}"),
    7985            0 :                 None => {} // node is shutting down
    7986              :             }
    7987              :         }
    7988              : 
    7989            0 :         top.sort_by_key(|i| i.max_logical_size);
    7990            0 :         top.reverse();
    7991            0 :         top
    7992            0 :     }
    7993              : 
    7994              :     /// Useful for tests: run whatever work a background [`Self::reconcile_all`] would have done, but
    7995              :     /// also wait for any generated Reconcilers to complete.  Calling this until it returns zero should
    7996              :     /// put the system into a quiescent state where future background reconciliations won't do anything.
    7997            0 :     pub(crate) async fn reconcile_all_now(&self) -> Result<usize, ReconcileWaitError> {
    7998            0 :         let reconciles_spawned = self.reconcile_all();
    7999            0 :         let reconciles_spawned = if reconciles_spawned == 0 {
    8000              :             // Only optimize when we are otherwise idle
    8001            0 :             self.optimize_all().await
    8002              :         } else {
    8003            0 :             reconciles_spawned
    8004              :         };
    8005              : 
    8006            0 :         let waiters = {
    8007            0 :             let mut waiters = Vec::new();
    8008            0 :             let locked = self.inner.read().unwrap();
    8009            0 :             for (_tenant_shard_id, shard) in locked.tenants.iter() {
    8010            0 :                 if let Some(waiter) = shard.get_waiter() {
    8011            0 :                     waiters.push(waiter);
    8012            0 :                 }
    8013              :             }
    8014            0 :             waiters
    8015            0 :         };
    8016            0 : 
    8017            0 :         let waiter_count = waiters.len();
    8018            0 :         match self.await_waiters(waiters, RECONCILE_TIMEOUT).await {
    8019            0 :             Ok(()) => {}
    8020            0 :             Err(ReconcileWaitError::Failed(_, reconcile_error))
    8021            0 :                 if matches!(*reconcile_error, ReconcileError::Cancel) =>
    8022            0 :             {
    8023            0 :                 // Ignore reconciler cancel errors: this reconciler might have shut down
    8024            0 :                 // because some other change superceded it.  We will return a nonzero number,
    8025            0 :                 // so the caller knows they might have to call again to quiesce the system.
    8026            0 :             }
    8027            0 :             Err(e) => {
    8028            0 :                 return Err(e);
    8029              :             }
    8030              :         };
    8031              : 
    8032            0 :         tracing::info!(
    8033            0 :             "{} reconciles in reconcile_all, {} waiters",
    8034              :             reconciles_spawned,
    8035              :             waiter_count
    8036              :         );
    8037              : 
    8038            0 :         Ok(std::cmp::max(waiter_count, reconciles_spawned))
    8039            0 :     }
    8040              : 
    8041            0 :     async fn stop_reconciliations(&self, reason: StopReconciliationsReason) {
    8042            0 :         // Cancel all on-going reconciles and wait for them to exit the gate.
    8043            0 :         tracing::info!("{reason}: cancelling and waiting for in-flight reconciles");
    8044            0 :         self.reconcilers_cancel.cancel();
    8045            0 :         self.reconcilers_gate.close().await;
    8046              : 
    8047              :         // Signal the background loop in [`Service::process_results`] to exit once
    8048              :         // it has proccessed the results from all the reconciles we cancelled earlier.
    8049            0 :         tracing::info!("{reason}: processing results from previously in-flight reconciles");
    8050            0 :         self.result_tx.send(ReconcileResultRequest::Stop).ok();
    8051            0 :         self.result_tx.closed().await;
    8052            0 :     }
    8053              : 
    8054            0 :     pub async fn shutdown(&self) {
    8055            0 :         self.stop_reconciliations(StopReconciliationsReason::ShuttingDown)
    8056            0 :             .await;
    8057              : 
    8058              :         // Background tasks hold gate guards: this notifies them of the cancellation and
    8059              :         // waits for them all to complete.
    8060            0 :         tracing::info!("Shutting down: cancelling and waiting for background tasks to exit");
    8061            0 :         self.cancel.cancel();
    8062            0 :         self.gate.close().await;
    8063            0 :     }
    8064              : 
    8065              :     /// Spot check the download lag for a secondary location of a shard.
    8066              :     /// Should be used as a heuristic, since it's not always precise: the
    8067              :     /// secondary might have not downloaded the new heat map yet and, hence,
    8068              :     /// is not aware of the lag.
    8069              :     ///
    8070              :     /// Returns:
    8071              :     /// * Ok(None) if the lag could not be determined from the status,
    8072              :     /// * Ok(Some(_)) if the lag could be determind
    8073              :     /// * Err on failures to query the pageserver.
    8074            0 :     async fn secondary_lag(
    8075            0 :         &self,
    8076            0 :         secondary: &NodeId,
    8077            0 :         tenant_shard_id: TenantShardId,
    8078            0 :     ) -> Result<Option<u64>, mgmt_api::Error> {
    8079            0 :         let nodes = self.inner.read().unwrap().nodes.clone();
    8080            0 :         let node = nodes.get(secondary).ok_or(mgmt_api::Error::ApiError(
    8081            0 :             StatusCode::NOT_FOUND,
    8082            0 :             format!("Node with id {} not found", secondary),
    8083            0 :         ))?;
    8084              : 
    8085            0 :         match node
    8086            0 :             .with_client_retries(
    8087            0 :                 |client| async move { client.tenant_secondary_status(tenant_shard_id).await },
    8088            0 :                 &self.config.pageserver_jwt_token,
    8089            0 :                 &self.config.ssl_ca_cert,
    8090            0 :                 1,
    8091            0 :                 3,
    8092            0 :                 Duration::from_millis(250),
    8093            0 :                 &self.cancel,
    8094            0 :             )
    8095            0 :             .await
    8096              :         {
    8097            0 :             Some(Ok(status)) => match status.heatmap_mtime {
    8098            0 :                 Some(_) => Ok(Some(status.bytes_total - status.bytes_downloaded)),
    8099            0 :                 None => Ok(None),
    8100              :             },
    8101            0 :             Some(Err(e)) => Err(e),
    8102            0 :             None => Err(mgmt_api::Error::Cancelled),
    8103              :         }
    8104            0 :     }
    8105              : 
    8106              :     /// Drain a node by moving the shards attached to it as primaries.
    8107              :     /// This is a long running operation and it should run as a separate Tokio task.
    8108            0 :     pub(crate) async fn drain_node(
    8109            0 :         self: &Arc<Self>,
    8110            0 :         node_id: NodeId,
    8111            0 :         cancel: CancellationToken,
    8112            0 :     ) -> Result<(), OperationError> {
    8113              :         const MAX_SECONDARY_LAG_BYTES_DEFAULT: u64 = 256 * 1024 * 1024;
    8114            0 :         let max_secondary_lag_bytes = self
    8115            0 :             .config
    8116            0 :             .max_secondary_lag_bytes
    8117            0 :             .unwrap_or(MAX_SECONDARY_LAG_BYTES_DEFAULT);
    8118              : 
    8119              :         // By default, live migrations are generous about the wait time for getting
    8120              :         // the secondary location up to speed. When draining, give up earlier in order
    8121              :         // to not stall the operation when a cold secondary is encountered.
    8122              :         const SECONDARY_WARMUP_TIMEOUT: Duration = Duration::from_secs(20);
    8123              :         const SECONDARY_DOWNLOAD_REQUEST_TIMEOUT: Duration = Duration::from_secs(5);
    8124            0 :         let reconciler_config = ReconcilerConfigBuilder::new(ReconcilerPriority::Normal)
    8125            0 :             .secondary_warmup_timeout(SECONDARY_WARMUP_TIMEOUT)
    8126            0 :             .secondary_download_request_timeout(SECONDARY_DOWNLOAD_REQUEST_TIMEOUT)
    8127            0 :             .build();
    8128            0 : 
    8129            0 :         let mut waiters = Vec::new();
    8130            0 : 
    8131            0 :         let mut tid_iter = TenantShardIterator::new({
    8132            0 :             let service = self.clone();
    8133            0 :             move |last_inspected_shard: Option<TenantShardId>| {
    8134            0 :                 let locked = &service.inner.read().unwrap();
    8135            0 :                 let tenants = &locked.tenants;
    8136            0 :                 let entry = match last_inspected_shard {
    8137            0 :                     Some(skip_past) => {
    8138            0 :                         // Skip to the last seen tenant shard id
    8139            0 :                         let mut cursor = tenants.iter().skip_while(|(tid, _)| **tid != skip_past);
    8140            0 : 
    8141            0 :                         // Skip past the last seen
    8142            0 :                         cursor.nth(1)
    8143              :                     }
    8144            0 :                     None => tenants.first_key_value(),
    8145              :                 };
    8146              : 
    8147            0 :                 entry.map(|(tid, _)| tid).copied()
    8148            0 :             }
    8149            0 :         });
    8150              : 
    8151            0 :         while !tid_iter.finished() {
    8152            0 :             if cancel.is_cancelled() {
    8153            0 :                 match self
    8154            0 :                     .node_configure(node_id, None, Some(NodeSchedulingPolicy::Active))
    8155            0 :                     .await
    8156              :                 {
    8157            0 :                     Ok(()) => return Err(OperationError::Cancelled),
    8158            0 :                     Err(err) => {
    8159            0 :                         return Err(OperationError::FinalizeError(
    8160            0 :                             format!(
    8161            0 :                                 "Failed to finalise drain cancel of {} by setting scheduling policy to Active: {}",
    8162            0 :                                 node_id, err
    8163            0 :                             )
    8164            0 :                             .into(),
    8165            0 :                         ));
    8166              :                     }
    8167              :                 }
    8168            0 :             }
    8169            0 : 
    8170            0 :             drain_utils::validate_node_state(&node_id, self.inner.read().unwrap().nodes.clone())?;
    8171              : 
    8172            0 :             while waiters.len() < MAX_RECONCILES_PER_OPERATION {
    8173            0 :                 let tid = match tid_iter.next() {
    8174            0 :                     Some(tid) => tid,
    8175              :                     None => {
    8176            0 :                         break;
    8177              :                     }
    8178              :                 };
    8179              : 
    8180            0 :                 let tid_drain = TenantShardDrain {
    8181            0 :                     drained_node: node_id,
    8182            0 :                     tenant_shard_id: tid,
    8183            0 :                 };
    8184              : 
    8185            0 :                 let dest_node_id = {
    8186            0 :                     let locked = self.inner.read().unwrap();
    8187            0 : 
    8188            0 :                     match tid_drain
    8189            0 :                         .tenant_shard_eligible_for_drain(&locked.tenants, &locked.scheduler)
    8190              :                     {
    8191            0 :                         Some(node_id) => node_id,
    8192              :                         None => {
    8193            0 :                             continue;
    8194              :                         }
    8195              :                     }
    8196              :                 };
    8197              : 
    8198            0 :                 match self.secondary_lag(&dest_node_id, tid).await {
    8199            0 :                     Ok(Some(lag)) if lag <= max_secondary_lag_bytes => {
    8200            0 :                         // The secondary is reasonably up to date.
    8201            0 :                         // Migrate to it
    8202            0 :                     }
    8203            0 :                     Ok(Some(lag)) => {
    8204            0 :                         tracing::info!(
    8205            0 :                             tenant_id=%tid.tenant_id, shard_id=%tid.shard_slug(),
    8206            0 :                             "Secondary on node {dest_node_id} is lagging by {lag}. Skipping reconcile."
    8207              :                         );
    8208            0 :                         continue;
    8209              :                     }
    8210              :                     Ok(None) => {
    8211            0 :                         tracing::info!(
    8212            0 :                             tenant_id=%tid.tenant_id, shard_id=%tid.shard_slug(),
    8213            0 :                             "Could not determine lag for secondary on node {dest_node_id}. Skipping reconcile."
    8214              :                         );
    8215            0 :                         continue;
    8216              :                     }
    8217            0 :                     Err(err) => {
    8218            0 :                         tracing::warn!(
    8219            0 :                             tenant_id=%tid.tenant_id, shard_id=%tid.shard_slug(),
    8220            0 :                             "Failed to get secondary lag from node {dest_node_id}. Skipping reconcile: {err}"
    8221              :                         );
    8222            0 :                         continue;
    8223              :                     }
    8224              :                 }
    8225              : 
    8226              :                 {
    8227            0 :                     let mut locked = self.inner.write().unwrap();
    8228            0 :                     let (nodes, tenants, scheduler) = locked.parts_mut();
    8229            0 :                     let rescheduled = tid_drain.reschedule_to_secondary(
    8230            0 :                         dest_node_id,
    8231            0 :                         tenants,
    8232            0 :                         scheduler,
    8233            0 :                         nodes,
    8234            0 :                     )?;
    8235              : 
    8236            0 :                     if let Some(tenant_shard) = rescheduled {
    8237            0 :                         let waiter = self.maybe_configured_reconcile_shard(
    8238            0 :                             tenant_shard,
    8239            0 :                             nodes,
    8240            0 :                             reconciler_config,
    8241            0 :                         );
    8242            0 :                         if let Some(some) = waiter {
    8243            0 :                             waiters.push(some);
    8244            0 :                         }
    8245            0 :                     }
    8246              :                 }
    8247              :             }
    8248              : 
    8249            0 :             waiters = self
    8250            0 :                 .await_waiters_remainder(waiters, WAITER_FILL_DRAIN_POLL_TIMEOUT)
    8251            0 :                 .await;
    8252              : 
    8253            0 :             failpoint_support::sleep_millis_async!("sleepy-drain-loop", &cancel);
    8254              :         }
    8255              : 
    8256            0 :         while !waiters.is_empty() {
    8257            0 :             if cancel.is_cancelled() {
    8258            0 :                 match self
    8259            0 :                     .node_configure(node_id, None, Some(NodeSchedulingPolicy::Active))
    8260            0 :                     .await
    8261              :                 {
    8262            0 :                     Ok(()) => return Err(OperationError::Cancelled),
    8263            0 :                     Err(err) => {
    8264            0 :                         return Err(OperationError::FinalizeError(
    8265            0 :                             format!(
    8266            0 :                                 "Failed to finalise drain cancel of {} by setting scheduling policy to Active: {}",
    8267            0 :                                 node_id, err
    8268            0 :                             )
    8269            0 :                             .into(),
    8270            0 :                         ));
    8271              :                     }
    8272              :                 }
    8273            0 :             }
    8274            0 : 
    8275            0 :             tracing::info!("Awaiting {} pending drain reconciliations", waiters.len());
    8276              : 
    8277            0 :             waiters = self
    8278            0 :                 .await_waiters_remainder(waiters, SHORT_RECONCILE_TIMEOUT)
    8279            0 :                 .await;
    8280              :         }
    8281              : 
    8282              :         // At this point we have done the best we could to drain shards from this node.
    8283              :         // Set the node scheduling policy to `[NodeSchedulingPolicy::PauseForRestart]`
    8284              :         // to complete the drain.
    8285            0 :         if let Err(err) = self
    8286            0 :             .node_configure(node_id, None, Some(NodeSchedulingPolicy::PauseForRestart))
    8287            0 :             .await
    8288              :         {
    8289              :             // This is not fatal. Anything that is polling the node scheduling policy to detect
    8290              :             // the end of the drain operations will hang, but all such places should enforce an
    8291              :             // overall timeout. The scheduling policy will be updated upon node re-attach and/or
    8292              :             // by the counterpart fill operation.
    8293            0 :             return Err(OperationError::FinalizeError(
    8294            0 :                 format!(
    8295            0 :                     "Failed to finalise drain of {node_id} by setting scheduling policy to PauseForRestart: {err}"
    8296            0 :                 )
    8297            0 :                 .into(),
    8298            0 :             ));
    8299            0 :         }
    8300            0 : 
    8301            0 :         Ok(())
    8302            0 :     }
    8303              : 
    8304              :     /// Create a node fill plan (pick secondaries to promote), based on:
    8305              :     /// 1. Shards which have a secondary on this node, and this node is in their home AZ, and are currently attached to a node
    8306              :     ///    outside their home AZ, should be migrated back here.
    8307              :     /// 2. If after step 1 we have not migrated enough shards for this node to have its fair share of
    8308              :     ///    attached shards, we will promote more shards from the nodes with the most attached shards, unless
    8309              :     ///    those shards have a home AZ that doesn't match the node we're filling.
    8310            0 :     fn fill_node_plan(&self, node_id: NodeId) -> Vec<TenantShardId> {
    8311            0 :         let mut locked = self.inner.write().unwrap();
    8312            0 :         let (nodes, tenants, _scheduler) = locked.parts_mut();
    8313            0 : 
    8314            0 :         let node_az = nodes
    8315            0 :             .get(&node_id)
    8316            0 :             .expect("Node must exist")
    8317            0 :             .get_availability_zone_id()
    8318            0 :             .clone();
    8319            0 : 
    8320            0 :         // The tenant shard IDs that we plan to promote from secondary to attached on this node
    8321            0 :         let mut plan = Vec::new();
    8322            0 : 
    8323            0 :         // Collect shards which do not have a preferred AZ & are elegible for moving in stage 2
    8324            0 :         let mut free_tids_by_node: HashMap<NodeId, Vec<TenantShardId>> = HashMap::new();
    8325            0 : 
    8326            0 :         // Don't respect AZ preferences if there is only one AZ.  This comes up in tests, but it could
    8327            0 :         // conceivably come up in real life if deploying a single-AZ region intentionally.
    8328            0 :         let respect_azs = nodes
    8329            0 :             .values()
    8330            0 :             .map(|n| n.get_availability_zone_id())
    8331            0 :             .unique()
    8332            0 :             .count()
    8333            0 :             > 1;
    8334              : 
    8335              :         // Step 1: collect all shards that we are required to migrate back to this node because their AZ preference
    8336              :         // requires it.
    8337            0 :         for (tsid, tenant_shard) in tenants {
    8338            0 :             if !tenant_shard.intent.get_secondary().contains(&node_id) {
    8339              :                 // Shard doesn't have a secondary on this node, ignore it.
    8340            0 :                 continue;
    8341            0 :             }
    8342            0 : 
    8343            0 :             // AZ check: when filling nodes after a restart, our intent is to move _back_ the
    8344            0 :             // shards which belong on this node, not to promote shards whose scheduling preference
    8345            0 :             // would be on their currently attached node.  So will avoid promoting shards whose
    8346            0 :             // home AZ doesn't match the AZ of the node we're filling.
    8347            0 :             match tenant_shard.preferred_az() {
    8348            0 :                 _ if !respect_azs => {
    8349            0 :                     if let Some(primary) = tenant_shard.intent.get_attached() {
    8350            0 :                         free_tids_by_node.entry(*primary).or_default().push(*tsid);
    8351            0 :                     }
    8352              :                 }
    8353              :                 None => {
    8354              :                     // Shard doesn't have an AZ preference: it is elegible to be moved, but we
    8355              :                     // will only do so if our target shard count requires it.
    8356            0 :                     if let Some(primary) = tenant_shard.intent.get_attached() {
    8357            0 :                         free_tids_by_node.entry(*primary).or_default().push(*tsid);
    8358            0 :                     }
    8359              :                 }
    8360            0 :                 Some(az) if az == &node_az => {
    8361              :                     // This shard's home AZ is equal to the node we're filling: it should
    8362              :                     // be moved back to this node as part of filling, unless its currently
    8363              :                     // attached location is also in its home AZ.
    8364            0 :                     if let Some(primary) = tenant_shard.intent.get_attached() {
    8365            0 :                         if nodes
    8366            0 :                             .get(primary)
    8367            0 :                             .expect("referenced node must exist")
    8368            0 :                             .get_availability_zone_id()
    8369            0 :                             != tenant_shard
    8370            0 :                                 .preferred_az()
    8371            0 :                                 .expect("tenant must have an AZ preference")
    8372              :                         {
    8373            0 :                             plan.push(*tsid)
    8374            0 :                         }
    8375              :                     } else {
    8376            0 :                         plan.push(*tsid)
    8377              :                     }
    8378              :                 }
    8379            0 :                 Some(_) => {
    8380            0 :                     // This shard's home AZ is somewhere other than the node we're filling,
    8381            0 :                     // it may not be moved back to this node as part of filling.  Ignore it
    8382            0 :                 }
    8383              :             }
    8384              :         }
    8385              : 
    8386              :         // Step 2: also promote any AZ-agnostic shards as required to achieve the target number of attachments
    8387            0 :         let fill_requirement = locked.scheduler.compute_fill_requirement(node_id);
    8388            0 : 
    8389            0 :         let expected_attached = locked.scheduler.expected_attached_shard_count();
    8390            0 :         let nodes_by_load = locked.scheduler.nodes_by_attached_shard_count();
    8391            0 : 
    8392            0 :         let mut promoted_per_tenant: HashMap<TenantId, usize> = HashMap::new();
    8393              : 
    8394            0 :         for (node_id, attached) in nodes_by_load {
    8395            0 :             let available = locked.nodes.get(&node_id).is_some_and(|n| n.is_available());
    8396            0 :             if !available {
    8397            0 :                 continue;
    8398            0 :             }
    8399            0 : 
    8400            0 :             if plan.len() >= fill_requirement
    8401            0 :                 || free_tids_by_node.is_empty()
    8402            0 :                 || attached <= expected_attached
    8403              :             {
    8404            0 :                 break;
    8405            0 :             }
    8406            0 : 
    8407            0 :             let can_take = attached - expected_attached;
    8408            0 :             let needed = fill_requirement - plan.len();
    8409            0 :             let mut take = std::cmp::min(can_take, needed);
    8410            0 : 
    8411            0 :             let mut remove_node = false;
    8412            0 :             while take > 0 {
    8413            0 :                 match free_tids_by_node.get_mut(&node_id) {
    8414            0 :                     Some(tids) => match tids.pop() {
    8415            0 :                         Some(tid) => {
    8416            0 :                             let max_promote_for_tenant = std::cmp::max(
    8417            0 :                                 tid.shard_count.count() as usize / locked.nodes.len(),
    8418            0 :                                 1,
    8419            0 :                             );
    8420            0 :                             let promoted = promoted_per_tenant.entry(tid.tenant_id).or_default();
    8421            0 :                             if *promoted < max_promote_for_tenant {
    8422            0 :                                 plan.push(tid);
    8423            0 :                                 *promoted += 1;
    8424            0 :                                 take -= 1;
    8425            0 :                             }
    8426              :                         }
    8427              :                         None => {
    8428            0 :                             remove_node = true;
    8429            0 :                             break;
    8430              :                         }
    8431              :                     },
    8432              :                     None => {
    8433            0 :                         break;
    8434              :                     }
    8435              :                 }
    8436              :             }
    8437              : 
    8438            0 :             if remove_node {
    8439            0 :                 free_tids_by_node.remove(&node_id);
    8440            0 :             }
    8441              :         }
    8442              : 
    8443            0 :         plan
    8444            0 :     }
    8445              : 
    8446              :     /// Fill a node by promoting its secondaries until the cluster is balanced
    8447              :     /// with regards to attached shard counts. Note that this operation only
    8448              :     /// makes sense as a counterpart to the drain implemented in [`Service::drain_node`].
    8449              :     /// This is a long running operation and it should run as a separate Tokio task.
    8450            0 :     pub(crate) async fn fill_node(
    8451            0 :         &self,
    8452            0 :         node_id: NodeId,
    8453            0 :         cancel: CancellationToken,
    8454            0 :     ) -> Result<(), OperationError> {
    8455              :         const SECONDARY_WARMUP_TIMEOUT: Duration = Duration::from_secs(20);
    8456              :         const SECONDARY_DOWNLOAD_REQUEST_TIMEOUT: Duration = Duration::from_secs(5);
    8457            0 :         let reconciler_config = ReconcilerConfigBuilder::new(ReconcilerPriority::Normal)
    8458            0 :             .secondary_warmup_timeout(SECONDARY_WARMUP_TIMEOUT)
    8459            0 :             .secondary_download_request_timeout(SECONDARY_DOWNLOAD_REQUEST_TIMEOUT)
    8460            0 :             .build();
    8461            0 : 
    8462            0 :         let mut tids_to_promote = self.fill_node_plan(node_id);
    8463            0 :         let mut waiters = Vec::new();
    8464              : 
    8465              :         // Execute the plan we've composed above. Before aplying each move from the plan,
    8466              :         // we validate to ensure that it has not gone stale in the meantime.
    8467            0 :         while !tids_to_promote.is_empty() {
    8468            0 :             if cancel.is_cancelled() {
    8469            0 :                 match self
    8470            0 :                     .node_configure(node_id, None, Some(NodeSchedulingPolicy::Active))
    8471            0 :                     .await
    8472              :                 {
    8473            0 :                     Ok(()) => return Err(OperationError::Cancelled),
    8474            0 :                     Err(err) => {
    8475            0 :                         return Err(OperationError::FinalizeError(
    8476            0 :                             format!(
    8477            0 :                                 "Failed to finalise drain cancel of {} by setting scheduling policy to Active: {}",
    8478            0 :                                 node_id, err
    8479            0 :                             )
    8480            0 :                             .into(),
    8481            0 :                         ));
    8482              :                     }
    8483              :                 }
    8484            0 :             }
    8485            0 : 
    8486            0 :             {
    8487            0 :                 let mut locked = self.inner.write().unwrap();
    8488            0 :                 let (nodes, tenants, scheduler) = locked.parts_mut();
    8489              : 
    8490            0 :                 let node = nodes.get(&node_id).ok_or(OperationError::NodeStateChanged(
    8491            0 :                     format!("node {node_id} was removed").into(),
    8492            0 :                 ))?;
    8493              : 
    8494            0 :                 let current_policy = node.get_scheduling();
    8495            0 :                 if !matches!(current_policy, NodeSchedulingPolicy::Filling) {
    8496              :                     // TODO(vlad): maybe cancel pending reconciles before erroring out. need to think
    8497              :                     // about it
    8498            0 :                     return Err(OperationError::NodeStateChanged(
    8499            0 :                         format!("node {node_id} changed state to {current_policy:?}").into(),
    8500            0 :                     ));
    8501            0 :                 }
    8502              : 
    8503            0 :                 while waiters.len() < MAX_RECONCILES_PER_OPERATION {
    8504            0 :                     if let Some(tid) = tids_to_promote.pop() {
    8505            0 :                         if let Some(tenant_shard) = tenants.get_mut(&tid) {
    8506              :                             // If the node being filled is not a secondary anymore,
    8507              :                             // skip the promotion.
    8508            0 :                             if !tenant_shard.intent.get_secondary().contains(&node_id) {
    8509            0 :                                 continue;
    8510            0 :                             }
    8511            0 : 
    8512            0 :                             let previously_attached_to = *tenant_shard.intent.get_attached();
    8513            0 :                             match tenant_shard.reschedule_to_secondary(Some(node_id), scheduler) {
    8514            0 :                                 Err(e) => {
    8515            0 :                                     tracing::warn!(
    8516            0 :                                         tenant_id=%tid.tenant_id, shard_id=%tid.shard_slug(),
    8517            0 :                                         "Scheduling error when filling pageserver {} : {e}", node_id
    8518              :                                     );
    8519              :                                 }
    8520              :                                 Ok(()) => {
    8521            0 :                                     tracing::info!(
    8522            0 :                                         tenant_id=%tid.tenant_id, shard_id=%tid.shard_slug(),
    8523            0 :                                         "Rescheduled shard while filling node {}: {:?} -> {}",
    8524              :                                         node_id,
    8525              :                                         previously_attached_to,
    8526              :                                         node_id
    8527              :                                     );
    8528              : 
    8529            0 :                                     if let Some(waiter) = self.maybe_configured_reconcile_shard(
    8530            0 :                                         tenant_shard,
    8531            0 :                                         nodes,
    8532            0 :                                         reconciler_config,
    8533            0 :                                     ) {
    8534            0 :                                         waiters.push(waiter);
    8535            0 :                                     }
    8536              :                                 }
    8537              :                             }
    8538            0 :                         }
    8539              :                     } else {
    8540            0 :                         break;
    8541              :                     }
    8542              :                 }
    8543              :             }
    8544              : 
    8545            0 :             waiters = self
    8546            0 :                 .await_waiters_remainder(waiters, WAITER_FILL_DRAIN_POLL_TIMEOUT)
    8547            0 :                 .await;
    8548              :         }
    8549              : 
    8550            0 :         while !waiters.is_empty() {
    8551            0 :             if cancel.is_cancelled() {
    8552            0 :                 match self
    8553            0 :                     .node_configure(node_id, None, Some(NodeSchedulingPolicy::Active))
    8554            0 :                     .await
    8555              :                 {
    8556            0 :                     Ok(()) => return Err(OperationError::Cancelled),
    8557            0 :                     Err(err) => {
    8558            0 :                         return Err(OperationError::FinalizeError(
    8559            0 :                             format!(
    8560            0 :                                 "Failed to finalise drain cancel of {} by setting scheduling policy to Active: {}",
    8561            0 :                                 node_id, err
    8562            0 :                             )
    8563            0 :                             .into(),
    8564            0 :                         ));
    8565              :                     }
    8566              :                 }
    8567            0 :             }
    8568            0 : 
    8569            0 :             tracing::info!("Awaiting {} pending fill reconciliations", waiters.len());
    8570              : 
    8571            0 :             waiters = self
    8572            0 :                 .await_waiters_remainder(waiters, SHORT_RECONCILE_TIMEOUT)
    8573            0 :                 .await;
    8574              :         }
    8575              : 
    8576            0 :         if let Err(err) = self
    8577            0 :             .node_configure(node_id, None, Some(NodeSchedulingPolicy::Active))
    8578            0 :             .await
    8579              :         {
    8580              :             // This isn't a huge issue since the filling process starts upon request. However, it
    8581              :             // will prevent the next drain from starting. The only case in which this can fail
    8582              :             // is database unavailability. Such a case will require manual intervention.
    8583            0 :             return Err(OperationError::FinalizeError(
    8584            0 :                 format!("Failed to finalise fill of {node_id} by setting scheduling policy to Active: {err}")
    8585            0 :                     .into(),
    8586            0 :             ));
    8587            0 :         }
    8588            0 : 
    8589            0 :         Ok(())
    8590            0 :     }
    8591              : 
    8592              :     /// Updates scrubber metadata health check results.
    8593            0 :     pub(crate) async fn metadata_health_update(
    8594            0 :         &self,
    8595            0 :         update_req: MetadataHealthUpdateRequest,
    8596            0 :     ) -> Result<(), ApiError> {
    8597            0 :         let now = chrono::offset::Utc::now();
    8598            0 :         let (healthy_records, unhealthy_records) = {
    8599            0 :             let locked = self.inner.read().unwrap();
    8600            0 :             let healthy_records = update_req
    8601            0 :                 .healthy_tenant_shards
    8602            0 :                 .into_iter()
    8603            0 :                 // Retain only health records associated with tenant shards managed by storage controller.
    8604            0 :                 .filter(|tenant_shard_id| locked.tenants.contains_key(tenant_shard_id))
    8605            0 :                 .map(|tenant_shard_id| MetadataHealthPersistence::new(tenant_shard_id, true, now))
    8606            0 :                 .collect();
    8607            0 :             let unhealthy_records = update_req
    8608            0 :                 .unhealthy_tenant_shards
    8609            0 :                 .into_iter()
    8610            0 :                 .filter(|tenant_shard_id| locked.tenants.contains_key(tenant_shard_id))
    8611            0 :                 .map(|tenant_shard_id| MetadataHealthPersistence::new(tenant_shard_id, false, now))
    8612            0 :                 .collect();
    8613            0 : 
    8614            0 :             (healthy_records, unhealthy_records)
    8615            0 :         };
    8616            0 : 
    8617            0 :         self.persistence
    8618            0 :             .update_metadata_health_records(healthy_records, unhealthy_records, now)
    8619            0 :             .await?;
    8620            0 :         Ok(())
    8621            0 :     }
    8622              : 
    8623              :     /// Lists the tenant shards that has unhealthy metadata status.
    8624            0 :     pub(crate) async fn metadata_health_list_unhealthy(
    8625            0 :         &self,
    8626            0 :     ) -> Result<Vec<TenantShardId>, ApiError> {
    8627            0 :         let result = self
    8628            0 :             .persistence
    8629            0 :             .list_unhealthy_metadata_health_records()
    8630            0 :             .await?
    8631            0 :             .iter()
    8632            0 :             .map(|p| p.get_tenant_shard_id().unwrap())
    8633            0 :             .collect();
    8634            0 : 
    8635            0 :         Ok(result)
    8636            0 :     }
    8637              : 
    8638              :     /// Lists the tenant shards that have not been scrubbed for some duration.
    8639            0 :     pub(crate) async fn metadata_health_list_outdated(
    8640            0 :         &self,
    8641            0 :         not_scrubbed_for: Duration,
    8642            0 :     ) -> Result<Vec<MetadataHealthRecord>, ApiError> {
    8643            0 :         let earlier = chrono::offset::Utc::now() - not_scrubbed_for;
    8644            0 :         let result = self
    8645            0 :             .persistence
    8646            0 :             .list_outdated_metadata_health_records(earlier)
    8647            0 :             .await?
    8648            0 :             .into_iter()
    8649            0 :             .map(|record| record.into())
    8650            0 :             .collect();
    8651            0 :         Ok(result)
    8652            0 :     }
    8653              : 
    8654            0 :     pub(crate) fn get_leadership_status(&self) -> LeadershipStatus {
    8655            0 :         self.inner.read().unwrap().get_leadership_status()
    8656            0 :     }
    8657              : 
    8658            0 :     pub(crate) async fn step_down(&self) -> GlobalObservedState {
    8659            0 :         tracing::info!("Received step down request from peer");
    8660            0 :         failpoint_support::sleep_millis_async!("sleep-on-step-down-handling");
    8661              : 
    8662            0 :         self.inner.write().unwrap().step_down();
    8663            0 :         // TODO: would it make sense to have a time-out for this?
    8664            0 :         self.stop_reconciliations(StopReconciliationsReason::SteppingDown)
    8665            0 :             .await;
    8666              : 
    8667            0 :         let mut global_observed = GlobalObservedState::default();
    8668            0 :         let locked = self.inner.read().unwrap();
    8669            0 :         for (tid, tenant_shard) in locked.tenants.iter() {
    8670            0 :             global_observed
    8671            0 :                 .0
    8672            0 :                 .insert(*tid, tenant_shard.observed.clone());
    8673            0 :         }
    8674              : 
    8675            0 :         global_observed
    8676            0 :     }
    8677              : 
    8678              :     /// Choose safekeepers for the new timeline: 3 in different azs.
    8679            0 :     pub(crate) async fn safekeepers_for_new_timeline(
    8680            0 :         &self,
    8681            0 :     ) -> Result<Vec<SafekeeperInfo>, ApiError> {
    8682            0 :         let mut all_safekeepers = {
    8683            0 :             let locked = self.inner.read().unwrap();
    8684            0 :             locked
    8685            0 :                 .safekeepers
    8686            0 :                 .iter()
    8687            0 :                 .filter_map(|sk| {
    8688            0 :                     if sk.1.scheduling_policy() != SkSchedulingPolicy::Active {
    8689              :                         // If we don't want to schedule stuff onto the safekeeper, respect that.
    8690            0 :                         return None;
    8691            0 :                     }
    8692            0 :                     let utilization_opt = if let SafekeeperState::Available {
    8693              :                         last_seen_at: _,
    8694            0 :                         utilization,
    8695            0 :                     } = sk.1.availability()
    8696              :                     {
    8697            0 :                         Some(utilization)
    8698              :                     } else {
    8699              :                         // non-available safekeepers still get a chance for new timelines,
    8700              :                         // but put them last in the list.
    8701            0 :                         None
    8702              :                     };
    8703            0 :                     let info = SafekeeperInfo {
    8704            0 :                         hostname: sk.1.skp.host.clone(),
    8705            0 :                         id: NodeId(sk.1.skp.id as u64),
    8706            0 :                     };
    8707            0 :                     Some((utilization_opt, info, sk.1.skp.availability_zone_id.clone()))
    8708            0 :                 })
    8709            0 :                 .collect::<Vec<_>>()
    8710            0 :         };
    8711            0 :         all_safekeepers.sort_by_key(|sk| {
    8712            0 :             (
    8713            0 :                 sk.0.as_ref()
    8714            0 :                     .map(|ut| ut.timeline_count)
    8715            0 :                     .unwrap_or(u64::MAX),
    8716            0 :                 // Use the id to decide on equal scores for reliability
    8717            0 :                 sk.1.id.0,
    8718            0 :             )
    8719            0 :         });
    8720            0 :         let mut sks = Vec::new();
    8721            0 :         let mut azs = HashSet::new();
    8722            0 :         for (_sk_util, sk_info, az_id) in all_safekeepers.iter() {
    8723            0 :             if !azs.insert(az_id) {
    8724            0 :                 continue;
    8725            0 :             }
    8726            0 :             sks.push(sk_info.clone());
    8727            0 :             if sks.len() == 3 {
    8728            0 :                 break;
    8729            0 :             }
    8730              :         }
    8731            0 :         if sks.len() == 3 {
    8732            0 :             Ok(sks)
    8733              :         } else {
    8734            0 :             Err(ApiError::InternalServerError(anyhow::anyhow!(
    8735            0 :                 "couldn't find three safekeepers in different AZs for new timeline"
    8736            0 :             )))
    8737              :         }
    8738            0 :     }
    8739              : 
    8740            0 :     pub(crate) async fn safekeepers_list(
    8741            0 :         &self,
    8742            0 :     ) -> Result<Vec<SafekeeperDescribeResponse>, DatabaseError> {
    8743            0 :         let locked = self.inner.read().unwrap();
    8744            0 :         let mut list = locked
    8745            0 :             .safekeepers
    8746            0 :             .iter()
    8747            0 :             .map(|sk| sk.1.describe_response())
    8748            0 :             .collect::<Result<Vec<_>, _>>()?;
    8749            0 :         list.sort_by_key(|v| v.id);
    8750            0 :         Ok(list)
    8751            0 :     }
    8752              : 
    8753            0 :     pub(crate) async fn get_safekeeper(
    8754            0 :         &self,
    8755            0 :         id: i64,
    8756            0 :     ) -> Result<SafekeeperDescribeResponse, DatabaseError> {
    8757            0 :         let locked = self.inner.read().unwrap();
    8758            0 :         let sk = locked
    8759            0 :             .safekeepers
    8760            0 :             .get(&NodeId(id as u64))
    8761            0 :             .ok_or(diesel::result::Error::NotFound)?;
    8762            0 :         sk.describe_response()
    8763            0 :     }
    8764              : 
    8765            0 :     pub(crate) async fn upsert_safekeeper(
    8766            0 :         &self,
    8767            0 :         record: crate::persistence::SafekeeperUpsert,
    8768            0 :     ) -> Result<(), ApiError> {
    8769            0 :         let node_id = NodeId(record.id as u64);
    8770            0 :         let use_https = self.config.use_https_safekeeper_api;
    8771            0 : 
    8772            0 :         if use_https && record.https_port.is_none() {
    8773            0 :             return Err(ApiError::PreconditionFailed(
    8774            0 :                 format!(
    8775            0 :                     "cannot upsert safekeeper {node_id}: \
    8776            0 :                     https is enabled, but https port is not specified"
    8777            0 :                 )
    8778            0 :                 .into(),
    8779            0 :             ));
    8780            0 :         }
    8781            0 : 
    8782            0 :         self.persistence.safekeeper_upsert(record.clone()).await?;
    8783              :         {
    8784            0 :             let mut locked = self.inner.write().unwrap();
    8785            0 :             let mut safekeepers = (*locked.safekeepers).clone();
    8786            0 :             match safekeepers.entry(node_id) {
    8787            0 :                 std::collections::hash_map::Entry::Occupied(mut entry) => entry
    8788            0 :                     .get_mut()
    8789            0 :                     .update_from_record(record)
    8790            0 :                     .expect("all preconditions should be checked before upsert to database"),
    8791            0 :                 std::collections::hash_map::Entry::Vacant(entry) => {
    8792            0 :                     entry.insert(
    8793            0 :                         Safekeeper::from_persistence(
    8794            0 :                             crate::persistence::SafekeeperPersistence::from_upsert(
    8795            0 :                                 record,
    8796            0 :                                 SkSchedulingPolicy::Pause,
    8797            0 :                             ),
    8798            0 :                             CancellationToken::new(),
    8799            0 :                             use_https,
    8800            0 :                         )
    8801            0 :                         .expect("all preconditions should be checked before upsert to database"),
    8802            0 :                     );
    8803            0 :                 }
    8804              :             }
    8805            0 :             locked.safekeepers = Arc::new(safekeepers);
    8806            0 :         }
    8807            0 :         Ok(())
    8808            0 :     }
    8809              : 
    8810            0 :     pub(crate) async fn set_safekeeper_scheduling_policy(
    8811            0 :         &self,
    8812            0 :         id: i64,
    8813            0 :         scheduling_policy: SkSchedulingPolicy,
    8814            0 :     ) -> Result<(), DatabaseError> {
    8815            0 :         self.persistence
    8816            0 :             .set_safekeeper_scheduling_policy(id, scheduling_policy)
    8817            0 :             .await?;
    8818            0 :         let node_id = NodeId(id as u64);
    8819            0 :         // After the change has been persisted successfully, update the in-memory state
    8820            0 :         {
    8821            0 :             let mut locked = self.inner.write().unwrap();
    8822            0 :             let mut safekeepers = (*locked.safekeepers).clone();
    8823            0 :             let sk = safekeepers
    8824            0 :                 .get_mut(&node_id)
    8825            0 :                 .ok_or(DatabaseError::Logical("Not found".to_string()))?;
    8826            0 :             sk.set_scheduling_policy(scheduling_policy);
    8827            0 : 
    8828            0 :             match scheduling_policy {
    8829            0 :                 SkSchedulingPolicy::Active => (),
    8830            0 :                 SkSchedulingPolicy::Decomissioned | SkSchedulingPolicy::Pause => {
    8831            0 :                     locked.safekeeper_reconcilers.cancel_safekeeper(node_id);
    8832            0 :                 }
    8833              :             }
    8834              : 
    8835            0 :             locked.safekeepers = Arc::new(safekeepers);
    8836            0 :         }
    8837            0 :         Ok(())
    8838            0 :     }
    8839              : 
    8840            0 :     pub(crate) async fn update_shards_preferred_azs(
    8841            0 :         &self,
    8842            0 :         req: ShardsPreferredAzsRequest,
    8843            0 :     ) -> Result<ShardsPreferredAzsResponse, ApiError> {
    8844            0 :         let preferred_azs = req.preferred_az_ids.into_iter().collect::<Vec<_>>();
    8845            0 :         let updated = self
    8846            0 :             .persistence
    8847            0 :             .set_tenant_shard_preferred_azs(preferred_azs)
    8848            0 :             .await
    8849            0 :             .map_err(|err| {
    8850            0 :                 ApiError::InternalServerError(anyhow::anyhow!(
    8851            0 :                     "Failed to persist preferred AZs: {err}"
    8852            0 :                 ))
    8853            0 :             })?;
    8854              : 
    8855            0 :         let mut updated_in_mem_and_db = Vec::default();
    8856            0 : 
    8857            0 :         let mut locked = self.inner.write().unwrap();
    8858            0 :         let state = locked.deref_mut();
    8859            0 :         for (tid, az_id) in updated {
    8860            0 :             let shard = state.tenants.get_mut(&tid);
    8861            0 :             if let Some(shard) = shard {
    8862            0 :                 shard.set_preferred_az(&mut state.scheduler, az_id);
    8863            0 :                 updated_in_mem_and_db.push(tid);
    8864            0 :             }
    8865              :         }
    8866              : 
    8867            0 :         Ok(ShardsPreferredAzsResponse {
    8868            0 :             updated: updated_in_mem_and_db,
    8869            0 :         })
    8870            0 :     }
    8871              : }
        

Generated by: LCOV version 2.1-beta