LCOV - code coverage report
Current view: top level - storage_controller/src - service.rs (source / functions) Coverage Total Hit
Test: 5445d246133daeceb0507e6cc0797ab7c1c70cb8.info Lines: 0.0 % 5506 0
Test Date: 2025-03-12 18:05:02 Functions: 0.0 % 484 0

            Line data    Source code
       1              : pub mod chaos_injector;
       2              : mod context_iterator;
       3              : pub(crate) mod safekeeper_reconciler;
       4              : 
       5              : use std::borrow::Cow;
       6              : use std::cmp::Ordering;
       7              : use std::collections::{BTreeMap, HashMap, HashSet};
       8              : use std::error::Error;
       9              : use std::num::NonZeroU32;
      10              : use std::ops::{Deref, DerefMut};
      11              : use std::path::PathBuf;
      12              : use std::str::FromStr;
      13              : use std::sync::Arc;
      14              : use std::time::{Duration, Instant};
      15              : 
      16              : use anyhow::Context;
      17              : use context_iterator::TenantShardContextIterator;
      18              : use control_plane::storage_controller::{
      19              :     AttachHookRequest, AttachHookResponse, InspectRequest, InspectResponse,
      20              : };
      21              : use diesel::result::DatabaseErrorKind;
      22              : use futures::StreamExt;
      23              : use futures::stream::FuturesUnordered;
      24              : use http_utils::error::ApiError;
      25              : use hyper::Uri;
      26              : use itertools::Itertools;
      27              : use pageserver_api::controller_api::{
      28              :     AvailabilityZone, MetadataHealthRecord, MetadataHealthUpdateRequest, NodeAvailability,
      29              :     NodeRegisterRequest, NodeSchedulingPolicy, NodeShard, NodeShardResponse, PlacementPolicy,
      30              :     SafekeeperDescribeResponse, ShardSchedulingPolicy, ShardsPreferredAzsRequest,
      31              :     ShardsPreferredAzsResponse, SkSchedulingPolicy, TenantCreateRequest, TenantCreateResponse,
      32              :     TenantCreateResponseShard, TenantDescribeResponse, TenantDescribeResponseShard,
      33              :     TenantLocateResponse, TenantPolicyRequest, TenantShardMigrateRequest,
      34              :     TenantShardMigrateResponse,
      35              : };
      36              : use pageserver_api::models::{
      37              :     self, LocationConfig, LocationConfigListResponse, LocationConfigMode, PageserverUtilization,
      38              :     SafekeeperInfo, SafekeepersInfo, SecondaryProgress, ShardParameters, TenantConfig,
      39              :     TenantConfigPatchRequest, TenantConfigRequest, TenantLocationConfigRequest,
      40              :     TenantLocationConfigResponse, TenantShardLocation, TenantShardSplitRequest,
      41              :     TenantShardSplitResponse, TenantSorting, TenantTimeTravelRequest,
      42              :     TimelineArchivalConfigRequest, TimelineCreateRequest, TimelineCreateResponseStorcon,
      43              :     TimelineInfo, TopTenantShardItem, TopTenantShardsRequest,
      44              : };
      45              : use pageserver_api::shard::{
      46              :     ShardCount, ShardIdentity, ShardNumber, ShardStripeSize, TenantShardId,
      47              : };
      48              : use pageserver_api::upcall_api::{
      49              :     ReAttachRequest, ReAttachResponse, ReAttachResponseTenant, ValidateRequest, ValidateResponse,
      50              :     ValidateResponseTenant,
      51              : };
      52              : use pageserver_client::{BlockUnblock, mgmt_api};
      53              : use reqwest::{Certificate, StatusCode};
      54              : use safekeeper_api::membership::{MemberSet, SafekeeperId};
      55              : use safekeeper_api::models::SafekeeperUtilization;
      56              : use safekeeper_reconciler::{SafekeeperReconcilers, ScheduleRequest};
      57              : use tokio::sync::TryAcquireError;
      58              : use tokio::sync::mpsc::error::TrySendError;
      59              : use tokio::task::JoinSet;
      60              : use tokio_util::sync::CancellationToken;
      61              : use tracing::{Instrument, debug, error, info, info_span, instrument, warn};
      62              : use utils::completion::Barrier;
      63              : use utils::generation::Generation;
      64              : use utils::id::{NodeId, TenantId, TimelineId};
      65              : use utils::logging::SecretString;
      66              : use utils::sync::gate::Gate;
      67              : use utils::{failpoint_support, pausable_failpoint};
      68              : 
      69              : use crate::background_node_operations::{
      70              :     Drain, Fill, MAX_RECONCILES_PER_OPERATION, Operation, OperationError, OperationHandler,
      71              : };
      72              : use crate::compute_hook::{self, ComputeHook, NotifyError};
      73              : use crate::drain_utils::{self, TenantShardDrain, TenantShardIterator};
      74              : use crate::heartbeater::{Heartbeater, PageserverState, SafekeeperState};
      75              : use crate::id_lock_map::{
      76              :     IdLockMap, TracingExclusiveGuard, trace_exclusive_lock, trace_shared_lock,
      77              : };
      78              : use crate::leadership::Leadership;
      79              : use crate::metrics;
      80              : use crate::node::{AvailabilityTransition, Node};
      81              : use crate::pageserver_client::PageserverClient;
      82              : use crate::peer_client::GlobalObservedState;
      83              : use crate::persistence::split_state::SplitState;
      84              : use crate::persistence::{
      85              :     AbortShardSplitStatus, ControllerPersistence, DatabaseError, DatabaseResult,
      86              :     MetadataHealthPersistence, Persistence, SafekeeperTimelineOpKind, ShardGenerationState,
      87              :     TenantFilter, TenantShardPersistence, TimelinePendingOpPersistence, TimelinePersistence,
      88              : };
      89              : use crate::reconciler::{
      90              :     ReconcileError, ReconcileUnits, ReconcilerConfig, ReconcilerConfigBuilder, ReconcilerPriority,
      91              :     attached_location_conf,
      92              : };
      93              : use crate::safekeeper::Safekeeper;
      94              : use crate::scheduler::{
      95              :     AttachedShardTag, MaySchedule, ScheduleContext, ScheduleError, ScheduleMode, Scheduler,
      96              : };
      97              : use crate::tenant_shard::{
      98              :     IntentState, MigrateAttachment, ObservedState, ObservedStateDelta, ObservedStateLocation,
      99              :     ReconcileNeeded, ReconcileResult, ReconcileWaitError, ReconcilerStatus, ReconcilerWaiter,
     100              :     ScheduleOptimization, ScheduleOptimizationAction, TenantShard,
     101              : };
     102              : 
     103              : const WAITER_FILL_DRAIN_POLL_TIMEOUT: Duration = Duration::from_millis(500);
     104              : 
     105              : // For operations that should be quick, like attaching a new tenant
     106              : const SHORT_RECONCILE_TIMEOUT: Duration = Duration::from_secs(5);
     107              : 
     108              : // For operations that might be slow, like migrating a tenant with
     109              : // some data in it.
     110              : pub const RECONCILE_TIMEOUT: Duration = Duration::from_secs(30);
     111              : 
     112              : // If we receive a call using Secondary mode initially, it will omit generation.  We will initialize
     113              : // tenant shards into this generation, and as long as it remains in this generation, we will accept
     114              : // input generation from future requests as authoritative.
     115              : const INITIAL_GENERATION: Generation = Generation::new(0);
     116              : 
     117              : /// How long [`Service::startup_reconcile`] is allowed to take before it should give
     118              : /// up on unresponsive pageservers and proceed.
     119              : pub(crate) const STARTUP_RECONCILE_TIMEOUT: Duration = Duration::from_secs(30);
     120              : 
     121              : /// How long a node may be unresponsive to heartbeats before we declare it offline.
     122              : /// This must be long enough to cover node restarts as well as normal operations: in future
     123              : pub const MAX_OFFLINE_INTERVAL_DEFAULT: Duration = Duration::from_secs(30);
     124              : 
     125              : /// How long a node may be unresponsive to heartbeats during start up before we declare it
     126              : /// offline.
     127              : ///
     128              : /// This is much more lenient than [`MAX_OFFLINE_INTERVAL_DEFAULT`] since the pageserver's
     129              : /// handling of the re-attach response may take a long time and blocks heartbeats from
     130              : /// being handled on the pageserver side.
     131              : pub const MAX_WARMING_UP_INTERVAL_DEFAULT: Duration = Duration::from_secs(300);
     132              : 
     133              : /// How often to send heartbeats to registered nodes?
     134              : pub const HEARTBEAT_INTERVAL_DEFAULT: Duration = Duration::from_secs(5);
     135              : 
     136              : /// How long is too long for a reconciliation?
     137              : pub const LONG_RECONCILE_THRESHOLD_DEFAULT: Duration = Duration::from_secs(120);
     138              : 
     139              : #[derive(Clone, strum_macros::Display)]
     140              : enum TenantOperations {
     141              :     Create,
     142              :     LocationConfig,
     143              :     ConfigSet,
     144              :     ConfigPatch,
     145              :     TimeTravelRemoteStorage,
     146              :     Delete,
     147              :     UpdatePolicy,
     148              :     ShardSplit,
     149              :     SecondaryDownload,
     150              :     TimelineCreate,
     151              :     TimelineDelete,
     152              :     AttachHook,
     153              :     TimelineArchivalConfig,
     154              :     TimelineDetachAncestor,
     155              :     TimelineGcBlockUnblock,
     156              :     DropDetached,
     157              :     DownloadHeatmapLayers,
     158              : }
     159              : 
     160              : #[derive(Clone, strum_macros::Display)]
     161              : enum NodeOperations {
     162              :     Register,
     163              :     Configure,
     164              :     Delete,
     165              : }
     166              : 
     167              : /// The leadership status for the storage controller process.
     168              : /// Allowed transitions are:
     169              : /// 1. Leader -> SteppedDown
     170              : /// 2. Candidate -> Leader
     171              : #[derive(
     172              :     Eq,
     173              :     PartialEq,
     174              :     Copy,
     175              :     Clone,
     176              :     strum_macros::Display,
     177            0 :     strum_macros::EnumIter,
     178              :     measured::FixedCardinalityLabel,
     179              : )]
     180              : #[strum(serialize_all = "snake_case")]
     181              : pub(crate) enum LeadershipStatus {
     182              :     /// This is the steady state where the storage controller can produce
     183              :     /// side effects in the cluster.
     184              :     Leader,
     185              :     /// We've been notified to step down by another candidate. No reconciliations
     186              :     /// take place in this state.
     187              :     SteppedDown,
     188              :     /// Initial state for a new storage controller instance. Will attempt to assume leadership.
     189              :     #[allow(unused)]
     190              :     Candidate,
     191              : }
     192              : 
     193              : pub const RECONCILER_CONCURRENCY_DEFAULT: usize = 128;
     194              : pub const PRIORITY_RECONCILER_CONCURRENCY_DEFAULT: usize = 256;
     195              : 
     196              : // Depth of the channel used to enqueue shards for reconciliation when they can't do it immediately.
     197              : // This channel is finite-size to avoid using excessive memory if we get into a state where reconciles are finishing more slowly
     198              : // than they're being pushed onto the queue.
     199              : const MAX_DELAYED_RECONCILES: usize = 10000;
     200              : 
     201              : // Top level state available to all HTTP handlers
     202              : struct ServiceState {
     203              :     leadership_status: LeadershipStatus,
     204              : 
     205              :     tenants: BTreeMap<TenantShardId, TenantShard>,
     206              : 
     207              :     nodes: Arc<HashMap<NodeId, Node>>,
     208              : 
     209              :     safekeepers: Arc<HashMap<NodeId, Safekeeper>>,
     210              : 
     211              :     safekeeper_reconcilers: SafekeeperReconcilers,
     212              : 
     213              :     scheduler: Scheduler,
     214              : 
     215              :     /// Ongoing background operation on the cluster if any is running.
     216              :     /// Note that only one such operation may run at any given time,
     217              :     /// hence the type choice.
     218              :     ongoing_operation: Option<OperationHandler>,
     219              : 
     220              :     /// Queue of tenants who are waiting for concurrency limits to permit them to reconcile
     221              :     delayed_reconcile_rx: tokio::sync::mpsc::Receiver<TenantShardId>,
     222              : }
     223              : 
     224              : /// Transform an error from a pageserver into an error to return to callers of a storage
     225              : /// controller API.
     226            0 : fn passthrough_api_error(node: &Node, e: mgmt_api::Error) -> ApiError {
     227            0 :     match e {
     228            0 :         mgmt_api::Error::SendRequest(e) => {
     229            0 :             // Presume errors sending requests are connectivity/availability issues
     230            0 :             ApiError::ResourceUnavailable(format!("{node} error sending request: {e}").into())
     231              :         }
     232            0 :         mgmt_api::Error::ReceiveErrorBody(str) => {
     233            0 :             // Presume errors receiving body are connectivity/availability issues
     234            0 :             ApiError::ResourceUnavailable(
     235            0 :                 format!("{node} error receiving error body: {str}").into(),
     236            0 :             )
     237              :         }
     238            0 :         mgmt_api::Error::ReceiveBody(err) if err.is_decode() => {
     239            0 :             // Return 500 for decoding errors.
     240            0 :             ApiError::InternalServerError(anyhow::Error::from(err).context("error decoding body"))
     241              :         }
     242            0 :         mgmt_api::Error::ReceiveBody(err) => {
     243            0 :             // Presume errors receiving body are connectivity/availability issues except for decoding errors
     244            0 :             let src_str = err.source().map(|e| e.to_string()).unwrap_or_default();
     245            0 :             ApiError::ResourceUnavailable(
     246            0 :                 format!("{node} error receiving error body: {err} {}", src_str).into(),
     247            0 :             )
     248              :         }
     249            0 :         mgmt_api::Error::ApiError(StatusCode::NOT_FOUND, msg) => {
     250            0 :             ApiError::NotFound(anyhow::anyhow!(format!("{node}: {msg}")).into())
     251              :         }
     252            0 :         mgmt_api::Error::ApiError(StatusCode::SERVICE_UNAVAILABLE, msg) => {
     253            0 :             ApiError::ResourceUnavailable(format!("{node}: {msg}").into())
     254              :         }
     255            0 :         mgmt_api::Error::ApiError(status @ StatusCode::UNAUTHORIZED, msg)
     256            0 :         | mgmt_api::Error::ApiError(status @ StatusCode::FORBIDDEN, msg) => {
     257              :             // Auth errors talking to a pageserver are not auth errors for the caller: they are
     258              :             // internal server errors, showing that something is wrong with the pageserver or
     259              :             // storage controller's auth configuration.
     260            0 :             ApiError::InternalServerError(anyhow::anyhow!("{node} {status}: {msg}"))
     261              :         }
     262            0 :         mgmt_api::Error::ApiError(status @ StatusCode::TOO_MANY_REQUESTS, msg) => {
     263            0 :             // Pass through 429 errors: if pageserver is asking us to wait + retry, we in
     264            0 :             // turn ask our clients to wait + retry
     265            0 :             ApiError::Conflict(format!("{node} {status}: {status} {msg}"))
     266              :         }
     267            0 :         mgmt_api::Error::ApiError(status, msg) => {
     268            0 :             // Presume general case of pageserver API errors is that we tried to do something
     269            0 :             // that can't be done right now.
     270            0 :             ApiError::Conflict(format!("{node} {status}: {status} {msg}"))
     271              :         }
     272            0 :         mgmt_api::Error::Cancelled => ApiError::ShuttingDown,
     273            0 :         mgmt_api::Error::CreateClient(e) => ApiError::InternalServerError(anyhow::anyhow!(e)),
     274              :     }
     275            0 : }
     276              : 
     277              : impl ServiceState {
     278            0 :     fn new(
     279            0 :         nodes: HashMap<NodeId, Node>,
     280            0 :         safekeepers: HashMap<NodeId, Safekeeper>,
     281            0 :         tenants: BTreeMap<TenantShardId, TenantShard>,
     282            0 :         scheduler: Scheduler,
     283            0 :         delayed_reconcile_rx: tokio::sync::mpsc::Receiver<TenantShardId>,
     284            0 :         initial_leadership_status: LeadershipStatus,
     285            0 :         reconcilers_cancel: CancellationToken,
     286            0 :     ) -> Self {
     287            0 :         metrics::update_leadership_status(initial_leadership_status);
     288            0 : 
     289            0 :         Self {
     290            0 :             leadership_status: initial_leadership_status,
     291            0 :             tenants,
     292            0 :             nodes: Arc::new(nodes),
     293            0 :             safekeepers: Arc::new(safekeepers),
     294            0 :             safekeeper_reconcilers: SafekeeperReconcilers::new(reconcilers_cancel),
     295            0 :             scheduler,
     296            0 :             ongoing_operation: None,
     297            0 :             delayed_reconcile_rx,
     298            0 :         }
     299            0 :     }
     300              : 
     301            0 :     fn parts_mut(
     302            0 :         &mut self,
     303            0 :     ) -> (
     304            0 :         &mut Arc<HashMap<NodeId, Node>>,
     305            0 :         &mut BTreeMap<TenantShardId, TenantShard>,
     306            0 :         &mut Scheduler,
     307            0 :     ) {
     308            0 :         (&mut self.nodes, &mut self.tenants, &mut self.scheduler)
     309            0 :     }
     310              : 
     311              :     #[allow(clippy::type_complexity)]
     312            0 :     fn parts_mut_sk(
     313            0 :         &mut self,
     314            0 :     ) -> (
     315            0 :         &mut Arc<HashMap<NodeId, Node>>,
     316            0 :         &mut Arc<HashMap<NodeId, Safekeeper>>,
     317            0 :         &mut BTreeMap<TenantShardId, TenantShard>,
     318            0 :         &mut Scheduler,
     319            0 :     ) {
     320            0 :         (
     321            0 :             &mut self.nodes,
     322            0 :             &mut self.safekeepers,
     323            0 :             &mut self.tenants,
     324            0 :             &mut self.scheduler,
     325            0 :         )
     326            0 :     }
     327              : 
     328            0 :     fn get_leadership_status(&self) -> LeadershipStatus {
     329            0 :         self.leadership_status
     330            0 :     }
     331              : 
     332            0 :     fn step_down(&mut self) {
     333            0 :         self.leadership_status = LeadershipStatus::SteppedDown;
     334            0 :         metrics::update_leadership_status(self.leadership_status);
     335            0 :     }
     336              : 
     337            0 :     fn become_leader(&mut self) {
     338            0 :         self.leadership_status = LeadershipStatus::Leader;
     339            0 :         metrics::update_leadership_status(self.leadership_status);
     340            0 :     }
     341              : }
     342              : 
     343              : #[derive(Clone)]
     344              : pub struct Config {
     345              :     // All pageservers managed by one instance of this service must have
     346              :     // the same public key.  This JWT token will be used to authenticate
     347              :     // this service to the pageservers it manages.
     348              :     pub pageserver_jwt_token: Option<String>,
     349              : 
     350              :     // All safekeepers managed by one instance of this service must have
     351              :     // the same public key. This JWT token will be used to authenticate
     352              :     // this service to the safekeepers it manages.
     353              :     pub safekeeper_jwt_token: Option<String>,
     354              : 
     355              :     // This JWT token will be used to authenticate this service to the control plane.
     356              :     pub control_plane_jwt_token: Option<String>,
     357              : 
     358              :     // This JWT token will be used to authenticate with other storage controller instances
     359              :     pub peer_jwt_token: Option<String>,
     360              : 
     361              :     /// Where the compute hook should send notifications of pageserver attachment locations
     362              :     /// (this URL points to the control plane in prod). If this is None, the compute hook will
     363              :     /// assume it is running in a test environment and try to update neon_local.
     364              :     pub compute_hook_url: Option<String>,
     365              : 
     366              :     /// Prefix for storage API endpoints of the control plane. We use this prefix to compute
     367              :     /// URLs that we use to send pageserver and safekeeper attachment locations.
     368              :     /// If this is None, the compute hook will assume it is running in a test environment
     369              :     /// and try to invoke neon_local instead.
     370              :     ///
     371              :     /// For now, there is also `compute_hook_url` which allows configuration of the pageserver
     372              :     /// specific endpoint, but it is in the process of being phased out.
     373              :     pub control_plane_url: Option<String>,
     374              : 
     375              :     /// Grace period within which a pageserver does not respond to heartbeats, but is still
     376              :     /// considered active. Once the grace period elapses, the next heartbeat failure will
     377              :     /// mark the pagseserver offline.
     378              :     pub max_offline_interval: Duration,
     379              : 
     380              :     /// Extended grace period within which pageserver may not respond to heartbeats.
     381              :     /// This extended grace period kicks in after the node has been drained for restart
     382              :     /// and/or upon handling the re-attach request from a node.
     383              :     pub max_warming_up_interval: Duration,
     384              : 
     385              :     /// How many normal-priority Reconcilers may be spawned concurrently
     386              :     pub reconciler_concurrency: usize,
     387              : 
     388              :     /// How many high-priority Reconcilers may be spawned concurrently
     389              :     pub priority_reconciler_concurrency: usize,
     390              : 
     391              :     /// How many API requests per second to allow per tenant, across all
     392              :     /// tenant-scoped API endpoints. Further API requests queue until ready.
     393              :     pub tenant_rate_limit: NonZeroU32,
     394              : 
     395              :     /// The size at which an unsharded tenant should be split (into 8 shards). This uses the logical
     396              :     /// size of the largest timeline in the shard (i.e. max_logical_size).
     397              :     ///
     398              :     /// None or 0 disables auto-splitting.
     399              :     ///
     400              :     /// TODO: consider using total logical size of all timelines instead.
     401              :     pub split_threshold: Option<u64>,
     402              : 
     403              :     // TODO: make this cfg(feature  = "testing")
     404              :     pub neon_local_repo_dir: Option<PathBuf>,
     405              : 
     406              :     // Maximum acceptable download lag for the secondary location
     407              :     // while draining a node. If the secondary location is lagging
     408              :     // by more than the configured amount, then the secondary is not
     409              :     // upgraded to primary.
     410              :     pub max_secondary_lag_bytes: Option<u64>,
     411              : 
     412              :     pub heartbeat_interval: Duration,
     413              : 
     414              :     pub address_for_peers: Option<Uri>,
     415              : 
     416              :     pub start_as_candidate: bool,
     417              : 
     418              :     pub http_service_port: i32,
     419              : 
     420              :     pub long_reconcile_threshold: Duration,
     421              : 
     422              :     pub use_https_pageserver_api: bool,
     423              : 
     424              :     pub use_https_safekeeper_api: bool,
     425              : 
     426              :     pub ssl_ca_cert: Option<Certificate>,
     427              : 
     428              :     pub timelines_onto_safekeepers: bool,
     429              : }
     430              : 
     431              : impl From<DatabaseError> for ApiError {
     432            0 :     fn from(err: DatabaseError) -> ApiError {
     433            0 :         match err {
     434            0 :             DatabaseError::Query(e) => ApiError::InternalServerError(e.into()),
     435              :             // FIXME: ApiError doesn't have an Unavailable variant, but ShuttingDown maps to 503.
     436              :             DatabaseError::Connection(_) | DatabaseError::ConnectionPool(_) => {
     437            0 :                 ApiError::ShuttingDown
     438              :             }
     439            0 :             DatabaseError::Logical(reason) | DatabaseError::Migration(reason) => {
     440            0 :                 ApiError::InternalServerError(anyhow::anyhow!(reason))
     441              :             }
     442              :         }
     443            0 :     }
     444              : }
     445              : 
     446              : enum InitialShardScheduleOutcome {
     447              :     Scheduled(TenantCreateResponseShard),
     448              :     NotScheduled,
     449              :     ShardScheduleError(ScheduleError),
     450              : }
     451              : 
     452              : pub struct Service {
     453              :     inner: Arc<std::sync::RwLock<ServiceState>>,
     454              :     config: Config,
     455              :     persistence: Arc<Persistence>,
     456              :     compute_hook: Arc<ComputeHook>,
     457              :     result_tx: tokio::sync::mpsc::UnboundedSender<ReconcileResultRequest>,
     458              : 
     459              :     heartbeater_ps: Heartbeater<Node, PageserverState>,
     460              :     heartbeater_sk: Heartbeater<Safekeeper, SafekeeperState>,
     461              : 
     462              :     // Channel for background cleanup from failed operations that require cleanup, such as shard split
     463              :     abort_tx: tokio::sync::mpsc::UnboundedSender<TenantShardSplitAbort>,
     464              : 
     465              :     // Locking on a tenant granularity (covers all shards in the tenant):
     466              :     // - Take exclusively for rare operations that mutate the tenant's persistent state (e.g. create/delete/split)
     467              :     // - Take in shared mode for operations that need the set of shards to stay the same to complete reliably (e.g. timeline CRUD)
     468              :     tenant_op_locks: IdLockMap<TenantId, TenantOperations>,
     469              : 
     470              :     // Locking for node-mutating operations: take exclusively for operations that modify the node's persistent state, or
     471              :     // that transition it to/from Active.
     472              :     node_op_locks: IdLockMap<NodeId, NodeOperations>,
     473              : 
     474              :     // Limit how many Reconcilers we will spawn concurrently for normal-priority tasks such as background reconciliations
     475              :     // and reconciliation on startup.
     476              :     reconciler_concurrency: Arc<tokio::sync::Semaphore>,
     477              : 
     478              :     // Limit how many Reconcilers we will spawn concurrently for high-priority tasks such as tenant/timeline CRUD, which
     479              :     // a human user might be waiting for.
     480              :     priority_reconciler_concurrency: Arc<tokio::sync::Semaphore>,
     481              : 
     482              :     /// Queue of tenants who are waiting for concurrency limits to permit them to reconcile
     483              :     /// Send into this queue to promptly attempt to reconcile this shard next time units are available.
     484              :     ///
     485              :     /// Note that this state logically lives inside ServiceState, but carrying Sender here makes the code simpler
     486              :     /// by avoiding needing a &mut ref to something inside the ServiceState.  This could be optimized to
     487              :     /// use a VecDeque instead of a channel to reduce synchronization overhead, at the cost of some code complexity.
     488              :     delayed_reconcile_tx: tokio::sync::mpsc::Sender<TenantShardId>,
     489              : 
     490              :     // Process shutdown will fire this token
     491              :     cancel: CancellationToken,
     492              : 
     493              :     // Child token of [`Service::cancel`] used by reconcilers
     494              :     reconcilers_cancel: CancellationToken,
     495              : 
     496              :     // Background tasks will hold this gate
     497              :     gate: Gate,
     498              : 
     499              :     // Reconcilers background tasks will hold this gate
     500              :     reconcilers_gate: Gate,
     501              : 
     502              :     /// This waits for initial reconciliation with pageservers to complete.  Until this barrier
     503              :     /// passes, it isn't safe to do any actions that mutate tenants.
     504              :     pub(crate) startup_complete: Barrier,
     505              : }
     506              : 
     507              : impl From<ReconcileWaitError> for ApiError {
     508            0 :     fn from(value: ReconcileWaitError) -> Self {
     509            0 :         match value {
     510            0 :             ReconcileWaitError::Shutdown => ApiError::ShuttingDown,
     511            0 :             e @ ReconcileWaitError::Timeout(_) => ApiError::Timeout(format!("{e}").into()),
     512            0 :             e @ ReconcileWaitError::Failed(..) => ApiError::InternalServerError(anyhow::anyhow!(e)),
     513              :         }
     514            0 :     }
     515              : }
     516              : 
     517              : impl From<OperationError> for ApiError {
     518            0 :     fn from(value: OperationError) -> Self {
     519            0 :         match value {
     520            0 :             OperationError::NodeStateChanged(err) | OperationError::FinalizeError(err) => {
     521            0 :                 ApiError::InternalServerError(anyhow::anyhow!(err))
     522              :             }
     523            0 :             OperationError::Cancelled => ApiError::Conflict("Operation was cancelled".into()),
     524              :         }
     525            0 :     }
     526              : }
     527              : 
     528              : #[allow(clippy::large_enum_variant)]
     529              : enum TenantCreateOrUpdate {
     530              :     Create(TenantCreateRequest),
     531              :     Update(Vec<ShardUpdate>),
     532              : }
     533              : 
     534              : struct ShardSplitParams {
     535              :     old_shard_count: ShardCount,
     536              :     new_shard_count: ShardCount,
     537              :     new_stripe_size: Option<ShardStripeSize>,
     538              :     targets: Vec<ShardSplitTarget>,
     539              :     policy: PlacementPolicy,
     540              :     config: TenantConfig,
     541              :     shard_ident: ShardIdentity,
     542              :     preferred_az_id: Option<AvailabilityZone>,
     543              : }
     544              : 
     545              : // When preparing for a shard split, we may either choose to proceed with the split,
     546              : // or find that the work is already done and return NoOp.
     547              : enum ShardSplitAction {
     548              :     Split(Box<ShardSplitParams>),
     549              :     NoOp(TenantShardSplitResponse),
     550              : }
     551              : 
     552              : // A parent shard which will be split
     553              : struct ShardSplitTarget {
     554              :     parent_id: TenantShardId,
     555              :     node: Node,
     556              :     child_ids: Vec<TenantShardId>,
     557              : }
     558              : 
     559              : /// When we tenant shard split operation fails, we may not be able to clean up immediately, because nodes
     560              : /// might not be available.  We therefore use a queue of abort operations processed in the background.
     561              : struct TenantShardSplitAbort {
     562              :     tenant_id: TenantId,
     563              :     /// The target values from the request that failed
     564              :     new_shard_count: ShardCount,
     565              :     new_stripe_size: Option<ShardStripeSize>,
     566              :     /// Until this abort op is complete, no other operations may be done on the tenant
     567              :     _tenant_lock: TracingExclusiveGuard<TenantOperations>,
     568              : }
     569              : 
     570              : #[derive(thiserror::Error, Debug)]
     571              : enum TenantShardSplitAbortError {
     572              :     #[error(transparent)]
     573              :     Database(#[from] DatabaseError),
     574              :     #[error(transparent)]
     575              :     Remote(#[from] mgmt_api::Error),
     576              :     #[error("Unavailable")]
     577              :     Unavailable,
     578              : }
     579              : 
     580              : struct ShardUpdate {
     581              :     tenant_shard_id: TenantShardId,
     582              :     placement_policy: PlacementPolicy,
     583              :     tenant_config: TenantConfig,
     584              : 
     585              :     /// If this is None, generation is not updated.
     586              :     generation: Option<Generation>,
     587              : 
     588              :     /// If this is None, scheduling policy is not updated.
     589              :     scheduling_policy: Option<ShardSchedulingPolicy>,
     590              : }
     591              : 
     592              : enum StopReconciliationsReason {
     593              :     ShuttingDown,
     594              :     SteppingDown,
     595              : }
     596              : 
     597              : impl std::fmt::Display for StopReconciliationsReason {
     598            0 :     fn fmt(&self, writer: &mut std::fmt::Formatter) -> std::fmt::Result {
     599            0 :         let s = match self {
     600            0 :             Self::ShuttingDown => "Shutting down",
     601            0 :             Self::SteppingDown => "Stepping down",
     602              :         };
     603            0 :         write!(writer, "{}", s)
     604            0 :     }
     605              : }
     606              : 
     607              : pub(crate) enum ReconcileResultRequest {
     608              :     ReconcileResult(ReconcileResult),
     609              :     Stop,
     610              : }
     611              : 
     612              : #[derive(Clone)]
     613              : struct MutationLocation {
     614              :     node: Node,
     615              :     generation: Generation,
     616              : }
     617              : 
     618              : #[derive(Clone)]
     619              : struct ShardMutationLocations {
     620              :     latest: MutationLocation,
     621              :     other: Vec<MutationLocation>,
     622              : }
     623              : 
     624              : #[derive(Default, Clone)]
     625              : struct TenantMutationLocations(BTreeMap<TenantShardId, ShardMutationLocations>);
     626              : 
     627              : impl Service {
     628            0 :     pub fn get_config(&self) -> &Config {
     629            0 :         &self.config
     630            0 :     }
     631              : 
     632              :     /// Called once on startup, this function attempts to contact all pageservers to build an up-to-date
     633              :     /// view of the world, and determine which pageservers are responsive.
     634              :     #[instrument(skip_all)]
     635              :     async fn startup_reconcile(
     636              :         self: &Arc<Service>,
     637              :         current_leader: Option<ControllerPersistence>,
     638              :         leader_step_down_state: Option<GlobalObservedState>,
     639              :         bg_compute_notify_result_tx: tokio::sync::mpsc::Sender<
     640              :             Result<(), (TenantShardId, NotifyError)>,
     641              :         >,
     642              :     ) {
     643              :         // Startup reconciliation does I/O to other services: whether they
     644              :         // are responsive or not, we should aim to finish within our deadline, because:
     645              :         // - If we don't, a k8s readiness hook watching /ready will kill us.
     646              :         // - While we're waiting for startup reconciliation, we are not fully
     647              :         //   available for end user operations like creating/deleting tenants and timelines.
     648              :         //
     649              :         // We set multiple deadlines to break up the time available between the phases of work: this is
     650              :         // arbitrary, but avoids a situation where the first phase could burn our entire timeout period.
     651              :         let start_at = Instant::now();
     652              :         let node_scan_deadline = start_at
     653              :             .checked_add(STARTUP_RECONCILE_TIMEOUT / 2)
     654              :             .expect("Reconcile timeout is a modest constant");
     655              : 
     656              :         let observed = if let Some(state) = leader_step_down_state {
     657              :             tracing::info!(
     658              :                 "Using observed state received from leader at {}",
     659              :                 current_leader.as_ref().unwrap().address
     660              :             );
     661              : 
     662              :             state
     663              :         } else {
     664              :             self.build_global_observed_state(node_scan_deadline).await
     665              :         };
     666              : 
     667              :         // Accumulate a list of any tenant locations that ought to be detached
     668              :         let mut cleanup = Vec::new();
     669              : 
     670              :         // Send initial heartbeat requests to all nodes loaded from the database
     671              :         let all_nodes = {
     672              :             let locked = self.inner.read().unwrap();
     673              :             locked.nodes.clone()
     674              :         };
     675              :         let (mut nodes_online, mut sks_online) =
     676              :             self.initial_heartbeat_round(all_nodes.keys()).await;
     677              : 
     678              :         // List of tenants for which we will attempt to notify compute of their location at startup
     679              :         let mut compute_notifications = Vec::new();
     680              : 
     681              :         // Populate intent and observed states for all tenants, based on reported state on pageservers
     682              :         tracing::info!("Populating tenant shards' states from initial pageserver scan...");
     683              :         let shard_count = {
     684              :             let mut locked = self.inner.write().unwrap();
     685              :             let (nodes, safekeepers, tenants, scheduler) = locked.parts_mut_sk();
     686              : 
     687              :             // Mark nodes online if they responded to us: nodes are offline by default after a restart.
     688              :             let mut new_nodes = (**nodes).clone();
     689              :             for (node_id, node) in new_nodes.iter_mut() {
     690              :                 if let Some(utilization) = nodes_online.remove(node_id) {
     691              :                     node.set_availability(NodeAvailability::Active(utilization));
     692              :                     scheduler.node_upsert(node);
     693              :                 }
     694              :             }
     695              :             *nodes = Arc::new(new_nodes);
     696              : 
     697              :             let mut new_sks = (**safekeepers).clone();
     698              :             for (node_id, node) in new_sks.iter_mut() {
     699              :                 if let Some((utilization, last_seen_at)) = sks_online.remove(node_id) {
     700              :                     node.set_availability(SafekeeperState::Available {
     701              :                         utilization,
     702              :                         last_seen_at,
     703              :                     });
     704              :                 }
     705              :             }
     706              :             *safekeepers = Arc::new(new_sks);
     707              : 
     708              :             for (tenant_shard_id, observed_state) in observed.0 {
     709              :                 let Some(tenant_shard) = tenants.get_mut(&tenant_shard_id) else {
     710              :                     for node_id in observed_state.locations.keys() {
     711              :                         cleanup.push((tenant_shard_id, *node_id));
     712              :                     }
     713              : 
     714              :                     continue;
     715              :                 };
     716              : 
     717              :                 tenant_shard.observed = observed_state;
     718              :             }
     719              : 
     720              :             // Populate each tenant's intent state
     721              :             let mut schedule_context = ScheduleContext::default();
     722              :             for (tenant_shard_id, tenant_shard) in tenants.iter_mut() {
     723              :                 if tenant_shard_id.shard_number == ShardNumber(0) {
     724              :                     // Reset scheduling context each time we advance to the next Tenant
     725              :                     schedule_context = ScheduleContext::default();
     726              :                 }
     727              : 
     728              :                 tenant_shard.intent_from_observed(scheduler);
     729              :                 if let Err(e) = tenant_shard.schedule(scheduler, &mut schedule_context) {
     730              :                     // Non-fatal error: we are unable to properly schedule the tenant, perhaps because
     731              :                     // not enough pageservers are available.  The tenant may well still be available
     732              :                     // to clients.
     733              :                     tracing::error!("Failed to schedule tenant {tenant_shard_id} at startup: {e}");
     734              :                 } else {
     735              :                     // If we're both intending and observed to be attached at a particular node, we will
     736              :                     // emit a compute notification for this. In the case where our observed state does not
     737              :                     // yet match our intent, we will eventually reconcile, and that will emit a compute notification.
     738              :                     if let Some(attached_at) = tenant_shard.stably_attached() {
     739              :                         compute_notifications.push(compute_hook::ShardUpdate {
     740              :                             tenant_shard_id: *tenant_shard_id,
     741              :                             node_id: attached_at,
     742              :                             stripe_size: tenant_shard.shard.stripe_size,
     743              :                             preferred_az: tenant_shard
     744              :                                 .preferred_az()
     745            0 :                                 .map(|az| Cow::Owned(az.clone())),
     746              :                         });
     747              :                     }
     748              :                 }
     749              :             }
     750              : 
     751              :             tenants.len()
     752              :         };
     753              : 
     754              :         // Before making any obeservable changes to the cluster, persist self
     755              :         // as leader in database and memory.
     756              :         let leadership = Leadership::new(
     757              :             self.persistence.clone(),
     758              :             self.config.clone(),
     759              :             self.cancel.child_token(),
     760              :         );
     761              : 
     762              :         if let Err(e) = leadership.become_leader(current_leader).await {
     763              :             tracing::error!("Failed to persist self as leader: {e}. Aborting start-up ...");
     764              :             std::process::exit(1);
     765              :         }
     766              : 
     767              :         let safekeepers = self.inner.read().unwrap().safekeepers.clone();
     768              :         let sk_schedule_requests =
     769              :             match safekeeper_reconciler::load_schedule_requests(self, &safekeepers).await {
     770              :                 Ok(v) => v,
     771              :                 Err(e) => {
     772              :                     tracing::warn!(
     773              :                         "Failed to load safekeeper pending ops at startup: {e}." // Don't abort for now: " Aborting start-up..."
     774              :                     );
     775              :                     // std::process::exit(1);
     776              :                     Vec::new()
     777              :                 }
     778              :             };
     779              : 
     780              :         {
     781              :             let mut locked = self.inner.write().unwrap();
     782              :             locked.become_leader();
     783              : 
     784              :             locked
     785              :                 .safekeeper_reconcilers
     786              :                 .schedule_request_vec(self, sk_schedule_requests);
     787              :         }
     788              : 
     789              :         // TODO: if any tenant's intent now differs from its loaded generation_pageserver, we should clear that
     790              :         // generation_pageserver in the database.
     791              : 
     792              :         // Emit compute hook notifications for all tenants which are already stably attached.  Other tenants
     793              :         // will emit compute hook notifications when they reconcile.
     794              :         //
     795              :         // Ordering: our calls to notify_background synchronously establish a relative order for these notifications vs. any later
     796              :         // calls into the ComputeHook for the same tenant: we can leave these to run to completion in the background and any later
     797              :         // calls will be correctly ordered wrt these.
     798              :         //
     799              :         // Concurrency: we call notify_background for all tenants, which will create O(N) tokio tasks, but almost all of them
     800              :         // will just wait on the ComputeHook::API_CONCURRENCY semaphore immediately, so very cheap until they get that semaphore
     801              :         // unit and start doing I/O.
     802              :         tracing::info!(
     803              :             "Sending {} compute notifications",
     804              :             compute_notifications.len()
     805              :         );
     806              :         self.compute_hook.notify_background(
     807              :             compute_notifications,
     808              :             bg_compute_notify_result_tx.clone(),
     809              :             &self.cancel,
     810              :         );
     811              : 
     812              :         // Finally, now that the service is up and running, launch reconcile operations for any tenants
     813              :         // which require it: under normal circumstances this should only include tenants that were in some
     814              :         // transient state before we restarted, or any tenants whose compute hooks failed above.
     815              :         tracing::info!("Checking for shards in need of reconciliation...");
     816              :         let reconcile_tasks = self.reconcile_all();
     817              :         // We will not wait for these reconciliation tasks to run here: we're now done with startup and
     818              :         // normal operations may proceed.
     819              : 
     820              :         // Clean up any tenants that were found on pageservers but are not known to us.  Do this in the
     821              :         // background because it does not need to complete in order to proceed with other work.
     822              :         if !cleanup.is_empty() {
     823              :             tracing::info!("Cleaning up {} locations in the background", cleanup.len());
     824              :             tokio::task::spawn({
     825              :                 let cleanup_self = self.clone();
     826            0 :                 async move { cleanup_self.cleanup_locations(cleanup).await }
     827              :             });
     828              :         }
     829              : 
     830              :         tracing::info!(
     831              :             "Startup complete, spawned {reconcile_tasks} reconciliation tasks ({shard_count} shards total)"
     832              :         );
     833              :     }
     834              : 
     835            0 :     async fn initial_heartbeat_round<'a>(
     836            0 :         &self,
     837            0 :         node_ids: impl Iterator<Item = &'a NodeId>,
     838            0 :     ) -> (
     839            0 :         HashMap<NodeId, PageserverUtilization>,
     840            0 :         HashMap<NodeId, (SafekeeperUtilization, Instant)>,
     841            0 :     ) {
     842            0 :         assert!(!self.startup_complete.is_ready());
     843              : 
     844            0 :         let all_nodes = {
     845            0 :             let locked = self.inner.read().unwrap();
     846            0 :             locked.nodes.clone()
     847            0 :         };
     848            0 : 
     849            0 :         let mut nodes_to_heartbeat = HashMap::new();
     850            0 :         for node_id in node_ids {
     851            0 :             match all_nodes.get(node_id) {
     852            0 :                 Some(node) => {
     853            0 :                     nodes_to_heartbeat.insert(*node_id, node.clone());
     854            0 :                 }
     855              :                 None => {
     856            0 :                     tracing::warn!("Node {node_id} was removed during start-up");
     857              :                 }
     858              :             }
     859              :         }
     860              : 
     861            0 :         let all_sks = {
     862            0 :             let locked = self.inner.read().unwrap();
     863            0 :             locked.safekeepers.clone()
     864            0 :         };
     865            0 : 
     866            0 :         tracing::info!("Sending initial heartbeats...");
     867              :         // Put a small, but reasonable timeout to get the initial heartbeats of the safekeepers to avoid a storage controller downtime
     868              :         const SK_TIMEOUT: Duration = Duration::from_secs(5);
     869            0 :         let (res_ps, res_sk) = tokio::join!(
     870            0 :             self.heartbeater_ps.heartbeat(Arc::new(nodes_to_heartbeat)),
     871            0 :             tokio::time::timeout(SK_TIMEOUT, self.heartbeater_sk.heartbeat(all_sks))
     872            0 :         );
     873              : 
     874            0 :         let mut online_nodes = HashMap::new();
     875            0 :         if let Ok(deltas) = res_ps {
     876            0 :             for (node_id, status) in deltas.0 {
     877            0 :                 match status {
     878            0 :                     PageserverState::Available { utilization, .. } => {
     879            0 :                         online_nodes.insert(node_id, utilization);
     880            0 :                     }
     881            0 :                     PageserverState::Offline => {}
     882              :                     PageserverState::WarmingUp { .. } => {
     883            0 :                         unreachable!("Nodes are never marked warming-up during startup reconcile")
     884              :                     }
     885              :                 }
     886              :             }
     887            0 :         }
     888              : 
     889            0 :         let mut online_sks = HashMap::new();
     890            0 :         if let Ok(Ok(deltas)) = res_sk {
     891            0 :             for (node_id, status) in deltas.0 {
     892            0 :                 match status {
     893              :                     SafekeeperState::Available {
     894            0 :                         utilization,
     895            0 :                         last_seen_at,
     896            0 :                     } => {
     897            0 :                         online_sks.insert(node_id, (utilization, last_seen_at));
     898            0 :                     }
     899            0 :                     SafekeeperState::Offline => {}
     900              :                 }
     901              :             }
     902            0 :         }
     903              : 
     904            0 :         (online_nodes, online_sks)
     905            0 :     }
     906              : 
     907              :     /// Used during [`Self::startup_reconcile`]: issue GETs to all nodes concurrently, with a deadline.
     908              :     ///
     909              :     /// The result includes only nodes which responded within the deadline
     910            0 :     async fn scan_node_locations(
     911            0 :         &self,
     912            0 :         deadline: Instant,
     913            0 :     ) -> HashMap<NodeId, LocationConfigListResponse> {
     914            0 :         let nodes = {
     915            0 :             let locked = self.inner.read().unwrap();
     916            0 :             locked.nodes.clone()
     917            0 :         };
     918            0 : 
     919            0 :         let mut node_results = HashMap::new();
     920            0 : 
     921            0 :         let mut node_list_futs = FuturesUnordered::new();
     922            0 : 
     923            0 :         tracing::info!("Scanning shards on {} nodes...", nodes.len());
     924            0 :         for node in nodes.values() {
     925            0 :             node_list_futs.push({
     926            0 :                 async move {
     927            0 :                     tracing::info!("Scanning shards on node {node}...");
     928            0 :                     let timeout = Duration::from_secs(5);
     929            0 :                     let response = node
     930            0 :                         .with_client_retries(
     931            0 :                             |client| async move { client.list_location_config().await },
     932            0 :                             &self.config.pageserver_jwt_token,
     933            0 :                             &self.config.ssl_ca_cert,
     934            0 :                             1,
     935            0 :                             5,
     936            0 :                             timeout,
     937            0 :                             &self.cancel,
     938            0 :                         )
     939            0 :                         .await;
     940            0 :                     (node.get_id(), response)
     941            0 :                 }
     942            0 :             });
     943            0 :         }
     944              : 
     945              :         loop {
     946            0 :             let (node_id, result) = tokio::select! {
     947            0 :                 next = node_list_futs.next() => {
     948            0 :                     match next {
     949            0 :                         Some(result) => result,
     950              :                         None =>{
     951              :                             // We got results for all our nodes
     952            0 :                             break;
     953              :                         }
     954              : 
     955              :                     }
     956              :                 },
     957            0 :                 _ = tokio::time::sleep(deadline.duration_since(Instant::now())) => {
     958              :                     // Give up waiting for anyone who hasn't responded: we will yield the results that we have
     959            0 :                     tracing::info!("Reached deadline while waiting for nodes to respond to location listing requests");
     960            0 :                     break;
     961              :                 }
     962              :             };
     963              : 
     964            0 :             let Some(list_response) = result else {
     965            0 :                 tracing::info!("Shutdown during startup_reconcile");
     966            0 :                 break;
     967              :             };
     968              : 
     969            0 :             match list_response {
     970            0 :                 Err(e) => {
     971            0 :                     tracing::warn!("Could not scan node {} ({e})", node_id);
     972              :                 }
     973            0 :                 Ok(listing) => {
     974            0 :                     node_results.insert(node_id, listing);
     975            0 :                 }
     976              :             }
     977              :         }
     978              : 
     979            0 :         node_results
     980            0 :     }
     981              : 
     982            0 :     async fn build_global_observed_state(&self, deadline: Instant) -> GlobalObservedState {
     983            0 :         let node_listings = self.scan_node_locations(deadline).await;
     984            0 :         let mut observed = GlobalObservedState::default();
     985              : 
     986            0 :         for (node_id, location_confs) in node_listings {
     987            0 :             tracing::info!(
     988            0 :                 "Received {} shard statuses from pageserver {}",
     989            0 :                 location_confs.tenant_shards.len(),
     990              :                 node_id
     991              :             );
     992              : 
     993            0 :             for (tid, location_conf) in location_confs.tenant_shards {
     994            0 :                 let entry = observed.0.entry(tid).or_default();
     995            0 :                 entry.locations.insert(
     996            0 :                     node_id,
     997            0 :                     ObservedStateLocation {
     998            0 :                         conf: location_conf,
     999            0 :                     },
    1000            0 :                 );
    1001            0 :             }
    1002              :         }
    1003              : 
    1004            0 :         observed
    1005            0 :     }
    1006              : 
    1007              :     /// Used during [`Self::startup_reconcile`]: detach a list of unknown-to-us tenants from pageservers.
    1008              :     ///
    1009              :     /// This is safe to run in the background, because if we don't have this TenantShardId in our map of
    1010              :     /// tenants, then it is probably something incompletely deleted before: we will not fight with any
    1011              :     /// other task trying to attach it.
    1012              :     #[instrument(skip_all)]
    1013              :     async fn cleanup_locations(&self, cleanup: Vec<(TenantShardId, NodeId)>) {
    1014              :         let nodes = self.inner.read().unwrap().nodes.clone();
    1015              : 
    1016              :         for (tenant_shard_id, node_id) in cleanup {
    1017              :             // A node reported a tenant_shard_id which is unknown to us: detach it.
    1018              :             let Some(node) = nodes.get(&node_id) else {
    1019              :                 // This is legitimate; we run in the background and [`Self::startup_reconcile`] might have identified
    1020              :                 // a location to clean up on a node that has since been removed.
    1021              :                 tracing::info!(
    1022              :                     "Not cleaning up location {node_id}/{tenant_shard_id}: node not found"
    1023              :                 );
    1024              :                 continue;
    1025              :             };
    1026              : 
    1027              :             if self.cancel.is_cancelled() {
    1028              :                 break;
    1029              :             }
    1030              : 
    1031              :             let client = match PageserverClient::new(
    1032              :                 node.get_id(),
    1033              :                 node.base_url(),
    1034              :                 self.config.pageserver_jwt_token.as_deref(),
    1035              :                 self.config.ssl_ca_cert.clone(),
    1036              :             ) {
    1037              :                 Ok(client) => client,
    1038              :                 Err(e) => {
    1039              :                     tracing::error!(
    1040              :                         "Failed to create client to detach unknown shard {tenant_shard_id} on pageserver {node_id}: {e}"
    1041              :                     );
    1042              :                     continue;
    1043              :                 }
    1044              :             };
    1045              :             match client
    1046              :                 .location_config(
    1047              :                     tenant_shard_id,
    1048              :                     LocationConfig {
    1049              :                         mode: LocationConfigMode::Detached,
    1050              :                         generation: None,
    1051              :                         secondary_conf: None,
    1052              :                         shard_number: tenant_shard_id.shard_number.0,
    1053              :                         shard_count: tenant_shard_id.shard_count.literal(),
    1054              :                         shard_stripe_size: 0,
    1055              :                         tenant_conf: models::TenantConfig::default(),
    1056              :                     },
    1057              :                     None,
    1058              :                     false,
    1059              :                 )
    1060              :                 .await
    1061              :             {
    1062              :                 Ok(()) => {
    1063              :                     tracing::info!(
    1064              :                         "Detached unknown shard {tenant_shard_id} on pageserver {node_id}"
    1065              :                     );
    1066              :                 }
    1067              :                 Err(e) => {
    1068              :                     // Non-fatal error: leaving a tenant shard behind that we are not managing shouldn't
    1069              :                     // break anything.
    1070              :                     tracing::error!(
    1071              :                         "Failed to detach unknown shard {tenant_shard_id} on pageserver {node_id}: {e}"
    1072              :                     );
    1073              :                 }
    1074              :             }
    1075              :         }
    1076              :     }
    1077              : 
    1078              :     /// Long running background task that periodically wakes up and looks for shards that need
    1079              :     /// reconciliation.  Reconciliation is fallible, so any reconciliation tasks that fail during
    1080              :     /// e.g. a tenant create/attach/migrate must eventually be retried: this task is responsible
    1081              :     /// for those retries.
    1082              :     #[instrument(skip_all)]
    1083              :     async fn background_reconcile(self: &Arc<Self>) {
    1084              :         self.startup_complete.clone().wait().await;
    1085              : 
    1086              :         const BACKGROUND_RECONCILE_PERIOD: Duration = Duration::from_secs(20);
    1087              :         let mut interval = tokio::time::interval(BACKGROUND_RECONCILE_PERIOD);
    1088              :         while !self.reconcilers_cancel.is_cancelled() {
    1089              :             tokio::select! {
    1090              :               _ = interval.tick() => {
    1091              :                 let reconciles_spawned = self.reconcile_all();
    1092              :                 if reconciles_spawned == 0 {
    1093              :                     // Run optimizer only when we didn't find any other work to do
    1094              :                     self.optimize_all().await;
    1095              :                 }
    1096              :                 // Always attempt autosplits. Sharding is crucial for bulk ingest performance, so we
    1097              :                 // must be responsive when new projects begin ingesting and reach the threshold.
    1098              :                 self.autosplit_tenants().await;
    1099              :             }
    1100              :               _ = self.reconcilers_cancel.cancelled() => return
    1101              :             }
    1102              :         }
    1103              :     }
    1104              :     /// Heartbeat all storage nodes once in a while.
    1105              :     #[instrument(skip_all)]
    1106              :     async fn spawn_heartbeat_driver(&self) {
    1107              :         self.startup_complete.clone().wait().await;
    1108              : 
    1109              :         let mut interval = tokio::time::interval(self.config.heartbeat_interval);
    1110              :         while !self.cancel.is_cancelled() {
    1111              :             tokio::select! {
    1112              :               _ = interval.tick() => { }
    1113              :               _ = self.cancel.cancelled() => return
    1114              :             };
    1115              : 
    1116              :             let nodes = {
    1117              :                 let locked = self.inner.read().unwrap();
    1118              :                 locked.nodes.clone()
    1119              :             };
    1120              : 
    1121              :             let safekeepers = {
    1122              :                 let locked = self.inner.read().unwrap();
    1123              :                 locked.safekeepers.clone()
    1124              :             };
    1125              : 
    1126              :             const SK_TIMEOUT: Duration = Duration::from_secs(3);
    1127              :             let (res_ps, res_sk) = tokio::join!(
    1128              :                 self.heartbeater_ps.heartbeat(nodes),
    1129              :                 tokio::time::timeout(SK_TIMEOUT, self.heartbeater_sk.heartbeat(safekeepers))
    1130              :             );
    1131              : 
    1132              :             if let Ok(deltas) = res_ps {
    1133              :                 let mut to_handle = Vec::default();
    1134              : 
    1135              :                 for (node_id, state) in deltas.0 {
    1136              :                     let new_availability = match state {
    1137              :                         PageserverState::Available { utilization, .. } => {
    1138              :                             NodeAvailability::Active(utilization)
    1139              :                         }
    1140              :                         PageserverState::WarmingUp { started_at } => {
    1141              :                             NodeAvailability::WarmingUp(started_at)
    1142              :                         }
    1143              :                         PageserverState::Offline => {
    1144              :                             // The node might have been placed in the WarmingUp state
    1145              :                             // while the heartbeat round was on-going. Hence, filter out
    1146              :                             // offline transitions for WarmingUp nodes that are still within
    1147              :                             // their grace period.
    1148              :                             if let Ok(NodeAvailability::WarmingUp(started_at)) = self
    1149              :                                 .get_node(node_id)
    1150              :                                 .await
    1151              :                                 .as_ref()
    1152            0 :                                 .map(|n| n.get_availability())
    1153              :                             {
    1154              :                                 let now = Instant::now();
    1155              :                                 if now - *started_at >= self.config.max_warming_up_interval {
    1156              :                                     NodeAvailability::Offline
    1157              :                                 } else {
    1158              :                                     NodeAvailability::WarmingUp(*started_at)
    1159              :                                 }
    1160              :                             } else {
    1161              :                                 NodeAvailability::Offline
    1162              :                             }
    1163              :                         }
    1164              :                     };
    1165              : 
    1166              :                     let node_lock = trace_exclusive_lock(
    1167              :                         &self.node_op_locks,
    1168              :                         node_id,
    1169              :                         NodeOperations::Configure,
    1170              :                     )
    1171              :                     .await;
    1172              : 
    1173              :                     pausable_failpoint!("heartbeat-pre-node-state-configure");
    1174              : 
    1175              :                     // This is the code path for geniune availability transitions (i.e node
    1176              :                     // goes unavailable and/or comes back online).
    1177              :                     let res = self
    1178              :                         .node_state_configure(node_id, Some(new_availability), None, &node_lock)
    1179              :                         .await;
    1180              : 
    1181              :                     match res {
    1182              :                         Ok(transition) => {
    1183              :                             // Keep hold of the lock until the availability transitions
    1184              :                             // have been handled in
    1185              :                             // [`Service::handle_node_availability_transitions`] in order avoid
    1186              :                             // racing with [`Service::external_node_configure`].
    1187              :                             to_handle.push((node_id, node_lock, transition));
    1188              :                         }
    1189              :                         Err(ApiError::NotFound(_)) => {
    1190              :                             // This should be rare, but legitimate since the heartbeats are done
    1191              :                             // on a snapshot of the nodes.
    1192              :                             tracing::info!("Node {} was not found after heartbeat round", node_id);
    1193              :                         }
    1194              :                         Err(ApiError::ShuttingDown) => {
    1195              :                             // No-op: we're shutting down, no need to try and update any nodes' statuses
    1196              :                         }
    1197              :                         Err(err) => {
    1198              :                             // Transition to active involves reconciling: if a node responds to a heartbeat then
    1199              :                             // becomes unavailable again, we may get an error here.
    1200              :                             tracing::error!(
    1201              :                                 "Failed to update node state {} after heartbeat round: {}",
    1202              :                                 node_id,
    1203              :                                 err
    1204              :                             );
    1205              :                         }
    1206              :                     }
    1207              :                 }
    1208              : 
    1209              :                 // We collected all the transitions above and now we handle them.
    1210              :                 let res = self.handle_node_availability_transitions(to_handle).await;
    1211              :                 if let Err(errs) = res {
    1212              :                     for (node_id, err) in errs {
    1213              :                         match err {
    1214              :                             ApiError::NotFound(_) => {
    1215              :                                 // This should be rare, but legitimate since the heartbeats are done
    1216              :                                 // on a snapshot of the nodes.
    1217              :                                 tracing::info!(
    1218              :                                     "Node {} was not found after heartbeat round",
    1219              :                                     node_id
    1220              :                                 );
    1221              :                             }
    1222              :                             err => {
    1223              :                                 tracing::error!(
    1224              :                                     "Failed to handle availability transition for {} after heartbeat round: {}",
    1225              :                                     node_id,
    1226              :                                     err
    1227              :                                 );
    1228              :                             }
    1229              :                         }
    1230              :                     }
    1231              :                 }
    1232              :             }
    1233              :             if let Ok(Ok(deltas)) = res_sk {
    1234              :                 let mut locked = self.inner.write().unwrap();
    1235              :                 let mut safekeepers = (*locked.safekeepers).clone();
    1236              :                 for (id, state) in deltas.0 {
    1237              :                     let Some(sk) = safekeepers.get_mut(&id) else {
    1238              :                         tracing::info!(
    1239              :                             "Couldn't update safekeeper safekeeper state for id {id} from heartbeat={state:?}"
    1240              :                         );
    1241              :                         continue;
    1242              :                     };
    1243              :                     sk.set_availability(state);
    1244              :                 }
    1245              :                 locked.safekeepers = Arc::new(safekeepers);
    1246              :             }
    1247              :         }
    1248              :     }
    1249              : 
    1250              :     /// Apply the contents of a [`ReconcileResult`] to our in-memory state: if the reconciliation
    1251              :     /// was successful and intent hasn't changed since the Reconciler was spawned, this will update
    1252              :     /// the observed state of the tenant such that subsequent calls to [`TenantShard::get_reconcile_needed`]
    1253              :     /// will indicate that reconciliation is not needed.
    1254              :     #[instrument(skip_all, fields(
    1255              :         seq=%result.sequence,
    1256              :         tenant_id=%result.tenant_shard_id.tenant_id,
    1257              :         shard_id=%result.tenant_shard_id.shard_slug(),
    1258              :     ))]
    1259              :     fn process_result(&self, result: ReconcileResult) {
    1260              :         let mut locked = self.inner.write().unwrap();
    1261              :         let (nodes, tenants, _scheduler) = locked.parts_mut();
    1262              :         let Some(tenant) = tenants.get_mut(&result.tenant_shard_id) else {
    1263              :             // A reconciliation result might race with removing a tenant: drop results for
    1264              :             // tenants that aren't in our map.
    1265              :             return;
    1266              :         };
    1267              : 
    1268              :         // Usually generation should only be updated via this path, so the max() isn't
    1269              :         // needed, but it is used to handle out-of-band updates via. e.g. test hook.
    1270              :         tenant.generation = std::cmp::max(tenant.generation, result.generation);
    1271              : 
    1272              :         // If the reconciler signals that it failed to notify compute, set this state on
    1273              :         // the shard so that a future [`TenantShard::maybe_reconcile`] will try again.
    1274              :         tenant.pending_compute_notification = result.pending_compute_notification;
    1275              : 
    1276              :         // Let the TenantShard know it is idle.
    1277              :         tenant.reconcile_complete(result.sequence);
    1278              : 
    1279              :         // In case a node was deleted while this reconcile is in flight, filter it out of the update we will
    1280              :         // make to the tenant
    1281            0 :         let deltas = result.observed_deltas.into_iter().flat_map(|delta| {
    1282              :             // In case a node was deleted while this reconcile is in flight, filter it out of the update we will
    1283              :             // make to the tenant
    1284            0 :             let node = nodes.get(delta.node_id())?;
    1285              : 
    1286            0 :             if node.is_available() {
    1287            0 :                 return Some(delta);
    1288            0 :             }
    1289            0 : 
    1290            0 :             // In case a node became unavailable concurrently with the reconcile, observed
    1291            0 :             // locations on it are now uncertain. By convention, set them to None in order
    1292            0 :             // for them to get refreshed when the node comes back online.
    1293            0 :             Some(ObservedStateDelta::Upsert(Box::new((
    1294            0 :                 node.get_id(),
    1295            0 :                 ObservedStateLocation { conf: None },
    1296            0 :             ))))
    1297            0 :         });
    1298              : 
    1299              :         match result.result {
    1300              :             Ok(()) => {
    1301              :                 tenant.apply_observed_deltas(deltas);
    1302              :                 tenant.waiter.advance(result.sequence);
    1303              :             }
    1304              :             Err(e) => {
    1305              :                 match e {
    1306              :                     ReconcileError::Cancel => {
    1307              :                         tracing::info!("Reconciler was cancelled");
    1308              :                     }
    1309              :                     ReconcileError::Remote(mgmt_api::Error::Cancelled) => {
    1310              :                         // This might be due to the reconciler getting cancelled, or it might
    1311              :                         // be due to the `Node` being marked offline.
    1312              :                         tracing::info!("Reconciler cancelled during pageserver API call");
    1313              :                     }
    1314              :                     _ => {
    1315              :                         tracing::warn!("Reconcile error: {}", e);
    1316              :                     }
    1317              :                 }
    1318              : 
    1319              :                 // Ordering: populate last_error before advancing error_seq,
    1320              :                 // so that waiters will see the correct error after waiting.
    1321              :                 tenant.set_last_error(result.sequence, e);
    1322              : 
    1323              :                 // Skip deletions on reconcile failures
    1324              :                 let upsert_deltas =
    1325            0 :                     deltas.filter(|delta| matches!(delta, ObservedStateDelta::Upsert(_)));
    1326              :                 tenant.apply_observed_deltas(upsert_deltas);
    1327              :             }
    1328              :         }
    1329              : 
    1330              :         // If we just finished detaching all shards for a tenant, it might be time to drop it from memory.
    1331              :         if tenant.policy == PlacementPolicy::Detached {
    1332              :             // We may only drop a tenant from memory while holding the exclusive lock on the tenant ID: this protects us
    1333              :             // from concurrent execution wrt a request handler that might expect the tenant to remain in memory for the
    1334              :             // duration of the request.
    1335              :             let guard = self.tenant_op_locks.try_exclusive(
    1336              :                 tenant.tenant_shard_id.tenant_id,
    1337              :                 TenantOperations::DropDetached,
    1338              :             );
    1339              :             if let Some(guard) = guard {
    1340              :                 self.maybe_drop_tenant(tenant.tenant_shard_id.tenant_id, &mut locked, &guard);
    1341              :             }
    1342              :         }
    1343              : 
    1344              :         // Maybe some other work can proceed now that this job finished.
    1345              :         //
    1346              :         // Only bother with this if we have some semaphore units available in the normal-priority semaphore (these
    1347              :         // reconciles are scheduled at `[ReconcilerPriority::Normal]`).
    1348              :         if self.reconciler_concurrency.available_permits() > 0 {
    1349              :             while let Ok(tenant_shard_id) = locked.delayed_reconcile_rx.try_recv() {
    1350              :                 let (nodes, tenants, _scheduler) = locked.parts_mut();
    1351              :                 if let Some(shard) = tenants.get_mut(&tenant_shard_id) {
    1352              :                     shard.delayed_reconcile = false;
    1353              :                     self.maybe_reconcile_shard(shard, nodes, ReconcilerPriority::Normal);
    1354              :                 }
    1355              : 
    1356              :                 if self.reconciler_concurrency.available_permits() == 0 {
    1357              :                     break;
    1358              :                 }
    1359              :             }
    1360              :         }
    1361              :     }
    1362              : 
    1363            0 :     async fn process_results(
    1364            0 :         &self,
    1365            0 :         mut result_rx: tokio::sync::mpsc::UnboundedReceiver<ReconcileResultRequest>,
    1366            0 :         mut bg_compute_hook_result_rx: tokio::sync::mpsc::Receiver<
    1367            0 :             Result<(), (TenantShardId, NotifyError)>,
    1368            0 :         >,
    1369            0 :     ) {
    1370              :         loop {
    1371              :             // Wait for the next result, or for cancellation
    1372            0 :             tokio::select! {
    1373            0 :                 r = result_rx.recv() => {
    1374            0 :                     match r {
    1375            0 :                         Some(ReconcileResultRequest::ReconcileResult(result)) => {self.process_result(result);},
    1376            0 :                         None | Some(ReconcileResultRequest::Stop) => {break;}
    1377              :                     }
    1378              :                 }
    1379            0 :                 _ = async{
    1380            0 :                     match bg_compute_hook_result_rx.recv().await {
    1381            0 :                         Some(result) => {
    1382            0 :                             if let Err((tenant_shard_id, notify_error)) = result {
    1383            0 :                                 tracing::warn!("Marking shard {tenant_shard_id} for notification retry, due to error {notify_error}");
    1384            0 :                                 let mut locked = self.inner.write().unwrap();
    1385            0 :                                 if let Some(shard) = locked.tenants.get_mut(&tenant_shard_id) {
    1386            0 :                                     shard.pending_compute_notification = true;
    1387            0 :                                 }
    1388              : 
    1389            0 :                             }
    1390              :                         },
    1391              :                         None => {
    1392              :                             // This channel is dead, but we don't want to terminate the outer loop{}: just wait for shutdown
    1393            0 :                             self.cancel.cancelled().await;
    1394              :                         }
    1395              :                     }
    1396            0 :                 } => {},
    1397            0 :                 _ = self.cancel.cancelled() => {
    1398            0 :                     break;
    1399              :                 }
    1400              :             };
    1401              :         }
    1402            0 :     }
    1403              : 
    1404            0 :     async fn process_aborts(
    1405            0 :         &self,
    1406            0 :         mut abort_rx: tokio::sync::mpsc::UnboundedReceiver<TenantShardSplitAbort>,
    1407            0 :     ) {
    1408              :         loop {
    1409              :             // Wait for the next result, or for cancellation
    1410            0 :             let op = tokio::select! {
    1411            0 :                 r = abort_rx.recv() => {
    1412            0 :                     match r {
    1413            0 :                         Some(op) => {op},
    1414            0 :                         None => {break;}
    1415              :                     }
    1416              :                 }
    1417            0 :                 _ = self.cancel.cancelled() => {
    1418            0 :                     break;
    1419              :                 }
    1420              :             };
    1421              : 
    1422              :             // Retry until shutdown: we must keep this request object alive until it is properly
    1423              :             // processed, as it holds a lock guard that prevents other operations trying to do things
    1424              :             // to the tenant while it is in a weird part-split state.
    1425            0 :             while !self.cancel.is_cancelled() {
    1426            0 :                 match self.abort_tenant_shard_split(&op).await {
    1427            0 :                     Ok(_) => break,
    1428            0 :                     Err(e) => {
    1429            0 :                         tracing::warn!(
    1430            0 :                             "Failed to abort shard split on {}, will retry: {e}",
    1431              :                             op.tenant_id
    1432              :                         );
    1433              : 
    1434              :                         // If a node is unavailable, we hope that it has been properly marked Offline
    1435              :                         // when we retry, so that the abort op will succeed.  If the abort op is failing
    1436              :                         // for some other reason, we will keep retrying forever, or until a human notices
    1437              :                         // and does something about it (either fixing a pageserver or restarting the controller).
    1438            0 :                         tokio::time::timeout(Duration::from_secs(5), self.cancel.cancelled())
    1439            0 :                             .await
    1440            0 :                             .ok();
    1441              :                     }
    1442              :                 }
    1443              :             }
    1444              :         }
    1445            0 :     }
    1446              : 
    1447            0 :     pub async fn spawn(config: Config, persistence: Arc<Persistence>) -> anyhow::Result<Arc<Self>> {
    1448            0 :         let (result_tx, result_rx) = tokio::sync::mpsc::unbounded_channel();
    1449            0 :         let (abort_tx, abort_rx) = tokio::sync::mpsc::unbounded_channel();
    1450            0 : 
    1451            0 :         let leadership_cancel = CancellationToken::new();
    1452            0 :         let leadership = Leadership::new(persistence.clone(), config.clone(), leadership_cancel);
    1453            0 :         let (leader, leader_step_down_state) = leadership.step_down_current_leader().await?;
    1454              : 
    1455              :         // Apply the migrations **after** the current leader has stepped down
    1456              :         // (or we've given up waiting for it), but **before** reading from the
    1457              :         // database. The only exception is reading the current leader before
    1458              :         // migrating.
    1459            0 :         persistence.migration_run().await?;
    1460              : 
    1461            0 :         tracing::info!("Loading nodes from database...");
    1462            0 :         let nodes = persistence
    1463            0 :             .list_nodes()
    1464            0 :             .await?
    1465            0 :             .into_iter()
    1466            0 :             .map(|x| Node::from_persistent(x, config.use_https_pageserver_api))
    1467            0 :             .collect::<anyhow::Result<Vec<Node>>>()?;
    1468            0 :         let nodes: HashMap<NodeId, Node> = nodes.into_iter().map(|n| (n.get_id(), n)).collect();
    1469            0 :         tracing::info!("Loaded {} nodes from database.", nodes.len());
    1470            0 :         metrics::METRICS_REGISTRY
    1471            0 :             .metrics_group
    1472            0 :             .storage_controller_pageserver_nodes
    1473            0 :             .set(nodes.len() as i64);
    1474            0 : 
    1475            0 :         tracing::info!("Loading safekeepers from database...");
    1476            0 :         let safekeepers = persistence
    1477            0 :             .list_safekeepers()
    1478            0 :             .await?
    1479            0 :             .into_iter()
    1480            0 :             .map(|skp| {
    1481            0 :                 Safekeeper::from_persistence(
    1482            0 :                     skp,
    1483            0 :                     CancellationToken::new(),
    1484            0 :                     config.use_https_safekeeper_api,
    1485            0 :                 )
    1486            0 :             })
    1487            0 :             .collect::<anyhow::Result<Vec<_>>>()?;
    1488            0 :         let safekeepers: HashMap<NodeId, Safekeeper> =
    1489            0 :             safekeepers.into_iter().map(|n| (n.get_id(), n)).collect();
    1490            0 :         tracing::info!("Loaded {} safekeepers from database.", safekeepers.len());
    1491              : 
    1492            0 :         tracing::info!("Loading shards from database...");
    1493            0 :         let mut tenant_shard_persistence = persistence.load_active_tenant_shards().await?;
    1494            0 :         tracing::info!(
    1495            0 :             "Loaded {} shards from database.",
    1496            0 :             tenant_shard_persistence.len()
    1497              :         );
    1498              : 
    1499              :         // If any shard splits were in progress, reset the database state to abort them
    1500            0 :         let mut tenant_shard_count_min_max: HashMap<TenantId, (ShardCount, ShardCount)> =
    1501            0 :             HashMap::new();
    1502            0 :         for tsp in &mut tenant_shard_persistence {
    1503            0 :             let shard = tsp.get_shard_identity()?;
    1504            0 :             let tenant_shard_id = tsp.get_tenant_shard_id()?;
    1505            0 :             let entry = tenant_shard_count_min_max
    1506            0 :                 .entry(tenant_shard_id.tenant_id)
    1507            0 :                 .or_insert_with(|| (shard.count, shard.count));
    1508            0 :             entry.0 = std::cmp::min(entry.0, shard.count);
    1509            0 :             entry.1 = std::cmp::max(entry.1, shard.count);
    1510            0 :         }
    1511              : 
    1512            0 :         for (tenant_id, (count_min, count_max)) in tenant_shard_count_min_max {
    1513            0 :             if count_min != count_max {
    1514              :                 // Aborting the split in the database and dropping the child shards is sufficient: the reconciliation in
    1515              :                 // [`Self::startup_reconcile`] will implicitly drop the child shards on remote pageservers, or they'll
    1516              :                 // be dropped later in [`Self::node_activate_reconcile`] if it isn't available right now.
    1517            0 :                 tracing::info!("Aborting shard split {tenant_id} {count_min:?} -> {count_max:?}");
    1518            0 :                 let abort_status = persistence.abort_shard_split(tenant_id, count_max).await?;
    1519              : 
    1520              :                 // We may never see the Complete status here: if the split was complete, we wouldn't have
    1521              :                 // identified this tenant has having mismatching min/max counts.
    1522            0 :                 assert!(matches!(abort_status, AbortShardSplitStatus::Aborted));
    1523              : 
    1524              :                 // Clear the splitting status in-memory, to reflect that we just aborted in the database
    1525            0 :                 tenant_shard_persistence.iter_mut().for_each(|tsp| {
    1526            0 :                     // Set idle split state on those shards that we will retain.
    1527            0 :                     let tsp_tenant_id = TenantId::from_str(tsp.tenant_id.as_str()).unwrap();
    1528            0 :                     if tsp_tenant_id == tenant_id
    1529            0 :                         && tsp.get_shard_identity().unwrap().count == count_min
    1530            0 :                     {
    1531            0 :                         tsp.splitting = SplitState::Idle;
    1532            0 :                     } else if tsp_tenant_id == tenant_id {
    1533              :                         // Leave the splitting state on the child shards: this will be used next to
    1534              :                         // drop them.
    1535            0 :                         tracing::info!(
    1536            0 :                             "Shard {tsp_tenant_id} will be dropped after shard split abort",
    1537              :                         );
    1538            0 :                     }
    1539            0 :                 });
    1540            0 : 
    1541            0 :                 // Drop shards for this tenant which we didn't just mark idle (i.e. child shards of the aborted split)
    1542            0 :                 tenant_shard_persistence.retain(|tsp| {
    1543            0 :                     TenantId::from_str(tsp.tenant_id.as_str()).unwrap() != tenant_id
    1544            0 :                         || tsp.splitting == SplitState::Idle
    1545            0 :                 });
    1546            0 :             }
    1547              :         }
    1548              : 
    1549            0 :         let mut tenants = BTreeMap::new();
    1550            0 : 
    1551            0 :         let mut scheduler = Scheduler::new(nodes.values());
    1552              : 
    1553              :         #[cfg(feature = "testing")]
    1554              :         {
    1555              :             use pageserver_api::controller_api::AvailabilityZone;
    1556              : 
    1557              :             // Hack: insert scheduler state for all nodes referenced by shards, as compatibility
    1558              :             // tests only store the shards, not the nodes.  The nodes will be loaded shortly
    1559              :             // after when pageservers start up and register.
    1560            0 :             let mut node_ids = HashSet::new();
    1561            0 :             for tsp in &tenant_shard_persistence {
    1562            0 :                 if let Some(node_id) = tsp.generation_pageserver {
    1563            0 :                     node_ids.insert(node_id);
    1564            0 :                 }
    1565              :             }
    1566            0 :             for node_id in node_ids {
    1567            0 :                 tracing::info!("Creating node {} in scheduler for tests", node_id);
    1568            0 :                 let node = Node::new(
    1569            0 :                     NodeId(node_id as u64),
    1570            0 :                     "".to_string(),
    1571            0 :                     123,
    1572            0 :                     None,
    1573            0 :                     "".to_string(),
    1574            0 :                     123,
    1575            0 :                     AvailabilityZone("test_az".to_string()),
    1576            0 :                     false,
    1577            0 :                 )
    1578            0 :                 .unwrap();
    1579            0 : 
    1580            0 :                 scheduler.node_upsert(&node);
    1581              :             }
    1582              :         }
    1583            0 :         for tsp in tenant_shard_persistence {
    1584            0 :             let tenant_shard_id = tsp.get_tenant_shard_id()?;
    1585              : 
    1586              :             // We will populate intent properly later in [`Self::startup_reconcile`], initially populate
    1587              :             // it with what we can infer: the node for which a generation was most recently issued.
    1588            0 :             let mut intent = IntentState::new(
    1589            0 :                 tsp.preferred_az_id
    1590            0 :                     .as_ref()
    1591            0 :                     .map(|az| AvailabilityZone(az.clone())),
    1592            0 :             );
    1593            0 :             if let Some(generation_pageserver) = tsp.generation_pageserver.map(|n| NodeId(n as u64))
    1594              :             {
    1595            0 :                 if nodes.contains_key(&generation_pageserver) {
    1596            0 :                     intent.set_attached(&mut scheduler, Some(generation_pageserver));
    1597            0 :                 } else {
    1598              :                     // If a node was removed before being completely drained, it is legal for it to leave behind a `generation_pageserver` referring
    1599              :                     // to a non-existent node, because node deletion doesn't block on completing the reconciliations that will issue new generations
    1600              :                     // on different pageservers.
    1601            0 :                     tracing::warn!(
    1602            0 :                         "Tenant shard {tenant_shard_id} references non-existent node {generation_pageserver} in database, will be rescheduled"
    1603              :                     );
    1604              :                 }
    1605            0 :             }
    1606            0 :             let new_tenant = TenantShard::from_persistent(tsp, intent)?;
    1607              : 
    1608            0 :             tenants.insert(tenant_shard_id, new_tenant);
    1609              :         }
    1610              : 
    1611            0 :         let (startup_completion, startup_complete) = utils::completion::channel();
    1612            0 : 
    1613            0 :         // This channel is continuously consumed by process_results, so doesn't need to be very large.
    1614            0 :         let (bg_compute_notify_result_tx, bg_compute_notify_result_rx) =
    1615            0 :             tokio::sync::mpsc::channel(512);
    1616            0 : 
    1617            0 :         let (delayed_reconcile_tx, delayed_reconcile_rx) =
    1618            0 :             tokio::sync::mpsc::channel(MAX_DELAYED_RECONCILES);
    1619            0 : 
    1620            0 :         let cancel = CancellationToken::new();
    1621            0 :         let reconcilers_cancel = cancel.child_token();
    1622            0 : 
    1623            0 :         let heartbeater_ps = Heartbeater::new(
    1624            0 :             config.pageserver_jwt_token.clone(),
    1625            0 :             config.ssl_ca_cert.clone(),
    1626            0 :             config.max_offline_interval,
    1627            0 :             config.max_warming_up_interval,
    1628            0 :             cancel.clone(),
    1629            0 :         );
    1630            0 : 
    1631            0 :         let heartbeater_sk = Heartbeater::new(
    1632            0 :             config.safekeeper_jwt_token.clone(),
    1633            0 :             config.ssl_ca_cert.clone(),
    1634            0 :             config.max_offline_interval,
    1635            0 :             config.max_warming_up_interval,
    1636            0 :             cancel.clone(),
    1637            0 :         );
    1638              : 
    1639            0 :         let initial_leadership_status = if config.start_as_candidate {
    1640            0 :             LeadershipStatus::Candidate
    1641              :         } else {
    1642            0 :             LeadershipStatus::Leader
    1643              :         };
    1644              : 
    1645            0 :         let this = Arc::new(Self {
    1646            0 :             inner: Arc::new(std::sync::RwLock::new(ServiceState::new(
    1647            0 :                 nodes,
    1648            0 :                 safekeepers,
    1649            0 :                 tenants,
    1650            0 :                 scheduler,
    1651            0 :                 delayed_reconcile_rx,
    1652            0 :                 initial_leadership_status,
    1653            0 :                 reconcilers_cancel.clone(),
    1654            0 :             ))),
    1655            0 :             config: config.clone(),
    1656            0 :             persistence,
    1657            0 :             compute_hook: Arc::new(ComputeHook::new(config.clone())),
    1658            0 :             result_tx,
    1659            0 :             heartbeater_ps,
    1660            0 :             heartbeater_sk,
    1661            0 :             reconciler_concurrency: Arc::new(tokio::sync::Semaphore::new(
    1662            0 :                 config.reconciler_concurrency,
    1663            0 :             )),
    1664            0 :             priority_reconciler_concurrency: Arc::new(tokio::sync::Semaphore::new(
    1665            0 :                 config.priority_reconciler_concurrency,
    1666            0 :             )),
    1667            0 :             delayed_reconcile_tx,
    1668            0 :             abort_tx,
    1669            0 :             startup_complete: startup_complete.clone(),
    1670            0 :             cancel,
    1671            0 :             reconcilers_cancel,
    1672            0 :             gate: Gate::default(),
    1673            0 :             reconcilers_gate: Gate::default(),
    1674            0 :             tenant_op_locks: Default::default(),
    1675            0 :             node_op_locks: Default::default(),
    1676            0 :         });
    1677            0 : 
    1678            0 :         let result_task_this = this.clone();
    1679            0 :         tokio::task::spawn(async move {
    1680              :             // Block shutdown until we're done (we must respect self.cancel)
    1681            0 :             if let Ok(_gate) = result_task_this.gate.enter() {
    1682            0 :                 result_task_this
    1683            0 :                     .process_results(result_rx, bg_compute_notify_result_rx)
    1684            0 :                     .await
    1685            0 :             }
    1686            0 :         });
    1687            0 : 
    1688            0 :         tokio::task::spawn({
    1689            0 :             let this = this.clone();
    1690            0 :             async move {
    1691              :                 // Block shutdown until we're done (we must respect self.cancel)
    1692            0 :                 if let Ok(_gate) = this.gate.enter() {
    1693            0 :                     this.process_aborts(abort_rx).await
    1694            0 :                 }
    1695            0 :             }
    1696            0 :         });
    1697            0 : 
    1698            0 :         tokio::task::spawn({
    1699            0 :             let this = this.clone();
    1700            0 :             async move {
    1701            0 :                 if let Ok(_gate) = this.gate.enter() {
    1702              :                     loop {
    1703            0 :                         tokio::select! {
    1704            0 :                             _ = this.cancel.cancelled() => {
    1705            0 :                                 break;
    1706              :                             },
    1707            0 :                             _ = tokio::time::sleep(Duration::from_secs(60)) => {}
    1708            0 :                         };
    1709            0 :                         this.tenant_op_locks.housekeeping();
    1710              :                     }
    1711            0 :                 }
    1712            0 :             }
    1713            0 :         });
    1714            0 : 
    1715            0 :         tokio::task::spawn({
    1716            0 :             let this = this.clone();
    1717            0 :             // We will block the [`Service::startup_complete`] barrier until [`Self::startup_reconcile`]
    1718            0 :             // is done.
    1719            0 :             let startup_completion = startup_completion.clone();
    1720            0 :             async move {
    1721              :                 // Block shutdown until we're done (we must respect self.cancel)
    1722            0 :                 let Ok(_gate) = this.gate.enter() else {
    1723            0 :                     return;
    1724              :                 };
    1725              : 
    1726            0 :                 this.startup_reconcile(leader, leader_step_down_state, bg_compute_notify_result_tx)
    1727            0 :                     .await;
    1728              : 
    1729            0 :                 drop(startup_completion);
    1730            0 :             }
    1731            0 :         });
    1732            0 : 
    1733            0 :         tokio::task::spawn({
    1734            0 :             let this = this.clone();
    1735            0 :             let startup_complete = startup_complete.clone();
    1736            0 :             async move {
    1737            0 :                 startup_complete.wait().await;
    1738            0 :                 this.background_reconcile().await;
    1739            0 :             }
    1740            0 :         });
    1741            0 : 
    1742            0 :         tokio::task::spawn({
    1743            0 :             let this = this.clone();
    1744            0 :             let startup_complete = startup_complete.clone();
    1745            0 :             async move {
    1746            0 :                 startup_complete.wait().await;
    1747            0 :                 this.spawn_heartbeat_driver().await;
    1748            0 :             }
    1749            0 :         });
    1750            0 : 
    1751            0 :         Ok(this)
    1752            0 :     }
    1753              : 
    1754            0 :     pub(crate) async fn attach_hook(
    1755            0 :         &self,
    1756            0 :         attach_req: AttachHookRequest,
    1757            0 :     ) -> anyhow::Result<AttachHookResponse> {
    1758            0 :         let _tenant_lock = trace_exclusive_lock(
    1759            0 :             &self.tenant_op_locks,
    1760            0 :             attach_req.tenant_shard_id.tenant_id,
    1761            0 :             TenantOperations::AttachHook,
    1762            0 :         )
    1763            0 :         .await;
    1764              : 
    1765              :         // This is a test hook.  To enable using it on tenants that were created directly with
    1766              :         // the pageserver API (not via this service), we will auto-create any missing tenant
    1767              :         // shards with default state.
    1768            0 :         let insert = {
    1769            0 :             match self
    1770            0 :                 .maybe_load_tenant(attach_req.tenant_shard_id.tenant_id, &_tenant_lock)
    1771            0 :                 .await
    1772              :             {
    1773            0 :                 Ok(_) => false,
    1774            0 :                 Err(ApiError::NotFound(_)) => true,
    1775            0 :                 Err(e) => return Err(e.into()),
    1776              :             }
    1777              :         };
    1778              : 
    1779            0 :         if insert {
    1780            0 :             let tsp = TenantShardPersistence {
    1781            0 :                 tenant_id: attach_req.tenant_shard_id.tenant_id.to_string(),
    1782            0 :                 shard_number: attach_req.tenant_shard_id.shard_number.0 as i32,
    1783            0 :                 shard_count: attach_req.tenant_shard_id.shard_count.literal() as i32,
    1784            0 :                 shard_stripe_size: 0,
    1785            0 :                 generation: attach_req.generation_override.or(Some(0)),
    1786            0 :                 generation_pageserver: None,
    1787            0 :                 placement_policy: serde_json::to_string(&PlacementPolicy::Attached(0)).unwrap(),
    1788            0 :                 config: serde_json::to_string(&TenantConfig::default()).unwrap(),
    1789            0 :                 splitting: SplitState::default(),
    1790            0 :                 scheduling_policy: serde_json::to_string(&ShardSchedulingPolicy::default())
    1791            0 :                     .unwrap(),
    1792            0 :                 preferred_az_id: None,
    1793            0 :             };
    1794            0 : 
    1795            0 :             match self.persistence.insert_tenant_shards(vec![tsp]).await {
    1796            0 :                 Err(e) => match e {
    1797              :                     DatabaseError::Query(diesel::result::Error::DatabaseError(
    1798              :                         DatabaseErrorKind::UniqueViolation,
    1799              :                         _,
    1800              :                     )) => {
    1801            0 :                         tracing::info!(
    1802            0 :                             "Raced with another request to insert tenant {}",
    1803              :                             attach_req.tenant_shard_id
    1804              :                         )
    1805              :                     }
    1806            0 :                     _ => return Err(e.into()),
    1807              :                 },
    1808              :                 Ok(()) => {
    1809            0 :                     tracing::info!("Inserted shard {} in database", attach_req.tenant_shard_id);
    1810              : 
    1811            0 :                     let mut locked = self.inner.write().unwrap();
    1812            0 :                     locked.tenants.insert(
    1813            0 :                         attach_req.tenant_shard_id,
    1814            0 :                         TenantShard::new(
    1815            0 :                             attach_req.tenant_shard_id,
    1816            0 :                             ShardIdentity::unsharded(),
    1817            0 :                             PlacementPolicy::Attached(0),
    1818            0 :                             None,
    1819            0 :                         ),
    1820            0 :                     );
    1821            0 :                     tracing::info!("Inserted shard {} in memory", attach_req.tenant_shard_id);
    1822              :                 }
    1823              :             }
    1824            0 :         }
    1825              : 
    1826            0 :         let new_generation = if let Some(req_node_id) = attach_req.node_id {
    1827            0 :             let maybe_tenant_conf = {
    1828            0 :                 let locked = self.inner.write().unwrap();
    1829            0 :                 locked
    1830            0 :                     .tenants
    1831            0 :                     .get(&attach_req.tenant_shard_id)
    1832            0 :                     .map(|t| t.config.clone())
    1833            0 :             };
    1834            0 : 
    1835            0 :             match maybe_tenant_conf {
    1836            0 :                 Some(conf) => {
    1837            0 :                     let new_generation = self
    1838            0 :                         .persistence
    1839            0 :                         .increment_generation(attach_req.tenant_shard_id, req_node_id)
    1840            0 :                         .await?;
    1841              : 
    1842              :                     // Persist the placement policy update. This is required
    1843              :                     // when we reattaching a detached tenant.
    1844            0 :                     self.persistence
    1845            0 :                         .update_tenant_shard(
    1846            0 :                             TenantFilter::Shard(attach_req.tenant_shard_id),
    1847            0 :                             Some(PlacementPolicy::Attached(0)),
    1848            0 :                             Some(conf),
    1849            0 :                             None,
    1850            0 :                             None,
    1851            0 :                         )
    1852            0 :                         .await?;
    1853            0 :                     Some(new_generation)
    1854              :                 }
    1855              :                 None => {
    1856            0 :                     anyhow::bail!("Attach hook handling raced with tenant removal")
    1857              :                 }
    1858              :             }
    1859              :         } else {
    1860            0 :             self.persistence.detach(attach_req.tenant_shard_id).await?;
    1861            0 :             None
    1862              :         };
    1863              : 
    1864            0 :         let mut locked = self.inner.write().unwrap();
    1865            0 :         let (_nodes, tenants, scheduler) = locked.parts_mut();
    1866            0 : 
    1867            0 :         let tenant_shard = tenants
    1868            0 :             .get_mut(&attach_req.tenant_shard_id)
    1869            0 :             .expect("Checked for existence above");
    1870              : 
    1871            0 :         if let Some(new_generation) = new_generation {
    1872            0 :             tenant_shard.generation = Some(new_generation);
    1873            0 :             tenant_shard.policy = PlacementPolicy::Attached(0);
    1874            0 :         } else {
    1875              :             // This is a detach notification.  We must update placement policy to avoid re-attaching
    1876              :             // during background scheduling/reconciliation, or during storage controller restart.
    1877            0 :             assert!(attach_req.node_id.is_none());
    1878            0 :             tenant_shard.policy = PlacementPolicy::Detached;
    1879              :         }
    1880              : 
    1881            0 :         if let Some(attaching_pageserver) = attach_req.node_id.as_ref() {
    1882            0 :             tracing::info!(
    1883              :                 tenant_id = %attach_req.tenant_shard_id,
    1884              :                 ps_id = %attaching_pageserver,
    1885              :                 generation = ?tenant_shard.generation,
    1886            0 :                 "issuing",
    1887              :             );
    1888            0 :         } else if let Some(ps_id) = tenant_shard.intent.get_attached() {
    1889            0 :             tracing::info!(
    1890              :                 tenant_id = %attach_req.tenant_shard_id,
    1891              :                 %ps_id,
    1892              :                 generation = ?tenant_shard.generation,
    1893            0 :                 "dropping",
    1894              :             );
    1895              :         } else {
    1896            0 :             tracing::info!(
    1897              :             tenant_id = %attach_req.tenant_shard_id,
    1898            0 :             "no-op: tenant already has no pageserver");
    1899              :         }
    1900            0 :         tenant_shard
    1901            0 :             .intent
    1902            0 :             .set_attached(scheduler, attach_req.node_id);
    1903            0 : 
    1904            0 :         tracing::info!(
    1905            0 :             "attach_hook: tenant {} set generation {:?}, pageserver {}",
    1906            0 :             attach_req.tenant_shard_id,
    1907            0 :             tenant_shard.generation,
    1908            0 :             // TODO: this is an odd number of 0xf's
    1909            0 :             attach_req.node_id.unwrap_or(utils::id::NodeId(0xfffffff))
    1910              :         );
    1911              : 
    1912              :         // Trick the reconciler into not doing anything for this tenant: this helps
    1913              :         // tests that manually configure a tenant on the pagesrever, and then call this
    1914              :         // attach hook: they don't want background reconciliation to modify what they
    1915              :         // did to the pageserver.
    1916              :         #[cfg(feature = "testing")]
    1917              :         {
    1918            0 :             if let Some(node_id) = attach_req.node_id {
    1919            0 :                 tenant_shard.observed.locations = HashMap::from([(
    1920            0 :                     node_id,
    1921            0 :                     ObservedStateLocation {
    1922            0 :                         conf: Some(attached_location_conf(
    1923            0 :                             tenant_shard.generation.unwrap(),
    1924            0 :                             &tenant_shard.shard,
    1925            0 :                             &tenant_shard.config,
    1926            0 :                             &PlacementPolicy::Attached(0),
    1927            0 :                         )),
    1928            0 :                     },
    1929            0 :                 )]);
    1930            0 :             } else {
    1931            0 :                 tenant_shard.observed.locations.clear();
    1932            0 :             }
    1933              :         }
    1934              : 
    1935            0 :         Ok(AttachHookResponse {
    1936            0 :             generation: attach_req
    1937            0 :                 .node_id
    1938            0 :                 .map(|_| tenant_shard.generation.expect("Test hook, not used on tenants that are mid-onboarding with a NULL generation").into().unwrap()),
    1939            0 :         })
    1940            0 :     }
    1941              : 
    1942            0 :     pub(crate) fn inspect(&self, inspect_req: InspectRequest) -> InspectResponse {
    1943            0 :         let locked = self.inner.read().unwrap();
    1944            0 : 
    1945            0 :         let tenant_shard = locked.tenants.get(&inspect_req.tenant_shard_id);
    1946            0 : 
    1947            0 :         InspectResponse {
    1948            0 :             attachment: tenant_shard.and_then(|s| {
    1949            0 :                 s.intent
    1950            0 :                     .get_attached()
    1951            0 :                     .map(|ps| (s.generation.expect("Test hook, not used on tenants that are mid-onboarding with a NULL generation").into().unwrap(), ps))
    1952            0 :             }),
    1953            0 :         }
    1954            0 :     }
    1955              : 
    1956              :     // When the availability state of a node transitions to active, we must do a full reconciliation
    1957              :     // of LocationConfigs on that node.  This is because while a node was offline:
    1958              :     // - we might have proceeded through startup_reconcile without checking for extraneous LocationConfigs on this node
    1959              :     // - aborting a tenant shard split might have left rogue child shards behind on this node.
    1960              :     //
    1961              :     // This function must complete _before_ setting a `Node` to Active: once it is set to Active, other
    1962              :     // Reconcilers might communicate with the node, and these must not overlap with the work we do in
    1963              :     // this function.
    1964              :     //
    1965              :     // The reconciliation logic in here is very similar to what [`Self::startup_reconcile`] does, but
    1966              :     // for written for a single node rather than as a batch job for all nodes.
    1967              :     #[tracing::instrument(skip_all, fields(node_id=%node.get_id()))]
    1968              :     async fn node_activate_reconcile(
    1969              :         &self,
    1970              :         mut node: Node,
    1971              :         _lock: &TracingExclusiveGuard<NodeOperations>,
    1972              :     ) -> Result<(), ApiError> {
    1973              :         // This Node is a mutable local copy: we will set it active so that we can use its
    1974              :         // API client to reconcile with the node.  The Node in [`Self::nodes`] will get updated
    1975              :         // later.
    1976              :         node.set_availability(NodeAvailability::Active(PageserverUtilization::full()));
    1977              : 
    1978              :         let configs = match node
    1979              :             .with_client_retries(
    1980            0 :                 |client| async move { client.list_location_config().await },
    1981              :                 &self.config.pageserver_jwt_token,
    1982              :                 &self.config.ssl_ca_cert,
    1983              :                 1,
    1984              :                 5,
    1985              :                 SHORT_RECONCILE_TIMEOUT,
    1986              :                 &self.cancel,
    1987              :             )
    1988              :             .await
    1989              :         {
    1990              :             None => {
    1991              :                 // We're shutting down (the Node's cancellation token can't have fired, because
    1992              :                 // we're the only scope that has a reference to it, and we didn't fire it).
    1993              :                 return Err(ApiError::ShuttingDown);
    1994              :             }
    1995              :             Some(Err(e)) => {
    1996              :                 // This node didn't succeed listing its locations: it may not proceed to active state
    1997              :                 // as it is apparently unavailable.
    1998              :                 return Err(ApiError::PreconditionFailed(
    1999              :                     format!("Failed to query node location configs, cannot activate ({e})").into(),
    2000              :                 ));
    2001              :             }
    2002              :             Some(Ok(configs)) => configs,
    2003              :         };
    2004              :         tracing::info!("Loaded {} LocationConfigs", configs.tenant_shards.len());
    2005              : 
    2006              :         let mut cleanup = Vec::new();
    2007              :         let mut mismatched_locations = 0;
    2008              :         {
    2009              :             let mut locked = self.inner.write().unwrap();
    2010              : 
    2011              :             for (tenant_shard_id, reported) in configs.tenant_shards {
    2012              :                 let Some(tenant_shard) = locked.tenants.get_mut(&tenant_shard_id) else {
    2013              :                     cleanup.push(tenant_shard_id);
    2014              :                     continue;
    2015              :                 };
    2016              : 
    2017              :                 let on_record = &mut tenant_shard
    2018              :                     .observed
    2019              :                     .locations
    2020              :                     .entry(node.get_id())
    2021            0 :                     .or_insert_with(|| ObservedStateLocation { conf: None })
    2022              :                     .conf;
    2023              : 
    2024              :                 // If the location reported by the node does not match our observed state,
    2025              :                 // then we mark it as uncertain and let the background reconciliation loop
    2026              :                 // deal with it.
    2027              :                 //
    2028              :                 // Note that this also covers net new locations reported by the node.
    2029              :                 if *on_record != reported {
    2030              :                     mismatched_locations += 1;
    2031              :                     *on_record = None;
    2032              :                 }
    2033              :             }
    2034              :         }
    2035              : 
    2036              :         if mismatched_locations > 0 {
    2037              :             tracing::info!(
    2038              :                 "Set observed state to None for {mismatched_locations} mismatched locations"
    2039              :             );
    2040              :         }
    2041              : 
    2042              :         for tenant_shard_id in cleanup {
    2043              :             tracing::info!("Detaching {tenant_shard_id}");
    2044              :             match node
    2045              :                 .with_client_retries(
    2046            0 :                     |client| async move {
    2047            0 :                         let config = LocationConfig {
    2048            0 :                             mode: LocationConfigMode::Detached,
    2049            0 :                             generation: None,
    2050            0 :                             secondary_conf: None,
    2051            0 :                             shard_number: tenant_shard_id.shard_number.0,
    2052            0 :                             shard_count: tenant_shard_id.shard_count.literal(),
    2053            0 :                             shard_stripe_size: 0,
    2054            0 :                             tenant_conf: models::TenantConfig::default(),
    2055            0 :                         };
    2056            0 :                         client
    2057            0 :                             .location_config(tenant_shard_id, config, None, false)
    2058            0 :                             .await
    2059            0 :                     },
    2060              :                     &self.config.pageserver_jwt_token,
    2061              :                     &self.config.ssl_ca_cert,
    2062              :                     1,
    2063              :                     5,
    2064              :                     SHORT_RECONCILE_TIMEOUT,
    2065              :                     &self.cancel,
    2066              :                 )
    2067              :                 .await
    2068              :             {
    2069              :                 None => {
    2070              :                     // We're shutting down (the Node's cancellation token can't have fired, because
    2071              :                     // we're the only scope that has a reference to it, and we didn't fire it).
    2072              :                     return Err(ApiError::ShuttingDown);
    2073              :                 }
    2074              :                 Some(Err(e)) => {
    2075              :                     // Do not let the node proceed to Active state if it is not responsive to requests
    2076              :                     // to detach.  This could happen if e.g. a shutdown bug in the pageserver is preventing
    2077              :                     // detach completing: we should not let this node back into the set of nodes considered
    2078              :                     // okay for scheduling.
    2079              :                     return Err(ApiError::Conflict(format!(
    2080              :                         "Node {node} failed to detach {tenant_shard_id}: {e}"
    2081              :                     )));
    2082              :                 }
    2083              :                 Some(Ok(_)) => {}
    2084              :             };
    2085              :         }
    2086              : 
    2087              :         Ok(())
    2088              :     }
    2089              : 
    2090            0 :     pub(crate) async fn re_attach(
    2091            0 :         &self,
    2092            0 :         reattach_req: ReAttachRequest,
    2093            0 :     ) -> Result<ReAttachResponse, ApiError> {
    2094            0 :         if let Some(register_req) = reattach_req.register {
    2095            0 :             self.node_register(register_req).await?;
    2096            0 :         }
    2097              : 
    2098              :         // Ordering: we must persist generation number updates before making them visible in the in-memory state
    2099            0 :         let incremented_generations = self.persistence.re_attach(reattach_req.node_id).await?;
    2100              : 
    2101            0 :         tracing::info!(
    2102              :             node_id=%reattach_req.node_id,
    2103            0 :             "Incremented {} tenant shards' generations",
    2104            0 :             incremented_generations.len()
    2105              :         );
    2106              : 
    2107              :         // Apply the updated generation to our in-memory state, and
    2108              :         // gather discover secondary locations.
    2109            0 :         let mut locked = self.inner.write().unwrap();
    2110            0 :         let (nodes, tenants, scheduler) = locked.parts_mut();
    2111            0 : 
    2112            0 :         let mut response = ReAttachResponse {
    2113            0 :             tenants: Vec::new(),
    2114            0 :         };
    2115              : 
    2116              :         // TODO: cancel/restart any running reconciliation for this tenant, it might be trying
    2117              :         // to call location_conf API with an old generation.  Wait for cancellation to complete
    2118              :         // before responding to this request.  Requires well implemented CancellationToken logic
    2119              :         // all the way to where we call location_conf.  Even then, there can still be a location_conf
    2120              :         // request in flight over the network: TODO handle that by making location_conf API refuse
    2121              :         // to go backward in generations.
    2122              : 
    2123              :         // Scan through all shards, applying updates for ones where we updated generation
    2124              :         // and identifying shards that intend to have a secondary location on this node.
    2125            0 :         for (tenant_shard_id, shard) in tenants {
    2126            0 :             if let Some(new_gen) = incremented_generations.get(tenant_shard_id) {
    2127            0 :                 let new_gen = *new_gen;
    2128            0 :                 response.tenants.push(ReAttachResponseTenant {
    2129            0 :                     id: *tenant_shard_id,
    2130            0 :                     r#gen: Some(new_gen.into().unwrap()),
    2131            0 :                     // A tenant is only put into multi or stale modes in the middle of a [`Reconciler::live_migrate`]
    2132            0 :                     // execution.  If a pageserver is restarted during that process, then the reconcile pass will
    2133            0 :                     // fail, and start from scratch, so it doesn't make sense for us to try and preserve
    2134            0 :                     // the stale/multi states at this point.
    2135            0 :                     mode: LocationConfigMode::AttachedSingle,
    2136            0 :                 });
    2137            0 : 
    2138            0 :                 shard.generation = std::cmp::max(shard.generation, Some(new_gen));
    2139            0 :                 if let Some(observed) = shard.observed.locations.get_mut(&reattach_req.node_id) {
    2140              :                     // Why can we update `observed` even though we're not sure our response will be received
    2141              :                     // by the pageserver?  Because the pageserver will not proceed with startup until
    2142              :                     // it has processed response: if it loses it, we'll see another request and increment
    2143              :                     // generation again, avoiding any uncertainty about dirtiness of tenant's state.
    2144            0 :                     if let Some(conf) = observed.conf.as_mut() {
    2145            0 :                         conf.generation = new_gen.into();
    2146            0 :                     }
    2147            0 :                 } else {
    2148            0 :                     // This node has no observed state for the shard: perhaps it was offline
    2149            0 :                     // when the pageserver restarted.  Insert a None, so that the Reconciler
    2150            0 :                     // will be prompted to learn the location's state before it makes changes.
    2151            0 :                     shard
    2152            0 :                         .observed
    2153            0 :                         .locations
    2154            0 :                         .insert(reattach_req.node_id, ObservedStateLocation { conf: None });
    2155            0 :                 }
    2156            0 :             } else if shard.intent.get_secondary().contains(&reattach_req.node_id) {
    2157            0 :                 // Ordering: pageserver will not accept /location_config requests until it has
    2158            0 :                 // finished processing the response from re-attach.  So we can update our in-memory state
    2159            0 :                 // now, and be confident that we are not stamping on the result of some later location config.
    2160            0 :                 // TODO: however, we are not strictly ordered wrt ReconcileResults queue,
    2161            0 :                 // so we might update observed state here, and then get over-written by some racing
    2162            0 :                 // ReconcileResult.  The impact is low however, since we have set state on pageserver something
    2163            0 :                 // that matches intent, so worst case if we race then we end up doing a spurious reconcile.
    2164            0 : 
    2165            0 :                 response.tenants.push(ReAttachResponseTenant {
    2166            0 :                     id: *tenant_shard_id,
    2167            0 :                     r#gen: None,
    2168            0 :                     mode: LocationConfigMode::Secondary,
    2169            0 :                 });
    2170            0 : 
    2171            0 :                 // We must not update observed, because we have no guarantee that our
    2172            0 :                 // response will be received by the pageserver. This could leave it
    2173            0 :                 // falsely dirty, but the resulting reconcile should be idempotent.
    2174            0 :             }
    2175              :         }
    2176              : 
    2177              :         // We consider a node Active once we have composed a re-attach response, but we
    2178              :         // do not call [`Self::node_activate_reconcile`]: the handling of the re-attach response
    2179              :         // implicitly synchronizes the LocationConfigs on the node.
    2180              :         //
    2181              :         // Setting a node active unblocks any Reconcilers that might write to the location config API,
    2182              :         // but those requests will not be accepted by the node until it has finished processing
    2183              :         // the re-attach response.
    2184              :         //
    2185              :         // Additionally, reset the nodes scheduling policy to match the conditional update done
    2186              :         // in [`Persistence::re_attach`].
    2187            0 :         if let Some(node) = nodes.get(&reattach_req.node_id) {
    2188            0 :             let reset_scheduling = matches!(
    2189            0 :                 node.get_scheduling(),
    2190              :                 NodeSchedulingPolicy::PauseForRestart
    2191              :                     | NodeSchedulingPolicy::Draining
    2192              :                     | NodeSchedulingPolicy::Filling
    2193              :             );
    2194              : 
    2195            0 :             let mut new_nodes = (**nodes).clone();
    2196            0 :             if let Some(node) = new_nodes.get_mut(&reattach_req.node_id) {
    2197            0 :                 if reset_scheduling {
    2198            0 :                     node.set_scheduling(NodeSchedulingPolicy::Active);
    2199            0 :                 }
    2200              : 
    2201            0 :                 tracing::info!("Marking {} warming-up on reattach", reattach_req.node_id);
    2202            0 :                 node.set_availability(NodeAvailability::WarmingUp(std::time::Instant::now()));
    2203            0 : 
    2204            0 :                 scheduler.node_upsert(node);
    2205            0 :                 let new_nodes = Arc::new(new_nodes);
    2206            0 :                 *nodes = new_nodes;
    2207              :             } else {
    2208            0 :                 tracing::error!(
    2209            0 :                     "Reattaching node {} was removed while processing the request",
    2210              :                     reattach_req.node_id
    2211              :                 );
    2212              :             }
    2213            0 :         }
    2214              : 
    2215            0 :         Ok(response)
    2216            0 :     }
    2217              : 
    2218            0 :     pub(crate) async fn validate(
    2219            0 :         &self,
    2220            0 :         validate_req: ValidateRequest,
    2221            0 :     ) -> Result<ValidateResponse, DatabaseError> {
    2222              :         // Fast in-memory check: we may reject validation on anything that doesn't match our
    2223              :         // in-memory generation for a shard
    2224            0 :         let in_memory_result = {
    2225            0 :             let mut in_memory_result = Vec::new();
    2226            0 :             let locked = self.inner.read().unwrap();
    2227            0 :             for req_tenant in validate_req.tenants {
    2228            0 :                 if let Some(tenant_shard) = locked.tenants.get(&req_tenant.id) {
    2229            0 :                     let valid = tenant_shard.generation == Some(Generation::new(req_tenant.r#gen));
    2230            0 :                     tracing::info!(
    2231            0 :                         "handle_validate: {}(gen {}): valid={valid} (latest {:?})",
    2232              :                         req_tenant.id,
    2233              :                         req_tenant.r#gen,
    2234              :                         tenant_shard.generation
    2235              :                     );
    2236              : 
    2237            0 :                     in_memory_result.push((
    2238            0 :                         req_tenant.id,
    2239            0 :                         Generation::new(req_tenant.r#gen),
    2240            0 :                         valid,
    2241            0 :                     ));
    2242              :                 } else {
    2243              :                     // This is legal: for example during a shard split the pageserver may still
    2244              :                     // have deletions in its queue from the old pre-split shard, or after deletion
    2245              :                     // of a tenant that was busy with compaction/gc while being deleted.
    2246            0 :                     tracing::info!(
    2247            0 :                         "Refusing deletion validation for missing shard {}",
    2248              :                         req_tenant.id
    2249              :                     );
    2250              :                 }
    2251              :             }
    2252              : 
    2253            0 :             in_memory_result
    2254              :         };
    2255              : 
    2256              :         // Database calls to confirm validity for anything that passed the in-memory check.  We must do this
    2257              :         // in case of controller split-brain, where some other controller process might have incremented the generation.
    2258            0 :         let db_generations = self
    2259            0 :             .persistence
    2260            0 :             .shard_generations(
    2261            0 :                 in_memory_result
    2262            0 :                     .iter()
    2263            0 :                     .filter_map(|i| if i.2 { Some(&i.0) } else { None }),
    2264            0 :             )
    2265            0 :             .await?;
    2266            0 :         let db_generations = db_generations.into_iter().collect::<HashMap<_, _>>();
    2267            0 : 
    2268            0 :         let mut response = ValidateResponse {
    2269            0 :             tenants: Vec::new(),
    2270            0 :         };
    2271            0 :         for (tenant_shard_id, validate_generation, valid) in in_memory_result.into_iter() {
    2272            0 :             let valid = if valid {
    2273            0 :                 let db_generation = db_generations.get(&tenant_shard_id);
    2274            0 :                 db_generation == Some(&Some(validate_generation))
    2275              :             } else {
    2276              :                 // If in-memory state says it's invalid, trust that.  It's always safe to fail a validation, at worst
    2277              :                 // this prevents a pageserver from cleaning up an object in S3.
    2278            0 :                 false
    2279              :             };
    2280              : 
    2281            0 :             response.tenants.push(ValidateResponseTenant {
    2282            0 :                 id: tenant_shard_id,
    2283            0 :                 valid,
    2284            0 :             })
    2285              :         }
    2286              : 
    2287            0 :         Ok(response)
    2288            0 :     }
    2289              : 
    2290            0 :     pub(crate) async fn tenant_create(
    2291            0 :         &self,
    2292            0 :         create_req: TenantCreateRequest,
    2293            0 :     ) -> Result<TenantCreateResponse, ApiError> {
    2294            0 :         let tenant_id = create_req.new_tenant_id.tenant_id;
    2295              : 
    2296              :         // Exclude any concurrent attempts to create/access the same tenant ID
    2297            0 :         let _tenant_lock = trace_exclusive_lock(
    2298            0 :             &self.tenant_op_locks,
    2299            0 :             create_req.new_tenant_id.tenant_id,
    2300            0 :             TenantOperations::Create,
    2301            0 :         )
    2302            0 :         .await;
    2303            0 :         let (response, waiters) = self.do_tenant_create(create_req).await?;
    2304              : 
    2305            0 :         if let Err(e) = self.await_waiters(waiters, RECONCILE_TIMEOUT).await {
    2306              :             // Avoid deadlock: reconcile may fail while notifying compute, if the cloud control plane refuses to
    2307              :             // accept compute notifications while it is in the process of creating.  Reconciliation will
    2308              :             // be retried in the background.
    2309            0 :             tracing::warn!(%tenant_id, "Reconcile not done yet while creating tenant ({e})");
    2310            0 :         }
    2311            0 :         Ok(response)
    2312            0 :     }
    2313              : 
    2314            0 :     pub(crate) async fn do_tenant_create(
    2315            0 :         &self,
    2316            0 :         create_req: TenantCreateRequest,
    2317            0 :     ) -> Result<(TenantCreateResponse, Vec<ReconcilerWaiter>), ApiError> {
    2318            0 :         let placement_policy = create_req
    2319            0 :             .placement_policy
    2320            0 :             .clone()
    2321            0 :             // As a default, zero secondaries is convenient for tests that don't choose a policy.
    2322            0 :             .unwrap_or(PlacementPolicy::Attached(0));
    2323              : 
    2324              :         // This service expects to handle sharding itself: it is an error to try and directly create
    2325              :         // a particular shard here.
    2326            0 :         let tenant_id = if !create_req.new_tenant_id.is_unsharded() {
    2327            0 :             return Err(ApiError::BadRequest(anyhow::anyhow!(
    2328            0 :                 "Attempted to create a specific shard, this API is for creating the whole tenant"
    2329            0 :             )));
    2330              :         } else {
    2331            0 :             create_req.new_tenant_id.tenant_id
    2332            0 :         };
    2333            0 : 
    2334            0 :         tracing::info!(
    2335            0 :             "Creating tenant {}, shard_count={:?}",
    2336              :             create_req.new_tenant_id,
    2337              :             create_req.shard_parameters.count,
    2338              :         );
    2339              : 
    2340            0 :         let create_ids = (0..create_req.shard_parameters.count.count())
    2341            0 :             .map(|i| TenantShardId {
    2342            0 :                 tenant_id,
    2343            0 :                 shard_number: ShardNumber(i),
    2344            0 :                 shard_count: create_req.shard_parameters.count,
    2345            0 :             })
    2346            0 :             .collect::<Vec<_>>();
    2347              : 
    2348              :         // If the caller specifies a None generation, it means "start from default".  This is different
    2349              :         // to [`Self::tenant_location_config`], where a None generation is used to represent
    2350              :         // an incompletely-onboarded tenant.
    2351            0 :         let initial_generation = if matches!(placement_policy, PlacementPolicy::Secondary) {
    2352            0 :             tracing::info!(
    2353            0 :                 "tenant_create: secondary mode, generation is_some={}",
    2354            0 :                 create_req.generation.is_some()
    2355              :             );
    2356            0 :             create_req.generation.map(Generation::new)
    2357              :         } else {
    2358            0 :             tracing::info!(
    2359            0 :                 "tenant_create: not secondary mode, generation is_some={}",
    2360            0 :                 create_req.generation.is_some()
    2361              :             );
    2362            0 :             Some(
    2363            0 :                 create_req
    2364            0 :                     .generation
    2365            0 :                     .map(Generation::new)
    2366            0 :                     .unwrap_or(INITIAL_GENERATION),
    2367            0 :             )
    2368              :         };
    2369              : 
    2370            0 :         let preferred_az_id = {
    2371            0 :             let locked = self.inner.read().unwrap();
    2372              :             // Idempotency: take the existing value if the tenant already exists
    2373            0 :             if let Some(shard) = locked.tenants.get(create_ids.first().unwrap()) {
    2374            0 :                 shard.preferred_az().cloned()
    2375              :             } else {
    2376            0 :                 locked.scheduler.get_az_for_new_tenant()
    2377              :             }
    2378              :         };
    2379              : 
    2380              :         // Ordering: we persist tenant shards before creating them on the pageserver.  This enables a caller
    2381              :         // to clean up after themselves by issuing a tenant deletion if something goes wrong and we restart
    2382              :         // during the creation, rather than risking leaving orphan objects in S3.
    2383            0 :         let persist_tenant_shards = create_ids
    2384            0 :             .iter()
    2385            0 :             .map(|tenant_shard_id| TenantShardPersistence {
    2386            0 :                 tenant_id: tenant_shard_id.tenant_id.to_string(),
    2387            0 :                 shard_number: tenant_shard_id.shard_number.0 as i32,
    2388            0 :                 shard_count: tenant_shard_id.shard_count.literal() as i32,
    2389            0 :                 shard_stripe_size: create_req.shard_parameters.stripe_size.0 as i32,
    2390            0 :                 generation: initial_generation.map(|g| g.into().unwrap() as i32),
    2391            0 :                 // The pageserver is not known until scheduling happens: we will set this column when
    2392            0 :                 // incrementing the generation the first time we attach to a pageserver.
    2393            0 :                 generation_pageserver: None,
    2394            0 :                 placement_policy: serde_json::to_string(&placement_policy).unwrap(),
    2395            0 :                 config: serde_json::to_string(&create_req.config).unwrap(),
    2396            0 :                 splitting: SplitState::default(),
    2397            0 :                 scheduling_policy: serde_json::to_string(&ShardSchedulingPolicy::default())
    2398            0 :                     .unwrap(),
    2399            0 :                 preferred_az_id: preferred_az_id.as_ref().map(|az| az.to_string()),
    2400            0 :             })
    2401            0 :             .collect();
    2402            0 : 
    2403            0 :         match self
    2404            0 :             .persistence
    2405            0 :             .insert_tenant_shards(persist_tenant_shards)
    2406            0 :             .await
    2407              :         {
    2408            0 :             Ok(_) => {}
    2409              :             Err(DatabaseError::Query(diesel::result::Error::DatabaseError(
    2410              :                 DatabaseErrorKind::UniqueViolation,
    2411              :                 _,
    2412              :             ))) => {
    2413              :                 // Unique key violation: this is probably a retry.  Because the shard count is part of the unique key,
    2414              :                 // if we see a unique key violation it means that the creation request's shard count matches the previous
    2415              :                 // creation's shard count.
    2416            0 :                 tracing::info!(
    2417            0 :                     "Tenant shards already present in database, proceeding with idempotent creation..."
    2418              :                 );
    2419              :             }
    2420              :             // Any other database error is unexpected and a bug.
    2421            0 :             Err(e) => return Err(ApiError::InternalServerError(anyhow::anyhow!(e))),
    2422              :         };
    2423              : 
    2424            0 :         let mut schedule_context = ScheduleContext::default();
    2425            0 :         let mut schedule_error = None;
    2426            0 :         let mut response_shards = Vec::new();
    2427            0 :         for tenant_shard_id in create_ids {
    2428            0 :             tracing::info!("Creating shard {tenant_shard_id}...");
    2429              : 
    2430            0 :             let outcome = self
    2431            0 :                 .do_initial_shard_scheduling(
    2432            0 :                     tenant_shard_id,
    2433            0 :                     initial_generation,
    2434            0 :                     &create_req.shard_parameters,
    2435            0 :                     create_req.config.clone(),
    2436            0 :                     placement_policy.clone(),
    2437            0 :                     preferred_az_id.as_ref(),
    2438            0 :                     &mut schedule_context,
    2439            0 :                 )
    2440            0 :                 .await;
    2441              : 
    2442            0 :             match outcome {
    2443            0 :                 InitialShardScheduleOutcome::Scheduled(resp) => response_shards.push(resp),
    2444            0 :                 InitialShardScheduleOutcome::NotScheduled => {}
    2445            0 :                 InitialShardScheduleOutcome::ShardScheduleError(err) => {
    2446            0 :                     schedule_error = Some(err);
    2447            0 :                 }
    2448              :             }
    2449              :         }
    2450              : 
    2451              :         // If we failed to schedule shards, then they are still created in the controller,
    2452              :         // but we return an error to the requester to avoid a silent failure when someone
    2453              :         // tries to e.g. create a tenant whose placement policy requires more nodes than
    2454              :         // are present in the system.  We do this here rather than in the above loop, to
    2455              :         // avoid situations where we only create a subset of shards in the tenant.
    2456            0 :         if let Some(e) = schedule_error {
    2457            0 :             return Err(ApiError::Conflict(format!(
    2458            0 :                 "Failed to schedule shard(s): {e}"
    2459            0 :             )));
    2460            0 :         }
    2461            0 : 
    2462            0 :         let waiters = {
    2463            0 :             let mut locked = self.inner.write().unwrap();
    2464            0 :             let (nodes, tenants, _scheduler) = locked.parts_mut();
    2465            0 :             let config = ReconcilerConfigBuilder::new(ReconcilerPriority::High)
    2466            0 :                 .tenant_creation_hint(true)
    2467            0 :                 .build();
    2468            0 :             tenants
    2469            0 :                 .range_mut(TenantShardId::tenant_range(tenant_id))
    2470            0 :                 .filter_map(|(_shard_id, shard)| {
    2471            0 :                     self.maybe_configured_reconcile_shard(shard, nodes, config)
    2472            0 :                 })
    2473            0 :                 .collect::<Vec<_>>()
    2474            0 :         };
    2475            0 : 
    2476            0 :         Ok((
    2477            0 :             TenantCreateResponse {
    2478            0 :                 shards: response_shards,
    2479            0 :             },
    2480            0 :             waiters,
    2481            0 :         ))
    2482            0 :     }
    2483              : 
    2484              :     /// Helper for tenant creation that does the scheduling for an individual shard. Covers both the
    2485              :     /// case of a new tenant and a pre-existing one.
    2486              :     #[allow(clippy::too_many_arguments)]
    2487            0 :     async fn do_initial_shard_scheduling(
    2488            0 :         &self,
    2489            0 :         tenant_shard_id: TenantShardId,
    2490            0 :         initial_generation: Option<Generation>,
    2491            0 :         shard_params: &ShardParameters,
    2492            0 :         config: TenantConfig,
    2493            0 :         placement_policy: PlacementPolicy,
    2494            0 :         preferred_az_id: Option<&AvailabilityZone>,
    2495            0 :         schedule_context: &mut ScheduleContext,
    2496            0 :     ) -> InitialShardScheduleOutcome {
    2497            0 :         let mut locked = self.inner.write().unwrap();
    2498            0 :         let (_nodes, tenants, scheduler) = locked.parts_mut();
    2499              : 
    2500              :         use std::collections::btree_map::Entry;
    2501            0 :         match tenants.entry(tenant_shard_id) {
    2502            0 :             Entry::Occupied(mut entry) => {
    2503            0 :                 tracing::info!("Tenant shard {tenant_shard_id} already exists while creating");
    2504              : 
    2505            0 :                 if let Err(err) = entry.get_mut().schedule(scheduler, schedule_context) {
    2506            0 :                     return InitialShardScheduleOutcome::ShardScheduleError(err);
    2507            0 :                 }
    2508              : 
    2509            0 :                 if let Some(node_id) = entry.get().intent.get_attached() {
    2510            0 :                     let generation = entry
    2511            0 :                         .get()
    2512            0 :                         .generation
    2513            0 :                         .expect("Generation is set when in attached mode");
    2514            0 :                     InitialShardScheduleOutcome::Scheduled(TenantCreateResponseShard {
    2515            0 :                         shard_id: tenant_shard_id,
    2516            0 :                         node_id: *node_id,
    2517            0 :                         generation: generation.into().unwrap(),
    2518            0 :                     })
    2519              :                 } else {
    2520            0 :                     InitialShardScheduleOutcome::NotScheduled
    2521              :                 }
    2522              :             }
    2523            0 :             Entry::Vacant(entry) => {
    2524            0 :                 let state = entry.insert(TenantShard::new(
    2525            0 :                     tenant_shard_id,
    2526            0 :                     ShardIdentity::from_params(tenant_shard_id.shard_number, shard_params),
    2527            0 :                     placement_policy,
    2528            0 :                     preferred_az_id.cloned(),
    2529            0 :                 ));
    2530            0 : 
    2531            0 :                 state.generation = initial_generation;
    2532            0 :                 state.config = config;
    2533            0 :                 if let Err(e) = state.schedule(scheduler, schedule_context) {
    2534            0 :                     return InitialShardScheduleOutcome::ShardScheduleError(e);
    2535            0 :                 }
    2536              : 
    2537              :                 // Only include shards in result if we are attaching: the purpose
    2538              :                 // of the response is to tell the caller where the shards are attached.
    2539            0 :                 if let Some(node_id) = state.intent.get_attached() {
    2540            0 :                     let generation = state
    2541            0 :                         .generation
    2542            0 :                         .expect("Generation is set when in attached mode");
    2543            0 :                     InitialShardScheduleOutcome::Scheduled(TenantCreateResponseShard {
    2544            0 :                         shard_id: tenant_shard_id,
    2545            0 :                         node_id: *node_id,
    2546            0 :                         generation: generation.into().unwrap(),
    2547            0 :                     })
    2548              :                 } else {
    2549            0 :                     InitialShardScheduleOutcome::NotScheduled
    2550              :                 }
    2551              :             }
    2552              :         }
    2553            0 :     }
    2554              : 
    2555              :     /// Helper for functions that reconcile a number of shards, and would like to do a timeout-bounded
    2556              :     /// wait for reconciliation to complete before responding.
    2557            0 :     async fn await_waiters(
    2558            0 :         &self,
    2559            0 :         waiters: Vec<ReconcilerWaiter>,
    2560            0 :         timeout: Duration,
    2561            0 :     ) -> Result<(), ReconcileWaitError> {
    2562            0 :         let deadline = Instant::now().checked_add(timeout).unwrap();
    2563            0 :         for waiter in waiters {
    2564            0 :             let timeout = deadline.duration_since(Instant::now());
    2565            0 :             waiter.wait_timeout(timeout).await?;
    2566              :         }
    2567              : 
    2568            0 :         Ok(())
    2569            0 :     }
    2570              : 
    2571              :     /// Same as [`Service::await_waiters`], but returns the waiters which are still
    2572              :     /// in progress
    2573            0 :     async fn await_waiters_remainder(
    2574            0 :         &self,
    2575            0 :         waiters: Vec<ReconcilerWaiter>,
    2576            0 :         timeout: Duration,
    2577            0 :     ) -> Vec<ReconcilerWaiter> {
    2578            0 :         let deadline = Instant::now().checked_add(timeout).unwrap();
    2579            0 :         for waiter in waiters.iter() {
    2580            0 :             let timeout = deadline.duration_since(Instant::now());
    2581            0 :             let _ = waiter.wait_timeout(timeout).await;
    2582              :         }
    2583              : 
    2584            0 :         waiters
    2585            0 :             .into_iter()
    2586            0 :             .filter(|waiter| matches!(waiter.get_status(), ReconcilerStatus::InProgress))
    2587            0 :             .collect::<Vec<_>>()
    2588            0 :     }
    2589              : 
    2590              :     /// Part of [`Self::tenant_location_config`]: dissect an incoming location config request,
    2591              :     /// and transform it into either a tenant creation of a series of shard updates.
    2592              :     ///
    2593              :     /// If the incoming request makes no changes, a [`TenantCreateOrUpdate::Update`] result will
    2594              :     /// still be returned.
    2595            0 :     fn tenant_location_config_prepare(
    2596            0 :         &self,
    2597            0 :         tenant_id: TenantId,
    2598            0 :         req: TenantLocationConfigRequest,
    2599            0 :     ) -> TenantCreateOrUpdate {
    2600            0 :         let mut updates = Vec::new();
    2601            0 :         let mut locked = self.inner.write().unwrap();
    2602            0 :         let (nodes, tenants, _scheduler) = locked.parts_mut();
    2603            0 :         let tenant_shard_id = TenantShardId::unsharded(tenant_id);
    2604              : 
    2605              :         // Use location config mode as an indicator of policy.
    2606            0 :         let placement_policy = match req.config.mode {
    2607            0 :             LocationConfigMode::Detached => PlacementPolicy::Detached,
    2608            0 :             LocationConfigMode::Secondary => PlacementPolicy::Secondary,
    2609              :             LocationConfigMode::AttachedMulti
    2610              :             | LocationConfigMode::AttachedSingle
    2611              :             | LocationConfigMode::AttachedStale => {
    2612            0 :                 if nodes.len() > 1 {
    2613            0 :                     PlacementPolicy::Attached(1)
    2614              :                 } else {
    2615              :                     // Convenience for dev/test: if we just have one pageserver, import
    2616              :                     // tenants into non-HA mode so that scheduling will succeed.
    2617            0 :                     PlacementPolicy::Attached(0)
    2618              :                 }
    2619              :             }
    2620              :         };
    2621              : 
    2622              :         // Ordinarily we do not update scheduling policy, but when making major changes
    2623              :         // like detaching or demoting to secondary-only, we need to force the scheduling
    2624              :         // mode to Active, or the caller's expected outcome (detach it) will not happen.
    2625            0 :         let scheduling_policy = match req.config.mode {
    2626              :             LocationConfigMode::Detached | LocationConfigMode::Secondary => {
    2627              :                 // Special case: when making major changes like detaching or demoting to secondary-only,
    2628              :                 // we need to force the scheduling mode to Active, or nothing will happen.
    2629            0 :                 Some(ShardSchedulingPolicy::Active)
    2630              :             }
    2631              :             LocationConfigMode::AttachedMulti
    2632              :             | LocationConfigMode::AttachedSingle
    2633              :             | LocationConfigMode::AttachedStale => {
    2634              :                 // While attached, continue to respect whatever the existing scheduling mode is.
    2635            0 :                 None
    2636              :             }
    2637              :         };
    2638              : 
    2639            0 :         let mut create = true;
    2640            0 :         for (shard_id, shard) in tenants.range_mut(TenantShardId::tenant_range(tenant_id)) {
    2641              :             // Saw an existing shard: this is not a creation
    2642            0 :             create = false;
    2643              : 
    2644              :             // Shards may have initially been created by a Secondary request, where we
    2645              :             // would have left generation as None.
    2646              :             //
    2647              :             // We only update generation the first time we see an attached-mode request,
    2648              :             // and if there is no existing generation set. The caller is responsible for
    2649              :             // ensuring that no non-storage-controller pageserver ever uses a higher
    2650              :             // generation than they passed in here.
    2651              :             use LocationConfigMode::*;
    2652            0 :             let set_generation = match req.config.mode {
    2653            0 :                 AttachedMulti | AttachedSingle | AttachedStale if shard.generation.is_none() => {
    2654            0 :                     req.config.generation.map(Generation::new)
    2655              :                 }
    2656            0 :                 _ => None,
    2657              :             };
    2658              : 
    2659            0 :             updates.push(ShardUpdate {
    2660            0 :                 tenant_shard_id: *shard_id,
    2661            0 :                 placement_policy: placement_policy.clone(),
    2662            0 :                 tenant_config: req.config.tenant_conf.clone(),
    2663            0 :                 generation: set_generation,
    2664            0 :                 scheduling_policy,
    2665            0 :             });
    2666              :         }
    2667              : 
    2668            0 :         if create {
    2669              :             use LocationConfigMode::*;
    2670            0 :             let generation = match req.config.mode {
    2671            0 :                 AttachedMulti | AttachedSingle | AttachedStale => req.config.generation,
    2672              :                 // If a caller provided a generation in a non-attached request, ignore it
    2673              :                 // and leave our generation as None: this enables a subsequent update to set
    2674              :                 // the generation when setting an attached mode for the first time.
    2675            0 :                 _ => None,
    2676              :             };
    2677              : 
    2678            0 :             TenantCreateOrUpdate::Create(
    2679            0 :                 // Synthesize a creation request
    2680            0 :                 TenantCreateRequest {
    2681            0 :                     new_tenant_id: tenant_shard_id,
    2682            0 :                     generation,
    2683            0 :                     shard_parameters: ShardParameters {
    2684            0 :                         count: tenant_shard_id.shard_count,
    2685            0 :                         // We only import un-sharded or single-sharded tenants, so stripe
    2686            0 :                         // size can be made up arbitrarily here.
    2687            0 :                         stripe_size: ShardParameters::DEFAULT_STRIPE_SIZE,
    2688            0 :                     },
    2689            0 :                     placement_policy: Some(placement_policy),
    2690            0 :                     config: req.config.tenant_conf,
    2691            0 :                 },
    2692            0 :             )
    2693              :         } else {
    2694            0 :             assert!(!updates.is_empty());
    2695            0 :             TenantCreateOrUpdate::Update(updates)
    2696              :         }
    2697            0 :     }
    2698              : 
    2699              :     /// For APIs that might act on tenants with [`PlacementPolicy::Detached`], first check if
    2700              :     /// the tenant is present in memory. If not, load it from the database.  If it is found
    2701              :     /// in neither location, return a NotFound error.
    2702              :     ///
    2703              :     /// Caller must demonstrate they hold a lock guard, as otherwise two callers might try and load
    2704              :     /// it at the same time, or we might race with [`Self::maybe_drop_tenant`]
    2705            0 :     async fn maybe_load_tenant(
    2706            0 :         &self,
    2707            0 :         tenant_id: TenantId,
    2708            0 :         _guard: &TracingExclusiveGuard<TenantOperations>,
    2709            0 :     ) -> Result<(), ApiError> {
    2710              :         // Check if the tenant is present in memory, and select an AZ to use when loading
    2711              :         // if we will load it.
    2712            0 :         let load_in_az = {
    2713            0 :             let locked = self.inner.read().unwrap();
    2714            0 :             let existing = locked
    2715            0 :                 .tenants
    2716            0 :                 .range(TenantShardId::tenant_range(tenant_id))
    2717            0 :                 .next();
    2718            0 : 
    2719            0 :             // If the tenant is not present in memory, we expect to load it from database,
    2720            0 :             // so let's figure out what AZ to load it into while we have self.inner locked.
    2721            0 :             if existing.is_none() {
    2722            0 :                 locked
    2723            0 :                     .scheduler
    2724            0 :                     .get_az_for_new_tenant()
    2725            0 :                     .ok_or(ApiError::BadRequest(anyhow::anyhow!(
    2726            0 :                         "No AZ with nodes found to load tenant"
    2727            0 :                     )))?
    2728              :             } else {
    2729              :                 // We already have this tenant in memory
    2730            0 :                 return Ok(());
    2731              :             }
    2732              :         };
    2733              : 
    2734            0 :         let tenant_shards = self.persistence.load_tenant(tenant_id).await?;
    2735            0 :         if tenant_shards.is_empty() {
    2736            0 :             return Err(ApiError::NotFound(
    2737            0 :                 anyhow::anyhow!("Tenant {} not found", tenant_id).into(),
    2738            0 :             ));
    2739            0 :         }
    2740            0 : 
    2741            0 :         // Update the persistent shards with the AZ that we are about to apply to in-memory state
    2742            0 :         self.persistence
    2743            0 :             .set_tenant_shard_preferred_azs(
    2744            0 :                 tenant_shards
    2745            0 :                     .iter()
    2746            0 :                     .map(|t| {
    2747            0 :                         (
    2748            0 :                             t.get_tenant_shard_id().expect("Corrupt shard in database"),
    2749            0 :                             Some(load_in_az.clone()),
    2750            0 :                         )
    2751            0 :                     })
    2752            0 :                     .collect(),
    2753            0 :             )
    2754            0 :             .await?;
    2755              : 
    2756            0 :         let mut locked = self.inner.write().unwrap();
    2757            0 :         tracing::info!(
    2758            0 :             "Loaded {} shards for tenant {}",
    2759            0 :             tenant_shards.len(),
    2760              :             tenant_id
    2761              :         );
    2762              : 
    2763            0 :         locked.tenants.extend(tenant_shards.into_iter().map(|p| {
    2764            0 :             let intent = IntentState::new(Some(load_in_az.clone()));
    2765            0 :             let shard =
    2766            0 :                 TenantShard::from_persistent(p, intent).expect("Corrupt shard row in database");
    2767            0 : 
    2768            0 :             // Sanity check: when loading on-demand, we should always be loaded something Detached
    2769            0 :             debug_assert!(shard.policy == PlacementPolicy::Detached);
    2770            0 :             if shard.policy != PlacementPolicy::Detached {
    2771            0 :                 tracing::error!(
    2772            0 :                     "Tenant shard {} loaded on-demand, but has non-Detached policy {:?}",
    2773              :                     shard.tenant_shard_id,
    2774              :                     shard.policy
    2775              :                 );
    2776            0 :             }
    2777              : 
    2778            0 :             (shard.tenant_shard_id, shard)
    2779            0 :         }));
    2780            0 : 
    2781            0 :         Ok(())
    2782            0 :     }
    2783              : 
    2784              :     /// If all shards for a tenant are detached, and in a fully quiescent state (no observed locations on pageservers),
    2785              :     /// and have no reconciler running, then we can drop the tenant from memory.  It will be reloaded on-demand
    2786              :     /// if we are asked to attach it again (see [`Self::maybe_load_tenant`]).
    2787              :     ///
    2788              :     /// Caller must demonstrate they hold a lock guard, as otherwise it is unsafe to drop a tenant from
    2789              :     /// memory while some other function might assume it continues to exist while not holding the lock on Self::inner.
    2790            0 :     fn maybe_drop_tenant(
    2791            0 :         &self,
    2792            0 :         tenant_id: TenantId,
    2793            0 :         locked: &mut std::sync::RwLockWriteGuard<ServiceState>,
    2794            0 :         _guard: &TracingExclusiveGuard<TenantOperations>,
    2795            0 :     ) {
    2796            0 :         let mut tenant_shards = locked.tenants.range(TenantShardId::tenant_range(tenant_id));
    2797            0 :         if tenant_shards.all(|(_id, shard)| {
    2798            0 :             shard.policy == PlacementPolicy::Detached
    2799            0 :                 && shard.reconciler.is_none()
    2800            0 :                 && shard.observed.is_empty()
    2801            0 :         }) {
    2802            0 :             let keys = locked
    2803            0 :                 .tenants
    2804            0 :                 .range(TenantShardId::tenant_range(tenant_id))
    2805            0 :                 .map(|(id, _)| id)
    2806            0 :                 .copied()
    2807            0 :                 .collect::<Vec<_>>();
    2808            0 :             for key in keys {
    2809            0 :                 tracing::info!("Dropping detached tenant shard {} from memory", key);
    2810            0 :                 locked.tenants.remove(&key);
    2811              :             }
    2812            0 :         }
    2813            0 :     }
    2814              : 
    2815              :     /// This API is used by the cloud control plane to migrate unsharded tenants that it created
    2816              :     /// directly with pageservers into this service.
    2817              :     ///
    2818              :     /// Cloud control plane MUST NOT continue issuing GENERATION NUMBERS for this tenant once it
    2819              :     /// has attempted to call this API. Failure to oblige to this rule may lead to S3 corruption.
    2820              :     /// Think of the first attempt to call this API as a transfer of absolute authority over the
    2821              :     /// tenant's source of generation numbers.
    2822              :     ///
    2823              :     /// The mode in this request coarse-grained control of tenants:
    2824              :     /// - Call with mode Attached* to upsert the tenant.
    2825              :     /// - Call with mode Secondary to either onboard a tenant without attaching it, or
    2826              :     ///   to set an existing tenant to PolicyMode::Secondary
    2827              :     /// - Call with mode Detached to switch to PolicyMode::Detached
    2828            0 :     pub(crate) async fn tenant_location_config(
    2829            0 :         &self,
    2830            0 :         tenant_shard_id: TenantShardId,
    2831            0 :         req: TenantLocationConfigRequest,
    2832            0 :     ) -> Result<TenantLocationConfigResponse, ApiError> {
    2833              :         // We require an exclusive lock, because we are updating both persistent and in-memory state
    2834            0 :         let _tenant_lock = trace_exclusive_lock(
    2835            0 :             &self.tenant_op_locks,
    2836            0 :             tenant_shard_id.tenant_id,
    2837            0 :             TenantOperations::LocationConfig,
    2838            0 :         )
    2839            0 :         .await;
    2840              : 
    2841            0 :         let tenant_id = if !tenant_shard_id.is_unsharded() {
    2842            0 :             return Err(ApiError::BadRequest(anyhow::anyhow!(
    2843            0 :                 "This API is for importing single-sharded or unsharded tenants"
    2844            0 :             )));
    2845              :         } else {
    2846            0 :             tenant_shard_id.tenant_id
    2847            0 :         };
    2848            0 : 
    2849            0 :         // In case we are waking up a Detached tenant
    2850            0 :         match self.maybe_load_tenant(tenant_id, &_tenant_lock).await {
    2851            0 :             Ok(()) | Err(ApiError::NotFound(_)) => {
    2852            0 :                 // This is a creation or an update
    2853            0 :             }
    2854            0 :             Err(e) => {
    2855            0 :                 return Err(e);
    2856              :             }
    2857              :         };
    2858              : 
    2859              :         // First check if this is a creation or an update
    2860            0 :         let create_or_update = self.tenant_location_config_prepare(tenant_id, req);
    2861            0 : 
    2862            0 :         let mut result = TenantLocationConfigResponse {
    2863            0 :             shards: Vec::new(),
    2864            0 :             stripe_size: None,
    2865            0 :         };
    2866            0 :         let waiters = match create_or_update {
    2867            0 :             TenantCreateOrUpdate::Create(create_req) => {
    2868            0 :                 let (create_resp, waiters) = self.do_tenant_create(create_req).await?;
    2869            0 :                 result.shards = create_resp
    2870            0 :                     .shards
    2871            0 :                     .into_iter()
    2872            0 :                     .map(|s| TenantShardLocation {
    2873            0 :                         node_id: s.node_id,
    2874            0 :                         shard_id: s.shard_id,
    2875            0 :                     })
    2876            0 :                     .collect();
    2877            0 :                 waiters
    2878              :             }
    2879            0 :             TenantCreateOrUpdate::Update(updates) => {
    2880            0 :                 // Persist updates
    2881            0 :                 // Ordering: write to the database before applying changes in-memory, so that
    2882            0 :                 // we will not appear time-travel backwards on a restart.
    2883            0 : 
    2884            0 :                 let mut schedule_context = ScheduleContext::default();
    2885              :                 for ShardUpdate {
    2886            0 :                     tenant_shard_id,
    2887            0 :                     placement_policy,
    2888            0 :                     tenant_config,
    2889            0 :                     generation,
    2890            0 :                     scheduling_policy,
    2891            0 :                 } in &updates
    2892              :                 {
    2893            0 :                     self.persistence
    2894            0 :                         .update_tenant_shard(
    2895            0 :                             TenantFilter::Shard(*tenant_shard_id),
    2896            0 :                             Some(placement_policy.clone()),
    2897            0 :                             Some(tenant_config.clone()),
    2898            0 :                             *generation,
    2899            0 :                             *scheduling_policy,
    2900            0 :                         )
    2901            0 :                         .await?;
    2902              :                 }
    2903              : 
    2904              :                 // Apply updates in-memory
    2905            0 :                 let mut waiters = Vec::new();
    2906            0 :                 {
    2907            0 :                     let mut locked = self.inner.write().unwrap();
    2908            0 :                     let (nodes, tenants, scheduler) = locked.parts_mut();
    2909              : 
    2910              :                     for ShardUpdate {
    2911            0 :                         tenant_shard_id,
    2912            0 :                         placement_policy,
    2913            0 :                         tenant_config,
    2914            0 :                         generation: update_generation,
    2915            0 :                         scheduling_policy,
    2916            0 :                     } in updates
    2917              :                     {
    2918            0 :                         let Some(shard) = tenants.get_mut(&tenant_shard_id) else {
    2919            0 :                             tracing::warn!("Shard {tenant_shard_id} removed while updating");
    2920            0 :                             continue;
    2921              :                         };
    2922              : 
    2923              :                         // Update stripe size
    2924            0 :                         if result.stripe_size.is_none() && shard.shard.count.count() > 1 {
    2925            0 :                             result.stripe_size = Some(shard.shard.stripe_size);
    2926            0 :                         }
    2927              : 
    2928            0 :                         shard.policy = placement_policy;
    2929            0 :                         shard.config = tenant_config;
    2930            0 :                         if let Some(generation) = update_generation {
    2931            0 :                             shard.generation = Some(generation);
    2932            0 :                         }
    2933              : 
    2934            0 :                         if let Some(scheduling_policy) = scheduling_policy {
    2935            0 :                             shard.set_scheduling_policy(scheduling_policy);
    2936            0 :                         }
    2937              : 
    2938            0 :                         shard.schedule(scheduler, &mut schedule_context)?;
    2939              : 
    2940            0 :                         let maybe_waiter =
    2941            0 :                             self.maybe_reconcile_shard(shard, nodes, ReconcilerPriority::High);
    2942            0 :                         if let Some(waiter) = maybe_waiter {
    2943            0 :                             waiters.push(waiter);
    2944            0 :                         }
    2945              : 
    2946            0 :                         if let Some(node_id) = shard.intent.get_attached() {
    2947            0 :                             result.shards.push(TenantShardLocation {
    2948            0 :                                 shard_id: tenant_shard_id,
    2949            0 :                                 node_id: *node_id,
    2950            0 :                             })
    2951            0 :                         }
    2952              :                     }
    2953              :                 }
    2954            0 :                 waiters
    2955              :             }
    2956              :         };
    2957              : 
    2958            0 :         if let Err(e) = self.await_waiters(waiters, SHORT_RECONCILE_TIMEOUT).await {
    2959              :             // Do not treat a reconcile error as fatal: we have already applied any requested
    2960              :             // Intent changes, and the reconcile can fail for external reasons like unavailable
    2961              :             // compute notification API.  In these cases, it is important that we do not
    2962              :             // cause the cloud control plane to retry forever on this API.
    2963            0 :             tracing::warn!(
    2964            0 :                 "Failed to reconcile after /location_config: {e}, returning success anyway"
    2965              :             );
    2966            0 :         }
    2967              : 
    2968              :         // Logging the full result is useful because it lets us cross-check what the cloud control
    2969              :         // plane's tenant_shards table should contain.
    2970            0 :         tracing::info!("Complete, returning {result:?}");
    2971              : 
    2972            0 :         Ok(result)
    2973            0 :     }
    2974              : 
    2975            0 :     pub(crate) async fn tenant_config_patch(
    2976            0 :         &self,
    2977            0 :         req: TenantConfigPatchRequest,
    2978            0 :     ) -> Result<(), ApiError> {
    2979            0 :         let _tenant_lock = trace_exclusive_lock(
    2980            0 :             &self.tenant_op_locks,
    2981            0 :             req.tenant_id,
    2982            0 :             TenantOperations::ConfigPatch,
    2983            0 :         )
    2984            0 :         .await;
    2985              : 
    2986            0 :         let tenant_id = req.tenant_id;
    2987            0 :         let patch = req.config;
    2988            0 : 
    2989            0 :         self.maybe_load_tenant(tenant_id, &_tenant_lock).await?;
    2990              : 
    2991            0 :         let base = {
    2992            0 :             let locked = self.inner.read().unwrap();
    2993            0 :             let shards = locked
    2994            0 :                 .tenants
    2995            0 :                 .range(TenantShardId::tenant_range(req.tenant_id));
    2996            0 : 
    2997            0 :             let mut configs = shards.map(|(_sid, shard)| &shard.config).peekable();
    2998              : 
    2999            0 :             let first = match configs.peek() {
    3000            0 :                 Some(first) => (*first).clone(),
    3001              :                 None => {
    3002            0 :                     return Err(ApiError::NotFound(
    3003            0 :                         anyhow::anyhow!("Tenant {} not found", req.tenant_id).into(),
    3004            0 :                     ));
    3005              :                 }
    3006              :             };
    3007              : 
    3008            0 :             if !configs.all_equal() {
    3009            0 :                 tracing::error!("Tenant configs for {} are mismatched. ", req.tenant_id);
    3010              :                 // This can't happen because we atomically update the database records
    3011              :                 // of all shards to the new value in [`Self::set_tenant_config_and_reconcile`].
    3012            0 :                 return Err(ApiError::InternalServerError(anyhow::anyhow!(
    3013            0 :                     "Tenant configs for {} are mismatched",
    3014            0 :                     req.tenant_id
    3015            0 :                 )));
    3016            0 :             }
    3017            0 : 
    3018            0 :             first
    3019              :         };
    3020              : 
    3021            0 :         let updated_config = base
    3022            0 :             .apply_patch(patch)
    3023            0 :             .map_err(|err| ApiError::BadRequest(anyhow::anyhow!(err)))?;
    3024            0 :         self.set_tenant_config_and_reconcile(tenant_id, updated_config)
    3025            0 :             .await
    3026            0 :     }
    3027              : 
    3028            0 :     pub(crate) async fn tenant_config_set(&self, req: TenantConfigRequest) -> Result<(), ApiError> {
    3029              :         // We require an exclusive lock, because we are updating persistent and in-memory state
    3030            0 :         let _tenant_lock = trace_exclusive_lock(
    3031            0 :             &self.tenant_op_locks,
    3032            0 :             req.tenant_id,
    3033            0 :             TenantOperations::ConfigSet,
    3034            0 :         )
    3035            0 :         .await;
    3036              : 
    3037            0 :         self.maybe_load_tenant(req.tenant_id, &_tenant_lock).await?;
    3038              : 
    3039            0 :         self.set_tenant_config_and_reconcile(req.tenant_id, req.config)
    3040            0 :             .await
    3041            0 :     }
    3042              : 
    3043            0 :     async fn set_tenant_config_and_reconcile(
    3044            0 :         &self,
    3045            0 :         tenant_id: TenantId,
    3046            0 :         config: TenantConfig,
    3047            0 :     ) -> Result<(), ApiError> {
    3048            0 :         self.persistence
    3049            0 :             .update_tenant_shard(
    3050            0 :                 TenantFilter::Tenant(tenant_id),
    3051            0 :                 None,
    3052            0 :                 Some(config.clone()),
    3053            0 :                 None,
    3054            0 :                 None,
    3055            0 :             )
    3056            0 :             .await?;
    3057              : 
    3058            0 :         let waiters = {
    3059            0 :             let mut waiters = Vec::new();
    3060            0 :             let mut locked = self.inner.write().unwrap();
    3061            0 :             let (nodes, tenants, _scheduler) = locked.parts_mut();
    3062            0 :             for (_shard_id, shard) in tenants.range_mut(TenantShardId::tenant_range(tenant_id)) {
    3063            0 :                 shard.config = config.clone();
    3064            0 :                 if let Some(waiter) =
    3065            0 :                     self.maybe_reconcile_shard(shard, nodes, ReconcilerPriority::High)
    3066            0 :                 {
    3067            0 :                     waiters.push(waiter);
    3068            0 :                 }
    3069              :             }
    3070            0 :             waiters
    3071              :         };
    3072              : 
    3073            0 :         if let Err(e) = self.await_waiters(waiters, SHORT_RECONCILE_TIMEOUT).await {
    3074              :             // Treat this as success because we have stored the configuration.  If e.g.
    3075              :             // a node was unavailable at this time, it should not stop us accepting a
    3076              :             // configuration change.
    3077            0 :             tracing::warn!(%tenant_id, "Accepted configuration update but reconciliation failed: {e}");
    3078            0 :         }
    3079              : 
    3080            0 :         Ok(())
    3081            0 :     }
    3082              : 
    3083            0 :     pub(crate) fn tenant_config_get(
    3084            0 :         &self,
    3085            0 :         tenant_id: TenantId,
    3086            0 :     ) -> Result<HashMap<&str, serde_json::Value>, ApiError> {
    3087            0 :         let config = {
    3088            0 :             let locked = self.inner.read().unwrap();
    3089            0 : 
    3090            0 :             match locked
    3091            0 :                 .tenants
    3092            0 :                 .range(TenantShardId::tenant_range(tenant_id))
    3093            0 :                 .next()
    3094              :             {
    3095            0 :                 Some((_tenant_shard_id, shard)) => shard.config.clone(),
    3096              :                 None => {
    3097            0 :                     return Err(ApiError::NotFound(
    3098            0 :                         anyhow::anyhow!("Tenant not found").into(),
    3099            0 :                     ));
    3100              :                 }
    3101              :             }
    3102              :         };
    3103              : 
    3104              :         // Unlike the pageserver, we do not have a set of global defaults: the config is
    3105              :         // entirely per-tenant.  Therefore the distinction between `tenant_specific_overrides`
    3106              :         // and `effective_config` in the response is meaningless, but we retain that syntax
    3107              :         // in order to remain compatible with the pageserver API.
    3108              : 
    3109            0 :         let response = HashMap::from([
    3110              :             (
    3111              :                 "tenant_specific_overrides",
    3112            0 :                 serde_json::to_value(&config)
    3113            0 :                     .context("serializing tenant specific overrides")
    3114            0 :                     .map_err(ApiError::InternalServerError)?,
    3115              :             ),
    3116              :             (
    3117            0 :                 "effective_config",
    3118            0 :                 serde_json::to_value(&config)
    3119            0 :                     .context("serializing effective config")
    3120            0 :                     .map_err(ApiError::InternalServerError)?,
    3121              :             ),
    3122              :         ]);
    3123              : 
    3124            0 :         Ok(response)
    3125            0 :     }
    3126              : 
    3127            0 :     pub(crate) async fn tenant_time_travel_remote_storage(
    3128            0 :         &self,
    3129            0 :         time_travel_req: &TenantTimeTravelRequest,
    3130            0 :         tenant_id: TenantId,
    3131            0 :         timestamp: Cow<'_, str>,
    3132            0 :         done_if_after: Cow<'_, str>,
    3133            0 :     ) -> Result<(), ApiError> {
    3134            0 :         let _tenant_lock = trace_exclusive_lock(
    3135            0 :             &self.tenant_op_locks,
    3136            0 :             tenant_id,
    3137            0 :             TenantOperations::TimeTravelRemoteStorage,
    3138            0 :         )
    3139            0 :         .await;
    3140              : 
    3141            0 :         let node = {
    3142            0 :             let mut locked = self.inner.write().unwrap();
    3143              :             // Just a sanity check to prevent misuse: the API expects that the tenant is fully
    3144              :             // detached everywhere, and nothing writes to S3 storage. Here, we verify that,
    3145              :             // but only at the start of the process, so it's really just to prevent operator
    3146              :             // mistakes.
    3147            0 :             for (shard_id, shard) in locked.tenants.range(TenantShardId::tenant_range(tenant_id)) {
    3148            0 :                 if shard.intent.get_attached().is_some() || !shard.intent.get_secondary().is_empty()
    3149              :                 {
    3150            0 :                     return Err(ApiError::InternalServerError(anyhow::anyhow!(
    3151            0 :                         "We want tenant to be attached in shard with tenant_shard_id={shard_id}"
    3152            0 :                     )));
    3153            0 :                 }
    3154            0 :                 let maybe_attached = shard
    3155            0 :                     .observed
    3156            0 :                     .locations
    3157            0 :                     .iter()
    3158            0 :                     .filter_map(|(node_id, observed_location)| {
    3159            0 :                         observed_location
    3160            0 :                             .conf
    3161            0 :                             .as_ref()
    3162            0 :                             .map(|loc| (node_id, observed_location, loc.mode))
    3163            0 :                     })
    3164            0 :                     .find(|(_, _, mode)| *mode != LocationConfigMode::Detached);
    3165            0 :                 if let Some((node_id, _observed_location, mode)) = maybe_attached {
    3166            0 :                     return Err(ApiError::InternalServerError(anyhow::anyhow!(
    3167            0 :                         "We observed attached={mode:?} tenant in node_id={node_id} shard with tenant_shard_id={shard_id}"
    3168            0 :                     )));
    3169            0 :                 }
    3170              :             }
    3171            0 :             let scheduler = &mut locked.scheduler;
    3172              :             // Right now we only perform the operation on a single node without parallelization
    3173              :             // TODO fan out the operation to multiple nodes for better performance
    3174            0 :             let node_id = scheduler.any_available_node()?;
    3175            0 :             let node = locked
    3176            0 :                 .nodes
    3177            0 :                 .get(&node_id)
    3178            0 :                 .expect("Pageservers may not be deleted while lock is active");
    3179            0 :             node.clone()
    3180            0 :         };
    3181            0 : 
    3182            0 :         // The shard count is encoded in the remote storage's URL, so we need to handle all historically used shard counts
    3183            0 :         let mut counts = time_travel_req
    3184            0 :             .shard_counts
    3185            0 :             .iter()
    3186            0 :             .copied()
    3187            0 :             .collect::<HashSet<_>>()
    3188            0 :             .into_iter()
    3189            0 :             .collect::<Vec<_>>();
    3190            0 :         counts.sort_unstable();
    3191              : 
    3192            0 :         for count in counts {
    3193            0 :             let shard_ids = (0..count.count())
    3194            0 :                 .map(|i| TenantShardId {
    3195            0 :                     tenant_id,
    3196            0 :                     shard_number: ShardNumber(i),
    3197            0 :                     shard_count: count,
    3198            0 :                 })
    3199            0 :                 .collect::<Vec<_>>();
    3200            0 :             for tenant_shard_id in shard_ids {
    3201            0 :                 let client = PageserverClient::new(
    3202            0 :                     node.get_id(),
    3203            0 :                     node.base_url(),
    3204            0 :                     self.config.pageserver_jwt_token.as_deref(),
    3205            0 :                     self.config.ssl_ca_cert.clone(),
    3206            0 :                 )
    3207            0 :                 .map_err(|e| passthrough_api_error(&node, e))?;
    3208              : 
    3209            0 :                 tracing::info!("Doing time travel recovery for shard {tenant_shard_id}",);
    3210              : 
    3211            0 :                 client
    3212            0 :                     .tenant_time_travel_remote_storage(
    3213            0 :                         tenant_shard_id,
    3214            0 :                         &timestamp,
    3215            0 :                         &done_if_after,
    3216            0 :                     )
    3217            0 :                     .await
    3218            0 :                     .map_err(|e| {
    3219            0 :                         ApiError::InternalServerError(anyhow::anyhow!(
    3220            0 :                             "Error doing time travel recovery for shard {tenant_shard_id} on node {}: {e}",
    3221            0 :                             node
    3222            0 :                         ))
    3223            0 :                     })?;
    3224              :             }
    3225              :         }
    3226            0 :         Ok(())
    3227            0 :     }
    3228              : 
    3229            0 :     pub(crate) async fn tenant_secondary_download(
    3230            0 :         &self,
    3231            0 :         tenant_id: TenantId,
    3232            0 :         wait: Option<Duration>,
    3233            0 :     ) -> Result<(StatusCode, SecondaryProgress), ApiError> {
    3234            0 :         let _tenant_lock = trace_shared_lock(
    3235            0 :             &self.tenant_op_locks,
    3236            0 :             tenant_id,
    3237            0 :             TenantOperations::SecondaryDownload,
    3238            0 :         )
    3239            0 :         .await;
    3240              : 
    3241              :         // Acquire lock and yield the collection of shard-node tuples which we will send requests onward to
    3242            0 :         let targets = {
    3243            0 :             let locked = self.inner.read().unwrap();
    3244            0 :             let mut targets = Vec::new();
    3245              : 
    3246            0 :             for (tenant_shard_id, shard) in
    3247            0 :                 locked.tenants.range(TenantShardId::tenant_range(tenant_id))
    3248              :             {
    3249            0 :                 for node_id in shard.intent.get_secondary() {
    3250            0 :                     let node = locked
    3251            0 :                         .nodes
    3252            0 :                         .get(node_id)
    3253            0 :                         .expect("Pageservers may not be deleted while referenced");
    3254            0 : 
    3255            0 :                     targets.push((*tenant_shard_id, node.clone()));
    3256            0 :                 }
    3257              :             }
    3258            0 :             targets
    3259            0 :         };
    3260            0 : 
    3261            0 :         // Issue concurrent requests to all shards' locations
    3262            0 :         let mut futs = FuturesUnordered::new();
    3263            0 :         for (tenant_shard_id, node) in targets {
    3264            0 :             let client = PageserverClient::new(
    3265            0 :                 node.get_id(),
    3266            0 :                 node.base_url(),
    3267            0 :                 self.config.pageserver_jwt_token.as_deref(),
    3268            0 :                 self.config.ssl_ca_cert.clone(),
    3269            0 :             )
    3270            0 :             .map_err(|e| passthrough_api_error(&node, e))?;
    3271            0 :             futs.push(async move {
    3272            0 :                 let result = client
    3273            0 :                     .tenant_secondary_download(tenant_shard_id, wait)
    3274            0 :                     .await;
    3275            0 :                 (result, node, tenant_shard_id)
    3276            0 :             })
    3277              :         }
    3278              : 
    3279              :         // Handle any errors returned by pageservers.  This includes cases like this request racing with
    3280              :         // a scheduling operation, such that the tenant shard we're calling doesn't exist on that pageserver any more, as
    3281              :         // well as more general cases like 503s, 500s, or timeouts.
    3282            0 :         let mut aggregate_progress = SecondaryProgress::default();
    3283            0 :         let mut aggregate_status: Option<StatusCode> = None;
    3284            0 :         let mut error: Option<mgmt_api::Error> = None;
    3285            0 :         while let Some((result, node, tenant_shard_id)) = futs.next().await {
    3286            0 :             match result {
    3287            0 :                 Err(e) => {
    3288            0 :                     // Secondary downloads are always advisory: if something fails, we nevertheless report success, so that whoever
    3289            0 :                     // is calling us will proceed with whatever migration they're doing, albeit with a slightly less warm cache
    3290            0 :                     // than they had hoped for.
    3291            0 :                     tracing::warn!("Secondary download error from pageserver {node}: {e}",);
    3292            0 :                     error = Some(e)
    3293              :                 }
    3294            0 :                 Ok((status_code, progress)) => {
    3295            0 :                     tracing::info!(%tenant_shard_id, "Shard status={status_code} progress: {progress:?}");
    3296            0 :                     aggregate_progress.layers_downloaded += progress.layers_downloaded;
    3297            0 :                     aggregate_progress.layers_total += progress.layers_total;
    3298            0 :                     aggregate_progress.bytes_downloaded += progress.bytes_downloaded;
    3299            0 :                     aggregate_progress.bytes_total += progress.bytes_total;
    3300            0 :                     aggregate_progress.heatmap_mtime =
    3301            0 :                         std::cmp::max(aggregate_progress.heatmap_mtime, progress.heatmap_mtime);
    3302            0 :                     aggregate_status = match aggregate_status {
    3303            0 :                         None => Some(status_code),
    3304            0 :                         Some(StatusCode::OK) => Some(status_code),
    3305            0 :                         Some(cur) => {
    3306            0 :                             // Other status codes (e.g. 202) -- do not overwrite.
    3307            0 :                             Some(cur)
    3308              :                         }
    3309              :                     };
    3310              :                 }
    3311              :             }
    3312              :         }
    3313              : 
    3314              :         // If any of the shards return 202, indicate our result as 202.
    3315            0 :         match aggregate_status {
    3316              :             None => {
    3317            0 :                 match error {
    3318            0 :                     Some(e) => {
    3319            0 :                         // No successes, and an error: surface it
    3320            0 :                         Err(ApiError::Conflict(format!("Error from pageserver: {e}")))
    3321              :                     }
    3322              :                     None => {
    3323              :                         // No shards found
    3324            0 :                         Err(ApiError::NotFound(
    3325            0 :                             anyhow::anyhow!("Tenant {} not found", tenant_id).into(),
    3326            0 :                         ))
    3327              :                     }
    3328              :                 }
    3329              :             }
    3330            0 :             Some(aggregate_status) => Ok((aggregate_status, aggregate_progress)),
    3331              :         }
    3332            0 :     }
    3333              : 
    3334            0 :     pub(crate) async fn tenant_delete(&self, tenant_id: TenantId) -> Result<StatusCode, ApiError> {
    3335            0 :         let _tenant_lock =
    3336            0 :             trace_exclusive_lock(&self.tenant_op_locks, tenant_id, TenantOperations::Delete).await;
    3337              : 
    3338            0 :         self.maybe_load_tenant(tenant_id, &_tenant_lock).await?;
    3339              : 
    3340              :         // Detach all shards. This also deletes local pageserver shard data.
    3341            0 :         let (detach_waiters, node) = {
    3342            0 :             let mut detach_waiters = Vec::new();
    3343            0 :             let mut locked = self.inner.write().unwrap();
    3344            0 :             let (nodes, tenants, scheduler) = locked.parts_mut();
    3345            0 :             for (_, shard) in tenants.range_mut(TenantShardId::tenant_range(tenant_id)) {
    3346              :                 // Update the tenant's intent to remove all attachments
    3347            0 :                 shard.policy = PlacementPolicy::Detached;
    3348            0 :                 shard
    3349            0 :                     .schedule(scheduler, &mut ScheduleContext::default())
    3350            0 :                     .expect("De-scheduling is infallible");
    3351            0 :                 debug_assert!(shard.intent.get_attached().is_none());
    3352            0 :                 debug_assert!(shard.intent.get_secondary().is_empty());
    3353              : 
    3354            0 :                 if let Some(waiter) =
    3355            0 :                     self.maybe_reconcile_shard(shard, nodes, ReconcilerPriority::High)
    3356            0 :                 {
    3357            0 :                     detach_waiters.push(waiter);
    3358            0 :                 }
    3359              :             }
    3360              : 
    3361              :             // Pick an arbitrary node to use for remote deletions (does not have to be where the tenant
    3362              :             // was attached, just has to be able to see the S3 content)
    3363            0 :             let node_id = scheduler.any_available_node()?;
    3364            0 :             let node = nodes
    3365            0 :                 .get(&node_id)
    3366            0 :                 .expect("Pageservers may not be deleted while lock is active");
    3367            0 :             (detach_waiters, node.clone())
    3368            0 :         };
    3369            0 : 
    3370            0 :         // This reconcile wait can fail in a few ways:
    3371            0 :         //  A there is a very long queue for the reconciler semaphore
    3372            0 :         //  B some pageserver is failing to handle a detach promptly
    3373            0 :         //  C some pageserver goes offline right at the moment we send it a request.
    3374            0 :         //
    3375            0 :         // A and C are transient: the semaphore will eventually become available, and once a node is marked offline
    3376            0 :         // the next attempt to reconcile will silently skip detaches for an offline node and succeed.  If B happens,
    3377            0 :         // it's a bug, and needs resolving at the pageserver level (we shouldn't just leave attachments behind while
    3378            0 :         // deleting the underlying data).
    3379            0 :         self.await_waiters(detach_waiters, RECONCILE_TIMEOUT)
    3380            0 :             .await?;
    3381              : 
    3382              :         // Delete the entire tenant (all shards) from remote storage via a random pageserver.
    3383              :         // Passing an unsharded tenant ID will cause the pageserver to remove all remote paths with
    3384              :         // the tenant ID prefix, including all shards (even possibly stale ones).
    3385            0 :         match node
    3386            0 :             .with_client_retries(
    3387            0 :                 |client| async move {
    3388            0 :                     client
    3389            0 :                         .tenant_delete(TenantShardId::unsharded(tenant_id))
    3390            0 :                         .await
    3391            0 :                 },
    3392            0 :                 &self.config.pageserver_jwt_token,
    3393            0 :                 &self.config.ssl_ca_cert,
    3394            0 :                 1,
    3395            0 :                 3,
    3396            0 :                 RECONCILE_TIMEOUT,
    3397            0 :                 &self.cancel,
    3398            0 :             )
    3399            0 :             .await
    3400            0 :             .unwrap_or(Err(mgmt_api::Error::Cancelled))
    3401              :         {
    3402            0 :             Ok(_) => {}
    3403              :             Err(mgmt_api::Error::Cancelled) => {
    3404            0 :                 return Err(ApiError::ShuttingDown);
    3405              :             }
    3406            0 :             Err(e) => {
    3407            0 :                 // This is unexpected: remote deletion should be infallible, unless the object store
    3408            0 :                 // at large is unavailable.
    3409            0 :                 tracing::error!("Error deleting via node {node}: {e}");
    3410            0 :                 return Err(ApiError::InternalServerError(anyhow::anyhow!(e)));
    3411              :             }
    3412              :         }
    3413              : 
    3414              :         // Fall through: deletion of the tenant on pageservers is complete, we may proceed to drop
    3415              :         // our in-memory state and database state.
    3416              : 
    3417              :         // Ordering: we delete persistent state first: if we then
    3418              :         // crash, we will drop the in-memory state.
    3419              : 
    3420              :         // Drop persistent state.
    3421            0 :         self.persistence.delete_tenant(tenant_id).await?;
    3422              : 
    3423              :         // Drop in-memory state
    3424              :         {
    3425            0 :             let mut locked = self.inner.write().unwrap();
    3426            0 :             let (_nodes, tenants, scheduler) = locked.parts_mut();
    3427              : 
    3428              :             // Dereference Scheduler from shards before dropping them
    3429            0 :             for (_tenant_shard_id, shard) in
    3430            0 :                 tenants.range_mut(TenantShardId::tenant_range(tenant_id))
    3431            0 :             {
    3432            0 :                 shard.intent.clear(scheduler);
    3433            0 :             }
    3434              : 
    3435            0 :             tenants.retain(|tenant_shard_id, _shard| tenant_shard_id.tenant_id != tenant_id);
    3436            0 :             tracing::info!(
    3437            0 :                 "Deleted tenant {tenant_id}, now have {} tenants",
    3438            0 :                 locked.tenants.len()
    3439              :             );
    3440              :         };
    3441              : 
    3442              :         // Success is represented as 404, to imitate the existing pageserver deletion API
    3443            0 :         Ok(StatusCode::NOT_FOUND)
    3444            0 :     }
    3445              : 
    3446              :     /// Naming: this configures the storage controller's policies for a tenant, whereas [`Self::tenant_config_set`] is "set the TenantConfig"
    3447              :     /// for a tenant.  The TenantConfig is passed through to pageservers, whereas this function modifies
    3448              :     /// the tenant's policies (configuration) within the storage controller
    3449            0 :     pub(crate) async fn tenant_update_policy(
    3450            0 :         &self,
    3451            0 :         tenant_id: TenantId,
    3452            0 :         req: TenantPolicyRequest,
    3453            0 :     ) -> Result<(), ApiError> {
    3454              :         // We require an exclusive lock, because we are updating persistent and in-memory state
    3455            0 :         let _tenant_lock = trace_exclusive_lock(
    3456            0 :             &self.tenant_op_locks,
    3457            0 :             tenant_id,
    3458            0 :             TenantOperations::UpdatePolicy,
    3459            0 :         )
    3460            0 :         .await;
    3461              : 
    3462            0 :         self.maybe_load_tenant(tenant_id, &_tenant_lock).await?;
    3463              : 
    3464            0 :         failpoint_support::sleep_millis_async!("tenant-update-policy-exclusive-lock");
    3465              : 
    3466              :         let TenantPolicyRequest {
    3467            0 :             placement,
    3468            0 :             mut scheduling,
    3469            0 :         } = req;
    3470              : 
    3471            0 :         if let Some(PlacementPolicy::Detached | PlacementPolicy::Secondary) = placement {
    3472              :             // When someone configures a tenant to detach, we force the scheduling policy to enable
    3473              :             // this to take effect.
    3474            0 :             if scheduling.is_none() {
    3475            0 :                 scheduling = Some(ShardSchedulingPolicy::Active);
    3476            0 :             }
    3477            0 :         }
    3478              : 
    3479            0 :         self.persistence
    3480            0 :             .update_tenant_shard(
    3481            0 :                 TenantFilter::Tenant(tenant_id),
    3482            0 :                 placement.clone(),
    3483            0 :                 None,
    3484            0 :                 None,
    3485            0 :                 scheduling,
    3486            0 :             )
    3487            0 :             .await?;
    3488              : 
    3489            0 :         let mut schedule_context = ScheduleContext::default();
    3490            0 :         let mut locked = self.inner.write().unwrap();
    3491            0 :         let (nodes, tenants, scheduler) = locked.parts_mut();
    3492            0 :         for (shard_id, shard) in tenants.range_mut(TenantShardId::tenant_range(tenant_id)) {
    3493            0 :             if let Some(placement) = &placement {
    3494            0 :                 shard.policy = placement.clone();
    3495            0 : 
    3496            0 :                 tracing::info!(tenant_id=%shard_id.tenant_id, shard_id=%shard_id.shard_slug(),
    3497            0 :                                "Updated placement policy to {placement:?}");
    3498            0 :             }
    3499              : 
    3500            0 :             if let Some(scheduling) = &scheduling {
    3501            0 :                 shard.set_scheduling_policy(*scheduling);
    3502            0 : 
    3503            0 :                 tracing::info!(tenant_id=%shard_id.tenant_id, shard_id=%shard_id.shard_slug(),
    3504            0 :                                "Updated scheduling policy to {scheduling:?}");
    3505            0 :             }
    3506              : 
    3507              :             // In case scheduling is being switched back on, try it now.
    3508            0 :             shard.schedule(scheduler, &mut schedule_context).ok();
    3509            0 :             self.maybe_reconcile_shard(shard, nodes, ReconcilerPriority::High);
    3510              :         }
    3511              : 
    3512            0 :         Ok(())
    3513            0 :     }
    3514              : 
    3515            0 :     pub(crate) async fn tenant_timeline_create_pageservers(
    3516            0 :         &self,
    3517            0 :         tenant_id: TenantId,
    3518            0 :         mut create_req: TimelineCreateRequest,
    3519            0 :     ) -> Result<TimelineInfo, ApiError> {
    3520            0 :         tracing::info!(
    3521            0 :             "Creating timeline {}/{}",
    3522              :             tenant_id,
    3523              :             create_req.new_timeline_id,
    3524              :         );
    3525              : 
    3526            0 :         self.tenant_remote_mutation(tenant_id, move |mut targets| async move {
    3527            0 :             if targets.0.is_empty() {
    3528            0 :                 return Err(ApiError::NotFound(
    3529            0 :                     anyhow::anyhow!("Tenant not found").into(),
    3530            0 :                 ));
    3531            0 :             };
    3532            0 : 
    3533            0 :             let (shard_zero_tid, shard_zero_locations) =
    3534            0 :                 targets.0.pop_first().expect("Must have at least one shard");
    3535            0 :             assert!(shard_zero_tid.is_shard_zero());
    3536              : 
    3537            0 :             async fn create_one(
    3538            0 :                 tenant_shard_id: TenantShardId,
    3539            0 :                 locations: ShardMutationLocations,
    3540            0 :                 jwt: Option<String>,
    3541            0 :                 ssl_ca_cert: Option<Certificate>,
    3542            0 :                 create_req: TimelineCreateRequest,
    3543            0 :             ) -> Result<TimelineInfo, ApiError> {
    3544            0 :                 let latest = locations.latest.node;
    3545            0 : 
    3546            0 :                 tracing::info!(
    3547            0 :                     "Creating timeline on shard {}/{}, attached to node {latest} in generation {:?}",
    3548              :                     tenant_shard_id,
    3549              :                     create_req.new_timeline_id,
    3550              :                     locations.latest.generation
    3551              :                 );
    3552              : 
    3553            0 :                 let client =
    3554            0 :                     PageserverClient::new(latest.get_id(), latest.base_url(), jwt.as_deref(), ssl_ca_cert.clone())
    3555            0 :                     .map_err(|e| passthrough_api_error(&latest, e))?;
    3556              : 
    3557            0 :                 let timeline_info = client
    3558            0 :                     .timeline_create(tenant_shard_id, &create_req)
    3559            0 :                     .await
    3560            0 :                     .map_err(|e| passthrough_api_error(&latest, e))?;
    3561              : 
    3562              :                 // We propagate timeline creations to all attached locations such that a compute
    3563              :                 // for the new timeline is able to start regardless of the current state of the
    3564              :                 // tenant shard reconciliation.
    3565            0 :                 for location in locations.other {
    3566            0 :                     tracing::info!(
    3567            0 :                         "Creating timeline on shard {}/{}, stale attached to node {} in generation {:?}",
    3568              :                         tenant_shard_id,
    3569              :                         create_req.new_timeline_id,
    3570              :                         location.node,
    3571              :                         location.generation
    3572              :                     );
    3573              : 
    3574            0 :                     let client = PageserverClient::new(
    3575            0 :                         location.node.get_id(),
    3576            0 :                         location.node.base_url(),
    3577            0 :                         jwt.as_deref(),
    3578            0 :                         ssl_ca_cert.clone(),
    3579            0 :                     )
    3580            0 :                     .map_err(|e| passthrough_api_error(&location.node, e))?;
    3581              : 
    3582            0 :                     let res = client
    3583            0 :                         .timeline_create(tenant_shard_id, &create_req)
    3584            0 :                         .await;
    3585              : 
    3586            0 :                     if let Err(e) = res {
    3587            0 :                         match e {
    3588            0 :                             mgmt_api::Error::ApiError(StatusCode::NOT_FOUND, _) => {
    3589            0 :                                 // Tenant might have been detached from the stale location,
    3590            0 :                                 // so ignore 404s.
    3591            0 :                             },
    3592              :                             _ => {
    3593            0 :                                 return Err(passthrough_api_error(&location.node, e));
    3594              :                             }
    3595              :                         }
    3596            0 :                     }
    3597              :                 }
    3598              : 
    3599            0 :                 Ok(timeline_info)
    3600            0 :             }
    3601              : 
    3602              :             // Because the caller might not provide an explicit LSN, we must do the creation first on a single shard, and then
    3603              :             // use whatever LSN that shard picked when creating on subsequent shards.  We arbitrarily use shard zero as the shard
    3604              :             // that will get the first creation request, and propagate the LSN to all the >0 shards.
    3605            0 :             let timeline_info = create_one(
    3606            0 :                 shard_zero_tid,
    3607            0 :                 shard_zero_locations,
    3608            0 :                 self.config.pageserver_jwt_token.clone(),
    3609            0 :                 self.config.ssl_ca_cert.clone(),
    3610            0 :                 create_req.clone(),
    3611            0 :             )
    3612            0 :             .await?;
    3613              : 
    3614              :             // Propagate the LSN that shard zero picked, if caller didn't provide one
    3615            0 :             match &mut create_req.mode {
    3616            0 :                 models::TimelineCreateRequestMode::Branch { ancestor_start_lsn, .. } if ancestor_start_lsn.is_none() => {
    3617            0 :                     *ancestor_start_lsn = timeline_info.ancestor_lsn;
    3618            0 :                 },
    3619            0 :                 _ => {}
    3620              :             }
    3621              : 
    3622              :             // Create timeline on remaining shards with number >0
    3623            0 :             if !targets.0.is_empty() {
    3624              :                 // If we had multiple shards, issue requests for the remainder now.
    3625            0 :                 let jwt = &self.config.pageserver_jwt_token;
    3626            0 :                 self.tenant_for_shards(
    3627            0 :                     targets
    3628            0 :                         .0
    3629            0 :                         .iter()
    3630            0 :                         .map(|t| (*t.0, t.1.latest.node.clone()))
    3631            0 :                         .collect(),
    3632            0 :                     |tenant_shard_id: TenantShardId, _node: Node| {
    3633            0 :                         let create_req = create_req.clone();
    3634            0 :                         let mutation_locations = targets.0.remove(&tenant_shard_id).unwrap();
    3635            0 :                         Box::pin(create_one(
    3636            0 :                             tenant_shard_id,
    3637            0 :                             mutation_locations,
    3638            0 :                             jwt.clone(),
    3639            0 :                             self.config.ssl_ca_cert.clone(),
    3640            0 :                             create_req,
    3641            0 :                         ))
    3642            0 :                     },
    3643            0 :                 )
    3644            0 :                 .await?;
    3645            0 :             }
    3646              : 
    3647            0 :             Ok(timeline_info)
    3648            0 :         })
    3649            0 :         .await?
    3650            0 :     }
    3651              : 
    3652              :     /// Timeline creation on safekeepers
    3653              :     ///
    3654              :     /// Returns `Ok(left)` if the timeline has been created on a quorum of safekeepers,
    3655              :     /// where `left` contains the list of safekeepers that didn't have a successful response.
    3656              :     /// Assumes tenant lock is held while calling this function.
    3657            0 :     async fn tenant_timeline_create_safekeepers_quorum(
    3658            0 :         &self,
    3659            0 :         tenant_id: TenantId,
    3660            0 :         timeline_id: TimelineId,
    3661            0 :         pg_version: u32,
    3662            0 :         timeline_persistence: &TimelinePersistence,
    3663            0 :     ) -> Result<Vec<NodeId>, ApiError> {
    3664            0 :         // If quorum is reached, return if we are outside of a specified timeout
    3665            0 :         let jwt = self
    3666            0 :             .config
    3667            0 :             .safekeeper_jwt_token
    3668            0 :             .clone()
    3669            0 :             .map(SecretString::from);
    3670            0 :         let mut joinset = JoinSet::new();
    3671            0 : 
    3672            0 :         let safekeepers = {
    3673            0 :             let locked = self.inner.read().unwrap();
    3674            0 :             locked.safekeepers.clone()
    3675            0 :         };
    3676            0 : 
    3677            0 :         let mut members = Vec::new();
    3678            0 :         for sk_id in timeline_persistence.sk_set.iter() {
    3679            0 :             let sk_id = NodeId(*sk_id as u64);
    3680            0 :             let Some(safekeeper) = safekeepers.get(&sk_id) else {
    3681            0 :                 return Err(ApiError::InternalServerError(anyhow::anyhow!(
    3682            0 :                     "couldn't find entry for safekeeper with id {sk_id}"
    3683            0 :                 )))?;
    3684              :             };
    3685            0 :             members.push(SafekeeperId {
    3686            0 :                 id: sk_id,
    3687            0 :                 host: safekeeper.skp.host.clone(),
    3688            0 :                 pg_port: safekeeper.skp.port as u16,
    3689            0 :             });
    3690              :         }
    3691            0 :         let mset = MemberSet::new(members).map_err(ApiError::InternalServerError)?;
    3692            0 :         let mconf = safekeeper_api::membership::Configuration::new(mset);
    3693            0 : 
    3694            0 :         let req = safekeeper_api::models::TimelineCreateRequest {
    3695            0 :             commit_lsn: None,
    3696            0 :             mconf,
    3697            0 :             pg_version,
    3698            0 :             start_lsn: timeline_persistence.start_lsn.0,
    3699            0 :             system_id: None,
    3700            0 :             tenant_id,
    3701            0 :             timeline_id,
    3702            0 :             wal_seg_size: None,
    3703            0 :         };
    3704              :         const SK_CREATE_TIMELINE_RECONCILE_TIMEOUT: Duration = Duration::from_secs(30);
    3705            0 :         for sk in timeline_persistence.sk_set.iter() {
    3706            0 :             let sk_id = NodeId(*sk as u64);
    3707            0 :             let safekeepers = safekeepers.clone();
    3708            0 :             let jwt = jwt.clone();
    3709            0 :             let ssl_ca_cert = self.config.ssl_ca_cert.clone();
    3710            0 :             let req = req.clone();
    3711            0 :             joinset.spawn(async move {
    3712            0 :                 // Unwrap is fine as we already would have returned error above
    3713            0 :                 let sk_p = safekeepers.get(&sk_id).unwrap();
    3714            0 :                 let res = sk_p
    3715            0 :                     .with_client_retries(
    3716            0 :                         |client| {
    3717            0 :                             let req = req.clone();
    3718            0 :                             async move { client.create_timeline(&req).await }
    3719            0 :                         },
    3720            0 :                         &jwt,
    3721            0 :                         &ssl_ca_cert,
    3722            0 :                         3,
    3723            0 :                         3,
    3724            0 :                         SK_CREATE_TIMELINE_RECONCILE_TIMEOUT,
    3725            0 :                         &CancellationToken::new(),
    3726            0 :                     )
    3727            0 :                     .await;
    3728            0 :                 (sk_id, sk_p.skp.host.clone(), res)
    3729            0 :             });
    3730            0 :         }
    3731              :         // After we have built the joinset, we now wait for the tasks to complete,
    3732              :         // but with a specified timeout to make sure we return swiftly, either with
    3733              :         // a failure or success.
    3734            0 :         let reconcile_deadline = tokio::time::Instant::now() + SK_CREATE_TIMELINE_RECONCILE_TIMEOUT;
    3735            0 : 
    3736            0 :         // Wait until all tasks finish or timeout is hit, whichever occurs
    3737            0 :         // first.
    3738            0 :         let mut reconcile_results = Vec::new();
    3739              :         loop {
    3740            0 :             if let Ok(res) = tokio::time::timeout_at(reconcile_deadline, joinset.join_next()).await
    3741              :             {
    3742            0 :                 let Some(res) = res else { break };
    3743            0 :                 match res {
    3744            0 :                     Ok(res) => {
    3745            0 :                         tracing::info!(
    3746            0 :                             "response from safekeeper id:{} at {}: {:?}",
    3747              :                             res.0,
    3748              :                             res.1,
    3749              :                             res.2
    3750              :                         );
    3751            0 :                         reconcile_results.push(res);
    3752              :                     }
    3753            0 :                     Err(join_err) => {
    3754            0 :                         tracing::info!("join_err for task in joinset: {join_err}");
    3755              :                     }
    3756              :                 }
    3757              :             } else {
    3758            0 :                 tracing::info!(
    3759            0 :                     "timeout for creation call after {} responses",
    3760            0 :                     reconcile_results.len()
    3761              :                 );
    3762            0 :                 break;
    3763              :             }
    3764              :         }
    3765              : 
    3766              :         // Now check now if quorum was reached in reconcile_results.
    3767            0 :         let total_result_count = reconcile_results.len();
    3768            0 :         let remaining = reconcile_results
    3769            0 :             .into_iter()
    3770            0 :             .filter_map(|res| res.2.is_err().then_some(res.0))
    3771            0 :             .collect::<Vec<_>>();
    3772            0 :         tracing::info!(
    3773            0 :             "Got {} non-successful responses from initial creation request of total {total_result_count} responses",
    3774            0 :             remaining.len()
    3775              :         );
    3776            0 :         if remaining.len() >= 2 {
    3777              :             // Failure
    3778            0 :             return Err(ApiError::InternalServerError(anyhow::anyhow!(
    3779            0 :                 "not enough successful reconciliations to reach quorum, please retry: {} errored",
    3780            0 :                 remaining.len()
    3781            0 :             )));
    3782            0 :         }
    3783            0 : 
    3784            0 :         Ok(remaining)
    3785            0 :     }
    3786              : 
    3787              :     /// Create timeline in controller database and on safekeepers.
    3788              :     /// `timeline_info` is result of timeline creation on pageserver.
    3789              :     ///
    3790              :     /// All actions must be idempotent as the call is retried until success. It
    3791              :     /// tries to create timeline in the db and on at least majority of
    3792              :     /// safekeepers + queue creation for safekeepers which missed it in the db
    3793              :     /// for infinite retries; after that, call returns Ok.
    3794              :     ///
    3795              :     /// The idea is that once this is reached as long as we have alive majority
    3796              :     /// of safekeepers it is expected to get eventually operational as storcon
    3797              :     /// will be able to seed timeline on nodes which missed creation by making
    3798              :     /// pull_timeline from peers. On the other hand we don't want to fail
    3799              :     /// timeline creation if one safekeeper is down.
    3800            0 :     async fn tenant_timeline_create_safekeepers(
    3801            0 :         self: &Arc<Self>,
    3802            0 :         tenant_id: TenantId,
    3803            0 :         timeline_info: &TimelineInfo,
    3804            0 :         create_mode: models::TimelineCreateRequestMode,
    3805            0 :     ) -> Result<SafekeepersInfo, ApiError> {
    3806            0 :         let timeline_id = timeline_info.timeline_id;
    3807            0 :         let pg_version = timeline_info.pg_version;
    3808              :         // Initially start_lsn is determined by last_record_lsn in pageserver
    3809              :         // response as it does initdb. However, later we persist it and in sk
    3810              :         // creation calls replace with the value from the timeline row if it
    3811              :         // previously existed as on retries in theory endpoint might have
    3812              :         // already written some data and advanced last_record_lsn, while we want
    3813              :         // safekeepers to have consistent start_lsn.
    3814            0 :         let start_lsn = match create_mode {
    3815            0 :             models::TimelineCreateRequestMode::Bootstrap { .. } => timeline_info.last_record_lsn,
    3816            0 :             models::TimelineCreateRequestMode::Branch { .. } => timeline_info.last_record_lsn,
    3817              :             models::TimelineCreateRequestMode::ImportPgdata { .. } => {
    3818            0 :                 return Err(ApiError::InternalServerError(anyhow::anyhow!(
    3819            0 :                     "import pgdata doesn't specify the start lsn, aborting creation on safekeepers"
    3820            0 :                 )))?;
    3821              :             }
    3822              :         };
    3823              :         // Choose initial set of safekeepers respecting affinity
    3824            0 :         let sks = self.safekeepers_for_new_timeline().await?;
    3825            0 :         let sks_persistence = sks.iter().map(|sk| sk.id.0 as i64).collect::<Vec<_>>();
    3826            0 :         // Add timeline to db
    3827            0 :         let mut timeline_persist = TimelinePersistence {
    3828            0 :             tenant_id: tenant_id.to_string(),
    3829            0 :             timeline_id: timeline_id.to_string(),
    3830            0 :             start_lsn: start_lsn.into(),
    3831            0 :             generation: 0,
    3832            0 :             sk_set: sks_persistence.clone(),
    3833            0 :             new_sk_set: None,
    3834            0 :             cplane_notified_generation: 0,
    3835            0 :             deleted_at: None,
    3836            0 :         };
    3837            0 :         let inserted = self
    3838            0 :             .persistence
    3839            0 :             .insert_timeline(timeline_persist.clone())
    3840            0 :             .await?;
    3841            0 :         if !inserted {
    3842            0 :             if let Some(existent_persist) = self
    3843            0 :                 .persistence
    3844            0 :                 .get_timeline(tenant_id, timeline_id)
    3845            0 :                 .await?
    3846            0 :             {
    3847            0 :                 // Replace with what we have in the db, to get stuff like the generation right.
    3848            0 :                 // We do still repeat the http calls to the safekeepers. After all, we could have
    3849            0 :                 // crashed right after the wrote to the DB.
    3850            0 :                 timeline_persist = existent_persist;
    3851            0 :             } else {
    3852            0 :                 return Err(ApiError::InternalServerError(anyhow::anyhow!(
    3853            0 :                     "insertion said timeline already in db, but looking it up, it was gone"
    3854            0 :                 )));
    3855              :             }
    3856            0 :         }
    3857              :         // Create the timeline on a quorum of safekeepers
    3858            0 :         let remaining = self
    3859            0 :             .tenant_timeline_create_safekeepers_quorum(
    3860            0 :                 tenant_id,
    3861            0 :                 timeline_id,
    3862            0 :                 pg_version,
    3863            0 :                 &timeline_persist,
    3864            0 :             )
    3865            0 :             .await?;
    3866              : 
    3867              :         // For the remaining safekeepers, take care of their reconciliation asynchronously
    3868            0 :         for &remaining_id in remaining.iter() {
    3869            0 :             let pending_op = TimelinePendingOpPersistence {
    3870            0 :                 tenant_id: tenant_id.to_string(),
    3871            0 :                 timeline_id: timeline_id.to_string(),
    3872            0 :                 generation: timeline_persist.generation,
    3873            0 :                 op_kind: crate::persistence::SafekeeperTimelineOpKind::Pull,
    3874            0 :                 sk_id: remaining_id.0 as i64,
    3875            0 :             };
    3876            0 :             tracing::info!("writing pending op for sk id {remaining_id}");
    3877            0 :             self.persistence.insert_pending_op(pending_op).await?;
    3878              :         }
    3879            0 :         if !remaining.is_empty() {
    3880            0 :             let mut locked = self.inner.write().unwrap();
    3881            0 :             for remaining_id in remaining {
    3882            0 :                 let Some(sk) = locked.safekeepers.get(&remaining_id) else {
    3883            0 :                     return Err(ApiError::InternalServerError(anyhow::anyhow!(
    3884            0 :                         "Couldn't find safekeeper with id {remaining_id}"
    3885            0 :                     )));
    3886              :                 };
    3887            0 :                 let Ok(host_list) = sks
    3888            0 :                     .iter()
    3889            0 :                     .map(|sk| {
    3890            0 :                         Ok((
    3891            0 :                             sk.id,
    3892            0 :                             locked
    3893            0 :                                 .safekeepers
    3894            0 :                                 .get(&sk.id)
    3895            0 :                                 .ok_or_else(|| {
    3896            0 :                                     ApiError::InternalServerError(anyhow::anyhow!(
    3897            0 :                                         "Couldn't find safekeeper with id {remaining_id} to pull from"
    3898            0 :                                     ))
    3899            0 :                                 })?
    3900            0 :                                 .base_url(),
    3901              :                         ))
    3902            0 :                     })
    3903            0 :                     .collect::<Result<_, ApiError>>()
    3904              :                 else {
    3905            0 :                     continue;
    3906              :                 };
    3907            0 :                 let req = ScheduleRequest {
    3908            0 :                     safekeeper: Box::new(sk.clone()),
    3909            0 :                     host_list,
    3910            0 :                     tenant_id,
    3911            0 :                     timeline_id,
    3912            0 :                     generation: timeline_persist.generation as u32,
    3913            0 :                     kind: crate::persistence::SafekeeperTimelineOpKind::Pull,
    3914            0 :                 };
    3915            0 :                 locked.safekeeper_reconcilers.schedule_request(self, req);
    3916              :             }
    3917            0 :         }
    3918              : 
    3919            0 :         Ok(SafekeepersInfo {
    3920            0 :             generation: timeline_persist.generation as u32,
    3921            0 :             safekeepers: sks,
    3922            0 :             tenant_id,
    3923            0 :             timeline_id,
    3924            0 :         })
    3925            0 :     }
    3926              : 
    3927            0 :     pub(crate) async fn tenant_timeline_create(
    3928            0 :         self: &Arc<Self>,
    3929            0 :         tenant_id: TenantId,
    3930            0 :         create_req: TimelineCreateRequest,
    3931            0 :     ) -> Result<TimelineCreateResponseStorcon, ApiError> {
    3932            0 :         let safekeepers = self.config.timelines_onto_safekeepers;
    3933            0 :         tracing::info!(
    3934              :             %safekeepers,
    3935            0 :             "Creating timeline {}/{}",
    3936              :             tenant_id,
    3937              :             create_req.new_timeline_id,
    3938              :         );
    3939              : 
    3940            0 :         let _tenant_lock = trace_shared_lock(
    3941            0 :             &self.tenant_op_locks,
    3942            0 :             tenant_id,
    3943            0 :             TenantOperations::TimelineCreate,
    3944            0 :         )
    3945            0 :         .await;
    3946            0 :         failpoint_support::sleep_millis_async!("tenant-create-timeline-shared-lock");
    3947            0 :         let create_mode = create_req.mode.clone();
    3948              : 
    3949            0 :         let timeline_info = self
    3950            0 :             .tenant_timeline_create_pageservers(tenant_id, create_req)
    3951            0 :             .await?;
    3952              : 
    3953            0 :         let safekeepers = if safekeepers {
    3954            0 :             let res = self
    3955            0 :                 .tenant_timeline_create_safekeepers(tenant_id, &timeline_info, create_mode)
    3956            0 :                 .instrument(tracing::info_span!("timeline_create_safekeepers", %tenant_id, timeline_id=%timeline_info.timeline_id))
    3957            0 :                 .await?;
    3958            0 :             Some(res)
    3959              :         } else {
    3960            0 :             None
    3961              :         };
    3962              : 
    3963            0 :         Ok(TimelineCreateResponseStorcon {
    3964            0 :             timeline_info,
    3965            0 :             safekeepers,
    3966            0 :         })
    3967            0 :     }
    3968              : 
    3969            0 :     pub(crate) async fn tenant_timeline_archival_config(
    3970            0 :         &self,
    3971            0 :         tenant_id: TenantId,
    3972            0 :         timeline_id: TimelineId,
    3973            0 :         req: TimelineArchivalConfigRequest,
    3974            0 :     ) -> Result<(), ApiError> {
    3975            0 :         tracing::info!(
    3976            0 :             "Setting archival config of timeline {tenant_id}/{timeline_id} to '{:?}'",
    3977              :             req.state
    3978              :         );
    3979              : 
    3980            0 :         let _tenant_lock = trace_shared_lock(
    3981            0 :             &self.tenant_op_locks,
    3982            0 :             tenant_id,
    3983            0 :             TenantOperations::TimelineArchivalConfig,
    3984            0 :         )
    3985            0 :         .await;
    3986              : 
    3987            0 :         self.tenant_remote_mutation(tenant_id, move |targets| async move {
    3988            0 :             if targets.0.is_empty() {
    3989            0 :                 return Err(ApiError::NotFound(
    3990            0 :                     anyhow::anyhow!("Tenant not found").into(),
    3991            0 :                 ));
    3992            0 :             }
    3993            0 :             async fn config_one(
    3994            0 :                 tenant_shard_id: TenantShardId,
    3995            0 :                 timeline_id: TimelineId,
    3996            0 :                 node: Node,
    3997            0 :                 jwt: Option<String>,
    3998            0 :                 ssl_ca_cert: Option<Certificate>,
    3999            0 :                 req: TimelineArchivalConfigRequest,
    4000            0 :             ) -> Result<(), ApiError> {
    4001            0 :                 tracing::info!(
    4002            0 :                     "Setting archival config of timeline on shard {tenant_shard_id}/{timeline_id}, attached to node {node}",
    4003              :                 );
    4004              : 
    4005            0 :                 let client = PageserverClient::new(node.get_id(), node.base_url(), jwt.as_deref(), ssl_ca_cert)
    4006            0 :                     .map_err(|e| passthrough_api_error(&node, e))?;
    4007              : 
    4008            0 :                 client
    4009            0 :                     .timeline_archival_config(tenant_shard_id, timeline_id, &req)
    4010            0 :                     .await
    4011            0 :                     .map_err(|e| match e {
    4012            0 :                         mgmt_api::Error::ApiError(StatusCode::PRECONDITION_FAILED, msg) => {
    4013            0 :                             ApiError::PreconditionFailed(msg.into_boxed_str())
    4014              :                         }
    4015            0 :                         _ => passthrough_api_error(&node, e),
    4016            0 :                     })
    4017            0 :             }
    4018              : 
    4019              :             // no shard needs to go first/last; the operation should be idempotent
    4020              :             // TODO: it would be great to ensure that all shards return the same error
    4021            0 :             let locations = targets.0.iter().map(|t| (*t.0, t.1.latest.node.clone())).collect();
    4022            0 :             let results = self
    4023            0 :                 .tenant_for_shards(locations, |tenant_shard_id, node| {
    4024            0 :                     futures::FutureExt::boxed(config_one(
    4025            0 :                         tenant_shard_id,
    4026            0 :                         timeline_id,
    4027            0 :                         node,
    4028            0 :                         self.config.pageserver_jwt_token.clone(),
    4029            0 :                         self.config.ssl_ca_cert.clone(),
    4030            0 :                         req.clone(),
    4031            0 :                     ))
    4032            0 :                 })
    4033            0 :                 .await?;
    4034            0 :             assert!(!results.is_empty(), "must have at least one result");
    4035              : 
    4036            0 :             Ok(())
    4037            0 :         }).await?
    4038            0 :     }
    4039              : 
    4040            0 :     pub(crate) async fn tenant_timeline_detach_ancestor(
    4041            0 :         &self,
    4042            0 :         tenant_id: TenantId,
    4043            0 :         timeline_id: TimelineId,
    4044            0 :     ) -> Result<models::detach_ancestor::AncestorDetached, ApiError> {
    4045            0 :         tracing::info!("Detaching timeline {tenant_id}/{timeline_id}",);
    4046              : 
    4047            0 :         let _tenant_lock = trace_shared_lock(
    4048            0 :             &self.tenant_op_locks,
    4049            0 :             tenant_id,
    4050            0 :             TenantOperations::TimelineDetachAncestor,
    4051            0 :         )
    4052            0 :         .await;
    4053              : 
    4054            0 :         self.tenant_remote_mutation(tenant_id, move |targets| async move {
    4055            0 :             if targets.0.is_empty() {
    4056            0 :                 return Err(ApiError::NotFound(
    4057            0 :                     anyhow::anyhow!("Tenant not found").into(),
    4058            0 :                 ));
    4059            0 :             }
    4060              : 
    4061            0 :             async fn detach_one(
    4062            0 :                 tenant_shard_id: TenantShardId,
    4063            0 :                 timeline_id: TimelineId,
    4064            0 :                 node: Node,
    4065            0 :                 jwt: Option<String>,
    4066            0 :                 ssl_ca_cert: Option<Certificate>,
    4067            0 :             ) -> Result<(ShardNumber, models::detach_ancestor::AncestorDetached), ApiError> {
    4068            0 :                 tracing::info!(
    4069            0 :                     "Detaching timeline on shard {tenant_shard_id}/{timeline_id}, attached to node {node}",
    4070              :                 );
    4071              : 
    4072            0 :                 let client = PageserverClient::new(node.get_id(), node.base_url(), jwt.as_deref(), ssl_ca_cert)
    4073            0 :                     .map_err(|e| passthrough_api_error(&node, e))?;
    4074              : 
    4075            0 :                 client
    4076            0 :                     .timeline_detach_ancestor(tenant_shard_id, timeline_id)
    4077            0 :                     .await
    4078            0 :                     .map_err(|e| {
    4079              :                         use mgmt_api::Error;
    4080              : 
    4081            0 :                         match e {
    4082              :                             // no ancestor (ever)
    4083            0 :                             Error::ApiError(StatusCode::CONFLICT, msg) => ApiError::Conflict(format!(
    4084            0 :                                 "{node}: {}",
    4085            0 :                                 msg.strip_prefix("Conflict: ").unwrap_or(&msg)
    4086            0 :                             )),
    4087              :                             // too many ancestors
    4088            0 :                             Error::ApiError(StatusCode::BAD_REQUEST, msg) => {
    4089            0 :                                 ApiError::BadRequest(anyhow::anyhow!("{node}: {msg}"))
    4090              :                             }
    4091            0 :                             Error::ApiError(StatusCode::INTERNAL_SERVER_ERROR, msg) => {
    4092            0 :                                 // avoid turning these into conflicts to remain compatible with
    4093            0 :                                 // pageservers, 500 errors are sadly retryable with timeline ancestor
    4094            0 :                                 // detach
    4095            0 :                                 ApiError::InternalServerError(anyhow::anyhow!("{node}: {msg}"))
    4096              :                             }
    4097              :                             // rest can be mapped as usual
    4098            0 :                             other => passthrough_api_error(&node, other),
    4099              :                         }
    4100            0 :                     })
    4101            0 :                     .map(|res| (tenant_shard_id.shard_number, res))
    4102            0 :             }
    4103              : 
    4104              :             // no shard needs to go first/last; the operation should be idempotent
    4105            0 :             let locations = targets.0.iter().map(|t| (*t.0, t.1.latest.node.clone())).collect();
    4106            0 :             let mut results = self
    4107            0 :                 .tenant_for_shards(locations, |tenant_shard_id, node| {
    4108            0 :                     futures::FutureExt::boxed(detach_one(
    4109            0 :                         tenant_shard_id,
    4110            0 :                         timeline_id,
    4111            0 :                         node,
    4112            0 :                         self.config.pageserver_jwt_token.clone(),
    4113            0 :                         self.config.ssl_ca_cert.clone(),
    4114            0 :                     ))
    4115            0 :                 })
    4116            0 :                 .await?;
    4117              : 
    4118            0 :             let any = results.pop().expect("we must have at least one response");
    4119            0 : 
    4120            0 :             let mismatching = results
    4121            0 :                 .iter()
    4122            0 :                 .filter(|(_, res)| res != &any.1)
    4123            0 :                 .collect::<Vec<_>>();
    4124            0 :             if !mismatching.is_empty() {
    4125              :                 // this can be hit by races which should not happen because operation lock on cplane
    4126            0 :                 let matching = results.len() - mismatching.len();
    4127            0 :                 tracing::error!(
    4128              :                     matching,
    4129              :                     compared_against=?any,
    4130              :                     ?mismatching,
    4131            0 :                     "shards returned different results"
    4132              :                 );
    4133              : 
    4134            0 :                 return Err(ApiError::InternalServerError(anyhow::anyhow!("pageservers returned mixed results for ancestor detach; manual intervention is required.")));
    4135            0 :             }
    4136            0 : 
    4137            0 :             Ok(any.1)
    4138            0 :         }).await?
    4139            0 :     }
    4140              : 
    4141            0 :     pub(crate) async fn tenant_timeline_block_unblock_gc(
    4142            0 :         &self,
    4143            0 :         tenant_id: TenantId,
    4144            0 :         timeline_id: TimelineId,
    4145            0 :         dir: BlockUnblock,
    4146            0 :     ) -> Result<(), ApiError> {
    4147            0 :         let _tenant_lock = trace_shared_lock(
    4148            0 :             &self.tenant_op_locks,
    4149            0 :             tenant_id,
    4150            0 :             TenantOperations::TimelineGcBlockUnblock,
    4151            0 :         )
    4152            0 :         .await;
    4153              : 
    4154            0 :         self.tenant_remote_mutation(tenant_id, move |targets| async move {
    4155            0 :             if targets.0.is_empty() {
    4156            0 :                 return Err(ApiError::NotFound(
    4157            0 :                     anyhow::anyhow!("Tenant not found").into(),
    4158            0 :                 ));
    4159            0 :             }
    4160              : 
    4161            0 :             async fn do_one(
    4162            0 :                 tenant_shard_id: TenantShardId,
    4163            0 :                 timeline_id: TimelineId,
    4164            0 :                 node: Node,
    4165            0 :                 jwt: Option<String>,
    4166            0 :                 ssl_ca_cert: Option<Certificate>,
    4167            0 :                 dir: BlockUnblock,
    4168            0 :             ) -> Result<(), ApiError> {
    4169            0 :                 let client = PageserverClient::new(
    4170            0 :                     node.get_id(),
    4171            0 :                     node.base_url(),
    4172            0 :                     jwt.as_deref(),
    4173            0 :                     ssl_ca_cert,
    4174            0 :                 )
    4175            0 :                 .map_err(|e| passthrough_api_error(&node, e))?;
    4176              : 
    4177            0 :                 client
    4178            0 :                     .timeline_block_unblock_gc(tenant_shard_id, timeline_id, dir)
    4179            0 :                     .await
    4180            0 :                     .map_err(|e| passthrough_api_error(&node, e))
    4181            0 :             }
    4182              : 
    4183              :             // no shard needs to go first/last; the operation should be idempotent
    4184            0 :             let locations = targets
    4185            0 :                 .0
    4186            0 :                 .iter()
    4187            0 :                 .map(|t| (*t.0, t.1.latest.node.clone()))
    4188            0 :                 .collect();
    4189            0 :             self.tenant_for_shards(locations, |tenant_shard_id, node| {
    4190            0 :                 futures::FutureExt::boxed(do_one(
    4191            0 :                     tenant_shard_id,
    4192            0 :                     timeline_id,
    4193            0 :                     node,
    4194            0 :                     self.config.pageserver_jwt_token.clone(),
    4195            0 :                     self.config.ssl_ca_cert.clone(),
    4196            0 :                     dir,
    4197            0 :                 ))
    4198            0 :             })
    4199            0 :             .await
    4200            0 :         })
    4201            0 :         .await??;
    4202            0 :         Ok(())
    4203            0 :     }
    4204              : 
    4205            0 :     pub(crate) async fn tenant_timeline_download_heatmap_layers(
    4206            0 :         &self,
    4207            0 :         tenant_shard_id: TenantShardId,
    4208            0 :         timeline_id: TimelineId,
    4209            0 :         concurrency: Option<usize>,
    4210            0 :         recurse: bool,
    4211            0 :     ) -> Result<(), ApiError> {
    4212            0 :         let _tenant_lock = trace_shared_lock(
    4213            0 :             &self.tenant_op_locks,
    4214            0 :             tenant_shard_id.tenant_id,
    4215            0 :             TenantOperations::DownloadHeatmapLayers,
    4216            0 :         )
    4217            0 :         .await;
    4218              : 
    4219            0 :         let targets = {
    4220            0 :             let locked = self.inner.read().unwrap();
    4221            0 :             let mut targets = Vec::new();
    4222              : 
    4223              :             // If the request got an unsharded tenant id, then apply
    4224              :             // the operation to all shards. Otherwise, apply it to a specific shard.
    4225            0 :             let shards_range = if tenant_shard_id.is_unsharded() {
    4226            0 :                 TenantShardId::tenant_range(tenant_shard_id.tenant_id)
    4227              :             } else {
    4228            0 :                 tenant_shard_id.range()
    4229              :             };
    4230              : 
    4231            0 :             for (tenant_shard_id, shard) in locked.tenants.range(shards_range) {
    4232            0 :                 if let Some(node_id) = shard.intent.get_attached() {
    4233            0 :                     let node = locked
    4234            0 :                         .nodes
    4235            0 :                         .get(node_id)
    4236            0 :                         .expect("Pageservers may not be deleted while referenced");
    4237            0 : 
    4238            0 :                     targets.push((*tenant_shard_id, node.clone()));
    4239            0 :                 }
    4240              :             }
    4241            0 :             targets
    4242            0 :         };
    4243            0 : 
    4244            0 :         self.tenant_for_shards_api(
    4245            0 :             targets,
    4246            0 :             |tenant_shard_id, client| async move {
    4247            0 :                 client
    4248            0 :                     .timeline_download_heatmap_layers(
    4249            0 :                         tenant_shard_id,
    4250            0 :                         timeline_id,
    4251            0 :                         concurrency,
    4252            0 :                         recurse,
    4253            0 :                     )
    4254            0 :                     .await
    4255            0 :             },
    4256            0 :             1,
    4257            0 :             1,
    4258            0 :             SHORT_RECONCILE_TIMEOUT,
    4259            0 :             &self.cancel,
    4260            0 :         )
    4261            0 :         .await;
    4262              : 
    4263            0 :         Ok(())
    4264            0 :     }
    4265              : 
    4266              :     /// Helper for concurrently calling a pageserver API on a number of shards, such as timeline creation.
    4267              :     ///
    4268              :     /// On success, the returned vector contains exactly the same number of elements as the input `locations`.
    4269            0 :     async fn tenant_for_shards<F, R>(
    4270            0 :         &self,
    4271            0 :         locations: Vec<(TenantShardId, Node)>,
    4272            0 :         mut req_fn: F,
    4273            0 :     ) -> Result<Vec<R>, ApiError>
    4274            0 :     where
    4275            0 :         F: FnMut(
    4276            0 :             TenantShardId,
    4277            0 :             Node,
    4278            0 :         )
    4279            0 :             -> std::pin::Pin<Box<dyn futures::Future<Output = Result<R, ApiError>> + Send>>,
    4280            0 :     {
    4281            0 :         let mut futs = FuturesUnordered::new();
    4282            0 :         let mut results = Vec::with_capacity(locations.len());
    4283              : 
    4284            0 :         for (tenant_shard_id, node) in locations {
    4285            0 :             futs.push(req_fn(tenant_shard_id, node));
    4286            0 :         }
    4287              : 
    4288            0 :         while let Some(r) = futs.next().await {
    4289            0 :             results.push(r?);
    4290              :         }
    4291              : 
    4292            0 :         Ok(results)
    4293            0 :     }
    4294              : 
    4295              :     /// Concurrently invoke a pageserver API call on many shards at once
    4296            0 :     pub(crate) async fn tenant_for_shards_api<T, O, F>(
    4297            0 :         &self,
    4298            0 :         locations: Vec<(TenantShardId, Node)>,
    4299            0 :         op: O,
    4300            0 :         warn_threshold: u32,
    4301            0 :         max_retries: u32,
    4302            0 :         timeout: Duration,
    4303            0 :         cancel: &CancellationToken,
    4304            0 :     ) -> Vec<mgmt_api::Result<T>>
    4305            0 :     where
    4306            0 :         O: Fn(TenantShardId, PageserverClient) -> F + Copy,
    4307            0 :         F: std::future::Future<Output = mgmt_api::Result<T>>,
    4308            0 :     {
    4309            0 :         let mut futs = FuturesUnordered::new();
    4310            0 :         let mut results = Vec::with_capacity(locations.len());
    4311              : 
    4312            0 :         for (tenant_shard_id, node) in locations {
    4313            0 :             futs.push(async move {
    4314            0 :                 node.with_client_retries(
    4315            0 :                     |client| op(tenant_shard_id, client),
    4316            0 :                     &self.config.pageserver_jwt_token,
    4317            0 :                     &self.config.ssl_ca_cert,
    4318            0 :                     warn_threshold,
    4319            0 :                     max_retries,
    4320            0 :                     timeout,
    4321            0 :                     cancel,
    4322            0 :                 )
    4323            0 :                 .await
    4324            0 :             });
    4325            0 :         }
    4326              : 
    4327            0 :         while let Some(r) = futs.next().await {
    4328            0 :             let r = r.unwrap_or(Err(mgmt_api::Error::Cancelled));
    4329            0 :             results.push(r);
    4330            0 :         }
    4331              : 
    4332            0 :         results
    4333            0 :     }
    4334              : 
    4335              :     /// Helper for safely working with the shards in a tenant remotely on pageservers, for example
    4336              :     /// when creating and deleting timelines:
    4337              :     /// - Makes sure shards are attached somewhere if they weren't already
    4338              :     /// - Looks up the shards and the nodes where they were most recently attached
    4339              :     /// - Guarantees that after the inner function returns, the shards' generations haven't moved on: this
    4340              :     ///   ensures that the remote operation acted on the most recent generation, and is therefore durable.
    4341            0 :     async fn tenant_remote_mutation<R, O, F>(
    4342            0 :         &self,
    4343            0 :         tenant_id: TenantId,
    4344            0 :         op: O,
    4345            0 :     ) -> Result<R, ApiError>
    4346            0 :     where
    4347            0 :         O: FnOnce(TenantMutationLocations) -> F,
    4348            0 :         F: std::future::Future<Output = R>,
    4349            0 :     {
    4350            0 :         let mutation_locations = {
    4351            0 :             let mut locations = TenantMutationLocations::default();
    4352              : 
    4353              :             // Load the currently attached pageservers for the latest generation of each shard.  This can
    4354              :             // run concurrently with reconciliations, and it is not guaranteed that the node we find here
    4355              :             // will still be the latest when we're done: we will check generations again at the end of
    4356              :             // this function to handle that.
    4357            0 :             let generations = self.persistence.tenant_generations(tenant_id).await?;
    4358              : 
    4359            0 :             if generations
    4360            0 :                 .iter()
    4361            0 :                 .any(|i| i.generation.is_none() || i.generation_pageserver.is_none())
    4362              :             {
    4363            0 :                 let shard_generations = generations
    4364            0 :                     .into_iter()
    4365            0 :                     .map(|i| (i.tenant_shard_id, (i.generation, i.generation_pageserver)))
    4366            0 :                     .collect::<HashMap<_, _>>();
    4367            0 : 
    4368            0 :                 // One or more shards has not been attached to a pageserver.  Check if this is because it's configured
    4369            0 :                 // to be detached (409: caller should give up), or because it's meant to be attached but isn't yet (503: caller should retry)
    4370            0 :                 let locked = self.inner.read().unwrap();
    4371            0 :                 for (shard_id, shard) in
    4372            0 :                     locked.tenants.range(TenantShardId::tenant_range(tenant_id))
    4373              :                 {
    4374            0 :                     match shard.policy {
    4375              :                         PlacementPolicy::Attached(_) => {
    4376              :                             // This shard is meant to be attached: the caller is not wrong to try and
    4377              :                             // use this function, but we can't service the request right now.
    4378            0 :                             let Some(generation) = shard_generations.get(shard_id) else {
    4379              :                                 // This can only happen if there is a split brain controller modifying the database.  This should
    4380              :                                 // never happen when testing, and if it happens in production we can only log the issue.
    4381            0 :                                 debug_assert!(false);
    4382            0 :                                 tracing::error!(
    4383            0 :                                     "Shard {shard_id} not found in generation state!  Is another rogue controller running?"
    4384              :                                 );
    4385            0 :                                 continue;
    4386              :                             };
    4387            0 :                             let (generation, generation_pageserver) = generation;
    4388            0 :                             if let Some(generation) = generation {
    4389            0 :                                 if generation_pageserver.is_none() {
    4390              :                                     // This is legitimate only in a very narrow window where the shard was only just configured into
    4391              :                                     // Attached mode after being created in Secondary or Detached mode, and it has had its generation
    4392              :                                     // set but not yet had a Reconciler run (reconciler is the only thing that sets generation_pageserver).
    4393            0 :                                     tracing::warn!(
    4394            0 :                                         "Shard {shard_id} generation is set ({generation:?}) but generation_pageserver is None, reconciler not run yet?"
    4395              :                                     );
    4396            0 :                                 }
    4397              :                             } else {
    4398              :                                 // This should never happen: a shard with no generation is only permitted when it was created in some state
    4399              :                                 // other than PlacementPolicy::Attached (and generation is always written to DB before setting Attached in memory)
    4400            0 :                                 debug_assert!(false);
    4401            0 :                                 tracing::error!(
    4402            0 :                                     "Shard {shard_id} generation is None, but it is in PlacementPolicy::Attached mode!"
    4403              :                                 );
    4404            0 :                                 continue;
    4405              :                             }
    4406              :                         }
    4407              :                         PlacementPolicy::Secondary | PlacementPolicy::Detached => {
    4408            0 :                             return Err(ApiError::Conflict(format!(
    4409            0 :                                 "Shard {shard_id} tenant has policy {:?}",
    4410            0 :                                 shard.policy
    4411            0 :                             )));
    4412              :                         }
    4413              :                     }
    4414              :                 }
    4415              : 
    4416            0 :                 return Err(ApiError::ResourceUnavailable(
    4417            0 :                     "One or more shards in tenant is not yet attached".into(),
    4418            0 :                 ));
    4419            0 :             }
    4420            0 : 
    4421            0 :             let locked = self.inner.read().unwrap();
    4422              :             for ShardGenerationState {
    4423            0 :                 tenant_shard_id,
    4424            0 :                 generation,
    4425            0 :                 generation_pageserver,
    4426            0 :             } in generations
    4427              :             {
    4428            0 :                 let node_id = generation_pageserver.expect("We checked for None above");
    4429            0 :                 let node = locked
    4430            0 :                     .nodes
    4431            0 :                     .get(&node_id)
    4432            0 :                     .ok_or(ApiError::Conflict(format!(
    4433            0 :                         "Raced with removal of node {node_id}"
    4434            0 :                     )))?;
    4435            0 :                 let generation = generation.expect("Checked above");
    4436            0 : 
    4437            0 :                 let tenant = locked.tenants.get(&tenant_shard_id);
    4438              : 
    4439              :                 // TODO(vlad): Abstract the logic that finds stale attached locations
    4440              :                 // from observed state into a [`Service`] method.
    4441            0 :                 let other_locations = match tenant {
    4442            0 :                     Some(tenant) => {
    4443            0 :                         let mut other = tenant.attached_locations();
    4444            0 :                         let latest_location_index =
    4445            0 :                             other.iter().position(|&l| l == (node.get_id(), generation));
    4446            0 :                         if let Some(idx) = latest_location_index {
    4447            0 :                             other.remove(idx);
    4448            0 :                         }
    4449              : 
    4450            0 :                         other
    4451              :                     }
    4452            0 :                     None => Vec::default(),
    4453              :                 };
    4454              : 
    4455            0 :                 let location = ShardMutationLocations {
    4456            0 :                     latest: MutationLocation {
    4457            0 :                         node: node.clone(),
    4458            0 :                         generation,
    4459            0 :                     },
    4460            0 :                     other: other_locations
    4461            0 :                         .into_iter()
    4462            0 :                         .filter_map(|(node_id, generation)| {
    4463            0 :                             let node = locked.nodes.get(&node_id)?;
    4464              : 
    4465            0 :                             Some(MutationLocation {
    4466            0 :                                 node: node.clone(),
    4467            0 :                                 generation,
    4468            0 :                             })
    4469            0 :                         })
    4470            0 :                         .collect(),
    4471            0 :                 };
    4472            0 :                 locations.0.insert(tenant_shard_id, location);
    4473            0 :             }
    4474              : 
    4475            0 :             locations
    4476              :         };
    4477              : 
    4478            0 :         let result = op(mutation_locations.clone()).await;
    4479              : 
    4480              :         // Post-check: are all the generations of all the shards the same as they were initially?  This proves that
    4481              :         // our remote operation executed on the latest generation and is therefore persistent.
    4482              :         {
    4483            0 :             let latest_generations = self.persistence.tenant_generations(tenant_id).await?;
    4484            0 :             if latest_generations
    4485            0 :                 .into_iter()
    4486            0 :                 .map(
    4487            0 :                     |ShardGenerationState {
    4488              :                          tenant_shard_id,
    4489              :                          generation,
    4490              :                          generation_pageserver: _,
    4491            0 :                      }| (tenant_shard_id, generation),
    4492            0 :                 )
    4493            0 :                 .collect::<Vec<_>>()
    4494            0 :                 != mutation_locations
    4495            0 :                     .0
    4496            0 :                     .into_iter()
    4497            0 :                     .map(|i| (i.0, Some(i.1.latest.generation)))
    4498            0 :                     .collect::<Vec<_>>()
    4499              :             {
    4500              :                 // We raced with something that incremented the generation, and therefore cannot be
    4501              :                 // confident that our actions are persistent (they might have hit an old generation).
    4502              :                 //
    4503              :                 // This is safe but requires a retry: ask the client to do that by giving them a 503 response.
    4504            0 :                 return Err(ApiError::ResourceUnavailable(
    4505            0 :                     "Tenant attachment changed, please retry".into(),
    4506            0 :                 ));
    4507            0 :             }
    4508            0 :         }
    4509            0 : 
    4510            0 :         Ok(result)
    4511            0 :     }
    4512              : 
    4513            0 :     pub(crate) async fn tenant_timeline_delete(
    4514            0 :         self: &Arc<Self>,
    4515            0 :         tenant_id: TenantId,
    4516            0 :         timeline_id: TimelineId,
    4517            0 :     ) -> Result<StatusCode, ApiError> {
    4518            0 :         tracing::info!("Deleting timeline {}/{}", tenant_id, timeline_id,);
    4519            0 :         let _tenant_lock = trace_shared_lock(
    4520            0 :             &self.tenant_op_locks,
    4521            0 :             tenant_id,
    4522            0 :             TenantOperations::TimelineDelete,
    4523            0 :         )
    4524            0 :         .await;
    4525              : 
    4526            0 :         let status_code = self.tenant_remote_mutation(tenant_id, move |mut targets| async move {
    4527            0 :             if targets.0.is_empty() {
    4528            0 :                 return Err(ApiError::NotFound(
    4529            0 :                     anyhow::anyhow!("Tenant not found").into(),
    4530            0 :                 ));
    4531            0 :             }
    4532            0 : 
    4533            0 :             let (shard_zero_tid, shard_zero_locations) = targets.0.pop_first().expect("Must have at least one shard");
    4534            0 :             assert!(shard_zero_tid.is_shard_zero());
    4535              : 
    4536            0 :             async fn delete_one(
    4537            0 :                 tenant_shard_id: TenantShardId,
    4538            0 :                 timeline_id: TimelineId,
    4539            0 :                 node: Node,
    4540            0 :                 jwt: Option<String>,
    4541            0 :                 ssl_ca_cert: Option<Certificate>,
    4542            0 :             ) -> Result<StatusCode, ApiError> {
    4543            0 :                 tracing::info!(
    4544            0 :                     "Deleting timeline on shard {tenant_shard_id}/{timeline_id}, attached to node {node}",
    4545              :                 );
    4546              : 
    4547            0 :                 let client = PageserverClient::new(node.get_id(), node.base_url(), jwt.as_deref(), ssl_ca_cert)
    4548            0 :                     .map_err(|e| passthrough_api_error(&node, e))?;
    4549            0 :                 let res = client
    4550            0 :                     .timeline_delete(tenant_shard_id, timeline_id)
    4551            0 :                     .await;
    4552              : 
    4553            0 :                 match res {
    4554            0 :                     Ok(ok) => Ok(ok),
    4555            0 :                     Err(mgmt_api::Error::ApiError(StatusCode::CONFLICT, _)) => Ok(StatusCode::CONFLICT),
    4556            0 :                     Err(mgmt_api::Error::ApiError(StatusCode::SERVICE_UNAVAILABLE, msg)) => Err(ApiError::ResourceUnavailable(msg.into())),
    4557            0 :                     Err(e) => {
    4558            0 :                         Err(
    4559            0 :                             ApiError::InternalServerError(anyhow::anyhow!(
    4560            0 :                                 "Error deleting timeline {timeline_id} on {tenant_shard_id} on node {node}: {e}",
    4561            0 :                             ))
    4562            0 :                         )
    4563              :                     }
    4564              :                 }
    4565            0 :             }
    4566              : 
    4567            0 :             let locations = targets.0.iter().map(|t| (*t.0, t.1.latest.node.clone())).collect();
    4568            0 :             let statuses = self
    4569            0 :                 .tenant_for_shards(locations, |tenant_shard_id: TenantShardId, node: Node| {
    4570            0 :                     Box::pin(delete_one(
    4571            0 :                         tenant_shard_id,
    4572            0 :                         timeline_id,
    4573            0 :                         node,
    4574            0 :                         self.config.pageserver_jwt_token.clone(),
    4575            0 :                         self.config.ssl_ca_cert.clone(),
    4576            0 :                     ))
    4577            0 :                 })
    4578            0 :                 .await?;
    4579              : 
    4580              :             // If any shards >0 haven't finished deletion yet, don't start deletion on shard zero.
    4581              :             // We return 409 (Conflict) if deletion was already in progress on any of the shards
    4582              :             // and 202 (Accepted) if deletion was not already in progress on any of the shards.
    4583            0 :             if statuses.iter().any(|s| s == &StatusCode::CONFLICT) {
    4584            0 :                 return Ok(StatusCode::CONFLICT);
    4585            0 :             }
    4586            0 : 
    4587            0 :             if statuses.iter().any(|s| s != &StatusCode::NOT_FOUND) {
    4588            0 :                 return Ok(StatusCode::ACCEPTED);
    4589            0 :             }
    4590              : 
    4591              :             // Delete shard zero last: this is not strictly necessary, but since a caller's GET on a timeline will be routed
    4592              :             // to shard zero, it gives a more obvious behavior that a GET returns 404 once the deletion is done.
    4593            0 :             let shard_zero_status = delete_one(
    4594            0 :                 shard_zero_tid,
    4595            0 :                 timeline_id,
    4596            0 :                 shard_zero_locations.latest.node,
    4597            0 :                 self.config.pageserver_jwt_token.clone(),
    4598            0 :                 self.config.ssl_ca_cert.clone(),
    4599            0 :             )
    4600            0 :             .await?;
    4601            0 :             Ok(shard_zero_status)
    4602            0 :         }).await?;
    4603              : 
    4604            0 :         self.tenant_timeline_delete_safekeepers(tenant_id, timeline_id)
    4605            0 :             .await?;
    4606              : 
    4607            0 :         status_code
    4608            0 :     }
    4609              :     /// Perform timeline deletion on safekeepers. Will return success: we persist the deletion into the reconciler.
    4610            0 :     async fn tenant_timeline_delete_safekeepers(
    4611            0 :         self: &Arc<Self>,
    4612            0 :         tenant_id: TenantId,
    4613            0 :         timeline_id: TimelineId,
    4614            0 :     ) -> Result<(), ApiError> {
    4615            0 :         let tl = self
    4616            0 :             .persistence
    4617            0 :             .get_timeline(tenant_id, timeline_id)
    4618            0 :             .await?;
    4619            0 :         let Some(tl) = tl else {
    4620            0 :             tracing::info!(
    4621            0 :                 "timeline {tenant_id}/{timeline_id} doesn't exist in timelines table, no deletions on safekeepers needed"
    4622              :             );
    4623            0 :             return Ok(());
    4624              :         };
    4625            0 :         let all_sks = tl
    4626            0 :             .new_sk_set
    4627            0 :             .iter()
    4628            0 :             .flat_map(|sks| {
    4629            0 :                 sks.iter()
    4630            0 :                     .map(|sk| (*sk, SafekeeperTimelineOpKind::Exclude))
    4631            0 :             })
    4632            0 :             .chain(
    4633            0 :                 tl.sk_set
    4634            0 :                     .iter()
    4635            0 :                     .map(|v| (*v, SafekeeperTimelineOpKind::Delete)),
    4636            0 :             )
    4637            0 :             .collect::<HashMap<_, _>>();
    4638            0 : 
    4639            0 :         // Schedule reconciliations
    4640            0 :         {
    4641            0 :             let mut locked = self.inner.write().unwrap();
    4642            0 :             for (sk_id, kind) in all_sks {
    4643            0 :                 let sk_id = NodeId(sk_id as u64);
    4644            0 :                 let Some(sk) = locked.safekeepers.get(&sk_id) else {
    4645            0 :                     return Err(ApiError::InternalServerError(anyhow::anyhow!(
    4646            0 :                         "Couldn't find safekeeper with id {sk_id}"
    4647            0 :                     )));
    4648              :                 };
    4649              : 
    4650            0 :                 let req = ScheduleRequest {
    4651            0 :                     safekeeper: Box::new(sk.clone()),
    4652            0 :                     // we don't use this for this kind, put a dummy value
    4653            0 :                     host_list: Vec::new(),
    4654            0 :                     tenant_id,
    4655            0 :                     timeline_id,
    4656            0 :                     generation: tl.generation as u32,
    4657            0 :                     kind,
    4658            0 :                 };
    4659            0 :                 locked.safekeeper_reconcilers.schedule_request(self, req);
    4660              :             }
    4661              :         }
    4662            0 :         Ok(())
    4663            0 :     }
    4664              : 
    4665              :     /// When you know the TenantId but not a specific shard, and would like to get the node holding shard 0.
    4666            0 :     pub(crate) async fn tenant_shard0_node(
    4667            0 :         &self,
    4668            0 :         tenant_id: TenantId,
    4669            0 :     ) -> Result<(Node, TenantShardId), ApiError> {
    4670            0 :         let tenant_shard_id = {
    4671            0 :             let locked = self.inner.read().unwrap();
    4672            0 :             let Some((tenant_shard_id, _shard)) = locked
    4673            0 :                 .tenants
    4674            0 :                 .range(TenantShardId::tenant_range(tenant_id))
    4675            0 :                 .next()
    4676              :             else {
    4677            0 :                 return Err(ApiError::NotFound(
    4678            0 :                     anyhow::anyhow!("Tenant {tenant_id} not found").into(),
    4679            0 :                 ));
    4680              :             };
    4681              : 
    4682            0 :             *tenant_shard_id
    4683            0 :         };
    4684            0 : 
    4685            0 :         self.tenant_shard_node(tenant_shard_id)
    4686            0 :             .await
    4687            0 :             .map(|node| (node, tenant_shard_id))
    4688            0 :     }
    4689              : 
    4690              :     /// When you need to send an HTTP request to the pageserver that holds a shard of a tenant, this
    4691              :     /// function looks up and returns node. If the shard isn't found, returns Err(ApiError::NotFound)
    4692            0 :     pub(crate) async fn tenant_shard_node(
    4693            0 :         &self,
    4694            0 :         tenant_shard_id: TenantShardId,
    4695            0 :     ) -> Result<Node, ApiError> {
    4696            0 :         // Look up in-memory state and maybe use the node from there.
    4697            0 :         {
    4698            0 :             let locked = self.inner.read().unwrap();
    4699            0 :             let Some(shard) = locked.tenants.get(&tenant_shard_id) else {
    4700            0 :                 return Err(ApiError::NotFound(
    4701            0 :                     anyhow::anyhow!("Tenant shard {tenant_shard_id} not found").into(),
    4702            0 :                 ));
    4703              :             };
    4704              : 
    4705            0 :             let Some(intent_node_id) = shard.intent.get_attached() else {
    4706            0 :                 tracing::warn!(
    4707            0 :                     tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug(),
    4708            0 :                     "Shard not scheduled (policy {:?}), cannot generate pass-through URL",
    4709              :                     shard.policy
    4710              :                 );
    4711            0 :                 return Err(ApiError::Conflict(
    4712            0 :                     "Cannot call timeline API on non-attached tenant".to_string(),
    4713            0 :                 ));
    4714              :             };
    4715              : 
    4716            0 :             if shard.reconciler.is_none() {
    4717              :                 // Optimization: while no reconcile is in flight, we may trust our in-memory state
    4718              :                 // to tell us which pageserver to use. Otherwise we will fall through and hit the database
    4719            0 :                 let Some(node) = locked.nodes.get(intent_node_id) else {
    4720              :                     // This should never happen
    4721            0 :                     return Err(ApiError::InternalServerError(anyhow::anyhow!(
    4722            0 :                         "Shard refers to nonexistent node"
    4723            0 :                     )));
    4724              :                 };
    4725            0 :                 return Ok(node.clone());
    4726            0 :             }
    4727              :         };
    4728              : 
    4729              :         // Look up the latest attached pageserver location from the database
    4730              :         // generation state: this will reflect the progress of any ongoing migration.
    4731              :         // Note that it is not guaranteed to _stay_ here, our caller must still handle
    4732              :         // the case where they call through to the pageserver and get a 404.
    4733            0 :         let db_result = self
    4734            0 :             .persistence
    4735            0 :             .tenant_generations(tenant_shard_id.tenant_id)
    4736            0 :             .await?;
    4737              :         let Some(ShardGenerationState {
    4738              :             tenant_shard_id: _,
    4739              :             generation: _,
    4740            0 :             generation_pageserver: Some(node_id),
    4741            0 :         }) = db_result
    4742            0 :             .into_iter()
    4743            0 :             .find(|s| s.tenant_shard_id == tenant_shard_id)
    4744              :         else {
    4745              :             // This can happen if we raced with a tenant deletion or a shard split.  On a retry
    4746              :             // the caller will either succeed (shard split case), get a proper 404 (deletion case),
    4747              :             // or a conflict response (case where tenant was detached in background)
    4748            0 :             return Err(ApiError::ResourceUnavailable(
    4749            0 :                 format!("Shard {tenant_shard_id} not found in database, or is not attached").into(),
    4750            0 :             ));
    4751              :         };
    4752            0 :         let locked = self.inner.read().unwrap();
    4753            0 :         let Some(node) = locked.nodes.get(&node_id) else {
    4754              :             // This should never happen
    4755            0 :             return Err(ApiError::InternalServerError(anyhow::anyhow!(
    4756            0 :                 "Shard refers to nonexistent node"
    4757            0 :             )));
    4758              :         };
    4759              : 
    4760            0 :         Ok(node.clone())
    4761            0 :     }
    4762              : 
    4763            0 :     pub(crate) fn tenant_locate(
    4764            0 :         &self,
    4765            0 :         tenant_id: TenantId,
    4766            0 :     ) -> Result<TenantLocateResponse, ApiError> {
    4767            0 :         let locked = self.inner.read().unwrap();
    4768            0 :         tracing::info!("Locating shards for tenant {tenant_id}");
    4769              : 
    4770            0 :         let mut result = Vec::new();
    4771            0 :         let mut shard_params: Option<ShardParameters> = None;
    4772              : 
    4773            0 :         for (tenant_shard_id, shard) in locked.tenants.range(TenantShardId::tenant_range(tenant_id))
    4774              :         {
    4775            0 :             let node_id =
    4776            0 :                 shard
    4777            0 :                     .intent
    4778            0 :                     .get_attached()
    4779            0 :                     .ok_or(ApiError::BadRequest(anyhow::anyhow!(
    4780            0 :                         "Cannot locate a tenant that is not attached"
    4781            0 :                     )))?;
    4782              : 
    4783            0 :             let node = locked
    4784            0 :                 .nodes
    4785            0 :                 .get(&node_id)
    4786            0 :                 .expect("Pageservers may not be deleted while referenced");
    4787            0 : 
    4788            0 :             result.push(node.shard_location(*tenant_shard_id));
    4789            0 : 
    4790            0 :             match &shard_params {
    4791            0 :                 None => {
    4792            0 :                     shard_params = Some(ShardParameters {
    4793            0 :                         stripe_size: shard.shard.stripe_size,
    4794            0 :                         count: shard.shard.count,
    4795            0 :                     });
    4796            0 :                 }
    4797            0 :                 Some(params) => {
    4798            0 :                     if params.stripe_size != shard.shard.stripe_size {
    4799              :                         // This should never happen.  We enforce at runtime because it's simpler than
    4800              :                         // adding an extra per-tenant data structure to store the things that should be the same
    4801            0 :                         return Err(ApiError::InternalServerError(anyhow::anyhow!(
    4802            0 :                             "Inconsistent shard stripe size parameters!"
    4803            0 :                         )));
    4804            0 :                     }
    4805              :                 }
    4806              :             }
    4807              :         }
    4808              : 
    4809            0 :         if result.is_empty() {
    4810            0 :             return Err(ApiError::NotFound(
    4811            0 :                 anyhow::anyhow!("No shards for this tenant ID found").into(),
    4812            0 :             ));
    4813            0 :         }
    4814            0 :         let shard_params = shard_params.expect("result is non-empty, therefore this is set");
    4815            0 :         tracing::info!(
    4816            0 :             "Located tenant {} with params {:?} on shards {}",
    4817            0 :             tenant_id,
    4818            0 :             shard_params,
    4819            0 :             result
    4820            0 :                 .iter()
    4821            0 :                 .map(|s| format!("{:?}", s))
    4822            0 :                 .collect::<Vec<_>>()
    4823            0 :                 .join(",")
    4824              :         );
    4825              : 
    4826            0 :         Ok(TenantLocateResponse {
    4827            0 :             shards: result,
    4828            0 :             shard_params,
    4829            0 :         })
    4830            0 :     }
    4831              : 
    4832              :     /// Returns None if the input iterator of shards does not include a shard with number=0
    4833            0 :     fn tenant_describe_impl<'a>(
    4834            0 :         &self,
    4835            0 :         shards: impl Iterator<Item = &'a TenantShard>,
    4836            0 :     ) -> Option<TenantDescribeResponse> {
    4837            0 :         let mut shard_zero = None;
    4838            0 :         let mut describe_shards = Vec::new();
    4839              : 
    4840            0 :         for shard in shards {
    4841            0 :             if shard.tenant_shard_id.is_shard_zero() {
    4842            0 :                 shard_zero = Some(shard);
    4843            0 :             }
    4844              : 
    4845            0 :             describe_shards.push(TenantDescribeResponseShard {
    4846            0 :                 tenant_shard_id: shard.tenant_shard_id,
    4847            0 :                 node_attached: *shard.intent.get_attached(),
    4848            0 :                 node_secondary: shard.intent.get_secondary().to_vec(),
    4849            0 :                 last_error: shard
    4850            0 :                     .last_error
    4851            0 :                     .lock()
    4852            0 :                     .unwrap()
    4853            0 :                     .as_ref()
    4854            0 :                     .map(|e| format!("{e}"))
    4855            0 :                     .unwrap_or("".to_string())
    4856            0 :                     .clone(),
    4857            0 :                 is_reconciling: shard.reconciler.is_some(),
    4858            0 :                 is_pending_compute_notification: shard.pending_compute_notification,
    4859            0 :                 is_splitting: matches!(shard.splitting, SplitState::Splitting),
    4860            0 :                 scheduling_policy: shard.get_scheduling_policy(),
    4861            0 :                 preferred_az_id: shard.preferred_az().map(ToString::to_string),
    4862              :             })
    4863              :         }
    4864              : 
    4865            0 :         let shard_zero = shard_zero?;
    4866              : 
    4867            0 :         Some(TenantDescribeResponse {
    4868            0 :             tenant_id: shard_zero.tenant_shard_id.tenant_id,
    4869            0 :             shards: describe_shards,
    4870            0 :             stripe_size: shard_zero.shard.stripe_size,
    4871            0 :             policy: shard_zero.policy.clone(),
    4872            0 :             config: shard_zero.config.clone(),
    4873            0 :         })
    4874            0 :     }
    4875              : 
    4876            0 :     pub(crate) fn tenant_describe(
    4877            0 :         &self,
    4878            0 :         tenant_id: TenantId,
    4879            0 :     ) -> Result<TenantDescribeResponse, ApiError> {
    4880            0 :         let locked = self.inner.read().unwrap();
    4881            0 : 
    4882            0 :         self.tenant_describe_impl(
    4883            0 :             locked
    4884            0 :                 .tenants
    4885            0 :                 .range(TenantShardId::tenant_range(tenant_id))
    4886            0 :                 .map(|(_k, v)| v),
    4887            0 :         )
    4888            0 :         .ok_or_else(|| ApiError::NotFound(anyhow::anyhow!("Tenant {tenant_id} not found").into()))
    4889            0 :     }
    4890              : 
    4891              :     /// limit & offset are pagination parameters. Since we are walking an in-memory HashMap, `offset` does not
    4892              :     /// avoid traversing data, it just avoid returning it. This is suitable for our purposes, since our in memory
    4893              :     /// maps are small enough to traverse fast, our pagination is just to avoid serializing huge JSON responses
    4894              :     /// in our external API.
    4895            0 :     pub(crate) fn tenant_list(
    4896            0 :         &self,
    4897            0 :         limit: Option<usize>,
    4898            0 :         start_after: Option<TenantId>,
    4899            0 :     ) -> Vec<TenantDescribeResponse> {
    4900            0 :         let locked = self.inner.read().unwrap();
    4901              : 
    4902              :         // Apply start_from parameter
    4903            0 :         let shard_range = match start_after {
    4904            0 :             None => locked.tenants.range(..),
    4905            0 :             Some(tenant_id) => locked.tenants.range(
    4906            0 :                 TenantShardId {
    4907            0 :                     tenant_id,
    4908            0 :                     shard_number: ShardNumber(u8::MAX),
    4909            0 :                     shard_count: ShardCount(u8::MAX),
    4910            0 :                 }..,
    4911            0 :             ),
    4912              :         };
    4913              : 
    4914            0 :         let mut result = Vec::new();
    4915            0 :         for (_tenant_id, tenant_shards) in &shard_range.group_by(|(id, _shard)| id.tenant_id) {
    4916            0 :             result.push(
    4917            0 :                 self.tenant_describe_impl(tenant_shards.map(|(_k, v)| v))
    4918            0 :                     .expect("Groups are always non-empty"),
    4919            0 :             );
    4920              : 
    4921              :             // Enforce `limit` parameter
    4922            0 :             if let Some(limit) = limit {
    4923            0 :                 if result.len() >= limit {
    4924            0 :                     break;
    4925            0 :                 }
    4926            0 :             }
    4927              :         }
    4928              : 
    4929            0 :         result
    4930            0 :     }
    4931              : 
    4932              :     #[instrument(skip_all, fields(tenant_id=%op.tenant_id))]
    4933              :     async fn abort_tenant_shard_split(
    4934              :         &self,
    4935              :         op: &TenantShardSplitAbort,
    4936              :     ) -> Result<(), TenantShardSplitAbortError> {
    4937              :         // Cleaning up a split:
    4938              :         // - Parent shards are not destroyed during a split, just detached.
    4939              :         // - Failed pageserver split API calls can leave the remote node with just the parent attached,
    4940              :         //   just the children attached, or both.
    4941              :         //
    4942              :         // Therefore our work to do is to:
    4943              :         // 1. Clean up storage controller's internal state to just refer to parents, no children
    4944              :         // 2. Call out to pageservers to ensure that children are detached
    4945              :         // 3. Call out to pageservers to ensure that parents are attached.
    4946              :         //
    4947              :         // Crash safety:
    4948              :         // - If the storage controller stops running during this cleanup *after* clearing the splitting state
    4949              :         //   from our database, then [`Self::startup_reconcile`] will regard child attachments as garbage
    4950              :         //   and detach them.
    4951              :         // - TODO: If the storage controller stops running during this cleanup *before* clearing the splitting state
    4952              :         //   from our database, then we will re-enter this cleanup routine on startup.
    4953              : 
    4954              :         let TenantShardSplitAbort {
    4955              :             tenant_id,
    4956              :             new_shard_count,
    4957              :             new_stripe_size,
    4958              :             ..
    4959              :         } = op;
    4960              : 
    4961              :         // First abort persistent state, if any exists.
    4962              :         match self
    4963              :             .persistence
    4964              :             .abort_shard_split(*tenant_id, *new_shard_count)
    4965              :             .await?
    4966              :         {
    4967              :             AbortShardSplitStatus::Aborted => {
    4968              :                 // Proceed to roll back any child shards created on pageservers
    4969              :             }
    4970              :             AbortShardSplitStatus::Complete => {
    4971              :                 // The split completed (we might hit that path if e.g. our database transaction
    4972              :                 // to write the completion landed in the database, but we dropped connection
    4973              :                 // before seeing the result).
    4974              :                 //
    4975              :                 // We must update in-memory state to reflect the successful split.
    4976              :                 self.tenant_shard_split_commit_inmem(
    4977              :                     *tenant_id,
    4978              :                     *new_shard_count,
    4979              :                     *new_stripe_size,
    4980              :                 );
    4981              :                 return Ok(());
    4982              :             }
    4983              :         }
    4984              : 
    4985              :         // Clean up in-memory state, and accumulate the list of child locations that need detaching
    4986              :         let detach_locations: Vec<(Node, TenantShardId)> = {
    4987              :             let mut detach_locations = Vec::new();
    4988              :             let mut locked = self.inner.write().unwrap();
    4989              :             let (nodes, tenants, scheduler) = locked.parts_mut();
    4990              : 
    4991              :             for (tenant_shard_id, shard) in
    4992              :                 tenants.range_mut(TenantShardId::tenant_range(op.tenant_id))
    4993              :             {
    4994              :                 if shard.shard.count == op.new_shard_count {
    4995              :                     // Surprising: the phase of [`Self::do_tenant_shard_split`] which inserts child shards in-memory
    4996              :                     // is infallible, so if we got an error we shouldn't have got that far.
    4997              :                     tracing::warn!(
    4998              :                         "During split abort, child shard {tenant_shard_id} found in-memory"
    4999              :                     );
    5000              :                     continue;
    5001              :                 }
    5002              : 
    5003              :                 // Add the children of this shard to this list of things to detach
    5004              :                 if let Some(node_id) = shard.intent.get_attached() {
    5005              :                     for child_id in tenant_shard_id.split(*new_shard_count) {
    5006              :                         detach_locations.push((
    5007              :                             nodes
    5008              :                                 .get(node_id)
    5009              :                                 .expect("Intent references nonexistent node")
    5010              :                                 .clone(),
    5011              :                             child_id,
    5012              :                         ));
    5013              :                     }
    5014              :                 } else {
    5015              :                     tracing::warn!(
    5016              :                         "During split abort, shard {tenant_shard_id} has no attached location"
    5017              :                     );
    5018              :                 }
    5019              : 
    5020              :                 tracing::info!("Restoring parent shard {tenant_shard_id}");
    5021              : 
    5022              :                 // Drop any intents that refer to unavailable nodes, to enable this abort to proceed even
    5023              :                 // if the original attachment location is offline.
    5024              :                 if let Some(node_id) = shard.intent.get_attached() {
    5025              :                     if !nodes.get(node_id).unwrap().is_available() {
    5026              :                         tracing::info!(
    5027              :                             "Demoting attached intent for {tenant_shard_id} on unavailable node {node_id}"
    5028              :                         );
    5029              :                         shard.intent.demote_attached(scheduler, *node_id);
    5030              :                     }
    5031              :                 }
    5032              :                 for node_id in shard.intent.get_secondary().clone() {
    5033              :                     if !nodes.get(&node_id).unwrap().is_available() {
    5034              :                         tracing::info!(
    5035              :                             "Dropping secondary intent for {tenant_shard_id} on unavailable node {node_id}"
    5036              :                         );
    5037              :                         shard.intent.remove_secondary(scheduler, node_id);
    5038              :                     }
    5039              :                 }
    5040              : 
    5041              :                 shard.splitting = SplitState::Idle;
    5042              :                 if let Err(e) = shard.schedule(scheduler, &mut ScheduleContext::default()) {
    5043              :                     // If this shard can't be scheduled now (perhaps due to offline nodes or
    5044              :                     // capacity issues), that must not prevent us rolling back a split.  In this
    5045              :                     // case it should be eventually scheduled in the background.
    5046              :                     tracing::warn!("Failed to schedule {tenant_shard_id} during shard abort: {e}")
    5047              :                 }
    5048              : 
    5049              :                 self.maybe_reconcile_shard(shard, nodes, ReconcilerPriority::High);
    5050              :             }
    5051              : 
    5052              :             // We don't expect any new_shard_count shards to exist here, but drop them just in case
    5053            0 :             tenants.retain(|_id, s| s.shard.count != *new_shard_count);
    5054              : 
    5055              :             detach_locations
    5056              :         };
    5057              : 
    5058              :         for (node, child_id) in detach_locations {
    5059              :             if !node.is_available() {
    5060              :                 // An unavailable node cannot be cleaned up now: to avoid blocking forever, we will permit this, and
    5061              :                 // rely on the reconciliation that happens when a node transitions to Active to clean up. Since we have
    5062              :                 // removed child shards from our in-memory state and database, the reconciliation will implicitly remove
    5063              :                 // them from the node.
    5064              :                 tracing::warn!(
    5065              :                     "Node {node} unavailable, can't clean up during split abort. It will be cleaned up when it is reactivated."
    5066              :                 );
    5067              :                 continue;
    5068              :             }
    5069              : 
    5070              :             // Detach the remote child.  If the pageserver split API call is still in progress, this call will get
    5071              :             // a 503 and retry, up to our limit.
    5072              :             tracing::info!("Detaching {child_id} on {node}...");
    5073              :             match node
    5074              :                 .with_client_retries(
    5075            0 :                     |client| async move {
    5076            0 :                         let config = LocationConfig {
    5077            0 :                             mode: LocationConfigMode::Detached,
    5078            0 :                             generation: None,
    5079            0 :                             secondary_conf: None,
    5080            0 :                             shard_number: child_id.shard_number.0,
    5081            0 :                             shard_count: child_id.shard_count.literal(),
    5082            0 :                             // Stripe size and tenant config don't matter when detaching
    5083            0 :                             shard_stripe_size: 0,
    5084            0 :                             tenant_conf: TenantConfig::default(),
    5085            0 :                         };
    5086            0 : 
    5087            0 :                         client.location_config(child_id, config, None, false).await
    5088            0 :                     },
    5089              :                     &self.config.pageserver_jwt_token,
    5090              :                     &self.config.ssl_ca_cert,
    5091              :                     1,
    5092              :                     10,
    5093              :                     Duration::from_secs(5),
    5094              :                     &self.cancel,
    5095              :                 )
    5096              :                 .await
    5097              :             {
    5098              :                 Some(Ok(_)) => {}
    5099              :                 Some(Err(e)) => {
    5100              :                     // We failed to communicate with the remote node.  This is problematic: we may be
    5101              :                     // leaving it with a rogue child shard.
    5102              :                     tracing::warn!(
    5103              :                         "Failed to detach child {child_id} from node {node} during abort"
    5104              :                     );
    5105              :                     return Err(e.into());
    5106              :                 }
    5107              :                 None => {
    5108              :                     // Cancellation: we were shutdown or the node went offline. Shutdown is fine, we'll
    5109              :                     // clean up on restart. The node going offline requires a retry.
    5110              :                     return Err(TenantShardSplitAbortError::Unavailable);
    5111              :                 }
    5112              :             };
    5113              :         }
    5114              : 
    5115              :         tracing::info!("Successfully aborted split");
    5116              :         Ok(())
    5117              :     }
    5118              : 
    5119              :     /// Infallible final stage of [`Self::tenant_shard_split`]: update the contents
    5120              :     /// of the tenant map to reflect the child shards that exist after the split.
    5121            0 :     fn tenant_shard_split_commit_inmem(
    5122            0 :         &self,
    5123            0 :         tenant_id: TenantId,
    5124            0 :         new_shard_count: ShardCount,
    5125            0 :         new_stripe_size: Option<ShardStripeSize>,
    5126            0 :     ) -> (
    5127            0 :         TenantShardSplitResponse,
    5128            0 :         Vec<(TenantShardId, NodeId, ShardStripeSize)>,
    5129            0 :         Vec<ReconcilerWaiter>,
    5130            0 :     ) {
    5131            0 :         let mut response = TenantShardSplitResponse {
    5132            0 :             new_shards: Vec::new(),
    5133            0 :         };
    5134            0 :         let mut child_locations = Vec::new();
    5135            0 :         let mut waiters = Vec::new();
    5136            0 : 
    5137            0 :         {
    5138            0 :             let mut locked = self.inner.write().unwrap();
    5139            0 : 
    5140            0 :             let parent_ids = locked
    5141            0 :                 .tenants
    5142            0 :                 .range(TenantShardId::tenant_range(tenant_id))
    5143            0 :                 .map(|(shard_id, _)| *shard_id)
    5144            0 :                 .collect::<Vec<_>>();
    5145            0 : 
    5146            0 :             let (nodes, tenants, scheduler) = locked.parts_mut();
    5147            0 :             for parent_id in parent_ids {
    5148            0 :                 let child_ids = parent_id.split(new_shard_count);
    5149              : 
    5150            0 :                 let (pageserver, generation, policy, parent_ident, config, preferred_az) = {
    5151            0 :                     let mut old_state = tenants
    5152            0 :                         .remove(&parent_id)
    5153            0 :                         .expect("It was present, we just split it");
    5154            0 : 
    5155            0 :                     // A non-splitting state is impossible, because [`Self::tenant_shard_split`] holds
    5156            0 :                     // a TenantId lock and passes it through to [`TenantShardSplitAbort`] in case of cleanup:
    5157            0 :                     // nothing else can clear this.
    5158            0 :                     assert!(matches!(old_state.splitting, SplitState::Splitting));
    5159              : 
    5160            0 :                     let old_attached = old_state.intent.get_attached().unwrap();
    5161            0 :                     old_state.intent.clear(scheduler);
    5162            0 :                     let generation = old_state.generation.expect("Shard must have been attached");
    5163            0 :                     (
    5164            0 :                         old_attached,
    5165            0 :                         generation,
    5166            0 :                         old_state.policy.clone(),
    5167            0 :                         old_state.shard,
    5168            0 :                         old_state.config.clone(),
    5169            0 :                         old_state.preferred_az().cloned(),
    5170            0 :                     )
    5171            0 :                 };
    5172            0 : 
    5173            0 :                 let mut schedule_context = ScheduleContext::default();
    5174            0 :                 for child in child_ids {
    5175            0 :                     let mut child_shard = parent_ident;
    5176            0 :                     child_shard.number = child.shard_number;
    5177            0 :                     child_shard.count = child.shard_count;
    5178            0 :                     if let Some(stripe_size) = new_stripe_size {
    5179            0 :                         child_shard.stripe_size = stripe_size;
    5180            0 :                     }
    5181              : 
    5182            0 :                     let mut child_observed: HashMap<NodeId, ObservedStateLocation> = HashMap::new();
    5183            0 :                     child_observed.insert(
    5184            0 :                         pageserver,
    5185            0 :                         ObservedStateLocation {
    5186            0 :                             conf: Some(attached_location_conf(
    5187            0 :                                 generation,
    5188            0 :                                 &child_shard,
    5189            0 :                                 &config,
    5190            0 :                                 &policy,
    5191            0 :                             )),
    5192            0 :                         },
    5193            0 :                     );
    5194            0 : 
    5195            0 :                     let mut child_state =
    5196            0 :                         TenantShard::new(child, child_shard, policy.clone(), preferred_az.clone());
    5197            0 :                     child_state.intent =
    5198            0 :                         IntentState::single(scheduler, Some(pageserver), preferred_az.clone());
    5199            0 :                     child_state.observed = ObservedState {
    5200            0 :                         locations: child_observed,
    5201            0 :                     };
    5202            0 :                     child_state.generation = Some(generation);
    5203            0 :                     child_state.config = config.clone();
    5204            0 : 
    5205            0 :                     // The child's TenantShard::splitting is intentionally left at the default value of Idle,
    5206            0 :                     // as at this point in the split process we have succeeded and this part is infallible:
    5207            0 :                     // we will never need to do any special recovery from this state.
    5208            0 : 
    5209            0 :                     child_locations.push((child, pageserver, child_shard.stripe_size));
    5210              : 
    5211            0 :                     if let Err(e) = child_state.schedule(scheduler, &mut schedule_context) {
    5212              :                         // This is not fatal, because we've implicitly already got an attached
    5213              :                         // location for the child shard.  Failure here just means we couldn't
    5214              :                         // find a secondary (e.g. because cluster is overloaded).
    5215            0 :                         tracing::warn!("Failed to schedule child shard {child}: {e}");
    5216            0 :                     }
    5217              :                     // In the background, attach secondary locations for the new shards
    5218            0 :                     if let Some(waiter) = self.maybe_reconcile_shard(
    5219            0 :                         &mut child_state,
    5220            0 :                         nodes,
    5221            0 :                         ReconcilerPriority::High,
    5222            0 :                     ) {
    5223            0 :                         waiters.push(waiter);
    5224            0 :                     }
    5225              : 
    5226            0 :                     tenants.insert(child, child_state);
    5227            0 :                     response.new_shards.push(child);
    5228              :                 }
    5229              :             }
    5230            0 :             (response, child_locations, waiters)
    5231            0 :         }
    5232            0 :     }
    5233              : 
    5234            0 :     async fn tenant_shard_split_start_secondaries(
    5235            0 :         &self,
    5236            0 :         tenant_id: TenantId,
    5237            0 :         waiters: Vec<ReconcilerWaiter>,
    5238            0 :     ) {
    5239              :         // Wait for initial reconcile of child shards, this creates the secondary locations
    5240            0 :         if let Err(e) = self.await_waiters(waiters, RECONCILE_TIMEOUT).await {
    5241              :             // This is not a failure to split: it's some issue reconciling the new child shards, perhaps
    5242              :             // their secondaries couldn't be attached.
    5243            0 :             tracing::warn!("Failed to reconcile after split: {e}");
    5244            0 :             return;
    5245            0 :         }
    5246              : 
    5247              :         // Take the state lock to discover the attached & secondary intents for all shards
    5248            0 :         let (attached, secondary) = {
    5249            0 :             let locked = self.inner.read().unwrap();
    5250            0 :             let mut attached = Vec::new();
    5251            0 :             let mut secondary = Vec::new();
    5252              : 
    5253            0 :             for (tenant_shard_id, shard) in
    5254            0 :                 locked.tenants.range(TenantShardId::tenant_range(tenant_id))
    5255              :             {
    5256            0 :                 let Some(node_id) = shard.intent.get_attached() else {
    5257              :                     // Unexpected.  Race with a PlacementPolicy change?
    5258            0 :                     tracing::warn!(
    5259            0 :                         "No attached node on {tenant_shard_id} immediately after shard split!"
    5260              :                     );
    5261            0 :                     continue;
    5262              :                 };
    5263              : 
    5264            0 :                 let Some(secondary_node_id) = shard.intent.get_secondary().first() else {
    5265              :                     // No secondary location.  Nothing for us to do.
    5266            0 :                     continue;
    5267              :                 };
    5268              : 
    5269            0 :                 let attached_node = locked
    5270            0 :                     .nodes
    5271            0 :                     .get(node_id)
    5272            0 :                     .expect("Pageservers may not be deleted while referenced");
    5273            0 : 
    5274            0 :                 let secondary_node = locked
    5275            0 :                     .nodes
    5276            0 :                     .get(secondary_node_id)
    5277            0 :                     .expect("Pageservers may not be deleted while referenced");
    5278            0 : 
    5279            0 :                 attached.push((*tenant_shard_id, attached_node.clone()));
    5280            0 :                 secondary.push((*tenant_shard_id, secondary_node.clone()));
    5281              :             }
    5282            0 :             (attached, secondary)
    5283            0 :         };
    5284            0 : 
    5285            0 :         if secondary.is_empty() {
    5286              :             // No secondary locations; nothing for us to do
    5287            0 :             return;
    5288            0 :         }
    5289              : 
    5290            0 :         for result in self
    5291            0 :             .tenant_for_shards_api(
    5292            0 :                 attached,
    5293            0 :                 |tenant_shard_id, client| async move {
    5294            0 :                     client.tenant_heatmap_upload(tenant_shard_id).await
    5295            0 :                 },
    5296            0 :                 1,
    5297            0 :                 1,
    5298            0 :                 SHORT_RECONCILE_TIMEOUT,
    5299            0 :                 &self.cancel,
    5300            0 :             )
    5301            0 :             .await
    5302              :         {
    5303            0 :             if let Err(e) = result {
    5304            0 :                 tracing::warn!("Error calling heatmap upload after shard split: {e}");
    5305            0 :                 return;
    5306            0 :             }
    5307              :         }
    5308              : 
    5309            0 :         for result in self
    5310            0 :             .tenant_for_shards_api(
    5311            0 :                 secondary,
    5312            0 :                 |tenant_shard_id, client| async move {
    5313            0 :                     client
    5314            0 :                         .tenant_secondary_download(tenant_shard_id, Some(Duration::ZERO))
    5315            0 :                         .await
    5316            0 :                 },
    5317            0 :                 1,
    5318            0 :                 1,
    5319            0 :                 SHORT_RECONCILE_TIMEOUT,
    5320            0 :                 &self.cancel,
    5321            0 :             )
    5322            0 :             .await
    5323              :         {
    5324            0 :             if let Err(e) = result {
    5325            0 :                 tracing::warn!("Error calling secondary download after shard split: {e}");
    5326            0 :                 return;
    5327            0 :             }
    5328              :         }
    5329            0 :     }
    5330              : 
    5331            0 :     pub(crate) async fn tenant_shard_split(
    5332            0 :         &self,
    5333            0 :         tenant_id: TenantId,
    5334            0 :         split_req: TenantShardSplitRequest,
    5335            0 :     ) -> Result<TenantShardSplitResponse, ApiError> {
    5336              :         // TODO: return 503 if we get stuck waiting for this lock
    5337              :         // (issue https://github.com/neondatabase/neon/issues/7108)
    5338            0 :         let _tenant_lock = trace_exclusive_lock(
    5339            0 :             &self.tenant_op_locks,
    5340            0 :             tenant_id,
    5341            0 :             TenantOperations::ShardSplit,
    5342            0 :         )
    5343            0 :         .await;
    5344              : 
    5345            0 :         let new_shard_count = ShardCount::new(split_req.new_shard_count);
    5346            0 :         let new_stripe_size = split_req.new_stripe_size;
    5347              : 
    5348              :         // Validate the request and construct parameters.  This phase is fallible, but does not require
    5349              :         // rollback on errors, as it does no I/O and mutates no state.
    5350            0 :         let shard_split_params = match self.prepare_tenant_shard_split(tenant_id, split_req)? {
    5351            0 :             ShardSplitAction::NoOp(resp) => return Ok(resp),
    5352            0 :             ShardSplitAction::Split(params) => params,
    5353              :         };
    5354              : 
    5355              :         // Execute this split: this phase mutates state and does remote I/O on pageservers.  If it fails,
    5356              :         // we must roll back.
    5357            0 :         let r = self
    5358            0 :             .do_tenant_shard_split(tenant_id, shard_split_params)
    5359            0 :             .await;
    5360              : 
    5361            0 :         let (response, waiters) = match r {
    5362            0 :             Ok(r) => r,
    5363            0 :             Err(e) => {
    5364            0 :                 // Split might be part-done, we must do work to abort it.
    5365            0 :                 tracing::warn!("Enqueuing background abort of split on {tenant_id}");
    5366            0 :                 self.abort_tx
    5367            0 :                     .send(TenantShardSplitAbort {
    5368            0 :                         tenant_id,
    5369            0 :                         new_shard_count,
    5370            0 :                         new_stripe_size,
    5371            0 :                         _tenant_lock,
    5372            0 :                     })
    5373            0 :                     // Ignore error sending: that just means we're shutting down: aborts are ephemeral so it's fine to drop it.
    5374            0 :                     .ok();
    5375            0 :                 return Err(e);
    5376              :             }
    5377              :         };
    5378              : 
    5379              :         // The split is now complete.  As an optimization, we will trigger all the child shards to upload
    5380              :         // a heatmap immediately, and all their secondary locations to start downloading: this avoids waiting
    5381              :         // for the background heatmap/download interval before secondaries get warm enough to migrate shards
    5382              :         // in [`Self::optimize_all`]
    5383            0 :         self.tenant_shard_split_start_secondaries(tenant_id, waiters)
    5384            0 :             .await;
    5385            0 :         Ok(response)
    5386            0 :     }
    5387              : 
    5388            0 :     fn prepare_tenant_shard_split(
    5389            0 :         &self,
    5390            0 :         tenant_id: TenantId,
    5391            0 :         split_req: TenantShardSplitRequest,
    5392            0 :     ) -> Result<ShardSplitAction, ApiError> {
    5393            0 :         fail::fail_point!("shard-split-validation", |_| Err(ApiError::BadRequest(
    5394            0 :             anyhow::anyhow!("failpoint")
    5395            0 :         )));
    5396              : 
    5397            0 :         let mut policy = None;
    5398            0 :         let mut config = None;
    5399            0 :         let mut shard_ident = None;
    5400            0 :         let mut preferred_az_id = None;
    5401              :         // Validate input, and calculate which shards we will create
    5402            0 :         let (old_shard_count, targets) =
    5403              :             {
    5404            0 :                 let locked = self.inner.read().unwrap();
    5405            0 : 
    5406            0 :                 let pageservers = locked.nodes.clone();
    5407            0 : 
    5408            0 :                 let mut targets = Vec::new();
    5409            0 : 
    5410            0 :                 // In case this is a retry, count how many already-split shards we found
    5411            0 :                 let mut children_found = Vec::new();
    5412            0 :                 let mut old_shard_count = None;
    5413              : 
    5414            0 :                 for (tenant_shard_id, shard) in
    5415            0 :                     locked.tenants.range(TenantShardId::tenant_range(tenant_id))
    5416              :                 {
    5417            0 :                     match shard.shard.count.count().cmp(&split_req.new_shard_count) {
    5418              :                         Ordering::Equal => {
    5419              :                             //  Already split this
    5420            0 :                             children_found.push(*tenant_shard_id);
    5421            0 :                             continue;
    5422              :                         }
    5423              :                         Ordering::Greater => {
    5424            0 :                             return Err(ApiError::BadRequest(anyhow::anyhow!(
    5425            0 :                                 "Requested count {} but already have shards at count {}",
    5426            0 :                                 split_req.new_shard_count,
    5427            0 :                                 shard.shard.count.count()
    5428            0 :                             )));
    5429              :                         }
    5430            0 :                         Ordering::Less => {
    5431            0 :                             // Fall through: this shard has lower count than requested,
    5432            0 :                             // is a candidate for splitting.
    5433            0 :                         }
    5434            0 :                     }
    5435            0 : 
    5436            0 :                     match old_shard_count {
    5437            0 :                         None => old_shard_count = Some(shard.shard.count),
    5438            0 :                         Some(old_shard_count) => {
    5439            0 :                             if old_shard_count != shard.shard.count {
    5440              :                                 // We may hit this case if a caller asked for two splits to
    5441              :                                 // different sizes, before the first one is complete.
    5442              :                                 // e.g. 1->2, 2->4, where the 4 call comes while we have a mixture
    5443              :                                 // of shard_count=1 and shard_count=2 shards in the map.
    5444            0 :                                 return Err(ApiError::Conflict(
    5445            0 :                                     "Cannot split, currently mid-split".to_string(),
    5446            0 :                                 ));
    5447            0 :                             }
    5448              :                         }
    5449              :                     }
    5450            0 :                     if policy.is_none() {
    5451            0 :                         policy = Some(shard.policy.clone());
    5452            0 :                     }
    5453            0 :                     if shard_ident.is_none() {
    5454            0 :                         shard_ident = Some(shard.shard);
    5455            0 :                     }
    5456            0 :                     if config.is_none() {
    5457            0 :                         config = Some(shard.config.clone());
    5458            0 :                     }
    5459            0 :                     if preferred_az_id.is_none() {
    5460            0 :                         preferred_az_id = shard.preferred_az().cloned();
    5461            0 :                     }
    5462              : 
    5463            0 :                     if tenant_shard_id.shard_count.count() == split_req.new_shard_count {
    5464            0 :                         tracing::info!(
    5465            0 :                             "Tenant shard {} already has shard count {}",
    5466              :                             tenant_shard_id,
    5467              :                             split_req.new_shard_count
    5468              :                         );
    5469            0 :                         continue;
    5470            0 :                     }
    5471              : 
    5472            0 :                     let node_id = shard.intent.get_attached().ok_or(ApiError::BadRequest(
    5473            0 :                         anyhow::anyhow!("Cannot split a tenant that is not attached"),
    5474            0 :                     ))?;
    5475              : 
    5476            0 :                     let node = pageservers
    5477            0 :                         .get(&node_id)
    5478            0 :                         .expect("Pageservers may not be deleted while referenced");
    5479            0 : 
    5480            0 :                     targets.push(ShardSplitTarget {
    5481            0 :                         parent_id: *tenant_shard_id,
    5482            0 :                         node: node.clone(),
    5483            0 :                         child_ids: tenant_shard_id
    5484            0 :                             .split(ShardCount::new(split_req.new_shard_count)),
    5485            0 :                     });
    5486              :                 }
    5487              : 
    5488            0 :                 if targets.is_empty() {
    5489            0 :                     if children_found.len() == split_req.new_shard_count as usize {
    5490            0 :                         return Ok(ShardSplitAction::NoOp(TenantShardSplitResponse {
    5491            0 :                             new_shards: children_found,
    5492            0 :                         }));
    5493              :                     } else {
    5494              :                         // No shards found to split, and no existing children found: the
    5495              :                         // tenant doesn't exist at all.
    5496            0 :                         return Err(ApiError::NotFound(
    5497            0 :                             anyhow::anyhow!("Tenant {} not found", tenant_id).into(),
    5498            0 :                         ));
    5499              :                     }
    5500            0 :                 }
    5501            0 : 
    5502            0 :                 (old_shard_count, targets)
    5503            0 :             };
    5504            0 : 
    5505            0 :         // unwrap safety: we would have returned above if we didn't find at least one shard to split
    5506            0 :         let old_shard_count = old_shard_count.unwrap();
    5507            0 :         let shard_ident = if let Some(new_stripe_size) = split_req.new_stripe_size {
    5508              :             // This ShardIdentity will be used as the template for all children, so this implicitly
    5509              :             // applies the new stripe size to the children.
    5510            0 :             let mut shard_ident = shard_ident.unwrap();
    5511            0 :             if shard_ident.count.count() > 1 && shard_ident.stripe_size != new_stripe_size {
    5512            0 :                 return Err(ApiError::BadRequest(anyhow::anyhow!(
    5513            0 :                     "Attempted to change stripe size ({:?}->{new_stripe_size:?}) on a tenant with multiple shards",
    5514            0 :                     shard_ident.stripe_size
    5515            0 :                 )));
    5516            0 :             }
    5517            0 : 
    5518            0 :             shard_ident.stripe_size = new_stripe_size;
    5519            0 :             tracing::info!("applied  stripe size {}", shard_ident.stripe_size.0);
    5520            0 :             shard_ident
    5521              :         } else {
    5522            0 :             shard_ident.unwrap()
    5523              :         };
    5524            0 :         let policy = policy.unwrap();
    5525            0 :         let config = config.unwrap();
    5526            0 : 
    5527            0 :         Ok(ShardSplitAction::Split(Box::new(ShardSplitParams {
    5528            0 :             old_shard_count,
    5529            0 :             new_shard_count: ShardCount::new(split_req.new_shard_count),
    5530            0 :             new_stripe_size: split_req.new_stripe_size,
    5531            0 :             targets,
    5532            0 :             policy,
    5533            0 :             config,
    5534            0 :             shard_ident,
    5535            0 :             preferred_az_id,
    5536            0 :         })))
    5537            0 :     }
    5538              : 
    5539            0 :     async fn do_tenant_shard_split(
    5540            0 :         &self,
    5541            0 :         tenant_id: TenantId,
    5542            0 :         params: Box<ShardSplitParams>,
    5543            0 :     ) -> Result<(TenantShardSplitResponse, Vec<ReconcilerWaiter>), ApiError> {
    5544            0 :         // FIXME: we have dropped self.inner lock, and not yet written anything to the database: another
    5545            0 :         // request could occur here, deleting or mutating the tenant.  begin_shard_split checks that the
    5546            0 :         // parent shards exist as expected, but it would be neater to do the above pre-checks within the
    5547            0 :         // same database transaction rather than pre-check in-memory and then maybe-fail the database write.
    5548            0 :         // (https://github.com/neondatabase/neon/issues/6676)
    5549            0 : 
    5550            0 :         let ShardSplitParams {
    5551            0 :             old_shard_count,
    5552            0 :             new_shard_count,
    5553            0 :             new_stripe_size,
    5554            0 :             mut targets,
    5555            0 :             policy,
    5556            0 :             config,
    5557            0 :             shard_ident,
    5558            0 :             preferred_az_id,
    5559            0 :         } = *params;
    5560              : 
    5561              :         // Drop any secondary locations: pageservers do not support splitting these, and in any case the
    5562              :         // end-state for a split tenant will usually be to have secondary locations on different nodes.
    5563              :         // The reconciliation calls in this block also implicitly cancel+barrier wrt any ongoing reconciliation
    5564              :         // at the time of split.
    5565            0 :         let waiters = {
    5566            0 :             let mut locked = self.inner.write().unwrap();
    5567            0 :             let mut waiters = Vec::new();
    5568            0 :             let (nodes, tenants, scheduler) = locked.parts_mut();
    5569            0 :             for target in &mut targets {
    5570            0 :                 let Some(shard) = tenants.get_mut(&target.parent_id) else {
    5571              :                     // Paranoia check: this shouldn't happen: we have the oplock for this tenant ID.
    5572            0 :                     return Err(ApiError::InternalServerError(anyhow::anyhow!(
    5573            0 :                         "Shard {} not found",
    5574            0 :                         target.parent_id
    5575            0 :                     )));
    5576              :                 };
    5577              : 
    5578            0 :                 if shard.intent.get_attached() != &Some(target.node.get_id()) {
    5579              :                     // Paranoia check: this shouldn't happen: we have the oplock for this tenant ID.
    5580            0 :                     return Err(ApiError::Conflict(format!(
    5581            0 :                         "Shard {} unexpectedly rescheduled during split",
    5582            0 :                         target.parent_id
    5583            0 :                     )));
    5584            0 :                 }
    5585            0 : 
    5586            0 :                 // Irrespective of PlacementPolicy, clear secondary locations from intent
    5587            0 :                 shard.intent.clear_secondary(scheduler);
    5588              : 
    5589              :                 // Run Reconciler to execute detach fo secondary locations.
    5590            0 :                 if let Some(waiter) =
    5591            0 :                     self.maybe_reconcile_shard(shard, nodes, ReconcilerPriority::High)
    5592            0 :                 {
    5593            0 :                     waiters.push(waiter);
    5594            0 :                 }
    5595              :             }
    5596            0 :             waiters
    5597            0 :         };
    5598            0 :         self.await_waiters(waiters, RECONCILE_TIMEOUT).await?;
    5599              : 
    5600              :         // Before creating any new child shards in memory or on the pageservers, persist them: this
    5601              :         // enables us to ensure that we will always be able to clean up if something goes wrong.  This also
    5602              :         // acts as the protection against two concurrent attempts to split: one of them will get a database
    5603              :         // error trying to insert the child shards.
    5604            0 :         let mut child_tsps = Vec::new();
    5605            0 :         for target in &targets {
    5606            0 :             let mut this_child_tsps = Vec::new();
    5607            0 :             for child in &target.child_ids {
    5608            0 :                 let mut child_shard = shard_ident;
    5609            0 :                 child_shard.number = child.shard_number;
    5610            0 :                 child_shard.count = child.shard_count;
    5611            0 : 
    5612            0 :                 tracing::info!(
    5613            0 :                     "Create child shard persistence with stripe size {}",
    5614              :                     shard_ident.stripe_size.0
    5615              :                 );
    5616              : 
    5617            0 :                 this_child_tsps.push(TenantShardPersistence {
    5618            0 :                     tenant_id: child.tenant_id.to_string(),
    5619            0 :                     shard_number: child.shard_number.0 as i32,
    5620            0 :                     shard_count: child.shard_count.literal() as i32,
    5621            0 :                     shard_stripe_size: shard_ident.stripe_size.0 as i32,
    5622            0 :                     // Note: this generation is a placeholder, [`Persistence::begin_shard_split`] will
    5623            0 :                     // populate the correct generation as part of its transaction, to protect us
    5624            0 :                     // against racing with changes in the state of the parent.
    5625            0 :                     generation: None,
    5626            0 :                     generation_pageserver: Some(target.node.get_id().0 as i64),
    5627            0 :                     placement_policy: serde_json::to_string(&policy).unwrap(),
    5628            0 :                     config: serde_json::to_string(&config).unwrap(),
    5629            0 :                     splitting: SplitState::Splitting,
    5630            0 : 
    5631            0 :                     // Scheduling policies and preferred AZ do not carry through to children
    5632            0 :                     scheduling_policy: serde_json::to_string(&ShardSchedulingPolicy::default())
    5633            0 :                         .unwrap(),
    5634            0 :                     preferred_az_id: preferred_az_id.as_ref().map(|az| az.0.clone()),
    5635            0 :                 });
    5636            0 :             }
    5637              : 
    5638            0 :             child_tsps.push((target.parent_id, this_child_tsps));
    5639              :         }
    5640              : 
    5641            0 :         if let Err(e) = self
    5642            0 :             .persistence
    5643            0 :             .begin_shard_split(old_shard_count, tenant_id, child_tsps)
    5644            0 :             .await
    5645              :         {
    5646            0 :             match e {
    5647              :                 DatabaseError::Query(diesel::result::Error::DatabaseError(
    5648              :                     DatabaseErrorKind::UniqueViolation,
    5649              :                     _,
    5650              :                 )) => {
    5651              :                     // Inserting a child shard violated a unique constraint: we raced with another call to
    5652              :                     // this function
    5653            0 :                     tracing::warn!("Conflicting attempt to split {tenant_id}: {e}");
    5654            0 :                     return Err(ApiError::Conflict("Tenant is already splitting".into()));
    5655              :                 }
    5656            0 :                 _ => return Err(ApiError::InternalServerError(e.into())),
    5657              :             }
    5658            0 :         }
    5659            0 :         fail::fail_point!("shard-split-post-begin", |_| Err(
    5660            0 :             ApiError::InternalServerError(anyhow::anyhow!("failpoint"))
    5661            0 :         ));
    5662              : 
    5663              :         // Now that I have persisted the splitting state, apply it in-memory.  This is infallible, so
    5664              :         // callers may assume that if splitting is set in memory, then it was persisted, and if splitting
    5665              :         // is not set in memory, then it was not persisted.
    5666              :         {
    5667            0 :             let mut locked = self.inner.write().unwrap();
    5668            0 :             for target in &targets {
    5669            0 :                 if let Some(parent_shard) = locked.tenants.get_mut(&target.parent_id) {
    5670            0 :                     parent_shard.splitting = SplitState::Splitting;
    5671            0 :                     // Put the observed state to None, to reflect that it is indeterminate once we start the
    5672            0 :                     // split operation.
    5673            0 :                     parent_shard
    5674            0 :                         .observed
    5675            0 :                         .locations
    5676            0 :                         .insert(target.node.get_id(), ObservedStateLocation { conf: None });
    5677            0 :                 }
    5678              :             }
    5679              :         }
    5680              : 
    5681              :         // TODO: issue split calls concurrently (this only matters once we're splitting
    5682              :         // N>1 shards into M shards -- initially we're usually splitting 1 shard into N).
    5683              : 
    5684            0 :         for target in &targets {
    5685              :             let ShardSplitTarget {
    5686            0 :                 parent_id,
    5687            0 :                 node,
    5688            0 :                 child_ids,
    5689            0 :             } = target;
    5690            0 :             let client = PageserverClient::new(
    5691            0 :                 node.get_id(),
    5692            0 :                 node.base_url(),
    5693            0 :                 self.config.pageserver_jwt_token.as_deref(),
    5694            0 :                 self.config.ssl_ca_cert.clone(),
    5695            0 :             )
    5696            0 :             .map_err(|e| passthrough_api_error(node, e))?;
    5697            0 :             let response = client
    5698            0 :                 .tenant_shard_split(
    5699            0 :                     *parent_id,
    5700            0 :                     TenantShardSplitRequest {
    5701            0 :                         new_shard_count: new_shard_count.literal(),
    5702            0 :                         new_stripe_size,
    5703            0 :                     },
    5704            0 :                 )
    5705            0 :                 .await
    5706            0 :                 .map_err(|e| ApiError::Conflict(format!("Failed to split {}: {}", parent_id, e)))?;
    5707              : 
    5708            0 :             fail::fail_point!("shard-split-post-remote", |_| Err(ApiError::Conflict(
    5709            0 :                 "failpoint".to_string()
    5710            0 :             )));
    5711              : 
    5712            0 :             failpoint_support::sleep_millis_async!("shard-split-post-remote-sleep", &self.cancel);
    5713              : 
    5714            0 :             tracing::info!(
    5715            0 :                 "Split {} into {}",
    5716            0 :                 parent_id,
    5717            0 :                 response
    5718            0 :                     .new_shards
    5719            0 :                     .iter()
    5720            0 :                     .map(|s| format!("{:?}", s))
    5721            0 :                     .collect::<Vec<_>>()
    5722            0 :                     .join(",")
    5723              :             );
    5724              : 
    5725            0 :             if &response.new_shards != child_ids {
    5726              :                 // This should never happen: the pageserver should agree with us on how shard splits work.
    5727            0 :                 return Err(ApiError::InternalServerError(anyhow::anyhow!(
    5728            0 :                     "Splitting shard {} resulted in unexpected IDs: {:?} (expected {:?})",
    5729            0 :                     parent_id,
    5730            0 :                     response.new_shards,
    5731            0 :                     child_ids
    5732            0 :                 )));
    5733            0 :             }
    5734              :         }
    5735              : 
    5736              :         // TODO: if the pageserver restarted concurrently with our split API call,
    5737              :         // the actual generation of the child shard might differ from the generation
    5738              :         // we expect it to have.  In order for our in-database generation to end up
    5739              :         // correct, we should carry the child generation back in the response and apply it here
    5740              :         // in complete_shard_split (and apply the correct generation in memory)
    5741              :         // (or, we can carry generation in the request and reject the request if
    5742              :         //  it doesn't match, but that requires more retry logic on this side)
    5743              : 
    5744            0 :         self.persistence
    5745            0 :             .complete_shard_split(tenant_id, old_shard_count)
    5746            0 :             .await?;
    5747              : 
    5748            0 :         fail::fail_point!("shard-split-post-complete", |_| Err(
    5749            0 :             ApiError::InternalServerError(anyhow::anyhow!("failpoint"))
    5750            0 :         ));
    5751              : 
    5752              :         // Replace all the shards we just split with their children: this phase is infallible.
    5753            0 :         let (response, child_locations, waiters) =
    5754            0 :             self.tenant_shard_split_commit_inmem(tenant_id, new_shard_count, new_stripe_size);
    5755            0 : 
    5756            0 :         // Send compute notifications for all the new shards
    5757            0 :         let mut failed_notifications = Vec::new();
    5758            0 :         for (child_id, child_ps, stripe_size) in child_locations {
    5759            0 :             if let Err(e) = self
    5760            0 :                 .compute_hook
    5761            0 :                 .notify(
    5762            0 :                     compute_hook::ShardUpdate {
    5763            0 :                         tenant_shard_id: child_id,
    5764            0 :                         node_id: child_ps,
    5765            0 :                         stripe_size,
    5766            0 :                         preferred_az: preferred_az_id.as_ref().map(Cow::Borrowed),
    5767            0 :                     },
    5768            0 :                     &self.cancel,
    5769            0 :                 )
    5770            0 :                 .await
    5771              :             {
    5772            0 :                 tracing::warn!(
    5773            0 :                     "Failed to update compute of {}->{} during split, proceeding anyway to complete split ({e})",
    5774              :                     child_id,
    5775              :                     child_ps
    5776              :                 );
    5777            0 :                 failed_notifications.push(child_id);
    5778            0 :             }
    5779              :         }
    5780              : 
    5781              :         // If we failed any compute notifications, make a note to retry later.
    5782            0 :         if !failed_notifications.is_empty() {
    5783            0 :             let mut locked = self.inner.write().unwrap();
    5784            0 :             for failed in failed_notifications {
    5785            0 :                 if let Some(shard) = locked.tenants.get_mut(&failed) {
    5786            0 :                     shard.pending_compute_notification = true;
    5787            0 :                 }
    5788              :             }
    5789            0 :         }
    5790              : 
    5791            0 :         Ok((response, waiters))
    5792            0 :     }
    5793              : 
    5794              :     /// A graceful migration: update the preferred node and let optimisation handle the migration
    5795              :     /// in the background (may take a long time as it will fully warm up a location before cutting over)
    5796              :     ///
    5797              :     /// Our external API calls this a 'prewarm=true' migration, but internally it isn't a special prewarm step: it's
    5798              :     /// just a migration that uses the same graceful procedure as our background scheduling optimisations would use.
    5799            0 :     fn tenant_shard_migrate_with_prewarm(
    5800            0 :         &self,
    5801            0 :         migrate_req: &TenantShardMigrateRequest,
    5802            0 :         shard: &mut TenantShard,
    5803            0 :         scheduler: &mut Scheduler,
    5804            0 :         schedule_context: ScheduleContext,
    5805            0 :     ) -> Result<Option<ScheduleOptimization>, ApiError> {
    5806            0 :         shard.set_preferred_node(Some(migrate_req.node_id));
    5807            0 : 
    5808            0 :         // Generate whatever the initial change to the intent is: this could be creation of a secondary, or
    5809            0 :         // cutting over to an existing secondary.  Caller is responsible for validating this before applying it,
    5810            0 :         // e.g. by checking secondary is warm enough.
    5811            0 :         Ok(shard.optimize_attachment(scheduler, &schedule_context))
    5812            0 :     }
    5813              : 
    5814              :     /// Immediate migration: directly update the intent state and kick off a reconciler
    5815            0 :     fn tenant_shard_migrate_immediate(
    5816            0 :         &self,
    5817            0 :         migrate_req: &TenantShardMigrateRequest,
    5818            0 :         nodes: &Arc<HashMap<NodeId, Node>>,
    5819            0 :         shard: &mut TenantShard,
    5820            0 :         scheduler: &mut Scheduler,
    5821            0 :     ) -> Result<Option<ReconcilerWaiter>, ApiError> {
    5822            0 :         // Non-graceful migration: update the intent state immediately
    5823            0 :         let old_attached = *shard.intent.get_attached();
    5824            0 :         match shard.policy {
    5825            0 :             PlacementPolicy::Attached(n) => {
    5826            0 :                 // If our new attached node was a secondary, it no longer should be.
    5827            0 :                 shard
    5828            0 :                     .intent
    5829            0 :                     .remove_secondary(scheduler, migrate_req.node_id);
    5830            0 : 
    5831            0 :                 shard
    5832            0 :                     .intent
    5833            0 :                     .set_attached(scheduler, Some(migrate_req.node_id));
    5834              : 
    5835              :                 // If we were already attached to something, demote that to a secondary
    5836            0 :                 if let Some(old_attached) = old_attached {
    5837            0 :                     if n > 0 {
    5838              :                         // Remove other secondaries to make room for the location we'll demote
    5839            0 :                         while shard.intent.get_secondary().len() >= n {
    5840            0 :                             shard.intent.pop_secondary(scheduler);
    5841            0 :                         }
    5842              : 
    5843            0 :                         shard.intent.push_secondary(scheduler, old_attached);
    5844            0 :                     }
    5845            0 :                 }
    5846              :             }
    5847            0 :             PlacementPolicy::Secondary => {
    5848            0 :                 shard.intent.clear(scheduler);
    5849            0 :                 shard.intent.push_secondary(scheduler, migrate_req.node_id);
    5850            0 :             }
    5851              :             PlacementPolicy::Detached => {
    5852            0 :                 return Err(ApiError::BadRequest(anyhow::anyhow!(
    5853            0 :                     "Cannot migrate a tenant that is PlacementPolicy::Detached: configure it to an attached policy first"
    5854            0 :                 )));
    5855              :             }
    5856              :         }
    5857              : 
    5858            0 :         tracing::info!("Migrating: new intent {:?}", shard.intent);
    5859            0 :         shard.sequence = shard.sequence.next();
    5860            0 :         shard.set_preferred_node(None); // Abort any in-flight graceful migration
    5861            0 :         Ok(self.maybe_configured_reconcile_shard(
    5862            0 :             shard,
    5863            0 :             nodes,
    5864            0 :             (&migrate_req.migration_config).into(),
    5865            0 :         ))
    5866            0 :     }
    5867              : 
    5868            0 :     pub(crate) async fn tenant_shard_migrate(
    5869            0 :         &self,
    5870            0 :         tenant_shard_id: TenantShardId,
    5871            0 :         migrate_req: TenantShardMigrateRequest,
    5872            0 :     ) -> Result<TenantShardMigrateResponse, ApiError> {
    5873              :         // Depending on whether the migration is a change and whether it's graceful or immediate, we might
    5874              :         // get a different outcome to handle
    5875              :         enum MigrationOutcome {
    5876              :             Optimization(Option<ScheduleOptimization>),
    5877              :             Reconcile(Option<ReconcilerWaiter>),
    5878              :         }
    5879              : 
    5880            0 :         let outcome = {
    5881            0 :             let mut locked = self.inner.write().unwrap();
    5882            0 :             let (nodes, tenants, scheduler) = locked.parts_mut();
    5883              : 
    5884            0 :             let Some(node) = nodes.get(&migrate_req.node_id) else {
    5885            0 :                 return Err(ApiError::BadRequest(anyhow::anyhow!(
    5886            0 :                     "Node {} not found",
    5887            0 :                     migrate_req.node_id
    5888            0 :                 )));
    5889              :             };
    5890              : 
    5891              :             // Migration to unavavailable node requires force flag
    5892            0 :             if !node.is_available() {
    5893            0 :                 if migrate_req.migration_config.override_scheduler {
    5894              :                     // Warn but proceed: the caller may intend to manually adjust the placement of
    5895              :                     // a shard even if the node is down, e.g. if intervening during an incident.
    5896            0 :                     tracing::warn!("Forcibly migrating to unavailable node {node}");
    5897              :                 } else {
    5898            0 :                     tracing::warn!("Node {node} is unavailable, refusing migration");
    5899            0 :                     return Err(ApiError::PreconditionFailed(
    5900            0 :                         format!("Node {node} is unavailable").into_boxed_str(),
    5901            0 :                     ));
    5902              :                 }
    5903            0 :             }
    5904              : 
    5905              :             // Calculate the ScheduleContext for this tenant
    5906            0 :             let mut schedule_context = ScheduleContext::default();
    5907            0 :             for (_shard_id, shard) in
    5908            0 :                 tenants.range(TenantShardId::tenant_range(tenant_shard_id.tenant_id))
    5909            0 :             {
    5910            0 :                 schedule_context.avoid(&shard.intent.all_pageservers());
    5911            0 :             }
    5912              : 
    5913              :             // Look up the specific shard we will migrate
    5914            0 :             let Some(shard) = tenants.get_mut(&tenant_shard_id) else {
    5915            0 :                 return Err(ApiError::NotFound(
    5916            0 :                     anyhow::anyhow!("Tenant shard not found").into(),
    5917            0 :                 ));
    5918              :             };
    5919              : 
    5920              :             // Migration to a node with unfavorable scheduling score requires a force flag, because it might just
    5921              :             // be migrated back by the optimiser.
    5922            0 :             if let Some(better_node) = shard.find_better_location::<AttachedShardTag>(
    5923            0 :                 scheduler,
    5924            0 :                 &schedule_context,
    5925            0 :                 migrate_req.node_id,
    5926            0 :                 &[],
    5927            0 :             ) {
    5928            0 :                 if !migrate_req.migration_config.override_scheduler {
    5929            0 :                     return Err(ApiError::PreconditionFailed(
    5930            0 :                         "Migration to a worse-scoring node".into(),
    5931            0 :                     ));
    5932              :                 } else {
    5933            0 :                     tracing::info!(
    5934            0 :                         "Migrating to a worse-scoring node {} (optimiser would prefer {better_node})",
    5935              :                         migrate_req.node_id
    5936              :                     );
    5937              :                 }
    5938            0 :             }
    5939              : 
    5940            0 :             if let Some(origin_node_id) = migrate_req.origin_node_id {
    5941            0 :                 if shard.intent.get_attached() != &Some(origin_node_id) {
    5942            0 :                     return Err(ApiError::PreconditionFailed(
    5943            0 :                         format!(
    5944            0 :                             "Migration expected to originate from {} but shard is on {:?}",
    5945            0 :                             origin_node_id,
    5946            0 :                             shard.intent.get_attached()
    5947            0 :                         )
    5948            0 :                         .into(),
    5949            0 :                     ));
    5950            0 :                 }
    5951            0 :             }
    5952              : 
    5953            0 :             if shard.intent.get_attached() == &Some(migrate_req.node_id) {
    5954              :                 // No-op case: we will still proceed to wait for reconciliation in case it is
    5955              :                 // incomplete from an earlier update to the intent.
    5956            0 :                 tracing::info!("Migrating: intent is unchanged {:?}", shard.intent);
    5957              : 
    5958              :                 // An instruction to migrate to the currently attached node should
    5959              :                 // cancel any pending graceful migration
    5960            0 :                 shard.set_preferred_node(None);
    5961            0 : 
    5962            0 :                 MigrationOutcome::Reconcile(self.maybe_configured_reconcile_shard(
    5963            0 :                     shard,
    5964            0 :                     nodes,
    5965            0 :                     (&migrate_req.migration_config).into(),
    5966            0 :                 ))
    5967            0 :             } else if migrate_req.migration_config.prewarm {
    5968            0 :                 MigrationOutcome::Optimization(self.tenant_shard_migrate_with_prewarm(
    5969            0 :                     &migrate_req,
    5970            0 :                     shard,
    5971            0 :                     scheduler,
    5972            0 :                     schedule_context,
    5973            0 :                 )?)
    5974              :             } else {
    5975            0 :                 MigrationOutcome::Reconcile(self.tenant_shard_migrate_immediate(
    5976            0 :                     &migrate_req,
    5977            0 :                     nodes,
    5978            0 :                     shard,
    5979            0 :                     scheduler,
    5980            0 :                 )?)
    5981              :             }
    5982              :         };
    5983              : 
    5984              :         // We may need to validate + apply an optimisation, or we may need to just retrive a reconcile waiter
    5985            0 :         let waiter = match outcome {
    5986            0 :             MigrationOutcome::Optimization(Some(optimization)) => {
    5987              :                 // Validate and apply the optimization -- this would happen anyway in background reconcile loop, but
    5988              :                 // we might as well do it more promptly as this is a direct external request.
    5989            0 :                 let mut validated = self
    5990            0 :                     .optimize_all_validate(vec![(tenant_shard_id, optimization)])
    5991            0 :                     .await;
    5992            0 :                 if let Some((_shard_id, optimization)) = validated.pop() {
    5993            0 :                     let mut locked = self.inner.write().unwrap();
    5994            0 :                     let (nodes, tenants, scheduler) = locked.parts_mut();
    5995            0 :                     let Some(shard) = tenants.get_mut(&tenant_shard_id) else {
    5996              :                         // Rare but possible: tenant is removed between generating optimisation and validating it.
    5997            0 :                         return Err(ApiError::NotFound(
    5998            0 :                             anyhow::anyhow!("Tenant shard not found").into(),
    5999            0 :                         ));
    6000              :                     };
    6001              : 
    6002            0 :                     if !shard.apply_optimization(scheduler, optimization) {
    6003              :                         // This can happen but is unusual enough to warn on: something else changed in the shard that made the optimisation stale
    6004              :                         // and therefore not applied.
    6005            0 :                         tracing::warn!(
    6006            0 :                             "Schedule optimisation generated during graceful migration was not applied, shard changed?"
    6007              :                         );
    6008            0 :                     }
    6009            0 :                     self.maybe_configured_reconcile_shard(
    6010            0 :                         shard,
    6011            0 :                         nodes,
    6012            0 :                         (&migrate_req.migration_config).into(),
    6013            0 :                     )
    6014              :                 } else {
    6015            0 :                     None
    6016              :                 }
    6017              :             }
    6018            0 :             MigrationOutcome::Optimization(None) => None,
    6019            0 :             MigrationOutcome::Reconcile(waiter) => waiter,
    6020              :         };
    6021              : 
    6022              :         // Finally, wait for any reconcile we started to complete.  In the case of immediate-mode migrations to cold
    6023              :         // locations, this has a good chance of timing out.
    6024            0 :         if let Some(waiter) = waiter {
    6025            0 :             waiter.wait_timeout(RECONCILE_TIMEOUT).await?;
    6026              :         } else {
    6027            0 :             tracing::info!("Migration is a no-op");
    6028              :         }
    6029              : 
    6030            0 :         Ok(TenantShardMigrateResponse {})
    6031            0 :     }
    6032              : 
    6033            0 :     pub(crate) async fn tenant_shard_migrate_secondary(
    6034            0 :         &self,
    6035            0 :         tenant_shard_id: TenantShardId,
    6036            0 :         migrate_req: TenantShardMigrateRequest,
    6037            0 :     ) -> Result<TenantShardMigrateResponse, ApiError> {
    6038            0 :         let waiter = {
    6039            0 :             let mut locked = self.inner.write().unwrap();
    6040            0 :             let (nodes, tenants, scheduler) = locked.parts_mut();
    6041              : 
    6042            0 :             let Some(node) = nodes.get(&migrate_req.node_id) else {
    6043            0 :                 return Err(ApiError::BadRequest(anyhow::anyhow!(
    6044            0 :                     "Node {} not found",
    6045            0 :                     migrate_req.node_id
    6046            0 :                 )));
    6047              :             };
    6048              : 
    6049            0 :             if !node.is_available() {
    6050              :                 // Warn but proceed: the caller may intend to manually adjust the placement of
    6051              :                 // a shard even if the node is down, e.g. if intervening during an incident.
    6052            0 :                 tracing::warn!("Migrating to unavailable node {node}");
    6053            0 :             }
    6054              : 
    6055            0 :             let Some(shard) = tenants.get_mut(&tenant_shard_id) else {
    6056            0 :                 return Err(ApiError::NotFound(
    6057            0 :                     anyhow::anyhow!("Tenant shard not found").into(),
    6058            0 :                 ));
    6059              :             };
    6060              : 
    6061            0 :             if shard.intent.get_secondary().len() == 1
    6062            0 :                 && shard.intent.get_secondary()[0] == migrate_req.node_id
    6063              :             {
    6064            0 :                 tracing::info!(
    6065            0 :                     "Migrating secondary to {node}: intent is unchanged {:?}",
    6066              :                     shard.intent
    6067              :                 );
    6068            0 :             } else if shard.intent.get_attached() == &Some(migrate_req.node_id) {
    6069            0 :                 tracing::info!(
    6070            0 :                     "Migrating secondary to {node}: already attached where we were asked to create a secondary"
    6071              :                 );
    6072              :             } else {
    6073            0 :                 let old_secondaries = shard.intent.get_secondary().clone();
    6074            0 :                 for secondary in old_secondaries {
    6075            0 :                     shard.intent.remove_secondary(scheduler, secondary);
    6076            0 :                 }
    6077              : 
    6078            0 :                 shard.intent.push_secondary(scheduler, migrate_req.node_id);
    6079            0 :                 shard.sequence = shard.sequence.next();
    6080            0 :                 tracing::info!(
    6081            0 :                     "Migrating secondary to {node}: new intent {:?}",
    6082              :                     shard.intent
    6083              :                 );
    6084              :             }
    6085              : 
    6086            0 :             self.maybe_reconcile_shard(shard, nodes, ReconcilerPriority::High)
    6087              :         };
    6088              : 
    6089            0 :         if let Some(waiter) = waiter {
    6090            0 :             waiter.wait_timeout(RECONCILE_TIMEOUT).await?;
    6091              :         } else {
    6092            0 :             tracing::info!("Migration is a no-op");
    6093              :         }
    6094              : 
    6095            0 :         Ok(TenantShardMigrateResponse {})
    6096            0 :     }
    6097              : 
    6098              :     /// 'cancel' in this context means cancel any ongoing reconcile
    6099            0 :     pub(crate) async fn tenant_shard_cancel_reconcile(
    6100            0 :         &self,
    6101            0 :         tenant_shard_id: TenantShardId,
    6102            0 :     ) -> Result<(), ApiError> {
    6103              :         // Take state lock and fire the cancellation token, after which we drop lock and wait for any ongoing reconcile to complete
    6104            0 :         let waiter = {
    6105            0 :             let locked = self.inner.write().unwrap();
    6106            0 :             let Some(shard) = locked.tenants.get(&tenant_shard_id) else {
    6107            0 :                 return Err(ApiError::NotFound(
    6108            0 :                     anyhow::anyhow!("Tenant shard not found").into(),
    6109            0 :                 ));
    6110              :             };
    6111              : 
    6112            0 :             let waiter = shard.get_waiter();
    6113            0 :             match waiter {
    6114              :                 None => {
    6115            0 :                     tracing::info!("Shard does not have an ongoing Reconciler");
    6116            0 :                     return Ok(());
    6117              :                 }
    6118            0 :                 Some(waiter) => {
    6119            0 :                     tracing::info!("Cancelling Reconciler");
    6120            0 :                     shard.cancel_reconciler();
    6121            0 :                     waiter
    6122            0 :                 }
    6123            0 :             }
    6124            0 :         };
    6125            0 : 
    6126            0 :         // Cancellation should be prompt.  If this fails we have still done our job of firing the
    6127            0 :         // cancellation token, but by returning an ApiError we will indicate to the caller that
    6128            0 :         // the Reconciler is misbehaving and not respecting the cancellation token
    6129            0 :         self.await_waiters(vec![waiter], SHORT_RECONCILE_TIMEOUT)
    6130            0 :             .await?;
    6131              : 
    6132            0 :         Ok(())
    6133            0 :     }
    6134              : 
    6135              :     /// This is for debug/support only: we simply drop all state for a tenant, without
    6136              :     /// detaching or deleting it on pageservers.
    6137            0 :     pub(crate) async fn tenant_drop(&self, tenant_id: TenantId) -> Result<(), ApiError> {
    6138            0 :         self.persistence.delete_tenant(tenant_id).await?;
    6139              : 
    6140            0 :         let mut locked = self.inner.write().unwrap();
    6141            0 :         let (_nodes, tenants, scheduler) = locked.parts_mut();
    6142            0 :         let mut shards = Vec::new();
    6143            0 :         for (tenant_shard_id, _) in tenants.range(TenantShardId::tenant_range(tenant_id)) {
    6144            0 :             shards.push(*tenant_shard_id);
    6145            0 :         }
    6146              : 
    6147            0 :         for shard_id in shards {
    6148            0 :             if let Some(mut shard) = tenants.remove(&shard_id) {
    6149            0 :                 shard.intent.clear(scheduler);
    6150            0 :             }
    6151              :         }
    6152              : 
    6153            0 :         Ok(())
    6154            0 :     }
    6155              : 
    6156              :     /// This is for debug/support only: assuming tenant data is already present in S3, we "create" a
    6157              :     /// tenant with a very high generation number so that it will see the existing data.
    6158            0 :     pub(crate) async fn tenant_import(
    6159            0 :         &self,
    6160            0 :         tenant_id: TenantId,
    6161            0 :     ) -> Result<TenantCreateResponse, ApiError> {
    6162            0 :         // Pick an arbitrary available pageserver to use for scanning the tenant in remote storage
    6163            0 :         let maybe_node = {
    6164            0 :             self.inner
    6165            0 :                 .read()
    6166            0 :                 .unwrap()
    6167            0 :                 .nodes
    6168            0 :                 .values()
    6169            0 :                 .find(|n| n.is_available())
    6170            0 :                 .cloned()
    6171              :         };
    6172            0 :         let Some(node) = maybe_node else {
    6173            0 :             return Err(ApiError::BadRequest(anyhow::anyhow!("No nodes available")));
    6174              :         };
    6175              : 
    6176            0 :         let client = PageserverClient::new(
    6177            0 :             node.get_id(),
    6178            0 :             node.base_url(),
    6179            0 :             self.config.pageserver_jwt_token.as_deref(),
    6180            0 :             self.config.ssl_ca_cert.clone(),
    6181            0 :         )
    6182            0 :         .map_err(|e| passthrough_api_error(&node, e))?;
    6183              : 
    6184            0 :         let scan_result = client
    6185            0 :             .tenant_scan_remote_storage(tenant_id)
    6186            0 :             .await
    6187            0 :             .map_err(|e| passthrough_api_error(&node, e))?;
    6188              : 
    6189              :         // A post-split tenant may contain a mixture of shard counts in remote storage: pick the highest count.
    6190            0 :         let Some(shard_count) = scan_result
    6191            0 :             .shards
    6192            0 :             .iter()
    6193            0 :             .map(|s| s.tenant_shard_id.shard_count)
    6194            0 :             .max()
    6195              :         else {
    6196            0 :             return Err(ApiError::NotFound(
    6197            0 :                 anyhow::anyhow!("No shards found").into(),
    6198            0 :             ));
    6199              :         };
    6200              : 
    6201              :         // Ideally we would set each newly imported shard's generation independently, but for correctness it is sufficient
    6202              :         // to
    6203            0 :         let generation = scan_result
    6204            0 :             .shards
    6205            0 :             .iter()
    6206            0 :             .map(|s| s.generation)
    6207            0 :             .max()
    6208            0 :             .expect("We already validated >0 shards");
    6209            0 : 
    6210            0 :         // FIXME: we have no way to recover the shard stripe size from contents of remote storage: this will
    6211            0 :         // only work if they were using the default stripe size.
    6212            0 :         let stripe_size = ShardParameters::DEFAULT_STRIPE_SIZE;
    6213              : 
    6214            0 :         let (response, waiters) = self
    6215            0 :             .do_tenant_create(TenantCreateRequest {
    6216            0 :                 new_tenant_id: TenantShardId::unsharded(tenant_id),
    6217            0 :                 generation,
    6218            0 : 
    6219            0 :                 shard_parameters: ShardParameters {
    6220            0 :                     count: shard_count,
    6221            0 :                     stripe_size,
    6222            0 :                 },
    6223            0 :                 placement_policy: Some(PlacementPolicy::Attached(0)), // No secondaries, for convenient debug/hacking
    6224            0 :                 config: TenantConfig::default(),
    6225            0 :             })
    6226            0 :             .await?;
    6227              : 
    6228            0 :         if let Err(e) = self.await_waiters(waiters, SHORT_RECONCILE_TIMEOUT).await {
    6229              :             // Since this is a debug/support operation, all kinds of weird issues are possible (e.g. this
    6230              :             // tenant doesn't exist in the control plane), so don't fail the request if it can't fully
    6231              :             // reconcile, as reconciliation includes notifying compute.
    6232            0 :             tracing::warn!(%tenant_id, "Reconcile not done yet while importing tenant ({e})");
    6233            0 :         }
    6234              : 
    6235            0 :         Ok(response)
    6236            0 :     }
    6237              : 
    6238              :     /// For debug/support: a full JSON dump of TenantShards.  Returns a response so that
    6239              :     /// we don't have to make TenantShard clonable in the return path.
    6240            0 :     pub(crate) fn tenants_dump(&self) -> Result<hyper::Response<hyper::Body>, ApiError> {
    6241            0 :         let serialized = {
    6242            0 :             let locked = self.inner.read().unwrap();
    6243            0 :             let result = locked.tenants.values().collect::<Vec<_>>();
    6244            0 :             serde_json::to_string(&result).map_err(|e| ApiError::InternalServerError(e.into()))?
    6245              :         };
    6246              : 
    6247            0 :         hyper::Response::builder()
    6248            0 :             .status(hyper::StatusCode::OK)
    6249            0 :             .header(hyper::header::CONTENT_TYPE, "application/json")
    6250            0 :             .body(hyper::Body::from(serialized))
    6251            0 :             .map_err(|e| ApiError::InternalServerError(e.into()))
    6252            0 :     }
    6253              : 
    6254              :     /// Check the consistency of in-memory state vs. persistent state, and check that the
    6255              :     /// scheduler's statistics are up to date.
    6256              :     ///
    6257              :     /// These consistency checks expect an **idle** system.  If changes are going on while
    6258              :     /// we run, then we can falsely indicate a consistency issue.  This is sufficient for end-of-test
    6259              :     /// checks, but not suitable for running continuously in the background in the field.
    6260            0 :     pub(crate) async fn consistency_check(&self) -> Result<(), ApiError> {
    6261            0 :         let (mut expect_nodes, mut expect_shards) = {
    6262            0 :             let locked = self.inner.read().unwrap();
    6263            0 : 
    6264            0 :             locked
    6265            0 :                 .scheduler
    6266            0 :                 .consistency_check(locked.nodes.values(), locked.tenants.values())
    6267            0 :                 .context("Scheduler checks")
    6268            0 :                 .map_err(ApiError::InternalServerError)?;
    6269              : 
    6270            0 :             let expect_nodes = locked
    6271            0 :                 .nodes
    6272            0 :                 .values()
    6273            0 :                 .map(|n| n.to_persistent())
    6274            0 :                 .collect::<Vec<_>>();
    6275            0 : 
    6276            0 :             let expect_shards = locked
    6277            0 :                 .tenants
    6278            0 :                 .values()
    6279            0 :                 .map(|t| t.to_persistent())
    6280            0 :                 .collect::<Vec<_>>();
    6281              : 
    6282              :             // This method can only validate the state of an idle system: if a reconcile is in
    6283              :             // progress, fail out early to avoid giving false errors on state that won't match
    6284              :             // between database and memory under a ReconcileResult is processed.
    6285            0 :             for t in locked.tenants.values() {
    6286            0 :                 if t.reconciler.is_some() {
    6287            0 :                     return Err(ApiError::InternalServerError(anyhow::anyhow!(
    6288            0 :                         "Shard {} reconciliation in progress",
    6289            0 :                         t.tenant_shard_id
    6290            0 :                     )));
    6291            0 :                 }
    6292              :             }
    6293              : 
    6294            0 :             (expect_nodes, expect_shards)
    6295              :         };
    6296              : 
    6297            0 :         let mut nodes = self.persistence.list_nodes().await?;
    6298            0 :         expect_nodes.sort_by_key(|n| n.node_id);
    6299            0 :         nodes.sort_by_key(|n| n.node_id);
    6300              : 
    6301              :         // Errors relating to nodes are deferred so that we don't skip the shard checks below if we have a node error
    6302            0 :         let node_result = if nodes != expect_nodes {
    6303            0 :             tracing::error!("Consistency check failed on nodes.");
    6304            0 :             tracing::error!(
    6305            0 :                 "Nodes in memory: {}",
    6306            0 :                 serde_json::to_string(&expect_nodes)
    6307            0 :                     .map_err(|e| ApiError::InternalServerError(e.into()))?
    6308              :             );
    6309            0 :             tracing::error!(
    6310            0 :                 "Nodes in database: {}",
    6311            0 :                 serde_json::to_string(&nodes)
    6312            0 :                     .map_err(|e| ApiError::InternalServerError(e.into()))?
    6313              :             );
    6314            0 :             Err(ApiError::InternalServerError(anyhow::anyhow!(
    6315            0 :                 "Node consistency failure"
    6316            0 :             )))
    6317              :         } else {
    6318            0 :             Ok(())
    6319              :         };
    6320              : 
    6321            0 :         let mut persistent_shards = self.persistence.load_active_tenant_shards().await?;
    6322            0 :         persistent_shards
    6323            0 :             .sort_by_key(|tsp| (tsp.tenant_id.clone(), tsp.shard_number, tsp.shard_count));
    6324            0 : 
    6325            0 :         expect_shards.sort_by_key(|tsp| (tsp.tenant_id.clone(), tsp.shard_number, tsp.shard_count));
    6326              : 
    6327              :         // Because JSON contents of persistent tenants might disagree with the fields in current `TenantConfig`
    6328              :         // definition, we will do an encode/decode cycle to ensure any legacy fields are dropped and any new
    6329              :         // fields are added, before doing a comparison.
    6330            0 :         for tsp in &mut persistent_shards {
    6331            0 :             let config: TenantConfig = serde_json::from_str(&tsp.config)
    6332            0 :                 .map_err(|e| ApiError::InternalServerError(e.into()))?;
    6333            0 :             tsp.config = serde_json::to_string(&config).expect("Encoding config is infallible");
    6334              :         }
    6335              : 
    6336            0 :         if persistent_shards != expect_shards {
    6337            0 :             tracing::error!("Consistency check failed on shards.");
    6338              : 
    6339            0 :             tracing::error!(
    6340            0 :                 "Shards in memory: {}",
    6341            0 :                 serde_json::to_string(&expect_shards)
    6342            0 :                     .map_err(|e| ApiError::InternalServerError(e.into()))?
    6343              :             );
    6344            0 :             tracing::error!(
    6345            0 :                 "Shards in database: {}",
    6346            0 :                 serde_json::to_string(&persistent_shards)
    6347            0 :                     .map_err(|e| ApiError::InternalServerError(e.into()))?
    6348              :             );
    6349              : 
    6350              :             // The total dump log lines above are useful in testing but in the field grafana will
    6351              :             // usually just drop them because they're so large. So we also do some explicit logging
    6352              :             // of just the diffs.
    6353            0 :             let persistent_shards = persistent_shards
    6354            0 :                 .into_iter()
    6355            0 :                 .map(|tsp| (tsp.get_tenant_shard_id().unwrap(), tsp))
    6356            0 :                 .collect::<HashMap<_, _>>();
    6357            0 :             let expect_shards = expect_shards
    6358            0 :                 .into_iter()
    6359            0 :                 .map(|tsp| (tsp.get_tenant_shard_id().unwrap(), tsp))
    6360            0 :                 .collect::<HashMap<_, _>>();
    6361            0 :             for (tenant_shard_id, persistent_tsp) in &persistent_shards {
    6362            0 :                 match expect_shards.get(tenant_shard_id) {
    6363              :                     None => {
    6364            0 :                         tracing::error!(
    6365            0 :                             "Shard {} found in database but not in memory",
    6366              :                             tenant_shard_id
    6367              :                         );
    6368              :                     }
    6369            0 :                     Some(expect_tsp) => {
    6370            0 :                         if expect_tsp != persistent_tsp {
    6371            0 :                             tracing::error!(
    6372            0 :                                 "Shard {} is inconsistent.  In memory: {}, database has: {}",
    6373            0 :                                 tenant_shard_id,
    6374            0 :                                 serde_json::to_string(expect_tsp).unwrap(),
    6375            0 :                                 serde_json::to_string(&persistent_tsp).unwrap()
    6376              :                             );
    6377            0 :                         }
    6378              :                     }
    6379              :                 }
    6380              :             }
    6381              : 
    6382              :             // Having already logged any differences, log any shards that simply aren't present in the database
    6383            0 :             for (tenant_shard_id, memory_tsp) in &expect_shards {
    6384            0 :                 if !persistent_shards.contains_key(tenant_shard_id) {
    6385            0 :                     tracing::error!(
    6386            0 :                         "Shard {} found in memory but not in database: {}",
    6387            0 :                         tenant_shard_id,
    6388            0 :                         serde_json::to_string(memory_tsp)
    6389            0 :                             .map_err(|e| ApiError::InternalServerError(e.into()))?
    6390              :                     );
    6391            0 :                 }
    6392              :             }
    6393              : 
    6394            0 :             return Err(ApiError::InternalServerError(anyhow::anyhow!(
    6395            0 :                 "Shard consistency failure"
    6396            0 :             )));
    6397            0 :         }
    6398            0 : 
    6399            0 :         node_result
    6400            0 :     }
    6401              : 
    6402              :     /// For debug/support: a JSON dump of the [`Scheduler`].  Returns a response so that
    6403              :     /// we don't have to make TenantShard clonable in the return path.
    6404            0 :     pub(crate) fn scheduler_dump(&self) -> Result<hyper::Response<hyper::Body>, ApiError> {
    6405            0 :         let serialized = {
    6406            0 :             let locked = self.inner.read().unwrap();
    6407            0 :             serde_json::to_string(&locked.scheduler)
    6408            0 :                 .map_err(|e| ApiError::InternalServerError(e.into()))?
    6409              :         };
    6410              : 
    6411            0 :         hyper::Response::builder()
    6412            0 :             .status(hyper::StatusCode::OK)
    6413            0 :             .header(hyper::header::CONTENT_TYPE, "application/json")
    6414            0 :             .body(hyper::Body::from(serialized))
    6415            0 :             .map_err(|e| ApiError::InternalServerError(e.into()))
    6416            0 :     }
    6417              : 
    6418              :     /// This is for debug/support only: we simply drop all state for a tenant, without
    6419              :     /// detaching or deleting it on pageservers.  We do not try and re-schedule any
    6420              :     /// tenants that were on this node.
    6421            0 :     pub(crate) async fn node_drop(&self, node_id: NodeId) -> Result<(), ApiError> {
    6422            0 :         self.persistence.delete_node(node_id).await?;
    6423              : 
    6424            0 :         let mut locked = self.inner.write().unwrap();
    6425              : 
    6426            0 :         for shard in locked.tenants.values_mut() {
    6427            0 :             shard.deref_node(node_id);
    6428            0 :             shard.observed.locations.remove(&node_id);
    6429            0 :         }
    6430              : 
    6431            0 :         let mut nodes = (*locked.nodes).clone();
    6432            0 :         nodes.remove(&node_id);
    6433            0 :         locked.nodes = Arc::new(nodes);
    6434            0 :         metrics::METRICS_REGISTRY
    6435            0 :             .metrics_group
    6436            0 :             .storage_controller_pageserver_nodes
    6437            0 :             .set(locked.nodes.len() as i64);
    6438            0 : 
    6439            0 :         locked.scheduler.node_remove(node_id);
    6440            0 : 
    6441            0 :         Ok(())
    6442            0 :     }
    6443              : 
    6444              :     /// If a node has any work on it, it will be rescheduled: this is "clean" in the sense
    6445              :     /// that we don't leave any bad state behind in the storage controller, but unclean
    6446              :     /// in the sense that we are not carefully draining the node.
    6447            0 :     pub(crate) async fn node_delete(&self, node_id: NodeId) -> Result<(), ApiError> {
    6448            0 :         let _node_lock =
    6449            0 :             trace_exclusive_lock(&self.node_op_locks, node_id, NodeOperations::Delete).await;
    6450              : 
    6451              :         // 1. Atomically update in-memory state:
    6452              :         //    - set the scheduling state to Pause to make subsequent scheduling ops skip it
    6453              :         //    - update shards' intents to exclude the node, and reschedule any shards whose intents we modified.
    6454              :         //    - drop the node from the main nodes map, so that when running reconciles complete they do not
    6455              :         //      re-insert references to this node into the ObservedState of shards
    6456              :         //    - drop the node from the scheduler
    6457              :         {
    6458            0 :             let mut locked = self.inner.write().unwrap();
    6459            0 :             let (nodes, tenants, scheduler) = locked.parts_mut();
    6460            0 : 
    6461            0 :             {
    6462            0 :                 let mut nodes_mut = (*nodes).deref().clone();
    6463            0 :                 match nodes_mut.get_mut(&node_id) {
    6464            0 :                     Some(node) => {
    6465            0 :                         // We do not bother setting this in the database, because we're about to delete the row anyway, and
    6466            0 :                         // if we crash it would not be desirable to leave the node paused after a restart.
    6467            0 :                         node.set_scheduling(NodeSchedulingPolicy::Pause);
    6468            0 :                     }
    6469              :                     None => {
    6470            0 :                         tracing::info!(
    6471            0 :                             "Node not found: presuming this is a retry and returning success"
    6472              :                         );
    6473            0 :                         return Ok(());
    6474              :                     }
    6475              :                 }
    6476              : 
    6477            0 :                 *nodes = Arc::new(nodes_mut);
    6478              :             }
    6479              : 
    6480            0 :             for (_tenant_id, mut schedule_context, shards) in
    6481            0 :                 TenantShardContextIterator::new(tenants, ScheduleMode::Normal)
    6482              :             {
    6483            0 :                 for shard in shards {
    6484            0 :                     if shard.deref_node(node_id) {
    6485            0 :                         if let Err(e) = shard.schedule(scheduler, &mut schedule_context) {
    6486              :                             // TODO: implement force flag to remove a node even if we can't reschedule
    6487              :                             // a tenant
    6488            0 :                             tracing::error!(
    6489            0 :                                 "Refusing to delete node, shard {} can't be rescheduled: {e}",
    6490              :                                 shard.tenant_shard_id
    6491              :                             );
    6492            0 :                             return Err(e.into());
    6493              :                         } else {
    6494            0 :                             tracing::info!(
    6495            0 :                                 "Rescheduled shard {} away from node during deletion",
    6496              :                                 shard.tenant_shard_id
    6497              :                             )
    6498              :                         }
    6499              : 
    6500            0 :                         self.maybe_reconcile_shard(shard, nodes, ReconcilerPriority::Normal);
    6501            0 :                     }
    6502              : 
    6503              :                     // Here we remove an existing observed location for the node we're removing, and it will
    6504              :                     // not be re-added by a reconciler's completion because we filter out removed nodes in
    6505              :                     // process_result.
    6506              :                     //
    6507              :                     // Note that we update the shard's observed state _after_ calling maybe_reconcile_shard: that
    6508              :                     // means any reconciles we spawned will know about the node we're deleting, enabling them
    6509              :                     // to do live migrations if it's still online.
    6510            0 :                     shard.observed.locations.remove(&node_id);
    6511              :                 }
    6512              :             }
    6513              : 
    6514            0 :             scheduler.node_remove(node_id);
    6515            0 : 
    6516            0 :             {
    6517            0 :                 let mut nodes_mut = (**nodes).clone();
    6518            0 :                 if let Some(mut removed_node) = nodes_mut.remove(&node_id) {
    6519            0 :                     // Ensure that any reconciler holding an Arc<> to this node will
    6520            0 :                     // drop out when trying to RPC to it (setting Offline state sets the
    6521            0 :                     // cancellation token on the Node object).
    6522            0 :                     removed_node.set_availability(NodeAvailability::Offline);
    6523            0 :                 }
    6524            0 :                 *nodes = Arc::new(nodes_mut);
    6525            0 :                 metrics::METRICS_REGISTRY
    6526            0 :                     .metrics_group
    6527            0 :                     .storage_controller_pageserver_nodes
    6528            0 :                     .set(nodes.len() as i64);
    6529            0 :             }
    6530            0 :         }
    6531            0 : 
    6532            0 :         // Note: some `generation_pageserver` columns on tenant shards in the database may still refer to
    6533            0 :         // the removed node, as this column means "The pageserver to which this generation was issued", and
    6534            0 :         // their generations won't get updated until the reconcilers moving them away from this node complete.
    6535            0 :         // That is safe because in Service::spawn we only use generation_pageserver if it refers to a node
    6536            0 :         // that exists.
    6537            0 : 
    6538            0 :         // 2. Actually delete the node from the database and from in-memory state
    6539            0 :         tracing::info!("Deleting node from database");
    6540            0 :         self.persistence.delete_node(node_id).await?;
    6541              : 
    6542            0 :         Ok(())
    6543            0 :     }
    6544              : 
    6545            0 :     pub(crate) async fn node_list(&self) -> Result<Vec<Node>, ApiError> {
    6546            0 :         let nodes = {
    6547            0 :             self.inner
    6548            0 :                 .read()
    6549            0 :                 .unwrap()
    6550            0 :                 .nodes
    6551            0 :                 .values()
    6552            0 :                 .cloned()
    6553            0 :                 .collect::<Vec<_>>()
    6554            0 :         };
    6555            0 : 
    6556            0 :         Ok(nodes)
    6557            0 :     }
    6558              : 
    6559            0 :     pub(crate) async fn get_node(&self, node_id: NodeId) -> Result<Node, ApiError> {
    6560            0 :         self.inner
    6561            0 :             .read()
    6562            0 :             .unwrap()
    6563            0 :             .nodes
    6564            0 :             .get(&node_id)
    6565            0 :             .cloned()
    6566            0 :             .ok_or(ApiError::NotFound(
    6567            0 :                 format!("Node {node_id} not registered").into(),
    6568            0 :             ))
    6569            0 :     }
    6570              : 
    6571            0 :     pub(crate) async fn get_node_shards(
    6572            0 :         &self,
    6573            0 :         node_id: NodeId,
    6574            0 :     ) -> Result<NodeShardResponse, ApiError> {
    6575            0 :         let locked = self.inner.read().unwrap();
    6576            0 :         let mut shards = Vec::new();
    6577            0 :         for (tid, tenant) in locked.tenants.iter() {
    6578            0 :             let is_intended_secondary = match (
    6579            0 :                 tenant.intent.get_attached() == &Some(node_id),
    6580            0 :                 tenant.intent.get_secondary().contains(&node_id),
    6581            0 :             ) {
    6582              :                 (true, true) => {
    6583            0 :                     return Err(ApiError::InternalServerError(anyhow::anyhow!(
    6584            0 :                         "{} attached as primary+secondary on the same node",
    6585            0 :                         tid
    6586            0 :                     )));
    6587              :                 }
    6588            0 :                 (true, false) => Some(false),
    6589            0 :                 (false, true) => Some(true),
    6590            0 :                 (false, false) => None,
    6591              :             };
    6592            0 :             let is_observed_secondary = if let Some(ObservedStateLocation { conf: Some(conf) }) =
    6593            0 :                 tenant.observed.locations.get(&node_id)
    6594              :             {
    6595            0 :                 Some(conf.secondary_conf.is_some())
    6596              :             } else {
    6597            0 :                 None
    6598              :             };
    6599            0 :             if is_intended_secondary.is_some() || is_observed_secondary.is_some() {
    6600            0 :                 shards.push(NodeShard {
    6601            0 :                     tenant_shard_id: *tid,
    6602            0 :                     is_intended_secondary,
    6603            0 :                     is_observed_secondary,
    6604            0 :                 });
    6605            0 :             }
    6606              :         }
    6607            0 :         Ok(NodeShardResponse { node_id, shards })
    6608            0 :     }
    6609              : 
    6610            0 :     pub(crate) async fn get_leader(&self) -> DatabaseResult<Option<ControllerPersistence>> {
    6611            0 :         self.persistence.get_leader().await
    6612            0 :     }
    6613              : 
    6614            0 :     pub(crate) async fn node_register(
    6615            0 :         &self,
    6616            0 :         register_req: NodeRegisterRequest,
    6617            0 :     ) -> Result<(), ApiError> {
    6618            0 :         let _node_lock = trace_exclusive_lock(
    6619            0 :             &self.node_op_locks,
    6620            0 :             register_req.node_id,
    6621            0 :             NodeOperations::Register,
    6622            0 :         )
    6623            0 :         .await;
    6624              : 
    6625              :         #[derive(PartialEq)]
    6626              :         enum RegistrationStatus {
    6627              :             UpToDate,
    6628              :             NeedUpdate,
    6629              :             Mismatched,
    6630              :             New,
    6631              :         }
    6632              : 
    6633            0 :         let registration_status = {
    6634            0 :             let locked = self.inner.read().unwrap();
    6635            0 :             if let Some(node) = locked.nodes.get(&register_req.node_id) {
    6636            0 :                 if node.registration_match(&register_req) {
    6637            0 :                     if node.need_update(&register_req) {
    6638            0 :                         RegistrationStatus::NeedUpdate
    6639              :                     } else {
    6640            0 :                         RegistrationStatus::UpToDate
    6641              :                     }
    6642              :                 } else {
    6643            0 :                     RegistrationStatus::Mismatched
    6644              :                 }
    6645              :             } else {
    6646            0 :                 RegistrationStatus::New
    6647              :             }
    6648              :         };
    6649              : 
    6650            0 :         match registration_status {
    6651              :             RegistrationStatus::UpToDate => {
    6652            0 :                 tracing::info!(
    6653            0 :                     "Node {} re-registered with matching address and is up to date",
    6654              :                     register_req.node_id
    6655              :                 );
    6656              : 
    6657            0 :                 return Ok(());
    6658              :             }
    6659              :             RegistrationStatus::Mismatched => {
    6660              :                 // TODO: decide if we want to allow modifying node addresses without removing and re-adding
    6661              :                 // the node.  Safest/simplest thing is to refuse it, and usually we deploy with
    6662              :                 // a fixed address through the lifetime of a node.
    6663            0 :                 tracing::warn!(
    6664            0 :                     "Node {} tried to register with different address",
    6665              :                     register_req.node_id
    6666              :                 );
    6667            0 :                 return Err(ApiError::Conflict(
    6668            0 :                     "Node is already registered with different address".to_string(),
    6669            0 :                 ));
    6670              :             }
    6671            0 :             RegistrationStatus::New | RegistrationStatus::NeedUpdate => {
    6672            0 :                 // fallthrough
    6673            0 :             }
    6674            0 :         }
    6675            0 : 
    6676            0 :         // We do not require that a node is actually online when registered (it will start life
    6677            0 :         // with it's  availability set to Offline), but we _do_ require that its DNS record exists. We're
    6678            0 :         // therefore not immune to asymmetric L3 connectivity issues, but we are protected against nodes
    6679            0 :         // that register themselves with a broken DNS config.  We check only the HTTP hostname, because
    6680            0 :         // the postgres hostname might only be resolvable to clients (e.g. if we're on a different VPC than clients).
    6681            0 :         if tokio::net::lookup_host(format!(
    6682            0 :             "{}:{}",
    6683            0 :             register_req.listen_http_addr, register_req.listen_http_port
    6684            0 :         ))
    6685            0 :         .await
    6686            0 :         .is_err()
    6687              :         {
    6688              :             // If we have a transient DNS issue, it's up to the caller to retry their registration.  Because
    6689              :             // we can't robustly distinguish between an intermittent issue and a totally bogus DNS situation,
    6690              :             // we return a soft 503 error, to encourage callers to retry past transient issues.
    6691            0 :             return Err(ApiError::ResourceUnavailable(
    6692            0 :                 format!(
    6693            0 :                     "Node {} tried to register with unknown DNS name '{}'",
    6694            0 :                     register_req.node_id, register_req.listen_http_addr
    6695            0 :                 )
    6696            0 :                 .into(),
    6697            0 :             ));
    6698            0 :         }
    6699            0 : 
    6700            0 :         if self.config.use_https_pageserver_api && register_req.listen_https_port.is_none() {
    6701            0 :             return Err(ApiError::PreconditionFailed(
    6702            0 :                 format!(
    6703            0 :                     "Node {} has no https port, but use_https is enabled",
    6704            0 :                     register_req.node_id
    6705            0 :                 )
    6706            0 :                 .into(),
    6707            0 :             ));
    6708            0 :         }
    6709            0 : 
    6710            0 :         // Ordering: we must persist the new node _before_ adding it to in-memory state.
    6711            0 :         // This ensures that before we use it for anything or expose it via any external
    6712            0 :         // API, it is guaranteed to be available after a restart.
    6713            0 :         let new_node = Node::new(
    6714            0 :             register_req.node_id,
    6715            0 :             register_req.listen_http_addr,
    6716            0 :             register_req.listen_http_port,
    6717            0 :             register_req.listen_https_port,
    6718            0 :             register_req.listen_pg_addr,
    6719            0 :             register_req.listen_pg_port,
    6720            0 :             register_req.availability_zone_id.clone(),
    6721            0 :             self.config.use_https_pageserver_api,
    6722            0 :         );
    6723            0 :         let new_node = match new_node {
    6724            0 :             Ok(new_node) => new_node,
    6725            0 :             Err(error) => return Err(ApiError::InternalServerError(error)),
    6726              :         };
    6727              : 
    6728            0 :         match registration_status {
    6729            0 :             RegistrationStatus::New => self.persistence.insert_node(&new_node).await?,
    6730              :             RegistrationStatus::NeedUpdate => {
    6731            0 :                 self.persistence
    6732            0 :                     .update_node_on_registration(
    6733            0 :                         register_req.node_id,
    6734            0 :                         register_req.listen_https_port,
    6735            0 :                     )
    6736            0 :                     .await?
    6737              :             }
    6738            0 :             _ => unreachable!("Other statuses have been processed earlier"),
    6739              :         }
    6740              : 
    6741            0 :         let mut locked = self.inner.write().unwrap();
    6742            0 :         let mut new_nodes = (*locked.nodes).clone();
    6743            0 : 
    6744            0 :         locked.scheduler.node_upsert(&new_node);
    6745            0 :         new_nodes.insert(register_req.node_id, new_node);
    6746            0 : 
    6747            0 :         locked.nodes = Arc::new(new_nodes);
    6748            0 : 
    6749            0 :         metrics::METRICS_REGISTRY
    6750            0 :             .metrics_group
    6751            0 :             .storage_controller_pageserver_nodes
    6752            0 :             .set(locked.nodes.len() as i64);
    6753            0 : 
    6754            0 :         match registration_status {
    6755              :             RegistrationStatus::New => {
    6756            0 :                 tracing::info!(
    6757            0 :                     "Registered pageserver {} ({}), now have {} pageservers",
    6758            0 :                     register_req.node_id,
    6759            0 :                     register_req.availability_zone_id,
    6760            0 :                     locked.nodes.len()
    6761              :                 );
    6762              :             }
    6763              :             RegistrationStatus::NeedUpdate => {
    6764            0 :                 tracing::info!(
    6765            0 :                     "Re-registered and updated node {} ({})",
    6766              :                     register_req.node_id,
    6767              :                     register_req.availability_zone_id,
    6768              :                 );
    6769              :             }
    6770            0 :             _ => unreachable!("Other statuses have been processed earlier"),
    6771              :         }
    6772            0 :         Ok(())
    6773            0 :     }
    6774              : 
    6775              :     /// Configure in-memory and persistent state of a node as requested
    6776              :     ///
    6777              :     /// Note that this function does not trigger any immediate side effects in response
    6778              :     /// to the changes. That part is handled by [`Self::handle_node_availability_transition`].
    6779            0 :     async fn node_state_configure(
    6780            0 :         &self,
    6781            0 :         node_id: NodeId,
    6782            0 :         availability: Option<NodeAvailability>,
    6783            0 :         scheduling: Option<NodeSchedulingPolicy>,
    6784            0 :         node_lock: &TracingExclusiveGuard<NodeOperations>,
    6785            0 :     ) -> Result<AvailabilityTransition, ApiError> {
    6786            0 :         if let Some(scheduling) = scheduling {
    6787              :             // Scheduling is a persistent part of Node: we must write updates to the database before
    6788              :             // applying them in memory
    6789            0 :             self.persistence
    6790            0 :                 .update_node_scheduling_policy(node_id, scheduling)
    6791            0 :                 .await?;
    6792            0 :         }
    6793              : 
    6794              :         // If we're activating a node, then before setting it active we must reconcile any shard locations
    6795              :         // on that node, in case it is out of sync, e.g. due to being unavailable during controller startup,
    6796              :         // by calling [`Self::node_activate_reconcile`]
    6797              :         //
    6798              :         // The transition we calculate here remains valid later in the function because we hold the op lock on the node:
    6799              :         // nothing else can mutate its availability while we run.
    6800            0 :         let availability_transition = if let Some(input_availability) = availability.as_ref() {
    6801            0 :             let (activate_node, availability_transition) = {
    6802            0 :                 let locked = self.inner.read().unwrap();
    6803            0 :                 let Some(node) = locked.nodes.get(&node_id) else {
    6804            0 :                     return Err(ApiError::NotFound(
    6805            0 :                         anyhow::anyhow!("Node {} not registered", node_id).into(),
    6806            0 :                     ));
    6807              :                 };
    6808              : 
    6809            0 :                 (
    6810            0 :                     node.clone(),
    6811            0 :                     node.get_availability_transition(input_availability),
    6812            0 :                 )
    6813              :             };
    6814              : 
    6815            0 :             if matches!(availability_transition, AvailabilityTransition::ToActive) {
    6816            0 :                 self.node_activate_reconcile(activate_node, node_lock)
    6817            0 :                     .await?;
    6818            0 :             }
    6819            0 :             availability_transition
    6820              :         } else {
    6821            0 :             AvailabilityTransition::Unchanged
    6822              :         };
    6823              : 
    6824              :         // Apply changes from the request to our in-memory state for the Node
    6825            0 :         let mut locked = self.inner.write().unwrap();
    6826            0 :         let (nodes, _tenants, scheduler) = locked.parts_mut();
    6827            0 : 
    6828            0 :         let mut new_nodes = (**nodes).clone();
    6829              : 
    6830            0 :         let Some(node) = new_nodes.get_mut(&node_id) else {
    6831            0 :             return Err(ApiError::NotFound(
    6832            0 :                 anyhow::anyhow!("Node not registered").into(),
    6833            0 :             ));
    6834              :         };
    6835              : 
    6836            0 :         if let Some(availability) = availability {
    6837            0 :             node.set_availability(availability);
    6838            0 :         }
    6839              : 
    6840            0 :         if let Some(scheduling) = scheduling {
    6841            0 :             node.set_scheduling(scheduling);
    6842            0 :         }
    6843              : 
    6844              :         // Update the scheduler, in case the elegibility of the node for new shards has changed
    6845            0 :         scheduler.node_upsert(node);
    6846            0 : 
    6847            0 :         let new_nodes = Arc::new(new_nodes);
    6848            0 :         locked.nodes = new_nodes;
    6849            0 : 
    6850            0 :         Ok(availability_transition)
    6851            0 :     }
    6852              : 
    6853              :     /// Handle availability transition of one node
    6854              :     ///
    6855              :     /// Note that you should first call [`Self::node_state_configure`] to update
    6856              :     /// the in-memory state referencing that node. If you need to handle more than one transition
    6857              :     /// consider using [`Self::handle_node_availability_transitions`].
    6858            0 :     async fn handle_node_availability_transition(
    6859            0 :         &self,
    6860            0 :         node_id: NodeId,
    6861            0 :         transition: AvailabilityTransition,
    6862            0 :         _node_lock: &TracingExclusiveGuard<NodeOperations>,
    6863            0 :     ) -> Result<(), ApiError> {
    6864            0 :         // Modify scheduling state for any Tenants that are affected by a change in the node's availability state.
    6865            0 :         match transition {
    6866              :             AvailabilityTransition::ToOffline => {
    6867            0 :                 tracing::info!("Node {} transition to offline", node_id);
    6868              : 
    6869            0 :                 let mut locked = self.inner.write().unwrap();
    6870            0 :                 let (nodes, tenants, scheduler) = locked.parts_mut();
    6871            0 : 
    6872            0 :                 let mut tenants_affected: usize = 0;
    6873              : 
    6874            0 :                 for (_tenant_id, mut schedule_context, shards) in
    6875            0 :                     TenantShardContextIterator::new(tenants, ScheduleMode::Normal)
    6876              :                 {
    6877            0 :                     for tenant_shard in shards {
    6878            0 :                         let tenant_shard_id = tenant_shard.tenant_shard_id;
    6879            0 :                         if let Some(observed_loc) =
    6880            0 :                             tenant_shard.observed.locations.get_mut(&node_id)
    6881            0 :                         {
    6882            0 :                             // When a node goes offline, we set its observed configuration to None, indicating unknown: we will
    6883            0 :                             // not assume our knowledge of the node's configuration is accurate until it comes back online
    6884            0 :                             observed_loc.conf = None;
    6885            0 :                         }
    6886              : 
    6887            0 :                         if nodes.len() == 1 {
    6888              :                             // Special case for single-node cluster: there is no point trying to reschedule
    6889              :                             // any tenant shards: avoid doing so, in order to avoid spewing warnings about
    6890              :                             // failures to schedule them.
    6891            0 :                             continue;
    6892            0 :                         }
    6893            0 : 
    6894            0 :                         if !nodes
    6895            0 :                             .values()
    6896            0 :                             .any(|n| matches!(n.may_schedule(), MaySchedule::Yes(_)))
    6897              :                         {
    6898              :                             // Special case for when all nodes are unavailable and/or unschedulable: there is no point
    6899              :                             // trying to reschedule since there's nowhere else to go. Without this
    6900              :                             // branch we incorrectly detach tenants in response to node unavailability.
    6901            0 :                             continue;
    6902            0 :                         }
    6903            0 : 
    6904            0 :                         if tenant_shard.intent.demote_attached(scheduler, node_id) {
    6905            0 :                             tenant_shard.sequence = tenant_shard.sequence.next();
    6906            0 : 
    6907            0 :                             match tenant_shard.schedule(scheduler, &mut schedule_context) {
    6908            0 :                                 Err(e) => {
    6909            0 :                                     // It is possible that some tenants will become unschedulable when too many pageservers
    6910            0 :                                     // go offline: in this case there isn't much we can do other than make the issue observable.
    6911            0 :                                     // TODO: give TenantShard a scheduling error attribute to be queried later.
    6912            0 :                                     tracing::warn!(%tenant_shard_id, "Scheduling error when marking pageserver {} offline: {e}", node_id);
    6913              :                                 }
    6914              :                                 Ok(()) => {
    6915            0 :                                     if self
    6916            0 :                                         .maybe_reconcile_shard(
    6917            0 :                                             tenant_shard,
    6918            0 :                                             nodes,
    6919            0 :                                             ReconcilerPriority::Normal,
    6920            0 :                                         )
    6921            0 :                                         .is_some()
    6922            0 :                                     {
    6923            0 :                                         tenants_affected += 1;
    6924            0 :                                     };
    6925              :                                 }
    6926              :                             }
    6927            0 :                         }
    6928              :                     }
    6929              :                 }
    6930            0 :                 tracing::info!(
    6931            0 :                     "Launched {} reconciler tasks for tenants affected by node {} going offline",
    6932              :                     tenants_affected,
    6933              :                     node_id
    6934              :                 )
    6935              :             }
    6936              :             AvailabilityTransition::ToActive => {
    6937            0 :                 tracing::info!("Node {} transition to active", node_id);
    6938              : 
    6939            0 :                 let mut locked = self.inner.write().unwrap();
    6940            0 :                 let (nodes, tenants, _scheduler) = locked.parts_mut();
    6941              : 
    6942              :                 // When a node comes back online, we must reconcile any tenant that has a None observed
    6943              :                 // location on the node.
    6944            0 :                 for tenant_shard in tenants.values_mut() {
    6945              :                     // If a reconciliation is already in progress, rely on the previous scheduling
    6946              :                     // decision and skip triggering a new reconciliation.
    6947            0 :                     if tenant_shard.reconciler.is_some() {
    6948            0 :                         continue;
    6949            0 :                     }
    6950              : 
    6951            0 :                     if let Some(observed_loc) = tenant_shard.observed.locations.get_mut(&node_id) {
    6952            0 :                         if observed_loc.conf.is_none() {
    6953            0 :                             self.maybe_reconcile_shard(
    6954            0 :                                 tenant_shard,
    6955            0 :                                 nodes,
    6956            0 :                                 ReconcilerPriority::Normal,
    6957            0 :                             );
    6958            0 :                         }
    6959            0 :                     }
    6960              :                 }
    6961              : 
    6962              :                 // TODO: in the background, we should balance work back onto this pageserver
    6963              :             }
    6964              :             // No action required for the intermediate unavailable state.
    6965              :             // When we transition into active or offline from the unavailable state,
    6966              :             // the correct handling above will kick in.
    6967              :             AvailabilityTransition::ToWarmingUpFromActive => {
    6968            0 :                 tracing::info!("Node {} transition to unavailable from active", node_id);
    6969              :             }
    6970              :             AvailabilityTransition::ToWarmingUpFromOffline => {
    6971            0 :                 tracing::info!("Node {} transition to unavailable from offline", node_id);
    6972              :             }
    6973              :             AvailabilityTransition::Unchanged => {
    6974            0 :                 tracing::debug!("Node {} no availability change during config", node_id);
    6975              :             }
    6976              :         }
    6977              : 
    6978            0 :         Ok(())
    6979            0 :     }
    6980              : 
    6981              :     /// Handle availability transition for multiple nodes
    6982              :     ///
    6983              :     /// Note that you should first call [`Self::node_state_configure`] for
    6984              :     /// all nodes being handled here for the handling to use fresh in-memory state.
    6985            0 :     async fn handle_node_availability_transitions(
    6986            0 :         &self,
    6987            0 :         transitions: Vec<(
    6988            0 :             NodeId,
    6989            0 :             TracingExclusiveGuard<NodeOperations>,
    6990            0 :             AvailabilityTransition,
    6991            0 :         )>,
    6992            0 :     ) -> Result<(), Vec<(NodeId, ApiError)>> {
    6993            0 :         let mut errors = Vec::default();
    6994            0 :         for (node_id, node_lock, transition) in transitions {
    6995            0 :             let res = self
    6996            0 :                 .handle_node_availability_transition(node_id, transition, &node_lock)
    6997            0 :                 .await;
    6998            0 :             if let Err(err) = res {
    6999            0 :                 errors.push((node_id, err));
    7000            0 :             }
    7001              :         }
    7002              : 
    7003            0 :         if errors.is_empty() {
    7004            0 :             Ok(())
    7005              :         } else {
    7006            0 :             Err(errors)
    7007              :         }
    7008            0 :     }
    7009              : 
    7010            0 :     pub(crate) async fn node_configure(
    7011            0 :         &self,
    7012            0 :         node_id: NodeId,
    7013            0 :         availability: Option<NodeAvailability>,
    7014            0 :         scheduling: Option<NodeSchedulingPolicy>,
    7015            0 :     ) -> Result<(), ApiError> {
    7016            0 :         let node_lock =
    7017            0 :             trace_exclusive_lock(&self.node_op_locks, node_id, NodeOperations::Configure).await;
    7018              : 
    7019            0 :         let transition = self
    7020            0 :             .node_state_configure(node_id, availability, scheduling, &node_lock)
    7021            0 :             .await?;
    7022            0 :         self.handle_node_availability_transition(node_id, transition, &node_lock)
    7023            0 :             .await
    7024            0 :     }
    7025              : 
    7026              :     /// Wrapper around [`Self::node_configure`] which only allows changes while there is no ongoing
    7027              :     /// operation for HTTP api.
    7028            0 :     pub(crate) async fn external_node_configure(
    7029            0 :         &self,
    7030            0 :         node_id: NodeId,
    7031            0 :         availability: Option<NodeAvailability>,
    7032            0 :         scheduling: Option<NodeSchedulingPolicy>,
    7033            0 :     ) -> Result<(), ApiError> {
    7034            0 :         {
    7035            0 :             let locked = self.inner.read().unwrap();
    7036            0 :             if let Some(op) = locked.ongoing_operation.as_ref().map(|op| op.operation) {
    7037            0 :                 return Err(ApiError::PreconditionFailed(
    7038            0 :                     format!("Ongoing background operation forbids configuring: {op}").into(),
    7039            0 :                 ));
    7040            0 :             }
    7041            0 :         }
    7042            0 : 
    7043            0 :         self.node_configure(node_id, availability, scheduling).await
    7044            0 :     }
    7045              : 
    7046            0 :     pub(crate) async fn start_node_drain(
    7047            0 :         self: &Arc<Self>,
    7048            0 :         node_id: NodeId,
    7049            0 :     ) -> Result<(), ApiError> {
    7050            0 :         let (ongoing_op, node_available, node_policy, schedulable_nodes_count) = {
    7051            0 :             let locked = self.inner.read().unwrap();
    7052            0 :             let nodes = &locked.nodes;
    7053            0 :             let node = nodes.get(&node_id).ok_or(ApiError::NotFound(
    7054            0 :                 anyhow::anyhow!("Node {} not registered", node_id).into(),
    7055            0 :             ))?;
    7056            0 :             let schedulable_nodes_count = nodes
    7057            0 :                 .iter()
    7058            0 :                 .filter(|(_, n)| matches!(n.may_schedule(), MaySchedule::Yes(_)))
    7059            0 :                 .count();
    7060            0 : 
    7061            0 :             (
    7062            0 :                 locked
    7063            0 :                     .ongoing_operation
    7064            0 :                     .as_ref()
    7065            0 :                     .map(|ongoing| ongoing.operation),
    7066            0 :                 node.is_available(),
    7067            0 :                 node.get_scheduling(),
    7068            0 :                 schedulable_nodes_count,
    7069            0 :             )
    7070            0 :         };
    7071              : 
    7072            0 :         if let Some(ongoing) = ongoing_op {
    7073            0 :             return Err(ApiError::PreconditionFailed(
    7074            0 :                 format!("Background operation already ongoing for node: {}", ongoing).into(),
    7075            0 :             ));
    7076            0 :         }
    7077            0 : 
    7078            0 :         if !node_available {
    7079            0 :             return Err(ApiError::ResourceUnavailable(
    7080            0 :                 format!("Node {node_id} is currently unavailable").into(),
    7081            0 :             ));
    7082            0 :         }
    7083            0 : 
    7084            0 :         if schedulable_nodes_count == 0 {
    7085            0 :             return Err(ApiError::PreconditionFailed(
    7086            0 :                 "No other schedulable nodes to drain to".into(),
    7087            0 :             ));
    7088            0 :         }
    7089            0 : 
    7090            0 :         match node_policy {
    7091              :             NodeSchedulingPolicy::Active => {
    7092            0 :                 self.node_configure(node_id, None, Some(NodeSchedulingPolicy::Draining))
    7093            0 :                     .await?;
    7094              : 
    7095            0 :                 let cancel = self.cancel.child_token();
    7096            0 :                 let gate_guard = self.gate.enter().map_err(|_| ApiError::ShuttingDown)?;
    7097              : 
    7098            0 :                 self.inner.write().unwrap().ongoing_operation = Some(OperationHandler {
    7099            0 :                     operation: Operation::Drain(Drain { node_id }),
    7100            0 :                     cancel: cancel.clone(),
    7101            0 :                 });
    7102              : 
    7103            0 :                 let span = tracing::info_span!(parent: None, "drain_node", %node_id);
    7104              : 
    7105            0 :                 tokio::task::spawn({
    7106            0 :                     let service = self.clone();
    7107            0 :                     let cancel = cancel.clone();
    7108            0 :                     async move {
    7109            0 :                         let _gate_guard = gate_guard;
    7110            0 : 
    7111            0 :                         scopeguard::defer! {
    7112            0 :                             let prev = service.inner.write().unwrap().ongoing_operation.take();
    7113            0 : 
    7114            0 :                             if let Some(Operation::Drain(removed_drain)) = prev.map(|h| h.operation) {
    7115            0 :                                 assert_eq!(removed_drain.node_id, node_id, "We always take the same operation");
    7116            0 :                             } else {
    7117            0 :                                 panic!("We always remove the same operation")
    7118            0 :                             }
    7119            0 :                         }
    7120            0 : 
    7121            0 :                         tracing::info!("Drain background operation starting");
    7122            0 :                         let res = service.drain_node(node_id, cancel).await;
    7123            0 :                         match res {
    7124              :                             Ok(()) => {
    7125            0 :                                 tracing::info!("Drain background operation completed successfully");
    7126              :                             }
    7127              :                             Err(OperationError::Cancelled) => {
    7128            0 :                                 tracing::info!("Drain background operation was cancelled");
    7129              :                             }
    7130            0 :                             Err(err) => {
    7131            0 :                                 tracing::error!("Drain background operation encountered: {err}")
    7132              :                             }
    7133              :                         }
    7134            0 :                     }
    7135            0 :                 }.instrument(span));
    7136            0 :             }
    7137              :             NodeSchedulingPolicy::Draining => {
    7138            0 :                 return Err(ApiError::Conflict(format!(
    7139            0 :                     "Node {node_id} has drain in progress"
    7140            0 :                 )));
    7141              :             }
    7142            0 :             policy => {
    7143            0 :                 return Err(ApiError::PreconditionFailed(
    7144            0 :                     format!("Node {node_id} cannot be drained due to {policy:?} policy").into(),
    7145            0 :                 ));
    7146              :             }
    7147              :         }
    7148              : 
    7149            0 :         Ok(())
    7150            0 :     }
    7151              : 
    7152            0 :     pub(crate) async fn cancel_node_drain(&self, node_id: NodeId) -> Result<(), ApiError> {
    7153            0 :         let node_available = {
    7154            0 :             let locked = self.inner.read().unwrap();
    7155            0 :             let nodes = &locked.nodes;
    7156            0 :             let node = nodes.get(&node_id).ok_or(ApiError::NotFound(
    7157            0 :                 anyhow::anyhow!("Node {} not registered", node_id).into(),
    7158            0 :             ))?;
    7159              : 
    7160            0 :             node.is_available()
    7161            0 :         };
    7162            0 : 
    7163            0 :         if !node_available {
    7164            0 :             return Err(ApiError::ResourceUnavailable(
    7165            0 :                 format!("Node {node_id} is currently unavailable").into(),
    7166            0 :             ));
    7167            0 :         }
    7168              : 
    7169            0 :         if let Some(op_handler) = self.inner.read().unwrap().ongoing_operation.as_ref() {
    7170            0 :             if let Operation::Drain(drain) = op_handler.operation {
    7171            0 :                 if drain.node_id == node_id {
    7172            0 :                     tracing::info!("Cancelling background drain operation for node {node_id}");
    7173            0 :                     op_handler.cancel.cancel();
    7174            0 :                     return Ok(());
    7175            0 :                 }
    7176            0 :             }
    7177            0 :         }
    7178              : 
    7179            0 :         Err(ApiError::PreconditionFailed(
    7180            0 :             format!("Node {node_id} has no drain in progress").into(),
    7181            0 :         ))
    7182            0 :     }
    7183              : 
    7184            0 :     pub(crate) async fn start_node_fill(self: &Arc<Self>, node_id: NodeId) -> Result<(), ApiError> {
    7185            0 :         let (ongoing_op, node_available, node_policy, total_nodes_count) = {
    7186            0 :             let locked = self.inner.read().unwrap();
    7187            0 :             let nodes = &locked.nodes;
    7188            0 :             let node = nodes.get(&node_id).ok_or(ApiError::NotFound(
    7189            0 :                 anyhow::anyhow!("Node {} not registered", node_id).into(),
    7190            0 :             ))?;
    7191              : 
    7192            0 :             (
    7193            0 :                 locked
    7194            0 :                     .ongoing_operation
    7195            0 :                     .as_ref()
    7196            0 :                     .map(|ongoing| ongoing.operation),
    7197            0 :                 node.is_available(),
    7198            0 :                 node.get_scheduling(),
    7199            0 :                 nodes.len(),
    7200            0 :             )
    7201            0 :         };
    7202              : 
    7203            0 :         if let Some(ongoing) = ongoing_op {
    7204            0 :             return Err(ApiError::PreconditionFailed(
    7205            0 :                 format!("Background operation already ongoing for node: {}", ongoing).into(),
    7206            0 :             ));
    7207            0 :         }
    7208            0 : 
    7209            0 :         if !node_available {
    7210            0 :             return Err(ApiError::ResourceUnavailable(
    7211            0 :                 format!("Node {node_id} is currently unavailable").into(),
    7212            0 :             ));
    7213            0 :         }
    7214            0 : 
    7215            0 :         if total_nodes_count <= 1 {
    7216            0 :             return Err(ApiError::PreconditionFailed(
    7217            0 :                 "No other nodes to fill from".into(),
    7218            0 :             ));
    7219            0 :         }
    7220            0 : 
    7221            0 :         match node_policy {
    7222              :             NodeSchedulingPolicy::Active => {
    7223            0 :                 self.node_configure(node_id, None, Some(NodeSchedulingPolicy::Filling))
    7224            0 :                     .await?;
    7225              : 
    7226            0 :                 let cancel = self.cancel.child_token();
    7227            0 :                 let gate_guard = self.gate.enter().map_err(|_| ApiError::ShuttingDown)?;
    7228              : 
    7229            0 :                 self.inner.write().unwrap().ongoing_operation = Some(OperationHandler {
    7230            0 :                     operation: Operation::Fill(Fill { node_id }),
    7231            0 :                     cancel: cancel.clone(),
    7232            0 :                 });
    7233              : 
    7234            0 :                 let span = tracing::info_span!(parent: None, "fill_node", %node_id);
    7235              : 
    7236            0 :                 tokio::task::spawn({
    7237            0 :                     let service = self.clone();
    7238            0 :                     let cancel = cancel.clone();
    7239            0 :                     async move {
    7240            0 :                         let _gate_guard = gate_guard;
    7241            0 : 
    7242            0 :                         scopeguard::defer! {
    7243            0 :                             let prev = service.inner.write().unwrap().ongoing_operation.take();
    7244            0 : 
    7245            0 :                             if let Some(Operation::Fill(removed_fill)) = prev.map(|h| h.operation) {
    7246            0 :                                 assert_eq!(removed_fill.node_id, node_id, "We always take the same operation");
    7247            0 :                             } else {
    7248            0 :                                 panic!("We always remove the same operation")
    7249            0 :                             }
    7250            0 :                         }
    7251            0 : 
    7252            0 :                         tracing::info!("Fill background operation starting");
    7253            0 :                         let res = service.fill_node(node_id, cancel).await;
    7254            0 :                         match res {
    7255              :                             Ok(()) => {
    7256            0 :                                 tracing::info!("Fill background operation completed successfully");
    7257              :                             }
    7258              :                             Err(OperationError::Cancelled) => {
    7259            0 :                                 tracing::info!("Fill background operation was cancelled");
    7260              :                             }
    7261            0 :                             Err(err) => {
    7262            0 :                                 tracing::error!("Fill background operation encountered: {err}")
    7263              :                             }
    7264              :                         }
    7265            0 :                     }
    7266            0 :                 }.instrument(span));
    7267            0 :             }
    7268              :             NodeSchedulingPolicy::Filling => {
    7269            0 :                 return Err(ApiError::Conflict(format!(
    7270            0 :                     "Node {node_id} has fill in progress"
    7271            0 :                 )));
    7272              :             }
    7273            0 :             policy => {
    7274            0 :                 return Err(ApiError::PreconditionFailed(
    7275            0 :                     format!("Node {node_id} cannot be filled due to {policy:?} policy").into(),
    7276            0 :                 ));
    7277              :             }
    7278              :         }
    7279              : 
    7280            0 :         Ok(())
    7281            0 :     }
    7282              : 
    7283            0 :     pub(crate) async fn cancel_node_fill(&self, node_id: NodeId) -> Result<(), ApiError> {
    7284            0 :         let node_available = {
    7285            0 :             let locked = self.inner.read().unwrap();
    7286            0 :             let nodes = &locked.nodes;
    7287            0 :             let node = nodes.get(&node_id).ok_or(ApiError::NotFound(
    7288            0 :                 anyhow::anyhow!("Node {} not registered", node_id).into(),
    7289            0 :             ))?;
    7290              : 
    7291            0 :             node.is_available()
    7292            0 :         };
    7293            0 : 
    7294            0 :         if !node_available {
    7295            0 :             return Err(ApiError::ResourceUnavailable(
    7296            0 :                 format!("Node {node_id} is currently unavailable").into(),
    7297            0 :             ));
    7298            0 :         }
    7299              : 
    7300            0 :         if let Some(op_handler) = self.inner.read().unwrap().ongoing_operation.as_ref() {
    7301            0 :             if let Operation::Fill(fill) = op_handler.operation {
    7302            0 :                 if fill.node_id == node_id {
    7303            0 :                     tracing::info!("Cancelling background drain operation for node {node_id}");
    7304            0 :                     op_handler.cancel.cancel();
    7305            0 :                     return Ok(());
    7306            0 :                 }
    7307            0 :             }
    7308            0 :         }
    7309              : 
    7310            0 :         Err(ApiError::PreconditionFailed(
    7311            0 :             format!("Node {node_id} has no fill in progress").into(),
    7312            0 :         ))
    7313            0 :     }
    7314              : 
    7315              :     /// Like [`Self::maybe_configured_reconcile_shard`], but uses the default reconciler
    7316              :     /// configuration
    7317            0 :     fn maybe_reconcile_shard(
    7318            0 :         &self,
    7319            0 :         shard: &mut TenantShard,
    7320            0 :         nodes: &Arc<HashMap<NodeId, Node>>,
    7321            0 :         priority: ReconcilerPriority,
    7322            0 :     ) -> Option<ReconcilerWaiter> {
    7323            0 :         self.maybe_configured_reconcile_shard(shard, nodes, ReconcilerConfig::new(priority))
    7324            0 :     }
    7325              : 
    7326              :     /// Before constructing a Reconciler, acquire semaphore units from the appropriate concurrency limit (depends on priority)
    7327            0 :     fn get_reconciler_units(
    7328            0 :         &self,
    7329            0 :         priority: ReconcilerPriority,
    7330            0 :     ) -> Result<ReconcileUnits, TryAcquireError> {
    7331            0 :         let units = match priority {
    7332            0 :             ReconcilerPriority::Normal => self.reconciler_concurrency.clone().try_acquire_owned(),
    7333              :             ReconcilerPriority::High => {
    7334            0 :                 match self
    7335            0 :                     .priority_reconciler_concurrency
    7336            0 :                     .clone()
    7337            0 :                     .try_acquire_owned()
    7338              :                 {
    7339            0 :                     Ok(u) => Ok(u),
    7340              :                     Err(TryAcquireError::NoPermits) => {
    7341              :                         // If the high priority semaphore is exhausted, then high priority tasks may steal units from
    7342              :                         // the normal priority semaphore.
    7343            0 :                         self.reconciler_concurrency.clone().try_acquire_owned()
    7344              :                     }
    7345            0 :                     Err(e) => Err(e),
    7346              :                 }
    7347              :             }
    7348              :         };
    7349              : 
    7350            0 :         units.map(ReconcileUnits::new)
    7351            0 :     }
    7352              : 
    7353              :     /// Wrap [`TenantShard`] reconciliation methods with acquisition of [`Gate`] and [`ReconcileUnits`],
    7354            0 :     fn maybe_configured_reconcile_shard(
    7355            0 :         &self,
    7356            0 :         shard: &mut TenantShard,
    7357            0 :         nodes: &Arc<HashMap<NodeId, Node>>,
    7358            0 :         reconciler_config: ReconcilerConfig,
    7359            0 :     ) -> Option<ReconcilerWaiter> {
    7360            0 :         let reconcile_needed = shard.get_reconcile_needed(nodes);
    7361              : 
    7362            0 :         let reconcile_reason = match reconcile_needed {
    7363            0 :             ReconcileNeeded::No => return None,
    7364            0 :             ReconcileNeeded::WaitExisting(waiter) => return Some(waiter),
    7365            0 :             ReconcileNeeded::Yes(reason) => {
    7366            0 :                 // Fall through to try and acquire units for spawning reconciler
    7367            0 :                 reason
    7368              :             }
    7369              :         };
    7370              : 
    7371            0 :         let units = match self.get_reconciler_units(reconciler_config.priority) {
    7372            0 :             Ok(u) => u,
    7373              :             Err(_) => {
    7374            0 :                 tracing::info!(tenant_id=%shard.tenant_shard_id.tenant_id, shard_id=%shard.tenant_shard_id.shard_slug(),
    7375            0 :                     "Concurrency limited: enqueued for reconcile later");
    7376            0 :                 if !shard.delayed_reconcile {
    7377            0 :                     match self.delayed_reconcile_tx.try_send(shard.tenant_shard_id) {
    7378            0 :                         Err(TrySendError::Closed(_)) => {
    7379            0 :                             // Weird mid-shutdown case?
    7380            0 :                         }
    7381              :                         Err(TrySendError::Full(_)) => {
    7382              :                             // It is safe to skip sending our ID in the channel: we will eventually get retried by the background reconcile task.
    7383            0 :                             tracing::warn!(
    7384            0 :                                 "Many shards are waiting to reconcile: delayed_reconcile queue is full"
    7385              :                             );
    7386              :                         }
    7387            0 :                         Ok(()) => {
    7388            0 :                             shard.delayed_reconcile = true;
    7389            0 :                         }
    7390              :                     }
    7391            0 :                 }
    7392              : 
    7393              :                 // We won't spawn a reconciler, but we will construct a waiter that waits for the shard's sequence
    7394              :                 // number to advance.  When this function is eventually called again and succeeds in getting units,
    7395              :                 // it will spawn a reconciler that makes this waiter complete.
    7396            0 :                 return Some(shard.future_reconcile_waiter());
    7397              :             }
    7398              :         };
    7399              : 
    7400            0 :         let Ok(gate_guard) = self.reconcilers_gate.enter() else {
    7401              :             // Gate closed: we're shutting down, drop out.
    7402            0 :             return None;
    7403              :         };
    7404              : 
    7405            0 :         shard.spawn_reconciler(
    7406            0 :             reconcile_reason,
    7407            0 :             &self.result_tx,
    7408            0 :             nodes,
    7409            0 :             &self.compute_hook,
    7410            0 :             reconciler_config,
    7411            0 :             &self.config,
    7412            0 :             &self.persistence,
    7413            0 :             units,
    7414            0 :             gate_guard,
    7415            0 :             &self.reconcilers_cancel,
    7416            0 :         )
    7417            0 :     }
    7418              : 
    7419              :     /// Check all tenants for pending reconciliation work, and reconcile those in need.
    7420              :     /// Additionally, reschedule tenants that require it.
    7421              :     ///
    7422              :     /// Returns how many reconciliation tasks were started, or `1` if no reconciles were
    7423              :     /// spawned but some _would_ have been spawned if `reconciler_concurrency` units where
    7424              :     /// available.  A return value of 0 indicates that everything is fully reconciled already.
    7425            0 :     fn reconcile_all(&self) -> usize {
    7426            0 :         let mut locked = self.inner.write().unwrap();
    7427            0 :         let (nodes, tenants, scheduler) = locked.parts_mut();
    7428            0 :         let pageservers = nodes.clone();
    7429            0 : 
    7430            0 :         // This function is an efficient place to update lazy statistics, since we are walking
    7431            0 :         // all tenants.
    7432            0 :         let mut pending_reconciles = 0;
    7433            0 :         let mut az_violations = 0;
    7434            0 : 
    7435            0 :         // If we find any tenants to drop from memory, stash them to offload after
    7436            0 :         // we're done traversing the map of tenants.
    7437            0 :         let mut drop_detached_tenants = Vec::new();
    7438            0 : 
    7439            0 :         let mut reconciles_spawned = 0;
    7440            0 :         for shard in tenants.values_mut() {
    7441              :             // Accumulate scheduling statistics
    7442            0 :             if let (Some(attached), Some(preferred)) =
    7443            0 :                 (shard.intent.get_attached(), shard.preferred_az())
    7444              :             {
    7445            0 :                 let node_az = nodes
    7446            0 :                     .get(attached)
    7447            0 :                     .expect("Nodes exist if referenced")
    7448            0 :                     .get_availability_zone_id();
    7449            0 :                 if node_az != preferred {
    7450            0 :                     az_violations += 1;
    7451            0 :                 }
    7452            0 :             }
    7453              : 
    7454              :             // Skip checking if this shard is already enqueued for reconciliation
    7455            0 :             if shard.delayed_reconcile && self.reconciler_concurrency.available_permits() == 0 {
    7456              :                 // If there is something delayed, then return a nonzero count so that
    7457              :                 // callers like reconcile_all_now do not incorrectly get the impression
    7458              :                 // that the system is in a quiescent state.
    7459            0 :                 reconciles_spawned = std::cmp::max(1, reconciles_spawned);
    7460            0 :                 pending_reconciles += 1;
    7461            0 :                 continue;
    7462            0 :             }
    7463            0 : 
    7464            0 :             // Eventual consistency: if an earlier reconcile job failed, and the shard is still
    7465            0 :             // dirty, spawn another rone
    7466            0 :             if self
    7467            0 :                 .maybe_reconcile_shard(shard, &pageservers, ReconcilerPriority::Normal)
    7468            0 :                 .is_some()
    7469            0 :             {
    7470            0 :                 reconciles_spawned += 1;
    7471            0 :             } else if shard.delayed_reconcile {
    7472            0 :                 // Shard wanted to reconcile but for some reason couldn't.
    7473            0 :                 pending_reconciles += 1;
    7474            0 :             }
    7475              : 
    7476              :             // If this tenant is detached, try dropping it from memory. This is usually done
    7477              :             // proactively in [`Self::process_results`], but we do it here to handle the edge
    7478              :             // case where a reconcile completes while someone else is holding an op lock for the tenant.
    7479            0 :             if shard.tenant_shard_id.shard_number == ShardNumber(0)
    7480            0 :                 && shard.policy == PlacementPolicy::Detached
    7481              :             {
    7482            0 :                 if let Some(guard) = self.tenant_op_locks.try_exclusive(
    7483            0 :                     shard.tenant_shard_id.tenant_id,
    7484            0 :                     TenantOperations::DropDetached,
    7485            0 :                 ) {
    7486            0 :                     drop_detached_tenants.push((shard.tenant_shard_id.tenant_id, guard));
    7487            0 :                 }
    7488            0 :             }
    7489              :         }
    7490              : 
    7491              :         // Some metrics are calculated from SchedulerNode state, update these periodically
    7492            0 :         scheduler.update_metrics();
    7493              : 
    7494              :         // Process any deferred tenant drops
    7495            0 :         for (tenant_id, guard) in drop_detached_tenants {
    7496            0 :             self.maybe_drop_tenant(tenant_id, &mut locked, &guard);
    7497            0 :         }
    7498              : 
    7499            0 :         metrics::METRICS_REGISTRY
    7500            0 :             .metrics_group
    7501            0 :             .storage_controller_schedule_az_violation
    7502            0 :             .set(az_violations as i64);
    7503            0 : 
    7504            0 :         metrics::METRICS_REGISTRY
    7505            0 :             .metrics_group
    7506            0 :             .storage_controller_pending_reconciles
    7507            0 :             .set(pending_reconciles as i64);
    7508            0 : 
    7509            0 :         reconciles_spawned
    7510            0 :     }
    7511              : 
    7512              :     /// `optimize` in this context means identifying shards which have valid scheduled locations, but
    7513              :     /// could be scheduled somewhere better:
    7514              :     /// - Cutting over to a secondary if the node with the secondary is more lightly loaded
    7515              :     ///    * e.g. after a node fails then recovers, to move some work back to it
    7516              :     /// - Cutting over to a secondary if it improves the spread of shard attachments within a tenant
    7517              :     ///    * e.g. after a shard split, the initial attached locations will all be on the node where
    7518              :     ///      we did the split, but are probably better placed elsewhere.
    7519              :     /// - Creating new secondary locations if it improves the spreading of a sharded tenant
    7520              :     ///    * e.g. after a shard split, some locations will be on the same node (where the split
    7521              :     ///      happened), and will probably be better placed elsewhere.
    7522              :     ///
    7523              :     /// To put it more briefly: whereas the scheduler respects soft constraints in a ScheduleContext at
    7524              :     /// the time of scheduling, this function looks for cases where a better-scoring location is available
    7525              :     /// according to those same soft constraints.
    7526            0 :     async fn optimize_all(&self) -> usize {
    7527              :         // Limit on how many shards' optmizations each call to this function will execute.  Combined
    7528              :         // with the frequency of background calls, this acts as an implicit rate limit that runs a small
    7529              :         // trickle of optimizations in the background, rather than executing a large number in parallel
    7530              :         // when a change occurs.
    7531              :         const MAX_OPTIMIZATIONS_EXEC_PER_PASS: usize = 16;
    7532              : 
    7533              :         // Synchronous prepare: scan shards for possible scheduling optimizations
    7534            0 :         let candidate_work = self.optimize_all_plan();
    7535            0 :         let candidate_work_len = candidate_work.len();
    7536              : 
    7537              :         // Asynchronous validate: I/O to pageservers to make sure shards are in a good state to apply validation
    7538            0 :         let validated_work = self.optimize_all_validate(candidate_work).await;
    7539              : 
    7540            0 :         let was_work_filtered = validated_work.len() != candidate_work_len;
    7541            0 : 
    7542            0 :         // Synchronous apply: update the shards' intent states according to validated optimisations
    7543            0 :         let mut reconciles_spawned = 0;
    7544            0 :         let mut optimizations_applied = 0;
    7545            0 :         let mut locked = self.inner.write().unwrap();
    7546            0 :         let (nodes, tenants, scheduler) = locked.parts_mut();
    7547            0 :         for (tenant_shard_id, optimization) in validated_work {
    7548            0 :             let Some(shard) = tenants.get_mut(&tenant_shard_id) else {
    7549              :                 // Shard was dropped between planning and execution;
    7550            0 :                 continue;
    7551              :             };
    7552            0 :             tracing::info!(tenant_shard_id=%tenant_shard_id, "Applying optimization: {optimization:?}");
    7553            0 :             if shard.apply_optimization(scheduler, optimization) {
    7554            0 :                 optimizations_applied += 1;
    7555            0 :                 if self
    7556            0 :                     .maybe_reconcile_shard(shard, nodes, ReconcilerPriority::Normal)
    7557            0 :                     .is_some()
    7558            0 :                 {
    7559            0 :                     reconciles_spawned += 1;
    7560            0 :                 }
    7561            0 :             }
    7562              : 
    7563            0 :             if optimizations_applied >= MAX_OPTIMIZATIONS_EXEC_PER_PASS {
    7564            0 :                 break;
    7565            0 :             }
    7566              :         }
    7567              : 
    7568            0 :         if was_work_filtered {
    7569            0 :             // If we filtered any work out during validation, ensure we return a nonzero value to indicate
    7570            0 :             // to callers that the system is not in a truly quiet state, it's going to do some work as soon
    7571            0 :             // as these validations start passing.
    7572            0 :             reconciles_spawned = std::cmp::max(reconciles_spawned, 1);
    7573            0 :         }
    7574              : 
    7575            0 :         reconciles_spawned
    7576            0 :     }
    7577              : 
    7578            0 :     fn optimize_all_plan(&self) -> Vec<(TenantShardId, ScheduleOptimization)> {
    7579              :         // How many candidate optimizations we will generate, before evaluating them for readniess: setting
    7580              :         // this higher than the execution limit gives us a chance to execute some work even if the first
    7581              :         // few optimizations we find are not ready.
    7582              :         const MAX_OPTIMIZATIONS_PLAN_PER_PASS: usize = 64;
    7583              : 
    7584            0 :         let mut work = Vec::new();
    7585            0 :         let mut locked = self.inner.write().unwrap();
    7586            0 :         let (_nodes, tenants, scheduler) = locked.parts_mut();
    7587              : 
    7588              :         // We are going to plan a bunch of optimisations before applying any of them, so the
    7589              :         // utilisation stats on nodes will be effectively stale for the >1st optimisation we
    7590              :         // generate.  To avoid this causing unstable migrations/flapping, it's important that the
    7591              :         // code in TenantShard for finding optimisations uses [`NodeAttachmentSchedulingScore::disregard_utilization`]
    7592              :         // to ignore the utilisation component of the score.
    7593              : 
    7594            0 :         for (_tenant_id, schedule_context, shards) in
    7595            0 :             TenantShardContextIterator::new(tenants, ScheduleMode::Speculative)
    7596              :         {
    7597            0 :             for shard in shards {
    7598            0 :                 if work.len() >= MAX_OPTIMIZATIONS_PLAN_PER_PASS {
    7599            0 :                     break;
    7600            0 :                 }
    7601            0 :                 match shard.get_scheduling_policy() {
    7602            0 :                     ShardSchedulingPolicy::Active => {
    7603            0 :                         // Ok to do optimization
    7604            0 :                     }
    7605            0 :                     ShardSchedulingPolicy::Essential if shard.get_preferred_node().is_some() => {
    7606            0 :                         // Ok to do optimization: we are executing a graceful migration that
    7607            0 :                         // has set preferred_node
    7608            0 :                     }
    7609              :                     ShardSchedulingPolicy::Essential
    7610              :                     | ShardSchedulingPolicy::Pause
    7611              :                     | ShardSchedulingPolicy::Stop => {
    7612              :                         // Policy prevents optimizing this shard.
    7613            0 :                         continue;
    7614              :                     }
    7615              :                 }
    7616              : 
    7617            0 :                 if !matches!(shard.splitting, SplitState::Idle)
    7618            0 :                     || matches!(shard.policy, PlacementPolicy::Detached)
    7619            0 :                     || shard.reconciler.is_some()
    7620              :                 {
    7621              :                     // Do not start any optimizations while another change to the tenant is ongoing: this
    7622              :                     // is not necessary for correctness, but simplifies operations and implicitly throttles
    7623              :                     // optimization changes to happen in a "trickle" over time.
    7624            0 :                     continue;
    7625            0 :                 }
    7626            0 : 
    7627            0 :                 // Fast path: we may quickly identify shards that don't have any possible optimisations
    7628            0 :                 if !shard.maybe_optimizable(scheduler, &schedule_context) {
    7629            0 :                     if cfg!(feature = "testing") {
    7630              :                         // Check that maybe_optimizable doesn't disagree with the actual optimization functions.
    7631              :                         // Only do this in testing builds because it is not a correctness-critical check, so we shouldn't
    7632              :                         // panic in prod if we hit this, or spend cycles on it in prod.
    7633            0 :                         assert!(
    7634            0 :                             shard
    7635            0 :                                 .optimize_attachment(scheduler, &schedule_context)
    7636            0 :                                 .is_none()
    7637            0 :                         );
    7638            0 :                         assert!(
    7639            0 :                             shard
    7640            0 :                                 .optimize_secondary(scheduler, &schedule_context)
    7641            0 :                                 .is_none()
    7642            0 :                         );
    7643            0 :                     }
    7644            0 :                     continue;
    7645            0 :                 }
    7646              : 
    7647            0 :                 if let Some(optimization) =
    7648              :                     // If idle, maybe optimize attachments: if a shard has a secondary location that is preferable to
    7649              :                     // its primary location based on soft constraints, cut it over.
    7650            0 :                     shard.optimize_attachment(scheduler, &schedule_context)
    7651              :                 {
    7652            0 :                     tracing::info!(tenant_shard_id=%shard.tenant_shard_id, "Identified optimization for attachment: {optimization:?}");
    7653            0 :                     work.push((shard.tenant_shard_id, optimization));
    7654            0 :                     break;
    7655            0 :                 } else if let Some(optimization) =
    7656              :                     // If idle, maybe optimize secondary locations: if a shard has a secondary location that would be
    7657              :                     // better placed on another node, based on ScheduleContext, then adjust it.  This
    7658              :                     // covers cases like after a shard split, where we might have too many shards
    7659              :                     // in the same tenant with secondary locations on the node where they originally split.
    7660            0 :                     shard.optimize_secondary(scheduler, &schedule_context)
    7661              :                 {
    7662            0 :                     tracing::info!(tenant_shard_id=%shard.tenant_shard_id, "Identified optimization for secondary: {optimization:?}");
    7663            0 :                     work.push((shard.tenant_shard_id, optimization));
    7664            0 :                     break;
    7665            0 :                 }
    7666              :             }
    7667              :         }
    7668              : 
    7669            0 :         work
    7670            0 :     }
    7671              : 
    7672            0 :     async fn optimize_all_validate(
    7673            0 :         &self,
    7674            0 :         candidate_work: Vec<(TenantShardId, ScheduleOptimization)>,
    7675            0 :     ) -> Vec<(TenantShardId, ScheduleOptimization)> {
    7676            0 :         // Take a clone of the node map to use outside the lock in async validation phase
    7677            0 :         let validation_nodes = { self.inner.read().unwrap().nodes.clone() };
    7678            0 : 
    7679            0 :         let mut want_secondary_status = Vec::new();
    7680            0 : 
    7681            0 :         // Validate our plans: this is an async phase where we may do I/O to pageservers to
    7682            0 :         // check that the state of locations is acceptable to run the optimization, such as
    7683            0 :         // checking that a secondary location is sufficiently warmed-up to cleanly cut over
    7684            0 :         // in a live migration.
    7685            0 :         let mut validated_work = Vec::new();
    7686            0 :         for (tenant_shard_id, optimization) in candidate_work {
    7687            0 :             match optimization.action {
    7688              :                 ScheduleOptimizationAction::MigrateAttachment(MigrateAttachment {
    7689              :                     old_attached_node_id: _,
    7690            0 :                     new_attached_node_id,
    7691            0 :                 }) => {
    7692            0 :                     match validation_nodes.get(&new_attached_node_id) {
    7693            0 :                         None => {
    7694            0 :                             // Node was dropped between planning and validation
    7695            0 :                         }
    7696            0 :                         Some(node) => {
    7697            0 :                             if !node.is_available() {
    7698            0 :                                 tracing::info!(
    7699            0 :                                     "Skipping optimization migration of {tenant_shard_id} to {new_attached_node_id} because node unavailable"
    7700              :                                 );
    7701            0 :                             } else {
    7702            0 :                                 // Accumulate optimizations that require fetching secondary status, so that we can execute these
    7703            0 :                                 // remote API requests concurrently.
    7704            0 :                                 want_secondary_status.push((
    7705            0 :                                     tenant_shard_id,
    7706            0 :                                     node.clone(),
    7707            0 :                                     optimization,
    7708            0 :                                 ));
    7709            0 :                             }
    7710              :                         }
    7711              :                     }
    7712              :                 }
    7713              :                 ScheduleOptimizationAction::ReplaceSecondary(_)
    7714              :                 | ScheduleOptimizationAction::CreateSecondary(_)
    7715              :                 | ScheduleOptimizationAction::RemoveSecondary(_) => {
    7716              :                     // No extra checks needed to manage secondaries: this does not interrupt client access
    7717            0 :                     validated_work.push((tenant_shard_id, optimization))
    7718              :                 }
    7719              :             };
    7720              :         }
    7721              : 
    7722              :         // Call into pageserver API to find out if the destination secondary location is warm enough for a reasonably smooth migration: we
    7723              :         // do this so that we avoid spawning a Reconciler that would have to wait minutes/hours for a destination to warm up: that reconciler
    7724              :         // would hold a precious reconcile semaphore unit the whole time it was waiting for the destination to warm up.
    7725            0 :         let results = self
    7726            0 :             .tenant_for_shards_api(
    7727            0 :                 want_secondary_status
    7728            0 :                     .iter()
    7729            0 :                     .map(|i| (i.0, i.1.clone()))
    7730            0 :                     .collect(),
    7731            0 :                 |tenant_shard_id, client| async move {
    7732            0 :                     client.tenant_secondary_status(tenant_shard_id).await
    7733            0 :                 },
    7734            0 :                 1,
    7735            0 :                 1,
    7736            0 :                 SHORT_RECONCILE_TIMEOUT,
    7737            0 :                 &self.cancel,
    7738            0 :             )
    7739            0 :             .await;
    7740              : 
    7741            0 :         for ((tenant_shard_id, node, optimization), secondary_status) in
    7742            0 :             want_secondary_status.into_iter().zip(results.into_iter())
    7743              :         {
    7744            0 :             match secondary_status {
    7745            0 :                 Err(e) => {
    7746            0 :                     tracing::info!(
    7747            0 :                         "Skipping migration of {tenant_shard_id} to {node}, error querying secondary: {e}"
    7748              :                     );
    7749              :                 }
    7750            0 :                 Ok(progress) => {
    7751              :                     // We require secondary locations to have less than 10GiB of downloads pending before we will use
    7752              :                     // them in an optimization
    7753              :                     const DOWNLOAD_FRESHNESS_THRESHOLD: u64 = 10 * 1024 * 1024 * 1024;
    7754              : 
    7755            0 :                     if progress.heatmap_mtime.is_none()
    7756            0 :                         || progress.bytes_total < DOWNLOAD_FRESHNESS_THRESHOLD
    7757            0 :                             && progress.bytes_downloaded != progress.bytes_total
    7758            0 :                         || progress.bytes_total - progress.bytes_downloaded
    7759            0 :                             > DOWNLOAD_FRESHNESS_THRESHOLD
    7760              :                     {
    7761            0 :                         tracing::info!(
    7762            0 :                             "Skipping migration of {tenant_shard_id} to {node} because secondary isn't ready: {progress:?}"
    7763              :                         );
    7764              : 
    7765              :                         #[cfg(feature = "testing")]
    7766            0 :                         if progress.heatmap_mtime.is_none() {
    7767              :                             // No heatmap might mean the attached location has never uploaded one, or that
    7768              :                             // the secondary download hasn't happened yet.  This is relatively unusual in the field,
    7769              :                             // but fairly common in tests.
    7770            0 :                             self.kick_secondary_download(tenant_shard_id).await;
    7771            0 :                         }
    7772              :                     } else {
    7773              :                         // Location looks ready: proceed
    7774            0 :                         tracing::info!(
    7775            0 :                             "{tenant_shard_id} secondary on {node} is warm enough for migration: {progress:?}"
    7776              :                         );
    7777            0 :                         validated_work.push((tenant_shard_id, optimization))
    7778              :                     }
    7779              :                 }
    7780              :             }
    7781              :         }
    7782              : 
    7783            0 :         validated_work
    7784            0 :     }
    7785              : 
    7786              :     /// Some aspects of scheduling optimisation wait for secondary locations to be warm.  This
    7787              :     /// happens on multi-minute timescales in the field, which is fine because optimisation is meant
    7788              :     /// to be a lazy background thing. However, when testing, it is not practical to wait around, so
    7789              :     /// we have this helper to move things along faster.
    7790              :     #[cfg(feature = "testing")]
    7791            0 :     async fn kick_secondary_download(&self, tenant_shard_id: TenantShardId) {
    7792            0 :         let (attached_node, secondaries) = {
    7793            0 :             let locked = self.inner.read().unwrap();
    7794            0 :             let Some(shard) = locked.tenants.get(&tenant_shard_id) else {
    7795            0 :                 tracing::warn!(
    7796            0 :                     "Skipping kick of secondary download for {tenant_shard_id}: not found"
    7797              :                 );
    7798            0 :                 return;
    7799              :             };
    7800              : 
    7801            0 :             let Some(attached) = shard.intent.get_attached() else {
    7802            0 :                 tracing::warn!(
    7803            0 :                     "Skipping kick of secondary download for {tenant_shard_id}: no attached"
    7804              :                 );
    7805            0 :                 return;
    7806              :             };
    7807              : 
    7808            0 :             let secondaries = shard
    7809            0 :                 .intent
    7810            0 :                 .get_secondary()
    7811            0 :                 .iter()
    7812            0 :                 .map(|n| locked.nodes.get(n).unwrap().clone())
    7813            0 :                 .collect::<Vec<_>>();
    7814            0 : 
    7815            0 :             (locked.nodes.get(attached).unwrap().clone(), secondaries)
    7816            0 :         };
    7817            0 : 
    7818            0 :         // Make remote API calls to upload + download heatmaps: we ignore errors because this is just
    7819            0 :         // a 'kick' to let scheduling optimisation run more promptly.
    7820            0 :         match attached_node
    7821            0 :             .with_client_retries(
    7822            0 :                 |client| async move { client.tenant_heatmap_upload(tenant_shard_id).await },
    7823            0 :                 &self.config.pageserver_jwt_token,
    7824            0 :                 &self.config.ssl_ca_cert,
    7825            0 :                 3,
    7826            0 :                 10,
    7827            0 :                 SHORT_RECONCILE_TIMEOUT,
    7828            0 :                 &self.cancel,
    7829            0 :             )
    7830            0 :             .await
    7831              :         {
    7832            0 :             Some(Err(e)) => {
    7833            0 :                 tracing::info!(
    7834            0 :                     "Failed to upload heatmap from {attached_node} for {tenant_shard_id}: {e}"
    7835              :                 );
    7836              :             }
    7837              :             None => {
    7838            0 :                 tracing::info!(
    7839            0 :                     "Cancelled while uploading heatmap from {attached_node} for {tenant_shard_id}"
    7840              :                 );
    7841              :             }
    7842              :             Some(Ok(_)) => {
    7843            0 :                 tracing::info!(
    7844            0 :                     "Successfully uploaded heatmap from {attached_node} for {tenant_shard_id}"
    7845              :                 );
    7846              :             }
    7847              :         }
    7848              : 
    7849            0 :         for secondary_node in secondaries {
    7850            0 :             match secondary_node
    7851            0 :                 .with_client_retries(
    7852            0 :                     |client| async move {
    7853            0 :                         client
    7854            0 :                             .tenant_secondary_download(
    7855            0 :                                 tenant_shard_id,
    7856            0 :                                 Some(Duration::from_secs(1)),
    7857            0 :                             )
    7858            0 :                             .await
    7859            0 :                     },
    7860            0 :                     &self.config.pageserver_jwt_token,
    7861            0 :                     &self.config.ssl_ca_cert,
    7862            0 :                     3,
    7863            0 :                     10,
    7864            0 :                     SHORT_RECONCILE_TIMEOUT,
    7865            0 :                     &self.cancel,
    7866            0 :                 )
    7867            0 :                 .await
    7868              :             {
    7869            0 :                 Some(Err(e)) => {
    7870            0 :                     tracing::info!(
    7871            0 :                         "Failed to download heatmap from {secondary_node} for {tenant_shard_id}: {e}"
    7872              :                     );
    7873              :                 }
    7874              :                 None => {
    7875            0 :                     tracing::info!(
    7876            0 :                         "Cancelled while downloading heatmap from {secondary_node} for {tenant_shard_id}"
    7877              :                     );
    7878              :                 }
    7879            0 :                 Some(Ok(progress)) => {
    7880            0 :                     tracing::info!(
    7881            0 :                         "Successfully downloaded heatmap from {secondary_node} for {tenant_shard_id}: {progress:?}"
    7882              :                     );
    7883              :                 }
    7884              :             }
    7885              :         }
    7886            0 :     }
    7887              : 
    7888              :     /// Asynchronously split a tenant that's eligible for automatic splits:
    7889              :     ///
    7890              :     /// * The tenant is unsharded.
    7891              :     /// * The logical size of its largest timeline exceeds split_threshold.
    7892              :     /// * The tenant's scheduling policy is active.
    7893              :     ///
    7894              :     /// At most one tenant will be split per call: the one with the largest max logical size. It
    7895              :     /// will split 1 → 8 shards.
    7896              :     ///
    7897              :     /// TODO: consider splitting based on total logical size rather than max logical size.
    7898              :     ///
    7899              :     /// TODO: consider spawning multiple splits in parallel: this is only called once every 20
    7900              :     /// seconds, so a large backlog can take a long time, and if a tenant fails to split it will
    7901              :     /// block all other splits.
    7902            0 :     async fn autosplit_tenants(self: &Arc<Self>) {
    7903            0 :         let Some(split_threshold) = self.config.split_threshold else {
    7904            0 :             return; // auto-splits are disabled
    7905              :         };
    7906            0 :         if split_threshold == 0 {
    7907            0 :             return;
    7908            0 :         }
    7909              : 
    7910              :         // Fetch the largest eligible shards by logical size.
    7911              :         const MAX_SHARDS: ShardCount = ShardCount::new(8);
    7912              : 
    7913            0 :         let mut top_n = self
    7914            0 :             .get_top_tenant_shards(&TopTenantShardsRequest {
    7915            0 :                 order_by: TenantSorting::MaxLogicalSize,
    7916            0 :                 limit: 10,
    7917            0 :                 where_shards_lt: Some(MAX_SHARDS),
    7918            0 :                 where_gt: Some(split_threshold),
    7919            0 :             })
    7920            0 :             .await;
    7921              : 
    7922              :         // Filter out tenants in a prohibiting scheduling mode.
    7923            0 :         {
    7924            0 :             let state = self.inner.read().unwrap();
    7925            0 :             top_n.retain(|i| {
    7926            0 :                 let policy = state.tenants.get(&i.id).map(|s| s.get_scheduling_policy());
    7927            0 :                 policy == Some(ShardSchedulingPolicy::Active)
    7928            0 :             });
    7929            0 :         }
    7930              : 
    7931            0 :         let Some(split_candidate) = top_n.into_iter().next() else {
    7932            0 :             debug!("No split-elegible shards found");
    7933            0 :             return;
    7934              :         };
    7935              : 
    7936              :         // We spawn a task to run this, so it's exactly like some external API client requesting it.
    7937              :         // We don't want to block the background reconcile loop on this.
    7938            0 :         info!(
    7939            0 :             "Auto-splitting tenant for size threshold {split_threshold}: current size {split_candidate:?}"
    7940              :         );
    7941              : 
    7942            0 :         let this = self.clone();
    7943            0 :         tokio::spawn(
    7944            0 :             async move {
    7945            0 :                 match this
    7946            0 :                     .tenant_shard_split(
    7947            0 :                         split_candidate.id.tenant_id,
    7948            0 :                         TenantShardSplitRequest {
    7949            0 :                             // Always split to the max number of shards: this avoids stepping
    7950            0 :                             // through intervening shard counts and encountering the overhead of a
    7951            0 :                             // split+cleanup each time as a tenant grows, and is not too expensive
    7952            0 :                             // because our max shard count is relatively low anyway. This policy
    7953            0 :                             // will be adjusted in future once we support higher shard count.
    7954            0 :                             new_shard_count: MAX_SHARDS.literal(),
    7955            0 :                             new_stripe_size: Some(ShardParameters::DEFAULT_STRIPE_SIZE),
    7956            0 :                         },
    7957            0 :                     )
    7958            0 :                     .await
    7959              :                 {
    7960            0 :                     Ok(_) => info!("Successful auto-split"),
    7961            0 :                     Err(err) => error!("Auto-split failed: {err}"),
    7962              :                 }
    7963            0 :             }
    7964            0 :             .instrument(info_span!("auto_split", tenant_id=%split_candidate.id.tenant_id)),
    7965              :         );
    7966            0 :     }
    7967              : 
    7968              :     /// Fetches the top tenant shards from every node, in descending order of
    7969              :     /// max logical size. Any node errors will be logged and ignored.
    7970            0 :     async fn get_top_tenant_shards(
    7971            0 :         &self,
    7972            0 :         request: &TopTenantShardsRequest,
    7973            0 :     ) -> Vec<TopTenantShardItem> {
    7974            0 :         let nodes = self
    7975            0 :             .inner
    7976            0 :             .read()
    7977            0 :             .unwrap()
    7978            0 :             .nodes
    7979            0 :             .values()
    7980            0 :             .cloned()
    7981            0 :             .collect_vec();
    7982            0 : 
    7983            0 :         let mut futures = FuturesUnordered::new();
    7984            0 :         for node in nodes {
    7985            0 :             futures.push(async move {
    7986            0 :                 node.with_client_retries(
    7987            0 :                     |client| async move { client.top_tenant_shards(request.clone()).await },
    7988            0 :                     &self.config.pageserver_jwt_token,
    7989            0 :                     &self.config.ssl_ca_cert,
    7990            0 :                     3,
    7991            0 :                     3,
    7992            0 :                     Duration::from_secs(5),
    7993            0 :                     &self.cancel,
    7994            0 :                 )
    7995            0 :                 .await
    7996            0 :             });
    7997            0 :         }
    7998              : 
    7999            0 :         let mut top = Vec::new();
    8000            0 :         while let Some(output) = futures.next().await {
    8001            0 :             match output {
    8002            0 :                 Some(Ok(response)) => top.extend(response.shards),
    8003            0 :                 Some(Err(mgmt_api::Error::Cancelled)) => {}
    8004            0 :                 Some(Err(err)) => warn!("failed to fetch top tenants: {err}"),
    8005            0 :                 None => {} // node is shutting down
    8006              :             }
    8007              :         }
    8008              : 
    8009            0 :         top.sort_by_key(|i| i.max_logical_size);
    8010            0 :         top.reverse();
    8011            0 :         top
    8012            0 :     }
    8013              : 
    8014              :     /// Useful for tests: run whatever work a background [`Self::reconcile_all`] would have done, but
    8015              :     /// also wait for any generated Reconcilers to complete.  Calling this until it returns zero should
    8016              :     /// put the system into a quiescent state where future background reconciliations won't do anything.
    8017            0 :     pub(crate) async fn reconcile_all_now(&self) -> Result<usize, ReconcileWaitError> {
    8018            0 :         let reconciles_spawned = self.reconcile_all();
    8019            0 :         let reconciles_spawned = if reconciles_spawned == 0 {
    8020              :             // Only optimize when we are otherwise idle
    8021            0 :             self.optimize_all().await
    8022              :         } else {
    8023            0 :             reconciles_spawned
    8024              :         };
    8025              : 
    8026            0 :         let waiters = {
    8027            0 :             let mut waiters = Vec::new();
    8028            0 :             let locked = self.inner.read().unwrap();
    8029            0 :             for (_tenant_shard_id, shard) in locked.tenants.iter() {
    8030            0 :                 if let Some(waiter) = shard.get_waiter() {
    8031            0 :                     waiters.push(waiter);
    8032            0 :                 }
    8033              :             }
    8034            0 :             waiters
    8035            0 :         };
    8036            0 : 
    8037            0 :         let waiter_count = waiters.len();
    8038            0 :         match self.await_waiters(waiters, RECONCILE_TIMEOUT).await {
    8039            0 :             Ok(()) => {}
    8040            0 :             Err(ReconcileWaitError::Failed(_, reconcile_error))
    8041            0 :                 if matches!(*reconcile_error, ReconcileError::Cancel) =>
    8042            0 :             {
    8043            0 :                 // Ignore reconciler cancel errors: this reconciler might have shut down
    8044            0 :                 // because some other change superceded it.  We will return a nonzero number,
    8045            0 :                 // so the caller knows they might have to call again to quiesce the system.
    8046            0 :             }
    8047            0 :             Err(e) => {
    8048            0 :                 return Err(e);
    8049              :             }
    8050              :         };
    8051              : 
    8052            0 :         tracing::info!(
    8053            0 :             "{} reconciles in reconcile_all, {} waiters",
    8054              :             reconciles_spawned,
    8055              :             waiter_count
    8056              :         );
    8057              : 
    8058            0 :         Ok(std::cmp::max(waiter_count, reconciles_spawned))
    8059            0 :     }
    8060              : 
    8061            0 :     async fn stop_reconciliations(&self, reason: StopReconciliationsReason) {
    8062            0 :         // Cancel all on-going reconciles and wait for them to exit the gate.
    8063            0 :         tracing::info!("{reason}: cancelling and waiting for in-flight reconciles");
    8064            0 :         self.reconcilers_cancel.cancel();
    8065            0 :         self.reconcilers_gate.close().await;
    8066              : 
    8067              :         // Signal the background loop in [`Service::process_results`] to exit once
    8068              :         // it has proccessed the results from all the reconciles we cancelled earlier.
    8069            0 :         tracing::info!("{reason}: processing results from previously in-flight reconciles");
    8070            0 :         self.result_tx.send(ReconcileResultRequest::Stop).ok();
    8071            0 :         self.result_tx.closed().await;
    8072            0 :     }
    8073              : 
    8074            0 :     pub async fn shutdown(&self) {
    8075            0 :         self.stop_reconciliations(StopReconciliationsReason::ShuttingDown)
    8076            0 :             .await;
    8077              : 
    8078              :         // Background tasks hold gate guards: this notifies them of the cancellation and
    8079              :         // waits for them all to complete.
    8080            0 :         tracing::info!("Shutting down: cancelling and waiting for background tasks to exit");
    8081            0 :         self.cancel.cancel();
    8082            0 :         self.gate.close().await;
    8083            0 :     }
    8084              : 
    8085              :     /// Spot check the download lag for a secondary location of a shard.
    8086              :     /// Should be used as a heuristic, since it's not always precise: the
    8087              :     /// secondary might have not downloaded the new heat map yet and, hence,
    8088              :     /// is not aware of the lag.
    8089              :     ///
    8090              :     /// Returns:
    8091              :     /// * Ok(None) if the lag could not be determined from the status,
    8092              :     /// * Ok(Some(_)) if the lag could be determind
    8093              :     /// * Err on failures to query the pageserver.
    8094            0 :     async fn secondary_lag(
    8095            0 :         &self,
    8096            0 :         secondary: &NodeId,
    8097            0 :         tenant_shard_id: TenantShardId,
    8098            0 :     ) -> Result<Option<u64>, mgmt_api::Error> {
    8099            0 :         let nodes = self.inner.read().unwrap().nodes.clone();
    8100            0 :         let node = nodes.get(secondary).ok_or(mgmt_api::Error::ApiError(
    8101            0 :             StatusCode::NOT_FOUND,
    8102            0 :             format!("Node with id {} not found", secondary),
    8103            0 :         ))?;
    8104              : 
    8105            0 :         match node
    8106            0 :             .with_client_retries(
    8107            0 :                 |client| async move { client.tenant_secondary_status(tenant_shard_id).await },
    8108            0 :                 &self.config.pageserver_jwt_token,
    8109            0 :                 &self.config.ssl_ca_cert,
    8110            0 :                 1,
    8111            0 :                 3,
    8112            0 :                 Duration::from_millis(250),
    8113            0 :                 &self.cancel,
    8114            0 :             )
    8115            0 :             .await
    8116              :         {
    8117            0 :             Some(Ok(status)) => match status.heatmap_mtime {
    8118            0 :                 Some(_) => Ok(Some(status.bytes_total - status.bytes_downloaded)),
    8119            0 :                 None => Ok(None),
    8120              :             },
    8121            0 :             Some(Err(e)) => Err(e),
    8122            0 :             None => Err(mgmt_api::Error::Cancelled),
    8123              :         }
    8124            0 :     }
    8125              : 
    8126              :     /// Drain a node by moving the shards attached to it as primaries.
    8127              :     /// This is a long running operation and it should run as a separate Tokio task.
    8128            0 :     pub(crate) async fn drain_node(
    8129            0 :         self: &Arc<Self>,
    8130            0 :         node_id: NodeId,
    8131            0 :         cancel: CancellationToken,
    8132            0 :     ) -> Result<(), OperationError> {
    8133              :         const MAX_SECONDARY_LAG_BYTES_DEFAULT: u64 = 256 * 1024 * 1024;
    8134            0 :         let max_secondary_lag_bytes = self
    8135            0 :             .config
    8136            0 :             .max_secondary_lag_bytes
    8137            0 :             .unwrap_or(MAX_SECONDARY_LAG_BYTES_DEFAULT);
    8138              : 
    8139              :         // By default, live migrations are generous about the wait time for getting
    8140              :         // the secondary location up to speed. When draining, give up earlier in order
    8141              :         // to not stall the operation when a cold secondary is encountered.
    8142              :         const SECONDARY_WARMUP_TIMEOUT: Duration = Duration::from_secs(20);
    8143              :         const SECONDARY_DOWNLOAD_REQUEST_TIMEOUT: Duration = Duration::from_secs(5);
    8144            0 :         let reconciler_config = ReconcilerConfigBuilder::new(ReconcilerPriority::Normal)
    8145            0 :             .secondary_warmup_timeout(SECONDARY_WARMUP_TIMEOUT)
    8146            0 :             .secondary_download_request_timeout(SECONDARY_DOWNLOAD_REQUEST_TIMEOUT)
    8147            0 :             .build();
    8148            0 : 
    8149            0 :         let mut waiters = Vec::new();
    8150            0 : 
    8151            0 :         let mut tid_iter = TenantShardIterator::new({
    8152            0 :             let service = self.clone();
    8153            0 :             move |last_inspected_shard: Option<TenantShardId>| {
    8154            0 :                 let locked = &service.inner.read().unwrap();
    8155            0 :                 let tenants = &locked.tenants;
    8156            0 :                 let entry = match last_inspected_shard {
    8157            0 :                     Some(skip_past) => {
    8158            0 :                         // Skip to the last seen tenant shard id
    8159            0 :                         let mut cursor = tenants.iter().skip_while(|(tid, _)| **tid != skip_past);
    8160            0 : 
    8161            0 :                         // Skip past the last seen
    8162            0 :                         cursor.nth(1)
    8163              :                     }
    8164            0 :                     None => tenants.first_key_value(),
    8165              :                 };
    8166              : 
    8167            0 :                 entry.map(|(tid, _)| tid).copied()
    8168            0 :             }
    8169            0 :         });
    8170              : 
    8171            0 :         while !tid_iter.finished() {
    8172            0 :             if cancel.is_cancelled() {
    8173            0 :                 match self
    8174            0 :                     .node_configure(node_id, None, Some(NodeSchedulingPolicy::Active))
    8175            0 :                     .await
    8176              :                 {
    8177            0 :                     Ok(()) => return Err(OperationError::Cancelled),
    8178            0 :                     Err(err) => {
    8179            0 :                         return Err(OperationError::FinalizeError(
    8180            0 :                             format!(
    8181            0 :                                 "Failed to finalise drain cancel of {} by setting scheduling policy to Active: {}",
    8182            0 :                                 node_id, err
    8183            0 :                             )
    8184            0 :                             .into(),
    8185            0 :                         ));
    8186              :                     }
    8187              :                 }
    8188            0 :             }
    8189            0 : 
    8190            0 :             drain_utils::validate_node_state(&node_id, self.inner.read().unwrap().nodes.clone())?;
    8191              : 
    8192            0 :             while waiters.len() < MAX_RECONCILES_PER_OPERATION {
    8193            0 :                 let tid = match tid_iter.next() {
    8194            0 :                     Some(tid) => tid,
    8195              :                     None => {
    8196            0 :                         break;
    8197              :                     }
    8198              :                 };
    8199              : 
    8200            0 :                 let tid_drain = TenantShardDrain {
    8201            0 :                     drained_node: node_id,
    8202            0 :                     tenant_shard_id: tid,
    8203            0 :                 };
    8204              : 
    8205            0 :                 let dest_node_id = {
    8206            0 :                     let locked = self.inner.read().unwrap();
    8207            0 : 
    8208            0 :                     match tid_drain
    8209            0 :                         .tenant_shard_eligible_for_drain(&locked.tenants, &locked.scheduler)
    8210              :                     {
    8211            0 :                         Some(node_id) => node_id,
    8212              :                         None => {
    8213            0 :                             continue;
    8214              :                         }
    8215              :                     }
    8216              :                 };
    8217              : 
    8218            0 :                 match self.secondary_lag(&dest_node_id, tid).await {
    8219            0 :                     Ok(Some(lag)) if lag <= max_secondary_lag_bytes => {
    8220            0 :                         // The secondary is reasonably up to date.
    8221            0 :                         // Migrate to it
    8222            0 :                     }
    8223            0 :                     Ok(Some(lag)) => {
    8224            0 :                         tracing::info!(
    8225            0 :                             tenant_id=%tid.tenant_id, shard_id=%tid.shard_slug(),
    8226            0 :                             "Secondary on node {dest_node_id} is lagging by {lag}. Skipping reconcile."
    8227              :                         );
    8228            0 :                         continue;
    8229              :                     }
    8230              :                     Ok(None) => {
    8231            0 :                         tracing::info!(
    8232            0 :                             tenant_id=%tid.tenant_id, shard_id=%tid.shard_slug(),
    8233            0 :                             "Could not determine lag for secondary on node {dest_node_id}. Skipping reconcile."
    8234              :                         );
    8235            0 :                         continue;
    8236              :                     }
    8237            0 :                     Err(err) => {
    8238            0 :                         tracing::warn!(
    8239            0 :                             tenant_id=%tid.tenant_id, shard_id=%tid.shard_slug(),
    8240            0 :                             "Failed to get secondary lag from node {dest_node_id}. Skipping reconcile: {err}"
    8241              :                         );
    8242            0 :                         continue;
    8243              :                     }
    8244              :                 }
    8245              : 
    8246              :                 {
    8247            0 :                     let mut locked = self.inner.write().unwrap();
    8248            0 :                     let (nodes, tenants, scheduler) = locked.parts_mut();
    8249            0 :                     let rescheduled = tid_drain.reschedule_to_secondary(
    8250            0 :                         dest_node_id,
    8251            0 :                         tenants,
    8252            0 :                         scheduler,
    8253            0 :                         nodes,
    8254            0 :                     )?;
    8255              : 
    8256            0 :                     if let Some(tenant_shard) = rescheduled {
    8257            0 :                         let waiter = self.maybe_configured_reconcile_shard(
    8258            0 :                             tenant_shard,
    8259            0 :                             nodes,
    8260            0 :                             reconciler_config,
    8261            0 :                         );
    8262            0 :                         if let Some(some) = waiter {
    8263            0 :                             waiters.push(some);
    8264            0 :                         }
    8265            0 :                     }
    8266              :                 }
    8267              :             }
    8268              : 
    8269            0 :             waiters = self
    8270            0 :                 .await_waiters_remainder(waiters, WAITER_FILL_DRAIN_POLL_TIMEOUT)
    8271            0 :                 .await;
    8272              : 
    8273            0 :             failpoint_support::sleep_millis_async!("sleepy-drain-loop", &cancel);
    8274              :         }
    8275              : 
    8276            0 :         while !waiters.is_empty() {
    8277            0 :             if cancel.is_cancelled() {
    8278            0 :                 match self
    8279            0 :                     .node_configure(node_id, None, Some(NodeSchedulingPolicy::Active))
    8280            0 :                     .await
    8281              :                 {
    8282            0 :                     Ok(()) => return Err(OperationError::Cancelled),
    8283            0 :                     Err(err) => {
    8284            0 :                         return Err(OperationError::FinalizeError(
    8285            0 :                             format!(
    8286            0 :                                 "Failed to finalise drain cancel of {} by setting scheduling policy to Active: {}",
    8287            0 :                                 node_id, err
    8288            0 :                             )
    8289            0 :                             .into(),
    8290            0 :                         ));
    8291              :                     }
    8292              :                 }
    8293            0 :             }
    8294            0 : 
    8295            0 :             tracing::info!("Awaiting {} pending drain reconciliations", waiters.len());
    8296              : 
    8297            0 :             waiters = self
    8298            0 :                 .await_waiters_remainder(waiters, SHORT_RECONCILE_TIMEOUT)
    8299            0 :                 .await;
    8300              :         }
    8301              : 
    8302              :         // At this point we have done the best we could to drain shards from this node.
    8303              :         // Set the node scheduling policy to `[NodeSchedulingPolicy::PauseForRestart]`
    8304              :         // to complete the drain.
    8305            0 :         if let Err(err) = self
    8306            0 :             .node_configure(node_id, None, Some(NodeSchedulingPolicy::PauseForRestart))
    8307            0 :             .await
    8308              :         {
    8309              :             // This is not fatal. Anything that is polling the node scheduling policy to detect
    8310              :             // the end of the drain operations will hang, but all such places should enforce an
    8311              :             // overall timeout. The scheduling policy will be updated upon node re-attach and/or
    8312              :             // by the counterpart fill operation.
    8313            0 :             return Err(OperationError::FinalizeError(
    8314            0 :                 format!(
    8315            0 :                     "Failed to finalise drain of {node_id} by setting scheduling policy to PauseForRestart: {err}"
    8316            0 :                 )
    8317            0 :                 .into(),
    8318            0 :             ));
    8319            0 :         }
    8320            0 : 
    8321            0 :         Ok(())
    8322            0 :     }
    8323              : 
    8324              :     /// Create a node fill plan (pick secondaries to promote), based on:
    8325              :     /// 1. Shards which have a secondary on this node, and this node is in their home AZ, and are currently attached to a node
    8326              :     ///    outside their home AZ, should be migrated back here.
    8327              :     /// 2. If after step 1 we have not migrated enough shards for this node to have its fair share of
    8328              :     ///    attached shards, we will promote more shards from the nodes with the most attached shards, unless
    8329              :     ///    those shards have a home AZ that doesn't match the node we're filling.
    8330            0 :     fn fill_node_plan(&self, node_id: NodeId) -> Vec<TenantShardId> {
    8331            0 :         let mut locked = self.inner.write().unwrap();
    8332            0 :         let (nodes, tenants, _scheduler) = locked.parts_mut();
    8333            0 : 
    8334            0 :         let node_az = nodes
    8335            0 :             .get(&node_id)
    8336            0 :             .expect("Node must exist")
    8337            0 :             .get_availability_zone_id()
    8338            0 :             .clone();
    8339            0 : 
    8340            0 :         // The tenant shard IDs that we plan to promote from secondary to attached on this node
    8341            0 :         let mut plan = Vec::new();
    8342            0 : 
    8343            0 :         // Collect shards which do not have a preferred AZ & are elegible for moving in stage 2
    8344            0 :         let mut free_tids_by_node: HashMap<NodeId, Vec<TenantShardId>> = HashMap::new();
    8345            0 : 
    8346            0 :         // Don't respect AZ preferences if there is only one AZ.  This comes up in tests, but it could
    8347            0 :         // conceivably come up in real life if deploying a single-AZ region intentionally.
    8348            0 :         let respect_azs = nodes
    8349            0 :             .values()
    8350            0 :             .map(|n| n.get_availability_zone_id())
    8351            0 :             .unique()
    8352            0 :             .count()
    8353            0 :             > 1;
    8354              : 
    8355              :         // Step 1: collect all shards that we are required to migrate back to this node because their AZ preference
    8356              :         // requires it.
    8357            0 :         for (tsid, tenant_shard) in tenants {
    8358            0 :             if !tenant_shard.intent.get_secondary().contains(&node_id) {
    8359              :                 // Shard doesn't have a secondary on this node, ignore it.
    8360            0 :                 continue;
    8361            0 :             }
    8362            0 : 
    8363            0 :             // AZ check: when filling nodes after a restart, our intent is to move _back_ the
    8364            0 :             // shards which belong on this node, not to promote shards whose scheduling preference
    8365            0 :             // would be on their currently attached node.  So will avoid promoting shards whose
    8366            0 :             // home AZ doesn't match the AZ of the node we're filling.
    8367            0 :             match tenant_shard.preferred_az() {
    8368            0 :                 _ if !respect_azs => {
    8369            0 :                     if let Some(primary) = tenant_shard.intent.get_attached() {
    8370            0 :                         free_tids_by_node.entry(*primary).or_default().push(*tsid);
    8371            0 :                     }
    8372              :                 }
    8373              :                 None => {
    8374              :                     // Shard doesn't have an AZ preference: it is elegible to be moved, but we
    8375              :                     // will only do so if our target shard count requires it.
    8376            0 :                     if let Some(primary) = tenant_shard.intent.get_attached() {
    8377            0 :                         free_tids_by_node.entry(*primary).or_default().push(*tsid);
    8378            0 :                     }
    8379              :                 }
    8380            0 :                 Some(az) if az == &node_az => {
    8381              :                     // This shard's home AZ is equal to the node we're filling: it should
    8382              :                     // be moved back to this node as part of filling, unless its currently
    8383              :                     // attached location is also in its home AZ.
    8384            0 :                     if let Some(primary) = tenant_shard.intent.get_attached() {
    8385            0 :                         if nodes
    8386            0 :                             .get(primary)
    8387            0 :                             .expect("referenced node must exist")
    8388            0 :                             .get_availability_zone_id()
    8389            0 :                             != tenant_shard
    8390            0 :                                 .preferred_az()
    8391            0 :                                 .expect("tenant must have an AZ preference")
    8392              :                         {
    8393            0 :                             plan.push(*tsid)
    8394            0 :                         }
    8395              :                     } else {
    8396            0 :                         plan.push(*tsid)
    8397              :                     }
    8398              :                 }
    8399            0 :                 Some(_) => {
    8400            0 :                     // This shard's home AZ is somewhere other than the node we're filling,
    8401            0 :                     // it may not be moved back to this node as part of filling.  Ignore it
    8402            0 :                 }
    8403              :             }
    8404              :         }
    8405              : 
    8406              :         // Step 2: also promote any AZ-agnostic shards as required to achieve the target number of attachments
    8407            0 :         let fill_requirement = locked.scheduler.compute_fill_requirement(node_id);
    8408            0 : 
    8409            0 :         let expected_attached = locked.scheduler.expected_attached_shard_count();
    8410            0 :         let nodes_by_load = locked.scheduler.nodes_by_attached_shard_count();
    8411            0 : 
    8412            0 :         let mut promoted_per_tenant: HashMap<TenantId, usize> = HashMap::new();
    8413              : 
    8414            0 :         for (node_id, attached) in nodes_by_load {
    8415            0 :             let available = locked.nodes.get(&node_id).is_some_and(|n| n.is_available());
    8416            0 :             if !available {
    8417            0 :                 continue;
    8418            0 :             }
    8419            0 : 
    8420            0 :             if plan.len() >= fill_requirement
    8421            0 :                 || free_tids_by_node.is_empty()
    8422            0 :                 || attached <= expected_attached
    8423              :             {
    8424            0 :                 break;
    8425            0 :             }
    8426            0 : 
    8427            0 :             let can_take = attached - expected_attached;
    8428            0 :             let needed = fill_requirement - plan.len();
    8429            0 :             let mut take = std::cmp::min(can_take, needed);
    8430            0 : 
    8431            0 :             let mut remove_node = false;
    8432            0 :             while take > 0 {
    8433            0 :                 match free_tids_by_node.get_mut(&node_id) {
    8434            0 :                     Some(tids) => match tids.pop() {
    8435            0 :                         Some(tid) => {
    8436            0 :                             let max_promote_for_tenant = std::cmp::max(
    8437            0 :                                 tid.shard_count.count() as usize / locked.nodes.len(),
    8438            0 :                                 1,
    8439            0 :                             );
    8440            0 :                             let promoted = promoted_per_tenant.entry(tid.tenant_id).or_default();
    8441            0 :                             if *promoted < max_promote_for_tenant {
    8442            0 :                                 plan.push(tid);
    8443            0 :                                 *promoted += 1;
    8444            0 :                                 take -= 1;
    8445            0 :                             }
    8446              :                         }
    8447              :                         None => {
    8448            0 :                             remove_node = true;
    8449            0 :                             break;
    8450              :                         }
    8451              :                     },
    8452              :                     None => {
    8453            0 :                         break;
    8454              :                     }
    8455              :                 }
    8456              :             }
    8457              : 
    8458            0 :             if remove_node {
    8459            0 :                 free_tids_by_node.remove(&node_id);
    8460            0 :             }
    8461              :         }
    8462              : 
    8463            0 :         plan
    8464            0 :     }
    8465              : 
    8466              :     /// Fill a node by promoting its secondaries until the cluster is balanced
    8467              :     /// with regards to attached shard counts. Note that this operation only
    8468              :     /// makes sense as a counterpart to the drain implemented in [`Service::drain_node`].
    8469              :     /// This is a long running operation and it should run as a separate Tokio task.
    8470            0 :     pub(crate) async fn fill_node(
    8471            0 :         &self,
    8472            0 :         node_id: NodeId,
    8473            0 :         cancel: CancellationToken,
    8474            0 :     ) -> Result<(), OperationError> {
    8475              :         const SECONDARY_WARMUP_TIMEOUT: Duration = Duration::from_secs(20);
    8476              :         const SECONDARY_DOWNLOAD_REQUEST_TIMEOUT: Duration = Duration::from_secs(5);
    8477            0 :         let reconciler_config = ReconcilerConfigBuilder::new(ReconcilerPriority::Normal)
    8478            0 :             .secondary_warmup_timeout(SECONDARY_WARMUP_TIMEOUT)
    8479            0 :             .secondary_download_request_timeout(SECONDARY_DOWNLOAD_REQUEST_TIMEOUT)
    8480            0 :             .build();
    8481            0 : 
    8482            0 :         let mut tids_to_promote = self.fill_node_plan(node_id);
    8483            0 :         let mut waiters = Vec::new();
    8484              : 
    8485              :         // Execute the plan we've composed above. Before aplying each move from the plan,
    8486              :         // we validate to ensure that it has not gone stale in the meantime.
    8487            0 :         while !tids_to_promote.is_empty() {
    8488            0 :             if cancel.is_cancelled() {
    8489            0 :                 match self
    8490            0 :                     .node_configure(node_id, None, Some(NodeSchedulingPolicy::Active))
    8491            0 :                     .await
    8492              :                 {
    8493            0 :                     Ok(()) => return Err(OperationError::Cancelled),
    8494            0 :                     Err(err) => {
    8495            0 :                         return Err(OperationError::FinalizeError(
    8496            0 :                             format!(
    8497            0 :                                 "Failed to finalise drain cancel of {} by setting scheduling policy to Active: {}",
    8498            0 :                                 node_id, err
    8499            0 :                             )
    8500            0 :                             .into(),
    8501            0 :                         ));
    8502              :                     }
    8503              :                 }
    8504            0 :             }
    8505            0 : 
    8506            0 :             {
    8507            0 :                 let mut locked = self.inner.write().unwrap();
    8508            0 :                 let (nodes, tenants, scheduler) = locked.parts_mut();
    8509              : 
    8510            0 :                 let node = nodes.get(&node_id).ok_or(OperationError::NodeStateChanged(
    8511            0 :                     format!("node {node_id} was removed").into(),
    8512            0 :                 ))?;
    8513              : 
    8514            0 :                 let current_policy = node.get_scheduling();
    8515            0 :                 if !matches!(current_policy, NodeSchedulingPolicy::Filling) {
    8516              :                     // TODO(vlad): maybe cancel pending reconciles before erroring out. need to think
    8517              :                     // about it
    8518            0 :                     return Err(OperationError::NodeStateChanged(
    8519            0 :                         format!("node {node_id} changed state to {current_policy:?}").into(),
    8520            0 :                     ));
    8521            0 :                 }
    8522              : 
    8523            0 :                 while waiters.len() < MAX_RECONCILES_PER_OPERATION {
    8524            0 :                     if let Some(tid) = tids_to_promote.pop() {
    8525            0 :                         if let Some(tenant_shard) = tenants.get_mut(&tid) {
    8526              :                             // If the node being filled is not a secondary anymore,
    8527              :                             // skip the promotion.
    8528            0 :                             if !tenant_shard.intent.get_secondary().contains(&node_id) {
    8529            0 :                                 continue;
    8530            0 :                             }
    8531            0 : 
    8532            0 :                             let previously_attached_to = *tenant_shard.intent.get_attached();
    8533            0 :                             match tenant_shard.reschedule_to_secondary(Some(node_id), scheduler) {
    8534            0 :                                 Err(e) => {
    8535            0 :                                     tracing::warn!(
    8536            0 :                                         tenant_id=%tid.tenant_id, shard_id=%tid.shard_slug(),
    8537            0 :                                         "Scheduling error when filling pageserver {} : {e}", node_id
    8538              :                                     );
    8539              :                                 }
    8540              :                                 Ok(()) => {
    8541            0 :                                     tracing::info!(
    8542            0 :                                         tenant_id=%tid.tenant_id, shard_id=%tid.shard_slug(),
    8543            0 :                                         "Rescheduled shard while filling node {}: {:?} -> {}",
    8544              :                                         node_id,
    8545              :                                         previously_attached_to,
    8546              :                                         node_id
    8547              :                                     );
    8548              : 
    8549            0 :                                     if let Some(waiter) = self.maybe_configured_reconcile_shard(
    8550            0 :                                         tenant_shard,
    8551            0 :                                         nodes,
    8552            0 :                                         reconciler_config,
    8553            0 :                                     ) {
    8554            0 :                                         waiters.push(waiter);
    8555            0 :                                     }
    8556              :                                 }
    8557              :                             }
    8558            0 :                         }
    8559              :                     } else {
    8560            0 :                         break;
    8561              :                     }
    8562              :                 }
    8563              :             }
    8564              : 
    8565            0 :             waiters = self
    8566            0 :                 .await_waiters_remainder(waiters, WAITER_FILL_DRAIN_POLL_TIMEOUT)
    8567            0 :                 .await;
    8568              :         }
    8569              : 
    8570            0 :         while !waiters.is_empty() {
    8571            0 :             if cancel.is_cancelled() {
    8572            0 :                 match self
    8573            0 :                     .node_configure(node_id, None, Some(NodeSchedulingPolicy::Active))
    8574            0 :                     .await
    8575              :                 {
    8576            0 :                     Ok(()) => return Err(OperationError::Cancelled),
    8577            0 :                     Err(err) => {
    8578            0 :                         return Err(OperationError::FinalizeError(
    8579            0 :                             format!(
    8580            0 :                                 "Failed to finalise drain cancel of {} by setting scheduling policy to Active: {}",
    8581            0 :                                 node_id, err
    8582            0 :                             )
    8583            0 :                             .into(),
    8584            0 :                         ));
    8585              :                     }
    8586              :                 }
    8587            0 :             }
    8588            0 : 
    8589            0 :             tracing::info!("Awaiting {} pending fill reconciliations", waiters.len());
    8590              : 
    8591            0 :             waiters = self
    8592            0 :                 .await_waiters_remainder(waiters, SHORT_RECONCILE_TIMEOUT)
    8593            0 :                 .await;
    8594              :         }
    8595              : 
    8596            0 :         if let Err(err) = self
    8597            0 :             .node_configure(node_id, None, Some(NodeSchedulingPolicy::Active))
    8598            0 :             .await
    8599              :         {
    8600              :             // This isn't a huge issue since the filling process starts upon request. However, it
    8601              :             // will prevent the next drain from starting. The only case in which this can fail
    8602              :             // is database unavailability. Such a case will require manual intervention.
    8603            0 :             return Err(OperationError::FinalizeError(
    8604            0 :                 format!("Failed to finalise fill of {node_id} by setting scheduling policy to Active: {err}")
    8605            0 :                     .into(),
    8606            0 :             ));
    8607            0 :         }
    8608            0 : 
    8609            0 :         Ok(())
    8610            0 :     }
    8611              : 
    8612              :     /// Updates scrubber metadata health check results.
    8613            0 :     pub(crate) async fn metadata_health_update(
    8614            0 :         &self,
    8615            0 :         update_req: MetadataHealthUpdateRequest,
    8616            0 :     ) -> Result<(), ApiError> {
    8617            0 :         let now = chrono::offset::Utc::now();
    8618            0 :         let (healthy_records, unhealthy_records) = {
    8619            0 :             let locked = self.inner.read().unwrap();
    8620            0 :             let healthy_records = update_req
    8621            0 :                 .healthy_tenant_shards
    8622            0 :                 .into_iter()
    8623            0 :                 // Retain only health records associated with tenant shards managed by storage controller.
    8624            0 :                 .filter(|tenant_shard_id| locked.tenants.contains_key(tenant_shard_id))
    8625            0 :                 .map(|tenant_shard_id| MetadataHealthPersistence::new(tenant_shard_id, true, now))
    8626            0 :                 .collect();
    8627            0 :             let unhealthy_records = update_req
    8628            0 :                 .unhealthy_tenant_shards
    8629            0 :                 .into_iter()
    8630            0 :                 .filter(|tenant_shard_id| locked.tenants.contains_key(tenant_shard_id))
    8631            0 :                 .map(|tenant_shard_id| MetadataHealthPersistence::new(tenant_shard_id, false, now))
    8632            0 :                 .collect();
    8633            0 : 
    8634            0 :             (healthy_records, unhealthy_records)
    8635            0 :         };
    8636            0 : 
    8637            0 :         self.persistence
    8638            0 :             .update_metadata_health_records(healthy_records, unhealthy_records, now)
    8639            0 :             .await?;
    8640            0 :         Ok(())
    8641            0 :     }
    8642              : 
    8643              :     /// Lists the tenant shards that has unhealthy metadata status.
    8644            0 :     pub(crate) async fn metadata_health_list_unhealthy(
    8645            0 :         &self,
    8646            0 :     ) -> Result<Vec<TenantShardId>, ApiError> {
    8647            0 :         let result = self
    8648            0 :             .persistence
    8649            0 :             .list_unhealthy_metadata_health_records()
    8650            0 :             .await?
    8651            0 :             .iter()
    8652            0 :             .map(|p| p.get_tenant_shard_id().unwrap())
    8653            0 :             .collect();
    8654            0 : 
    8655            0 :         Ok(result)
    8656            0 :     }
    8657              : 
    8658              :     /// Lists the tenant shards that have not been scrubbed for some duration.
    8659            0 :     pub(crate) async fn metadata_health_list_outdated(
    8660            0 :         &self,
    8661            0 :         not_scrubbed_for: Duration,
    8662            0 :     ) -> Result<Vec<MetadataHealthRecord>, ApiError> {
    8663            0 :         let earlier = chrono::offset::Utc::now() - not_scrubbed_for;
    8664            0 :         let result = self
    8665            0 :             .persistence
    8666            0 :             .list_outdated_metadata_health_records(earlier)
    8667            0 :             .await?
    8668            0 :             .into_iter()
    8669            0 :             .map(|record| record.into())
    8670            0 :             .collect();
    8671            0 :         Ok(result)
    8672            0 :     }
    8673              : 
    8674            0 :     pub(crate) fn get_leadership_status(&self) -> LeadershipStatus {
    8675            0 :         self.inner.read().unwrap().get_leadership_status()
    8676            0 :     }
    8677              : 
    8678            0 :     pub(crate) async fn step_down(&self) -> GlobalObservedState {
    8679            0 :         tracing::info!("Received step down request from peer");
    8680            0 :         failpoint_support::sleep_millis_async!("sleep-on-step-down-handling");
    8681              : 
    8682            0 :         self.inner.write().unwrap().step_down();
    8683            0 :         // TODO: would it make sense to have a time-out for this?
    8684            0 :         self.stop_reconciliations(StopReconciliationsReason::SteppingDown)
    8685            0 :             .await;
    8686              : 
    8687            0 :         let mut global_observed = GlobalObservedState::default();
    8688            0 :         let locked = self.inner.read().unwrap();
    8689            0 :         for (tid, tenant_shard) in locked.tenants.iter() {
    8690            0 :             global_observed
    8691            0 :                 .0
    8692            0 :                 .insert(*tid, tenant_shard.observed.clone());
    8693            0 :         }
    8694              : 
    8695            0 :         global_observed
    8696            0 :     }
    8697              : 
    8698              :     /// Choose safekeepers for the new timeline: 3 in different azs.
    8699            0 :     pub(crate) async fn safekeepers_for_new_timeline(
    8700            0 :         &self,
    8701            0 :     ) -> Result<Vec<SafekeeperInfo>, ApiError> {
    8702            0 :         let mut all_safekeepers = {
    8703            0 :             let locked = self.inner.read().unwrap();
    8704            0 :             locked
    8705            0 :                 .safekeepers
    8706            0 :                 .iter()
    8707            0 :                 .filter_map(|sk| {
    8708            0 :                     if sk.1.scheduling_policy() != SkSchedulingPolicy::Active {
    8709              :                         // If we don't want to schedule stuff onto the safekeeper, respect that.
    8710            0 :                         return None;
    8711            0 :                     }
    8712            0 :                     let utilization_opt = if let SafekeeperState::Available {
    8713              :                         last_seen_at: _,
    8714            0 :                         utilization,
    8715            0 :                     } = sk.1.availability()
    8716              :                     {
    8717            0 :                         Some(utilization)
    8718              :                     } else {
    8719              :                         // non-available safekeepers still get a chance for new timelines,
    8720              :                         // but put them last in the list.
    8721            0 :                         None
    8722              :                     };
    8723            0 :                     let info = SafekeeperInfo {
    8724            0 :                         hostname: sk.1.skp.host.clone(),
    8725            0 :                         id: NodeId(sk.1.skp.id as u64),
    8726            0 :                     };
    8727            0 :                     Some((utilization_opt, info, sk.1.skp.availability_zone_id.clone()))
    8728            0 :                 })
    8729            0 :                 .collect::<Vec<_>>()
    8730            0 :         };
    8731            0 :         all_safekeepers.sort_by_key(|sk| {
    8732            0 :             (
    8733            0 :                 sk.0.as_ref()
    8734            0 :                     .map(|ut| ut.timeline_count)
    8735            0 :                     .unwrap_or(u64::MAX),
    8736            0 :                 // Use the id to decide on equal scores for reliability
    8737            0 :                 sk.1.id.0,
    8738            0 :             )
    8739            0 :         });
    8740            0 :         let mut sks = Vec::new();
    8741            0 :         let mut azs = HashSet::new();
    8742            0 :         for (_sk_util, sk_info, az_id) in all_safekeepers.iter() {
    8743            0 :             if !azs.insert(az_id) {
    8744            0 :                 continue;
    8745            0 :             }
    8746            0 :             sks.push(sk_info.clone());
    8747            0 :             if sks.len() == 3 {
    8748            0 :                 break;
    8749            0 :             }
    8750              :         }
    8751            0 :         if sks.len() == 3 {
    8752            0 :             Ok(sks)
    8753              :         } else {
    8754            0 :             Err(ApiError::InternalServerError(anyhow::anyhow!(
    8755            0 :                 "couldn't find three safekeepers in different AZs for new timeline"
    8756            0 :             )))
    8757              :         }
    8758            0 :     }
    8759              : 
    8760            0 :     pub(crate) async fn safekeepers_list(
    8761            0 :         &self,
    8762            0 :     ) -> Result<Vec<SafekeeperDescribeResponse>, DatabaseError> {
    8763            0 :         let locked = self.inner.read().unwrap();
    8764            0 :         let mut list = locked
    8765            0 :             .safekeepers
    8766            0 :             .iter()
    8767            0 :             .map(|sk| sk.1.describe_response())
    8768            0 :             .collect::<Result<Vec<_>, _>>()?;
    8769            0 :         list.sort_by_key(|v| v.id);
    8770            0 :         Ok(list)
    8771            0 :     }
    8772              : 
    8773            0 :     pub(crate) async fn get_safekeeper(
    8774            0 :         &self,
    8775            0 :         id: i64,
    8776            0 :     ) -> Result<SafekeeperDescribeResponse, DatabaseError> {
    8777            0 :         let locked = self.inner.read().unwrap();
    8778            0 :         let sk = locked
    8779            0 :             .safekeepers
    8780            0 :             .get(&NodeId(id as u64))
    8781            0 :             .ok_or(diesel::result::Error::NotFound)?;
    8782            0 :         sk.describe_response()
    8783            0 :     }
    8784              : 
    8785            0 :     pub(crate) async fn upsert_safekeeper(
    8786            0 :         &self,
    8787            0 :         record: crate::persistence::SafekeeperUpsert,
    8788            0 :     ) -> Result<(), ApiError> {
    8789            0 :         let node_id = NodeId(record.id as u64);
    8790            0 :         let use_https = self.config.use_https_safekeeper_api;
    8791            0 : 
    8792            0 :         if use_https && record.https_port.is_none() {
    8793            0 :             return Err(ApiError::PreconditionFailed(
    8794            0 :                 format!(
    8795            0 :                     "cannot upsert safekeeper {node_id}: \
    8796            0 :                     https is enabled, but https port is not specified"
    8797            0 :                 )
    8798            0 :                 .into(),
    8799            0 :             ));
    8800            0 :         }
    8801            0 : 
    8802            0 :         self.persistence.safekeeper_upsert(record.clone()).await?;
    8803              :         {
    8804            0 :             let mut locked = self.inner.write().unwrap();
    8805            0 :             let mut safekeepers = (*locked.safekeepers).clone();
    8806            0 :             match safekeepers.entry(node_id) {
    8807            0 :                 std::collections::hash_map::Entry::Occupied(mut entry) => entry
    8808            0 :                     .get_mut()
    8809            0 :                     .update_from_record(record)
    8810            0 :                     .expect("all preconditions should be checked before upsert to database"),
    8811            0 :                 std::collections::hash_map::Entry::Vacant(entry) => {
    8812            0 :                     entry.insert(
    8813            0 :                         Safekeeper::from_persistence(
    8814            0 :                             crate::persistence::SafekeeperPersistence::from_upsert(
    8815            0 :                                 record,
    8816            0 :                                 SkSchedulingPolicy::Pause,
    8817            0 :                             ),
    8818            0 :                             CancellationToken::new(),
    8819            0 :                             use_https,
    8820            0 :                         )
    8821            0 :                         .expect("all preconditions should be checked before upsert to database"),
    8822            0 :                     );
    8823            0 :                 }
    8824              :             }
    8825            0 :             locked.safekeepers = Arc::new(safekeepers);
    8826            0 :         }
    8827            0 :         Ok(())
    8828            0 :     }
    8829              : 
    8830            0 :     pub(crate) async fn set_safekeeper_scheduling_policy(
    8831            0 :         &self,
    8832            0 :         id: i64,
    8833            0 :         scheduling_policy: SkSchedulingPolicy,
    8834            0 :     ) -> Result<(), DatabaseError> {
    8835            0 :         self.persistence
    8836            0 :             .set_safekeeper_scheduling_policy(id, scheduling_policy)
    8837            0 :             .await?;
    8838            0 :         let node_id = NodeId(id as u64);
    8839            0 :         // After the change has been persisted successfully, update the in-memory state
    8840            0 :         {
    8841            0 :             let mut locked = self.inner.write().unwrap();
    8842            0 :             let mut safekeepers = (*locked.safekeepers).clone();
    8843            0 :             let sk = safekeepers
    8844            0 :                 .get_mut(&node_id)
    8845            0 :                 .ok_or(DatabaseError::Logical("Not found".to_string()))?;
    8846            0 :             sk.set_scheduling_policy(scheduling_policy);
    8847            0 : 
    8848            0 :             match scheduling_policy {
    8849            0 :                 SkSchedulingPolicy::Active => (),
    8850            0 :                 SkSchedulingPolicy::Decomissioned | SkSchedulingPolicy::Pause => {
    8851            0 :                     locked.safekeeper_reconcilers.cancel_safekeeper(node_id);
    8852            0 :                 }
    8853              :             }
    8854              : 
    8855            0 :             locked.safekeepers = Arc::new(safekeepers);
    8856            0 :         }
    8857            0 :         Ok(())
    8858            0 :     }
    8859              : 
    8860            0 :     pub(crate) async fn update_shards_preferred_azs(
    8861            0 :         &self,
    8862            0 :         req: ShardsPreferredAzsRequest,
    8863            0 :     ) -> Result<ShardsPreferredAzsResponse, ApiError> {
    8864            0 :         let preferred_azs = req.preferred_az_ids.into_iter().collect::<Vec<_>>();
    8865            0 :         let updated = self
    8866            0 :             .persistence
    8867            0 :             .set_tenant_shard_preferred_azs(preferred_azs)
    8868            0 :             .await
    8869            0 :             .map_err(|err| {
    8870            0 :                 ApiError::InternalServerError(anyhow::anyhow!(
    8871            0 :                     "Failed to persist preferred AZs: {err}"
    8872            0 :                 ))
    8873            0 :             })?;
    8874              : 
    8875            0 :         let mut updated_in_mem_and_db = Vec::default();
    8876            0 : 
    8877            0 :         let mut locked = self.inner.write().unwrap();
    8878            0 :         let state = locked.deref_mut();
    8879            0 :         for (tid, az_id) in updated {
    8880            0 :             let shard = state.tenants.get_mut(&tid);
    8881            0 :             if let Some(shard) = shard {
    8882            0 :                 shard.set_preferred_az(&mut state.scheduler, az_id);
    8883            0 :                 updated_in_mem_and_db.push(tid);
    8884            0 :             }
    8885              :         }
    8886              : 
    8887            0 :         Ok(ShardsPreferredAzsResponse {
    8888            0 :             updated: updated_in_mem_and_db,
    8889            0 :         })
    8890            0 :     }
    8891              : }
        

Generated by: LCOV version 2.1-beta