LCOV - code coverage report
Current view: top level - control_plane/src - storage_controller.rs (source / functions) Coverage Total Hit
Test: bb522999b2ee0ee028df22bb188d3a84170ba700.info Lines: 0.0 % 365 0
Test Date: 2024-07-21 16:16:09 Functions: 0.0 % 98 0

            Line data    Source code
       1              : use crate::{
       2              :     background_process,
       3              :     local_env::{LocalEnv, NeonStorageControllerConf},
       4              : };
       5              : use camino::{Utf8Path, Utf8PathBuf};
       6              : use pageserver_api::{
       7              :     controller_api::{
       8              :         NodeConfigureRequest, NodeRegisterRequest, TenantCreateRequest, TenantCreateResponse,
       9              :         TenantLocateResponse, TenantShardMigrateRequest, TenantShardMigrateResponse,
      10              :     },
      11              :     models::{
      12              :         TenantShardSplitRequest, TenantShardSplitResponse, TimelineCreateRequest, TimelineInfo,
      13              :     },
      14              :     shard::{ShardStripeSize, TenantShardId},
      15              : };
      16              : use pageserver_client::mgmt_api::ResponseErrorMessageExt;
      17              : use postgres_backend::AuthType;
      18              : use reqwest::Method;
      19              : use serde::{de::DeserializeOwned, Deserialize, Serialize};
      20              : use std::{fs, str::FromStr, time::Duration};
      21              : use tokio::process::Command;
      22              : use tracing::instrument;
      23              : use url::Url;
      24              : use utils::{
      25              :     auth::{encode_from_key_file, Claims, Scope},
      26              :     id::{NodeId, TenantId},
      27              : };
      28              : 
      29              : pub struct StorageController {
      30              :     env: LocalEnv,
      31              :     listen: String,
      32              :     private_key: Option<Vec<u8>>,
      33              :     public_key: Option<String>,
      34              :     postgres_port: u16,
      35              :     client: reqwest::Client,
      36              :     config: NeonStorageControllerConf,
      37              : }
      38              : 
      39              : const COMMAND: &str = "storage_controller";
      40              : 
      41              : const STORAGE_CONTROLLER_POSTGRES_VERSION: u32 = 16;
      42              : 
      43              : const DB_NAME: &str = "storage_controller";
      44              : 
      45            0 : #[derive(Serialize, Deserialize)]
      46              : pub struct AttachHookRequest {
      47              :     pub tenant_shard_id: TenantShardId,
      48              :     pub node_id: Option<NodeId>,
      49              :     pub generation_override: Option<i32>,
      50              : }
      51              : 
      52            0 : #[derive(Serialize, Deserialize)]
      53              : pub struct AttachHookResponse {
      54              :     pub gen: Option<u32>,
      55              : }
      56              : 
      57            0 : #[derive(Serialize, Deserialize)]
      58              : pub struct InspectRequest {
      59              :     pub tenant_shard_id: TenantShardId,
      60              : }
      61              : 
      62            0 : #[derive(Serialize, Deserialize)]
      63              : pub struct InspectResponse {
      64              :     pub attachment: Option<(u32, NodeId)>,
      65              : }
      66              : 
      67              : impl StorageController {
      68            0 :     pub fn from_env(env: &LocalEnv) -> Self {
      69            0 :         // Makes no sense to construct this if pageservers aren't going to use it: assume
      70            0 :         // pageservers have control plane API set
      71            0 :         let listen_url = env.control_plane_api.clone().unwrap();
      72            0 : 
      73            0 :         let listen = format!(
      74            0 :             "{}:{}",
      75            0 :             listen_url.host_str().unwrap(),
      76            0 :             listen_url.port().unwrap()
      77            0 :         );
      78            0 : 
      79            0 :         // Convention: NeonEnv in python tests reserves the next port after the control_plane_api
      80            0 :         // port, for use by our captive postgres.
      81            0 :         let postgres_port = listen_url
      82            0 :             .port()
      83            0 :             .expect("Control plane API setting should always have a port")
      84            0 :             + 1;
      85            0 : 
      86            0 :         // Assume all pageservers have symmetric auth configuration: this service
      87            0 :         // expects to use one JWT token to talk to all of them.
      88            0 :         let ps_conf = env
      89            0 :             .pageservers
      90            0 :             .first()
      91            0 :             .expect("Config is validated to contain at least one pageserver");
      92            0 :         let (private_key, public_key) = match ps_conf.http_auth_type {
      93            0 :             AuthType::Trust => (None, None),
      94              :             AuthType::NeonJWT => {
      95            0 :                 let private_key_path = env.get_private_key_path();
      96            0 :                 let private_key = fs::read(private_key_path).expect("failed to read private key");
      97            0 : 
      98            0 :                 // If pageserver auth is enabled, this implicitly enables auth for this service,
      99            0 :                 // using the same credentials.
     100            0 :                 let public_key_path =
     101            0 :                     camino::Utf8PathBuf::try_from(env.base_data_dir.join("auth_public_key.pem"))
     102            0 :                         .unwrap();
     103              : 
     104              :                 // This service takes keys as a string rather than as a path to a file/dir: read the key into memory.
     105            0 :                 let public_key = if std::fs::metadata(&public_key_path)
     106            0 :                     .expect("Can't stat public key")
     107            0 :                     .is_dir()
     108              :                 {
     109              :                     // Our config may specify a directory: this is for the pageserver's ability to handle multiple
     110              :                     // keys.  We only use one key at a time, so, arbitrarily load the first one in the directory.
     111            0 :                     let mut dir =
     112            0 :                         std::fs::read_dir(&public_key_path).expect("Can't readdir public key path");
     113            0 :                     let dent = dir
     114            0 :                         .next()
     115            0 :                         .expect("Empty key dir")
     116            0 :                         .expect("Error reading key dir");
     117            0 : 
     118            0 :                     std::fs::read_to_string(dent.path()).expect("Can't read public key")
     119              :                 } else {
     120            0 :                     std::fs::read_to_string(&public_key_path).expect("Can't read public key")
     121              :                 };
     122            0 :                 (Some(private_key), Some(public_key))
     123              :             }
     124              :         };
     125              : 
     126            0 :         Self {
     127            0 :             env: env.clone(),
     128            0 :             listen,
     129            0 :             private_key,
     130            0 :             public_key,
     131            0 :             postgres_port,
     132            0 :             client: reqwest::ClientBuilder::new()
     133            0 :                 .build()
     134            0 :                 .expect("Failed to construct http client"),
     135            0 :             config: env.storage_controller.clone(),
     136            0 :         }
     137            0 :     }
     138              : 
     139            0 :     fn pid_file(&self) -> Utf8PathBuf {
     140            0 :         Utf8PathBuf::from_path_buf(self.env.base_data_dir.join("storage_controller.pid"))
     141            0 :             .expect("non-Unicode path")
     142            0 :     }
     143              : 
     144              :     /// PIDFile for the postgres instance used to store storage controller state
     145            0 :     fn postgres_pid_file(&self) -> Utf8PathBuf {
     146            0 :         Utf8PathBuf::from_path_buf(
     147            0 :             self.env
     148            0 :                 .base_data_dir
     149            0 :                 .join("storage_controller_postgres.pid"),
     150            0 :         )
     151            0 :         .expect("non-Unicode path")
     152            0 :     }
     153              : 
     154              :     /// Find the directory containing postgres subdirectories, such `bin` and `lib`
     155              :     ///
     156              :     /// This usually uses STORAGE_CONTROLLER_POSTGRES_VERSION of postgres, but will fall back
     157              :     /// to other versions if that one isn't found.  Some automated tests create circumstances
     158              :     /// where only one version is available in pg_distrib_dir, such as `test_remote_extensions`.
     159            0 :     async fn get_pg_dir(&self, dir_name: &str) -> anyhow::Result<Utf8PathBuf> {
     160            0 :         let prefer_versions = [STORAGE_CONTROLLER_POSTGRES_VERSION, 15, 14];
     161              : 
     162            0 :         for v in prefer_versions {
     163            0 :             let path = Utf8PathBuf::from_path_buf(self.env.pg_dir(v, dir_name)?).unwrap();
     164            0 :             if tokio::fs::try_exists(&path).await? {
     165            0 :                 return Ok(path);
     166            0 :             }
     167              :         }
     168              : 
     169              :         // Fall through
     170            0 :         anyhow::bail!(
     171            0 :             "Postgres directory '{}' not found in {}",
     172            0 :             dir_name,
     173            0 :             self.env.pg_distrib_dir.display(),
     174            0 :         );
     175            0 :     }
     176              : 
     177            0 :     pub async fn get_pg_bin_dir(&self) -> anyhow::Result<Utf8PathBuf> {
     178            0 :         self.get_pg_dir("bin").await
     179            0 :     }
     180              : 
     181            0 :     pub async fn get_pg_lib_dir(&self) -> anyhow::Result<Utf8PathBuf> {
     182            0 :         self.get_pg_dir("lib").await
     183            0 :     }
     184              : 
     185              :     /// Readiness check for our postgres process
     186            0 :     async fn pg_isready(&self, pg_bin_dir: &Utf8Path) -> anyhow::Result<bool> {
     187            0 :         let bin_path = pg_bin_dir.join("pg_isready");
     188            0 :         let args = ["-h", "localhost", "-p", &format!("{}", self.postgres_port)];
     189            0 :         let exitcode = Command::new(bin_path).args(args).spawn()?.wait().await?;
     190              : 
     191            0 :         Ok(exitcode.success())
     192            0 :     }
     193              : 
     194              :     /// Create our database if it doesn't exist, and run migrations.
     195              :     ///
     196              :     /// This function is equivalent to the `diesel setup` command in the diesel CLI.  We implement
     197              :     /// the same steps by hand to avoid imposing a dependency on installing diesel-cli for developers
     198              :     /// who just want to run `cargo neon_local` without knowing about diesel.
     199              :     ///
     200              :     /// Returns the database url
     201            0 :     pub async fn setup_database(&self) -> anyhow::Result<String> {
     202            0 :         let database_url = format!("postgresql://localhost:{}/{DB_NAME}", self.postgres_port);
     203              : 
     204            0 :         let pg_bin_dir = self.get_pg_bin_dir().await?;
     205            0 :         let createdb_path = pg_bin_dir.join("createdb");
     206            0 :         let output = Command::new(&createdb_path)
     207            0 :             .args([
     208            0 :                 "-h",
     209            0 :                 "localhost",
     210            0 :                 "-p",
     211            0 :                 &format!("{}", self.postgres_port),
     212            0 :                 DB_NAME,
     213            0 :             ])
     214            0 :             .output()
     215            0 :             .await
     216            0 :             .expect("Failed to spawn createdb");
     217            0 : 
     218            0 :         if !output.status.success() {
     219            0 :             let stderr = String::from_utf8(output.stderr).expect("Non-UTF8 output from createdb");
     220            0 :             if stderr.contains("already exists") {
     221            0 :                 tracing::info!("Database {DB_NAME} already exists");
     222              :             } else {
     223            0 :                 anyhow::bail!("createdb failed with status {}: {stderr}", output.status);
     224              :             }
     225            0 :         }
     226              : 
     227            0 :         Ok(database_url)
     228            0 :     }
     229              : 
     230            0 :     pub async fn connect_to_database(
     231            0 :         &self,
     232            0 :     ) -> anyhow::Result<(
     233            0 :         tokio_postgres::Client,
     234            0 :         tokio_postgres::Connection<tokio_postgres::Socket, tokio_postgres::tls::NoTlsStream>,
     235            0 :     )> {
     236            0 :         tokio_postgres::Config::new()
     237            0 :             .host("localhost")
     238            0 :             .port(self.postgres_port)
     239            0 :             // The user is the ambient operating system user name.
     240            0 :             // That is an impurity which we want to fix in => TODO https://github.com/neondatabase/neon/issues/8400
     241            0 :             //
     242            0 :             // Until we get there, use the ambient operating system user name.
     243            0 :             // Recent tokio-postgres versions default to this if the user isn't specified.
     244            0 :             // But tokio-postgres fork doesn't have this upstream commit:
     245            0 :             // https://github.com/sfackler/rust-postgres/commit/cb609be758f3fb5af537f04b584a2ee0cebd5e79
     246            0 :             // => we should rebase our fork => TODO https://github.com/neondatabase/neon/issues/8399
     247            0 :             .user(&whoami::username())
     248            0 :             .dbname(DB_NAME)
     249            0 :             .connect(tokio_postgres::NoTls)
     250            0 :             .await
     251            0 :             .map_err(anyhow::Error::new)
     252            0 :     }
     253              : 
     254            0 :     pub async fn start(&self, retry_timeout: &Duration) -> anyhow::Result<()> {
     255            0 :         // Start a vanilla Postgres process used by the storage controller for persistence.
     256            0 :         let pg_data_path = Utf8PathBuf::from_path_buf(self.env.base_data_dir.clone())
     257            0 :             .unwrap()
     258            0 :             .join("storage_controller_db");
     259            0 :         let pg_bin_dir = self.get_pg_bin_dir().await?;
     260            0 :         let pg_lib_dir = self.get_pg_lib_dir().await?;
     261            0 :         let pg_log_path = pg_data_path.join("postgres.log");
     262            0 : 
     263            0 :         if !tokio::fs::try_exists(&pg_data_path).await? {
     264              :             // Initialize empty database
     265            0 :             let initdb_path = pg_bin_dir.join("initdb");
     266            0 :             let mut child = Command::new(&initdb_path)
     267            0 :                 .envs(vec![
     268            0 :                     ("LD_LIBRARY_PATH".to_owned(), pg_lib_dir.to_string()),
     269            0 :                     ("DYLD_LIBRARY_PATH".to_owned(), pg_lib_dir.to_string()),
     270            0 :                 ])
     271            0 :                 .args(["-D", pg_data_path.as_ref()])
     272            0 :                 .spawn()
     273            0 :                 .expect("Failed to spawn initdb");
     274            0 :             let status = child.wait().await?;
     275            0 :             if !status.success() {
     276            0 :                 anyhow::bail!("initdb failed with status {status}");
     277            0 :             }
     278            0 :         };
     279              : 
     280              :         // Write a minimal config file:
     281              :         // - Specify the port, since this is chosen dynamically
     282              :         // - Switch off fsync, since we're running on lightweight test environments and when e.g. scale testing
     283              :         //   the storage controller we don't want a slow local disk to interfere with that.
     284              :         //
     285              :         // NB: it's important that we rewrite this file on each start command so we propagate changes
     286              :         // from `LocalEnv`'s config file (`.neon/config`).
     287            0 :         tokio::fs::write(
     288            0 :             &pg_data_path.join("postgresql.conf"),
     289            0 :             format!("port = {}\nfsync=off\n", self.postgres_port),
     290            0 :         )
     291            0 :         .await?;
     292              : 
     293            0 :         println!("Starting storage controller database...");
     294            0 :         let db_start_args = [
     295            0 :             "-w",
     296            0 :             "-D",
     297            0 :             pg_data_path.as_ref(),
     298            0 :             "-l",
     299            0 :             pg_log_path.as_ref(),
     300            0 :             "start",
     301            0 :         ];
     302            0 : 
     303            0 :         background_process::start_process(
     304            0 :             "storage_controller_db",
     305            0 :             &self.env.base_data_dir,
     306            0 :             pg_bin_dir.join("pg_ctl").as_std_path(),
     307            0 :             db_start_args,
     308            0 :             vec![
     309            0 :                 ("LD_LIBRARY_PATH".to_owned(), pg_lib_dir.to_string()),
     310            0 :                 ("DYLD_LIBRARY_PATH".to_owned(), pg_lib_dir.to_string()),
     311            0 :             ],
     312            0 :             background_process::InitialPidFile::Create(self.postgres_pid_file()),
     313            0 :             retry_timeout,
     314            0 :             || self.pg_isready(&pg_bin_dir),
     315            0 :         )
     316            0 :         .await?;
     317              : 
     318              :         // Run migrations on every startup, in case something changed.
     319            0 :         let database_url = self.setup_database().await?;
     320              : 
     321              :         // We support running a startup SQL script to fiddle with the database before we launch storcon.
     322              :         // This is used by the test suite.
     323            0 :         let startup_script_path = self
     324            0 :             .env
     325            0 :             .base_data_dir
     326            0 :             .join("storage_controller_db.startup.sql");
     327            0 :         let startup_script = match tokio::fs::read_to_string(&startup_script_path).await {
     328            0 :             Ok(script) => {
     329            0 :                 tokio::fs::remove_file(startup_script_path).await?;
     330            0 :                 script
     331              :             }
     332            0 :             Err(e) => {
     333            0 :                 if e.kind() == std::io::ErrorKind::NotFound {
     334              :                     // always run some startup script so that this code path doesn't bit rot
     335            0 :                     "BEGIN; COMMIT;".to_string()
     336              :                 } else {
     337            0 :                     anyhow::bail!("Failed to read startup script: {e}")
     338              :                 }
     339              :             }
     340              :         };
     341            0 :         let (mut client, conn) = self.connect_to_database().await?;
     342            0 :         let conn = tokio::spawn(conn);
     343            0 :         let tx = client.build_transaction();
     344            0 :         let tx = tx.start().await?;
     345            0 :         tx.batch_execute(&startup_script).await?;
     346            0 :         tx.commit().await?;
     347            0 :         drop(client);
     348            0 :         conn.await??;
     349              : 
     350            0 :         let mut args = vec![
     351            0 :             "-l",
     352            0 :             &self.listen,
     353            0 :             "--dev",
     354            0 :             "--database-url",
     355            0 :             &database_url,
     356            0 :             "--max-unavailable-interval",
     357            0 :             &humantime::Duration::from(self.config.max_unavailable).to_string(),
     358            0 :         ]
     359            0 :         .into_iter()
     360            0 :         .map(|s| s.to_string())
     361            0 :         .collect::<Vec<_>>();
     362            0 :         if let Some(private_key) = &self.private_key {
     363            0 :             let claims = Claims::new(None, Scope::PageServerApi);
     364            0 :             let jwt_token =
     365            0 :                 encode_from_key_file(&claims, private_key).expect("failed to generate jwt token");
     366            0 :             args.push(format!("--jwt-token={jwt_token}"));
     367            0 :         }
     368              : 
     369            0 :         if let Some(public_key) = &self.public_key {
     370            0 :             args.push(format!("--public-key=\"{public_key}\""));
     371            0 :         }
     372              : 
     373            0 :         if let Some(control_plane_compute_hook_api) = &self.env.control_plane_compute_hook_api {
     374            0 :             args.push(format!(
     375            0 :                 "--compute-hook-url={control_plane_compute_hook_api}"
     376            0 :             ));
     377            0 :         }
     378              : 
     379            0 :         if let Some(split_threshold) = self.config.split_threshold.as_ref() {
     380            0 :             args.push(format!("--split-threshold={split_threshold}"))
     381            0 :         }
     382              : 
     383            0 :         args.push(format!(
     384            0 :             "--neon-local-repo-dir={}",
     385            0 :             self.env.base_data_dir.display()
     386            0 :         ));
     387            0 : 
     388            0 :         background_process::start_process(
     389            0 :             COMMAND,
     390            0 :             &self.env.base_data_dir,
     391            0 :             &self.env.storage_controller_bin(),
     392            0 :             args,
     393            0 :             vec![
     394            0 :                 ("LD_LIBRARY_PATH".to_owned(), pg_lib_dir.to_string()),
     395            0 :                 ("DYLD_LIBRARY_PATH".to_owned(), pg_lib_dir.to_string()),
     396            0 :             ],
     397            0 :             background_process::InitialPidFile::Create(self.pid_file()),
     398            0 :             retry_timeout,
     399            0 :             || async {
     400            0 :                 match self.ready().await {
     401            0 :                     Ok(_) => Ok(true),
     402            0 :                     Err(_) => Ok(false),
     403            0 :                 }
     404            0 :             },
     405            0 :         )
     406            0 :         .await?;
     407              : 
     408            0 :         Ok(())
     409            0 :     }
     410              : 
     411            0 :     pub async fn stop(&self, immediate: bool) -> anyhow::Result<()> {
     412            0 :         background_process::stop_process(immediate, COMMAND, &self.pid_file())?;
     413              : 
     414            0 :         let pg_data_path = self.env.base_data_dir.join("storage_controller_db");
     415            0 :         let pg_bin_dir = self.get_pg_bin_dir().await?;
     416              : 
     417            0 :         println!("Stopping storage controller database...");
     418            0 :         let pg_stop_args = ["-D", &pg_data_path.to_string_lossy(), "stop"];
     419            0 :         let stop_status = Command::new(pg_bin_dir.join("pg_ctl"))
     420            0 :             .args(pg_stop_args)
     421            0 :             .spawn()?
     422            0 :             .wait()
     423            0 :             .await?;
     424            0 :         if !stop_status.success() {
     425            0 :             let pg_status_args = ["-D", &pg_data_path.to_string_lossy(), "status"];
     426            0 :             let status_exitcode = Command::new(pg_bin_dir.join("pg_ctl"))
     427            0 :                 .args(pg_status_args)
     428            0 :                 .spawn()?
     429            0 :                 .wait()
     430            0 :                 .await?;
     431              : 
     432              :             // pg_ctl status returns this exit code if postgres is not running: in this case it is
     433              :             // fine that stop failed.  Otherwise it is an error that stop failed.
     434              :             const PG_STATUS_NOT_RUNNING: i32 = 3;
     435            0 :             if Some(PG_STATUS_NOT_RUNNING) == status_exitcode.code() {
     436            0 :                 println!("Storage controller database is already stopped");
     437            0 :                 return Ok(());
     438              :             } else {
     439            0 :                 anyhow::bail!("Failed to stop storage controller database: {stop_status}")
     440              :             }
     441            0 :         }
     442            0 : 
     443            0 :         Ok(())
     444            0 :     }
     445              : 
     446            0 :     fn get_claims_for_path(path: &str) -> anyhow::Result<Option<Claims>> {
     447            0 :         let category = match path.find('/') {
     448            0 :             Some(idx) => &path[..idx],
     449            0 :             None => path,
     450              :         };
     451              : 
     452            0 :         match category {
     453            0 :             "status" | "ready" => Ok(None),
     454            0 :             "control" | "debug" => Ok(Some(Claims::new(None, Scope::Admin))),
     455            0 :             "v1" => Ok(Some(Claims::new(None, Scope::PageServerApi))),
     456            0 :             _ => Err(anyhow::anyhow!("Failed to determine claims for {}", path)),
     457              :         }
     458            0 :     }
     459              : 
     460              :     /// Simple HTTP request wrapper for calling into storage controller
     461            0 :     async fn dispatch<RQ, RS>(
     462            0 :         &self,
     463            0 :         method: reqwest::Method,
     464            0 :         path: String,
     465            0 :         body: Option<RQ>,
     466            0 :     ) -> anyhow::Result<RS>
     467            0 :     where
     468            0 :         RQ: Serialize + Sized,
     469            0 :         RS: DeserializeOwned + Sized,
     470            0 :     {
     471            0 :         // The configured URL has the /upcall path prefix for pageservers to use: we will strip that out
     472            0 :         // for general purpose API access.
     473            0 :         let listen_url = self.env.control_plane_api.clone().unwrap();
     474            0 :         let url = Url::from_str(&format!(
     475            0 :             "http://{}:{}/{path}",
     476            0 :             listen_url.host_str().unwrap(),
     477            0 :             listen_url.port().unwrap()
     478            0 :         ))
     479            0 :         .unwrap();
     480            0 : 
     481            0 :         let mut builder = self.client.request(method, url);
     482            0 :         if let Some(body) = body {
     483            0 :             builder = builder.json(&body)
     484            0 :         }
     485            0 :         if let Some(private_key) = &self.private_key {
     486            0 :             println!("Getting claims for path {}", path);
     487            0 :             if let Some(required_claims) = Self::get_claims_for_path(&path)? {
     488            0 :                 println!("Got claims {:?} for path {}", required_claims, path);
     489            0 :                 let jwt_token = encode_from_key_file(&required_claims, private_key)?;
     490            0 :                 builder = builder.header(
     491            0 :                     reqwest::header::AUTHORIZATION,
     492            0 :                     format!("Bearer {jwt_token}"),
     493            0 :                 );
     494            0 :             }
     495            0 :         }
     496              : 
     497            0 :         let response = builder.send().await?;
     498            0 :         let response = response.error_from_body().await?;
     499              : 
     500            0 :         Ok(response
     501            0 :             .json()
     502            0 :             .await
     503            0 :             .map_err(pageserver_client::mgmt_api::Error::ReceiveBody)?)
     504            0 :     }
     505              : 
     506              :     /// Call into the attach_hook API, for use before handing out attachments to pageservers
     507            0 :     #[instrument(skip(self))]
     508              :     pub async fn attach_hook(
     509              :         &self,
     510              :         tenant_shard_id: TenantShardId,
     511              :         pageserver_id: NodeId,
     512              :     ) -> anyhow::Result<Option<u32>> {
     513              :         let request = AttachHookRequest {
     514              :             tenant_shard_id,
     515              :             node_id: Some(pageserver_id),
     516              :             generation_override: None,
     517              :         };
     518              : 
     519              :         let response = self
     520              :             .dispatch::<_, AttachHookResponse>(
     521              :                 Method::POST,
     522              :                 "debug/v1/attach-hook".to_string(),
     523              :                 Some(request),
     524              :             )
     525              :             .await?;
     526              : 
     527              :         Ok(response.gen)
     528              :     }
     529              : 
     530            0 :     #[instrument(skip(self))]
     531              :     pub async fn inspect(
     532              :         &self,
     533              :         tenant_shard_id: TenantShardId,
     534              :     ) -> anyhow::Result<Option<(u32, NodeId)>> {
     535              :         let request = InspectRequest { tenant_shard_id };
     536              : 
     537              :         let response = self
     538              :             .dispatch::<_, InspectResponse>(
     539              :                 Method::POST,
     540              :                 "debug/v1/inspect".to_string(),
     541              :                 Some(request),
     542              :             )
     543              :             .await?;
     544              : 
     545              :         Ok(response.attachment)
     546              :     }
     547              : 
     548            0 :     #[instrument(skip(self))]
     549              :     pub async fn tenant_create(
     550              :         &self,
     551              :         req: TenantCreateRequest,
     552              :     ) -> anyhow::Result<TenantCreateResponse> {
     553              :         self.dispatch(Method::POST, "v1/tenant".to_string(), Some(req))
     554              :             .await
     555              :     }
     556              : 
     557            0 :     #[instrument(skip(self))]
     558              :     pub async fn tenant_import(&self, tenant_id: TenantId) -> anyhow::Result<TenantCreateResponse> {
     559              :         self.dispatch::<(), TenantCreateResponse>(
     560              :             Method::POST,
     561              :             format!("debug/v1/tenant/{tenant_id}/import"),
     562              :             None,
     563              :         )
     564              :         .await
     565              :     }
     566              : 
     567            0 :     #[instrument(skip(self))]
     568              :     pub async fn tenant_locate(&self, tenant_id: TenantId) -> anyhow::Result<TenantLocateResponse> {
     569              :         self.dispatch::<(), _>(
     570              :             Method::GET,
     571              :             format!("debug/v1/tenant/{tenant_id}/locate"),
     572              :             None,
     573              :         )
     574              :         .await
     575              :     }
     576              : 
     577            0 :     #[instrument(skip(self))]
     578              :     pub async fn tenant_migrate(
     579              :         &self,
     580              :         tenant_shard_id: TenantShardId,
     581              :         node_id: NodeId,
     582              :     ) -> anyhow::Result<TenantShardMigrateResponse> {
     583              :         self.dispatch(
     584              :             Method::PUT,
     585              :             format!("control/v1/tenant/{tenant_shard_id}/migrate"),
     586              :             Some(TenantShardMigrateRequest {
     587              :                 tenant_shard_id,
     588              :                 node_id,
     589              :             }),
     590              :         )
     591              :         .await
     592              :     }
     593              : 
     594            0 :     #[instrument(skip(self), fields(%tenant_id, %new_shard_count))]
     595              :     pub async fn tenant_split(
     596              :         &self,
     597              :         tenant_id: TenantId,
     598              :         new_shard_count: u8,
     599              :         new_stripe_size: Option<ShardStripeSize>,
     600              :     ) -> anyhow::Result<TenantShardSplitResponse> {
     601              :         self.dispatch(
     602              :             Method::PUT,
     603              :             format!("control/v1/tenant/{tenant_id}/shard_split"),
     604              :             Some(TenantShardSplitRequest {
     605              :                 new_shard_count,
     606              :                 new_stripe_size,
     607              :             }),
     608              :         )
     609              :         .await
     610              :     }
     611              : 
     612            0 :     #[instrument(skip_all, fields(node_id=%req.node_id))]
     613              :     pub async fn node_register(&self, req: NodeRegisterRequest) -> anyhow::Result<()> {
     614              :         self.dispatch::<_, ()>(Method::POST, "control/v1/node".to_string(), Some(req))
     615              :             .await
     616              :     }
     617              : 
     618            0 :     #[instrument(skip_all, fields(node_id=%req.node_id))]
     619              :     pub async fn node_configure(&self, req: NodeConfigureRequest) -> anyhow::Result<()> {
     620              :         self.dispatch::<_, ()>(
     621              :             Method::PUT,
     622              :             format!("control/v1/node/{}/config", req.node_id),
     623              :             Some(req),
     624              :         )
     625              :         .await
     626              :     }
     627              : 
     628            0 :     #[instrument(skip(self))]
     629              :     pub async fn ready(&self) -> anyhow::Result<()> {
     630              :         self.dispatch::<(), ()>(Method::GET, "ready".to_string(), None)
     631              :             .await
     632              :     }
     633              : 
     634            0 :     #[instrument(skip_all, fields(%tenant_id, timeline_id=%req.new_timeline_id))]
     635              :     pub async fn tenant_timeline_create(
     636              :         &self,
     637              :         tenant_id: TenantId,
     638              :         req: TimelineCreateRequest,
     639              :     ) -> anyhow::Result<TimelineInfo> {
     640              :         self.dispatch(
     641              :             Method::POST,
     642              :             format!("v1/tenant/{tenant_id}/timeline"),
     643              :             Some(req),
     644              :         )
     645              :         .await
     646              :     }
     647              : }
        

Generated by: LCOV version 2.1-beta