LCOV - code coverage report
Current view: top level - pageserver/src - config.rs (source / functions) Coverage Total Hit
Test: 07bee600374ccd486c69370d0972d9035964fe68.info Lines: 84.1 % 271 228
Test Date: 2025-02-20 13:11:02 Functions: 53.7 % 41 22

            Line data    Source code
       1              : //! Functions for handling page server configuration options
       2              : //!
       3              : //! Configuration options can be set in the pageserver.toml configuration
       4              : //! file, or on the command line.
       5              : //! See also `settings.md` for better description on every parameter.
       6              : 
       7              : use anyhow::{bail, ensure, Context};
       8              : use pageserver_api::models::ImageCompressionAlgorithm;
       9              : use pageserver_api::{
      10              :     config::{DiskUsageEvictionTaskConfig, MaxVectoredReadBytes},
      11              :     shard::TenantShardId,
      12              : };
      13              : use remote_storage::{RemotePath, RemoteStorageConfig};
      14              : use std::env;
      15              : use storage_broker::Uri;
      16              : use utils::logging::SecretString;
      17              : use utils::postgres_client::PostgresClientProtocol;
      18              : 
      19              : use once_cell::sync::OnceCell;
      20              : use reqwest::Url;
      21              : use std::num::NonZeroUsize;
      22              : use std::sync::Arc;
      23              : use std::time::Duration;
      24              : 
      25              : use camino::{Utf8Path, Utf8PathBuf};
      26              : use postgres_backend::AuthType;
      27              : use utils::{
      28              :     id::{NodeId, TimelineId},
      29              :     logging::LogFormat,
      30              : };
      31              : 
      32              : use crate::tenant::storage_layer::inmemory_layer::IndexEntry;
      33              : use crate::tenant::{TENANTS_SEGMENT_NAME, TIMELINES_SEGMENT_NAME};
      34              : use crate::virtual_file;
      35              : use crate::virtual_file::io_engine;
      36              : use crate::{TENANT_HEATMAP_BASENAME, TENANT_LOCATION_CONFIG_NAME};
      37              : 
      38              : /// Global state of pageserver.
      39              : ///
      40              : /// It's mostly immutable configuration, but some semaphores and the
      41              : /// like crept in over time and the name stuck.
      42              : ///
      43              : /// Instantiated by deserializing `pageserver.toml` into  [`pageserver_api::config::ConfigToml`]
      44              : /// and passing that to [`PageServerConf::parse_and_validate`].
      45              : ///
      46              : /// # Adding a New Field
      47              : ///
      48              : /// 1. Add the field to `pageserver_api::config::ConfigToml`.
      49              : /// 2. Fix compiler errors (exhaustive destructuring will guide you).
      50              : ///
      51              : /// For fields that require additional validation or filling in of defaults at runtime,
      52              : /// check for examples in the [`PageServerConf::parse_and_validate`] method.
      53              : #[derive(Debug, Clone, PartialEq, Eq)]
      54              : pub struct PageServerConf {
      55              :     // Identifier of that particular pageserver so e g safekeepers
      56              :     // can safely distinguish different pageservers
      57              :     pub id: NodeId,
      58              : 
      59              :     /// Example (default): 127.0.0.1:64000
      60              :     pub listen_pg_addr: String,
      61              :     /// Example (default): 127.0.0.1:9898
      62              :     pub listen_http_addr: String,
      63              : 
      64              :     /// Current availability zone. Used for traffic metrics.
      65              :     pub availability_zone: Option<String>,
      66              : 
      67              :     // Timeout when waiting for WAL receiver to catch up to an LSN given in a GetPage@LSN call.
      68              :     pub wait_lsn_timeout: Duration,
      69              :     // How long to wait for WAL redo to complete.
      70              :     pub wal_redo_timeout: Duration,
      71              : 
      72              :     pub superuser: String,
      73              :     pub locale: String,
      74              : 
      75              :     pub page_cache_size: usize,
      76              :     pub max_file_descriptors: usize,
      77              : 
      78              :     // Repository directory, relative to current working directory.
      79              :     // Normally, the page server changes the current working directory
      80              :     // to the repository, and 'workdir' is always '.'. But we don't do
      81              :     // that during unit testing, because the current directory is global
      82              :     // to the process but different unit tests work on different
      83              :     // repositories.
      84              :     pub workdir: Utf8PathBuf,
      85              : 
      86              :     pub pg_distrib_dir: Utf8PathBuf,
      87              : 
      88              :     // Authentication
      89              :     /// authentication method for the HTTP mgmt API
      90              :     pub http_auth_type: AuthType,
      91              :     /// authentication method for libpq connections from compute
      92              :     pub pg_auth_type: AuthType,
      93              :     /// Path to a file or directory containing public key(s) for verifying JWT tokens.
      94              :     /// Used for both mgmt and compute auth, if enabled.
      95              :     pub auth_validation_public_key_path: Option<Utf8PathBuf>,
      96              : 
      97              :     pub remote_storage_config: Option<RemoteStorageConfig>,
      98              : 
      99              :     pub default_tenant_conf: crate::tenant::config::TenantConf,
     100              : 
     101              :     /// Storage broker endpoints to connect to.
     102              :     pub broker_endpoint: Uri,
     103              :     pub broker_keepalive_interval: Duration,
     104              : 
     105              :     pub log_format: LogFormat,
     106              : 
     107              :     /// Number of tenants which will be concurrently loaded from remote storage proactively on startup or attach.
     108              :     ///
     109              :     /// A lower value implicitly deprioritizes loading such tenants, vs. other work in the system.
     110              :     pub concurrent_tenant_warmup: ConfigurableSemaphore,
     111              : 
     112              :     /// Number of concurrent [`Tenant::gather_size_inputs`](crate::tenant::Tenant::gather_size_inputs) allowed.
     113              :     pub concurrent_tenant_size_logical_size_queries: ConfigurableSemaphore,
     114              :     /// Limit of concurrent [`Tenant::gather_size_inputs`] issued by module `eviction_task`.
     115              :     /// The number of permits is the same as `concurrent_tenant_size_logical_size_queries`.
     116              :     /// See the comment in `eviction_task` for details.
     117              :     ///
     118              :     /// [`Tenant::gather_size_inputs`]: crate::tenant::Tenant::gather_size_inputs
     119              :     pub eviction_task_immitated_concurrent_logical_size_queries: ConfigurableSemaphore,
     120              : 
     121              :     // How often to collect metrics and send them to the metrics endpoint.
     122              :     pub metric_collection_interval: Duration,
     123              :     // How often to send unchanged cached metrics to the metrics endpoint.
     124              :     pub metric_collection_endpoint: Option<Url>,
     125              :     pub metric_collection_bucket: Option<RemoteStorageConfig>,
     126              :     pub synthetic_size_calculation_interval: Duration,
     127              : 
     128              :     pub disk_usage_based_eviction: Option<DiskUsageEvictionTaskConfig>,
     129              : 
     130              :     pub test_remote_failures: u64,
     131              : 
     132              :     pub ondemand_download_behavior_treat_error_as_warn: bool,
     133              : 
     134              :     /// How long will background tasks be delayed at most after initial load of tenants.
     135              :     ///
     136              :     /// Our largest initialization completions are in the range of 100-200s, so perhaps 10s works
     137              :     /// as we now isolate initial loading, initial logical size calculation and background tasks.
     138              :     /// Smaller nodes will have background tasks "not running" for this long unless every timeline
     139              :     /// has it's initial logical size calculated. Not running background tasks for some seconds is
     140              :     /// not terrible.
     141              :     pub background_task_maximum_delay: Duration,
     142              : 
     143              :     pub control_plane_api: Option<Url>,
     144              : 
     145              :     /// JWT token for use with the control plane API.
     146              :     pub control_plane_api_token: Option<SecretString>,
     147              : 
     148              :     pub import_pgdata_upcall_api: Option<Url>,
     149              :     pub import_pgdata_upcall_api_token: Option<SecretString>,
     150              :     pub import_pgdata_aws_endpoint_url: Option<Url>,
     151              : 
     152              :     /// If true, pageserver will make best-effort to operate without a control plane: only
     153              :     /// for use in major incidents.
     154              :     pub control_plane_emergency_mode: bool,
     155              : 
     156              :     /// How many heatmap uploads may be done concurrency: lower values implicitly deprioritize
     157              :     /// heatmap uploads vs. other remote storage operations.
     158              :     pub heatmap_upload_concurrency: usize,
     159              : 
     160              :     /// How many remote storage downloads may be done for secondary tenants concurrently.  Implicitly
     161              :     /// deprioritises secondary downloads vs. remote storage operations for attached tenants.
     162              :     pub secondary_download_concurrency: usize,
     163              : 
     164              :     /// Maximum number of WAL records to be ingested and committed at the same time
     165              :     pub ingest_batch_size: u64,
     166              : 
     167              :     pub virtual_file_io_engine: virtual_file::IoEngineKind,
     168              : 
     169              :     pub max_vectored_read_bytes: MaxVectoredReadBytes,
     170              : 
     171              :     pub image_compression: ImageCompressionAlgorithm,
     172              : 
     173              :     /// Whether to offload archived timelines automatically
     174              :     pub timeline_offloading: bool,
     175              : 
     176              :     /// How many bytes of ephemeral layer content will we allow per kilobyte of RAM.  When this
     177              :     /// is exceeded, we start proactively closing ephemeral layers to limit the total amount
     178              :     /// of ephemeral data.
     179              :     ///
     180              :     /// Setting this to zero disables limits on total ephemeral layer size.
     181              :     pub ephemeral_bytes_per_memory_kb: usize,
     182              : 
     183              :     pub l0_flush: crate::l0_flush::L0FlushConfig,
     184              : 
     185              :     /// Direct IO settings
     186              :     pub virtual_file_io_mode: virtual_file::IoMode,
     187              : 
     188              :     /// Optionally disable disk syncs (unsafe!)
     189              :     pub no_sync: bool,
     190              : 
     191              :     pub wal_receiver_protocol: PostgresClientProtocol,
     192              : 
     193              :     pub page_service_pipelining: pageserver_api::config::PageServicePipeliningConfig,
     194              : 
     195              :     pub get_vectored_concurrent_io: pageserver_api::config::GetVectoredConcurrentIo,
     196              : 
     197              :     /// Enable read path debugging. If enabled, read key errors will print a backtrace of the layer
     198              :     /// files read.
     199              :     pub enable_read_path_debugging: bool,
     200              : }
     201              : 
     202              : /// Token for authentication to safekeepers
     203              : ///
     204              : /// We do not want to store this in a PageServerConf because the latter may be logged
     205              : /// and/or serialized at a whim, while the token is secret. Currently this token is the
     206              : /// same for accessing all tenants/timelines, but may become per-tenant/per-timeline in
     207              : /// the future, more tokens and auth may arrive for storage broker, completely changing the logic.
     208              : /// Hence, we resort to a global variable for now instead of passing the token from the
     209              : /// startup code to the connection code through a dozen layers.
     210              : pub static SAFEKEEPER_AUTH_TOKEN: OnceCell<Arc<String>> = OnceCell::new();
     211              : 
     212              : impl PageServerConf {
     213              :     //
     214              :     // Repository paths, relative to workdir.
     215              :     //
     216              : 
     217        15408 :     pub fn tenants_path(&self) -> Utf8PathBuf {
     218        15408 :         self.workdir.join(TENANTS_SEGMENT_NAME)
     219        15408 :     }
     220              : 
     221          144 :     pub fn deletion_prefix(&self) -> Utf8PathBuf {
     222          144 :         self.workdir.join("deletion")
     223          144 :     }
     224              : 
     225            0 :     pub fn metadata_path(&self) -> Utf8PathBuf {
     226            0 :         self.workdir.join("metadata.json")
     227            0 :     }
     228              : 
     229           56 :     pub fn deletion_list_path(&self, sequence: u64) -> Utf8PathBuf {
     230              :         // Encode a version in the filename, so that if we ever switch away from JSON we can
     231              :         // increment this.
     232              :         const VERSION: u8 = 1;
     233              : 
     234           56 :         self.deletion_prefix()
     235           56 :             .join(format!("{sequence:016x}-{VERSION:02x}.list"))
     236           56 :     }
     237              : 
     238           48 :     pub fn deletion_header_path(&self) -> Utf8PathBuf {
     239              :         // Encode a version in the filename, so that if we ever switch away from JSON we can
     240              :         // increment this.
     241              :         const VERSION: u8 = 1;
     242              : 
     243           48 :         self.deletion_prefix().join(format!("header-{VERSION:02x}"))
     244           48 :     }
     245              : 
     246        15304 :     pub fn tenant_path(&self, tenant_shard_id: &TenantShardId) -> Utf8PathBuf {
     247        15304 :         self.tenants_path().join(tenant_shard_id.to_string())
     248        15304 :     }
     249              : 
     250              :     /// Points to a place in pageserver's local directory,
     251              :     /// where certain tenant's LocationConf be stored.
     252            0 :     pub(crate) fn tenant_location_config_path(
     253            0 :         &self,
     254            0 :         tenant_shard_id: &TenantShardId,
     255            0 :     ) -> Utf8PathBuf {
     256            0 :         self.tenant_path(tenant_shard_id)
     257            0 :             .join(TENANT_LOCATION_CONFIG_NAME)
     258            0 :     }
     259              : 
     260          444 :     pub(crate) fn tenant_heatmap_path(&self, tenant_shard_id: &TenantShardId) -> Utf8PathBuf {
     261          444 :         self.tenant_path(tenant_shard_id)
     262          444 :             .join(TENANT_HEATMAP_BASENAME)
     263          444 :     }
     264              : 
     265        14408 :     pub fn timelines_path(&self, tenant_shard_id: &TenantShardId) -> Utf8PathBuf {
     266        14408 :         self.tenant_path(tenant_shard_id)
     267        14408 :             .join(TIMELINES_SEGMENT_NAME)
     268        14408 :     }
     269              : 
     270        13508 :     pub fn timeline_path(
     271        13508 :         &self,
     272        13508 :         tenant_shard_id: &TenantShardId,
     273        13508 :         timeline_id: &TimelineId,
     274        13508 :     ) -> Utf8PathBuf {
     275        13508 :         self.timelines_path(tenant_shard_id)
     276        13508 :             .join(timeline_id.to_string())
     277        13508 :     }
     278              : 
     279              :     /// Turns storage remote path of a file into its local path.
     280            0 :     pub fn local_path(&self, remote_path: &RemotePath) -> Utf8PathBuf {
     281            0 :         remote_path.with_base(&self.workdir)
     282            0 :     }
     283              : 
     284              :     //
     285              :     // Postgres distribution paths
     286              :     //
     287           40 :     pub fn pg_distrib_dir(&self, pg_version: u32) -> anyhow::Result<Utf8PathBuf> {
     288           40 :         let path = self.pg_distrib_dir.clone();
     289           40 : 
     290           40 :         #[allow(clippy::manual_range_patterns)]
     291           40 :         match pg_version {
     292           40 :             14 | 15 | 16 | 17 => Ok(path.join(format!("v{pg_version}"))),
     293            0 :             _ => bail!("Unsupported postgres version: {}", pg_version),
     294              :         }
     295           40 :     }
     296              : 
     297           20 :     pub fn pg_bin_dir(&self, pg_version: u32) -> anyhow::Result<Utf8PathBuf> {
     298           20 :         Ok(self.pg_distrib_dir(pg_version)?.join("bin"))
     299           20 :     }
     300           20 :     pub fn pg_lib_dir(&self, pg_version: u32) -> anyhow::Result<Utf8PathBuf> {
     301           20 :         Ok(self.pg_distrib_dir(pg_version)?.join("lib"))
     302           20 :     }
     303              : 
     304              :     /// Parse a configuration file (pageserver.toml) into a PageServerConf struct,
     305              :     /// validating the input and failing on errors.
     306              :     ///
     307              :     /// This leaves any options not present in the file in the built-in defaults.
     308          480 :     pub fn parse_and_validate(
     309          480 :         id: NodeId,
     310          480 :         config_toml: pageserver_api::config::ConfigToml,
     311          480 :         workdir: &Utf8Path,
     312          480 :     ) -> anyhow::Result<Self> {
     313          480 :         let pageserver_api::config::ConfigToml {
     314          480 :             listen_pg_addr,
     315          480 :             listen_http_addr,
     316          480 :             availability_zone,
     317          480 :             wait_lsn_timeout,
     318          480 :             wal_redo_timeout,
     319          480 :             superuser,
     320          480 :             locale,
     321          480 :             page_cache_size,
     322          480 :             max_file_descriptors,
     323          480 :             pg_distrib_dir,
     324          480 :             http_auth_type,
     325          480 :             pg_auth_type,
     326          480 :             auth_validation_public_key_path,
     327          480 :             remote_storage,
     328          480 :             broker_endpoint,
     329          480 :             broker_keepalive_interval,
     330          480 :             log_format,
     331          480 :             metric_collection_interval,
     332          480 :             metric_collection_endpoint,
     333          480 :             metric_collection_bucket,
     334          480 :             synthetic_size_calculation_interval,
     335          480 :             disk_usage_based_eviction,
     336          480 :             test_remote_failures,
     337          480 :             ondemand_download_behavior_treat_error_as_warn,
     338          480 :             background_task_maximum_delay,
     339          480 :             control_plane_api,
     340          480 :             control_plane_api_token,
     341          480 :             control_plane_emergency_mode,
     342          480 :             import_pgdata_upcall_api,
     343          480 :             import_pgdata_upcall_api_token,
     344          480 :             import_pgdata_aws_endpoint_url,
     345          480 :             heatmap_upload_concurrency,
     346          480 :             secondary_download_concurrency,
     347          480 :             ingest_batch_size,
     348          480 :             max_vectored_read_bytes,
     349          480 :             image_compression,
     350          480 :             timeline_offloading,
     351          480 :             ephemeral_bytes_per_memory_kb,
     352          480 :             l0_flush,
     353          480 :             virtual_file_io_mode,
     354          480 :             concurrent_tenant_warmup,
     355          480 :             concurrent_tenant_size_logical_size_queries,
     356          480 :             virtual_file_io_engine,
     357          480 :             tenant_config,
     358          480 :             no_sync,
     359          480 :             wal_receiver_protocol,
     360          480 :             page_service_pipelining,
     361          480 :             get_vectored_concurrent_io,
     362          480 :             enable_read_path_debugging,
     363          480 :         } = config_toml;
     364              : 
     365          480 :         let mut conf = PageServerConf {
     366              :             // ------------------------------------------------------------
     367              :             // fields that are already fully validated by the ConfigToml Deserialize impl
     368              :             // ------------------------------------------------------------
     369          480 :             listen_pg_addr,
     370          480 :             listen_http_addr,
     371          480 :             availability_zone,
     372          480 :             wait_lsn_timeout,
     373          480 :             wal_redo_timeout,
     374          480 :             superuser,
     375          480 :             locale,
     376          480 :             page_cache_size,
     377          480 :             max_file_descriptors,
     378          480 :             http_auth_type,
     379          480 :             pg_auth_type,
     380          480 :             auth_validation_public_key_path,
     381          480 :             remote_storage_config: remote_storage,
     382          480 :             broker_endpoint,
     383          480 :             broker_keepalive_interval,
     384          480 :             log_format,
     385          480 :             metric_collection_interval,
     386          480 :             metric_collection_endpoint,
     387          480 :             metric_collection_bucket,
     388          480 :             synthetic_size_calculation_interval,
     389          480 :             disk_usage_based_eviction,
     390          480 :             test_remote_failures,
     391          480 :             ondemand_download_behavior_treat_error_as_warn,
     392          480 :             background_task_maximum_delay,
     393          480 :             control_plane_api,
     394          480 :             control_plane_emergency_mode,
     395          480 :             heatmap_upload_concurrency,
     396          480 :             secondary_download_concurrency,
     397          480 :             ingest_batch_size,
     398          480 :             max_vectored_read_bytes,
     399          480 :             image_compression,
     400          480 :             timeline_offloading,
     401          480 :             ephemeral_bytes_per_memory_kb,
     402          480 :             import_pgdata_upcall_api,
     403          480 :             import_pgdata_upcall_api_token: import_pgdata_upcall_api_token.map(SecretString::from),
     404          480 :             import_pgdata_aws_endpoint_url,
     405          480 :             wal_receiver_protocol,
     406          480 :             page_service_pipelining,
     407          480 :             get_vectored_concurrent_io,
     408          480 : 
     409          480 :             // ------------------------------------------------------------
     410          480 :             // fields that require additional validation or custom handling
     411          480 :             // ------------------------------------------------------------
     412          480 :             workdir: workdir.to_owned(),
     413          480 :             pg_distrib_dir: pg_distrib_dir.unwrap_or_else(|| {
     414            4 :                 std::env::current_dir()
     415            4 :                     .expect("current_dir() failed")
     416            4 :                     .try_into()
     417            4 :                     .expect("current_dir() is not a valid Utf8Path")
     418          480 :             }),
     419          480 :             control_plane_api_token: control_plane_api_token.map(SecretString::from),
     420          480 :             id,
     421          480 :             default_tenant_conf: tenant_config,
     422          480 :             concurrent_tenant_warmup: ConfigurableSemaphore::new(concurrent_tenant_warmup),
     423          480 :             concurrent_tenant_size_logical_size_queries: ConfigurableSemaphore::new(
     424          480 :                 concurrent_tenant_size_logical_size_queries,
     425          480 :             ),
     426          480 :             eviction_task_immitated_concurrent_logical_size_queries: ConfigurableSemaphore::new(
     427          480 :                 // re-use `concurrent_tenant_size_logical_size_queries`
     428          480 :                 concurrent_tenant_size_logical_size_queries,
     429          480 :             ),
     430          480 :             virtual_file_io_engine: match virtual_file_io_engine {
     431            0 :                 Some(v) => v,
     432          480 :                 None => match crate::virtual_file::io_engine_feature_test()
     433          480 :                     .context("auto-detect virtual_file_io_engine")?
     434              :                 {
     435          480 :                     io_engine::FeatureTestResult::PlatformPreferred(v) => v, // make no noise
     436            0 :                     io_engine::FeatureTestResult::Worse { engine, remark } => {
     437            0 :                         // TODO: bubble this up to the caller so we can tracing::warn! it.
     438            0 :                         eprintln!("auto-detected IO engine is not platform-preferred: engine={engine:?} remark={remark:?}");
     439            0 :                         engine
     440              :                     }
     441              :                 },
     442              :             },
     443          480 :             l0_flush: l0_flush
     444          480 :                 .map(crate::l0_flush::L0FlushConfig::from)
     445          480 :                 .unwrap_or_default(),
     446          480 :             virtual_file_io_mode: virtual_file_io_mode.unwrap_or(virtual_file::IoMode::preferred()),
     447          480 :             no_sync: no_sync.unwrap_or(false),
     448          480 :             enable_read_path_debugging: enable_read_path_debugging.unwrap_or(false),
     449          480 :         };
     450          480 : 
     451          480 :         // ------------------------------------------------------------
     452          480 :         // custom validation code that covers more than one field in isolation
     453          480 :         // ------------------------------------------------------------
     454          480 : 
     455          480 :         if conf.http_auth_type == AuthType::NeonJWT || conf.pg_auth_type == AuthType::NeonJWT {
     456            0 :             let auth_validation_public_key_path = conf
     457            0 :                 .auth_validation_public_key_path
     458            0 :                 .get_or_insert_with(|| workdir.join("auth_public_key.pem"));
     459            0 :             ensure!(
     460            0 :                 auth_validation_public_key_path.exists(),
     461            0 :                 format!(
     462            0 :                     "Can't find auth_validation_public_key at '{auth_validation_public_key_path}'",
     463            0 :                 )
     464              :             );
     465          480 :         }
     466              : 
     467          480 :         IndexEntry::validate_checkpoint_distance(conf.default_tenant_conf.checkpoint_distance)
     468          480 :             .map_err(anyhow::Error::msg)
     469          480 :             .with_context(|| {
     470            0 :                 format!(
     471            0 :                     "effective checkpoint distance is unsupported: {}",
     472            0 :                     conf.default_tenant_conf.checkpoint_distance
     473            0 :                 )
     474          480 :             })?;
     475              : 
     476          480 :         Ok(conf)
     477          480 :     }
     478              : 
     479              :     #[cfg(test)]
     480          480 :     pub fn test_repo_dir(test_name: &str) -> Utf8PathBuf {
     481          480 :         let test_output_dir = std::env::var("TEST_OUTPUT").unwrap_or("../tmp_check".into());
     482          480 :         Utf8PathBuf::from(format!("{test_output_dir}/test_{test_name}"))
     483          480 :     }
     484              : 
     485          476 :     pub fn dummy_conf(repo_dir: Utf8PathBuf) -> Self {
     486          476 :         let pg_distrib_dir = Utf8PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../pg_install");
     487          476 : 
     488          476 :         let config_toml = pageserver_api::config::ConfigToml {
     489          476 :             wait_lsn_timeout: Duration::from_secs(60),
     490          476 :             wal_redo_timeout: Duration::from_secs(60),
     491          476 :             pg_distrib_dir: Some(pg_distrib_dir),
     492          476 :             metric_collection_interval: Duration::from_secs(60),
     493          476 :             synthetic_size_calculation_interval: Duration::from_secs(60),
     494          476 :             background_task_maximum_delay: Duration::ZERO,
     495          476 :             ..Default::default()
     496          476 :         };
     497          476 :         PageServerConf::parse_and_validate(NodeId(0), config_toml, &repo_dir).unwrap()
     498          476 :     }
     499              : }
     500              : 
     501            0 : #[derive(serde::Deserialize, serde::Serialize)]
     502              : #[serde(deny_unknown_fields)]
     503              : pub struct PageserverIdentity {
     504              :     pub id: NodeId,
     505              : }
     506              : 
     507              : /// Configurable semaphore permits setting.
     508              : ///
     509              : /// Does not allow semaphore permits to be zero, because at runtime initially zero permits and empty
     510              : /// semaphore cannot be distinguished, leading any feature using these to await forever (or until
     511              : /// new permits are added).
     512              : #[derive(Debug, Clone)]
     513              : pub struct ConfigurableSemaphore {
     514              :     initial_permits: NonZeroUsize,
     515              :     inner: std::sync::Arc<tokio::sync::Semaphore>,
     516              : }
     517              : 
     518              : impl ConfigurableSemaphore {
     519              :     /// Initializse using a non-zero amount of permits.
     520              :     ///
     521              :     /// Require a non-zero initial permits, because using permits == 0 is a crude way to disable a
     522              :     /// feature such as [`Tenant::gather_size_inputs`]. Otherwise any semaphore using future will
     523              :     /// behave like [`futures::future::pending`], just waiting until new permits are added.
     524              :     ///
     525              :     /// [`Tenant::gather_size_inputs`]: crate::tenant::Tenant::gather_size_inputs
     526         1440 :     pub fn new(initial_permits: NonZeroUsize) -> Self {
     527         1440 :         ConfigurableSemaphore {
     528         1440 :             initial_permits,
     529         1440 :             inner: std::sync::Arc::new(tokio::sync::Semaphore::new(initial_permits.get())),
     530         1440 :         }
     531         1440 :     }
     532              : 
     533              :     /// Returns the configured amount of permits.
     534            0 :     pub fn initial_permits(&self) -> NonZeroUsize {
     535            0 :         self.initial_permits
     536            0 :     }
     537              : }
     538              : 
     539              : impl PartialEq for ConfigurableSemaphore {
     540            0 :     fn eq(&self, other: &Self) -> bool {
     541            0 :         // the number of permits can be increased at runtime, so we cannot really fulfill the
     542            0 :         // PartialEq value equality otherwise
     543            0 :         self.initial_permits == other.initial_permits
     544            0 :     }
     545              : }
     546              : 
     547              : impl Eq for ConfigurableSemaphore {}
     548              : 
     549              : impl ConfigurableSemaphore {
     550            0 :     pub fn inner(&self) -> &std::sync::Arc<tokio::sync::Semaphore> {
     551            0 :         &self.inner
     552            0 :     }
     553              : }
     554              : 
     555              : #[cfg(test)]
     556              : mod tests {
     557              : 
     558              :     use camino::Utf8PathBuf;
     559              :     use utils::id::NodeId;
     560              : 
     561              :     use super::PageServerConf;
     562              : 
     563              :     #[test]
     564            4 :     fn test_empty_config_toml_is_valid() {
     565            4 :         // we use Default impl of everything in this situation
     566            4 :         let input = r#"
     567            4 :         "#;
     568            4 :         let config_toml = toml_edit::de::from_str::<pageserver_api::config::ConfigToml>(input)
     569            4 :             .expect("empty config is valid");
     570            4 :         let workdir = Utf8PathBuf::from("/nonexistent");
     571            4 :         PageServerConf::parse_and_validate(NodeId(0), config_toml, &workdir)
     572            4 :             .expect("parse_and_validate");
     573            4 :     }
     574              : 
     575              :     /// If there's a typo in the pageserver config, we'd rather catch that typo
     576              :     /// and fail pageserver startup than silently ignoring the typo, leaving whoever
     577              :     /// made it in the believe that their config change is effective.
     578              :     ///
     579              :     /// The default in serde is to allow unknown fields, so, we rely
     580              :     /// on developer+review discipline to add `deny_unknown_fields` when adding
     581              :     /// new structs to the config, and these tests here as a regression test.
     582              :     ///
     583              :     /// The alternative to all of this would be to allow unknown fields in the config.
     584              :     /// To catch them, we could have a config check tool or mgmt API endpoint that
     585              :     /// compares the effective config with the TOML on disk and makes sure that
     586              :     /// the on-disk TOML is a strict subset of the effective config.
     587              :     mod unknown_fields_handling {
     588              :         macro_rules! test {
     589              :             ($short_name:ident, $input:expr) => {
     590              :                 #[test]
     591           20 :                 fn $short_name() {
     592           20 :                     let input = $input;
     593           20 :                     let err = toml_edit::de::from_str::<pageserver_api::config::ConfigToml>(&input)
     594           20 :                         .expect_err("some_invalid_field is an invalid field");
     595           20 :                     dbg!(&err);
     596           20 :                     assert!(err.to_string().contains("some_invalid_field"));
     597           20 :                 }
     598              :             };
     599              :         }
     600              :         use indoc::indoc;
     601              : 
     602              :         test!(
     603              :             toplevel,
     604              :             indoc! {r#"
     605              :                 some_invalid_field = 23
     606              :             "#}
     607              :         );
     608              : 
     609              :         test!(
     610              :             toplevel_nested,
     611              :             indoc! {r#"
     612              :                 [some_invalid_field]
     613              :                 foo = 23
     614              :             "#}
     615              :         );
     616              : 
     617              :         test!(
     618              :             disk_usage_based_eviction,
     619              :             indoc! {r#"
     620              :                 [disk_usage_based_eviction]
     621              :                 some_invalid_field = 23
     622              :             "#}
     623              :         );
     624              : 
     625              :         test!(
     626              :             tenant_config,
     627              :             indoc! {r#"
     628              :                 [tenant_config]
     629              :                 some_invalid_field = 23
     630              :             "#}
     631              :         );
     632              : 
     633              :         test!(
     634              :             l0_flush,
     635              :             indoc! {r#"
     636              :                 [l0_flush]
     637              :                 mode = "direct"
     638              :                 some_invalid_field = 23
     639              :             "#}
     640              :         );
     641              : 
     642              :         // TODO: fix this => https://github.com/neondatabase/neon/issues/8915
     643              :         // test!(
     644              :         //     remote_storage_config,
     645              :         //     indoc! {r#"
     646              :         //         [remote_storage_config]
     647              :         //         local_path = "/nonexistent"
     648              :         //         some_invalid_field = 23
     649              :         //     "#}
     650              :         // );
     651              :     }
     652              : }
        

Generated by: LCOV version 2.1-beta