Line data Source code
1 : //! Functions for handling page server configuration options
2 : //!
3 : //! Configuration options can be set in the pageserver.toml configuration
4 : //! file, or on the command line.
5 : //! See also `settings.md` for better description on every parameter.
6 :
7 : use anyhow::{bail, ensure, Context};
8 : use pageserver_api::models::ImageCompressionAlgorithm;
9 : use pageserver_api::{
10 : config::{DiskUsageEvictionTaskConfig, MaxVectoredReadBytes},
11 : shard::TenantShardId,
12 : };
13 : use remote_storage::{RemotePath, RemoteStorageConfig};
14 : use std::env;
15 : use storage_broker::Uri;
16 : use utils::logging::SecretString;
17 :
18 : use once_cell::sync::OnceCell;
19 : use reqwest::Url;
20 : use std::num::NonZeroUsize;
21 : use std::sync::Arc;
22 : use std::time::Duration;
23 :
24 : use camino::{Utf8Path, Utf8PathBuf};
25 : use postgres_backend::AuthType;
26 : use utils::{
27 : id::{NodeId, TimelineId},
28 : logging::LogFormat,
29 : };
30 :
31 : use crate::tenant::storage_layer::inmemory_layer::IndexEntry;
32 : use crate::tenant::{TENANTS_SEGMENT_NAME, TIMELINES_SEGMENT_NAME};
33 : use crate::virtual_file;
34 : use crate::virtual_file::io_engine;
35 : use crate::{TENANT_HEATMAP_BASENAME, TENANT_LOCATION_CONFIG_NAME};
36 :
37 : /// Global state of pageserver.
38 : ///
39 : /// It's mostly immutable configuration, but some semaphores and the
40 : /// like crept in over time and the name stuck.
41 : ///
42 : /// Instantiated by deserializing `pageserver.toml` into [`pageserver_api::config::ConfigToml`]
43 : /// and passing that to [`PageServerConf::parse_and_validate`].
44 : ///
45 : /// # Adding a New Field
46 : ///
47 : /// 1. Add the field to `pageserver_api::config::ConfigToml`.
48 : /// 2. Fix compiler errors (exhaustive destructuring will guide you).
49 : ///
50 : /// For fields that require additional validation or filling in of defaults at runtime,
51 : /// check for examples in the [`PageServerConf::parse_and_validate`] method.
52 : #[derive(Debug, Clone, PartialEq, Eq)]
53 : pub struct PageServerConf {
54 : // Identifier of that particular pageserver so e g safekeepers
55 : // can safely distinguish different pageservers
56 : pub id: NodeId,
57 :
58 : /// Example (default): 127.0.0.1:64000
59 : pub listen_pg_addr: String,
60 : /// Example (default): 127.0.0.1:9898
61 : pub listen_http_addr: String,
62 :
63 : /// Current availability zone. Used for traffic metrics.
64 : pub availability_zone: Option<String>,
65 :
66 : // Timeout when waiting for WAL receiver to catch up to an LSN given in a GetPage@LSN call.
67 : pub wait_lsn_timeout: Duration,
68 : // How long to wait for WAL redo to complete.
69 : pub wal_redo_timeout: Duration,
70 :
71 : pub superuser: String,
72 : pub locale: String,
73 :
74 : pub page_cache_size: usize,
75 : pub max_file_descriptors: usize,
76 :
77 : // Repository directory, relative to current working directory.
78 : // Normally, the page server changes the current working directory
79 : // to the repository, and 'workdir' is always '.'. But we don't do
80 : // that during unit testing, because the current directory is global
81 : // to the process but different unit tests work on different
82 : // repositories.
83 : pub workdir: Utf8PathBuf,
84 :
85 : pub pg_distrib_dir: Utf8PathBuf,
86 :
87 : // Authentication
88 : /// authentication method for the HTTP mgmt API
89 : pub http_auth_type: AuthType,
90 : /// authentication method for libpq connections from compute
91 : pub pg_auth_type: AuthType,
92 : /// Path to a file or directory containing public key(s) for verifying JWT tokens.
93 : /// Used for both mgmt and compute auth, if enabled.
94 : pub auth_validation_public_key_path: Option<Utf8PathBuf>,
95 :
96 : pub remote_storage_config: Option<RemoteStorageConfig>,
97 :
98 : pub default_tenant_conf: crate::tenant::config::TenantConf,
99 :
100 : /// Storage broker endpoints to connect to.
101 : pub broker_endpoint: Uri,
102 : pub broker_keepalive_interval: Duration,
103 :
104 : pub log_format: LogFormat,
105 :
106 : /// Number of tenants which will be concurrently loaded from remote storage proactively on startup or attach.
107 : ///
108 : /// A lower value implicitly deprioritizes loading such tenants, vs. other work in the system.
109 : pub concurrent_tenant_warmup: ConfigurableSemaphore,
110 :
111 : /// Number of concurrent [`Tenant::gather_size_inputs`](crate::tenant::Tenant::gather_size_inputs) allowed.
112 : pub concurrent_tenant_size_logical_size_queries: ConfigurableSemaphore,
113 : /// Limit of concurrent [`Tenant::gather_size_inputs`] issued by module `eviction_task`.
114 : /// The number of permits is the same as `concurrent_tenant_size_logical_size_queries`.
115 : /// See the comment in `eviction_task` for details.
116 : ///
117 : /// [`Tenant::gather_size_inputs`]: crate::tenant::Tenant::gather_size_inputs
118 : pub eviction_task_immitated_concurrent_logical_size_queries: ConfigurableSemaphore,
119 :
120 : // How often to collect metrics and send them to the metrics endpoint.
121 : pub metric_collection_interval: Duration,
122 : // How often to send unchanged cached metrics to the metrics endpoint.
123 : pub metric_collection_endpoint: Option<Url>,
124 : pub metric_collection_bucket: Option<RemoteStorageConfig>,
125 : pub synthetic_size_calculation_interval: Duration,
126 :
127 : pub disk_usage_based_eviction: Option<DiskUsageEvictionTaskConfig>,
128 :
129 : pub test_remote_failures: u64,
130 :
131 : pub ondemand_download_behavior_treat_error_as_warn: bool,
132 :
133 : /// How long will background tasks be delayed at most after initial load of tenants.
134 : ///
135 : /// Our largest initialization completions are in the range of 100-200s, so perhaps 10s works
136 : /// as we now isolate initial loading, initial logical size calculation and background tasks.
137 : /// Smaller nodes will have background tasks "not running" for this long unless every timeline
138 : /// has it's initial logical size calculated. Not running background tasks for some seconds is
139 : /// not terrible.
140 : pub background_task_maximum_delay: Duration,
141 :
142 : pub control_plane_api: Option<Url>,
143 :
144 : /// JWT token for use with the control plane API.
145 : pub control_plane_api_token: Option<SecretString>,
146 :
147 : pub import_pgdata_upcall_api: Option<Url>,
148 : pub import_pgdata_upcall_api_token: Option<SecretString>,
149 : pub import_pgdata_aws_endpoint_url: Option<Url>,
150 :
151 : /// If true, pageserver will make best-effort to operate without a control plane: only
152 : /// for use in major incidents.
153 : pub control_plane_emergency_mode: bool,
154 :
155 : /// How many heatmap uploads may be done concurrency: lower values implicitly deprioritize
156 : /// heatmap uploads vs. other remote storage operations.
157 : pub heatmap_upload_concurrency: usize,
158 :
159 : /// How many remote storage downloads may be done for secondary tenants concurrently. Implicitly
160 : /// deprioritises secondary downloads vs. remote storage operations for attached tenants.
161 : pub secondary_download_concurrency: usize,
162 :
163 : /// Maximum number of WAL records to be ingested and committed at the same time
164 : pub ingest_batch_size: u64,
165 :
166 : pub virtual_file_io_engine: virtual_file::IoEngineKind,
167 :
168 : pub max_vectored_read_bytes: MaxVectoredReadBytes,
169 :
170 : pub image_compression: ImageCompressionAlgorithm,
171 :
172 : /// Whether to offload archived timelines automatically
173 : pub timeline_offloading: bool,
174 :
175 : /// How many bytes of ephemeral layer content will we allow per kilobyte of RAM. When this
176 : /// is exceeded, we start proactively closing ephemeral layers to limit the total amount
177 : /// of ephemeral data.
178 : ///
179 : /// Setting this to zero disables limits on total ephemeral layer size.
180 : pub ephemeral_bytes_per_memory_kb: usize,
181 :
182 : pub l0_flush: crate::l0_flush::L0FlushConfig,
183 :
184 : /// Direct IO settings
185 : pub virtual_file_io_mode: virtual_file::IoMode,
186 :
187 : /// Optionally disable disk syncs (unsafe!)
188 : pub no_sync: bool,
189 :
190 : /// Maximum amount of time for which a get page request request
191 : /// might be held up for request merging.
192 : pub server_side_batch_timeout: Option<Duration>,
193 : }
194 :
195 : /// Token for authentication to safekeepers
196 : ///
197 : /// We do not want to store this in a PageServerConf because the latter may be logged
198 : /// and/or serialized at a whim, while the token is secret. Currently this token is the
199 : /// same for accessing all tenants/timelines, but may become per-tenant/per-timeline in
200 : /// the future, more tokens and auth may arrive for storage broker, completely changing the logic.
201 : /// Hence, we resort to a global variable for now instead of passing the token from the
202 : /// startup code to the connection code through a dozen layers.
203 : pub static SAFEKEEPER_AUTH_TOKEN: OnceCell<Arc<String>> = OnceCell::new();
204 :
205 : impl PageServerConf {
206 : //
207 : // Repository paths, relative to workdir.
208 : //
209 :
210 7042 : pub fn tenants_path(&self) -> Utf8PathBuf {
211 7042 : self.workdir.join(TENANTS_SEGMENT_NAME)
212 7042 : }
213 :
214 72 : pub fn deletion_prefix(&self) -> Utf8PathBuf {
215 72 : self.workdir.join("deletion")
216 72 : }
217 :
218 0 : pub fn metadata_path(&self) -> Utf8PathBuf {
219 0 : self.workdir.join("metadata.json")
220 0 : }
221 :
222 28 : pub fn deletion_list_path(&self, sequence: u64) -> Utf8PathBuf {
223 : // Encode a version in the filename, so that if we ever switch away from JSON we can
224 : // increment this.
225 : const VERSION: u8 = 1;
226 :
227 28 : self.deletion_prefix()
228 28 : .join(format!("{sequence:016x}-{VERSION:02x}.list"))
229 28 : }
230 :
231 24 : pub fn deletion_header_path(&self) -> Utf8PathBuf {
232 : // Encode a version in the filename, so that if we ever switch away from JSON we can
233 : // increment this.
234 : const VERSION: u8 = 1;
235 :
236 24 : self.deletion_prefix().join(format!("header-{VERSION:02x}"))
237 24 : }
238 :
239 7002 : pub fn tenant_path(&self, tenant_shard_id: &TenantShardId) -> Utf8PathBuf {
240 7002 : self.tenants_path().join(tenant_shard_id.to_string())
241 7002 : }
242 :
243 : /// Points to a place in pageserver's local directory,
244 : /// where certain tenant's LocationConf be stored.
245 0 : pub(crate) fn tenant_location_config_path(
246 0 : &self,
247 0 : tenant_shard_id: &TenantShardId,
248 0 : ) -> Utf8PathBuf {
249 0 : self.tenant_path(tenant_shard_id)
250 0 : .join(TENANT_LOCATION_CONFIG_NAME)
251 0 : }
252 :
253 0 : pub(crate) fn tenant_heatmap_path(&self, tenant_shard_id: &TenantShardId) -> Utf8PathBuf {
254 0 : self.tenant_path(tenant_shard_id)
255 0 : .join(TENANT_HEATMAP_BASENAME)
256 0 : }
257 :
258 6806 : pub fn timelines_path(&self, tenant_shard_id: &TenantShardId) -> Utf8PathBuf {
259 6806 : self.tenant_path(tenant_shard_id)
260 6806 : .join(TIMELINES_SEGMENT_NAME)
261 6806 : }
262 :
263 6416 : pub fn timeline_path(
264 6416 : &self,
265 6416 : tenant_shard_id: &TenantShardId,
266 6416 : timeline_id: &TimelineId,
267 6416 : ) -> Utf8PathBuf {
268 6416 : self.timelines_path(tenant_shard_id)
269 6416 : .join(timeline_id.to_string())
270 6416 : }
271 :
272 : /// Turns storage remote path of a file into its local path.
273 0 : pub fn local_path(&self, remote_path: &RemotePath) -> Utf8PathBuf {
274 0 : remote_path.with_base(&self.workdir)
275 0 : }
276 :
277 : //
278 : // Postgres distribution paths
279 : //
280 20 : pub fn pg_distrib_dir(&self, pg_version: u32) -> anyhow::Result<Utf8PathBuf> {
281 20 : let path = self.pg_distrib_dir.clone();
282 20 :
283 20 : #[allow(clippy::manual_range_patterns)]
284 20 : match pg_version {
285 20 : 14 | 15 | 16 | 17 => Ok(path.join(format!("v{pg_version}"))),
286 0 : _ => bail!("Unsupported postgres version: {}", pg_version),
287 : }
288 20 : }
289 :
290 10 : pub fn pg_bin_dir(&self, pg_version: u32) -> anyhow::Result<Utf8PathBuf> {
291 10 : Ok(self.pg_distrib_dir(pg_version)?.join("bin"))
292 10 : }
293 10 : pub fn pg_lib_dir(&self, pg_version: u32) -> anyhow::Result<Utf8PathBuf> {
294 10 : Ok(self.pg_distrib_dir(pg_version)?.join("lib"))
295 10 : }
296 :
297 : /// Parse a configuration file (pageserver.toml) into a PageServerConf struct,
298 : /// validating the input and failing on errors.
299 : ///
300 : /// This leaves any options not present in the file in the built-in defaults.
301 210 : pub fn parse_and_validate(
302 210 : id: NodeId,
303 210 : config_toml: pageserver_api::config::ConfigToml,
304 210 : workdir: &Utf8Path,
305 210 : ) -> anyhow::Result<Self> {
306 210 : let pageserver_api::config::ConfigToml {
307 210 : listen_pg_addr,
308 210 : listen_http_addr,
309 210 : availability_zone,
310 210 : wait_lsn_timeout,
311 210 : wal_redo_timeout,
312 210 : superuser,
313 210 : locale,
314 210 : page_cache_size,
315 210 : max_file_descriptors,
316 210 : pg_distrib_dir,
317 210 : http_auth_type,
318 210 : pg_auth_type,
319 210 : auth_validation_public_key_path,
320 210 : remote_storage,
321 210 : broker_endpoint,
322 210 : broker_keepalive_interval,
323 210 : log_format,
324 210 : metric_collection_interval,
325 210 : metric_collection_endpoint,
326 210 : metric_collection_bucket,
327 210 : synthetic_size_calculation_interval,
328 210 : disk_usage_based_eviction,
329 210 : test_remote_failures,
330 210 : ondemand_download_behavior_treat_error_as_warn,
331 210 : background_task_maximum_delay,
332 210 : control_plane_api,
333 210 : control_plane_api_token,
334 210 : control_plane_emergency_mode,
335 210 : import_pgdata_upcall_api,
336 210 : import_pgdata_upcall_api_token,
337 210 : import_pgdata_aws_endpoint_url,
338 210 : heatmap_upload_concurrency,
339 210 : secondary_download_concurrency,
340 210 : ingest_batch_size,
341 210 : max_vectored_read_bytes,
342 210 : image_compression,
343 210 : timeline_offloading,
344 210 : ephemeral_bytes_per_memory_kb,
345 210 : l0_flush,
346 210 : virtual_file_io_mode,
347 210 : concurrent_tenant_warmup,
348 210 : concurrent_tenant_size_logical_size_queries,
349 210 : virtual_file_io_engine,
350 210 : server_side_batch_timeout,
351 210 : tenant_config,
352 210 : no_sync,
353 210 : } = config_toml;
354 :
355 210 : let mut conf = PageServerConf {
356 : // ------------------------------------------------------------
357 : // fields that are already fully validated by the ConfigToml Deserialize impl
358 : // ------------------------------------------------------------
359 210 : listen_pg_addr,
360 210 : listen_http_addr,
361 210 : availability_zone,
362 210 : wait_lsn_timeout,
363 210 : wal_redo_timeout,
364 210 : superuser,
365 210 : locale,
366 210 : page_cache_size,
367 210 : max_file_descriptors,
368 210 : http_auth_type,
369 210 : pg_auth_type,
370 210 : auth_validation_public_key_path,
371 210 : remote_storage_config: remote_storage,
372 210 : broker_endpoint,
373 210 : broker_keepalive_interval,
374 210 : log_format,
375 210 : metric_collection_interval,
376 210 : metric_collection_endpoint,
377 210 : metric_collection_bucket,
378 210 : synthetic_size_calculation_interval,
379 210 : disk_usage_based_eviction,
380 210 : test_remote_failures,
381 210 : ondemand_download_behavior_treat_error_as_warn,
382 210 : background_task_maximum_delay,
383 210 : control_plane_api,
384 210 : control_plane_emergency_mode,
385 210 : heatmap_upload_concurrency,
386 210 : secondary_download_concurrency,
387 210 : ingest_batch_size,
388 210 : max_vectored_read_bytes,
389 210 : image_compression,
390 210 : timeline_offloading,
391 210 : ephemeral_bytes_per_memory_kb,
392 210 : server_side_batch_timeout,
393 210 : import_pgdata_upcall_api,
394 210 : import_pgdata_upcall_api_token: import_pgdata_upcall_api_token.map(SecretString::from),
395 210 : import_pgdata_aws_endpoint_url,
396 210 :
397 210 : // ------------------------------------------------------------
398 210 : // fields that require additional validation or custom handling
399 210 : // ------------------------------------------------------------
400 210 : workdir: workdir.to_owned(),
401 210 : pg_distrib_dir: pg_distrib_dir.unwrap_or_else(|| {
402 2 : std::env::current_dir()
403 2 : .expect("current_dir() failed")
404 2 : .try_into()
405 2 : .expect("current_dir() is not a valid Utf8Path")
406 210 : }),
407 210 : control_plane_api_token: control_plane_api_token.map(SecretString::from),
408 210 : id,
409 210 : default_tenant_conf: tenant_config,
410 210 : concurrent_tenant_warmup: ConfigurableSemaphore::new(concurrent_tenant_warmup),
411 210 : concurrent_tenant_size_logical_size_queries: ConfigurableSemaphore::new(
412 210 : concurrent_tenant_size_logical_size_queries,
413 210 : ),
414 210 : eviction_task_immitated_concurrent_logical_size_queries: ConfigurableSemaphore::new(
415 210 : // re-use `concurrent_tenant_size_logical_size_queries`
416 210 : concurrent_tenant_size_logical_size_queries,
417 210 : ),
418 210 : virtual_file_io_engine: match virtual_file_io_engine {
419 0 : Some(v) => v,
420 210 : None => match crate::virtual_file::io_engine_feature_test()
421 210 : .context("auto-detect virtual_file_io_engine")?
422 : {
423 210 : io_engine::FeatureTestResult::PlatformPreferred(v) => v, // make no noise
424 0 : io_engine::FeatureTestResult::Worse { engine, remark } => {
425 0 : // TODO: bubble this up to the caller so we can tracing::warn! it.
426 0 : eprintln!("auto-detected IO engine is not platform-preferred: engine={engine:?} remark={remark:?}");
427 0 : engine
428 : }
429 : },
430 : },
431 210 : l0_flush: l0_flush
432 210 : .map(crate::l0_flush::L0FlushConfig::from)
433 210 : .unwrap_or_default(),
434 210 : virtual_file_io_mode: virtual_file_io_mode.unwrap_or(virtual_file::IoMode::preferred()),
435 210 : no_sync: no_sync.unwrap_or(false),
436 210 : };
437 210 :
438 210 : // ------------------------------------------------------------
439 210 : // custom validation code that covers more than one field in isolation
440 210 : // ------------------------------------------------------------
441 210 :
442 210 : if conf.http_auth_type == AuthType::NeonJWT || conf.pg_auth_type == AuthType::NeonJWT {
443 0 : let auth_validation_public_key_path = conf
444 0 : .auth_validation_public_key_path
445 0 : .get_or_insert_with(|| workdir.join("auth_public_key.pem"));
446 0 : ensure!(
447 0 : auth_validation_public_key_path.exists(),
448 0 : format!(
449 0 : "Can't find auth_validation_public_key at '{auth_validation_public_key_path}'",
450 0 : )
451 : );
452 210 : }
453 :
454 210 : IndexEntry::validate_checkpoint_distance(conf.default_tenant_conf.checkpoint_distance)
455 210 : .map_err(anyhow::Error::msg)
456 210 : .with_context(|| {
457 0 : format!(
458 0 : "effective checkpoint distance is unsupported: {}",
459 0 : conf.default_tenant_conf.checkpoint_distance
460 0 : )
461 210 : })?;
462 :
463 210 : Ok(conf)
464 210 : }
465 :
466 : #[cfg(test)]
467 210 : pub fn test_repo_dir(test_name: &str) -> Utf8PathBuf {
468 210 : let test_output_dir = std::env::var("TEST_OUTPUT").unwrap_or("../tmp_check".into());
469 210 : Utf8PathBuf::from(format!("{test_output_dir}/test_{test_name}"))
470 210 : }
471 :
472 208 : pub fn dummy_conf(repo_dir: Utf8PathBuf) -> Self {
473 208 : let pg_distrib_dir = Utf8PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../pg_install");
474 208 :
475 208 : let config_toml = pageserver_api::config::ConfigToml {
476 208 : wait_lsn_timeout: Duration::from_secs(60),
477 208 : wal_redo_timeout: Duration::from_secs(60),
478 208 : pg_distrib_dir: Some(pg_distrib_dir),
479 208 : metric_collection_interval: Duration::from_secs(60),
480 208 : synthetic_size_calculation_interval: Duration::from_secs(60),
481 208 : background_task_maximum_delay: Duration::ZERO,
482 208 : ..Default::default()
483 208 : };
484 208 : PageServerConf::parse_and_validate(NodeId(0), config_toml, &repo_dir).unwrap()
485 208 : }
486 : }
487 :
488 0 : #[derive(serde::Deserialize, serde::Serialize)]
489 : #[serde(deny_unknown_fields)]
490 : pub struct PageserverIdentity {
491 : pub id: NodeId,
492 : }
493 :
494 : /// Configurable semaphore permits setting.
495 : ///
496 : /// Does not allow semaphore permits to be zero, because at runtime initially zero permits and empty
497 : /// semaphore cannot be distinguished, leading any feature using these to await forever (or until
498 : /// new permits are added).
499 : #[derive(Debug, Clone)]
500 : pub struct ConfigurableSemaphore {
501 : initial_permits: NonZeroUsize,
502 : inner: std::sync::Arc<tokio::sync::Semaphore>,
503 : }
504 :
505 : impl ConfigurableSemaphore {
506 : /// Initializse using a non-zero amount of permits.
507 : ///
508 : /// Require a non-zero initial permits, because using permits == 0 is a crude way to disable a
509 : /// feature such as [`Tenant::gather_size_inputs`]. Otherwise any semaphore using future will
510 : /// behave like [`futures::future::pending`], just waiting until new permits are added.
511 : ///
512 : /// [`Tenant::gather_size_inputs`]: crate::tenant::Tenant::gather_size_inputs
513 630 : pub fn new(initial_permits: NonZeroUsize) -> Self {
514 630 : ConfigurableSemaphore {
515 630 : initial_permits,
516 630 : inner: std::sync::Arc::new(tokio::sync::Semaphore::new(initial_permits.get())),
517 630 : }
518 630 : }
519 :
520 : /// Returns the configured amount of permits.
521 0 : pub fn initial_permits(&self) -> NonZeroUsize {
522 0 : self.initial_permits
523 0 : }
524 : }
525 :
526 : impl PartialEq for ConfigurableSemaphore {
527 0 : fn eq(&self, other: &Self) -> bool {
528 0 : // the number of permits can be increased at runtime, so we cannot really fulfill the
529 0 : // PartialEq value equality otherwise
530 0 : self.initial_permits == other.initial_permits
531 0 : }
532 : }
533 :
534 : impl Eq for ConfigurableSemaphore {}
535 :
536 : impl ConfigurableSemaphore {
537 0 : pub fn inner(&self) -> &std::sync::Arc<tokio::sync::Semaphore> {
538 0 : &self.inner
539 0 : }
540 : }
541 :
542 : #[cfg(test)]
543 : mod tests {
544 :
545 : use camino::Utf8PathBuf;
546 : use utils::id::NodeId;
547 :
548 : use super::PageServerConf;
549 :
550 : #[test]
551 2 : fn test_empty_config_toml_is_valid() {
552 2 : // we use Default impl of everything in this situation
553 2 : let input = r#"
554 2 : "#;
555 2 : let config_toml = toml_edit::de::from_str::<pageserver_api::config::ConfigToml>(input)
556 2 : .expect("empty config is valid");
557 2 : let workdir = Utf8PathBuf::from("/nonexistent");
558 2 : PageServerConf::parse_and_validate(NodeId(0), config_toml, &workdir)
559 2 : .expect("parse_and_validate");
560 2 : }
561 :
562 : /// If there's a typo in the pageserver config, we'd rather catch that typo
563 : /// and fail pageserver startup than silently ignoring the typo, leaving whoever
564 : /// made it in the believe that their config change is effective.
565 : ///
566 : /// The default in serde is to allow unknown fields, so, we rely
567 : /// on developer+review discipline to add `deny_unknown_fields` when adding
568 : /// new structs to the config, and these tests here as a regression test.
569 : ///
570 : /// The alternative to all of this would be to allow unknown fields in the config.
571 : /// To catch them, we could have a config check tool or mgmt API endpoint that
572 : /// compares the effective config with the TOML on disk and makes sure that
573 : /// the on-disk TOML is a strict subset of the effective config.
574 : mod unknown_fields_handling {
575 : macro_rules! test {
576 : ($short_name:ident, $input:expr) => {
577 : #[test]
578 10 : fn $short_name() {
579 10 : let input = $input;
580 10 : let err = toml_edit::de::from_str::<pageserver_api::config::ConfigToml>(&input)
581 10 : .expect_err("some_invalid_field is an invalid field");
582 10 : dbg!(&err);
583 10 : assert!(err.to_string().contains("some_invalid_field"));
584 10 : }
585 : };
586 : }
587 : use indoc::indoc;
588 :
589 : test!(
590 : toplevel,
591 : indoc! {r#"
592 : some_invalid_field = 23
593 : "#}
594 : );
595 :
596 : test!(
597 : toplevel_nested,
598 : indoc! {r#"
599 : [some_invalid_field]
600 : foo = 23
601 : "#}
602 : );
603 :
604 : test!(
605 : disk_usage_based_eviction,
606 : indoc! {r#"
607 : [disk_usage_based_eviction]
608 : some_invalid_field = 23
609 : "#}
610 : );
611 :
612 : test!(
613 : tenant_config,
614 : indoc! {r#"
615 : [tenant_config]
616 : some_invalid_field = 23
617 : "#}
618 : );
619 :
620 : test!(
621 : l0_flush,
622 : indoc! {r#"
623 : [l0_flush]
624 : mode = "direct"
625 : some_invalid_field = 23
626 : "#}
627 : );
628 :
629 : // TODO: fix this => https://github.com/neondatabase/neon/issues/8915
630 : // test!(
631 : // remote_storage_config,
632 : // indoc! {r#"
633 : // [remote_storage_config]
634 : // local_path = "/nonexistent"
635 : // some_invalid_field = 23
636 : // "#}
637 : // );
638 : }
639 : }
|
