Line data Source code
1 : //! Functions for handling page server configuration options
2 : //!
3 : //! Configuration options can be set in the pageserver.toml configuration
4 : //! file, or on the command line.
5 : //! See also `settings.md` for better description on every parameter.
6 :
7 : use anyhow::{bail, ensure, Context};
8 : use pageserver_api::models::ImageCompressionAlgorithm;
9 : use pageserver_api::{
10 : config::{DiskUsageEvictionTaskConfig, MaxVectoredReadBytes},
11 : shard::TenantShardId,
12 : };
13 : use remote_storage::{RemotePath, RemoteStorageConfig};
14 : use std::env;
15 : use storage_broker::Uri;
16 : use utils::logging::SecretString;
17 :
18 : use once_cell::sync::OnceCell;
19 : use reqwest::Url;
20 : use std::num::NonZeroUsize;
21 : use std::sync::Arc;
22 : use std::time::Duration;
23 :
24 : use camino::{Utf8Path, Utf8PathBuf};
25 : use postgres_backend::AuthType;
26 : use utils::{
27 : id::{NodeId, TimelineId},
28 : logging::LogFormat,
29 : };
30 :
31 : use crate::tenant::storage_layer::inmemory_layer::IndexEntry;
32 : use crate::tenant::{TENANTS_SEGMENT_NAME, TIMELINES_SEGMENT_NAME};
33 : use crate::virtual_file;
34 : use crate::virtual_file::io_engine;
35 : use crate::{TENANT_HEATMAP_BASENAME, TENANT_LOCATION_CONFIG_NAME};
36 :
37 : /// Global state of pageserver.
38 : ///
39 : /// It's mostly immutable configuration, but some semaphores and the
40 : /// like crept in over time and the name stuck.
41 : ///
42 : /// Instantiated by deserializing `pageserver.toml` into [`pageserver_api::config::ConfigToml`]
43 : /// and passing that to [`PageServerConf::parse_and_validate`].
44 : ///
45 : /// # Adding a New Field
46 : ///
47 : /// 1. Add the field to `pageserver_api::config::ConfigToml`.
48 : /// 2. Fix compiler errors (exhaustive destructuring will guide you).
49 : ///
50 : /// For fields that require additional validation or filling in of defaults at runtime,
51 : /// check for examples in the [`PageServerConf::parse_and_validate`] method.
52 : #[derive(Debug, Clone, PartialEq, Eq)]
53 : pub struct PageServerConf {
54 : // Identifier of that particular pageserver so e g safekeepers
55 : // can safely distinguish different pageservers
56 : pub id: NodeId,
57 :
58 : /// Example (default): 127.0.0.1:64000
59 : pub listen_pg_addr: String,
60 : /// Example (default): 127.0.0.1:9898
61 : pub listen_http_addr: String,
62 :
63 : /// Current availability zone. Used for traffic metrics.
64 : pub availability_zone: Option<String>,
65 :
66 : // Timeout when waiting for WAL receiver to catch up to an LSN given in a GetPage@LSN call.
67 : pub wait_lsn_timeout: Duration,
68 : // How long to wait for WAL redo to complete.
69 : pub wal_redo_timeout: Duration,
70 :
71 : pub superuser: String,
72 :
73 : pub page_cache_size: usize,
74 : pub max_file_descriptors: usize,
75 :
76 : // Repository directory, relative to current working directory.
77 : // Normally, the page server changes the current working directory
78 : // to the repository, and 'workdir' is always '.'. But we don't do
79 : // that during unit testing, because the current directory is global
80 : // to the process but different unit tests work on different
81 : // repositories.
82 : pub workdir: Utf8PathBuf,
83 :
84 : pub pg_distrib_dir: Utf8PathBuf,
85 :
86 : // Authentication
87 : /// authentication method for the HTTP mgmt API
88 : pub http_auth_type: AuthType,
89 : /// authentication method for libpq connections from compute
90 : pub pg_auth_type: AuthType,
91 : /// Path to a file or directory containing public key(s) for verifying JWT tokens.
92 : /// Used for both mgmt and compute auth, if enabled.
93 : pub auth_validation_public_key_path: Option<Utf8PathBuf>,
94 :
95 : pub remote_storage_config: Option<RemoteStorageConfig>,
96 :
97 : pub default_tenant_conf: crate::tenant::config::TenantConf,
98 :
99 : /// Storage broker endpoints to connect to.
100 : pub broker_endpoint: Uri,
101 : pub broker_keepalive_interval: Duration,
102 :
103 : pub log_format: LogFormat,
104 :
105 : /// Number of tenants which will be concurrently loaded from remote storage proactively on startup or attach.
106 : ///
107 : /// A lower value implicitly deprioritizes loading such tenants, vs. other work in the system.
108 : pub concurrent_tenant_warmup: ConfigurableSemaphore,
109 :
110 : /// Number of concurrent [`Tenant::gather_size_inputs`](crate::tenant::Tenant::gather_size_inputs) allowed.
111 : pub concurrent_tenant_size_logical_size_queries: ConfigurableSemaphore,
112 : /// Limit of concurrent [`Tenant::gather_size_inputs`] issued by module `eviction_task`.
113 : /// The number of permits is the same as `concurrent_tenant_size_logical_size_queries`.
114 : /// See the comment in `eviction_task` for details.
115 : ///
116 : /// [`Tenant::gather_size_inputs`]: crate::tenant::Tenant::gather_size_inputs
117 : pub eviction_task_immitated_concurrent_logical_size_queries: ConfigurableSemaphore,
118 :
119 : // How often to collect metrics and send them to the metrics endpoint.
120 : pub metric_collection_interval: Duration,
121 : // How often to send unchanged cached metrics to the metrics endpoint.
122 : pub metric_collection_endpoint: Option<Url>,
123 : pub metric_collection_bucket: Option<RemoteStorageConfig>,
124 : pub synthetic_size_calculation_interval: Duration,
125 :
126 : pub disk_usage_based_eviction: Option<DiskUsageEvictionTaskConfig>,
127 :
128 : pub test_remote_failures: u64,
129 :
130 : pub ondemand_download_behavior_treat_error_as_warn: bool,
131 :
132 : /// How long will background tasks be delayed at most after initial load of tenants.
133 : ///
134 : /// Our largest initialization completions are in the range of 100-200s, so perhaps 10s works
135 : /// as we now isolate initial loading, initial logical size calculation and background tasks.
136 : /// Smaller nodes will have background tasks "not running" for this long unless every timeline
137 : /// has it's initial logical size calculated. Not running background tasks for some seconds is
138 : /// not terrible.
139 : pub background_task_maximum_delay: Duration,
140 :
141 : pub control_plane_api: Option<Url>,
142 :
143 : /// JWT token for use with the control plane API.
144 : pub control_plane_api_token: Option<SecretString>,
145 :
146 : /// If true, pageserver will make best-effort to operate without a control plane: only
147 : /// for use in major incidents.
148 : pub control_plane_emergency_mode: bool,
149 :
150 : /// How many heatmap uploads may be done concurrency: lower values implicitly deprioritize
151 : /// heatmap uploads vs. other remote storage operations.
152 : pub heatmap_upload_concurrency: usize,
153 :
154 : /// How many remote storage downloads may be done for secondary tenants concurrently. Implicitly
155 : /// deprioritises secondary downloads vs. remote storage operations for attached tenants.
156 : pub secondary_download_concurrency: usize,
157 :
158 : /// Maximum number of WAL records to be ingested and committed at the same time
159 : pub ingest_batch_size: u64,
160 :
161 : pub virtual_file_io_engine: virtual_file::IoEngineKind,
162 :
163 : pub max_vectored_read_bytes: MaxVectoredReadBytes,
164 :
165 : pub image_compression: ImageCompressionAlgorithm,
166 :
167 : /// How many bytes of ephemeral layer content will we allow per kilobyte of RAM. When this
168 : /// is exceeded, we start proactively closing ephemeral layers to limit the total amount
169 : /// of ephemeral data.
170 : ///
171 : /// Setting this to zero disables limits on total ephemeral layer size.
172 : pub ephemeral_bytes_per_memory_kb: usize,
173 :
174 : pub l0_flush: crate::l0_flush::L0FlushConfig,
175 :
176 : /// Direct IO settings
177 : pub virtual_file_direct_io: virtual_file::DirectIoMode,
178 :
179 : pub io_buffer_alignment: usize,
180 : }
181 :
182 : /// Token for authentication to safekeepers
183 : ///
184 : /// We do not want to store this in a PageServerConf because the latter may be logged
185 : /// and/or serialized at a whim, while the token is secret. Currently this token is the
186 : /// same for accessing all tenants/timelines, but may become per-tenant/per-timeline in
187 : /// the future, more tokens and auth may arrive for storage broker, completely changing the logic.
188 : /// Hence, we resort to a global variable for now instead of passing the token from the
189 : /// startup code to the connection code through a dozen layers.
190 : pub static SAFEKEEPER_AUTH_TOKEN: OnceCell<Arc<String>> = OnceCell::new();
191 :
192 : impl PageServerConf {
193 : //
194 : // Repository paths, relative to workdir.
195 : //
196 :
197 20778 : pub fn tenants_path(&self) -> Utf8PathBuf {
198 20778 : self.workdir.join(TENANTS_SEGMENT_NAME)
199 20778 : }
200 :
201 216 : pub fn deletion_prefix(&self) -> Utf8PathBuf {
202 216 : self.workdir.join("deletion")
203 216 : }
204 :
205 0 : pub fn metadata_path(&self) -> Utf8PathBuf {
206 0 : self.workdir.join("metadata.json")
207 0 : }
208 :
209 84 : pub fn deletion_list_path(&self, sequence: u64) -> Utf8PathBuf {
210 : // Encode a version in the filename, so that if we ever switch away from JSON we can
211 : // increment this.
212 : const VERSION: u8 = 1;
213 :
214 84 : self.deletion_prefix()
215 84 : .join(format!("{sequence:016x}-{VERSION:02x}.list"))
216 84 : }
217 :
218 72 : pub fn deletion_header_path(&self) -> Utf8PathBuf {
219 : // Encode a version in the filename, so that if we ever switch away from JSON we can
220 : // increment this.
221 : const VERSION: u8 = 1;
222 :
223 72 : self.deletion_prefix().join(format!("header-{VERSION:02x}"))
224 72 : }
225 :
226 20778 : pub fn tenant_path(&self, tenant_shard_id: &TenantShardId) -> Utf8PathBuf {
227 20778 : self.tenants_path().join(tenant_shard_id.to_string())
228 20778 : }
229 :
230 : /// Points to a place in pageserver's local directory,
231 : /// where certain tenant's LocationConf be stored.
232 0 : pub(crate) fn tenant_location_config_path(
233 0 : &self,
234 0 : tenant_shard_id: &TenantShardId,
235 0 : ) -> Utf8PathBuf {
236 0 : self.tenant_path(tenant_shard_id)
237 0 : .join(TENANT_LOCATION_CONFIG_NAME)
238 0 : }
239 :
240 0 : pub(crate) fn tenant_heatmap_path(&self, tenant_shard_id: &TenantShardId) -> Utf8PathBuf {
241 0 : self.tenant_path(tenant_shard_id)
242 0 : .join(TENANT_HEATMAP_BASENAME)
243 0 : }
244 :
245 20196 : pub fn timelines_path(&self, tenant_shard_id: &TenantShardId) -> Utf8PathBuf {
246 20196 : self.tenant_path(tenant_shard_id)
247 20196 : .join(TIMELINES_SEGMENT_NAME)
248 20196 : }
249 :
250 19044 : pub fn timeline_path(
251 19044 : &self,
252 19044 : tenant_shard_id: &TenantShardId,
253 19044 : timeline_id: &TimelineId,
254 19044 : ) -> Utf8PathBuf {
255 19044 : self.timelines_path(tenant_shard_id)
256 19044 : .join(timeline_id.to_string())
257 19044 : }
258 :
259 : /// Turns storage remote path of a file into its local path.
260 0 : pub fn local_path(&self, remote_path: &RemotePath) -> Utf8PathBuf {
261 0 : remote_path.with_base(&self.workdir)
262 0 : }
263 :
264 : //
265 : // Postgres distribution paths
266 : //
267 48 : pub fn pg_distrib_dir(&self, pg_version: u32) -> anyhow::Result<Utf8PathBuf> {
268 48 : let path = self.pg_distrib_dir.clone();
269 48 :
270 48 : #[allow(clippy::manual_range_patterns)]
271 48 : match pg_version {
272 48 : 14 | 15 | 16 | 17 => Ok(path.join(format!("v{pg_version}"))),
273 0 : _ => bail!("Unsupported postgres version: {}", pg_version),
274 : }
275 48 : }
276 :
277 24 : pub fn pg_bin_dir(&self, pg_version: u32) -> anyhow::Result<Utf8PathBuf> {
278 24 : Ok(self.pg_distrib_dir(pg_version)?.join("bin"))
279 24 : }
280 24 : pub fn pg_lib_dir(&self, pg_version: u32) -> anyhow::Result<Utf8PathBuf> {
281 24 : Ok(self.pg_distrib_dir(pg_version)?.join("lib"))
282 24 : }
283 :
284 : /// Parse a configuration file (pageserver.toml) into a PageServerConf struct,
285 : /// validating the input and failing on errors.
286 : ///
287 : /// This leaves any options not present in the file in the built-in defaults.
288 618 : pub fn parse_and_validate(
289 618 : id: NodeId,
290 618 : config_toml: pageserver_api::config::ConfigToml,
291 618 : workdir: &Utf8Path,
292 618 : ) -> anyhow::Result<Self> {
293 618 : let pageserver_api::config::ConfigToml {
294 618 : listen_pg_addr,
295 618 : listen_http_addr,
296 618 : availability_zone,
297 618 : wait_lsn_timeout,
298 618 : wal_redo_timeout,
299 618 : superuser,
300 618 : page_cache_size,
301 618 : max_file_descriptors,
302 618 : pg_distrib_dir,
303 618 : http_auth_type,
304 618 : pg_auth_type,
305 618 : auth_validation_public_key_path,
306 618 : remote_storage,
307 618 : broker_endpoint,
308 618 : broker_keepalive_interval,
309 618 : log_format,
310 618 : metric_collection_interval,
311 618 : metric_collection_endpoint,
312 618 : metric_collection_bucket,
313 618 : synthetic_size_calculation_interval,
314 618 : disk_usage_based_eviction,
315 618 : test_remote_failures,
316 618 : ondemand_download_behavior_treat_error_as_warn,
317 618 : background_task_maximum_delay,
318 618 : control_plane_api,
319 618 : control_plane_api_token,
320 618 : control_plane_emergency_mode,
321 618 : heatmap_upload_concurrency,
322 618 : secondary_download_concurrency,
323 618 : ingest_batch_size,
324 618 : max_vectored_read_bytes,
325 618 : image_compression,
326 618 : ephemeral_bytes_per_memory_kb,
327 618 : compact_level0_phase1_value_access: _,
328 618 : l0_flush,
329 618 : virtual_file_direct_io,
330 618 : concurrent_tenant_warmup,
331 618 : concurrent_tenant_size_logical_size_queries,
332 618 : virtual_file_io_engine,
333 618 : io_buffer_alignment,
334 618 : tenant_config,
335 618 : } = config_toml;
336 :
337 618 : let mut conf = PageServerConf {
338 : // ------------------------------------------------------------
339 : // fields that are already fully validated by the ConfigToml Deserialize impl
340 : // ------------------------------------------------------------
341 618 : listen_pg_addr,
342 618 : listen_http_addr,
343 618 : availability_zone,
344 618 : wait_lsn_timeout,
345 618 : wal_redo_timeout,
346 618 : superuser,
347 618 : page_cache_size,
348 618 : max_file_descriptors,
349 618 : http_auth_type,
350 618 : pg_auth_type,
351 618 : auth_validation_public_key_path,
352 618 : remote_storage_config: remote_storage,
353 618 : broker_endpoint,
354 618 : broker_keepalive_interval,
355 618 : log_format,
356 618 : metric_collection_interval,
357 618 : metric_collection_endpoint,
358 618 : metric_collection_bucket,
359 618 : synthetic_size_calculation_interval,
360 618 : disk_usage_based_eviction,
361 618 : test_remote_failures,
362 618 : ondemand_download_behavior_treat_error_as_warn,
363 618 : background_task_maximum_delay,
364 618 : control_plane_api,
365 618 : control_plane_emergency_mode,
366 618 : heatmap_upload_concurrency,
367 618 : secondary_download_concurrency,
368 618 : ingest_batch_size,
369 618 : max_vectored_read_bytes,
370 618 : image_compression,
371 618 : ephemeral_bytes_per_memory_kb,
372 618 : virtual_file_direct_io,
373 618 : io_buffer_alignment,
374 618 :
375 618 : // ------------------------------------------------------------
376 618 : // fields that require additional validation or custom handling
377 618 : // ------------------------------------------------------------
378 618 : workdir: workdir.to_owned(),
379 618 : pg_distrib_dir: pg_distrib_dir.unwrap_or_else(|| {
380 6 : std::env::current_dir()
381 6 : .expect("current_dir() failed")
382 6 : .try_into()
383 6 : .expect("current_dir() is not a valid Utf8Path")
384 618 : }),
385 618 : control_plane_api_token: control_plane_api_token.map(SecretString::from),
386 618 : id,
387 618 : default_tenant_conf: tenant_config,
388 618 : concurrent_tenant_warmup: ConfigurableSemaphore::new(concurrent_tenant_warmup),
389 618 : concurrent_tenant_size_logical_size_queries: ConfigurableSemaphore::new(
390 618 : concurrent_tenant_size_logical_size_queries,
391 618 : ),
392 618 : eviction_task_immitated_concurrent_logical_size_queries: ConfigurableSemaphore::new(
393 618 : // re-use `concurrent_tenant_size_logical_size_queries`
394 618 : concurrent_tenant_size_logical_size_queries,
395 618 : ),
396 618 : virtual_file_io_engine: match virtual_file_io_engine {
397 0 : Some(v) => v,
398 618 : None => match crate::virtual_file::io_engine_feature_test()
399 618 : .context("auto-detect virtual_file_io_engine")?
400 : {
401 618 : io_engine::FeatureTestResult::PlatformPreferred(v) => v, // make no noise
402 0 : io_engine::FeatureTestResult::Worse { engine, remark } => {
403 0 : // TODO: bubble this up to the caller so we can tracing::warn! it.
404 0 : eprintln!("auto-detected IO engine is not platform-preferred: engine={engine:?} remark={remark:?}");
405 0 : engine
406 : }
407 : },
408 : },
409 618 : l0_flush: l0_flush
410 618 : .map(crate::l0_flush::L0FlushConfig::from)
411 618 : .unwrap_or_default(),
412 618 : };
413 618 :
414 618 : // ------------------------------------------------------------
415 618 : // custom validation code that covers more than one field in isolation
416 618 : // ------------------------------------------------------------
417 618 :
418 618 : if conf.http_auth_type == AuthType::NeonJWT || conf.pg_auth_type == AuthType::NeonJWT {
419 0 : let auth_validation_public_key_path = conf
420 0 : .auth_validation_public_key_path
421 0 : .get_or_insert_with(|| workdir.join("auth_public_key.pem"));
422 0 : ensure!(
423 0 : auth_validation_public_key_path.exists(),
424 0 : format!(
425 0 : "Can't find auth_validation_public_key at '{auth_validation_public_key_path}'",
426 0 : )
427 : );
428 618 : }
429 :
430 618 : IndexEntry::validate_checkpoint_distance(conf.default_tenant_conf.checkpoint_distance)
431 618 : .map_err(anyhow::Error::msg)
432 618 : .with_context(|| {
433 0 : format!(
434 0 : "effective checkpoint distance is unsupported: {}",
435 0 : conf.default_tenant_conf.checkpoint_distance
436 0 : )
437 618 : })?;
438 :
439 618 : Ok(conf)
440 618 : }
441 :
442 : #[cfg(test)]
443 624 : pub fn test_repo_dir(test_name: &str) -> Utf8PathBuf {
444 624 : let test_output_dir = std::env::var("TEST_OUTPUT").unwrap_or("../tmp_check".into());
445 624 : Utf8PathBuf::from(format!("{test_output_dir}/test_{test_name}"))
446 624 : }
447 :
448 612 : pub fn dummy_conf(repo_dir: Utf8PathBuf) -> Self {
449 612 : let pg_distrib_dir = Utf8PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../pg_install");
450 612 :
451 612 : let config_toml = pageserver_api::config::ConfigToml {
452 612 : wait_lsn_timeout: Duration::from_secs(60),
453 612 : wal_redo_timeout: Duration::from_secs(60),
454 612 : pg_distrib_dir: Some(pg_distrib_dir),
455 612 : metric_collection_interval: Duration::from_secs(60),
456 612 : synthetic_size_calculation_interval: Duration::from_secs(60),
457 612 : background_task_maximum_delay: Duration::ZERO,
458 612 : ..Default::default()
459 612 : };
460 612 : PageServerConf::parse_and_validate(NodeId(0), config_toml, &repo_dir).unwrap()
461 612 : }
462 : }
463 :
464 0 : #[derive(serde::Deserialize, serde::Serialize)]
465 : #[serde(deny_unknown_fields)]
466 : pub struct PageserverIdentity {
467 : pub id: NodeId,
468 : }
469 :
470 : /// Configurable semaphore permits setting.
471 : ///
472 : /// Does not allow semaphore permits to be zero, because at runtime initially zero permits and empty
473 : /// semaphore cannot be distinguished, leading any feature using these to await forever (or until
474 : /// new permits are added).
475 : #[derive(Debug, Clone)]
476 : pub struct ConfigurableSemaphore {
477 : initial_permits: NonZeroUsize,
478 : inner: std::sync::Arc<tokio::sync::Semaphore>,
479 : }
480 :
481 : impl ConfigurableSemaphore {
482 : /// Initializse using a non-zero amount of permits.
483 : ///
484 : /// Require a non-zero initial permits, because using permits == 0 is a crude way to disable a
485 : /// feature such as [`Tenant::gather_size_inputs`]. Otherwise any semaphore using future will
486 : /// behave like [`futures::future::pending`], just waiting until new permits are added.
487 : ///
488 : /// [`Tenant::gather_size_inputs`]: crate::tenant::Tenant::gather_size_inputs
489 1854 : pub fn new(initial_permits: NonZeroUsize) -> Self {
490 1854 : ConfigurableSemaphore {
491 1854 : initial_permits,
492 1854 : inner: std::sync::Arc::new(tokio::sync::Semaphore::new(initial_permits.get())),
493 1854 : }
494 1854 : }
495 :
496 : /// Returns the configured amount of permits.
497 0 : pub fn initial_permits(&self) -> NonZeroUsize {
498 0 : self.initial_permits
499 0 : }
500 : }
501 :
502 : impl PartialEq for ConfigurableSemaphore {
503 0 : fn eq(&self, other: &Self) -> bool {
504 0 : // the number of permits can be increased at runtime, so we cannot really fulfill the
505 0 : // PartialEq value equality otherwise
506 0 : self.initial_permits == other.initial_permits
507 0 : }
508 : }
509 :
510 : impl Eq for ConfigurableSemaphore {}
511 :
512 : impl ConfigurableSemaphore {
513 0 : pub fn inner(&self) -> &std::sync::Arc<tokio::sync::Semaphore> {
514 0 : &self.inner
515 0 : }
516 : }
517 :
518 : #[cfg(test)]
519 : mod tests {
520 :
521 : use camino::Utf8PathBuf;
522 : use utils::id::NodeId;
523 :
524 : use super::PageServerConf;
525 :
526 : #[test]
527 6 : fn test_empty_config_toml_is_valid() {
528 6 : // we use Default impl of everything in this situation
529 6 : let input = r#"
530 6 : "#;
531 6 : let config_toml = toml_edit::de::from_str::<pageserver_api::config::ConfigToml>(input)
532 6 : .expect("empty config is valid");
533 6 : let workdir = Utf8PathBuf::from("/nonexistent");
534 6 : PageServerConf::parse_and_validate(NodeId(0), config_toml, &workdir)
535 6 : .expect("parse_and_validate");
536 6 : }
537 :
538 : #[test]
539 6 : fn test_compactl0_phase1_access_mode_is_ignored_silently() {
540 6 : let input = indoc::indoc! {r#"
541 6 : [compact_level0_phase1_value_access]
542 6 : mode = "streaming-kmerge"
543 6 : validate = "key-lsn-value"
544 6 : "#};
545 6 : toml_edit::de::from_str::<pageserver_api::config::ConfigToml>(input).unwrap();
546 6 : }
547 :
548 : /// If there's a typo in the pageserver config, we'd rather catch that typo
549 : /// and fail pageserver startup than silently ignoring the typo, leaving whoever
550 : /// made it in the believe that their config change is effective.
551 : ///
552 : /// The default in serde is to allow unknown fields, so, we rely
553 : /// on developer+review discipline to add `deny_unknown_fields` when adding
554 : /// new structs to the config, and these tests here as a regression test.
555 : ///
556 : /// The alternative to all of this would be to allow unknown fields in the config.
557 : /// To catch them, we could have a config check tool or mgmt API endpoint that
558 : /// compares the effective config with the TOML on disk and makes sure that
559 : /// the on-disk TOML is a strict subset of the effective config.
560 : mod unknown_fields_handling {
561 : macro_rules! test {
562 : ($short_name:ident, $input:expr) => {
563 : #[test]
564 30 : fn $short_name() {
565 30 : let input = $input;
566 30 : let err = toml_edit::de::from_str::<pageserver_api::config::ConfigToml>(&input)
567 30 : .expect_err("some_invalid_field is an invalid field");
568 30 : dbg!(&err);
569 30 : assert!(err.to_string().contains("some_invalid_field"));
570 30 : }
571 : };
572 : }
573 : use indoc::indoc;
574 :
575 : test!(
576 : toplevel,
577 : indoc! {r#"
578 : some_invalid_field = 23
579 : "#}
580 : );
581 :
582 : test!(
583 : toplevel_nested,
584 : indoc! {r#"
585 : [some_invalid_field]
586 : foo = 23
587 : "#}
588 : );
589 :
590 : test!(
591 : disk_usage_based_eviction,
592 : indoc! {r#"
593 : [disk_usage_based_eviction]
594 : some_invalid_field = 23
595 : "#}
596 : );
597 :
598 : test!(
599 : tenant_config,
600 : indoc! {r#"
601 : [tenant_config]
602 : some_invalid_field = 23
603 : "#}
604 : );
605 :
606 : test!(
607 : l0_flush,
608 : indoc! {r#"
609 : [l0_flush]
610 : mode = "direct"
611 : some_invalid_field = 23
612 : "#}
613 : );
614 :
615 : // TODO: fix this => https://github.com/neondatabase/neon/issues/8915
616 : // test!(
617 : // remote_storage_config,
618 : // indoc! {r#"
619 : // [remote_storage_config]
620 : // local_path = "/nonexistent"
621 : // some_invalid_field = 23
622 : // "#}
623 : // );
624 : }
625 : }
|
