Line data Source code
1 : //! Functions for handling page server configuration options
2 : //!
3 : //! Configuration options can be set in the pageserver.toml configuration
4 : //! file, or on the command line.
5 : //! See also `settings.md` for better description on every parameter.
6 :
7 : use anyhow::{bail, ensure, Context};
8 : use pageserver_api::models::ImageCompressionAlgorithm;
9 : use pageserver_api::{
10 : config::{DiskUsageEvictionTaskConfig, MaxVectoredReadBytes},
11 : shard::TenantShardId,
12 : };
13 : use remote_storage::{RemotePath, RemoteStorageConfig};
14 : use std::env;
15 : use storage_broker::Uri;
16 : use utils::crashsafe::path_with_suffix_extension;
17 : use utils::logging::SecretString;
18 :
19 : use once_cell::sync::OnceCell;
20 : use reqwest::Url;
21 : use std::num::NonZeroUsize;
22 : use std::sync::Arc;
23 : use std::time::Duration;
24 :
25 : use camino::{Utf8Path, Utf8PathBuf};
26 : use postgres_backend::AuthType;
27 : use utils::{
28 : id::{NodeId, TimelineId},
29 : logging::LogFormat,
30 : };
31 :
32 : use crate::tenant::storage_layer::inmemory_layer::IndexEntry;
33 : use crate::tenant::{TENANTS_SEGMENT_NAME, TIMELINES_SEGMENT_NAME};
34 : use crate::virtual_file;
35 : use crate::virtual_file::io_engine;
36 : use crate::{TENANT_HEATMAP_BASENAME, TENANT_LOCATION_CONFIG_NAME, TIMELINE_DELETE_MARK_SUFFIX};
37 :
38 : /// Global state of pageserver.
39 : ///
40 : /// It's mostly immutable configuration, but some semaphores and the
41 : /// like crept in over time and the name stuck.
42 : ///
43 : /// Instantiated by deserializing `pageserver.toml` into [`pageserver_api::config::ConfigToml`]
44 : /// and passing that to [`PageServerConf::parse_and_validate`].
45 : ///
46 : /// # Adding a New Field
47 : ///
48 : /// 1. Add the field to `pageserver_api::config::ConfigToml`.
49 : /// 2. Fix compiler errors (exhaustive destructuring will guide you).
50 : ///
51 : /// For fields that require additional validation or filling in of defaults at runtime,
52 : /// check for examples in the [`PageServerConf::parse_and_validate`] method.
53 : #[derive(Debug, Clone, PartialEq, Eq)]
54 : pub struct PageServerConf {
55 : // Identifier of that particular pageserver so e g safekeepers
56 : // can safely distinguish different pageservers
57 : pub id: NodeId,
58 :
59 : /// Example (default): 127.0.0.1:64000
60 : pub listen_pg_addr: String,
61 : /// Example (default): 127.0.0.1:9898
62 : pub listen_http_addr: String,
63 :
64 : /// Current availability zone. Used for traffic metrics.
65 : pub availability_zone: Option<String>,
66 :
67 : // Timeout when waiting for WAL receiver to catch up to an LSN given in a GetPage@LSN call.
68 : pub wait_lsn_timeout: Duration,
69 : // How long to wait for WAL redo to complete.
70 : pub wal_redo_timeout: Duration,
71 :
72 : pub superuser: String,
73 :
74 : pub page_cache_size: usize,
75 : pub max_file_descriptors: usize,
76 :
77 : // Repository directory, relative to current working directory.
78 : // Normally, the page server changes the current working directory
79 : // to the repository, and 'workdir' is always '.'. But we don't do
80 : // that during unit testing, because the current directory is global
81 : // to the process but different unit tests work on different
82 : // repositories.
83 : pub workdir: Utf8PathBuf,
84 :
85 : pub pg_distrib_dir: Utf8PathBuf,
86 :
87 : // Authentication
88 : /// authentication method for the HTTP mgmt API
89 : pub http_auth_type: AuthType,
90 : /// authentication method for libpq connections from compute
91 : pub pg_auth_type: AuthType,
92 : /// Path to a file or directory containing public key(s) for verifying JWT tokens.
93 : /// Used for both mgmt and compute auth, if enabled.
94 : pub auth_validation_public_key_path: Option<Utf8PathBuf>,
95 :
96 : pub remote_storage_config: Option<RemoteStorageConfig>,
97 :
98 : pub default_tenant_conf: crate::tenant::config::TenantConf,
99 :
100 : /// Storage broker endpoints to connect to.
101 : pub broker_endpoint: Uri,
102 : pub broker_keepalive_interval: Duration,
103 :
104 : pub log_format: LogFormat,
105 :
106 : /// Number of tenants which will be concurrently loaded from remote storage proactively on startup or attach.
107 : ///
108 : /// A lower value implicitly deprioritizes loading such tenants, vs. other work in the system.
109 : pub concurrent_tenant_warmup: ConfigurableSemaphore,
110 :
111 : /// Number of concurrent [`Tenant::gather_size_inputs`](crate::tenant::Tenant::gather_size_inputs) allowed.
112 : pub concurrent_tenant_size_logical_size_queries: ConfigurableSemaphore,
113 : /// Limit of concurrent [`Tenant::gather_size_inputs`] issued by module `eviction_task`.
114 : /// The number of permits is the same as `concurrent_tenant_size_logical_size_queries`.
115 : /// See the comment in `eviction_task` for details.
116 : ///
117 : /// [`Tenant::gather_size_inputs`]: crate::tenant::Tenant::gather_size_inputs
118 : pub eviction_task_immitated_concurrent_logical_size_queries: ConfigurableSemaphore,
119 :
120 : // How often to collect metrics and send them to the metrics endpoint.
121 : pub metric_collection_interval: Duration,
122 : // How often to send unchanged cached metrics to the metrics endpoint.
123 : pub metric_collection_endpoint: Option<Url>,
124 : pub metric_collection_bucket: Option<RemoteStorageConfig>,
125 : pub synthetic_size_calculation_interval: Duration,
126 :
127 : pub disk_usage_based_eviction: Option<DiskUsageEvictionTaskConfig>,
128 :
129 : pub test_remote_failures: u64,
130 :
131 : pub ondemand_download_behavior_treat_error_as_warn: bool,
132 :
133 : /// How long will background tasks be delayed at most after initial load of tenants.
134 : ///
135 : /// Our largest initialization completions are in the range of 100-200s, so perhaps 10s works
136 : /// as we now isolate initial loading, initial logical size calculation and background tasks.
137 : /// Smaller nodes will have background tasks "not running" for this long unless every timeline
138 : /// has it's initial logical size calculated. Not running background tasks for some seconds is
139 : /// not terrible.
140 : pub background_task_maximum_delay: Duration,
141 :
142 : pub control_plane_api: Option<Url>,
143 :
144 : /// JWT token for use with the control plane API.
145 : pub control_plane_api_token: Option<SecretString>,
146 :
147 : /// If true, pageserver will make best-effort to operate without a control plane: only
148 : /// for use in major incidents.
149 : pub control_plane_emergency_mode: bool,
150 :
151 : /// How many heatmap uploads may be done concurrency: lower values implicitly deprioritize
152 : /// heatmap uploads vs. other remote storage operations.
153 : pub heatmap_upload_concurrency: usize,
154 :
155 : /// How many remote storage downloads may be done for secondary tenants concurrently. Implicitly
156 : /// deprioritises secondary downloads vs. remote storage operations for attached tenants.
157 : pub secondary_download_concurrency: usize,
158 :
159 : /// Maximum number of WAL records to be ingested and committed at the same time
160 : pub ingest_batch_size: u64,
161 :
162 : pub virtual_file_io_engine: virtual_file::IoEngineKind,
163 :
164 : pub max_vectored_read_bytes: MaxVectoredReadBytes,
165 :
166 : pub image_compression: ImageCompressionAlgorithm,
167 :
168 : /// How many bytes of ephemeral layer content will we allow per kilobyte of RAM. When this
169 : /// is exceeded, we start proactively closing ephemeral layers to limit the total amount
170 : /// of ephemeral data.
171 : ///
172 : /// Setting this to zero disables limits on total ephemeral layer size.
173 : pub ephemeral_bytes_per_memory_kb: usize,
174 :
175 : pub l0_flush: crate::l0_flush::L0FlushConfig,
176 :
177 : /// Direct IO settings
178 : pub virtual_file_direct_io: virtual_file::DirectIoMode,
179 :
180 : pub io_buffer_alignment: usize,
181 : }
182 :
183 : /// Token for authentication to safekeepers
184 : ///
185 : /// We do not want to store this in a PageServerConf because the latter may be logged
186 : /// and/or serialized at a whim, while the token is secret. Currently this token is the
187 : /// same for accessing all tenants/timelines, but may become per-tenant/per-timeline in
188 : /// the future, more tokens and auth may arrive for storage broker, completely changing the logic.
189 : /// Hence, we resort to a global variable for now instead of passing the token from the
190 : /// startup code to the connection code through a dozen layers.
191 : pub static SAFEKEEPER_AUTH_TOKEN: OnceCell<Arc<String>> = OnceCell::new();
192 :
193 : impl PageServerConf {
194 : //
195 : // Repository paths, relative to workdir.
196 : //
197 :
198 20694 : pub fn tenants_path(&self) -> Utf8PathBuf {
199 20694 : self.workdir.join(TENANTS_SEGMENT_NAME)
200 20694 : }
201 :
202 216 : pub fn deletion_prefix(&self) -> Utf8PathBuf {
203 216 : self.workdir.join("deletion")
204 216 : }
205 :
206 0 : pub fn metadata_path(&self) -> Utf8PathBuf {
207 0 : self.workdir.join("metadata.json")
208 0 : }
209 :
210 84 : pub fn deletion_list_path(&self, sequence: u64) -> Utf8PathBuf {
211 : // Encode a version in the filename, so that if we ever switch away from JSON we can
212 : // increment this.
213 : const VERSION: u8 = 1;
214 :
215 84 : self.deletion_prefix()
216 84 : .join(format!("{sequence:016x}-{VERSION:02x}.list"))
217 84 : }
218 :
219 72 : pub fn deletion_header_path(&self) -> Utf8PathBuf {
220 : // Encode a version in the filename, so that if we ever switch away from JSON we can
221 : // increment this.
222 : const VERSION: u8 = 1;
223 :
224 72 : self.deletion_prefix().join(format!("header-{VERSION:02x}"))
225 72 : }
226 :
227 20694 : pub fn tenant_path(&self, tenant_shard_id: &TenantShardId) -> Utf8PathBuf {
228 20694 : self.tenants_path().join(tenant_shard_id.to_string())
229 20694 : }
230 :
231 : /// Points to a place in pageserver's local directory,
232 : /// where certain tenant's LocationConf be stored.
233 0 : pub(crate) fn tenant_location_config_path(
234 0 : &self,
235 0 : tenant_shard_id: &TenantShardId,
236 0 : ) -> Utf8PathBuf {
237 0 : self.tenant_path(tenant_shard_id)
238 0 : .join(TENANT_LOCATION_CONFIG_NAME)
239 0 : }
240 :
241 0 : pub(crate) fn tenant_heatmap_path(&self, tenant_shard_id: &TenantShardId) -> Utf8PathBuf {
242 0 : self.tenant_path(tenant_shard_id)
243 0 : .join(TENANT_HEATMAP_BASENAME)
244 0 : }
245 :
246 20118 : pub fn timelines_path(&self, tenant_shard_id: &TenantShardId) -> Utf8PathBuf {
247 20118 : self.tenant_path(tenant_shard_id)
248 20118 : .join(TIMELINES_SEGMENT_NAME)
249 20118 : }
250 :
251 18978 : pub fn timeline_path(
252 18978 : &self,
253 18978 : tenant_shard_id: &TenantShardId,
254 18978 : timeline_id: &TimelineId,
255 18978 : ) -> Utf8PathBuf {
256 18978 : self.timelines_path(tenant_shard_id)
257 18978 : .join(timeline_id.to_string())
258 18978 : }
259 :
260 0 : pub(crate) fn timeline_delete_mark_file_path(
261 0 : &self,
262 0 : tenant_shard_id: TenantShardId,
263 0 : timeline_id: TimelineId,
264 0 : ) -> Utf8PathBuf {
265 0 : path_with_suffix_extension(
266 0 : self.timeline_path(&tenant_shard_id, &timeline_id),
267 0 : TIMELINE_DELETE_MARK_SUFFIX,
268 0 : )
269 0 : }
270 :
271 : /// Turns storage remote path of a file into its local path.
272 0 : pub fn local_path(&self, remote_path: &RemotePath) -> Utf8PathBuf {
273 0 : remote_path.with_base(&self.workdir)
274 0 : }
275 :
276 : //
277 : // Postgres distribution paths
278 : //
279 48 : pub fn pg_distrib_dir(&self, pg_version: u32) -> anyhow::Result<Utf8PathBuf> {
280 48 : let path = self.pg_distrib_dir.clone();
281 48 :
282 48 : #[allow(clippy::manual_range_patterns)]
283 48 : match pg_version {
284 48 : 14 | 15 | 16 | 17 => Ok(path.join(format!("v{pg_version}"))),
285 0 : _ => bail!("Unsupported postgres version: {}", pg_version),
286 : }
287 48 : }
288 :
289 24 : pub fn pg_bin_dir(&self, pg_version: u32) -> anyhow::Result<Utf8PathBuf> {
290 24 : Ok(self.pg_distrib_dir(pg_version)?.join("bin"))
291 24 : }
292 24 : pub fn pg_lib_dir(&self, pg_version: u32) -> anyhow::Result<Utf8PathBuf> {
293 24 : Ok(self.pg_distrib_dir(pg_version)?.join("lib"))
294 24 : }
295 :
296 : /// Parse a configuration file (pageserver.toml) into a PageServerConf struct,
297 : /// validating the input and failing on errors.
298 : ///
299 : /// This leaves any options not present in the file in the built-in defaults.
300 612 : pub fn parse_and_validate(
301 612 : id: NodeId,
302 612 : config_toml: pageserver_api::config::ConfigToml,
303 612 : workdir: &Utf8Path,
304 612 : ) -> anyhow::Result<Self> {
305 612 : let pageserver_api::config::ConfigToml {
306 612 : listen_pg_addr,
307 612 : listen_http_addr,
308 612 : availability_zone,
309 612 : wait_lsn_timeout,
310 612 : wal_redo_timeout,
311 612 : superuser,
312 612 : page_cache_size,
313 612 : max_file_descriptors,
314 612 : pg_distrib_dir,
315 612 : http_auth_type,
316 612 : pg_auth_type,
317 612 : auth_validation_public_key_path,
318 612 : remote_storage,
319 612 : broker_endpoint,
320 612 : broker_keepalive_interval,
321 612 : log_format,
322 612 : metric_collection_interval,
323 612 : metric_collection_endpoint,
324 612 : metric_collection_bucket,
325 612 : synthetic_size_calculation_interval,
326 612 : disk_usage_based_eviction,
327 612 : test_remote_failures,
328 612 : ondemand_download_behavior_treat_error_as_warn,
329 612 : background_task_maximum_delay,
330 612 : control_plane_api,
331 612 : control_plane_api_token,
332 612 : control_plane_emergency_mode,
333 612 : heatmap_upload_concurrency,
334 612 : secondary_download_concurrency,
335 612 : ingest_batch_size,
336 612 : max_vectored_read_bytes,
337 612 : image_compression,
338 612 : ephemeral_bytes_per_memory_kb,
339 612 : compact_level0_phase1_value_access: _,
340 612 : l0_flush,
341 612 : virtual_file_direct_io,
342 612 : concurrent_tenant_warmup,
343 612 : concurrent_tenant_size_logical_size_queries,
344 612 : virtual_file_io_engine,
345 612 : io_buffer_alignment,
346 612 : tenant_config,
347 612 : } = config_toml;
348 :
349 612 : let mut conf = PageServerConf {
350 : // ------------------------------------------------------------
351 : // fields that are already fully validated by the ConfigToml Deserialize impl
352 : // ------------------------------------------------------------
353 612 : listen_pg_addr,
354 612 : listen_http_addr,
355 612 : availability_zone,
356 612 : wait_lsn_timeout,
357 612 : wal_redo_timeout,
358 612 : superuser,
359 612 : page_cache_size,
360 612 : max_file_descriptors,
361 612 : http_auth_type,
362 612 : pg_auth_type,
363 612 : auth_validation_public_key_path,
364 612 : remote_storage_config: remote_storage,
365 612 : broker_endpoint,
366 612 : broker_keepalive_interval,
367 612 : log_format,
368 612 : metric_collection_interval,
369 612 : metric_collection_endpoint,
370 612 : metric_collection_bucket,
371 612 : synthetic_size_calculation_interval,
372 612 : disk_usage_based_eviction,
373 612 : test_remote_failures,
374 612 : ondemand_download_behavior_treat_error_as_warn,
375 612 : background_task_maximum_delay,
376 612 : control_plane_api,
377 612 : control_plane_emergency_mode,
378 612 : heatmap_upload_concurrency,
379 612 : secondary_download_concurrency,
380 612 : ingest_batch_size,
381 612 : max_vectored_read_bytes,
382 612 : image_compression,
383 612 : ephemeral_bytes_per_memory_kb,
384 612 : virtual_file_direct_io,
385 612 : io_buffer_alignment,
386 612 :
387 612 : // ------------------------------------------------------------
388 612 : // fields that require additional validation or custom handling
389 612 : // ------------------------------------------------------------
390 612 : workdir: workdir.to_owned(),
391 612 : pg_distrib_dir: pg_distrib_dir.unwrap_or_else(|| {
392 6 : std::env::current_dir()
393 6 : .expect("current_dir() failed")
394 6 : .try_into()
395 6 : .expect("current_dir() is not a valid Utf8Path")
396 612 : }),
397 612 : control_plane_api_token: control_plane_api_token.map(SecretString::from),
398 612 : id,
399 612 : default_tenant_conf: tenant_config,
400 612 : concurrent_tenant_warmup: ConfigurableSemaphore::new(concurrent_tenant_warmup),
401 612 : concurrent_tenant_size_logical_size_queries: ConfigurableSemaphore::new(
402 612 : concurrent_tenant_size_logical_size_queries,
403 612 : ),
404 612 : eviction_task_immitated_concurrent_logical_size_queries: ConfigurableSemaphore::new(
405 612 : // re-use `concurrent_tenant_size_logical_size_queries`
406 612 : concurrent_tenant_size_logical_size_queries,
407 612 : ),
408 612 : virtual_file_io_engine: match virtual_file_io_engine {
409 0 : Some(v) => v,
410 612 : None => match crate::virtual_file::io_engine_feature_test()
411 612 : .context("auto-detect virtual_file_io_engine")?
412 : {
413 612 : io_engine::FeatureTestResult::PlatformPreferred(v) => v, // make no noise
414 0 : io_engine::FeatureTestResult::Worse { engine, remark } => {
415 0 : // TODO: bubble this up to the caller so we can tracing::warn! it.
416 0 : eprintln!("auto-detected IO engine is not platform-preferred: engine={engine:?} remark={remark:?}");
417 0 : engine
418 : }
419 : },
420 : },
421 612 : l0_flush: l0_flush
422 612 : .map(crate::l0_flush::L0FlushConfig::from)
423 612 : .unwrap_or_default(),
424 612 : };
425 612 :
426 612 : // ------------------------------------------------------------
427 612 : // custom validation code that covers more than one field in isolation
428 612 : // ------------------------------------------------------------
429 612 :
430 612 : if conf.http_auth_type == AuthType::NeonJWT || conf.pg_auth_type == AuthType::NeonJWT {
431 0 : let auth_validation_public_key_path = conf
432 0 : .auth_validation_public_key_path
433 0 : .get_or_insert_with(|| workdir.join("auth_public_key.pem"));
434 0 : ensure!(
435 0 : auth_validation_public_key_path.exists(),
436 0 : format!(
437 0 : "Can't find auth_validation_public_key at '{auth_validation_public_key_path}'",
438 0 : )
439 : );
440 612 : }
441 :
442 612 : IndexEntry::validate_checkpoint_distance(conf.default_tenant_conf.checkpoint_distance)
443 612 : .map_err(anyhow::Error::msg)
444 612 : .with_context(|| {
445 0 : format!(
446 0 : "effective checkpoint distance is unsupported: {}",
447 0 : conf.default_tenant_conf.checkpoint_distance
448 0 : )
449 612 : })?;
450 :
451 612 : Ok(conf)
452 612 : }
453 :
454 : #[cfg(test)]
455 618 : pub fn test_repo_dir(test_name: &str) -> Utf8PathBuf {
456 618 : let test_output_dir = std::env::var("TEST_OUTPUT").unwrap_or("../tmp_check".into());
457 618 : Utf8PathBuf::from(format!("{test_output_dir}/test_{test_name}"))
458 618 : }
459 :
460 606 : pub fn dummy_conf(repo_dir: Utf8PathBuf) -> Self {
461 606 : let pg_distrib_dir = Utf8PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../pg_install");
462 606 :
463 606 : let config_toml = pageserver_api::config::ConfigToml {
464 606 : wait_lsn_timeout: Duration::from_secs(60),
465 606 : wal_redo_timeout: Duration::from_secs(60),
466 606 : pg_distrib_dir: Some(pg_distrib_dir),
467 606 : metric_collection_interval: Duration::from_secs(60),
468 606 : synthetic_size_calculation_interval: Duration::from_secs(60),
469 606 : background_task_maximum_delay: Duration::ZERO,
470 606 : ..Default::default()
471 606 : };
472 606 : PageServerConf::parse_and_validate(NodeId(0), config_toml, &repo_dir).unwrap()
473 606 : }
474 : }
475 :
476 0 : #[derive(serde::Deserialize, serde::Serialize)]
477 : #[serde(deny_unknown_fields)]
478 : pub struct PageserverIdentity {
479 : pub id: NodeId,
480 : }
481 :
482 : /// Configurable semaphore permits setting.
483 : ///
484 : /// Does not allow semaphore permits to be zero, because at runtime initially zero permits and empty
485 : /// semaphore cannot be distinguished, leading any feature using these to await forever (or until
486 : /// new permits are added).
487 : #[derive(Debug, Clone)]
488 : pub struct ConfigurableSemaphore {
489 : initial_permits: NonZeroUsize,
490 : inner: std::sync::Arc<tokio::sync::Semaphore>,
491 : }
492 :
493 : impl ConfigurableSemaphore {
494 : pub const DEFAULT_INITIAL: NonZeroUsize = match NonZeroUsize::new(1) {
495 : Some(x) => x,
496 : None => panic!("const unwrap is not yet stable"),
497 : };
498 :
499 : /// Initializse using a non-zero amount of permits.
500 : ///
501 : /// Require a non-zero initial permits, because using permits == 0 is a crude way to disable a
502 : /// feature such as [`Tenant::gather_size_inputs`]. Otherwise any semaphore using future will
503 : /// behave like [`futures::future::pending`], just waiting until new permits are added.
504 : ///
505 : /// [`Tenant::gather_size_inputs`]: crate::tenant::Tenant::gather_size_inputs
506 1836 : pub fn new(initial_permits: NonZeroUsize) -> Self {
507 1836 : ConfigurableSemaphore {
508 1836 : initial_permits,
509 1836 : inner: std::sync::Arc::new(tokio::sync::Semaphore::new(initial_permits.get())),
510 1836 : }
511 1836 : }
512 :
513 : /// Returns the configured amount of permits.
514 0 : pub fn initial_permits(&self) -> NonZeroUsize {
515 0 : self.initial_permits
516 0 : }
517 : }
518 :
519 : impl Default for ConfigurableSemaphore {
520 0 : fn default() -> Self {
521 0 : Self::new(Self::DEFAULT_INITIAL)
522 0 : }
523 : }
524 :
525 : impl PartialEq for ConfigurableSemaphore {
526 0 : fn eq(&self, other: &Self) -> bool {
527 0 : // the number of permits can be increased at runtime, so we cannot really fulfill the
528 0 : // PartialEq value equality otherwise
529 0 : self.initial_permits == other.initial_permits
530 0 : }
531 : }
532 :
533 : impl Eq for ConfigurableSemaphore {}
534 :
535 : impl ConfigurableSemaphore {
536 0 : pub fn inner(&self) -> &std::sync::Arc<tokio::sync::Semaphore> {
537 0 : &self.inner
538 0 : }
539 : }
540 :
541 : #[cfg(test)]
542 : mod tests {
543 :
544 : use camino::Utf8PathBuf;
545 : use utils::id::NodeId;
546 :
547 : use super::PageServerConf;
548 :
549 : #[test]
550 6 : fn test_empty_config_toml_is_valid() {
551 6 : // we use Default impl of everything in this situation
552 6 : let input = r#"
553 6 : "#;
554 6 : let config_toml = toml_edit::de::from_str::<pageserver_api::config::ConfigToml>(input)
555 6 : .expect("empty config is valid");
556 6 : let workdir = Utf8PathBuf::from("/nonexistent");
557 6 : PageServerConf::parse_and_validate(NodeId(0), config_toml, &workdir)
558 6 : .expect("parse_and_validate");
559 6 : }
560 :
561 : #[test]
562 6 : fn test_compactl0_phase1_access_mode_is_ignored_silently() {
563 6 : let input = indoc::indoc! {r#"
564 6 : [compact_level0_phase1_value_access]
565 6 : mode = "streaming-kmerge"
566 6 : validate = "key-lsn-value"
567 6 : "#};
568 6 : toml_edit::de::from_str::<pageserver_api::config::ConfigToml>(input).unwrap();
569 6 : }
570 :
571 : /// If there's a typo in the pageserver config, we'd rather catch that typo
572 : /// and fail pageserver startup than silently ignoring the typo, leaving whoever
573 : /// made it in the believe that their config change is effective.
574 : ///
575 : /// The default in serde is to allow unknown fields, so, we rely
576 : /// on developer+review discipline to add `deny_unknown_fields` when adding
577 : /// new structs to the config, and these tests here as a regression test.
578 : ///
579 : /// The alternative to all of this would be to allow unknown fields in the config.
580 : /// To catch them, we could have a config check tool or mgmt API endpoint that
581 : /// compares the effective config with the TOML on disk and makes sure that
582 : /// the on-disk TOML is a strict subset of the effective config.
583 : mod unknown_fields_handling {
584 : macro_rules! test {
585 : ($short_name:ident, $input:expr) => {
586 : #[test]
587 30 : fn $short_name() {
588 30 : let input = $input;
589 30 : let err = toml_edit::de::from_str::<pageserver_api::config::ConfigToml>(&input)
590 30 : .expect_err("some_invalid_field is an invalid field");
591 30 : dbg!(&err);
592 30 : assert!(err.to_string().contains("some_invalid_field"));
593 30 : }
594 : };
595 : }
596 : use indoc::indoc;
597 :
598 : test!(
599 : toplevel,
600 : indoc! {r#"
601 : some_invalid_field = 23
602 : "#}
603 : );
604 :
605 : test!(
606 : toplevel_nested,
607 : indoc! {r#"
608 : [some_invalid_field]
609 : foo = 23
610 : "#}
611 : );
612 :
613 : test!(
614 : disk_usage_based_eviction,
615 : indoc! {r#"
616 : [disk_usage_based_eviction]
617 : some_invalid_field = 23
618 : "#}
619 : );
620 :
621 : test!(
622 : tenant_config,
623 : indoc! {r#"
624 : [tenant_config]
625 : some_invalid_field = 23
626 : "#}
627 : );
628 :
629 : test!(
630 : l0_flush,
631 : indoc! {r#"
632 : [l0_flush]
633 : mode = "direct"
634 : some_invalid_field = 23
635 : "#}
636 : );
637 :
638 : // TODO: fix this => https://github.com/neondatabase/neon/issues/8915
639 : // test!(
640 : // remote_storage_config,
641 : // indoc! {r#"
642 : // [remote_storage_config]
643 : // local_path = "/nonexistent"
644 : // some_invalid_field = 23
645 : // "#}
646 : // );
647 : }
648 : }
|
