Line data Source code
1 : //! Functions for handling page server configuration options
2 : //!
3 : //! Configuration options can be set in the pageserver.toml configuration
4 : //! file, or on the command line.
5 : //! See also `settings.md` for better description on every parameter.
6 :
7 : use anyhow::{bail, ensure, Context};
8 : use pageserver_api::models::ImageCompressionAlgorithm;
9 : use pageserver_api::{
10 : config::{DiskUsageEvictionTaskConfig, MaxVectoredReadBytes},
11 : shard::TenantShardId,
12 : };
13 : use remote_storage::{RemotePath, RemoteStorageConfig};
14 : use std::env;
15 : use storage_broker::Uri;
16 : use utils::logging::SecretString;
17 : use utils::postgres_client::PostgresClientProtocol;
18 :
19 : use once_cell::sync::OnceCell;
20 : use reqwest::Url;
21 : use std::num::NonZeroUsize;
22 : use std::sync::Arc;
23 : use std::time::Duration;
24 :
25 : use camino::{Utf8Path, Utf8PathBuf};
26 : use postgres_backend::AuthType;
27 : use utils::{
28 : id::{NodeId, TimelineId},
29 : logging::LogFormat,
30 : };
31 :
32 : use crate::tenant::storage_layer::inmemory_layer::IndexEntry;
33 : use crate::tenant::{TENANTS_SEGMENT_NAME, TIMELINES_SEGMENT_NAME};
34 : use crate::virtual_file;
35 : use crate::virtual_file::io_engine;
36 : use crate::{TENANT_HEATMAP_BASENAME, TENANT_LOCATION_CONFIG_NAME};
37 :
38 : /// Global state of pageserver.
39 : ///
40 : /// It's mostly immutable configuration, but some semaphores and the
41 : /// like crept in over time and the name stuck.
42 : ///
43 : /// Instantiated by deserializing `pageserver.toml` into [`pageserver_api::config::ConfigToml`]
44 : /// and passing that to [`PageServerConf::parse_and_validate`].
45 : ///
46 : /// # Adding a New Field
47 : ///
48 : /// 1. Add the field to `pageserver_api::config::ConfigToml`.
49 : /// 2. Fix compiler errors (exhaustive destructuring will guide you).
50 : ///
51 : /// For fields that require additional validation or filling in of defaults at runtime,
52 : /// check for examples in the [`PageServerConf::parse_and_validate`] method.
53 : #[derive(Debug, Clone, PartialEq, Eq)]
54 : pub struct PageServerConf {
55 : // Identifier of that particular pageserver so e g safekeepers
56 : // can safely distinguish different pageservers
57 : pub id: NodeId,
58 :
59 : /// Example (default): 127.0.0.1:64000
60 : pub listen_pg_addr: String,
61 : /// Example (default): 127.0.0.1:9898
62 : pub listen_http_addr: String,
63 :
64 : /// Current availability zone. Used for traffic metrics.
65 : pub availability_zone: Option<String>,
66 :
67 : // Timeout when waiting for WAL receiver to catch up to an LSN given in a GetPage@LSN call.
68 : pub wait_lsn_timeout: Duration,
69 : // How long to wait for WAL redo to complete.
70 : pub wal_redo_timeout: Duration,
71 :
72 : pub superuser: String,
73 : pub locale: String,
74 :
75 : pub page_cache_size: usize,
76 : pub max_file_descriptors: usize,
77 :
78 : // Repository directory, relative to current working directory.
79 : // Normally, the page server changes the current working directory
80 : // to the repository, and 'workdir' is always '.'. But we don't do
81 : // that during unit testing, because the current directory is global
82 : // to the process but different unit tests work on different
83 : // repositories.
84 : pub workdir: Utf8PathBuf,
85 :
86 : pub pg_distrib_dir: Utf8PathBuf,
87 :
88 : // Authentication
89 : /// authentication method for the HTTP mgmt API
90 : pub http_auth_type: AuthType,
91 : /// authentication method for libpq connections from compute
92 : pub pg_auth_type: AuthType,
93 : /// Path to a file or directory containing public key(s) for verifying JWT tokens.
94 : /// Used for both mgmt and compute auth, if enabled.
95 : pub auth_validation_public_key_path: Option<Utf8PathBuf>,
96 :
97 : pub remote_storage_config: Option<RemoteStorageConfig>,
98 :
99 : pub default_tenant_conf: crate::tenant::config::TenantConf,
100 :
101 : /// Storage broker endpoints to connect to.
102 : pub broker_endpoint: Uri,
103 : pub broker_keepalive_interval: Duration,
104 :
105 : pub log_format: LogFormat,
106 :
107 : /// Number of tenants which will be concurrently loaded from remote storage proactively on startup or attach.
108 : ///
109 : /// A lower value implicitly deprioritizes loading such tenants, vs. other work in the system.
110 : pub concurrent_tenant_warmup: ConfigurableSemaphore,
111 :
112 : /// Number of concurrent [`Tenant::gather_size_inputs`](crate::tenant::Tenant::gather_size_inputs) allowed.
113 : pub concurrent_tenant_size_logical_size_queries: ConfigurableSemaphore,
114 : /// Limit of concurrent [`Tenant::gather_size_inputs`] issued by module `eviction_task`.
115 : /// The number of permits is the same as `concurrent_tenant_size_logical_size_queries`.
116 : /// See the comment in `eviction_task` for details.
117 : ///
118 : /// [`Tenant::gather_size_inputs`]: crate::tenant::Tenant::gather_size_inputs
119 : pub eviction_task_immitated_concurrent_logical_size_queries: ConfigurableSemaphore,
120 :
121 : // How often to collect metrics and send them to the metrics endpoint.
122 : pub metric_collection_interval: Duration,
123 : // How often to send unchanged cached metrics to the metrics endpoint.
124 : pub metric_collection_endpoint: Option<Url>,
125 : pub metric_collection_bucket: Option<RemoteStorageConfig>,
126 : pub synthetic_size_calculation_interval: Duration,
127 :
128 : pub disk_usage_based_eviction: Option<DiskUsageEvictionTaskConfig>,
129 :
130 : pub test_remote_failures: u64,
131 :
132 : pub ondemand_download_behavior_treat_error_as_warn: bool,
133 :
134 : /// How long will background tasks be delayed at most after initial load of tenants.
135 : ///
136 : /// Our largest initialization completions are in the range of 100-200s, so perhaps 10s works
137 : /// as we now isolate initial loading, initial logical size calculation and background tasks.
138 : /// Smaller nodes will have background tasks "not running" for this long unless every timeline
139 : /// has it's initial logical size calculated. Not running background tasks for some seconds is
140 : /// not terrible.
141 : pub background_task_maximum_delay: Duration,
142 :
143 : pub control_plane_api: Option<Url>,
144 :
145 : /// JWT token for use with the control plane API.
146 : pub control_plane_api_token: Option<SecretString>,
147 :
148 : pub import_pgdata_upcall_api: Option<Url>,
149 : pub import_pgdata_upcall_api_token: Option<SecretString>,
150 : pub import_pgdata_aws_endpoint_url: Option<Url>,
151 :
152 : /// If true, pageserver will make best-effort to operate without a control plane: only
153 : /// for use in major incidents.
154 : pub control_plane_emergency_mode: bool,
155 :
156 : /// How many heatmap uploads may be done concurrency: lower values implicitly deprioritize
157 : /// heatmap uploads vs. other remote storage operations.
158 : pub heatmap_upload_concurrency: usize,
159 :
160 : /// How many remote storage downloads may be done for secondary tenants concurrently. Implicitly
161 : /// deprioritises secondary downloads vs. remote storage operations for attached tenants.
162 : pub secondary_download_concurrency: usize,
163 :
164 : /// Maximum number of WAL records to be ingested and committed at the same time
165 : pub ingest_batch_size: u64,
166 :
167 : pub virtual_file_io_engine: virtual_file::IoEngineKind,
168 :
169 : pub max_vectored_read_bytes: MaxVectoredReadBytes,
170 :
171 : pub image_compression: ImageCompressionAlgorithm,
172 :
173 : /// Whether to offload archived timelines automatically
174 : pub timeline_offloading: bool,
175 :
176 : /// How many bytes of ephemeral layer content will we allow per kilobyte of RAM. When this
177 : /// is exceeded, we start proactively closing ephemeral layers to limit the total amount
178 : /// of ephemeral data.
179 : ///
180 : /// Setting this to zero disables limits on total ephemeral layer size.
181 : pub ephemeral_bytes_per_memory_kb: usize,
182 :
183 : pub l0_flush: crate::l0_flush::L0FlushConfig,
184 :
185 : /// Direct IO settings
186 : pub virtual_file_io_mode: virtual_file::IoMode,
187 :
188 : /// Optionally disable disk syncs (unsafe!)
189 : pub no_sync: bool,
190 :
191 : pub wal_receiver_protocol: PostgresClientProtocol,
192 :
193 : pub page_service_pipelining: pageserver_api::config::PageServicePipeliningConfig,
194 :
195 : pub get_vectored_concurrent_io: pageserver_api::config::GetVectoredConcurrentIo,
196 :
197 : /// Enable read path debugging. If enabled, read key errors will print a backtrace of the layer
198 : /// files read.
199 : pub enable_read_path_debugging: bool,
200 :
201 : /// Interpreted protocol feature: if enabled, validate that the logical WAL received from
202 : /// safekeepers does not have gaps.
203 : pub validate_wal_contiguity: bool,
204 : }
205 :
206 : /// Token for authentication to safekeepers
207 : ///
208 : /// We do not want to store this in a PageServerConf because the latter may be logged
209 : /// and/or serialized at a whim, while the token is secret. Currently this token is the
210 : /// same for accessing all tenants/timelines, but may become per-tenant/per-timeline in
211 : /// the future, more tokens and auth may arrive for storage broker, completely changing the logic.
212 : /// Hence, we resort to a global variable for now instead of passing the token from the
213 : /// startup code to the connection code through a dozen layers.
214 : pub static SAFEKEEPER_AUTH_TOKEN: OnceCell<Arc<String>> = OnceCell::new();
215 :
216 : impl PageServerConf {
217 : //
218 : // Repository paths, relative to workdir.
219 : //
220 :
221 15426 : pub fn tenants_path(&self) -> Utf8PathBuf {
222 15426 : self.workdir.join(TENANTS_SEGMENT_NAME)
223 15426 : }
224 :
225 144 : pub fn deletion_prefix(&self) -> Utf8PathBuf {
226 144 : self.workdir.join("deletion")
227 144 : }
228 :
229 0 : pub fn metadata_path(&self) -> Utf8PathBuf {
230 0 : self.workdir.join("metadata.json")
231 0 : }
232 :
233 56 : pub fn deletion_list_path(&self, sequence: u64) -> Utf8PathBuf {
234 : // Encode a version in the filename, so that if we ever switch away from JSON we can
235 : // increment this.
236 : const VERSION: u8 = 1;
237 :
238 56 : self.deletion_prefix()
239 56 : .join(format!("{sequence:016x}-{VERSION:02x}.list"))
240 56 : }
241 :
242 48 : pub fn deletion_header_path(&self) -> Utf8PathBuf {
243 : // Encode a version in the filename, so that if we ever switch away from JSON we can
244 : // increment this.
245 : const VERSION: u8 = 1;
246 :
247 48 : self.deletion_prefix().join(format!("header-{VERSION:02x}"))
248 48 : }
249 :
250 15322 : pub fn tenant_path(&self, tenant_shard_id: &TenantShardId) -> Utf8PathBuf {
251 15322 : self.tenants_path().join(tenant_shard_id.to_string())
252 15322 : }
253 :
254 : /// Points to a place in pageserver's local directory,
255 : /// where certain tenant's LocationConf be stored.
256 0 : pub(crate) fn tenant_location_config_path(
257 0 : &self,
258 0 : tenant_shard_id: &TenantShardId,
259 0 : ) -> Utf8PathBuf {
260 0 : self.tenant_path(tenant_shard_id)
261 0 : .join(TENANT_LOCATION_CONFIG_NAME)
262 0 : }
263 :
264 445 : pub(crate) fn tenant_heatmap_path(&self, tenant_shard_id: &TenantShardId) -> Utf8PathBuf {
265 445 : self.tenant_path(tenant_shard_id)
266 445 : .join(TENANT_HEATMAP_BASENAME)
267 445 : }
268 :
269 14424 : pub fn timelines_path(&self, tenant_shard_id: &TenantShardId) -> Utf8PathBuf {
270 14424 : self.tenant_path(tenant_shard_id)
271 14424 : .join(TIMELINES_SEGMENT_NAME)
272 14424 : }
273 :
274 13522 : pub fn timeline_path(
275 13522 : &self,
276 13522 : tenant_shard_id: &TenantShardId,
277 13522 : timeline_id: &TimelineId,
278 13522 : ) -> Utf8PathBuf {
279 13522 : self.timelines_path(tenant_shard_id)
280 13522 : .join(timeline_id.to_string())
281 13522 : }
282 :
283 : /// Turns storage remote path of a file into its local path.
284 0 : pub fn local_path(&self, remote_path: &RemotePath) -> Utf8PathBuf {
285 0 : remote_path.with_base(&self.workdir)
286 0 : }
287 :
288 : //
289 : // Postgres distribution paths
290 : //
291 40 : pub fn pg_distrib_dir(&self, pg_version: u32) -> anyhow::Result<Utf8PathBuf> {
292 40 : let path = self.pg_distrib_dir.clone();
293 40 :
294 40 : #[allow(clippy::manual_range_patterns)]
295 40 : match pg_version {
296 40 : 14 | 15 | 16 | 17 => Ok(path.join(format!("v{pg_version}"))),
297 0 : _ => bail!("Unsupported postgres version: {}", pg_version),
298 : }
299 40 : }
300 :
301 20 : pub fn pg_bin_dir(&self, pg_version: u32) -> anyhow::Result<Utf8PathBuf> {
302 20 : Ok(self.pg_distrib_dir(pg_version)?.join("bin"))
303 20 : }
304 20 : pub fn pg_lib_dir(&self, pg_version: u32) -> anyhow::Result<Utf8PathBuf> {
305 20 : Ok(self.pg_distrib_dir(pg_version)?.join("lib"))
306 20 : }
307 :
308 : /// Parse a configuration file (pageserver.toml) into a PageServerConf struct,
309 : /// validating the input and failing on errors.
310 : ///
311 : /// This leaves any options not present in the file in the built-in defaults.
312 481 : pub fn parse_and_validate(
313 481 : id: NodeId,
314 481 : config_toml: pageserver_api::config::ConfigToml,
315 481 : workdir: &Utf8Path,
316 481 : ) -> anyhow::Result<Self> {
317 481 : let pageserver_api::config::ConfigToml {
318 481 : listen_pg_addr,
319 481 : listen_http_addr,
320 481 : availability_zone,
321 481 : wait_lsn_timeout,
322 481 : wal_redo_timeout,
323 481 : superuser,
324 481 : locale,
325 481 : page_cache_size,
326 481 : max_file_descriptors,
327 481 : pg_distrib_dir,
328 481 : http_auth_type,
329 481 : pg_auth_type,
330 481 : auth_validation_public_key_path,
331 481 : remote_storage,
332 481 : broker_endpoint,
333 481 : broker_keepalive_interval,
334 481 : log_format,
335 481 : metric_collection_interval,
336 481 : metric_collection_endpoint,
337 481 : metric_collection_bucket,
338 481 : synthetic_size_calculation_interval,
339 481 : disk_usage_based_eviction,
340 481 : test_remote_failures,
341 481 : ondemand_download_behavior_treat_error_as_warn,
342 481 : background_task_maximum_delay,
343 481 : control_plane_api,
344 481 : control_plane_api_token,
345 481 : control_plane_emergency_mode,
346 481 : import_pgdata_upcall_api,
347 481 : import_pgdata_upcall_api_token,
348 481 : import_pgdata_aws_endpoint_url,
349 481 : heatmap_upload_concurrency,
350 481 : secondary_download_concurrency,
351 481 : ingest_batch_size,
352 481 : max_vectored_read_bytes,
353 481 : image_compression,
354 481 : timeline_offloading,
355 481 : ephemeral_bytes_per_memory_kb,
356 481 : l0_flush,
357 481 : virtual_file_io_mode,
358 481 : concurrent_tenant_warmup,
359 481 : concurrent_tenant_size_logical_size_queries,
360 481 : virtual_file_io_engine,
361 481 : tenant_config,
362 481 : no_sync,
363 481 : wal_receiver_protocol,
364 481 : page_service_pipelining,
365 481 : get_vectored_concurrent_io,
366 481 : enable_read_path_debugging,
367 481 : validate_wal_contiguity,
368 481 : } = config_toml;
369 :
370 481 : let mut conf = PageServerConf {
371 : // ------------------------------------------------------------
372 : // fields that are already fully validated by the ConfigToml Deserialize impl
373 : // ------------------------------------------------------------
374 481 : listen_pg_addr,
375 481 : listen_http_addr,
376 481 : availability_zone,
377 481 : wait_lsn_timeout,
378 481 : wal_redo_timeout,
379 481 : superuser,
380 481 : locale,
381 481 : page_cache_size,
382 481 : max_file_descriptors,
383 481 : http_auth_type,
384 481 : pg_auth_type,
385 481 : auth_validation_public_key_path,
386 481 : remote_storage_config: remote_storage,
387 481 : broker_endpoint,
388 481 : broker_keepalive_interval,
389 481 : log_format,
390 481 : metric_collection_interval,
391 481 : metric_collection_endpoint,
392 481 : metric_collection_bucket,
393 481 : synthetic_size_calculation_interval,
394 481 : disk_usage_based_eviction,
395 481 : test_remote_failures,
396 481 : ondemand_download_behavior_treat_error_as_warn,
397 481 : background_task_maximum_delay,
398 481 : control_plane_api,
399 481 : control_plane_emergency_mode,
400 481 : heatmap_upload_concurrency,
401 481 : secondary_download_concurrency,
402 481 : ingest_batch_size,
403 481 : max_vectored_read_bytes,
404 481 : image_compression,
405 481 : timeline_offloading,
406 481 : ephemeral_bytes_per_memory_kb,
407 481 : import_pgdata_upcall_api,
408 481 : import_pgdata_upcall_api_token: import_pgdata_upcall_api_token.map(SecretString::from),
409 481 : import_pgdata_aws_endpoint_url,
410 481 : wal_receiver_protocol,
411 481 : page_service_pipelining,
412 481 : get_vectored_concurrent_io,
413 481 :
414 481 : // ------------------------------------------------------------
415 481 : // fields that require additional validation or custom handling
416 481 : // ------------------------------------------------------------
417 481 : workdir: workdir.to_owned(),
418 481 : pg_distrib_dir: pg_distrib_dir.unwrap_or_else(|| {
419 4 : std::env::current_dir()
420 4 : .expect("current_dir() failed")
421 4 : .try_into()
422 4 : .expect("current_dir() is not a valid Utf8Path")
423 481 : }),
424 481 : control_plane_api_token: control_plane_api_token.map(SecretString::from),
425 481 : id,
426 481 : default_tenant_conf: tenant_config,
427 481 : concurrent_tenant_warmup: ConfigurableSemaphore::new(concurrent_tenant_warmup),
428 481 : concurrent_tenant_size_logical_size_queries: ConfigurableSemaphore::new(
429 481 : concurrent_tenant_size_logical_size_queries,
430 481 : ),
431 481 : eviction_task_immitated_concurrent_logical_size_queries: ConfigurableSemaphore::new(
432 481 : // re-use `concurrent_tenant_size_logical_size_queries`
433 481 : concurrent_tenant_size_logical_size_queries,
434 481 : ),
435 481 : virtual_file_io_engine: match virtual_file_io_engine {
436 0 : Some(v) => v,
437 481 : None => match crate::virtual_file::io_engine_feature_test()
438 481 : .context("auto-detect virtual_file_io_engine")?
439 : {
440 481 : io_engine::FeatureTestResult::PlatformPreferred(v) => v, // make no noise
441 0 : io_engine::FeatureTestResult::Worse { engine, remark } => {
442 0 : // TODO: bubble this up to the caller so we can tracing::warn! it.
443 0 : eprintln!("auto-detected IO engine is not platform-preferred: engine={engine:?} remark={remark:?}");
444 0 : engine
445 : }
446 : },
447 : },
448 481 : l0_flush: l0_flush
449 481 : .map(crate::l0_flush::L0FlushConfig::from)
450 481 : .unwrap_or_default(),
451 481 : virtual_file_io_mode: virtual_file_io_mode.unwrap_or(virtual_file::IoMode::preferred()),
452 481 : no_sync: no_sync.unwrap_or(false),
453 481 : enable_read_path_debugging: enable_read_path_debugging.unwrap_or(false),
454 481 : validate_wal_contiguity: validate_wal_contiguity.unwrap_or(false),
455 481 : };
456 481 :
457 481 : // ------------------------------------------------------------
458 481 : // custom validation code that covers more than one field in isolation
459 481 : // ------------------------------------------------------------
460 481 :
461 481 : if conf.http_auth_type == AuthType::NeonJWT || conf.pg_auth_type == AuthType::NeonJWT {
462 0 : let auth_validation_public_key_path = conf
463 0 : .auth_validation_public_key_path
464 0 : .get_or_insert_with(|| workdir.join("auth_public_key.pem"));
465 0 : ensure!(
466 0 : auth_validation_public_key_path.exists(),
467 0 : format!(
468 0 : "Can't find auth_validation_public_key at '{auth_validation_public_key_path}'",
469 0 : )
470 : );
471 481 : }
472 :
473 481 : IndexEntry::validate_checkpoint_distance(conf.default_tenant_conf.checkpoint_distance)
474 481 : .map_err(anyhow::Error::msg)
475 481 : .with_context(|| {
476 0 : format!(
477 0 : "effective checkpoint distance is unsupported: {}",
478 0 : conf.default_tenant_conf.checkpoint_distance
479 0 : )
480 481 : })?;
481 :
482 481 : Ok(conf)
483 481 : }
484 :
485 : #[cfg(test)]
486 481 : pub fn test_repo_dir(test_name: &str) -> Utf8PathBuf {
487 481 : let test_output_dir = std::env::var("TEST_OUTPUT").unwrap_or("../tmp_check".into());
488 481 : Utf8PathBuf::from(format!("{test_output_dir}/test_{test_name}"))
489 481 : }
490 :
491 477 : pub fn dummy_conf(repo_dir: Utf8PathBuf) -> Self {
492 477 : let pg_distrib_dir = Utf8PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../pg_install");
493 477 :
494 477 : let config_toml = pageserver_api::config::ConfigToml {
495 477 : wait_lsn_timeout: Duration::from_secs(60),
496 477 : wal_redo_timeout: Duration::from_secs(60),
497 477 : pg_distrib_dir: Some(pg_distrib_dir),
498 477 : metric_collection_interval: Duration::from_secs(60),
499 477 : synthetic_size_calculation_interval: Duration::from_secs(60),
500 477 : background_task_maximum_delay: Duration::ZERO,
501 477 : ..Default::default()
502 477 : };
503 477 : PageServerConf::parse_and_validate(NodeId(0), config_toml, &repo_dir).unwrap()
504 477 : }
505 : }
506 :
507 0 : #[derive(serde::Deserialize, serde::Serialize)]
508 : #[serde(deny_unknown_fields)]
509 : pub struct PageserverIdentity {
510 : pub id: NodeId,
511 : }
512 :
513 : /// Configurable semaphore permits setting.
514 : ///
515 : /// Does not allow semaphore permits to be zero, because at runtime initially zero permits and empty
516 : /// semaphore cannot be distinguished, leading any feature using these to await forever (or until
517 : /// new permits are added).
518 : #[derive(Debug, Clone)]
519 : pub struct ConfigurableSemaphore {
520 : initial_permits: NonZeroUsize,
521 : inner: std::sync::Arc<tokio::sync::Semaphore>,
522 : }
523 :
524 : impl ConfigurableSemaphore {
525 : /// Initializse using a non-zero amount of permits.
526 : ///
527 : /// Require a non-zero initial permits, because using permits == 0 is a crude way to disable a
528 : /// feature such as [`Tenant::gather_size_inputs`]. Otherwise any semaphore using future will
529 : /// behave like [`futures::future::pending`], just waiting until new permits are added.
530 : ///
531 : /// [`Tenant::gather_size_inputs`]: crate::tenant::Tenant::gather_size_inputs
532 1443 : pub fn new(initial_permits: NonZeroUsize) -> Self {
533 1443 : ConfigurableSemaphore {
534 1443 : initial_permits,
535 1443 : inner: std::sync::Arc::new(tokio::sync::Semaphore::new(initial_permits.get())),
536 1443 : }
537 1443 : }
538 :
539 : /// Returns the configured amount of permits.
540 0 : pub fn initial_permits(&self) -> NonZeroUsize {
541 0 : self.initial_permits
542 0 : }
543 : }
544 :
545 : impl PartialEq for ConfigurableSemaphore {
546 0 : fn eq(&self, other: &Self) -> bool {
547 0 : // the number of permits can be increased at runtime, so we cannot really fulfill the
548 0 : // PartialEq value equality otherwise
549 0 : self.initial_permits == other.initial_permits
550 0 : }
551 : }
552 :
553 : impl Eq for ConfigurableSemaphore {}
554 :
555 : impl ConfigurableSemaphore {
556 0 : pub fn inner(&self) -> &std::sync::Arc<tokio::sync::Semaphore> {
557 0 : &self.inner
558 0 : }
559 : }
560 :
561 : #[cfg(test)]
562 : mod tests {
563 :
564 : use camino::Utf8PathBuf;
565 : use utils::id::NodeId;
566 :
567 : use super::PageServerConf;
568 :
569 : #[test]
570 4 : fn test_empty_config_toml_is_valid() {
571 4 : // we use Default impl of everything in this situation
572 4 : let input = r#"
573 4 : "#;
574 4 : let config_toml = toml_edit::de::from_str::<pageserver_api::config::ConfigToml>(input)
575 4 : .expect("empty config is valid");
576 4 : let workdir = Utf8PathBuf::from("/nonexistent");
577 4 : PageServerConf::parse_and_validate(NodeId(0), config_toml, &workdir)
578 4 : .expect("parse_and_validate");
579 4 : }
580 :
581 : /// If there's a typo in the pageserver config, we'd rather catch that typo
582 : /// and fail pageserver startup than silently ignoring the typo, leaving whoever
583 : /// made it in the believe that their config change is effective.
584 : ///
585 : /// The default in serde is to allow unknown fields, so, we rely
586 : /// on developer+review discipline to add `deny_unknown_fields` when adding
587 : /// new structs to the config, and these tests here as a regression test.
588 : ///
589 : /// The alternative to all of this would be to allow unknown fields in the config.
590 : /// To catch them, we could have a config check tool or mgmt API endpoint that
591 : /// compares the effective config with the TOML on disk and makes sure that
592 : /// the on-disk TOML is a strict subset of the effective config.
593 : mod unknown_fields_handling {
594 : macro_rules! test {
595 : ($short_name:ident, $input:expr) => {
596 : #[test]
597 20 : fn $short_name() {
598 20 : let input = $input;
599 20 : let err = toml_edit::de::from_str::<pageserver_api::config::ConfigToml>(&input)
600 20 : .expect_err("some_invalid_field is an invalid field");
601 20 : dbg!(&err);
602 20 : assert!(err.to_string().contains("some_invalid_field"));
603 20 : }
604 : };
605 : }
606 : use indoc::indoc;
607 :
608 : test!(
609 : toplevel,
610 : indoc! {r#"
611 : some_invalid_field = 23
612 : "#}
613 : );
614 :
615 : test!(
616 : toplevel_nested,
617 : indoc! {r#"
618 : [some_invalid_field]
619 : foo = 23
620 : "#}
621 : );
622 :
623 : test!(
624 : disk_usage_based_eviction,
625 : indoc! {r#"
626 : [disk_usage_based_eviction]
627 : some_invalid_field = 23
628 : "#}
629 : );
630 :
631 : test!(
632 : tenant_config,
633 : indoc! {r#"
634 : [tenant_config]
635 : some_invalid_field = 23
636 : "#}
637 : );
638 :
639 : test!(
640 : l0_flush,
641 : indoc! {r#"
642 : [l0_flush]
643 : mode = "direct"
644 : some_invalid_field = 23
645 : "#}
646 : );
647 :
648 : // TODO: fix this => https://github.com/neondatabase/neon/issues/8915
649 : // test!(
650 : // remote_storage_config,
651 : // indoc! {r#"
652 : // [remote_storage_config]
653 : // local_path = "/nonexistent"
654 : // some_invalid_field = 23
655 : // "#}
656 : // );
657 : }
658 : }
|
