Line data Source code
1 : //! Functions for handling per-tenant configuration options
2 : //!
3 : //! If tenant is created with --config option,
4 : //! the tenant-specific config will be stored in tenant's directory.
5 : //! Otherwise, global pageserver's config is used.
6 : //!
7 : //! If the tenant config file is corrupted, the tenant will be disabled.
8 : //! We cannot use global or default config instead, because wrong settings
9 : //! may lead to a data loss.
10 : //!
11 : use anyhow::bail;
12 : use pageserver_api::models::AuxFilePolicy;
13 : use pageserver_api::models::CompactionAlgorithm;
14 : use pageserver_api::models::EvictionPolicy;
15 : use pageserver_api::models::{self, ThrottleConfig};
16 : use pageserver_api::shard::{ShardCount, ShardIdentity, ShardNumber, ShardStripeSize};
17 : use serde::de::IntoDeserializer;
18 : use serde::{Deserialize, Serialize};
19 : use serde_json::Value;
20 : use std::num::NonZeroU64;
21 : use std::time::Duration;
22 : use utils::generation::Generation;
23 :
24 : pub mod defaults {
25 :
26 : // FIXME: This current value is very low. I would imagine something like 1 GB or 10 GB
27 : // would be more appropriate. But a low value forces the code to be exercised more,
28 : // which is good for now to trigger bugs.
29 : // This parameter actually determines L0 layer file size.
30 : pub const DEFAULT_CHECKPOINT_DISTANCE: u64 = 256 * 1024 * 1024;
31 : pub const DEFAULT_CHECKPOINT_TIMEOUT: &str = "10 m";
32 :
33 : // FIXME the below configs are only used by legacy algorithm. The new algorithm
34 : // has different parameters.
35 :
36 : // Target file size, when creating image and delta layers.
37 : // This parameter determines L1 layer file size.
38 : pub const DEFAULT_COMPACTION_TARGET_SIZE: u64 = 128 * 1024 * 1024;
39 :
40 : pub const DEFAULT_COMPACTION_PERIOD: &str = "20 s";
41 : pub const DEFAULT_COMPACTION_THRESHOLD: usize = 10;
42 : pub const DEFAULT_COMPACTION_ALGORITHM: super::CompactionAlgorithm =
43 : super::CompactionAlgorithm::Legacy;
44 :
45 : pub const DEFAULT_GC_HORIZON: u64 = 64 * 1024 * 1024;
46 :
47 : // Large DEFAULT_GC_PERIOD is fine as long as PITR_INTERVAL is larger.
48 : // If there's a need to decrease this value, first make sure that GC
49 : // doesn't hold a layer map write lock for non-trivial operations.
50 : // Relevant: https://github.com/neondatabase/neon/issues/3394
51 : pub const DEFAULT_GC_PERIOD: &str = "1 hr";
52 : pub const DEFAULT_IMAGE_CREATION_THRESHOLD: usize = 3;
53 : pub const DEFAULT_PITR_INTERVAL: &str = "7 days";
54 : pub const DEFAULT_WALRECEIVER_CONNECT_TIMEOUT: &str = "10 seconds";
55 : pub const DEFAULT_WALRECEIVER_LAGGING_WAL_TIMEOUT: &str = "10 seconds";
56 : // The default limit on WAL lag should be set to avoid causing disconnects under high throughput
57 : // scenarios: since the broker stats are updated ~1/s, a value of 1GiB should be sufficient for
58 : // throughputs up to 1GiB/s per timeline.
59 : pub const DEFAULT_MAX_WALRECEIVER_LSN_WAL_LAG: u64 = 1024 * 1024 * 1024;
60 : pub const DEFAULT_EVICTIONS_LOW_RESIDENCE_DURATION_METRIC_THRESHOLD: &str = "24 hour";
61 : // By default ingest enough WAL for two new L0 layers before checking if new image
62 : // image layers should be created.
63 : pub const DEFAULT_IMAGE_LAYER_CREATION_CHECK_THRESHOLD: u8 = 2;
64 :
65 : pub const DEFAULT_INGEST_BATCH_SIZE: u64 = 100;
66 : }
67 :
68 0 : #[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)]
69 : pub(crate) enum AttachmentMode {
70 : /// Our generation is current as far as we know, and as far as we know we are the only attached
71 : /// pageserver. This is the "normal" attachment mode.
72 : Single,
73 : /// Our generation number is current as far as we know, but we are advised that another
74 : /// pageserver is still attached, and therefore to avoid executing deletions. This is
75 : /// the attachment mode of a pagesever that is the destination of a migration.
76 : Multi,
77 : /// Our generation number is superseded, or about to be superseded. We are advised
78 : /// to avoid remote storage writes if possible, and to avoid sending billing data. This
79 : /// is the attachment mode of a pageserver that is the origin of a migration.
80 : Stale,
81 : }
82 :
83 0 : #[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)]
84 : pub(crate) struct AttachedLocationConfig {
85 : pub(crate) generation: Generation,
86 : pub(crate) attach_mode: AttachmentMode,
87 : // TODO: add a flag to override AttachmentMode's policies under
88 : // disk pressure (i.e. unblock uploads under disk pressure in Stale
89 : // state, unblock deletions after timeout in Multi state)
90 : }
91 :
92 0 : #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
93 : pub(crate) struct SecondaryLocationConfig {
94 : /// If true, keep the local cache warm by polling remote storage
95 : pub(crate) warm: bool,
96 : }
97 :
98 0 : #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
99 : pub(crate) enum LocationMode {
100 : Attached(AttachedLocationConfig),
101 : Secondary(SecondaryLocationConfig),
102 : }
103 :
104 : /// Per-tenant, per-pageserver configuration. All pageservers use the same TenantConf,
105 : /// but have distinct LocationConf.
106 0 : #[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
107 : pub(crate) struct LocationConf {
108 : /// The location-specific part of the configuration, describes the operating
109 : /// mode of this pageserver for this tenant.
110 : pub(crate) mode: LocationMode,
111 :
112 : /// The detailed shard identity. This structure is already scoped within
113 : /// a TenantShardId, but we need the full ShardIdentity to enable calculating
114 : /// key->shard mappings.
115 : #[serde(default = "ShardIdentity::unsharded")]
116 : #[serde(skip_serializing_if = "ShardIdentity::is_unsharded")]
117 : pub(crate) shard: ShardIdentity,
118 :
119 : /// The pan-cluster tenant configuration, the same on all locations
120 : pub(crate) tenant_conf: TenantConfOpt,
121 : }
122 :
123 : impl std::fmt::Debug for LocationConf {
124 0 : fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
125 0 : match &self.mode {
126 0 : LocationMode::Attached(conf) => {
127 0 : write!(
128 0 : f,
129 0 : "Attached {:?}, gen={:?}",
130 0 : conf.attach_mode, conf.generation
131 0 : )
132 : }
133 0 : LocationMode::Secondary(conf) => {
134 0 : write!(f, "Secondary, warm={}", conf.warm)
135 : }
136 : }
137 0 : }
138 : }
139 :
140 : impl AttachedLocationConfig {
141 : /// Consult attachment mode to determine whether we are currently permitted
142 : /// to delete layers. This is only advisory, not required for data safety.
143 : /// See [`AttachmentMode`] for more context.
144 756 : pub(crate) fn may_delete_layers_hint(&self) -> bool {
145 756 : // TODO: add an override for disk pressure in AttachedLocationConfig,
146 756 : // and respect it here.
147 756 : match &self.attach_mode {
148 756 : AttachmentMode::Single => true,
149 : AttachmentMode::Multi | AttachmentMode::Stale => {
150 : // In Multi mode we avoid doing deletions because some other
151 : // attached pageserver might get 404 while trying to read
152 : // a layer we delete which is still referenced in their metadata.
153 : //
154 : // In Stale mode, we avoid doing deletions because we expect
155 : // that they would ultimately fail validation in the deletion
156 : // queue due to our stale generation.
157 0 : false
158 : }
159 : }
160 756 : }
161 :
162 : /// Whether we are currently hinted that it is worthwhile to upload layers.
163 : /// This is only advisory, not required for data safety.
164 : /// See [`AttachmentMode`] for more context.
165 0 : pub(crate) fn may_upload_layers_hint(&self) -> bool {
166 0 : // TODO: add an override for disk pressure in AttachedLocationConfig,
167 0 : // and respect it here.
168 0 : match &self.attach_mode {
169 0 : AttachmentMode::Single | AttachmentMode::Multi => true,
170 : AttachmentMode::Stale => {
171 : // In Stale mode, we avoid doing uploads because we expect that
172 : // our replacement pageserver will already have started its own
173 : // IndexPart that will never reference layers we upload: it is
174 : // wasteful.
175 0 : false
176 : }
177 : }
178 0 : }
179 : }
180 :
181 : impl LocationConf {
182 : /// For use when loading from a legacy configuration: presence of a tenant
183 : /// implies it is in AttachmentMode::Single, which used to be the only
184 : /// possible state. This function should eventually be removed.
185 138 : pub(crate) fn attached_single(
186 138 : tenant_conf: TenantConfOpt,
187 138 : generation: Generation,
188 138 : shard_params: &models::ShardParameters,
189 138 : ) -> Self {
190 138 : Self {
191 138 : mode: LocationMode::Attached(AttachedLocationConfig {
192 138 : generation,
193 138 : attach_mode: AttachmentMode::Single,
194 138 : }),
195 138 : shard: ShardIdentity::from_params(ShardNumber(0), shard_params),
196 138 : tenant_conf,
197 138 : }
198 138 : }
199 :
200 : /// For use when attaching/re-attaching: update the generation stored in this
201 : /// structure. If we were in a secondary state, promote to attached (posession
202 : /// of a fresh generation implies this).
203 0 : pub(crate) fn attach_in_generation(&mut self, mode: AttachmentMode, generation: Generation) {
204 0 : match &mut self.mode {
205 0 : LocationMode::Attached(attach_conf) => {
206 0 : attach_conf.generation = generation;
207 0 : attach_conf.attach_mode = mode;
208 0 : }
209 : LocationMode::Secondary(_) => {
210 : // We are promoted to attached by the control plane's re-attach response
211 0 : self.mode = LocationMode::Attached(AttachedLocationConfig {
212 0 : generation,
213 0 : attach_mode: mode,
214 0 : })
215 : }
216 : }
217 0 : }
218 :
219 0 : pub(crate) fn try_from(conf: &'_ models::LocationConfig) -> anyhow::Result<Self> {
220 0 : let tenant_conf = TenantConfOpt::try_from(&conf.tenant_conf)?;
221 :
222 0 : fn get_generation(conf: &'_ models::LocationConfig) -> Result<Generation, anyhow::Error> {
223 0 : conf.generation
224 0 : .map(Generation::new)
225 0 : .ok_or_else(|| anyhow::anyhow!("Generation must be set when attaching"))
226 0 : }
227 :
228 0 : let mode = match &conf.mode {
229 : models::LocationConfigMode::AttachedMulti => {
230 : LocationMode::Attached(AttachedLocationConfig {
231 0 : generation: get_generation(conf)?,
232 0 : attach_mode: AttachmentMode::Multi,
233 : })
234 : }
235 : models::LocationConfigMode::AttachedSingle => {
236 : LocationMode::Attached(AttachedLocationConfig {
237 0 : generation: get_generation(conf)?,
238 0 : attach_mode: AttachmentMode::Single,
239 : })
240 : }
241 : models::LocationConfigMode::AttachedStale => {
242 : LocationMode::Attached(AttachedLocationConfig {
243 0 : generation: get_generation(conf)?,
244 0 : attach_mode: AttachmentMode::Stale,
245 : })
246 : }
247 : models::LocationConfigMode::Secondary => {
248 0 : anyhow::ensure!(conf.generation.is_none());
249 :
250 0 : let warm = conf
251 0 : .secondary_conf
252 0 : .as_ref()
253 0 : .map(|c| c.warm)
254 0 : .unwrap_or(false);
255 0 : LocationMode::Secondary(SecondaryLocationConfig { warm })
256 : }
257 : models::LocationConfigMode::Detached => {
258 : // Should not have been called: API code should translate this mode
259 : // into a detach rather than trying to decode it as a LocationConf
260 0 : return Err(anyhow::anyhow!("Cannot decode a Detached configuration"));
261 : }
262 : };
263 :
264 0 : let shard = if conf.shard_count == 0 {
265 0 : ShardIdentity::unsharded()
266 : } else {
267 0 : ShardIdentity::new(
268 0 : ShardNumber(conf.shard_number),
269 0 : ShardCount::new(conf.shard_count),
270 0 : ShardStripeSize(conf.shard_stripe_size),
271 0 : )?
272 : };
273 :
274 0 : Ok(Self {
275 0 : shard,
276 0 : mode,
277 0 : tenant_conf,
278 0 : })
279 0 : }
280 : }
281 :
282 : impl Default for LocationConf {
283 : // TODO: this should be removed once tenant loading can guarantee that we are never
284 : // loading from a directory without a configuration.
285 : // => tech debt since https://github.com/neondatabase/neon/issues/1555
286 0 : fn default() -> Self {
287 0 : Self {
288 0 : mode: LocationMode::Attached(AttachedLocationConfig {
289 0 : generation: Generation::none(),
290 0 : attach_mode: AttachmentMode::Single,
291 0 : }),
292 0 : tenant_conf: TenantConfOpt::default(),
293 0 : shard: ShardIdentity::unsharded(),
294 0 : }
295 0 : }
296 : }
297 :
298 : /// A tenant's calcuated configuration, which is the result of merging a
299 : /// tenant's TenantConfOpt with the global TenantConf from PageServerConf.
300 : ///
301 : /// For storing and transmitting individual tenant's configuration, see
302 : /// TenantConfOpt.
303 0 : #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
304 : pub struct TenantConf {
305 : // Flush out an inmemory layer, if it's holding WAL older than this
306 : // This puts a backstop on how much WAL needs to be re-digested if the
307 : // page server crashes.
308 : // This parameter actually determines L0 layer file size.
309 : pub checkpoint_distance: u64,
310 : // Inmemory layer is also flushed at least once in checkpoint_timeout to
311 : // eventually upload WAL after activity is stopped.
312 : #[serde(with = "humantime_serde")]
313 : pub checkpoint_timeout: Duration,
314 : // Target file size, when creating image and delta layers.
315 : // This parameter determines L1 layer file size.
316 : pub compaction_target_size: u64,
317 : // How often to check if there's compaction work to be done.
318 : // Duration::ZERO means automatic compaction is disabled.
319 : #[serde(with = "humantime_serde")]
320 : pub compaction_period: Duration,
321 : // Level0 delta layer threshold for compaction.
322 : pub compaction_threshold: usize,
323 : pub compaction_algorithm: CompactionAlgorithm,
324 : // Determines how much history is retained, to allow
325 : // branching and read replicas at an older point in time.
326 : // The unit is #of bytes of WAL.
327 : // Page versions older than this are garbage collected away.
328 : pub gc_horizon: u64,
329 : // Interval at which garbage collection is triggered.
330 : // Duration::ZERO means automatic GC is disabled
331 : #[serde(with = "humantime_serde")]
332 : pub gc_period: Duration,
333 : // Delta layer churn threshold to create L1 image layers.
334 : pub image_creation_threshold: usize,
335 : // Determines how much history is retained, to allow
336 : // branching and read replicas at an older point in time.
337 : // The unit is time.
338 : // Page versions older than this are garbage collected away.
339 : #[serde(with = "humantime_serde")]
340 : pub pitr_interval: Duration,
341 : /// Maximum amount of time to wait while opening a connection to receive wal, before erroring.
342 : #[serde(with = "humantime_serde")]
343 : pub walreceiver_connect_timeout: Duration,
344 : /// Considers safekeepers stalled after no WAL updates were received longer than this threshold.
345 : /// A stalled safekeeper will be changed to a newer one when it appears.
346 : #[serde(with = "humantime_serde")]
347 : pub lagging_wal_timeout: Duration,
348 : /// Considers safekeepers lagging when their WAL is behind another safekeeper for more than this threshold.
349 : /// A lagging safekeeper will be changed after `lagging_wal_timeout` time elapses since the last WAL update,
350 : /// to avoid eager reconnects.
351 : pub max_lsn_wal_lag: NonZeroU64,
352 : pub trace_read_requests: bool,
353 : pub eviction_policy: EvictionPolicy,
354 : pub min_resident_size_override: Option<u64>,
355 : // See the corresponding metric's help string.
356 : #[serde(with = "humantime_serde")]
357 : pub evictions_low_residence_duration_metric_threshold: Duration,
358 :
359 : /// If non-zero, the period between uploads of a heatmap from attached tenants. This
360 : /// may be disabled if a Tenant will not have secondary locations: only secondary
361 : /// locations will use the heatmap uploaded by attached locations.
362 : #[serde(with = "humantime_serde")]
363 : pub heatmap_period: Duration,
364 :
365 : /// If true then SLRU segments are dowloaded on demand, if false SLRU segments are included in basebackup
366 : pub lazy_slru_download: bool,
367 :
368 : pub timeline_get_throttle: pageserver_api::models::ThrottleConfig,
369 :
370 : // How much WAL must be ingested before checking again whether a new image layer is required.
371 : // Expresed in multiples of checkpoint distance.
372 : pub image_layer_creation_check_threshold: u8,
373 :
374 : /// Switch to a new aux file policy. Switching this flag requires the user has not written any aux file into
375 : /// the storage before, and this flag cannot be switched back. Otherwise there will be data corruptions.
376 : /// There is a `last_aux_file_policy` flag which gets persisted in `index_part.json` once the first aux
377 : /// file is written.
378 : pub switch_aux_file_policy: AuxFilePolicy,
379 : }
380 :
381 : /// Same as TenantConf, but this struct preserves the information about
382 : /// which parameters are set and which are not.
383 144 : #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
384 : pub struct TenantConfOpt {
385 : #[serde(skip_serializing_if = "Option::is_none")]
386 : #[serde(default)]
387 : pub checkpoint_distance: Option<u64>,
388 :
389 : #[serde(skip_serializing_if = "Option::is_none")]
390 : #[serde(with = "humantime_serde")]
391 : #[serde(default)]
392 : pub checkpoint_timeout: Option<Duration>,
393 :
394 : #[serde(skip_serializing_if = "Option::is_none")]
395 : #[serde(default)]
396 : pub compaction_target_size: Option<u64>,
397 :
398 : #[serde(skip_serializing_if = "Option::is_none")]
399 : #[serde(with = "humantime_serde")]
400 : #[serde(default)]
401 : pub compaction_period: Option<Duration>,
402 :
403 : #[serde(skip_serializing_if = "Option::is_none")]
404 : #[serde(default)]
405 : pub compaction_threshold: Option<usize>,
406 :
407 : #[serde(skip_serializing_if = "Option::is_none")]
408 : #[serde(default)]
409 : pub compaction_algorithm: Option<CompactionAlgorithm>,
410 :
411 : #[serde(skip_serializing_if = "Option::is_none")]
412 : #[serde(default)]
413 : pub gc_horizon: Option<u64>,
414 :
415 : #[serde(skip_serializing_if = "Option::is_none")]
416 : #[serde(with = "humantime_serde")]
417 : #[serde(default)]
418 : pub gc_period: Option<Duration>,
419 :
420 : #[serde(skip_serializing_if = "Option::is_none")]
421 : #[serde(default)]
422 : pub image_creation_threshold: Option<usize>,
423 :
424 : #[serde(skip_serializing_if = "Option::is_none")]
425 : #[serde(with = "humantime_serde")]
426 : #[serde(default)]
427 : pub pitr_interval: Option<Duration>,
428 :
429 : #[serde(skip_serializing_if = "Option::is_none")]
430 : #[serde(with = "humantime_serde")]
431 : #[serde(default)]
432 : pub walreceiver_connect_timeout: Option<Duration>,
433 :
434 : #[serde(skip_serializing_if = "Option::is_none")]
435 : #[serde(with = "humantime_serde")]
436 : #[serde(default)]
437 : pub lagging_wal_timeout: Option<Duration>,
438 :
439 : #[serde(skip_serializing_if = "Option::is_none")]
440 : #[serde(default)]
441 : pub max_lsn_wal_lag: Option<NonZeroU64>,
442 :
443 : #[serde(skip_serializing_if = "Option::is_none")]
444 : #[serde(default)]
445 : pub trace_read_requests: Option<bool>,
446 :
447 : #[serde(skip_serializing_if = "Option::is_none")]
448 : #[serde(default)]
449 : pub eviction_policy: Option<EvictionPolicy>,
450 :
451 : #[serde(skip_serializing_if = "Option::is_none")]
452 : #[serde(default)]
453 : pub min_resident_size_override: Option<u64>,
454 :
455 : #[serde(skip_serializing_if = "Option::is_none")]
456 : #[serde(with = "humantime_serde")]
457 : #[serde(default)]
458 : pub evictions_low_residence_duration_metric_threshold: Option<Duration>,
459 :
460 : #[serde(skip_serializing_if = "Option::is_none")]
461 : #[serde(with = "humantime_serde")]
462 : #[serde(default)]
463 : pub heatmap_period: Option<Duration>,
464 :
465 : #[serde(skip_serializing_if = "Option::is_none")]
466 : #[serde(default)]
467 : pub lazy_slru_download: Option<bool>,
468 :
469 : #[serde(skip_serializing_if = "Option::is_none")]
470 : pub timeline_get_throttle: Option<pageserver_api::models::ThrottleConfig>,
471 :
472 : #[serde(skip_serializing_if = "Option::is_none")]
473 : pub image_layer_creation_check_threshold: Option<u8>,
474 :
475 : #[serde(skip_serializing_if = "Option::is_none")]
476 : #[serde(default)]
477 : pub switch_aux_file_policy: Option<AuxFilePolicy>,
478 : }
479 :
480 : impl TenantConfOpt {
481 18 : pub fn merge(&self, global_conf: TenantConf) -> TenantConf {
482 18 : TenantConf {
483 18 : checkpoint_distance: self
484 18 : .checkpoint_distance
485 18 : .unwrap_or(global_conf.checkpoint_distance),
486 18 : checkpoint_timeout: self
487 18 : .checkpoint_timeout
488 18 : .unwrap_or(global_conf.checkpoint_timeout),
489 18 : compaction_target_size: self
490 18 : .compaction_target_size
491 18 : .unwrap_or(global_conf.compaction_target_size),
492 18 : compaction_period: self
493 18 : .compaction_period
494 18 : .unwrap_or(global_conf.compaction_period),
495 18 : compaction_threshold: self
496 18 : .compaction_threshold
497 18 : .unwrap_or(global_conf.compaction_threshold),
498 18 : compaction_algorithm: self
499 18 : .compaction_algorithm
500 18 : .unwrap_or(global_conf.compaction_algorithm),
501 18 : gc_horizon: self.gc_horizon.unwrap_or(global_conf.gc_horizon),
502 18 : gc_period: self.gc_period.unwrap_or(global_conf.gc_period),
503 18 : image_creation_threshold: self
504 18 : .image_creation_threshold
505 18 : .unwrap_or(global_conf.image_creation_threshold),
506 18 : pitr_interval: self.pitr_interval.unwrap_or(global_conf.pitr_interval),
507 18 : walreceiver_connect_timeout: self
508 18 : .walreceiver_connect_timeout
509 18 : .unwrap_or(global_conf.walreceiver_connect_timeout),
510 18 : lagging_wal_timeout: self
511 18 : .lagging_wal_timeout
512 18 : .unwrap_or(global_conf.lagging_wal_timeout),
513 18 : max_lsn_wal_lag: self.max_lsn_wal_lag.unwrap_or(global_conf.max_lsn_wal_lag),
514 18 : trace_read_requests: self
515 18 : .trace_read_requests
516 18 : .unwrap_or(global_conf.trace_read_requests),
517 18 : eviction_policy: self.eviction_policy.unwrap_or(global_conf.eviction_policy),
518 18 : min_resident_size_override: self
519 18 : .min_resident_size_override
520 18 : .or(global_conf.min_resident_size_override),
521 18 : evictions_low_residence_duration_metric_threshold: self
522 18 : .evictions_low_residence_duration_metric_threshold
523 18 : .unwrap_or(global_conf.evictions_low_residence_duration_metric_threshold),
524 18 : heatmap_period: self.heatmap_period.unwrap_or(global_conf.heatmap_period),
525 18 : lazy_slru_download: self
526 18 : .lazy_slru_download
527 18 : .unwrap_or(global_conf.lazy_slru_download),
528 18 : timeline_get_throttle: self
529 18 : .timeline_get_throttle
530 18 : .clone()
531 18 : .unwrap_or(global_conf.timeline_get_throttle),
532 18 : image_layer_creation_check_threshold: self
533 18 : .image_layer_creation_check_threshold
534 18 : .unwrap_or(global_conf.image_layer_creation_check_threshold),
535 18 : switch_aux_file_policy: self
536 18 : .switch_aux_file_policy
537 18 : .unwrap_or(global_conf.switch_aux_file_policy),
538 18 : }
539 18 : }
540 : }
541 :
542 : impl Default for TenantConf {
543 308 : fn default() -> Self {
544 308 : use defaults::*;
545 308 : Self {
546 308 : checkpoint_distance: DEFAULT_CHECKPOINT_DISTANCE,
547 308 : checkpoint_timeout: humantime::parse_duration(DEFAULT_CHECKPOINT_TIMEOUT)
548 308 : .expect("cannot parse default checkpoint timeout"),
549 308 : compaction_target_size: DEFAULT_COMPACTION_TARGET_SIZE,
550 308 : compaction_period: humantime::parse_duration(DEFAULT_COMPACTION_PERIOD)
551 308 : .expect("cannot parse default compaction period"),
552 308 : compaction_threshold: DEFAULT_COMPACTION_THRESHOLD,
553 308 : compaction_algorithm: DEFAULT_COMPACTION_ALGORITHM,
554 308 : gc_horizon: DEFAULT_GC_HORIZON,
555 308 : gc_period: humantime::parse_duration(DEFAULT_GC_PERIOD)
556 308 : .expect("cannot parse default gc period"),
557 308 : image_creation_threshold: DEFAULT_IMAGE_CREATION_THRESHOLD,
558 308 : pitr_interval: humantime::parse_duration(DEFAULT_PITR_INTERVAL)
559 308 : .expect("cannot parse default PITR interval"),
560 308 : walreceiver_connect_timeout: humantime::parse_duration(
561 308 : DEFAULT_WALRECEIVER_CONNECT_TIMEOUT,
562 308 : )
563 308 : .expect("cannot parse default walreceiver connect timeout"),
564 308 : lagging_wal_timeout: humantime::parse_duration(DEFAULT_WALRECEIVER_LAGGING_WAL_TIMEOUT)
565 308 : .expect("cannot parse default walreceiver lagging wal timeout"),
566 308 : max_lsn_wal_lag: NonZeroU64::new(DEFAULT_MAX_WALRECEIVER_LSN_WAL_LAG)
567 308 : .expect("cannot parse default max walreceiver Lsn wal lag"),
568 308 : trace_read_requests: false,
569 308 : eviction_policy: EvictionPolicy::NoEviction,
570 308 : min_resident_size_override: None,
571 308 : evictions_low_residence_duration_metric_threshold: humantime::parse_duration(
572 308 : DEFAULT_EVICTIONS_LOW_RESIDENCE_DURATION_METRIC_THRESHOLD,
573 308 : )
574 308 : .expect("cannot parse default evictions_low_residence_duration_metric_threshold"),
575 308 : heatmap_period: Duration::ZERO,
576 308 : lazy_slru_download: false,
577 308 : timeline_get_throttle: crate::tenant::throttle::Config::disabled(),
578 308 : image_layer_creation_check_threshold: DEFAULT_IMAGE_LAYER_CREATION_CHECK_THRESHOLD,
579 308 : switch_aux_file_policy: AuxFilePolicy::default_tenant_config(),
580 308 : }
581 308 : }
582 : }
583 :
584 : impl TryFrom<&'_ models::TenantConfig> for TenantConfOpt {
585 : type Error = anyhow::Error;
586 :
587 4 : fn try_from(request_data: &'_ models::TenantConfig) -> Result<Self, Self::Error> {
588 : // Convert the request_data to a JSON Value
589 4 : let json_value: Value = serde_json::to_value(request_data)?;
590 :
591 : // Create a Deserializer from the JSON Value
592 4 : let deserializer = json_value.into_deserializer();
593 :
594 : // Use serde_path_to_error to deserialize the JSON Value into TenantConfOpt
595 4 : let tenant_conf: TenantConfOpt = serde_path_to_error::deserialize(deserializer)?;
596 :
597 2 : Ok(tenant_conf)
598 4 : }
599 : }
600 :
601 : impl TryFrom<toml_edit::Item> for TenantConfOpt {
602 : type Error = anyhow::Error;
603 :
604 10 : fn try_from(item: toml_edit::Item) -> Result<Self, Self::Error> {
605 10 : match item {
606 2 : toml_edit::Item::Value(value) => {
607 2 : let d = value.into_deserializer();
608 2 : return serde_path_to_error::deserialize(d)
609 2 : .map_err(|e| anyhow::anyhow!("{}: {}", e.path(), e.inner().message()));
610 : }
611 8 : toml_edit::Item::Table(table) => {
612 8 : let deserializer = toml_edit::de::Deserializer::new(table.into());
613 8 : return serde_path_to_error::deserialize(deserializer)
614 8 : .map_err(|e| anyhow::anyhow!("{}: {}", e.path(), e.inner().message()));
615 : }
616 : _ => {
617 0 : bail!("expected non-inline table but found {item}")
618 : }
619 : }
620 10 : }
621 : }
622 :
623 : /// This is a conversion from our internal tenant config object to the one used
624 : /// in external APIs.
625 : impl From<TenantConfOpt> for models::TenantConfig {
626 0 : fn from(value: TenantConfOpt) -> Self {
627 0 : fn humantime(d: Duration) -> String {
628 0 : format!("{}s", d.as_secs())
629 0 : }
630 0 : Self {
631 0 : checkpoint_distance: value.checkpoint_distance,
632 0 : checkpoint_timeout: value.checkpoint_timeout.map(humantime),
633 0 : compaction_algorithm: value.compaction_algorithm,
634 0 : compaction_target_size: value.compaction_target_size,
635 0 : compaction_period: value.compaction_period.map(humantime),
636 0 : compaction_threshold: value.compaction_threshold,
637 0 : gc_horizon: value.gc_horizon,
638 0 : gc_period: value.gc_period.map(humantime),
639 0 : image_creation_threshold: value.image_creation_threshold,
640 0 : pitr_interval: value.pitr_interval.map(humantime),
641 0 : walreceiver_connect_timeout: value.walreceiver_connect_timeout.map(humantime),
642 0 : lagging_wal_timeout: value.lagging_wal_timeout.map(humantime),
643 0 : max_lsn_wal_lag: value.max_lsn_wal_lag,
644 0 : trace_read_requests: value.trace_read_requests,
645 0 : eviction_policy: value.eviction_policy,
646 0 : min_resident_size_override: value.min_resident_size_override,
647 0 : evictions_low_residence_duration_metric_threshold: value
648 0 : .evictions_low_residence_duration_metric_threshold
649 0 : .map(humantime),
650 0 : heatmap_period: value.heatmap_period.map(humantime),
651 0 : lazy_slru_download: value.lazy_slru_download,
652 0 : timeline_get_throttle: value.timeline_get_throttle.map(ThrottleConfig::from),
653 0 : image_layer_creation_check_threshold: value.image_layer_creation_check_threshold,
654 0 : switch_aux_file_policy: value.switch_aux_file_policy,
655 0 : }
656 0 : }
657 : }
658 :
659 : #[cfg(test)]
660 : mod tests {
661 : use super::*;
662 : use models::TenantConfig;
663 :
664 : #[test]
665 2 : fn de_serializing_pageserver_config_omits_empty_values() {
666 2 : let small_conf = TenantConfOpt {
667 2 : gc_horizon: Some(42),
668 2 : ..TenantConfOpt::default()
669 2 : };
670 2 :
671 2 : let toml_form = toml_edit::ser::to_string(&small_conf).unwrap();
672 2 : assert_eq!(toml_form, "gc_horizon = 42\n");
673 2 : assert_eq!(small_conf, toml_edit::de::from_str(&toml_form).unwrap());
674 :
675 2 : let json_form = serde_json::to_string(&small_conf).unwrap();
676 2 : assert_eq!(json_form, "{\"gc_horizon\":42}");
677 2 : assert_eq!(small_conf, serde_json::from_str(&json_form).unwrap());
678 2 : }
679 :
680 : #[test]
681 2 : fn test_try_from_models_tenant_config_err() {
682 2 : let tenant_config = models::TenantConfig {
683 2 : lagging_wal_timeout: Some("5a".to_string()),
684 2 : ..TenantConfig::default()
685 2 : };
686 2 :
687 2 : let tenant_conf_opt = TenantConfOpt::try_from(&tenant_config);
688 2 :
689 2 : assert!(
690 2 : tenant_conf_opt.is_err(),
691 0 : "Suceeded to convert TenantConfig to TenantConfOpt"
692 : );
693 :
694 2 : let expected_error_str =
695 2 : "lagging_wal_timeout: invalid value: string \"5a\", expected a duration";
696 2 : assert_eq!(tenant_conf_opt.unwrap_err().to_string(), expected_error_str);
697 2 : }
698 :
699 : #[test]
700 2 : fn test_try_from_models_tenant_config_success() {
701 2 : let tenant_config = models::TenantConfig {
702 2 : lagging_wal_timeout: Some("5s".to_string()),
703 2 : ..TenantConfig::default()
704 2 : };
705 2 :
706 2 : let tenant_conf_opt = TenantConfOpt::try_from(&tenant_config).unwrap();
707 2 :
708 2 : assert_eq!(
709 2 : tenant_conf_opt.lagging_wal_timeout,
710 2 : Some(Duration::from_secs(5))
711 2 : );
712 2 : }
713 : }
|
