Line data Source code
1 : //! Functions for handling per-tenant configuration options
2 : //!
3 : //! If tenant is created with --config option,
4 : //! the tenant-specific config will be stored in tenant's directory.
5 : //! Otherwise, global pageserver's config is used.
6 : //!
7 : //! If the tenant config file is corrupted, the tenant will be disabled.
8 : //! We cannot use global or default config instead, because wrong settings
9 : //! may lead to a data loss.
10 : //!
11 : use anyhow::bail;
12 : use pageserver_api::models::AuxFilePolicy;
13 : use pageserver_api::models::CompactionAlgorithm;
14 : use pageserver_api::models::CompactionAlgorithmSettings;
15 : use pageserver_api::models::EvictionPolicy;
16 : use pageserver_api::models::LsnLease;
17 : use pageserver_api::models::{self, ThrottleConfig};
18 : use pageserver_api::shard::{ShardCount, ShardIdentity, ShardNumber, ShardStripeSize};
19 : use serde::de::IntoDeserializer;
20 : use serde::{Deserialize, Serialize};
21 : use serde_json::Value;
22 : use std::num::NonZeroU64;
23 : use std::time::Duration;
24 : use utils::generation::Generation;
25 :
26 : pub mod defaults {
27 :
28 : // FIXME: This current value is very low. I would imagine something like 1 GB or 10 GB
29 : // would be more appropriate. But a low value forces the code to be exercised more,
30 : // which is good for now to trigger bugs.
31 : // This parameter actually determines L0 layer file size.
32 : pub const DEFAULT_CHECKPOINT_DISTANCE: u64 = 256 * 1024 * 1024;
33 : pub const DEFAULT_CHECKPOINT_TIMEOUT: &str = "10 m";
34 :
35 : // FIXME the below configs are only used by legacy algorithm. The new algorithm
36 : // has different parameters.
37 :
38 : // Target file size, when creating image and delta layers.
39 : // This parameter determines L1 layer file size.
40 : pub const DEFAULT_COMPACTION_TARGET_SIZE: u64 = 128 * 1024 * 1024;
41 :
42 : pub const DEFAULT_COMPACTION_PERIOD: &str = "20 s";
43 : pub const DEFAULT_COMPACTION_THRESHOLD: usize = 10;
44 : pub const DEFAULT_COMPACTION_ALGORITHM: super::CompactionAlgorithm =
45 : super::CompactionAlgorithm::Legacy;
46 :
47 : pub const DEFAULT_GC_HORIZON: u64 = 64 * 1024 * 1024;
48 :
49 : // Large DEFAULT_GC_PERIOD is fine as long as PITR_INTERVAL is larger.
50 : // If there's a need to decrease this value, first make sure that GC
51 : // doesn't hold a layer map write lock for non-trivial operations.
52 : // Relevant: https://github.com/neondatabase/neon/issues/3394
53 : pub const DEFAULT_GC_PERIOD: &str = "1 hr";
54 : pub const DEFAULT_IMAGE_CREATION_THRESHOLD: usize = 3;
55 : pub const DEFAULT_PITR_INTERVAL: &str = "7 days";
56 : pub const DEFAULT_WALRECEIVER_CONNECT_TIMEOUT: &str = "10 seconds";
57 : pub const DEFAULT_WALRECEIVER_LAGGING_WAL_TIMEOUT: &str = "10 seconds";
58 : // The default limit on WAL lag should be set to avoid causing disconnects under high throughput
59 : // scenarios: since the broker stats are updated ~1/s, a value of 1GiB should be sufficient for
60 : // throughputs up to 1GiB/s per timeline.
61 : pub const DEFAULT_MAX_WALRECEIVER_LSN_WAL_LAG: u64 = 1024 * 1024 * 1024;
62 : pub const DEFAULT_EVICTIONS_LOW_RESIDENCE_DURATION_METRIC_THRESHOLD: &str = "24 hour";
63 : // By default ingest enough WAL for two new L0 layers before checking if new image
64 : // image layers should be created.
65 : pub const DEFAULT_IMAGE_LAYER_CREATION_CHECK_THRESHOLD: u8 = 2;
66 :
67 : pub const DEFAULT_INGEST_BATCH_SIZE: u64 = 100;
68 : }
69 :
70 0 : #[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)]
71 : pub(crate) enum AttachmentMode {
72 : /// Our generation is current as far as we know, and as far as we know we are the only attached
73 : /// pageserver. This is the "normal" attachment mode.
74 : Single,
75 : /// Our generation number is current as far as we know, but we are advised that another
76 : /// pageserver is still attached, and therefore to avoid executing deletions. This is
77 : /// the attachment mode of a pagesever that is the destination of a migration.
78 : Multi,
79 : /// Our generation number is superseded, or about to be superseded. We are advised
80 : /// to avoid remote storage writes if possible, and to avoid sending billing data. This
81 : /// is the attachment mode of a pageserver that is the origin of a migration.
82 : Stale,
83 : }
84 :
85 0 : #[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)]
86 : pub(crate) struct AttachedLocationConfig {
87 : pub(crate) generation: Generation,
88 : pub(crate) attach_mode: AttachmentMode,
89 : // TODO: add a flag to override AttachmentMode's policies under
90 : // disk pressure (i.e. unblock uploads under disk pressure in Stale
91 : // state, unblock deletions after timeout in Multi state)
92 : }
93 :
94 0 : #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
95 : pub(crate) struct SecondaryLocationConfig {
96 : /// If true, keep the local cache warm by polling remote storage
97 : pub(crate) warm: bool,
98 : }
99 :
100 0 : #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
101 : pub(crate) enum LocationMode {
102 : Attached(AttachedLocationConfig),
103 : Secondary(SecondaryLocationConfig),
104 : }
105 :
106 : /// Per-tenant, per-pageserver configuration. All pageservers use the same TenantConf,
107 : /// but have distinct LocationConf.
108 0 : #[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
109 : pub(crate) struct LocationConf {
110 : /// The location-specific part of the configuration, describes the operating
111 : /// mode of this pageserver for this tenant.
112 : pub(crate) mode: LocationMode,
113 :
114 : /// The detailed shard identity. This structure is already scoped within
115 : /// a TenantShardId, but we need the full ShardIdentity to enable calculating
116 : /// key->shard mappings.
117 : #[serde(default = "ShardIdentity::unsharded")]
118 : #[serde(skip_serializing_if = "ShardIdentity::is_unsharded")]
119 : pub(crate) shard: ShardIdentity,
120 :
121 : /// The pan-cluster tenant configuration, the same on all locations
122 : pub(crate) tenant_conf: TenantConfOpt,
123 : }
124 :
125 : impl std::fmt::Debug for LocationConf {
126 0 : fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
127 0 : match &self.mode {
128 0 : LocationMode::Attached(conf) => {
129 0 : write!(
130 0 : f,
131 0 : "Attached {:?}, gen={:?}",
132 0 : conf.attach_mode, conf.generation
133 0 : )
134 : }
135 0 : LocationMode::Secondary(conf) => {
136 0 : write!(f, "Secondary, warm={}", conf.warm)
137 : }
138 : }
139 0 : }
140 : }
141 :
142 : impl AttachedLocationConfig {
143 : /// Consult attachment mode to determine whether we are currently permitted
144 : /// to delete layers. This is only advisory, not required for data safety.
145 : /// See [`AttachmentMode`] for more context.
146 754 : pub(crate) fn may_delete_layers_hint(&self) -> bool {
147 754 : // TODO: add an override for disk pressure in AttachedLocationConfig,
148 754 : // and respect it here.
149 754 : match &self.attach_mode {
150 754 : AttachmentMode::Single => true,
151 : AttachmentMode::Multi | AttachmentMode::Stale => {
152 : // In Multi mode we avoid doing deletions because some other
153 : // attached pageserver might get 404 while trying to read
154 : // a layer we delete which is still referenced in their metadata.
155 : //
156 : // In Stale mode, we avoid doing deletions because we expect
157 : // that they would ultimately fail validation in the deletion
158 : // queue due to our stale generation.
159 0 : false
160 : }
161 : }
162 754 : }
163 :
164 : /// Whether we are currently hinted that it is worthwhile to upload layers.
165 : /// This is only advisory, not required for data safety.
166 : /// See [`AttachmentMode`] for more context.
167 0 : pub(crate) fn may_upload_layers_hint(&self) -> bool {
168 0 : // TODO: add an override for disk pressure in AttachedLocationConfig,
169 0 : // and respect it here.
170 0 : match &self.attach_mode {
171 0 : AttachmentMode::Single | AttachmentMode::Multi => true,
172 : AttachmentMode::Stale => {
173 : // In Stale mode, we avoid doing uploads because we expect that
174 : // our replacement pageserver will already have started its own
175 : // IndexPart that will never reference layers we upload: it is
176 : // wasteful.
177 0 : false
178 : }
179 : }
180 0 : }
181 : }
182 :
183 : impl LocationConf {
184 : /// For use when loading from a legacy configuration: presence of a tenant
185 : /// implies it is in AttachmentMode::Single, which used to be the only
186 : /// possible state. This function should eventually be removed.
187 166 : pub(crate) fn attached_single(
188 166 : tenant_conf: TenantConfOpt,
189 166 : generation: Generation,
190 166 : shard_params: &models::ShardParameters,
191 166 : ) -> Self {
192 166 : Self {
193 166 : mode: LocationMode::Attached(AttachedLocationConfig {
194 166 : generation,
195 166 : attach_mode: AttachmentMode::Single,
196 166 : }),
197 166 : shard: ShardIdentity::from_params(ShardNumber(0), shard_params),
198 166 : tenant_conf,
199 166 : }
200 166 : }
201 :
202 : /// For use when attaching/re-attaching: update the generation stored in this
203 : /// structure. If we were in a secondary state, promote to attached (posession
204 : /// of a fresh generation implies this).
205 0 : pub(crate) fn attach_in_generation(&mut self, mode: AttachmentMode, generation: Generation) {
206 0 : match &mut self.mode {
207 0 : LocationMode::Attached(attach_conf) => {
208 0 : attach_conf.generation = generation;
209 0 : attach_conf.attach_mode = mode;
210 0 : }
211 : LocationMode::Secondary(_) => {
212 : // We are promoted to attached by the control plane's re-attach response
213 0 : self.mode = LocationMode::Attached(AttachedLocationConfig {
214 0 : generation,
215 0 : attach_mode: mode,
216 0 : })
217 : }
218 : }
219 0 : }
220 :
221 0 : pub(crate) fn try_from(conf: &'_ models::LocationConfig) -> anyhow::Result<Self> {
222 0 : let tenant_conf = TenantConfOpt::try_from(&conf.tenant_conf)?;
223 :
224 0 : fn get_generation(conf: &'_ models::LocationConfig) -> Result<Generation, anyhow::Error> {
225 0 : conf.generation
226 0 : .map(Generation::new)
227 0 : .ok_or_else(|| anyhow::anyhow!("Generation must be set when attaching"))
228 0 : }
229 :
230 0 : let mode = match &conf.mode {
231 : models::LocationConfigMode::AttachedMulti => {
232 : LocationMode::Attached(AttachedLocationConfig {
233 0 : generation: get_generation(conf)?,
234 0 : attach_mode: AttachmentMode::Multi,
235 : })
236 : }
237 : models::LocationConfigMode::AttachedSingle => {
238 : LocationMode::Attached(AttachedLocationConfig {
239 0 : generation: get_generation(conf)?,
240 0 : attach_mode: AttachmentMode::Single,
241 : })
242 : }
243 : models::LocationConfigMode::AttachedStale => {
244 : LocationMode::Attached(AttachedLocationConfig {
245 0 : generation: get_generation(conf)?,
246 0 : attach_mode: AttachmentMode::Stale,
247 : })
248 : }
249 : models::LocationConfigMode::Secondary => {
250 0 : anyhow::ensure!(conf.generation.is_none());
251 :
252 0 : let warm = conf
253 0 : .secondary_conf
254 0 : .as_ref()
255 0 : .map(|c| c.warm)
256 0 : .unwrap_or(false);
257 0 : LocationMode::Secondary(SecondaryLocationConfig { warm })
258 : }
259 : models::LocationConfigMode::Detached => {
260 : // Should not have been called: API code should translate this mode
261 : // into a detach rather than trying to decode it as a LocationConf
262 0 : return Err(anyhow::anyhow!("Cannot decode a Detached configuration"));
263 : }
264 : };
265 :
266 0 : let shard = if conf.shard_count == 0 {
267 0 : ShardIdentity::unsharded()
268 : } else {
269 0 : ShardIdentity::new(
270 0 : ShardNumber(conf.shard_number),
271 0 : ShardCount::new(conf.shard_count),
272 0 : ShardStripeSize(conf.shard_stripe_size),
273 0 : )?
274 : };
275 :
276 0 : Ok(Self {
277 0 : shard,
278 0 : mode,
279 0 : tenant_conf,
280 0 : })
281 0 : }
282 : }
283 :
284 : impl Default for LocationConf {
285 : // TODO: this should be removed once tenant loading can guarantee that we are never
286 : // loading from a directory without a configuration.
287 : // => tech debt since https://github.com/neondatabase/neon/issues/1555
288 0 : fn default() -> Self {
289 0 : Self {
290 0 : mode: LocationMode::Attached(AttachedLocationConfig {
291 0 : generation: Generation::none(),
292 0 : attach_mode: AttachmentMode::Single,
293 0 : }),
294 0 : tenant_conf: TenantConfOpt::default(),
295 0 : shard: ShardIdentity::unsharded(),
296 0 : }
297 0 : }
298 : }
299 :
300 : /// A tenant's calcuated configuration, which is the result of merging a
301 : /// tenant's TenantConfOpt with the global TenantConf from PageServerConf.
302 : ///
303 : /// For storing and transmitting individual tenant's configuration, see
304 : /// TenantConfOpt.
305 0 : #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
306 : pub struct TenantConf {
307 : // Flush out an inmemory layer, if it's holding WAL older than this
308 : // This puts a backstop on how much WAL needs to be re-digested if the
309 : // page server crashes.
310 : // This parameter actually determines L0 layer file size.
311 : pub checkpoint_distance: u64,
312 : // Inmemory layer is also flushed at least once in checkpoint_timeout to
313 : // eventually upload WAL after activity is stopped.
314 : #[serde(with = "humantime_serde")]
315 : pub checkpoint_timeout: Duration,
316 : // Target file size, when creating image and delta layers.
317 : // This parameter determines L1 layer file size.
318 : pub compaction_target_size: u64,
319 : // How often to check if there's compaction work to be done.
320 : // Duration::ZERO means automatic compaction is disabled.
321 : #[serde(with = "humantime_serde")]
322 : pub compaction_period: Duration,
323 : // Level0 delta layer threshold for compaction.
324 : pub compaction_threshold: usize,
325 : pub compaction_algorithm: CompactionAlgorithmSettings,
326 : // Determines how much history is retained, to allow
327 : // branching and read replicas at an older point in time.
328 : // The unit is #of bytes of WAL.
329 : // Page versions older than this are garbage collected away.
330 : pub gc_horizon: u64,
331 : // Interval at which garbage collection is triggered.
332 : // Duration::ZERO means automatic GC is disabled
333 : #[serde(with = "humantime_serde")]
334 : pub gc_period: Duration,
335 : // Delta layer churn threshold to create L1 image layers.
336 : pub image_creation_threshold: usize,
337 : // Determines how much history is retained, to allow
338 : // branching and read replicas at an older point in time.
339 : // The unit is time.
340 : // Page versions older than this are garbage collected away.
341 : #[serde(with = "humantime_serde")]
342 : pub pitr_interval: Duration,
343 : /// Maximum amount of time to wait while opening a connection to receive wal, before erroring.
344 : #[serde(with = "humantime_serde")]
345 : pub walreceiver_connect_timeout: Duration,
346 : /// Considers safekeepers stalled after no WAL updates were received longer than this threshold.
347 : /// A stalled safekeeper will be changed to a newer one when it appears.
348 : #[serde(with = "humantime_serde")]
349 : pub lagging_wal_timeout: Duration,
350 : /// Considers safekeepers lagging when their WAL is behind another safekeeper for more than this threshold.
351 : /// A lagging safekeeper will be changed after `lagging_wal_timeout` time elapses since the last WAL update,
352 : /// to avoid eager reconnects.
353 : pub max_lsn_wal_lag: NonZeroU64,
354 : pub trace_read_requests: bool,
355 : pub eviction_policy: EvictionPolicy,
356 : pub min_resident_size_override: Option<u64>,
357 : // See the corresponding metric's help string.
358 : #[serde(with = "humantime_serde")]
359 : pub evictions_low_residence_duration_metric_threshold: Duration,
360 :
361 : /// If non-zero, the period between uploads of a heatmap from attached tenants. This
362 : /// may be disabled if a Tenant will not have secondary locations: only secondary
363 : /// locations will use the heatmap uploaded by attached locations.
364 : #[serde(with = "humantime_serde")]
365 : pub heatmap_period: Duration,
366 :
367 : /// If true then SLRU segments are dowloaded on demand, if false SLRU segments are included in basebackup
368 : pub lazy_slru_download: bool,
369 :
370 : pub timeline_get_throttle: pageserver_api::models::ThrottleConfig,
371 :
372 : // How much WAL must be ingested before checking again whether a new image layer is required.
373 : // Expresed in multiples of checkpoint distance.
374 : pub image_layer_creation_check_threshold: u8,
375 :
376 : /// Switch to a new aux file policy. Switching this flag requires the user has not written any aux file into
377 : /// the storage before, and this flag cannot be switched back. Otherwise there will be data corruptions.
378 : /// There is a `last_aux_file_policy` flag which gets persisted in `index_part.json` once the first aux
379 : /// file is written.
380 : pub switch_aux_file_policy: AuxFilePolicy,
381 :
382 : /// The length for an explicit LSN lease request.
383 : /// Layers needed to reconstruct pages at LSN will not be GC-ed during this interval.
384 : #[serde(with = "humantime_serde")]
385 : pub lsn_lease_length: Duration,
386 :
387 : /// The length for an implicit LSN lease granted as part of `get_lsn_by_timestamp` request.
388 : /// Layers needed to reconstruct pages at LSN will not be GC-ed during this interval.
389 : #[serde(with = "humantime_serde")]
390 : pub lsn_lease_length_for_ts: Duration,
391 : }
392 :
393 : /// Same as TenantConf, but this struct preserves the information about
394 : /// which parameters are set and which are not.
395 152 : #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
396 : pub struct TenantConfOpt {
397 : #[serde(skip_serializing_if = "Option::is_none")]
398 : #[serde(default)]
399 : pub checkpoint_distance: Option<u64>,
400 :
401 : #[serde(skip_serializing_if = "Option::is_none")]
402 : #[serde(with = "humantime_serde")]
403 : #[serde(default)]
404 : pub checkpoint_timeout: Option<Duration>,
405 :
406 : #[serde(skip_serializing_if = "Option::is_none")]
407 : #[serde(default)]
408 : pub compaction_target_size: Option<u64>,
409 :
410 : #[serde(skip_serializing_if = "Option::is_none")]
411 : #[serde(with = "humantime_serde")]
412 : #[serde(default)]
413 : pub compaction_period: Option<Duration>,
414 :
415 : #[serde(skip_serializing_if = "Option::is_none")]
416 : #[serde(default)]
417 : pub compaction_threshold: Option<usize>,
418 :
419 : #[serde(skip_serializing_if = "Option::is_none")]
420 : #[serde(default)]
421 : pub compaction_algorithm: Option<CompactionAlgorithmSettings>,
422 :
423 : #[serde(skip_serializing_if = "Option::is_none")]
424 : #[serde(default)]
425 : pub gc_horizon: Option<u64>,
426 :
427 : #[serde(skip_serializing_if = "Option::is_none")]
428 : #[serde(with = "humantime_serde")]
429 : #[serde(default)]
430 : pub gc_period: Option<Duration>,
431 :
432 : #[serde(skip_serializing_if = "Option::is_none")]
433 : #[serde(default)]
434 : pub image_creation_threshold: Option<usize>,
435 :
436 : #[serde(skip_serializing_if = "Option::is_none")]
437 : #[serde(with = "humantime_serde")]
438 : #[serde(default)]
439 : pub pitr_interval: Option<Duration>,
440 :
441 : #[serde(skip_serializing_if = "Option::is_none")]
442 : #[serde(with = "humantime_serde")]
443 : #[serde(default)]
444 : pub walreceiver_connect_timeout: Option<Duration>,
445 :
446 : #[serde(skip_serializing_if = "Option::is_none")]
447 : #[serde(with = "humantime_serde")]
448 : #[serde(default)]
449 : pub lagging_wal_timeout: Option<Duration>,
450 :
451 : #[serde(skip_serializing_if = "Option::is_none")]
452 : #[serde(default)]
453 : pub max_lsn_wal_lag: Option<NonZeroU64>,
454 :
455 : #[serde(skip_serializing_if = "Option::is_none")]
456 : #[serde(default)]
457 : pub trace_read_requests: Option<bool>,
458 :
459 : #[serde(skip_serializing_if = "Option::is_none")]
460 : #[serde(default)]
461 : pub eviction_policy: Option<EvictionPolicy>,
462 :
463 : #[serde(skip_serializing_if = "Option::is_none")]
464 : #[serde(default)]
465 : pub min_resident_size_override: Option<u64>,
466 :
467 : #[serde(skip_serializing_if = "Option::is_none")]
468 : #[serde(with = "humantime_serde")]
469 : #[serde(default)]
470 : pub evictions_low_residence_duration_metric_threshold: Option<Duration>,
471 :
472 : #[serde(skip_serializing_if = "Option::is_none")]
473 : #[serde(with = "humantime_serde")]
474 : #[serde(default)]
475 : pub heatmap_period: Option<Duration>,
476 :
477 : #[serde(skip_serializing_if = "Option::is_none")]
478 : #[serde(default)]
479 : pub lazy_slru_download: Option<bool>,
480 :
481 : #[serde(skip_serializing_if = "Option::is_none")]
482 : pub timeline_get_throttle: Option<pageserver_api::models::ThrottleConfig>,
483 :
484 : #[serde(skip_serializing_if = "Option::is_none")]
485 : pub image_layer_creation_check_threshold: Option<u8>,
486 :
487 : #[serde(skip_serializing_if = "Option::is_none")]
488 : #[serde(default)]
489 : pub switch_aux_file_policy: Option<AuxFilePolicy>,
490 :
491 : #[serde(skip_serializing_if = "Option::is_none")]
492 : #[serde(with = "humantime_serde")]
493 : #[serde(default)]
494 : pub lsn_lease_length: Option<Duration>,
495 :
496 : #[serde(skip_serializing_if = "Option::is_none")]
497 : #[serde(with = "humantime_serde")]
498 : #[serde(default)]
499 : pub lsn_lease_length_for_ts: Option<Duration>,
500 : }
501 :
502 : impl TenantConfOpt {
503 18 : pub fn merge(&self, global_conf: TenantConf) -> TenantConf {
504 18 : TenantConf {
505 18 : checkpoint_distance: self
506 18 : .checkpoint_distance
507 18 : .unwrap_or(global_conf.checkpoint_distance),
508 18 : checkpoint_timeout: self
509 18 : .checkpoint_timeout
510 18 : .unwrap_or(global_conf.checkpoint_timeout),
511 18 : compaction_target_size: self
512 18 : .compaction_target_size
513 18 : .unwrap_or(global_conf.compaction_target_size),
514 18 : compaction_period: self
515 18 : .compaction_period
516 18 : .unwrap_or(global_conf.compaction_period),
517 18 : compaction_threshold: self
518 18 : .compaction_threshold
519 18 : .unwrap_or(global_conf.compaction_threshold),
520 18 : compaction_algorithm: self
521 18 : .compaction_algorithm
522 18 : .as_ref()
523 18 : .unwrap_or(&global_conf.compaction_algorithm)
524 18 : .clone(),
525 18 : gc_horizon: self.gc_horizon.unwrap_or(global_conf.gc_horizon),
526 18 : gc_period: self.gc_period.unwrap_or(global_conf.gc_period),
527 18 : image_creation_threshold: self
528 18 : .image_creation_threshold
529 18 : .unwrap_or(global_conf.image_creation_threshold),
530 18 : pitr_interval: self.pitr_interval.unwrap_or(global_conf.pitr_interval),
531 18 : walreceiver_connect_timeout: self
532 18 : .walreceiver_connect_timeout
533 18 : .unwrap_or(global_conf.walreceiver_connect_timeout),
534 18 : lagging_wal_timeout: self
535 18 : .lagging_wal_timeout
536 18 : .unwrap_or(global_conf.lagging_wal_timeout),
537 18 : max_lsn_wal_lag: self.max_lsn_wal_lag.unwrap_or(global_conf.max_lsn_wal_lag),
538 18 : trace_read_requests: self
539 18 : .trace_read_requests
540 18 : .unwrap_or(global_conf.trace_read_requests),
541 18 : eviction_policy: self.eviction_policy.unwrap_or(global_conf.eviction_policy),
542 18 : min_resident_size_override: self
543 18 : .min_resident_size_override
544 18 : .or(global_conf.min_resident_size_override),
545 18 : evictions_low_residence_duration_metric_threshold: self
546 18 : .evictions_low_residence_duration_metric_threshold
547 18 : .unwrap_or(global_conf.evictions_low_residence_duration_metric_threshold),
548 18 : heatmap_period: self.heatmap_period.unwrap_or(global_conf.heatmap_period),
549 18 : lazy_slru_download: self
550 18 : .lazy_slru_download
551 18 : .unwrap_or(global_conf.lazy_slru_download),
552 18 : timeline_get_throttle: self
553 18 : .timeline_get_throttle
554 18 : .clone()
555 18 : .unwrap_or(global_conf.timeline_get_throttle),
556 18 : image_layer_creation_check_threshold: self
557 18 : .image_layer_creation_check_threshold
558 18 : .unwrap_or(global_conf.image_layer_creation_check_threshold),
559 18 : switch_aux_file_policy: self
560 18 : .switch_aux_file_policy
561 18 : .unwrap_or(global_conf.switch_aux_file_policy),
562 18 : lsn_lease_length: self
563 18 : .lsn_lease_length
564 18 : .unwrap_or(global_conf.lsn_lease_length),
565 18 : lsn_lease_length_for_ts: self
566 18 : .lsn_lease_length_for_ts
567 18 : .unwrap_or(global_conf.lsn_lease_length_for_ts),
568 18 : }
569 18 : }
570 : }
571 :
572 : impl Default for TenantConf {
573 356 : fn default() -> Self {
574 356 : use defaults::*;
575 356 : Self {
576 356 : checkpoint_distance: DEFAULT_CHECKPOINT_DISTANCE,
577 356 : checkpoint_timeout: humantime::parse_duration(DEFAULT_CHECKPOINT_TIMEOUT)
578 356 : .expect("cannot parse default checkpoint timeout"),
579 356 : compaction_target_size: DEFAULT_COMPACTION_TARGET_SIZE,
580 356 : compaction_period: humantime::parse_duration(DEFAULT_COMPACTION_PERIOD)
581 356 : .expect("cannot parse default compaction period"),
582 356 : compaction_threshold: DEFAULT_COMPACTION_THRESHOLD,
583 356 : compaction_algorithm: CompactionAlgorithmSettings {
584 356 : kind: DEFAULT_COMPACTION_ALGORITHM,
585 356 : },
586 356 : gc_horizon: DEFAULT_GC_HORIZON,
587 356 : gc_period: humantime::parse_duration(DEFAULT_GC_PERIOD)
588 356 : .expect("cannot parse default gc period"),
589 356 : image_creation_threshold: DEFAULT_IMAGE_CREATION_THRESHOLD,
590 356 : pitr_interval: humantime::parse_duration(DEFAULT_PITR_INTERVAL)
591 356 : .expect("cannot parse default PITR interval"),
592 356 : walreceiver_connect_timeout: humantime::parse_duration(
593 356 : DEFAULT_WALRECEIVER_CONNECT_TIMEOUT,
594 356 : )
595 356 : .expect("cannot parse default walreceiver connect timeout"),
596 356 : lagging_wal_timeout: humantime::parse_duration(DEFAULT_WALRECEIVER_LAGGING_WAL_TIMEOUT)
597 356 : .expect("cannot parse default walreceiver lagging wal timeout"),
598 356 : max_lsn_wal_lag: NonZeroU64::new(DEFAULT_MAX_WALRECEIVER_LSN_WAL_LAG)
599 356 : .expect("cannot parse default max walreceiver Lsn wal lag"),
600 356 : trace_read_requests: false,
601 356 : eviction_policy: EvictionPolicy::NoEviction,
602 356 : min_resident_size_override: None,
603 356 : evictions_low_residence_duration_metric_threshold: humantime::parse_duration(
604 356 : DEFAULT_EVICTIONS_LOW_RESIDENCE_DURATION_METRIC_THRESHOLD,
605 356 : )
606 356 : .expect("cannot parse default evictions_low_residence_duration_metric_threshold"),
607 356 : heatmap_period: Duration::ZERO,
608 356 : lazy_slru_download: false,
609 356 : timeline_get_throttle: crate::tenant::throttle::Config::disabled(),
610 356 : image_layer_creation_check_threshold: DEFAULT_IMAGE_LAYER_CREATION_CHECK_THRESHOLD,
611 356 : switch_aux_file_policy: AuxFilePolicy::default_tenant_config(),
612 356 : lsn_lease_length: LsnLease::DEFAULT_LENGTH,
613 356 : lsn_lease_length_for_ts: LsnLease::DEFAULT_LENGTH_FOR_TS,
614 356 : }
615 356 : }
616 : }
617 :
618 : impl TryFrom<&'_ models::TenantConfig> for TenantConfOpt {
619 : type Error = anyhow::Error;
620 :
621 4 : fn try_from(request_data: &'_ models::TenantConfig) -> Result<Self, Self::Error> {
622 : // Convert the request_data to a JSON Value
623 4 : let json_value: Value = serde_json::to_value(request_data)?;
624 :
625 : // Create a Deserializer from the JSON Value
626 4 : let deserializer = json_value.into_deserializer();
627 :
628 : // Use serde_path_to_error to deserialize the JSON Value into TenantConfOpt
629 4 : let tenant_conf: TenantConfOpt = serde_path_to_error::deserialize(deserializer)?;
630 :
631 2 : Ok(tenant_conf)
632 4 : }
633 : }
634 :
635 : impl TryFrom<toml_edit::Item> for TenantConfOpt {
636 : type Error = anyhow::Error;
637 :
638 10 : fn try_from(item: toml_edit::Item) -> Result<Self, Self::Error> {
639 10 : match item {
640 2 : toml_edit::Item::Value(value) => {
641 2 : let d = value.into_deserializer();
642 2 : return serde_path_to_error::deserialize(d)
643 2 : .map_err(|e| anyhow::anyhow!("{}: {}", e.path(), e.inner().message()));
644 : }
645 8 : toml_edit::Item::Table(table) => {
646 8 : let deserializer = toml_edit::de::Deserializer::new(table.into());
647 8 : return serde_path_to_error::deserialize(deserializer)
648 8 : .map_err(|e| anyhow::anyhow!("{}: {}", e.path(), e.inner().message()));
649 : }
650 : _ => {
651 0 : bail!("expected non-inline table but found {item}")
652 : }
653 : }
654 10 : }
655 : }
656 :
657 : /// This is a conversion from our internal tenant config object to the one used
658 : /// in external APIs.
659 : impl From<TenantConfOpt> for models::TenantConfig {
660 0 : fn from(value: TenantConfOpt) -> Self {
661 0 : fn humantime(d: Duration) -> String {
662 0 : format!("{}s", d.as_secs())
663 0 : }
664 0 : Self {
665 0 : checkpoint_distance: value.checkpoint_distance,
666 0 : checkpoint_timeout: value.checkpoint_timeout.map(humantime),
667 0 : compaction_algorithm: value.compaction_algorithm,
668 0 : compaction_target_size: value.compaction_target_size,
669 0 : compaction_period: value.compaction_period.map(humantime),
670 0 : compaction_threshold: value.compaction_threshold,
671 0 : gc_horizon: value.gc_horizon,
672 0 : gc_period: value.gc_period.map(humantime),
673 0 : image_creation_threshold: value.image_creation_threshold,
674 0 : pitr_interval: value.pitr_interval.map(humantime),
675 0 : walreceiver_connect_timeout: value.walreceiver_connect_timeout.map(humantime),
676 0 : lagging_wal_timeout: value.lagging_wal_timeout.map(humantime),
677 0 : max_lsn_wal_lag: value.max_lsn_wal_lag,
678 0 : trace_read_requests: value.trace_read_requests,
679 0 : eviction_policy: value.eviction_policy,
680 0 : min_resident_size_override: value.min_resident_size_override,
681 0 : evictions_low_residence_duration_metric_threshold: value
682 0 : .evictions_low_residence_duration_metric_threshold
683 0 : .map(humantime),
684 0 : heatmap_period: value.heatmap_period.map(humantime),
685 0 : lazy_slru_download: value.lazy_slru_download,
686 0 : timeline_get_throttle: value.timeline_get_throttle.map(ThrottleConfig::from),
687 0 : image_layer_creation_check_threshold: value.image_layer_creation_check_threshold,
688 0 : switch_aux_file_policy: value.switch_aux_file_policy,
689 0 : lsn_lease_length: value.lsn_lease_length.map(humantime),
690 0 : lsn_lease_length_for_ts: value.lsn_lease_length_for_ts.map(humantime),
691 0 : }
692 0 : }
693 : }
694 :
695 : #[cfg(test)]
696 : mod tests {
697 : use super::*;
698 : use models::TenantConfig;
699 :
700 : #[test]
701 2 : fn de_serializing_pageserver_config_omits_empty_values() {
702 2 : let small_conf = TenantConfOpt {
703 2 : gc_horizon: Some(42),
704 2 : ..TenantConfOpt::default()
705 2 : };
706 2 :
707 2 : let toml_form = toml_edit::ser::to_string(&small_conf).unwrap();
708 2 : assert_eq!(toml_form, "gc_horizon = 42\n");
709 2 : assert_eq!(small_conf, toml_edit::de::from_str(&toml_form).unwrap());
710 :
711 2 : let json_form = serde_json::to_string(&small_conf).unwrap();
712 2 : assert_eq!(json_form, "{\"gc_horizon\":42}");
713 2 : assert_eq!(small_conf, serde_json::from_str(&json_form).unwrap());
714 2 : }
715 :
716 : #[test]
717 2 : fn test_try_from_models_tenant_config_err() {
718 2 : let tenant_config = models::TenantConfig {
719 2 : lagging_wal_timeout: Some("5a".to_string()),
720 2 : ..TenantConfig::default()
721 2 : };
722 2 :
723 2 : let tenant_conf_opt = TenantConfOpt::try_from(&tenant_config);
724 2 :
725 2 : assert!(
726 2 : tenant_conf_opt.is_err(),
727 0 : "Suceeded to convert TenantConfig to TenantConfOpt"
728 : );
729 :
730 2 : let expected_error_str =
731 2 : "lagging_wal_timeout: invalid value: string \"5a\", expected a duration";
732 2 : assert_eq!(tenant_conf_opt.unwrap_err().to_string(), expected_error_str);
733 2 : }
734 :
735 : #[test]
736 2 : fn test_try_from_models_tenant_config_success() {
737 2 : let tenant_config = models::TenantConfig {
738 2 : lagging_wal_timeout: Some("5s".to_string()),
739 2 : ..TenantConfig::default()
740 2 : };
741 2 :
742 2 : let tenant_conf_opt = TenantConfOpt::try_from(&tenant_config).unwrap();
743 2 :
744 2 : assert_eq!(
745 2 : tenant_conf_opt.lagging_wal_timeout,
746 2 : Some(Duration::from_secs(5))
747 2 : );
748 2 : }
749 : }
|
