Line data Source code
1 : //! Functions for handling per-tenant configuration options
2 : //!
3 : //! If tenant is created with --config option,
4 : //! the tenant-specific config will be stored in tenant's directory.
5 : //! Otherwise, global pageserver's config is used.
6 : //!
7 : //! If the tenant config file is corrupted, the tenant will be disabled.
8 : //! We cannot use global or default config instead, because wrong settings
9 : //! may lead to a data loss.
10 : //!
11 : use anyhow::bail;
12 : use pageserver_api::models::CompactionAlgorithm;
13 : use pageserver_api::models::EvictionPolicy;
14 : use pageserver_api::models::{self, ThrottleConfig};
15 : use pageserver_api::shard::{ShardCount, ShardIdentity, ShardNumber, ShardStripeSize};
16 : use serde::de::IntoDeserializer;
17 : use serde::{Deserialize, Serialize};
18 : use serde_json::Value;
19 : use std::num::NonZeroU64;
20 : use std::time::Duration;
21 : use utils::generation::Generation;
22 :
23 : pub mod defaults {
24 :
25 : // FIXME: This current value is very low. I would imagine something like 1 GB or 10 GB
26 : // would be more appropriate. But a low value forces the code to be exercised more,
27 : // which is good for now to trigger bugs.
28 : // This parameter actually determines L0 layer file size.
29 : pub const DEFAULT_CHECKPOINT_DISTANCE: u64 = 256 * 1024 * 1024;
30 : pub const DEFAULT_CHECKPOINT_TIMEOUT: &str = "10 m";
31 :
32 : // FIXME the below configs are only used by legacy algorithm. The new algorithm
33 : // has different parameters.
34 :
35 : // Target file size, when creating image and delta layers.
36 : // This parameter determines L1 layer file size.
37 : pub const DEFAULT_COMPACTION_TARGET_SIZE: u64 = 128 * 1024 * 1024;
38 :
39 : pub const DEFAULT_COMPACTION_PERIOD: &str = "20 s";
40 : pub const DEFAULT_COMPACTION_THRESHOLD: usize = 10;
41 : pub const DEFAULT_COMPACTION_ALGORITHM: super::CompactionAlgorithm =
42 : super::CompactionAlgorithm::Legacy;
43 :
44 : pub const DEFAULT_GC_HORIZON: u64 = 64 * 1024 * 1024;
45 :
46 : // Large DEFAULT_GC_PERIOD is fine as long as PITR_INTERVAL is larger.
47 : // If there's a need to decrease this value, first make sure that GC
48 : // doesn't hold a layer map write lock for non-trivial operations.
49 : // Relevant: https://github.com/neondatabase/neon/issues/3394
50 : pub const DEFAULT_GC_PERIOD: &str = "1 hr";
51 : pub const DEFAULT_IMAGE_CREATION_THRESHOLD: usize = 3;
52 : pub const DEFAULT_PITR_INTERVAL: &str = "7 days";
53 : pub const DEFAULT_WALRECEIVER_CONNECT_TIMEOUT: &str = "10 seconds";
54 : pub const DEFAULT_WALRECEIVER_LAGGING_WAL_TIMEOUT: &str = "10 seconds";
55 : pub const DEFAULT_MAX_WALRECEIVER_LSN_WAL_LAG: u64 = 10 * 1024 * 1024;
56 : pub const DEFAULT_EVICTIONS_LOW_RESIDENCE_DURATION_METRIC_THRESHOLD: &str = "24 hour";
57 :
58 : pub const DEFAULT_INGEST_BATCH_SIZE: u64 = 100;
59 : }
60 :
61 0 : #[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)]
62 : pub(crate) enum AttachmentMode {
63 : /// Our generation is current as far as we know, and as far as we know we are the only attached
64 : /// pageserver. This is the "normal" attachment mode.
65 : Single,
66 : /// Our generation number is current as far as we know, but we are advised that another
67 : /// pageserver is still attached, and therefore to avoid executing deletions. This is
68 : /// the attachment mode of a pagesever that is the destination of a migration.
69 : Multi,
70 : /// Our generation number is superseded, or about to be superseded. We are advised
71 : /// to avoid remote storage writes if possible, and to avoid sending billing data. This
72 : /// is the attachment mode of a pageserver that is the origin of a migration.
73 : Stale,
74 : }
75 :
76 0 : #[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)]
77 : pub(crate) struct AttachedLocationConfig {
78 : pub(crate) generation: Generation,
79 : pub(crate) attach_mode: AttachmentMode,
80 : // TODO: add a flag to override AttachmentMode's policies under
81 : // disk pressure (i.e. unblock uploads under disk pressure in Stale
82 : // state, unblock deletions after timeout in Multi state)
83 : }
84 :
85 0 : #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
86 : pub(crate) struct SecondaryLocationConfig {
87 : /// If true, keep the local cache warm by polling remote storage
88 : pub(crate) warm: bool,
89 : }
90 :
91 0 : #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
92 : pub(crate) enum LocationMode {
93 : Attached(AttachedLocationConfig),
94 : Secondary(SecondaryLocationConfig),
95 : }
96 :
97 : /// Per-tenant, per-pageserver configuration. All pageservers use the same TenantConf,
98 : /// but have distinct LocationConf.
99 0 : #[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
100 : pub(crate) struct LocationConf {
101 : /// The location-specific part of the configuration, describes the operating
102 : /// mode of this pageserver for this tenant.
103 : pub(crate) mode: LocationMode,
104 :
105 : /// The detailed shard identity. This structure is already scoped within
106 : /// a TenantShardId, but we need the full ShardIdentity to enable calculating
107 : /// key->shard mappings.
108 : #[serde(default = "ShardIdentity::unsharded")]
109 : #[serde(skip_serializing_if = "ShardIdentity::is_unsharded")]
110 : pub(crate) shard: ShardIdentity,
111 :
112 : /// The pan-cluster tenant configuration, the same on all locations
113 : pub(crate) tenant_conf: TenantConfOpt,
114 : }
115 :
116 : impl std::fmt::Debug for LocationConf {
117 0 : fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
118 0 : match &self.mode {
119 0 : LocationMode::Attached(conf) => {
120 0 : write!(
121 0 : f,
122 0 : "Attached {:?}, gen={:?}",
123 0 : conf.attach_mode, conf.generation
124 0 : )
125 : }
126 0 : LocationMode::Secondary(conf) => {
127 0 : write!(f, "Secondary, warm={}", conf.warm)
128 : }
129 : }
130 0 : }
131 : }
132 :
133 : impl AttachedLocationConfig {
134 : /// Consult attachment mode to determine whether we are currently permitted
135 : /// to delete layers. This is only advisory, not required for data safety.
136 : /// See [`AttachmentMode`] for more context.
137 8 : pub(crate) fn may_delete_layers_hint(&self) -> bool {
138 8 : // TODO: add an override for disk pressure in AttachedLocationConfig,
139 8 : // and respect it here.
140 8 : match &self.attach_mode {
141 8 : AttachmentMode::Single => true,
142 : AttachmentMode::Multi | AttachmentMode::Stale => {
143 : // In Multi mode we avoid doing deletions because some other
144 : // attached pageserver might get 404 while trying to read
145 : // a layer we delete which is still referenced in their metadata.
146 : //
147 : // In Stale mode, we avoid doing deletions because we expect
148 : // that they would ultimately fail validation in the deletion
149 : // queue due to our stale generation.
150 0 : false
151 : }
152 : }
153 8 : }
154 :
155 : /// Whether we are currently hinted that it is worthwhile to upload layers.
156 : /// This is only advisory, not required for data safety.
157 : /// See [`AttachmentMode`] for more context.
158 0 : pub(crate) fn may_upload_layers_hint(&self) -> bool {
159 0 : // TODO: add an override for disk pressure in AttachedLocationConfig,
160 0 : // and respect it here.
161 0 : match &self.attach_mode {
162 0 : AttachmentMode::Single | AttachmentMode::Multi => true,
163 : AttachmentMode::Stale => {
164 : // In Stale mode, we avoid doing uploads because we expect that
165 : // our replacement pageserver will already have started its own
166 : // IndexPart that will never reference layers we upload: it is
167 : // wasteful.
168 0 : false
169 : }
170 : }
171 0 : }
172 : }
173 :
174 : impl LocationConf {
175 : /// For use when loading from a legacy configuration: presence of a tenant
176 : /// implies it is in AttachmentMode::Single, which used to be the only
177 : /// possible state. This function should eventually be removed.
178 88 : pub(crate) fn attached_single(
179 88 : tenant_conf: TenantConfOpt,
180 88 : generation: Generation,
181 88 : shard_params: &models::ShardParameters,
182 88 : ) -> Self {
183 88 : Self {
184 88 : mode: LocationMode::Attached(AttachedLocationConfig {
185 88 : generation,
186 88 : attach_mode: AttachmentMode::Single,
187 88 : }),
188 88 : shard: ShardIdentity::from_params(ShardNumber(0), shard_params),
189 88 : tenant_conf,
190 88 : }
191 88 : }
192 :
193 : /// For use when attaching/re-attaching: update the generation stored in this
194 : /// structure. If we were in a secondary state, promote to attached (posession
195 : /// of a fresh generation implies this).
196 0 : pub(crate) fn attach_in_generation(&mut self, generation: Generation) {
197 0 : match &mut self.mode {
198 0 : LocationMode::Attached(attach_conf) => {
199 0 : attach_conf.generation = generation;
200 0 : }
201 : LocationMode::Secondary(_) => {
202 : // We are promoted to attached by the control plane's re-attach response
203 0 : self.mode = LocationMode::Attached(AttachedLocationConfig {
204 0 : generation,
205 0 : attach_mode: AttachmentMode::Single,
206 0 : })
207 : }
208 : }
209 0 : }
210 :
211 0 : pub(crate) fn try_from(conf: &'_ models::LocationConfig) -> anyhow::Result<Self> {
212 0 : let tenant_conf = TenantConfOpt::try_from(&conf.tenant_conf)?;
213 :
214 0 : fn get_generation(conf: &'_ models::LocationConfig) -> Result<Generation, anyhow::Error> {
215 0 : conf.generation
216 0 : .map(Generation::new)
217 0 : .ok_or_else(|| anyhow::anyhow!("Generation must be set when attaching"))
218 0 : }
219 :
220 0 : let mode = match &conf.mode {
221 : models::LocationConfigMode::AttachedMulti => {
222 : LocationMode::Attached(AttachedLocationConfig {
223 0 : generation: get_generation(conf)?,
224 0 : attach_mode: AttachmentMode::Multi,
225 : })
226 : }
227 : models::LocationConfigMode::AttachedSingle => {
228 : LocationMode::Attached(AttachedLocationConfig {
229 0 : generation: get_generation(conf)?,
230 0 : attach_mode: AttachmentMode::Single,
231 : })
232 : }
233 : models::LocationConfigMode::AttachedStale => {
234 : LocationMode::Attached(AttachedLocationConfig {
235 0 : generation: get_generation(conf)?,
236 0 : attach_mode: AttachmentMode::Stale,
237 : })
238 : }
239 : models::LocationConfigMode::Secondary => {
240 0 : anyhow::ensure!(conf.generation.is_none());
241 :
242 0 : let warm = conf
243 0 : .secondary_conf
244 0 : .as_ref()
245 0 : .map(|c| c.warm)
246 0 : .unwrap_or(false);
247 0 : LocationMode::Secondary(SecondaryLocationConfig { warm })
248 : }
249 : models::LocationConfigMode::Detached => {
250 : // Should not have been called: API code should translate this mode
251 : // into a detach rather than trying to decode it as a LocationConf
252 0 : return Err(anyhow::anyhow!("Cannot decode a Detached configuration"));
253 : }
254 : };
255 :
256 0 : let shard = if conf.shard_count == 0 {
257 0 : ShardIdentity::unsharded()
258 : } else {
259 0 : ShardIdentity::new(
260 0 : ShardNumber(conf.shard_number),
261 0 : ShardCount::new(conf.shard_count),
262 0 : ShardStripeSize(conf.shard_stripe_size),
263 0 : )?
264 : };
265 :
266 0 : Ok(Self {
267 0 : shard,
268 0 : mode,
269 0 : tenant_conf,
270 0 : })
271 0 : }
272 : }
273 :
274 : impl Default for LocationConf {
275 : // TODO: this should be removed once tenant loading can guarantee that we are never
276 : // loading from a directory without a configuration.
277 : // => tech debt since https://github.com/neondatabase/neon/issues/1555
278 0 : fn default() -> Self {
279 0 : Self {
280 0 : mode: LocationMode::Attached(AttachedLocationConfig {
281 0 : generation: Generation::none(),
282 0 : attach_mode: AttachmentMode::Single,
283 0 : }),
284 0 : tenant_conf: TenantConfOpt::default(),
285 0 : shard: ShardIdentity::unsharded(),
286 0 : }
287 0 : }
288 : }
289 :
290 : /// A tenant's calcuated configuration, which is the result of merging a
291 : /// tenant's TenantConfOpt with the global TenantConf from PageServerConf.
292 : ///
293 : /// For storing and transmitting individual tenant's configuration, see
294 : /// TenantConfOpt.
295 88 : #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
296 : pub struct TenantConf {
297 : // Flush out an inmemory layer, if it's holding WAL older than this
298 : // This puts a backstop on how much WAL needs to be re-digested if the
299 : // page server crashes.
300 : // This parameter actually determines L0 layer file size.
301 : pub checkpoint_distance: u64,
302 : // Inmemory layer is also flushed at least once in checkpoint_timeout to
303 : // eventually upload WAL after activity is stopped.
304 : #[serde(with = "humantime_serde")]
305 : pub checkpoint_timeout: Duration,
306 : // Target file size, when creating image and delta layers.
307 : // This parameter determines L1 layer file size.
308 : pub compaction_target_size: u64,
309 : // How often to check if there's compaction work to be done.
310 : // Duration::ZERO means automatic compaction is disabled.
311 : #[serde(with = "humantime_serde")]
312 : pub compaction_period: Duration,
313 : // Level0 delta layer threshold for compaction.
314 : pub compaction_threshold: usize,
315 : pub compaction_algorithm: CompactionAlgorithm,
316 : // Determines how much history is retained, to allow
317 : // branching and read replicas at an older point in time.
318 : // The unit is #of bytes of WAL.
319 : // Page versions older than this are garbage collected away.
320 : pub gc_horizon: u64,
321 : // Interval at which garbage collection is triggered.
322 : // Duration::ZERO means automatic GC is disabled
323 : #[serde(with = "humantime_serde")]
324 : pub gc_period: Duration,
325 : // Delta layer churn threshold to create L1 image layers.
326 : pub image_creation_threshold: usize,
327 : // Determines how much history is retained, to allow
328 : // branching and read replicas at an older point in time.
329 : // The unit is time.
330 : // Page versions older than this are garbage collected away.
331 : #[serde(with = "humantime_serde")]
332 : pub pitr_interval: Duration,
333 : /// Maximum amount of time to wait while opening a connection to receive wal, before erroring.
334 : #[serde(with = "humantime_serde")]
335 : pub walreceiver_connect_timeout: Duration,
336 : /// Considers safekeepers stalled after no WAL updates were received longer than this threshold.
337 : /// A stalled safekeeper will be changed to a newer one when it appears.
338 : #[serde(with = "humantime_serde")]
339 : pub lagging_wal_timeout: Duration,
340 : /// Considers safekeepers lagging when their WAL is behind another safekeeper for more than this threshold.
341 : /// A lagging safekeeper will be changed after `lagging_wal_timeout` time elapses since the last WAL update,
342 : /// to avoid eager reconnects.
343 : pub max_lsn_wal_lag: NonZeroU64,
344 : pub trace_read_requests: bool,
345 : pub eviction_policy: EvictionPolicy,
346 : pub min_resident_size_override: Option<u64>,
347 : // See the corresponding metric's help string.
348 : #[serde(with = "humantime_serde")]
349 : pub evictions_low_residence_duration_metric_threshold: Duration,
350 :
351 : /// If non-zero, the period between uploads of a heatmap from attached tenants. This
352 : /// may be disabled if a Tenant will not have secondary locations: only secondary
353 : /// locations will use the heatmap uploaded by attached locations.
354 : pub heatmap_period: Duration,
355 :
356 : /// If true then SLRU segments are dowloaded on demand, if false SLRU segments are included in basebackup
357 : pub lazy_slru_download: bool,
358 :
359 : pub timeline_get_throttle: pageserver_api::models::ThrottleConfig,
360 : }
361 :
362 : /// Same as TenantConf, but this struct preserves the information about
363 : /// which parameters are set and which are not.
364 2002 : #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
365 : pub struct TenantConfOpt {
366 : #[serde(skip_serializing_if = "Option::is_none")]
367 : #[serde(default)]
368 : pub checkpoint_distance: Option<u64>,
369 :
370 : #[serde(skip_serializing_if = "Option::is_none")]
371 : #[serde(with = "humantime_serde")]
372 : #[serde(default)]
373 : pub checkpoint_timeout: Option<Duration>,
374 :
375 : #[serde(skip_serializing_if = "Option::is_none")]
376 : #[serde(default)]
377 : pub compaction_target_size: Option<u64>,
378 :
379 : #[serde(skip_serializing_if = "Option::is_none")]
380 : #[serde(with = "humantime_serde")]
381 : #[serde(default)]
382 : pub compaction_period: Option<Duration>,
383 :
384 : #[serde(skip_serializing_if = "Option::is_none")]
385 : #[serde(default)]
386 : pub compaction_threshold: Option<usize>,
387 :
388 : #[serde(skip_serializing_if = "Option::is_none")]
389 : #[serde(default)]
390 : pub compaction_algorithm: Option<CompactionAlgorithm>,
391 :
392 : #[serde(skip_serializing_if = "Option::is_none")]
393 : #[serde(default)]
394 : pub gc_horizon: Option<u64>,
395 :
396 : #[serde(skip_serializing_if = "Option::is_none")]
397 : #[serde(with = "humantime_serde")]
398 : #[serde(default)]
399 : pub gc_period: Option<Duration>,
400 :
401 : #[serde(skip_serializing_if = "Option::is_none")]
402 : #[serde(default)]
403 : pub image_creation_threshold: Option<usize>,
404 :
405 : #[serde(skip_serializing_if = "Option::is_none")]
406 : #[serde(with = "humantime_serde")]
407 : #[serde(default)]
408 : pub pitr_interval: Option<Duration>,
409 :
410 : #[serde(skip_serializing_if = "Option::is_none")]
411 : #[serde(with = "humantime_serde")]
412 : #[serde(default)]
413 : pub walreceiver_connect_timeout: Option<Duration>,
414 :
415 : #[serde(skip_serializing_if = "Option::is_none")]
416 : #[serde(with = "humantime_serde")]
417 : #[serde(default)]
418 : pub lagging_wal_timeout: Option<Duration>,
419 :
420 : #[serde(skip_serializing_if = "Option::is_none")]
421 : #[serde(default)]
422 : pub max_lsn_wal_lag: Option<NonZeroU64>,
423 :
424 : #[serde(skip_serializing_if = "Option::is_none")]
425 : #[serde(default)]
426 : pub trace_read_requests: Option<bool>,
427 :
428 : #[serde(skip_serializing_if = "Option::is_none")]
429 : #[serde(default)]
430 : pub eviction_policy: Option<EvictionPolicy>,
431 :
432 : #[serde(skip_serializing_if = "Option::is_none")]
433 : #[serde(default)]
434 : pub min_resident_size_override: Option<u64>,
435 :
436 : #[serde(skip_serializing_if = "Option::is_none")]
437 : #[serde(with = "humantime_serde")]
438 : #[serde(default)]
439 : pub evictions_low_residence_duration_metric_threshold: Option<Duration>,
440 :
441 : #[serde(skip_serializing_if = "Option::is_none")]
442 : #[serde(with = "humantime_serde")]
443 : #[serde(default)]
444 : pub heatmap_period: Option<Duration>,
445 :
446 : #[serde(skip_serializing_if = "Option::is_none")]
447 : #[serde(default)]
448 : pub lazy_slru_download: Option<bool>,
449 :
450 : #[serde(skip_serializing_if = "Option::is_none")]
451 : pub timeline_get_throttle: Option<pageserver_api::models::ThrottleConfig>,
452 : }
453 :
454 : impl TenantConfOpt {
455 18 : pub fn merge(&self, global_conf: TenantConf) -> TenantConf {
456 18 : TenantConf {
457 18 : checkpoint_distance: self
458 18 : .checkpoint_distance
459 18 : .unwrap_or(global_conf.checkpoint_distance),
460 18 : checkpoint_timeout: self
461 18 : .checkpoint_timeout
462 18 : .unwrap_or(global_conf.checkpoint_timeout),
463 18 : compaction_target_size: self
464 18 : .compaction_target_size
465 18 : .unwrap_or(global_conf.compaction_target_size),
466 18 : compaction_period: self
467 18 : .compaction_period
468 18 : .unwrap_or(global_conf.compaction_period),
469 18 : compaction_threshold: self
470 18 : .compaction_threshold
471 18 : .unwrap_or(global_conf.compaction_threshold),
472 18 : compaction_algorithm: self
473 18 : .compaction_algorithm
474 18 : .unwrap_or(global_conf.compaction_algorithm),
475 18 : gc_horizon: self.gc_horizon.unwrap_or(global_conf.gc_horizon),
476 18 : gc_period: self.gc_period.unwrap_or(global_conf.gc_period),
477 18 : image_creation_threshold: self
478 18 : .image_creation_threshold
479 18 : .unwrap_or(global_conf.image_creation_threshold),
480 18 : pitr_interval: self.pitr_interval.unwrap_or(global_conf.pitr_interval),
481 18 : walreceiver_connect_timeout: self
482 18 : .walreceiver_connect_timeout
483 18 : .unwrap_or(global_conf.walreceiver_connect_timeout),
484 18 : lagging_wal_timeout: self
485 18 : .lagging_wal_timeout
486 18 : .unwrap_or(global_conf.lagging_wal_timeout),
487 18 : max_lsn_wal_lag: self.max_lsn_wal_lag.unwrap_or(global_conf.max_lsn_wal_lag),
488 18 : trace_read_requests: self
489 18 : .trace_read_requests
490 18 : .unwrap_or(global_conf.trace_read_requests),
491 18 : eviction_policy: self.eviction_policy.unwrap_or(global_conf.eviction_policy),
492 18 : min_resident_size_override: self
493 18 : .min_resident_size_override
494 18 : .or(global_conf.min_resident_size_override),
495 18 : evictions_low_residence_duration_metric_threshold: self
496 18 : .evictions_low_residence_duration_metric_threshold
497 18 : .unwrap_or(global_conf.evictions_low_residence_duration_metric_threshold),
498 18 : heatmap_period: self.heatmap_period.unwrap_or(global_conf.heatmap_period),
499 18 : lazy_slru_download: self
500 18 : .lazy_slru_download
501 18 : .unwrap_or(global_conf.lazy_slru_download),
502 18 : timeline_get_throttle: self
503 18 : .timeline_get_throttle
504 18 : .clone()
505 18 : .unwrap_or(global_conf.timeline_get_throttle),
506 18 : }
507 18 : }
508 : }
509 :
510 : impl Default for TenantConf {
511 226 : fn default() -> Self {
512 226 : use defaults::*;
513 226 : Self {
514 226 : checkpoint_distance: DEFAULT_CHECKPOINT_DISTANCE,
515 226 : checkpoint_timeout: humantime::parse_duration(DEFAULT_CHECKPOINT_TIMEOUT)
516 226 : .expect("cannot parse default checkpoint timeout"),
517 226 : compaction_target_size: DEFAULT_COMPACTION_TARGET_SIZE,
518 226 : compaction_period: humantime::parse_duration(DEFAULT_COMPACTION_PERIOD)
519 226 : .expect("cannot parse default compaction period"),
520 226 : compaction_threshold: DEFAULT_COMPACTION_THRESHOLD,
521 226 : compaction_algorithm: DEFAULT_COMPACTION_ALGORITHM,
522 226 : gc_horizon: DEFAULT_GC_HORIZON,
523 226 : gc_period: humantime::parse_duration(DEFAULT_GC_PERIOD)
524 226 : .expect("cannot parse default gc period"),
525 226 : image_creation_threshold: DEFAULT_IMAGE_CREATION_THRESHOLD,
526 226 : pitr_interval: humantime::parse_duration(DEFAULT_PITR_INTERVAL)
527 226 : .expect("cannot parse default PITR interval"),
528 226 : walreceiver_connect_timeout: humantime::parse_duration(
529 226 : DEFAULT_WALRECEIVER_CONNECT_TIMEOUT,
530 226 : )
531 226 : .expect("cannot parse default walreceiver connect timeout"),
532 226 : lagging_wal_timeout: humantime::parse_duration(DEFAULT_WALRECEIVER_LAGGING_WAL_TIMEOUT)
533 226 : .expect("cannot parse default walreceiver lagging wal timeout"),
534 226 : max_lsn_wal_lag: NonZeroU64::new(DEFAULT_MAX_WALRECEIVER_LSN_WAL_LAG)
535 226 : .expect("cannot parse default max walreceiver Lsn wal lag"),
536 226 : trace_read_requests: false,
537 226 : eviction_policy: EvictionPolicy::NoEviction,
538 226 : min_resident_size_override: None,
539 226 : evictions_low_residence_duration_metric_threshold: humantime::parse_duration(
540 226 : DEFAULT_EVICTIONS_LOW_RESIDENCE_DURATION_METRIC_THRESHOLD,
541 226 : )
542 226 : .expect("cannot parse default evictions_low_residence_duration_metric_threshold"),
543 226 : heatmap_period: Duration::ZERO,
544 226 : lazy_slru_download: false,
545 226 : timeline_get_throttle: crate::tenant::throttle::Config::disabled(),
546 226 : }
547 226 : }
548 : }
549 :
550 : impl TryFrom<&'_ models::TenantConfig> for TenantConfOpt {
551 : type Error = anyhow::Error;
552 :
553 4 : fn try_from(request_data: &'_ models::TenantConfig) -> Result<Self, Self::Error> {
554 : // Convert the request_data to a JSON Value
555 4 : let json_value: Value = serde_json::to_value(request_data)?;
556 :
557 : // Create a Deserializer from the JSON Value
558 4 : let deserializer = json_value.into_deserializer();
559 :
560 : // Use serde_path_to_error to deserialize the JSON Value into TenantConfOpt
561 4 : let tenant_conf: TenantConfOpt = serde_path_to_error::deserialize(deserializer)?;
562 :
563 2 : Ok(tenant_conf)
564 4 : }
565 : }
566 :
567 : impl TryFrom<toml_edit::Item> for TenantConfOpt {
568 : type Error = anyhow::Error;
569 :
570 10 : fn try_from(item: toml_edit::Item) -> Result<Self, Self::Error> {
571 10 : match item {
572 2 : toml_edit::Item::Value(value) => {
573 2 : let d = value.into_deserializer();
574 2 : return serde_path_to_error::deserialize(d)
575 2 : .map_err(|e| anyhow::anyhow!("{}: {}", e.path(), e.inner().message()));
576 : }
577 8 : toml_edit::Item::Table(table) => {
578 8 : let deserializer = toml_edit::de::Deserializer::new(table.into());
579 8 : return serde_path_to_error::deserialize(deserializer)
580 8 : .map_err(|e| anyhow::anyhow!("{}: {}", e.path(), e.inner().message()));
581 : }
582 : _ => {
583 0 : bail!("expected non-inline table but found {item}")
584 : }
585 : }
586 10 : }
587 : }
588 :
589 : /// This is a conversion from our internal tenant config object to the one used
590 : /// in external APIs.
591 : impl From<TenantConfOpt> for models::TenantConfig {
592 0 : fn from(value: TenantConfOpt) -> Self {
593 0 : fn humantime(d: Duration) -> String {
594 0 : format!("{}s", d.as_secs())
595 0 : }
596 0 : Self {
597 0 : checkpoint_distance: value.checkpoint_distance,
598 0 : checkpoint_timeout: value.checkpoint_timeout.map(humantime),
599 0 : compaction_algorithm: value.compaction_algorithm,
600 0 : compaction_target_size: value.compaction_target_size,
601 0 : compaction_period: value.compaction_period.map(humantime),
602 0 : compaction_threshold: value.compaction_threshold,
603 0 : gc_horizon: value.gc_horizon,
604 0 : gc_period: value.gc_period.map(humantime),
605 0 : image_creation_threshold: value.image_creation_threshold,
606 0 : pitr_interval: value.pitr_interval.map(humantime),
607 0 : walreceiver_connect_timeout: value.walreceiver_connect_timeout.map(humantime),
608 0 : lagging_wal_timeout: value.lagging_wal_timeout.map(humantime),
609 0 : max_lsn_wal_lag: value.max_lsn_wal_lag,
610 0 : trace_read_requests: value.trace_read_requests,
611 0 : eviction_policy: value.eviction_policy,
612 0 : min_resident_size_override: value.min_resident_size_override,
613 0 : evictions_low_residence_duration_metric_threshold: value
614 0 : .evictions_low_residence_duration_metric_threshold
615 0 : .map(humantime),
616 0 : heatmap_period: value.heatmap_period.map(humantime),
617 0 : lazy_slru_download: value.lazy_slru_download,
618 0 : timeline_get_throttle: value.timeline_get_throttle.map(ThrottleConfig::from),
619 0 : }
620 0 : }
621 : }
622 :
623 : #[cfg(test)]
624 : mod tests {
625 : use super::*;
626 : use models::TenantConfig;
627 :
628 2 : #[test]
629 2 : fn de_serializing_pageserver_config_omits_empty_values() {
630 2 : let small_conf = TenantConfOpt {
631 2 : gc_horizon: Some(42),
632 2 : ..TenantConfOpt::default()
633 2 : };
634 2 :
635 2 : let toml_form = toml_edit::ser::to_string(&small_conf).unwrap();
636 2 : assert_eq!(toml_form, "gc_horizon = 42\n");
637 2 : assert_eq!(small_conf, toml_edit::de::from_str(&toml_form).unwrap());
638 :
639 2 : let json_form = serde_json::to_string(&small_conf).unwrap();
640 2 : assert_eq!(json_form, "{\"gc_horizon\":42}");
641 2 : assert_eq!(small_conf, serde_json::from_str(&json_form).unwrap());
642 2 : }
643 :
644 2 : #[test]
645 2 : fn test_try_from_models_tenant_config_err() {
646 2 : let tenant_config = models::TenantConfig {
647 2 : lagging_wal_timeout: Some("5a".to_string()),
648 2 : ..TenantConfig::default()
649 2 : };
650 2 :
651 2 : let tenant_conf_opt = TenantConfOpt::try_from(&tenant_config);
652 2 :
653 2 : assert!(
654 2 : tenant_conf_opt.is_err(),
655 0 : "Suceeded to convert TenantConfig to TenantConfOpt"
656 : );
657 :
658 2 : let expected_error_str =
659 2 : "lagging_wal_timeout: invalid value: string \"5a\", expected a duration";
660 2 : assert_eq!(tenant_conf_opt.unwrap_err().to_string(), expected_error_str);
661 2 : }
662 :
663 2 : #[test]
664 2 : fn test_try_from_models_tenant_config_success() {
665 2 : let tenant_config = models::TenantConfig {
666 2 : lagging_wal_timeout: Some("5s".to_string()),
667 2 : ..TenantConfig::default()
668 2 : };
669 2 :
670 2 : let tenant_conf_opt = TenantConfOpt::try_from(&tenant_config).unwrap();
671 2 :
672 2 : assert_eq!(
673 2 : tenant_conf_opt.lagging_wal_timeout,
674 2 : Some(Duration::from_secs(5))
675 2 : );
676 2 : }
677 : }
|
