Line data Source code
1 : //! Functions for handling per-tenant configuration options
2 : //!
3 : //! If tenant is created with --config option,
4 : //! the tenant-specific config will be stored in tenant's directory.
5 : //! Otherwise, global pageserver's config is used.
6 : //!
7 : //! If the tenant config file is corrupted, the tenant will be disabled.
8 : //! We cannot use global or default config instead, because wrong settings
9 : //! may lead to a data loss.
10 : //!
11 : use std::num::NonZeroU64;
12 : use std::time::Duration;
13 :
14 : pub(crate) use pageserver_api::config::TenantConfigToml as TenantConf;
15 : use pageserver_api::models::{
16 : self, CompactionAlgorithmSettings, EvictionPolicy, TenantConfigPatch,
17 : };
18 : use pageserver_api::shard::{ShardCount, ShardIdentity, ShardNumber, ShardStripeSize};
19 : use serde::de::IntoDeserializer;
20 : use serde::{Deserialize, Serialize};
21 : use serde_json::Value;
22 : use utils::generation::Generation;
23 : use utils::postgres_client::PostgresClientProtocol;
24 :
25 0 : #[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)]
26 : pub(crate) enum AttachmentMode {
27 : /// Our generation is current as far as we know, and as far as we know we are the only attached
28 : /// pageserver. This is the "normal" attachment mode.
29 : Single,
30 : /// Our generation number is current as far as we know, but we are advised that another
31 : /// pageserver is still attached, and therefore to avoid executing deletions. This is
32 : /// the attachment mode of a pagesever that is the destination of a migration.
33 : Multi,
34 : /// Our generation number is superseded, or about to be superseded. We are advised
35 : /// to avoid remote storage writes if possible, and to avoid sending billing data. This
36 : /// is the attachment mode of a pageserver that is the origin of a migration.
37 : Stale,
38 : }
39 :
40 0 : #[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)]
41 : pub(crate) struct AttachedLocationConfig {
42 : pub(crate) generation: Generation,
43 : pub(crate) attach_mode: AttachmentMode,
44 : // TODO: add a flag to override AttachmentMode's policies under
45 : // disk pressure (i.e. unblock uploads under disk pressure in Stale
46 : // state, unblock deletions after timeout in Multi state)
47 : }
48 :
49 0 : #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
50 : pub(crate) struct SecondaryLocationConfig {
51 : /// If true, keep the local cache warm by polling remote storage
52 : pub(crate) warm: bool,
53 : }
54 :
55 0 : #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
56 : pub(crate) enum LocationMode {
57 : Attached(AttachedLocationConfig),
58 : Secondary(SecondaryLocationConfig),
59 : }
60 :
61 : /// Per-tenant, per-pageserver configuration. All pageservers use the same TenantConf,
62 : /// but have distinct LocationConf.
63 0 : #[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
64 : pub(crate) struct LocationConf {
65 : /// The location-specific part of the configuration, describes the operating
66 : /// mode of this pageserver for this tenant.
67 : pub(crate) mode: LocationMode,
68 :
69 : /// The detailed shard identity. This structure is already scoped within
70 : /// a TenantShardId, but we need the full ShardIdentity to enable calculating
71 : /// key->shard mappings.
72 : #[serde(default = "ShardIdentity::unsharded")]
73 : #[serde(skip_serializing_if = "ShardIdentity::is_unsharded")]
74 : pub(crate) shard: ShardIdentity,
75 :
76 : /// The pan-cluster tenant configuration, the same on all locations
77 : pub(crate) tenant_conf: TenantConfOpt,
78 : }
79 :
80 : impl std::fmt::Debug for LocationConf {
81 0 : fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
82 0 : match &self.mode {
83 0 : LocationMode::Attached(conf) => {
84 0 : write!(
85 0 : f,
86 0 : "Attached {:?}, gen={:?}",
87 0 : conf.attach_mode, conf.generation
88 0 : )
89 : }
90 0 : LocationMode::Secondary(conf) => {
91 0 : write!(f, "Secondary, warm={}", conf.warm)
92 : }
93 : }
94 0 : }
95 : }
96 :
97 : impl AttachedLocationConfig {
98 : /// Consult attachment mode to determine whether we are currently permitted
99 : /// to delete layers. This is only advisory, not required for data safety.
100 : /// See [`AttachmentMode`] for more context.
101 2432 : pub(crate) fn may_delete_layers_hint(&self) -> bool {
102 2432 : // TODO: add an override for disk pressure in AttachedLocationConfig,
103 2432 : // and respect it here.
104 2432 : match &self.attach_mode {
105 2432 : AttachmentMode::Single => true,
106 : AttachmentMode::Multi | AttachmentMode::Stale => {
107 : // In Multi mode we avoid doing deletions because some other
108 : // attached pageserver might get 404 while trying to read
109 : // a layer we delete which is still referenced in their metadata.
110 : //
111 : // In Stale mode, we avoid doing deletions because we expect
112 : // that they would ultimately fail validation in the deletion
113 : // queue due to our stale generation.
114 0 : false
115 : }
116 : }
117 2432 : }
118 :
119 : /// Whether we are currently hinted that it is worthwhile to upload layers.
120 : /// This is only advisory, not required for data safety.
121 : /// See [`AttachmentMode`] for more context.
122 924 : pub(crate) fn may_upload_layers_hint(&self) -> bool {
123 924 : // TODO: add an override for disk pressure in AttachedLocationConfig,
124 924 : // and respect it here.
125 924 : match &self.attach_mode {
126 924 : AttachmentMode::Single | AttachmentMode::Multi => true,
127 : AttachmentMode::Stale => {
128 : // In Stale mode, we avoid doing uploads because we expect that
129 : // our replacement pageserver will already have started its own
130 : // IndexPart that will never reference layers we upload: it is
131 : // wasteful.
132 0 : false
133 : }
134 : }
135 924 : }
136 : }
137 :
138 : impl LocationConf {
139 : /// For use when loading from a legacy configuration: presence of a tenant
140 : /// implies it is in AttachmentMode::Single, which used to be the only
141 : /// possible state. This function should eventually be removed.
142 452 : pub(crate) fn attached_single(
143 452 : tenant_conf: TenantConfOpt,
144 452 : generation: Generation,
145 452 : shard_params: &models::ShardParameters,
146 452 : ) -> Self {
147 452 : Self {
148 452 : mode: LocationMode::Attached(AttachedLocationConfig {
149 452 : generation,
150 452 : attach_mode: AttachmentMode::Single,
151 452 : }),
152 452 : shard: ShardIdentity::from_params(ShardNumber(0), shard_params),
153 452 : tenant_conf,
154 452 : }
155 452 : }
156 :
157 : /// For use when attaching/re-attaching: update the generation stored in this
158 : /// structure. If we were in a secondary state, promote to attached (posession
159 : /// of a fresh generation implies this).
160 0 : pub(crate) fn attach_in_generation(&mut self, mode: AttachmentMode, generation: Generation) {
161 0 : match &mut self.mode {
162 0 : LocationMode::Attached(attach_conf) => {
163 0 : attach_conf.generation = generation;
164 0 : attach_conf.attach_mode = mode;
165 0 : }
166 : LocationMode::Secondary(_) => {
167 : // We are promoted to attached by the control plane's re-attach response
168 0 : self.mode = LocationMode::Attached(AttachedLocationConfig {
169 0 : generation,
170 0 : attach_mode: mode,
171 0 : })
172 : }
173 : }
174 0 : }
175 :
176 0 : pub(crate) fn try_from(conf: &'_ models::LocationConfig) -> anyhow::Result<Self> {
177 0 : let tenant_conf = TenantConfOpt::try_from(&conf.tenant_conf)?;
178 :
179 0 : fn get_generation(conf: &'_ models::LocationConfig) -> Result<Generation, anyhow::Error> {
180 0 : conf.generation
181 0 : .map(Generation::new)
182 0 : .ok_or_else(|| anyhow::anyhow!("Generation must be set when attaching"))
183 0 : }
184 :
185 0 : let mode = match &conf.mode {
186 : models::LocationConfigMode::AttachedMulti => {
187 : LocationMode::Attached(AttachedLocationConfig {
188 0 : generation: get_generation(conf)?,
189 0 : attach_mode: AttachmentMode::Multi,
190 : })
191 : }
192 : models::LocationConfigMode::AttachedSingle => {
193 : LocationMode::Attached(AttachedLocationConfig {
194 0 : generation: get_generation(conf)?,
195 0 : attach_mode: AttachmentMode::Single,
196 : })
197 : }
198 : models::LocationConfigMode::AttachedStale => {
199 : LocationMode::Attached(AttachedLocationConfig {
200 0 : generation: get_generation(conf)?,
201 0 : attach_mode: AttachmentMode::Stale,
202 : })
203 : }
204 : models::LocationConfigMode::Secondary => {
205 0 : anyhow::ensure!(conf.generation.is_none());
206 :
207 0 : let warm = conf
208 0 : .secondary_conf
209 0 : .as_ref()
210 0 : .map(|c| c.warm)
211 0 : .unwrap_or(false);
212 0 : LocationMode::Secondary(SecondaryLocationConfig { warm })
213 : }
214 : models::LocationConfigMode::Detached => {
215 : // Should not have been called: API code should translate this mode
216 : // into a detach rather than trying to decode it as a LocationConf
217 0 : return Err(anyhow::anyhow!("Cannot decode a Detached configuration"));
218 : }
219 : };
220 :
221 0 : let shard = if conf.shard_count == 0 {
222 : // NB: carry over the persisted stripe size instead of using the default. This doesn't
223 : // matter for most practical purposes, since unsharded tenants don't use the stripe
224 : // size, but can cause inconsistencies between storcon and Pageserver and cause manual
225 : // splits without `new_stripe_size` to use an unintended stripe size.
226 0 : ShardIdentity::unsharded_with_stripe_size(ShardStripeSize(conf.shard_stripe_size))
227 : } else {
228 0 : ShardIdentity::new(
229 0 : ShardNumber(conf.shard_number),
230 0 : ShardCount::new(conf.shard_count),
231 0 : ShardStripeSize(conf.shard_stripe_size),
232 0 : )?
233 : };
234 :
235 0 : Ok(Self {
236 0 : shard,
237 0 : mode,
238 0 : tenant_conf,
239 0 : })
240 0 : }
241 : }
242 :
243 : impl Default for LocationConf {
244 : // TODO: this should be removed once tenant loading can guarantee that we are never
245 : // loading from a directory without a configuration.
246 : // => tech debt since https://github.com/neondatabase/neon/issues/1555
247 0 : fn default() -> Self {
248 0 : Self {
249 0 : mode: LocationMode::Attached(AttachedLocationConfig {
250 0 : generation: Generation::none(),
251 0 : attach_mode: AttachmentMode::Single,
252 0 : }),
253 0 : tenant_conf: TenantConfOpt::default(),
254 0 : shard: ShardIdentity::unsharded(),
255 0 : }
256 0 : }
257 : }
258 :
259 : /// Same as TenantConf, but this struct preserves the information about
260 : /// which parameters are set and which are not.
261 148 : #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
262 : pub struct TenantConfOpt {
263 : #[serde(skip_serializing_if = "Option::is_none")]
264 : #[serde(default)]
265 : pub checkpoint_distance: Option<u64>,
266 :
267 : #[serde(skip_serializing_if = "Option::is_none")]
268 : #[serde(with = "humantime_serde")]
269 : #[serde(default)]
270 : pub checkpoint_timeout: Option<Duration>,
271 :
272 : #[serde(skip_serializing_if = "Option::is_none")]
273 : #[serde(default)]
274 : pub compaction_target_size: Option<u64>,
275 :
276 : #[serde(skip_serializing_if = "Option::is_none")]
277 : #[serde(with = "humantime_serde")]
278 : #[serde(default)]
279 : pub compaction_period: Option<Duration>,
280 :
281 : #[serde(skip_serializing_if = "Option::is_none")]
282 : #[serde(default)]
283 : pub compaction_threshold: Option<usize>,
284 :
285 : #[serde(skip_serializing_if = "Option::is_none")]
286 : #[serde(default)]
287 : pub compaction_upper_limit: Option<usize>,
288 :
289 : #[serde(skip_serializing_if = "Option::is_none")]
290 : #[serde(default)]
291 : pub compaction_algorithm: Option<CompactionAlgorithmSettings>,
292 :
293 : #[serde(skip_serializing_if = "Option::is_none")]
294 : #[serde(default)]
295 : pub compaction_l0_first: Option<bool>,
296 :
297 : #[serde(skip_serializing_if = "Option::is_none")]
298 : #[serde(default)]
299 : pub compaction_l0_semaphore: Option<bool>,
300 :
301 : #[serde(skip_serializing_if = "Option::is_none")]
302 : #[serde(default)]
303 : pub l0_flush_delay_threshold: Option<usize>,
304 :
305 : #[serde(skip_serializing_if = "Option::is_none")]
306 : #[serde(default)]
307 : pub l0_flush_stall_threshold: Option<usize>,
308 :
309 : #[serde(skip_serializing_if = "Option::is_none")]
310 : #[serde(default)]
311 : pub l0_flush_wait_upload: Option<bool>,
312 :
313 : #[serde(skip_serializing_if = "Option::is_none")]
314 : #[serde(default)]
315 : pub gc_horizon: Option<u64>,
316 :
317 : #[serde(skip_serializing_if = "Option::is_none")]
318 : #[serde(with = "humantime_serde")]
319 : #[serde(default)]
320 : pub gc_period: Option<Duration>,
321 :
322 : #[serde(skip_serializing_if = "Option::is_none")]
323 : #[serde(default)]
324 : pub image_creation_threshold: Option<usize>,
325 :
326 : #[serde(skip_serializing_if = "Option::is_none")]
327 : #[serde(with = "humantime_serde")]
328 : #[serde(default)]
329 : pub pitr_interval: Option<Duration>,
330 :
331 : #[serde(skip_serializing_if = "Option::is_none")]
332 : #[serde(with = "humantime_serde")]
333 : #[serde(default)]
334 : pub walreceiver_connect_timeout: Option<Duration>,
335 :
336 : #[serde(skip_serializing_if = "Option::is_none")]
337 : #[serde(with = "humantime_serde")]
338 : #[serde(default)]
339 : pub lagging_wal_timeout: Option<Duration>,
340 :
341 : #[serde(skip_serializing_if = "Option::is_none")]
342 : #[serde(default)]
343 : pub max_lsn_wal_lag: Option<NonZeroU64>,
344 :
345 : #[serde(skip_serializing_if = "Option::is_none")]
346 : #[serde(default)]
347 : pub eviction_policy: Option<EvictionPolicy>,
348 :
349 : #[serde(skip_serializing_if = "Option::is_none")]
350 : #[serde(default)]
351 : pub min_resident_size_override: Option<u64>,
352 :
353 : #[serde(skip_serializing_if = "Option::is_none")]
354 : #[serde(with = "humantime_serde")]
355 : #[serde(default)]
356 : pub evictions_low_residence_duration_metric_threshold: Option<Duration>,
357 :
358 : #[serde(skip_serializing_if = "Option::is_none")]
359 : #[serde(with = "humantime_serde")]
360 : #[serde(default)]
361 : pub heatmap_period: Option<Duration>,
362 :
363 : #[serde(skip_serializing_if = "Option::is_none")]
364 : #[serde(default)]
365 : pub lazy_slru_download: Option<bool>,
366 :
367 : #[serde(skip_serializing_if = "Option::is_none")]
368 : pub timeline_get_throttle: Option<pageserver_api::models::ThrottleConfig>,
369 :
370 : #[serde(skip_serializing_if = "Option::is_none")]
371 : pub image_layer_creation_check_threshold: Option<u8>,
372 :
373 : #[serde(skip_serializing_if = "Option::is_none")]
374 : pub image_creation_preempt_threshold: Option<usize>,
375 :
376 : #[serde(skip_serializing_if = "Option::is_none")]
377 : #[serde(with = "humantime_serde")]
378 : #[serde(default)]
379 : pub lsn_lease_length: Option<Duration>,
380 :
381 : #[serde(skip_serializing_if = "Option::is_none")]
382 : #[serde(with = "humantime_serde")]
383 : #[serde(default)]
384 : pub lsn_lease_length_for_ts: Option<Duration>,
385 :
386 : #[serde(skip_serializing_if = "Option::is_none")]
387 : #[serde(default)]
388 : pub timeline_offloading: Option<bool>,
389 :
390 : #[serde(skip_serializing_if = "Option::is_none")]
391 : pub wal_receiver_protocol_override: Option<PostgresClientProtocol>,
392 :
393 : #[serde(skip_serializing_if = "Option::is_none")]
394 : pub rel_size_v2_enabled: Option<bool>,
395 :
396 : #[serde(skip_serializing_if = "Option::is_none")]
397 : pub gc_compaction_enabled: Option<bool>,
398 :
399 : #[serde(skip_serializing_if = "Option::is_none")]
400 : pub gc_compaction_initial_threshold_kb: Option<u64>,
401 :
402 : #[serde(skip_serializing_if = "Option::is_none")]
403 : pub gc_compaction_ratio_percent: Option<u64>,
404 : }
405 :
406 : impl TenantConfOpt {
407 0 : pub fn merge(&self, global_conf: TenantConf) -> TenantConf {
408 0 : TenantConf {
409 0 : checkpoint_distance: self
410 0 : .checkpoint_distance
411 0 : .unwrap_or(global_conf.checkpoint_distance),
412 0 : checkpoint_timeout: self
413 0 : .checkpoint_timeout
414 0 : .unwrap_or(global_conf.checkpoint_timeout),
415 0 : compaction_target_size: self
416 0 : .compaction_target_size
417 0 : .unwrap_or(global_conf.compaction_target_size),
418 0 : compaction_period: self
419 0 : .compaction_period
420 0 : .unwrap_or(global_conf.compaction_period),
421 0 : compaction_threshold: self
422 0 : .compaction_threshold
423 0 : .unwrap_or(global_conf.compaction_threshold),
424 0 : compaction_upper_limit: self
425 0 : .compaction_upper_limit
426 0 : .unwrap_or(global_conf.compaction_upper_limit),
427 0 : compaction_algorithm: self
428 0 : .compaction_algorithm
429 0 : .as_ref()
430 0 : .unwrap_or(&global_conf.compaction_algorithm)
431 0 : .clone(),
432 0 : compaction_l0_first: self
433 0 : .compaction_l0_first
434 0 : .unwrap_or(global_conf.compaction_l0_first),
435 0 : compaction_l0_semaphore: self
436 0 : .compaction_l0_semaphore
437 0 : .unwrap_or(global_conf.compaction_l0_semaphore),
438 0 : l0_flush_delay_threshold: self
439 0 : .l0_flush_delay_threshold
440 0 : .or(global_conf.l0_flush_delay_threshold),
441 0 : l0_flush_stall_threshold: self
442 0 : .l0_flush_stall_threshold
443 0 : .or(global_conf.l0_flush_stall_threshold),
444 0 : l0_flush_wait_upload: self
445 0 : .l0_flush_wait_upload
446 0 : .unwrap_or(global_conf.l0_flush_wait_upload),
447 0 : gc_horizon: self.gc_horizon.unwrap_or(global_conf.gc_horizon),
448 0 : gc_period: self.gc_period.unwrap_or(global_conf.gc_period),
449 0 : image_creation_threshold: self
450 0 : .image_creation_threshold
451 0 : .unwrap_or(global_conf.image_creation_threshold),
452 0 : pitr_interval: self.pitr_interval.unwrap_or(global_conf.pitr_interval),
453 0 : walreceiver_connect_timeout: self
454 0 : .walreceiver_connect_timeout
455 0 : .unwrap_or(global_conf.walreceiver_connect_timeout),
456 0 : lagging_wal_timeout: self
457 0 : .lagging_wal_timeout
458 0 : .unwrap_or(global_conf.lagging_wal_timeout),
459 0 : max_lsn_wal_lag: self.max_lsn_wal_lag.unwrap_or(global_conf.max_lsn_wal_lag),
460 0 : eviction_policy: self.eviction_policy.unwrap_or(global_conf.eviction_policy),
461 0 : min_resident_size_override: self
462 0 : .min_resident_size_override
463 0 : .or(global_conf.min_resident_size_override),
464 0 : evictions_low_residence_duration_metric_threshold: self
465 0 : .evictions_low_residence_duration_metric_threshold
466 0 : .unwrap_or(global_conf.evictions_low_residence_duration_metric_threshold),
467 0 : heatmap_period: self.heatmap_period.unwrap_or(global_conf.heatmap_period),
468 0 : lazy_slru_download: self
469 0 : .lazy_slru_download
470 0 : .unwrap_or(global_conf.lazy_slru_download),
471 0 : timeline_get_throttle: self
472 0 : .timeline_get_throttle
473 0 : .clone()
474 0 : .unwrap_or(global_conf.timeline_get_throttle),
475 0 : image_layer_creation_check_threshold: self
476 0 : .image_layer_creation_check_threshold
477 0 : .unwrap_or(global_conf.image_layer_creation_check_threshold),
478 0 : image_creation_preempt_threshold: self
479 0 : .image_creation_preempt_threshold
480 0 : .unwrap_or(global_conf.image_creation_preempt_threshold),
481 0 : lsn_lease_length: self
482 0 : .lsn_lease_length
483 0 : .unwrap_or(global_conf.lsn_lease_length),
484 0 : lsn_lease_length_for_ts: self
485 0 : .lsn_lease_length_for_ts
486 0 : .unwrap_or(global_conf.lsn_lease_length_for_ts),
487 0 : timeline_offloading: self
488 0 : .timeline_offloading
489 0 : .unwrap_or(global_conf.timeline_offloading),
490 0 : wal_receiver_protocol_override: self
491 0 : .wal_receiver_protocol_override
492 0 : .or(global_conf.wal_receiver_protocol_override),
493 0 : rel_size_v2_enabled: self
494 0 : .rel_size_v2_enabled
495 0 : .unwrap_or(global_conf.rel_size_v2_enabled),
496 0 : gc_compaction_enabled: self
497 0 : .gc_compaction_enabled
498 0 : .unwrap_or(global_conf.gc_compaction_enabled),
499 0 : gc_compaction_initial_threshold_kb: self
500 0 : .gc_compaction_initial_threshold_kb
501 0 : .unwrap_or(global_conf.gc_compaction_initial_threshold_kb),
502 0 : gc_compaction_ratio_percent: self
503 0 : .gc_compaction_ratio_percent
504 0 : .unwrap_or(global_conf.gc_compaction_ratio_percent),
505 0 : }
506 0 : }
507 :
508 0 : pub fn apply_patch(self, patch: TenantConfigPatch) -> anyhow::Result<TenantConfOpt> {
509 0 : let Self {
510 0 : mut checkpoint_distance,
511 0 : mut checkpoint_timeout,
512 0 : mut compaction_target_size,
513 0 : mut compaction_period,
514 0 : mut compaction_threshold,
515 0 : mut compaction_upper_limit,
516 0 : mut compaction_algorithm,
517 0 : mut compaction_l0_first,
518 0 : mut compaction_l0_semaphore,
519 0 : mut l0_flush_delay_threshold,
520 0 : mut l0_flush_stall_threshold,
521 0 : mut l0_flush_wait_upload,
522 0 : mut gc_horizon,
523 0 : mut gc_period,
524 0 : mut image_creation_threshold,
525 0 : mut pitr_interval,
526 0 : mut walreceiver_connect_timeout,
527 0 : mut lagging_wal_timeout,
528 0 : mut max_lsn_wal_lag,
529 0 : mut eviction_policy,
530 0 : mut min_resident_size_override,
531 0 : mut evictions_low_residence_duration_metric_threshold,
532 0 : mut heatmap_period,
533 0 : mut lazy_slru_download,
534 0 : mut timeline_get_throttle,
535 0 : mut image_layer_creation_check_threshold,
536 0 : mut image_creation_preempt_threshold,
537 0 : mut lsn_lease_length,
538 0 : mut lsn_lease_length_for_ts,
539 0 : mut timeline_offloading,
540 0 : mut wal_receiver_protocol_override,
541 0 : mut rel_size_v2_enabled,
542 0 : mut gc_compaction_enabled,
543 0 : mut gc_compaction_initial_threshold_kb,
544 0 : mut gc_compaction_ratio_percent,
545 0 : } = self;
546 0 :
547 0 : patch.checkpoint_distance.apply(&mut checkpoint_distance);
548 0 : patch
549 0 : .checkpoint_timeout
550 0 : .map(|v| humantime::parse_duration(&v))?
551 0 : .apply(&mut checkpoint_timeout);
552 0 : patch
553 0 : .compaction_target_size
554 0 : .apply(&mut compaction_target_size);
555 0 : patch
556 0 : .compaction_period
557 0 : .map(|v| humantime::parse_duration(&v))?
558 0 : .apply(&mut compaction_period);
559 0 : patch.compaction_threshold.apply(&mut compaction_threshold);
560 0 : patch
561 0 : .compaction_upper_limit
562 0 : .apply(&mut compaction_upper_limit);
563 0 : patch.compaction_algorithm.apply(&mut compaction_algorithm);
564 0 : patch.compaction_l0_first.apply(&mut compaction_l0_first);
565 0 : patch
566 0 : .compaction_l0_semaphore
567 0 : .apply(&mut compaction_l0_semaphore);
568 0 : patch
569 0 : .l0_flush_delay_threshold
570 0 : .apply(&mut l0_flush_delay_threshold);
571 0 : patch
572 0 : .l0_flush_stall_threshold
573 0 : .apply(&mut l0_flush_stall_threshold);
574 0 : patch.l0_flush_wait_upload.apply(&mut l0_flush_wait_upload);
575 0 : patch.gc_horizon.apply(&mut gc_horizon);
576 0 : patch
577 0 : .gc_period
578 0 : .map(|v| humantime::parse_duration(&v))?
579 0 : .apply(&mut gc_period);
580 0 : patch
581 0 : .image_creation_threshold
582 0 : .apply(&mut image_creation_threshold);
583 0 : patch
584 0 : .pitr_interval
585 0 : .map(|v| humantime::parse_duration(&v))?
586 0 : .apply(&mut pitr_interval);
587 0 : patch
588 0 : .walreceiver_connect_timeout
589 0 : .map(|v| humantime::parse_duration(&v))?
590 0 : .apply(&mut walreceiver_connect_timeout);
591 0 : patch
592 0 : .lagging_wal_timeout
593 0 : .map(|v| humantime::parse_duration(&v))?
594 0 : .apply(&mut lagging_wal_timeout);
595 0 : patch.max_lsn_wal_lag.apply(&mut max_lsn_wal_lag);
596 0 : patch.eviction_policy.apply(&mut eviction_policy);
597 0 : patch
598 0 : .min_resident_size_override
599 0 : .apply(&mut min_resident_size_override);
600 0 : patch
601 0 : .evictions_low_residence_duration_metric_threshold
602 0 : .map(|v| humantime::parse_duration(&v))?
603 0 : .apply(&mut evictions_low_residence_duration_metric_threshold);
604 0 : patch
605 0 : .heatmap_period
606 0 : .map(|v| humantime::parse_duration(&v))?
607 0 : .apply(&mut heatmap_period);
608 0 : patch.lazy_slru_download.apply(&mut lazy_slru_download);
609 0 : patch
610 0 : .timeline_get_throttle
611 0 : .apply(&mut timeline_get_throttle);
612 0 : patch
613 0 : .image_layer_creation_check_threshold
614 0 : .apply(&mut image_layer_creation_check_threshold);
615 0 : patch
616 0 : .image_creation_preempt_threshold
617 0 : .apply(&mut image_creation_preempt_threshold);
618 0 : patch
619 0 : .lsn_lease_length
620 0 : .map(|v| humantime::parse_duration(&v))?
621 0 : .apply(&mut lsn_lease_length);
622 0 : patch
623 0 : .lsn_lease_length_for_ts
624 0 : .map(|v| humantime::parse_duration(&v))?
625 0 : .apply(&mut lsn_lease_length_for_ts);
626 0 : patch.timeline_offloading.apply(&mut timeline_offloading);
627 0 : patch
628 0 : .wal_receiver_protocol_override
629 0 : .apply(&mut wal_receiver_protocol_override);
630 0 : patch.rel_size_v2_enabled.apply(&mut rel_size_v2_enabled);
631 0 : patch
632 0 : .gc_compaction_enabled
633 0 : .apply(&mut gc_compaction_enabled);
634 0 : patch
635 0 : .gc_compaction_initial_threshold_kb
636 0 : .apply(&mut gc_compaction_initial_threshold_kb);
637 0 : patch
638 0 : .gc_compaction_ratio_percent
639 0 : .apply(&mut gc_compaction_ratio_percent);
640 0 :
641 0 : Ok(Self {
642 0 : checkpoint_distance,
643 0 : checkpoint_timeout,
644 0 : compaction_target_size,
645 0 : compaction_period,
646 0 : compaction_threshold,
647 0 : compaction_upper_limit,
648 0 : compaction_algorithm,
649 0 : compaction_l0_first,
650 0 : compaction_l0_semaphore,
651 0 : l0_flush_delay_threshold,
652 0 : l0_flush_stall_threshold,
653 0 : l0_flush_wait_upload,
654 0 : gc_horizon,
655 0 : gc_period,
656 0 : image_creation_threshold,
657 0 : pitr_interval,
658 0 : walreceiver_connect_timeout,
659 0 : lagging_wal_timeout,
660 0 : max_lsn_wal_lag,
661 0 : eviction_policy,
662 0 : min_resident_size_override,
663 0 : evictions_low_residence_duration_metric_threshold,
664 0 : heatmap_period,
665 0 : lazy_slru_download,
666 0 : timeline_get_throttle,
667 0 : image_layer_creation_check_threshold,
668 0 : image_creation_preempt_threshold,
669 0 : lsn_lease_length,
670 0 : lsn_lease_length_for_ts,
671 0 : timeline_offloading,
672 0 : wal_receiver_protocol_override,
673 0 : rel_size_v2_enabled,
674 0 : gc_compaction_enabled,
675 0 : gc_compaction_initial_threshold_kb,
676 0 : gc_compaction_ratio_percent,
677 0 : })
678 0 : }
679 : }
680 :
681 : impl TryFrom<&'_ models::TenantConfig> for TenantConfOpt {
682 : type Error = anyhow::Error;
683 :
684 4 : fn try_from(request_data: &'_ models::TenantConfig) -> Result<Self, Self::Error> {
685 : // Convert the request_data to a JSON Value
686 4 : let json_value: Value = serde_json::to_value(request_data)?;
687 :
688 : // Create a Deserializer from the JSON Value
689 4 : let deserializer = json_value.into_deserializer();
690 :
691 : // Use serde_path_to_error to deserialize the JSON Value into TenantConfOpt
692 4 : let tenant_conf: TenantConfOpt = serde_path_to_error::deserialize(deserializer)?;
693 :
694 4 : Ok(tenant_conf)
695 4 : }
696 : }
697 :
698 : /// This is a conversion from our internal tenant config object to the one used
699 : /// in external APIs.
700 : impl From<TenantConfOpt> for models::TenantConfig {
701 : // TODO(vlad): These are now the same, but they have different serialization logic.
702 : // Can we merge them?
703 0 : fn from(value: TenantConfOpt) -> Self {
704 0 : Self {
705 0 : checkpoint_distance: value.checkpoint_distance,
706 0 : checkpoint_timeout: value.checkpoint_timeout,
707 0 : compaction_algorithm: value.compaction_algorithm,
708 0 : compaction_target_size: value.compaction_target_size,
709 0 : compaction_period: value.compaction_period,
710 0 : compaction_threshold: value.compaction_threshold,
711 0 : compaction_upper_limit: value.compaction_upper_limit,
712 0 : compaction_l0_first: value.compaction_l0_first,
713 0 : compaction_l0_semaphore: value.compaction_l0_semaphore,
714 0 : l0_flush_delay_threshold: value.l0_flush_delay_threshold,
715 0 : l0_flush_stall_threshold: value.l0_flush_stall_threshold,
716 0 : l0_flush_wait_upload: value.l0_flush_wait_upload,
717 0 : gc_horizon: value.gc_horizon,
718 0 : gc_period: value.gc_period,
719 0 : image_creation_threshold: value.image_creation_threshold,
720 0 : pitr_interval: value.pitr_interval,
721 0 : walreceiver_connect_timeout: value.walreceiver_connect_timeout,
722 0 : lagging_wal_timeout: value.lagging_wal_timeout,
723 0 : max_lsn_wal_lag: value.max_lsn_wal_lag,
724 0 : eviction_policy: value.eviction_policy,
725 0 : min_resident_size_override: value.min_resident_size_override,
726 0 : evictions_low_residence_duration_metric_threshold: value
727 0 : .evictions_low_residence_duration_metric_threshold,
728 0 : heatmap_period: value.heatmap_period,
729 0 : lazy_slru_download: value.lazy_slru_download,
730 0 : timeline_get_throttle: value.timeline_get_throttle,
731 0 : image_layer_creation_check_threshold: value.image_layer_creation_check_threshold,
732 0 : image_creation_preempt_threshold: value.image_creation_preempt_threshold,
733 0 : lsn_lease_length: value.lsn_lease_length,
734 0 : lsn_lease_length_for_ts: value.lsn_lease_length_for_ts,
735 0 : timeline_offloading: value.timeline_offloading,
736 0 : wal_receiver_protocol_override: value.wal_receiver_protocol_override,
737 0 : rel_size_v2_enabled: value.rel_size_v2_enabled,
738 0 : gc_compaction_enabled: value.gc_compaction_enabled,
739 0 : gc_compaction_initial_threshold_kb: value.gc_compaction_initial_threshold_kb,
740 0 : gc_compaction_ratio_percent: value.gc_compaction_ratio_percent,
741 0 : }
742 0 : }
743 : }
744 :
745 : #[cfg(test)]
746 : mod tests {
747 : use models::TenantConfig;
748 :
749 : use super::*;
750 :
751 : #[test]
752 4 : fn de_serializing_pageserver_config_omits_empty_values() {
753 4 : let small_conf = TenantConfOpt {
754 4 : gc_horizon: Some(42),
755 4 : ..TenantConfOpt::default()
756 4 : };
757 4 :
758 4 : let toml_form = toml_edit::ser::to_string(&small_conf).unwrap();
759 4 : assert_eq!(toml_form, "gc_horizon = 42\n");
760 4 : assert_eq!(small_conf, toml_edit::de::from_str(&toml_form).unwrap());
761 :
762 4 : let json_form = serde_json::to_string(&small_conf).unwrap();
763 4 : assert_eq!(json_form, "{\"gc_horizon\":42}");
764 4 : assert_eq!(small_conf, serde_json::from_str(&json_form).unwrap());
765 4 : }
766 :
767 : #[test]
768 4 : fn test_try_from_models_tenant_config_success() {
769 4 : let tenant_config = models::TenantConfig {
770 4 : lagging_wal_timeout: Some(Duration::from_secs(5)),
771 4 : ..TenantConfig::default()
772 4 : };
773 4 :
774 4 : let tenant_conf_opt = TenantConfOpt::try_from(&tenant_config).unwrap();
775 4 :
776 4 : assert_eq!(
777 4 : tenant_conf_opt.lagging_wal_timeout,
778 4 : Some(Duration::from_secs(5))
779 4 : );
780 4 : }
781 : }
|
