Line data Source code
1 : pub mod detach_ancestor;
2 : pub mod partitioning;
3 : pub mod utilization;
4 :
5 : use core::ops::Range;
6 : use std::collections::HashMap;
7 : use std::fmt::Display;
8 : use std::num::{NonZeroU32, NonZeroU64, NonZeroUsize};
9 : use std::str::FromStr;
10 : use std::time::{Duration, SystemTime};
11 :
12 : #[cfg(feature = "testing")]
13 : use camino::Utf8PathBuf;
14 : use postgres_versioninfo::PgMajorVersion;
15 : use serde::{Deserialize, Deserializer, Serialize, Serializer};
16 : use serde_with::serde_as;
17 : pub use utilization::PageserverUtilization;
18 : use utils::id::{NodeId, TenantId, TimelineId};
19 : use utils::lsn::Lsn;
20 : use utils::{completion, serde_system_time};
21 :
22 : use crate::config::Ratio;
23 : use crate::key::{CompactKey, Key};
24 : use crate::shard::{
25 : DEFAULT_STRIPE_SIZE, ShardCount, ShardIdentity, ShardStripeSize, TenantShardId,
26 : };
27 :
28 : /// The state of a tenant in this pageserver.
29 : ///
30 : /// ```mermaid
31 : /// stateDiagram-v2
32 : ///
33 : /// [*] --> Attaching: spawn_attach()
34 : ///
35 : /// Attaching --> Activating: activate()
36 : /// Activating --> Active: infallible
37 : ///
38 : /// Attaching --> Broken: attach() failure
39 : ///
40 : /// Active --> Stopping: set_stopping(), part of shutdown & detach
41 : /// Stopping --> Broken: late error in remove_tenant_from_memory
42 : ///
43 : /// Broken --> [*]: ignore / detach / shutdown
44 : /// Stopping --> [*]: remove_from_memory complete
45 : ///
46 : /// Active --> Broken: cfg(testing)-only tenant break point
47 : /// ```
48 : #[derive(
49 : Clone,
50 : PartialEq,
51 : Eq,
52 0 : serde::Serialize,
53 0 : serde::Deserialize,
54 : strum_macros::Display,
55 : strum_macros::VariantNames,
56 : strum_macros::AsRefStr,
57 : strum_macros::IntoStaticStr,
58 : )]
59 : #[serde(tag = "slug", content = "data")]
60 : pub enum TenantState {
61 : /// This tenant is being attached to the pageserver.
62 : ///
63 : /// `set_stopping()` and `set_broken()` do not work in this state and wait for it to pass.
64 : Attaching,
65 : /// The tenant is transitioning from Loading/Attaching to Active.
66 : ///
67 : /// While in this state, the individual timelines are being activated.
68 : ///
69 : /// `set_stopping()` and `set_broken()` do not work in this state and wait for it to pass.
70 : Activating(ActivatingFrom),
71 : /// The tenant has finished activating and is open for business.
72 : ///
73 : /// Transitions out of this state are possible through `set_stopping()` and `set_broken()`.
74 : Active,
75 : /// The tenant is recognized by pageserver, but it is being detached or the
76 : /// system is being shut down.
77 : ///
78 : /// Transitions out of this state are possible through `set_broken()`.
79 : Stopping {
80 : /// The barrier can be used to wait for shutdown to complete. The first caller to set
81 : /// Some(Barrier) is responsible for driving shutdown to completion. Subsequent callers
82 : /// will wait for the first caller's existing barrier.
83 : ///
84 : /// None is set when an attach is cancelled, to signal to shutdown that the attach has in
85 : /// fact cancelled:
86 : ///
87 : /// 1. `shutdown` sees `TenantState::Attaching`, and cancels the tenant.
88 : /// 2. `attach` sets `TenantState::Stopping(None)` and exits.
89 : /// 3. `set_stopping` waits for `TenantState::Stopping(None)` and sets
90 : /// `TenantState::Stopping(Some)` to claim the barrier as the shutdown owner.
91 : //
92 : // Because of https://github.com/serde-rs/serde/issues/2105 this has to be a named field,
93 : // otherwise it will not be skipped during deserialization
94 : #[serde(skip)]
95 : progress: Option<completion::Barrier>,
96 : },
97 : /// The tenant is recognized by the pageserver, but can no longer be used for
98 : /// any operations.
99 : ///
100 : /// If the tenant fails to load or attach, it will transition to this state
101 : /// and it is guaranteed that no background tasks are running in its name.
102 : ///
103 : /// The other way to transition into this state is from `Stopping` state
104 : /// through `set_broken()` called from `remove_tenant_from_memory()`. That happens
105 : /// if the cleanup future executed by `remove_tenant_from_memory()` fails.
106 : Broken { reason: String, backtrace: String },
107 : }
108 :
109 : impl TenantState {
110 0 : pub fn attachment_status(&self) -> TenantAttachmentStatus {
111 : use TenantAttachmentStatus::*;
112 :
113 : // Below TenantState::Activating is used as "transient" or "transparent" state for
114 : // attachment_status determining.
115 0 : match self {
116 : // The attach procedure writes the marker file before adding the Attaching tenant to the tenants map.
117 : // So, technically, we can return Attached here.
118 : // However, as soon as Console observes Attached, it will proceed with the Postgres-level health check.
119 : // But, our attach task might still be fetching the remote timelines, etc.
120 : // So, return `Maybe` while Attaching, making Console wait for the attach task to finish.
121 0 : Self::Attaching | Self::Activating(ActivatingFrom::Attaching) => Maybe,
122 : // We only reach Active after successful load / attach.
123 : // So, call atttachment status Attached.
124 0 : Self::Active => Attached,
125 : // If the (initial or resumed) attach procedure fails, the tenant becomes Broken.
126 : // However, it also becomes Broken if the regular load fails.
127 : // From Console's perspective there's no practical difference
128 : // because attachment_status is polled by console only during attach operation execution.
129 0 : Self::Broken { reason, .. } => Failed {
130 0 : reason: reason.to_owned(),
131 0 : },
132 : // Why is Stopping a Maybe case? Because, during pageserver shutdown,
133 : // we set the Stopping state irrespective of whether the tenant
134 : // has finished attaching or not.
135 0 : Self::Stopping { .. } => Maybe,
136 : }
137 0 : }
138 :
139 0 : pub fn broken_from_reason(reason: String) -> Self {
140 0 : let backtrace_str: String = format!("{}", std::backtrace::Backtrace::force_capture());
141 0 : Self::Broken {
142 0 : reason,
143 0 : backtrace: backtrace_str,
144 0 : }
145 0 : }
146 : }
147 :
148 : impl std::fmt::Debug for TenantState {
149 2 : fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
150 2 : match self {
151 2 : Self::Broken { reason, backtrace } if !reason.is_empty() => {
152 2 : write!(f, "Broken due to: {reason}. Backtrace:\n{backtrace}")
153 : }
154 0 : _ => write!(f, "{self}"),
155 : }
156 2 : }
157 : }
158 :
159 : /// A temporary lease to a specific lsn inside a timeline.
160 : /// Access to the lsn is guaranteed by the pageserver until the expiration indicated by `valid_until`.
161 : #[serde_as]
162 : #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
163 : pub struct LsnLease {
164 : #[serde_as(as = "SystemTimeAsRfc3339Millis")]
165 : pub valid_until: SystemTime,
166 : }
167 :
168 : serde_with::serde_conv!(
169 : SystemTimeAsRfc3339Millis,
170 : SystemTime,
171 0 : |time: &SystemTime| humantime::format_rfc3339_millis(*time).to_string(),
172 0 : |value: String| -> Result<_, humantime::TimestampError> { humantime::parse_rfc3339(&value) }
173 : );
174 :
175 : impl LsnLease {
176 : /// The default length for an explicit LSN lease request (10 minutes).
177 : pub const DEFAULT_LENGTH: Duration = Duration::from_secs(10 * 60);
178 :
179 : /// The default length for an implicit LSN lease granted during
180 : /// `get_lsn_by_timestamp` request (1 minutes).
181 : pub const DEFAULT_LENGTH_FOR_TS: Duration = Duration::from_secs(60);
182 :
183 : /// Checks whether the lease is expired.
184 3 : pub fn is_expired(&self, now: &SystemTime) -> bool {
185 3 : now > &self.valid_until
186 3 : }
187 : }
188 :
189 : /// Controls the detach ancestor behavior.
190 : /// - When set to `NoAncestorAndReparent`, we will only detach a branch if its ancestor is a root branch. It will automatically reparent any children of the ancestor before and at the branch point.
191 : /// - When set to `MultiLevelAndNoReparent`, we will detach a branch from multiple levels of ancestors, and no reparenting will happen at all.
192 : #[derive(Debug, Clone, Copy, Default)]
193 : pub enum DetachBehavior {
194 : #[default]
195 : NoAncestorAndReparent,
196 : MultiLevelAndNoReparent,
197 : }
198 :
199 : impl std::str::FromStr for DetachBehavior {
200 : type Err = &'static str;
201 :
202 0 : fn from_str(s: &str) -> Result<Self, Self::Err> {
203 0 : match s {
204 0 : "no_ancestor_and_reparent" => Ok(DetachBehavior::NoAncestorAndReparent),
205 0 : "multi_level_and_no_reparent" => Ok(DetachBehavior::MultiLevelAndNoReparent),
206 0 : "v1" => Ok(DetachBehavior::NoAncestorAndReparent),
207 0 : "v2" => Ok(DetachBehavior::MultiLevelAndNoReparent),
208 0 : _ => Err("cannot parse detach behavior"),
209 : }
210 0 : }
211 : }
212 :
213 : impl std::fmt::Display for DetachBehavior {
214 0 : fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
215 0 : match self {
216 0 : DetachBehavior::NoAncestorAndReparent => write!(f, "no_ancestor_and_reparent"),
217 0 : DetachBehavior::MultiLevelAndNoReparent => write!(f, "multi_level_and_no_reparent"),
218 : }
219 0 : }
220 : }
221 :
222 : /// The only [`TenantState`] variants we could be `TenantState::Activating` from.
223 : ///
224 : /// XXX: We used to have more variants here, but now it's just one, which makes this rather
225 : /// useless. Remove, once we've checked that there's no client code left that looks at this.
226 0 : #[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
227 : pub enum ActivatingFrom {
228 : /// Arrived to [`TenantState::Activating`] from [`TenantState::Attaching`]
229 : Attaching,
230 : }
231 :
232 : /// A state of a timeline in pageserver's memory.
233 0 : #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
234 : pub enum TimelineState {
235 : /// The timeline is recognized by the pageserver but is not yet operational.
236 : /// In particular, the walreceiver connection loop is not running for this timeline.
237 : /// It will eventually transition to state Active or Broken.
238 : Loading,
239 : /// The timeline is fully operational.
240 : /// It can be queried, and the walreceiver connection loop is running.
241 : Active,
242 : /// The timeline was previously Loading or Active but is shutting down.
243 : /// It cannot transition back into any other state.
244 : Stopping,
245 : /// The timeline is broken and not operational (previous states: Loading or Active).
246 : Broken { reason: String, backtrace: String },
247 : }
248 :
249 : #[serde_with::serde_as]
250 0 : #[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
251 : pub struct CompactLsnRange {
252 : pub start: Lsn,
253 : pub end: Lsn,
254 : }
255 :
256 : #[serde_with::serde_as]
257 : #[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
258 : pub struct CompactKeyRange {
259 : #[serde_as(as = "serde_with::DisplayFromStr")]
260 : pub start: Key,
261 : #[serde_as(as = "serde_with::DisplayFromStr")]
262 : pub end: Key,
263 : }
264 :
265 : impl From<Range<Lsn>> for CompactLsnRange {
266 3 : fn from(range: Range<Lsn>) -> Self {
267 3 : Self {
268 3 : start: range.start,
269 3 : end: range.end,
270 3 : }
271 3 : }
272 : }
273 :
274 : impl From<Range<Key>> for CompactKeyRange {
275 8 : fn from(range: Range<Key>) -> Self {
276 8 : Self {
277 8 : start: range.start,
278 8 : end: range.end,
279 8 : }
280 8 : }
281 : }
282 :
283 : impl From<CompactLsnRange> for Range<Lsn> {
284 5 : fn from(range: CompactLsnRange) -> Self {
285 5 : range.start..range.end
286 5 : }
287 : }
288 :
289 : impl From<CompactKeyRange> for Range<Key> {
290 8 : fn from(range: CompactKeyRange) -> Self {
291 8 : range.start..range.end
292 8 : }
293 : }
294 :
295 : impl CompactLsnRange {
296 2 : pub fn above(lsn: Lsn) -> Self {
297 2 : Self {
298 2 : start: lsn,
299 2 : end: Lsn::MAX,
300 2 : }
301 2 : }
302 : }
303 :
304 : #[derive(Debug, Clone, Serialize)]
305 : pub struct CompactInfoResponse {
306 : pub compact_key_range: Option<CompactKeyRange>,
307 : pub compact_lsn_range: Option<CompactLsnRange>,
308 : pub sub_compaction: bool,
309 : pub running: bool,
310 : pub job_id: usize,
311 : }
312 :
313 0 : #[derive(Serialize, Deserialize, Clone)]
314 : pub struct TimelineCreateRequest {
315 : pub new_timeline_id: TimelineId,
316 : #[serde(flatten)]
317 : pub mode: TimelineCreateRequestMode,
318 : }
319 :
320 : impl TimelineCreateRequest {
321 0 : pub fn mode_tag(&self) -> &'static str {
322 0 : match &self.mode {
323 0 : TimelineCreateRequestMode::Branch { .. } => "branch",
324 0 : TimelineCreateRequestMode::ImportPgdata { .. } => "import",
325 0 : TimelineCreateRequestMode::Bootstrap { .. } => "bootstrap",
326 : }
327 0 : }
328 :
329 0 : pub fn is_import(&self) -> bool {
330 0 : matches!(self.mode, TimelineCreateRequestMode::ImportPgdata { .. })
331 0 : }
332 : }
333 :
334 0 : #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
335 : pub enum ShardImportStatus {
336 : InProgress(Option<ShardImportProgress>),
337 : Done,
338 : Error(String),
339 : }
340 :
341 0 : #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
342 : pub enum ShardImportProgress {
343 : V1(ShardImportProgressV1),
344 : }
345 :
346 0 : #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
347 : pub struct ShardImportProgressV1 {
348 : /// Total number of jobs in the import plan
349 : pub jobs: usize,
350 : /// Number of jobs completed
351 : pub completed: usize,
352 : /// Hash of the plan
353 : pub import_plan_hash: u64,
354 : /// Soft limit for the job size
355 : /// This needs to remain constant throughout the import
356 : pub job_soft_size_limit: usize,
357 : }
358 :
359 : impl ShardImportStatus {
360 0 : pub fn is_terminal(&self) -> bool {
361 0 : match self {
362 0 : ShardImportStatus::InProgress(_) => false,
363 0 : ShardImportStatus::Done | ShardImportStatus::Error(_) => true,
364 : }
365 0 : }
366 : }
367 :
368 : /// Storage controller specific extensions to [`TimelineInfo`].
369 0 : #[derive(Serialize, Deserialize, Clone)]
370 : pub struct TimelineCreateResponseStorcon {
371 : #[serde(flatten)]
372 : pub timeline_info: TimelineInfo,
373 :
374 : pub safekeepers: Option<SafekeepersInfo>,
375 : }
376 :
377 : /// Safekeepers as returned in timeline creation request to storcon or pushed to
378 : /// cplane in the post migration hook.
379 0 : #[derive(Serialize, Deserialize, Clone)]
380 : pub struct SafekeepersInfo {
381 : pub tenant_id: TenantId,
382 : pub timeline_id: TimelineId,
383 : pub generation: u32,
384 : pub safekeepers: Vec<SafekeeperInfo>,
385 : }
386 :
387 0 : #[derive(Serialize, Deserialize, Clone, Debug)]
388 : pub struct SafekeeperInfo {
389 : pub id: NodeId,
390 : pub hostname: String,
391 : }
392 :
393 0 : #[derive(Serialize, Deserialize, Clone)]
394 : #[serde(untagged)]
395 : pub enum TimelineCreateRequestMode {
396 : Branch {
397 : ancestor_timeline_id: TimelineId,
398 : #[serde(default)]
399 : ancestor_start_lsn: Option<Lsn>,
400 : // TODO: cplane sets this, but, the branching code always
401 : // inherits the ancestor's pg_version. Earlier code wasn't
402 : // using a flattened enum, so, it was an accepted field, and
403 : // we continue to accept it by having it here.
404 : pg_version: Option<PgMajorVersion>,
405 : #[serde(default, skip_serializing_if = "std::ops::Not::not")]
406 : read_only: bool,
407 : },
408 : ImportPgdata {
409 : import_pgdata: TimelineCreateRequestModeImportPgdata,
410 : },
411 : // NB: Bootstrap is all-optional, and thus the serde(untagged) will cause serde to stop at Bootstrap.
412 : // (serde picks the first matching enum variant, in declaration order).
413 : Bootstrap {
414 : #[serde(default)]
415 : existing_initdb_timeline_id: Option<TimelineId>,
416 : pg_version: Option<PgMajorVersion>,
417 : },
418 : }
419 :
420 0 : #[derive(Serialize, Deserialize, Clone)]
421 : pub struct TimelineCreateRequestModeImportPgdata {
422 : pub location: ImportPgdataLocation,
423 : pub idempotency_key: ImportPgdataIdempotencyKey,
424 : }
425 :
426 0 : #[derive(Serialize, Deserialize, Clone, Debug)]
427 : pub enum ImportPgdataLocation {
428 : #[cfg(feature = "testing")]
429 : LocalFs { path: Utf8PathBuf },
430 : AwsS3 {
431 : region: String,
432 : bucket: String,
433 : /// A better name for this would be `prefix`; changing requires coordination with cplane.
434 : /// See <https://github.com/neondatabase/cloud/issues/20646>.
435 : key: String,
436 : },
437 : }
438 :
439 : #[derive(Serialize, Deserialize, Clone)]
440 : #[serde(transparent)]
441 : pub struct ImportPgdataIdempotencyKey(pub String);
442 :
443 : impl ImportPgdataIdempotencyKey {
444 0 : pub fn random() -> Self {
445 : use rand::Rng;
446 : use rand::distr::Alphanumeric;
447 0 : Self(
448 0 : rand::rng()
449 0 : .sample_iter(&Alphanumeric)
450 0 : .take(20)
451 0 : .map(char::from)
452 0 : .collect(),
453 0 : )
454 0 : }
455 : }
456 :
457 0 : #[derive(Serialize, Deserialize, Clone)]
458 : pub struct LsnLeaseRequest {
459 : pub lsn: Lsn,
460 : }
461 :
462 0 : #[derive(Serialize, Deserialize)]
463 : pub struct TenantShardSplitRequest {
464 : pub new_shard_count: u8,
465 :
466 : // A tenant's stripe size is only meaningful the first time their shard count goes
467 : // above 1: therefore during a split from 1->N shards, we may modify the stripe size.
468 : //
469 : // If this is set while the stripe count is being increased from an already >1 value,
470 : // then the request will fail with 400.
471 : pub new_stripe_size: Option<ShardStripeSize>,
472 : }
473 :
474 0 : #[derive(Serialize, Deserialize)]
475 : pub struct TenantShardSplitResponse {
476 : pub new_shards: Vec<TenantShardId>,
477 : }
478 :
479 : /// Parameters that apply to all shards in a tenant. Used during tenant creation.
480 0 : #[derive(Clone, Copy, Serialize, Deserialize, Debug)]
481 : #[serde(deny_unknown_fields)]
482 : pub struct ShardParameters {
483 : pub count: ShardCount,
484 : pub stripe_size: ShardStripeSize,
485 : }
486 :
487 : impl ShardParameters {
488 0 : pub fn is_unsharded(&self) -> bool {
489 0 : self.count.is_unsharded()
490 0 : }
491 : }
492 :
493 : impl Default for ShardParameters {
494 120 : fn default() -> Self {
495 120 : Self {
496 120 : count: ShardCount::new(0),
497 120 : stripe_size: DEFAULT_STRIPE_SIZE,
498 120 : }
499 120 : }
500 : }
501 :
502 : impl From<ShardIdentity> for ShardParameters {
503 0 : fn from(identity: ShardIdentity) -> Self {
504 0 : Self {
505 0 : count: identity.count,
506 0 : stripe_size: identity.stripe_size,
507 0 : }
508 0 : }
509 : }
510 :
511 : #[derive(Debug, Default, Clone, Eq, PartialEq)]
512 : pub enum FieldPatch<T> {
513 : Upsert(T),
514 : Remove,
515 : #[default]
516 : Noop,
517 : }
518 :
519 : impl<T> FieldPatch<T> {
520 80 : fn is_noop(&self) -> bool {
521 80 : matches!(self, FieldPatch::Noop)
522 80 : }
523 :
524 40 : pub fn apply(self, target: &mut Option<T>) {
525 40 : match self {
526 1 : Self::Upsert(v) => *target = Some(v),
527 1 : Self::Remove => *target = None,
528 38 : Self::Noop => {}
529 : }
530 40 : }
531 :
532 11 : pub fn map<U, E, F: FnOnce(T) -> Result<U, E>>(self, map: F) -> Result<FieldPatch<U>, E> {
533 11 : match self {
534 0 : Self::Upsert(v) => Ok(FieldPatch::<U>::Upsert(map(v)?)),
535 0 : Self::Remove => Ok(FieldPatch::<U>::Remove),
536 11 : Self::Noop => Ok(FieldPatch::<U>::Noop),
537 : }
538 11 : }
539 : }
540 :
541 : impl<'de, T: Deserialize<'de>> Deserialize<'de> for FieldPatch<T> {
542 2 : fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
543 2 : where
544 2 : D: Deserializer<'de>,
545 : {
546 2 : Option::deserialize(deserializer).map(|opt| match opt {
547 1 : None => FieldPatch::Remove,
548 1 : Some(val) => FieldPatch::Upsert(val),
549 2 : })
550 2 : }
551 : }
552 :
553 : impl<T: Serialize> Serialize for FieldPatch<T> {
554 2 : fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
555 2 : where
556 2 : S: Serializer,
557 : {
558 2 : match self {
559 1 : FieldPatch::Upsert(val) => serializer.serialize_some(val),
560 1 : FieldPatch::Remove => serializer.serialize_none(),
561 0 : FieldPatch::Noop => unreachable!(),
562 : }
563 2 : }
564 : }
565 :
566 0 : #[derive(Serialize, Deserialize, Debug, Default, Clone, Eq, PartialEq)]
567 : #[serde(default)]
568 : pub struct TenantConfigPatch {
569 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
570 : pub checkpoint_distance: FieldPatch<u64>,
571 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
572 : pub checkpoint_timeout: FieldPatch<String>,
573 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
574 : pub compaction_target_size: FieldPatch<u64>,
575 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
576 : pub compaction_period: FieldPatch<String>,
577 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
578 : pub compaction_threshold: FieldPatch<usize>,
579 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
580 : pub compaction_upper_limit: FieldPatch<usize>,
581 : // defer parsing compaction_algorithm, like eviction_policy
582 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
583 : pub compaction_algorithm: FieldPatch<CompactionAlgorithmSettings>,
584 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
585 : pub compaction_shard_ancestor: FieldPatch<bool>,
586 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
587 : pub compaction_l0_first: FieldPatch<bool>,
588 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
589 : pub compaction_l0_semaphore: FieldPatch<bool>,
590 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
591 : pub l0_flush_delay_threshold: FieldPatch<usize>,
592 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
593 : pub l0_flush_stall_threshold: FieldPatch<usize>,
594 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
595 : pub gc_horizon: FieldPatch<u64>,
596 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
597 : pub gc_period: FieldPatch<String>,
598 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
599 : pub image_creation_threshold: FieldPatch<usize>,
600 : // HADRON
601 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
602 : pub image_layer_force_creation_period: FieldPatch<String>,
603 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
604 : pub pitr_interval: FieldPatch<String>,
605 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
606 : pub walreceiver_connect_timeout: FieldPatch<String>,
607 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
608 : pub lagging_wal_timeout: FieldPatch<String>,
609 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
610 : pub max_lsn_wal_lag: FieldPatch<NonZeroU64>,
611 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
612 : pub eviction_policy: FieldPatch<EvictionPolicy>,
613 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
614 : pub min_resident_size_override: FieldPatch<u64>,
615 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
616 : pub evictions_low_residence_duration_metric_threshold: FieldPatch<String>,
617 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
618 : pub heatmap_period: FieldPatch<String>,
619 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
620 : pub lazy_slru_download: FieldPatch<bool>,
621 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
622 : pub timeline_get_throttle: FieldPatch<ThrottleConfig>,
623 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
624 : pub image_layer_creation_check_threshold: FieldPatch<u8>,
625 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
626 : pub image_creation_preempt_threshold: FieldPatch<usize>,
627 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
628 : pub lsn_lease_length: FieldPatch<String>,
629 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
630 : pub lsn_lease_length_for_ts: FieldPatch<String>,
631 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
632 : pub timeline_offloading: FieldPatch<bool>,
633 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
634 : pub rel_size_v2_enabled: FieldPatch<bool>,
635 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
636 : pub gc_compaction_enabled: FieldPatch<bool>,
637 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
638 : pub gc_compaction_verification: FieldPatch<bool>,
639 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
640 : pub gc_compaction_initial_threshold_kb: FieldPatch<u64>,
641 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
642 : pub gc_compaction_ratio_percent: FieldPatch<u64>,
643 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
644 : pub sampling_ratio: FieldPatch<Option<Ratio>>,
645 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
646 : pub relsize_snapshot_cache_capacity: FieldPatch<usize>,
647 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
648 : pub basebackup_cache_enabled: FieldPatch<bool>,
649 : #[serde(skip_serializing_if = "FieldPatch::is_noop")]
650 : pub rel_size_v1_access_disabled: FieldPatch<bool>,
651 : }
652 :
653 : /// Like [`crate::config::TenantConfigToml`], but preserves the information
654 : /// about which parameters are set and which are not.
655 : ///
656 : /// Used in many places, including durably stored ones.
657 : #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
658 : #[serde(default)] // this maps omitted fields in deserialization to None
659 : pub struct TenantConfig {
660 : #[serde(skip_serializing_if = "Option::is_none")]
661 : pub checkpoint_distance: Option<u64>,
662 :
663 : #[serde(skip_serializing_if = "Option::is_none")]
664 : #[serde(with = "humantime_serde")]
665 : pub checkpoint_timeout: Option<Duration>,
666 :
667 : #[serde(skip_serializing_if = "Option::is_none")]
668 : pub compaction_target_size: Option<u64>,
669 :
670 : #[serde(skip_serializing_if = "Option::is_none")]
671 : #[serde(with = "humantime_serde")]
672 : pub compaction_period: Option<Duration>,
673 :
674 : #[serde(skip_serializing_if = "Option::is_none")]
675 : pub compaction_threshold: Option<usize>,
676 :
677 : #[serde(skip_serializing_if = "Option::is_none")]
678 : pub compaction_upper_limit: Option<usize>,
679 :
680 : #[serde(skip_serializing_if = "Option::is_none")]
681 : pub compaction_algorithm: Option<CompactionAlgorithmSettings>,
682 :
683 : #[serde(skip_serializing_if = "Option::is_none")]
684 : pub compaction_shard_ancestor: Option<bool>,
685 :
686 : #[serde(skip_serializing_if = "Option::is_none")]
687 : pub compaction_l0_first: Option<bool>,
688 :
689 : #[serde(skip_serializing_if = "Option::is_none")]
690 : pub compaction_l0_semaphore: Option<bool>,
691 :
692 : #[serde(skip_serializing_if = "Option::is_none")]
693 : pub l0_flush_delay_threshold: Option<usize>,
694 :
695 : #[serde(skip_serializing_if = "Option::is_none")]
696 : pub l0_flush_stall_threshold: Option<usize>,
697 :
698 : #[serde(skip_serializing_if = "Option::is_none")]
699 : pub gc_horizon: Option<u64>,
700 :
701 : #[serde(skip_serializing_if = "Option::is_none")]
702 : #[serde(with = "humantime_serde")]
703 : pub gc_period: Option<Duration>,
704 :
705 : #[serde(skip_serializing_if = "Option::is_none")]
706 : pub image_creation_threshold: Option<usize>,
707 :
708 : // HADRON
709 : #[serde(skip_serializing_if = "Option::is_none")]
710 : #[serde(with = "humantime_serde")]
711 : pub image_layer_force_creation_period: Option<Duration>,
712 :
713 : #[serde(skip_serializing_if = "Option::is_none")]
714 : #[serde(with = "humantime_serde")]
715 : pub pitr_interval: Option<Duration>,
716 :
717 : #[serde(skip_serializing_if = "Option::is_none")]
718 : #[serde(with = "humantime_serde")]
719 : pub walreceiver_connect_timeout: Option<Duration>,
720 :
721 : #[serde(skip_serializing_if = "Option::is_none")]
722 : #[serde(with = "humantime_serde")]
723 : pub lagging_wal_timeout: Option<Duration>,
724 :
725 : #[serde(skip_serializing_if = "Option::is_none")]
726 : pub max_lsn_wal_lag: Option<NonZeroU64>,
727 :
728 : #[serde(skip_serializing_if = "Option::is_none")]
729 : pub eviction_policy: Option<EvictionPolicy>,
730 :
731 : #[serde(skip_serializing_if = "Option::is_none")]
732 : pub min_resident_size_override: Option<u64>,
733 :
734 : #[serde(skip_serializing_if = "Option::is_none")]
735 : #[serde(with = "humantime_serde")]
736 : pub evictions_low_residence_duration_metric_threshold: Option<Duration>,
737 :
738 : #[serde(skip_serializing_if = "Option::is_none")]
739 : #[serde(with = "humantime_serde")]
740 : pub heatmap_period: Option<Duration>,
741 :
742 : #[serde(skip_serializing_if = "Option::is_none")]
743 : pub lazy_slru_download: Option<bool>,
744 :
745 : #[serde(skip_serializing_if = "Option::is_none")]
746 : pub timeline_get_throttle: Option<ThrottleConfig>,
747 :
748 : #[serde(skip_serializing_if = "Option::is_none")]
749 : pub image_layer_creation_check_threshold: Option<u8>,
750 :
751 : #[serde(skip_serializing_if = "Option::is_none")]
752 : pub image_creation_preempt_threshold: Option<usize>,
753 :
754 : #[serde(skip_serializing_if = "Option::is_none")]
755 : #[serde(with = "humantime_serde")]
756 : pub lsn_lease_length: Option<Duration>,
757 :
758 : #[serde(skip_serializing_if = "Option::is_none")]
759 : #[serde(with = "humantime_serde")]
760 : pub lsn_lease_length_for_ts: Option<Duration>,
761 :
762 : #[serde(skip_serializing_if = "Option::is_none")]
763 : pub timeline_offloading: Option<bool>,
764 :
765 : #[serde(skip_serializing_if = "Option::is_none")]
766 : pub rel_size_v2_enabled: Option<bool>,
767 :
768 : #[serde(skip_serializing_if = "Option::is_none")]
769 : pub gc_compaction_enabled: Option<bool>,
770 :
771 : #[serde(skip_serializing_if = "Option::is_none")]
772 : pub gc_compaction_verification: Option<bool>,
773 :
774 : #[serde(skip_serializing_if = "Option::is_none")]
775 : pub gc_compaction_initial_threshold_kb: Option<u64>,
776 :
777 : #[serde(skip_serializing_if = "Option::is_none")]
778 : pub gc_compaction_ratio_percent: Option<u64>,
779 :
780 : #[serde(skip_serializing_if = "Option::is_none")]
781 : pub sampling_ratio: Option<Option<Ratio>>,
782 :
783 : #[serde(skip_serializing_if = "Option::is_none")]
784 : pub relsize_snapshot_cache_capacity: Option<usize>,
785 :
786 : #[serde(skip_serializing_if = "Option::is_none")]
787 : pub basebackup_cache_enabled: Option<bool>,
788 :
789 : #[serde(skip_serializing_if = "Option::is_none")]
790 : pub rel_size_v1_access_disabled: Option<bool>,
791 : }
792 :
793 : impl TenantConfig {
794 1 : pub fn apply_patch(
795 1 : self,
796 1 : patch: TenantConfigPatch,
797 1 : ) -> Result<TenantConfig, humantime::DurationError> {
798 : let Self {
799 1 : mut checkpoint_distance,
800 1 : mut checkpoint_timeout,
801 1 : mut compaction_target_size,
802 1 : mut compaction_period,
803 1 : mut compaction_threshold,
804 1 : mut compaction_upper_limit,
805 1 : mut compaction_algorithm,
806 1 : mut compaction_shard_ancestor,
807 1 : mut compaction_l0_first,
808 1 : mut compaction_l0_semaphore,
809 1 : mut l0_flush_delay_threshold,
810 1 : mut l0_flush_stall_threshold,
811 1 : mut gc_horizon,
812 1 : mut gc_period,
813 1 : mut image_creation_threshold,
814 1 : mut image_layer_force_creation_period,
815 1 : mut pitr_interval,
816 1 : mut walreceiver_connect_timeout,
817 1 : mut lagging_wal_timeout,
818 1 : mut max_lsn_wal_lag,
819 1 : mut eviction_policy,
820 1 : mut min_resident_size_override,
821 1 : mut evictions_low_residence_duration_metric_threshold,
822 1 : mut heatmap_period,
823 1 : mut lazy_slru_download,
824 1 : mut timeline_get_throttle,
825 1 : mut image_layer_creation_check_threshold,
826 1 : mut image_creation_preempt_threshold,
827 1 : mut lsn_lease_length,
828 1 : mut lsn_lease_length_for_ts,
829 1 : mut timeline_offloading,
830 1 : mut rel_size_v2_enabled,
831 1 : mut gc_compaction_enabled,
832 1 : mut gc_compaction_verification,
833 1 : mut gc_compaction_initial_threshold_kb,
834 1 : mut gc_compaction_ratio_percent,
835 1 : mut sampling_ratio,
836 1 : mut relsize_snapshot_cache_capacity,
837 1 : mut basebackup_cache_enabled,
838 1 : mut rel_size_v1_access_disabled,
839 1 : } = self;
840 :
841 1 : patch.checkpoint_distance.apply(&mut checkpoint_distance);
842 1 : patch
843 1 : .checkpoint_timeout
844 1 : .map(|v| humantime::parse_duration(&v))?
845 1 : .apply(&mut checkpoint_timeout);
846 1 : patch
847 1 : .compaction_target_size
848 1 : .apply(&mut compaction_target_size);
849 1 : patch
850 1 : .compaction_period
851 1 : .map(|v| humantime::parse_duration(&v))?
852 1 : .apply(&mut compaction_period);
853 1 : patch.compaction_threshold.apply(&mut compaction_threshold);
854 1 : patch
855 1 : .compaction_upper_limit
856 1 : .apply(&mut compaction_upper_limit);
857 1 : patch.compaction_algorithm.apply(&mut compaction_algorithm);
858 1 : patch
859 1 : .compaction_shard_ancestor
860 1 : .apply(&mut compaction_shard_ancestor);
861 1 : patch.compaction_l0_first.apply(&mut compaction_l0_first);
862 1 : patch
863 1 : .compaction_l0_semaphore
864 1 : .apply(&mut compaction_l0_semaphore);
865 1 : patch
866 1 : .l0_flush_delay_threshold
867 1 : .apply(&mut l0_flush_delay_threshold);
868 1 : patch
869 1 : .l0_flush_stall_threshold
870 1 : .apply(&mut l0_flush_stall_threshold);
871 1 : patch.gc_horizon.apply(&mut gc_horizon);
872 1 : patch
873 1 : .gc_period
874 1 : .map(|v| humantime::parse_duration(&v))?
875 1 : .apply(&mut gc_period);
876 1 : patch
877 1 : .image_creation_threshold
878 1 : .apply(&mut image_creation_threshold);
879 : // HADRON
880 1 : patch
881 1 : .image_layer_force_creation_period
882 1 : .map(|v| humantime::parse_duration(&v))?
883 1 : .apply(&mut image_layer_force_creation_period);
884 1 : patch
885 1 : .pitr_interval
886 1 : .map(|v| humantime::parse_duration(&v))?
887 1 : .apply(&mut pitr_interval);
888 1 : patch
889 1 : .walreceiver_connect_timeout
890 1 : .map(|v| humantime::parse_duration(&v))?
891 1 : .apply(&mut walreceiver_connect_timeout);
892 1 : patch
893 1 : .lagging_wal_timeout
894 1 : .map(|v| humantime::parse_duration(&v))?
895 1 : .apply(&mut lagging_wal_timeout);
896 1 : patch.max_lsn_wal_lag.apply(&mut max_lsn_wal_lag);
897 1 : patch.eviction_policy.apply(&mut eviction_policy);
898 1 : patch
899 1 : .min_resident_size_override
900 1 : .apply(&mut min_resident_size_override);
901 1 : patch
902 1 : .evictions_low_residence_duration_metric_threshold
903 1 : .map(|v| humantime::parse_duration(&v))?
904 1 : .apply(&mut evictions_low_residence_duration_metric_threshold);
905 1 : patch
906 1 : .heatmap_period
907 1 : .map(|v| humantime::parse_duration(&v))?
908 1 : .apply(&mut heatmap_period);
909 1 : patch.lazy_slru_download.apply(&mut lazy_slru_download);
910 1 : patch
911 1 : .timeline_get_throttle
912 1 : .apply(&mut timeline_get_throttle);
913 1 : patch
914 1 : .image_layer_creation_check_threshold
915 1 : .apply(&mut image_layer_creation_check_threshold);
916 1 : patch
917 1 : .image_creation_preempt_threshold
918 1 : .apply(&mut image_creation_preempt_threshold);
919 1 : patch
920 1 : .lsn_lease_length
921 1 : .map(|v| humantime::parse_duration(&v))?
922 1 : .apply(&mut lsn_lease_length);
923 1 : patch
924 1 : .lsn_lease_length_for_ts
925 1 : .map(|v| humantime::parse_duration(&v))?
926 1 : .apply(&mut lsn_lease_length_for_ts);
927 1 : patch.timeline_offloading.apply(&mut timeline_offloading);
928 1 : patch.rel_size_v2_enabled.apply(&mut rel_size_v2_enabled);
929 1 : patch
930 1 : .gc_compaction_enabled
931 1 : .apply(&mut gc_compaction_enabled);
932 1 : patch
933 1 : .gc_compaction_verification
934 1 : .apply(&mut gc_compaction_verification);
935 1 : patch
936 1 : .gc_compaction_initial_threshold_kb
937 1 : .apply(&mut gc_compaction_initial_threshold_kb);
938 1 : patch
939 1 : .gc_compaction_ratio_percent
940 1 : .apply(&mut gc_compaction_ratio_percent);
941 1 : patch.sampling_ratio.apply(&mut sampling_ratio);
942 1 : patch
943 1 : .relsize_snapshot_cache_capacity
944 1 : .apply(&mut relsize_snapshot_cache_capacity);
945 1 : patch
946 1 : .basebackup_cache_enabled
947 1 : .apply(&mut basebackup_cache_enabled);
948 1 : patch
949 1 : .rel_size_v1_access_disabled
950 1 : .apply(&mut rel_size_v1_access_disabled);
951 :
952 1 : Ok(Self {
953 1 : checkpoint_distance,
954 1 : checkpoint_timeout,
955 1 : compaction_target_size,
956 1 : compaction_period,
957 1 : compaction_threshold,
958 1 : compaction_upper_limit,
959 1 : compaction_algorithm,
960 1 : compaction_shard_ancestor,
961 1 : compaction_l0_first,
962 1 : compaction_l0_semaphore,
963 1 : l0_flush_delay_threshold,
964 1 : l0_flush_stall_threshold,
965 1 : gc_horizon,
966 1 : gc_period,
967 1 : image_creation_threshold,
968 1 : image_layer_force_creation_period,
969 1 : pitr_interval,
970 1 : walreceiver_connect_timeout,
971 1 : lagging_wal_timeout,
972 1 : max_lsn_wal_lag,
973 1 : eviction_policy,
974 1 : min_resident_size_override,
975 1 : evictions_low_residence_duration_metric_threshold,
976 1 : heatmap_period,
977 1 : lazy_slru_download,
978 1 : timeline_get_throttle,
979 1 : image_layer_creation_check_threshold,
980 1 : image_creation_preempt_threshold,
981 1 : lsn_lease_length,
982 1 : lsn_lease_length_for_ts,
983 1 : timeline_offloading,
984 1 : rel_size_v2_enabled,
985 1 : gc_compaction_enabled,
986 1 : gc_compaction_verification,
987 1 : gc_compaction_initial_threshold_kb,
988 1 : gc_compaction_ratio_percent,
989 1 : sampling_ratio,
990 1 : relsize_snapshot_cache_capacity,
991 1 : basebackup_cache_enabled,
992 1 : rel_size_v1_access_disabled,
993 1 : })
994 1 : }
995 :
996 0 : pub fn merge(
997 0 : &self,
998 0 : global_conf: crate::config::TenantConfigToml,
999 0 : ) -> crate::config::TenantConfigToml {
1000 0 : crate::config::TenantConfigToml {
1001 0 : checkpoint_distance: self
1002 0 : .checkpoint_distance
1003 0 : .unwrap_or(global_conf.checkpoint_distance),
1004 0 : checkpoint_timeout: self
1005 0 : .checkpoint_timeout
1006 0 : .unwrap_or(global_conf.checkpoint_timeout),
1007 0 : compaction_target_size: self
1008 0 : .compaction_target_size
1009 0 : .unwrap_or(global_conf.compaction_target_size),
1010 0 : compaction_period: self
1011 0 : .compaction_period
1012 0 : .unwrap_or(global_conf.compaction_period),
1013 0 : compaction_threshold: self
1014 0 : .compaction_threshold
1015 0 : .unwrap_or(global_conf.compaction_threshold),
1016 0 : compaction_upper_limit: self
1017 0 : .compaction_upper_limit
1018 0 : .unwrap_or(global_conf.compaction_upper_limit),
1019 0 : compaction_algorithm: self
1020 0 : .compaction_algorithm
1021 0 : .as_ref()
1022 0 : .unwrap_or(&global_conf.compaction_algorithm)
1023 0 : .clone(),
1024 0 : compaction_shard_ancestor: self
1025 0 : .compaction_shard_ancestor
1026 0 : .unwrap_or(global_conf.compaction_shard_ancestor),
1027 0 : compaction_l0_first: self
1028 0 : .compaction_l0_first
1029 0 : .unwrap_or(global_conf.compaction_l0_first),
1030 0 : compaction_l0_semaphore: self
1031 0 : .compaction_l0_semaphore
1032 0 : .unwrap_or(global_conf.compaction_l0_semaphore),
1033 0 : l0_flush_delay_threshold: self
1034 0 : .l0_flush_delay_threshold
1035 0 : .or(global_conf.l0_flush_delay_threshold),
1036 0 : l0_flush_stall_threshold: self
1037 0 : .l0_flush_stall_threshold
1038 0 : .or(global_conf.l0_flush_stall_threshold),
1039 0 : gc_horizon: self.gc_horizon.unwrap_or(global_conf.gc_horizon),
1040 0 : gc_period: self.gc_period.unwrap_or(global_conf.gc_period),
1041 0 : image_creation_threshold: self
1042 0 : .image_creation_threshold
1043 0 : .unwrap_or(global_conf.image_creation_threshold),
1044 0 : image_layer_force_creation_period: self
1045 0 : .image_layer_force_creation_period
1046 0 : .or(global_conf.image_layer_force_creation_period),
1047 0 : pitr_interval: self.pitr_interval.unwrap_or(global_conf.pitr_interval),
1048 0 : walreceiver_connect_timeout: self
1049 0 : .walreceiver_connect_timeout
1050 0 : .unwrap_or(global_conf.walreceiver_connect_timeout),
1051 0 : lagging_wal_timeout: self
1052 0 : .lagging_wal_timeout
1053 0 : .unwrap_or(global_conf.lagging_wal_timeout),
1054 0 : max_lsn_wal_lag: self.max_lsn_wal_lag.unwrap_or(global_conf.max_lsn_wal_lag),
1055 0 : eviction_policy: self.eviction_policy.unwrap_or(global_conf.eviction_policy),
1056 0 : min_resident_size_override: self
1057 0 : .min_resident_size_override
1058 0 : .or(global_conf.min_resident_size_override),
1059 0 : evictions_low_residence_duration_metric_threshold: self
1060 0 : .evictions_low_residence_duration_metric_threshold
1061 0 : .unwrap_or(global_conf.evictions_low_residence_duration_metric_threshold),
1062 0 : heatmap_period: self.heatmap_period.unwrap_or(global_conf.heatmap_period),
1063 0 : lazy_slru_download: self
1064 0 : .lazy_slru_download
1065 0 : .unwrap_or(global_conf.lazy_slru_download),
1066 0 : timeline_get_throttle: self
1067 0 : .timeline_get_throttle
1068 0 : .clone()
1069 0 : .unwrap_or(global_conf.timeline_get_throttle),
1070 0 : image_layer_creation_check_threshold: self
1071 0 : .image_layer_creation_check_threshold
1072 0 : .unwrap_or(global_conf.image_layer_creation_check_threshold),
1073 0 : image_creation_preempt_threshold: self
1074 0 : .image_creation_preempt_threshold
1075 0 : .unwrap_or(global_conf.image_creation_preempt_threshold),
1076 0 : lsn_lease_length: self
1077 0 : .lsn_lease_length
1078 0 : .unwrap_or(global_conf.lsn_lease_length),
1079 0 : lsn_lease_length_for_ts: self
1080 0 : .lsn_lease_length_for_ts
1081 0 : .unwrap_or(global_conf.lsn_lease_length_for_ts),
1082 0 : timeline_offloading: self
1083 0 : .timeline_offloading
1084 0 : .unwrap_or(global_conf.timeline_offloading),
1085 0 : rel_size_v2_enabled: self
1086 0 : .rel_size_v2_enabled
1087 0 : .unwrap_or(global_conf.rel_size_v2_enabled),
1088 0 : gc_compaction_enabled: self
1089 0 : .gc_compaction_enabled
1090 0 : .unwrap_or(global_conf.gc_compaction_enabled),
1091 0 : gc_compaction_verification: self
1092 0 : .gc_compaction_verification
1093 0 : .unwrap_or(global_conf.gc_compaction_verification),
1094 0 : gc_compaction_initial_threshold_kb: self
1095 0 : .gc_compaction_initial_threshold_kb
1096 0 : .unwrap_or(global_conf.gc_compaction_initial_threshold_kb),
1097 0 : gc_compaction_ratio_percent: self
1098 0 : .gc_compaction_ratio_percent
1099 0 : .unwrap_or(global_conf.gc_compaction_ratio_percent),
1100 0 : sampling_ratio: self.sampling_ratio.unwrap_or(global_conf.sampling_ratio),
1101 0 : relsize_snapshot_cache_capacity: self
1102 0 : .relsize_snapshot_cache_capacity
1103 0 : .unwrap_or(global_conf.relsize_snapshot_cache_capacity),
1104 0 : basebackup_cache_enabled: self
1105 0 : .basebackup_cache_enabled
1106 0 : .unwrap_or(global_conf.basebackup_cache_enabled),
1107 0 : rel_size_v1_access_disabled: self
1108 0 : .rel_size_v1_access_disabled
1109 0 : .unwrap_or(global_conf.rel_size_v1_access_disabled),
1110 0 : }
1111 0 : }
1112 : }
1113 :
1114 : /// The policy for the aux file storage.
1115 : ///
1116 : /// It can be switched through `switch_aux_file_policy` tenant config.
1117 : /// When the first aux file written, the policy will be persisted in the
1118 : /// `index_part.json` file and has a limited migration path.
1119 : ///
1120 : /// Currently, we only allow the following migration path:
1121 : ///
1122 : /// Unset -> V1
1123 : /// -> V2
1124 : /// -> CrossValidation -> V2
1125 : #[derive(
1126 : Eq,
1127 : PartialEq,
1128 : Debug,
1129 : Copy,
1130 : Clone,
1131 : strum_macros::EnumString,
1132 : strum_macros::Display,
1133 0 : serde_with::DeserializeFromStr,
1134 : serde_with::SerializeDisplay,
1135 : )]
1136 : #[strum(serialize_all = "kebab-case")]
1137 : pub enum AuxFilePolicy {
1138 : /// V1 aux file policy: store everything in AUX_FILE_KEY
1139 : #[strum(ascii_case_insensitive)]
1140 : V1,
1141 : /// V2 aux file policy: store in the AUX_FILE keyspace
1142 : #[strum(ascii_case_insensitive)]
1143 : V2,
1144 : /// Cross validation runs both formats on the write path and does validation
1145 : /// on the read path.
1146 : #[strum(ascii_case_insensitive)]
1147 : CrossValidation,
1148 : }
1149 :
1150 : #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
1151 : #[serde(tag = "kind")]
1152 : pub enum EvictionPolicy {
1153 : NoEviction,
1154 : LayerAccessThreshold(EvictionPolicyLayerAccessThreshold),
1155 : OnlyImitiate(EvictionPolicyLayerAccessThreshold),
1156 : }
1157 :
1158 : impl EvictionPolicy {
1159 0 : pub fn discriminant_str(&self) -> &'static str {
1160 0 : match self {
1161 0 : EvictionPolicy::NoEviction => "NoEviction",
1162 0 : EvictionPolicy::LayerAccessThreshold(_) => "LayerAccessThreshold",
1163 0 : EvictionPolicy::OnlyImitiate(_) => "OnlyImitiate",
1164 : }
1165 0 : }
1166 : }
1167 :
1168 : #[derive(
1169 : Eq,
1170 : PartialEq,
1171 : Debug,
1172 : Copy,
1173 : Clone,
1174 : strum_macros::EnumString,
1175 : strum_macros::Display,
1176 0 : serde_with::DeserializeFromStr,
1177 : serde_with::SerializeDisplay,
1178 : )]
1179 : #[strum(serialize_all = "kebab-case")]
1180 : pub enum CompactionAlgorithm {
1181 : Legacy,
1182 : Tiered,
1183 : }
1184 :
1185 : #[derive(
1186 0 : Debug, Clone, Copy, PartialEq, Eq, serde_with::DeserializeFromStr, serde_with::SerializeDisplay,
1187 : )]
1188 : pub enum ImageCompressionAlgorithm {
1189 : // Disabled for writes, support decompressing during read path
1190 : Disabled,
1191 : /// Zstandard compression. Level 0 means and None mean the same (default level). Levels can be negative as well.
1192 : /// For details, see the [manual](http://facebook.github.io/zstd/zstd_manual.html).
1193 : Zstd {
1194 : level: Option<i8>,
1195 : },
1196 : }
1197 :
1198 : impl FromStr for ImageCompressionAlgorithm {
1199 : type Err = anyhow::Error;
1200 8 : fn from_str(s: &str) -> Result<Self, Self::Err> {
1201 8 : let mut components = s.split(['(', ')']);
1202 8 : let first = components
1203 8 : .next()
1204 8 : .ok_or_else(|| anyhow::anyhow!("empty string"))?;
1205 8 : match first {
1206 8 : "disabled" => Ok(ImageCompressionAlgorithm::Disabled),
1207 6 : "zstd" => {
1208 6 : let level = if let Some(v) = components.next() {
1209 4 : let v: i8 = v.parse()?;
1210 4 : Some(v)
1211 : } else {
1212 2 : None
1213 : };
1214 :
1215 6 : Ok(ImageCompressionAlgorithm::Zstd { level })
1216 : }
1217 0 : _ => anyhow::bail!("invalid specifier '{first}'"),
1218 : }
1219 8 : }
1220 : }
1221 :
1222 : impl Display for ImageCompressionAlgorithm {
1223 12 : fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1224 12 : match self {
1225 3 : ImageCompressionAlgorithm::Disabled => write!(f, "disabled"),
1226 9 : ImageCompressionAlgorithm::Zstd { level } => {
1227 9 : if let Some(level) = level {
1228 6 : write!(f, "zstd({level})")
1229 : } else {
1230 3 : write!(f, "zstd")
1231 : }
1232 : }
1233 : }
1234 12 : }
1235 : }
1236 :
1237 0 : #[derive(Eq, PartialEq, Debug, Clone, Serialize, Deserialize)]
1238 : pub struct CompactionAlgorithmSettings {
1239 : pub kind: CompactionAlgorithm,
1240 : }
1241 :
1242 0 : #[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)]
1243 : #[serde(tag = "mode", rename_all = "kebab-case")]
1244 : pub enum L0FlushConfig {
1245 : #[serde(rename_all = "snake_case")]
1246 : Direct { max_concurrency: NonZeroUsize },
1247 : }
1248 :
1249 : #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
1250 : pub struct EvictionPolicyLayerAccessThreshold {
1251 : #[serde(with = "humantime_serde")]
1252 : pub period: Duration,
1253 : #[serde(with = "humantime_serde")]
1254 : pub threshold: Duration,
1255 : }
1256 :
1257 : #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
1258 : pub struct ThrottleConfig {
1259 : /// See [`ThrottleConfigTaskKinds`] for why we do the serde `rename`.
1260 : #[serde(rename = "task_kinds")]
1261 : pub enabled: ThrottleConfigTaskKinds,
1262 : pub initial: u32,
1263 : #[serde(with = "humantime_serde")]
1264 : pub refill_interval: Duration,
1265 : pub refill_amount: NonZeroU32,
1266 : pub max: u32,
1267 : }
1268 :
1269 : /// Before <https://github.com/neondatabase/neon/pull/9962>
1270 : /// the throttle was a per `Timeline::get`/`Timeline::get_vectored` call.
1271 : /// The `task_kinds` field controlled which Pageserver "Task Kind"s
1272 : /// were subject to the throttle.
1273 : ///
1274 : /// After that PR, the throttle is applied at pagestream request level
1275 : /// and the `task_kinds` field does not apply since the only task kind
1276 : /// that us subject to the throttle is that of the page service.
1277 : ///
1278 : /// However, we don't want to make a breaking config change right now
1279 : /// because it means we have to migrate all the tenant configs.
1280 : /// This will be done in a future PR.
1281 : ///
1282 : /// In the meantime, we use emptiness / non-emptsiness of the `task_kinds`
1283 : /// field to determine if the throttle is enabled or not.
1284 : #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
1285 : #[serde(transparent)]
1286 : pub struct ThrottleConfigTaskKinds(Vec<String>);
1287 :
1288 : impl ThrottleConfigTaskKinds {
1289 139 : pub fn disabled() -> Self {
1290 139 : Self(vec![])
1291 139 : }
1292 122 : pub fn is_enabled(&self) -> bool {
1293 122 : !self.0.is_empty()
1294 122 : }
1295 : }
1296 :
1297 : impl ThrottleConfig {
1298 139 : pub fn disabled() -> Self {
1299 139 : Self {
1300 139 : enabled: ThrottleConfigTaskKinds::disabled(),
1301 139 : // other values don't matter with emtpy `task_kinds`.
1302 139 : initial: 0,
1303 139 : refill_interval: Duration::from_millis(1),
1304 139 : refill_amount: NonZeroU32::new(1).unwrap(),
1305 139 : max: 1,
1306 139 : }
1307 139 : }
1308 : /// The requests per second allowed by the given config.
1309 0 : pub fn steady_rps(&self) -> f64 {
1310 0 : (self.refill_amount.get() as f64) / (self.refill_interval.as_secs_f64())
1311 0 : }
1312 : }
1313 :
1314 : #[cfg(test)]
1315 : mod throttle_config_tests {
1316 : use super::*;
1317 :
1318 : #[test]
1319 1 : fn test_disabled_is_disabled() {
1320 1 : let config = ThrottleConfig::disabled();
1321 1 : assert!(!config.enabled.is_enabled());
1322 1 : }
1323 : #[test]
1324 1 : fn test_enabled_backwards_compat() {
1325 1 : let input = serde_json::json!({
1326 1 : "task_kinds": ["PageRequestHandler"],
1327 1 : "initial": 40000,
1328 1 : "refill_interval": "50ms",
1329 1 : "refill_amount": 1000,
1330 1 : "max": 40000,
1331 1 : "fair": true
1332 : });
1333 1 : let config: ThrottleConfig = serde_json::from_value(input).unwrap();
1334 1 : assert!(config.enabled.is_enabled());
1335 1 : }
1336 : }
1337 :
1338 : /// A flattened analog of a `pagesever::tenant::LocationMode`, which
1339 : /// lists out all possible states (and the virtual "Detached" state)
1340 : /// in a flat form rather than using rust-style enums.
1341 0 : #[derive(Serialize, Deserialize, Debug, Clone, Copy, Eq, PartialEq)]
1342 : pub enum LocationConfigMode {
1343 : AttachedSingle,
1344 : AttachedMulti,
1345 : AttachedStale,
1346 : Secondary,
1347 : Detached,
1348 : }
1349 :
1350 0 : #[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq)]
1351 : pub struct LocationConfigSecondary {
1352 : pub warm: bool,
1353 : }
1354 :
1355 : /// An alternative representation of `pageserver::tenant::LocationConf`,
1356 : /// for use in external-facing APIs.
1357 0 : #[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq)]
1358 : pub struct LocationConfig {
1359 : pub mode: LocationConfigMode,
1360 : /// If attaching, in what generation?
1361 : #[serde(default)]
1362 : pub generation: Option<u32>,
1363 :
1364 : // If requesting mode `Secondary`, configuration for that.
1365 : #[serde(default)]
1366 : pub secondary_conf: Option<LocationConfigSecondary>,
1367 :
1368 : // Shard parameters: if shard_count is nonzero, then other shard_* fields
1369 : // must be set accurately.
1370 : #[serde(default)]
1371 : pub shard_number: u8,
1372 : #[serde(default)]
1373 : pub shard_count: u8,
1374 : #[serde(default)]
1375 : pub shard_stripe_size: u32,
1376 :
1377 : // This configuration only affects attached mode, but should be provided irrespective
1378 : // of the mode, as a secondary location might transition on startup if the response
1379 : // to the `/re-attach` control plane API requests it.
1380 : pub tenant_conf: TenantConfig,
1381 : }
1382 :
1383 0 : #[derive(Serialize, Deserialize)]
1384 : pub struct LocationConfigListResponse {
1385 : pub tenant_shards: Vec<(TenantShardId, Option<LocationConfig>)>,
1386 : }
1387 :
1388 : #[derive(Serialize)]
1389 : pub struct StatusResponse {
1390 : pub id: NodeId,
1391 : }
1392 :
1393 0 : #[derive(Serialize, Deserialize, Debug)]
1394 : #[serde(deny_unknown_fields)]
1395 : pub struct TenantLocationConfigRequest {
1396 : #[serde(flatten)]
1397 : pub config: LocationConfig, // as we have a flattened field, we should reject all unknown fields in it
1398 : }
1399 :
1400 0 : #[derive(Serialize, Deserialize, Debug)]
1401 : #[serde(deny_unknown_fields)]
1402 : pub struct TenantTimeTravelRequest {
1403 : pub shard_counts: Vec<ShardCount>,
1404 : }
1405 :
1406 0 : #[derive(Serialize, Deserialize, Debug)]
1407 : #[serde(deny_unknown_fields)]
1408 : pub struct TenantShardLocation {
1409 : pub shard_id: TenantShardId,
1410 : pub node_id: NodeId,
1411 : }
1412 :
1413 0 : #[derive(Serialize, Deserialize, Debug)]
1414 : #[serde(deny_unknown_fields)]
1415 : pub struct TenantLocationConfigResponse {
1416 : pub shards: Vec<TenantShardLocation>,
1417 : // If the shards' ShardCount count is >1, stripe_size will be set.
1418 : pub stripe_size: Option<ShardStripeSize>,
1419 : }
1420 :
1421 0 : #[derive(Serialize, Deserialize, Debug)]
1422 : #[serde(deny_unknown_fields)]
1423 : pub struct TenantConfigRequest {
1424 : pub tenant_id: TenantId,
1425 : #[serde(flatten)]
1426 : pub config: TenantConfig, // as we have a flattened field, we should reject all unknown fields in it
1427 : }
1428 :
1429 : impl std::ops::Deref for TenantConfigRequest {
1430 : type Target = TenantConfig;
1431 :
1432 0 : fn deref(&self) -> &Self::Target {
1433 0 : &self.config
1434 0 : }
1435 : }
1436 :
1437 : impl TenantConfigRequest {
1438 0 : pub fn new(tenant_id: TenantId) -> TenantConfigRequest {
1439 0 : let config = TenantConfig::default();
1440 0 : TenantConfigRequest { tenant_id, config }
1441 0 : }
1442 : }
1443 :
1444 0 : #[derive(Serialize, Deserialize, Debug)]
1445 : #[serde(deny_unknown_fields)]
1446 : pub struct TenantConfigPatchRequest {
1447 : pub tenant_id: TenantId,
1448 : #[serde(flatten)]
1449 : pub config: TenantConfigPatch, // as we have a flattened field, we should reject all unknown fields in it
1450 : }
1451 :
1452 0 : #[derive(Serialize, Deserialize, Debug)]
1453 : pub struct TenantWaitLsnRequest {
1454 : #[serde(flatten)]
1455 : pub timelines: HashMap<TimelineId, Lsn>,
1456 : pub timeout: Duration,
1457 : }
1458 :
1459 : /// See [`TenantState::attachment_status`] and the OpenAPI docs for context.
1460 0 : #[derive(Serialize, Deserialize, Clone)]
1461 : #[serde(tag = "slug", content = "data", rename_all = "snake_case")]
1462 : pub enum TenantAttachmentStatus {
1463 : Maybe,
1464 : Attached,
1465 : Failed { reason: String },
1466 : }
1467 :
1468 0 : #[derive(Serialize, Deserialize, Clone)]
1469 : pub struct TenantInfo {
1470 : pub id: TenantShardId,
1471 : // NB: intentionally not part of OpenAPI, we don't want to commit to a specific set of TenantState's
1472 : pub state: TenantState,
1473 : /// Sum of the size of all layer files.
1474 : /// If a layer is present in both local FS and S3, it counts only once.
1475 : pub current_physical_size: Option<u64>, // physical size is only included in `tenant_status` endpoint
1476 : pub attachment_status: TenantAttachmentStatus,
1477 : pub generation: u32,
1478 :
1479 : /// Opaque explanation if gc is being blocked.
1480 : ///
1481 : /// Only looked up for the individual tenant detail, not the listing.
1482 : #[serde(skip_serializing_if = "Option::is_none")]
1483 : pub gc_blocking: Option<String>,
1484 : }
1485 :
1486 0 : #[derive(Serialize, Deserialize, Clone)]
1487 : pub struct TenantDetails {
1488 : #[serde(flatten)]
1489 : pub tenant_info: TenantInfo,
1490 :
1491 : pub walredo: Option<WalRedoManagerStatus>,
1492 :
1493 : pub timelines: Vec<TimelineId>,
1494 : }
1495 :
1496 0 : #[derive(Serialize, Deserialize, PartialEq, Eq, Clone, Copy, Debug)]
1497 : pub enum TimelineArchivalState {
1498 : Archived,
1499 : Unarchived,
1500 : }
1501 :
1502 0 : #[derive(Serialize, Deserialize, PartialEq, Eq, Clone)]
1503 : pub enum TimelineVisibilityState {
1504 : Visible,
1505 : Invisible,
1506 : }
1507 :
1508 0 : #[derive(Serialize, Deserialize, PartialEq, Eq, Clone)]
1509 : pub struct TimelineArchivalConfigRequest {
1510 : pub state: TimelineArchivalState,
1511 : }
1512 :
1513 0 : #[derive(Serialize, Deserialize, PartialEq, Eq, Clone)]
1514 : pub struct TimelinePatchIndexPartRequest {
1515 : pub rel_size_migration: Option<RelSizeMigration>,
1516 : pub rel_size_migrated_at: Option<Lsn>,
1517 : pub gc_compaction_last_completed_lsn: Option<Lsn>,
1518 : pub applied_gc_cutoff_lsn: Option<Lsn>,
1519 : #[serde(default)]
1520 : pub force_index_update: bool,
1521 : }
1522 :
1523 0 : #[derive(Debug, Serialize, Deserialize, Clone)]
1524 : pub struct TimelinesInfoAndOffloaded {
1525 : pub timelines: Vec<TimelineInfo>,
1526 : pub offloaded: Vec<OffloadedTimelineInfo>,
1527 : }
1528 :
1529 : /// Analog of [`TimelineInfo`] for offloaded timelines.
1530 0 : #[derive(Debug, Serialize, Deserialize, Clone)]
1531 : pub struct OffloadedTimelineInfo {
1532 : pub tenant_id: TenantShardId,
1533 : pub timeline_id: TimelineId,
1534 : /// Whether the timeline has a parent it has been branched off from or not
1535 : pub ancestor_timeline_id: Option<TimelineId>,
1536 : /// Whether to retain the branch lsn at the ancestor or not
1537 : pub ancestor_retain_lsn: Option<Lsn>,
1538 : /// The time point when the timeline was archived
1539 : pub archived_at: chrono::DateTime<chrono::Utc>,
1540 : }
1541 :
1542 : /// The state of the rel size migration. This is persisted in the index part.
1543 : /// compatibility.
1544 0 : #[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1545 : #[serde(rename_all = "camelCase")]
1546 : pub enum RelSizeMigration {
1547 : /// The tenant is using the old rel_size format.
1548 : /// Note that this enum is persisted as `Option<RelSizeMigration>` in the index part, so
1549 : /// `None` is the same as `Some(RelSizeMigration::Legacy)`.
1550 : #[default]
1551 : Legacy,
1552 : /// The tenant is migrating to the new rel_size format. Both old and new rel_size format are
1553 : /// persisted in the storage. The read path will read both formats and validate them.
1554 : Migrating,
1555 : /// The tenant has migrated to the new rel_size format. Only the new rel_size format is persisted
1556 : /// in the storage, and the read path will not read the old format.
1557 : Migrated,
1558 : }
1559 :
1560 : /// This represents the output of the "timeline_detail" and "timeline_list" API calls.
1561 0 : #[derive(Debug, Serialize, Deserialize, Clone)]
1562 : pub struct TimelineInfo {
1563 : pub tenant_id: TenantShardId,
1564 : pub timeline_id: TimelineId,
1565 :
1566 : pub ancestor_timeline_id: Option<TimelineId>,
1567 : pub ancestor_lsn: Option<Lsn>,
1568 : pub last_record_lsn: Lsn,
1569 : pub prev_record_lsn: Option<Lsn>,
1570 :
1571 : /// The LSN up to which GC has advanced: older data may still exist but it is not available for clients.
1572 : /// This LSN is not suitable for deciding where to create branches etc: use [`TimelineInfo::min_readable_lsn`] instead,
1573 : /// as it is easier to reason about.
1574 : #[serde(default)]
1575 : pub applied_gc_cutoff_lsn: Lsn,
1576 :
1577 : /// The upper bound of data which is either already GC'ed, or elegible to be GC'ed at any time based on PITR interval.
1578 : /// This LSN represents the "end of history" for this timeline, and callers should use it to figure out the oldest
1579 : /// LSN at which it is legal to create a branch or ephemeral endpoint.
1580 : ///
1581 : /// Note that holders of valid LSN leases may be able to create branches and read pages earlier
1582 : /// than this LSN, but new leases may not be taken out earlier than this LSN.
1583 : #[serde(default)]
1584 : pub min_readable_lsn: Lsn,
1585 :
1586 : pub disk_consistent_lsn: Lsn,
1587 :
1588 : /// The LSN that we have succesfully uploaded to remote storage
1589 : pub remote_consistent_lsn: Lsn,
1590 :
1591 : /// The LSN that we are advertizing to safekeepers
1592 : pub remote_consistent_lsn_visible: Lsn,
1593 :
1594 : /// The LSN from the start of the root timeline (never changes)
1595 : pub initdb_lsn: Lsn,
1596 :
1597 : pub current_logical_size: u64,
1598 : pub current_logical_size_is_accurate: bool,
1599 :
1600 : pub directory_entries_counts: Vec<u64>,
1601 :
1602 : /// Sum of the size of all layer files.
1603 : /// If a layer is present in both local FS and S3, it counts only once.
1604 : pub current_physical_size: Option<u64>, // is None when timeline is Unloaded
1605 : pub current_logical_size_non_incremental: Option<u64>,
1606 :
1607 : /// How many bytes of WAL are within this branch's pitr_interval. If the pitr_interval goes
1608 : /// beyond the branch's branch point, we only count up to the branch point.
1609 : pub pitr_history_size: u64,
1610 :
1611 : /// Whether this branch's branch point is within its ancestor's PITR interval (i.e. any
1612 : /// ancestor data used by this branch would have been retained anyway). If this is false, then
1613 : /// this branch may be imposing a cost on the ancestor by causing it to retain layers that it would
1614 : /// otherwise be able to GC.
1615 : pub within_ancestor_pitr: bool,
1616 :
1617 : pub timeline_dir_layer_file_size_sum: Option<u64>,
1618 :
1619 : pub wal_source_connstr: Option<String>,
1620 : pub last_received_msg_lsn: Option<Lsn>,
1621 : /// the timestamp (in microseconds) of the last received message
1622 : pub last_received_msg_ts: Option<u128>,
1623 : pub pg_version: PgMajorVersion,
1624 :
1625 : pub state: TimelineState,
1626 :
1627 : pub walreceiver_status: String,
1628 :
1629 : // ALWAYS add new fields at the end of the struct with `Option` to ensure forward/backward compatibility.
1630 : // Backward compatibility: you will get a JSON not containing the newly-added field.
1631 : // Forward compatibility: a previous version of the pageserver will receive a JSON. serde::Deserialize does
1632 : // not deny unknown fields by default so it's safe to set the field to some value, though it won't be
1633 : // read.
1634 : /// Whether the timeline is archived.
1635 : pub is_archived: Option<bool>,
1636 :
1637 : /// The status of the rel_size migration.
1638 : pub rel_size_migration: Option<RelSizeMigration>,
1639 : pub rel_size_migrated_at: Option<Lsn>,
1640 :
1641 : /// Whether the timeline is invisible in synthetic size calculations.
1642 : pub is_invisible: Option<bool>,
1643 : // HADRON: the largest LSN below which all page updates have been included in the image layers.
1644 : #[serde(skip_serializing_if = "Option::is_none")]
1645 : pub image_consistent_lsn: Option<Lsn>,
1646 : }
1647 :
1648 0 : #[derive(Debug, Clone, Serialize, Deserialize)]
1649 : pub struct LayerMapInfo {
1650 : pub in_memory_layers: Vec<InMemoryLayerInfo>,
1651 : pub historic_layers: Vec<HistoricLayerInfo>,
1652 : }
1653 :
1654 : /// The residence status of a layer
1655 0 : #[derive(Debug, Clone, Copy, Serialize, Deserialize)]
1656 : pub enum LayerResidenceStatus {
1657 : /// Residence status for a layer file that exists locally.
1658 : /// It may also exist on the remote, we don't care here.
1659 : Resident,
1660 : /// Residence status for a layer file that only exists on the remote.
1661 : Evicted,
1662 : }
1663 :
1664 : #[serde_as]
1665 : #[derive(Debug, Clone, Serialize, Deserialize)]
1666 : pub struct LayerAccessStats {
1667 : #[serde_as(as = "serde_with::TimestampMilliSeconds")]
1668 : pub access_time: SystemTime,
1669 :
1670 : #[serde_as(as = "serde_with::TimestampMilliSeconds")]
1671 : pub residence_time: SystemTime,
1672 :
1673 : pub visible: bool,
1674 : }
1675 :
1676 0 : #[derive(Debug, Clone, Serialize, Deserialize)]
1677 : #[serde(tag = "kind")]
1678 : pub enum InMemoryLayerInfo {
1679 : Open { lsn_start: Lsn },
1680 : Frozen { lsn_start: Lsn, lsn_end: Lsn },
1681 : }
1682 :
1683 0 : #[derive(Debug, Clone, Serialize, Deserialize)]
1684 : #[serde(tag = "kind")]
1685 : pub enum HistoricLayerInfo {
1686 : Delta {
1687 : layer_file_name: String,
1688 : layer_file_size: u64,
1689 :
1690 : lsn_start: Lsn,
1691 : lsn_end: Lsn,
1692 : remote: bool,
1693 : access_stats: LayerAccessStats,
1694 :
1695 : l0: bool,
1696 : },
1697 : Image {
1698 : layer_file_name: String,
1699 : layer_file_size: u64,
1700 :
1701 : lsn_start: Lsn,
1702 : remote: bool,
1703 : access_stats: LayerAccessStats,
1704 : },
1705 : }
1706 :
1707 : impl HistoricLayerInfo {
1708 0 : pub fn layer_file_name(&self) -> &str {
1709 0 : match self {
1710 : HistoricLayerInfo::Delta {
1711 0 : layer_file_name, ..
1712 0 : } => layer_file_name,
1713 : HistoricLayerInfo::Image {
1714 0 : layer_file_name, ..
1715 0 : } => layer_file_name,
1716 : }
1717 0 : }
1718 0 : pub fn is_remote(&self) -> bool {
1719 0 : match self {
1720 0 : HistoricLayerInfo::Delta { remote, .. } => *remote,
1721 0 : HistoricLayerInfo::Image { remote, .. } => *remote,
1722 : }
1723 0 : }
1724 0 : pub fn set_remote(&mut self, value: bool) {
1725 0 : let field = match self {
1726 0 : HistoricLayerInfo::Delta { remote, .. } => remote,
1727 0 : HistoricLayerInfo::Image { remote, .. } => remote,
1728 : };
1729 0 : *field = value;
1730 0 : }
1731 0 : pub fn layer_file_size(&self) -> u64 {
1732 0 : match self {
1733 : HistoricLayerInfo::Delta {
1734 0 : layer_file_size, ..
1735 0 : } => *layer_file_size,
1736 : HistoricLayerInfo::Image {
1737 0 : layer_file_size, ..
1738 0 : } => *layer_file_size,
1739 : }
1740 0 : }
1741 : }
1742 :
1743 0 : #[derive(Debug, Serialize, Deserialize)]
1744 : pub struct DownloadRemoteLayersTaskSpawnRequest {
1745 : pub max_concurrent_downloads: NonZeroUsize,
1746 : }
1747 :
1748 0 : #[derive(Debug, Serialize, Deserialize)]
1749 : pub struct IngestAuxFilesRequest {
1750 : pub aux_files: HashMap<String, String>,
1751 : }
1752 :
1753 0 : #[derive(Debug, Serialize, Deserialize)]
1754 : pub struct ListAuxFilesRequest {
1755 : pub lsn: Lsn,
1756 : }
1757 :
1758 0 : #[derive(Debug, Serialize, Deserialize, Clone)]
1759 : pub struct DownloadRemoteLayersTaskInfo {
1760 : pub task_id: String,
1761 : pub state: DownloadRemoteLayersTaskState,
1762 : pub total_layer_count: u64, // stable once `completed`
1763 : pub successful_download_count: u64, // stable once `completed`
1764 : pub failed_download_count: u64, // stable once `completed`
1765 : }
1766 :
1767 0 : #[derive(Debug, Serialize, Deserialize, Clone)]
1768 : pub enum DownloadRemoteLayersTaskState {
1769 : Running,
1770 : Completed,
1771 : ShutDown,
1772 : }
1773 :
1774 0 : #[derive(Debug, Serialize, Deserialize)]
1775 : pub struct TimelineGcRequest {
1776 : pub gc_horizon: Option<u64>,
1777 : }
1778 :
1779 0 : #[derive(Debug, Clone, Serialize, Deserialize)]
1780 : pub struct WalRedoManagerProcessStatus {
1781 : pub pid: u32,
1782 : }
1783 :
1784 0 : #[derive(Debug, Clone, Serialize, Deserialize)]
1785 : pub struct WalRedoManagerStatus {
1786 : pub last_redo_at: Option<chrono::DateTime<chrono::Utc>>,
1787 : pub process: Option<WalRedoManagerProcessStatus>,
1788 : }
1789 :
1790 : /// The progress of a secondary tenant.
1791 : ///
1792 : /// It is mostly useful when doing a long running download: e.g. initiating
1793 : /// a download job, timing out while waiting for it to run, and then inspecting this status to understand
1794 : /// what's happening.
1795 0 : #[derive(Default, Debug, Serialize, Deserialize, Clone)]
1796 : pub struct SecondaryProgress {
1797 : /// The remote storage LastModified time of the heatmap object we last downloaded.
1798 : pub heatmap_mtime: Option<serde_system_time::SystemTime>,
1799 :
1800 : /// The number of layers currently on-disk
1801 : pub layers_downloaded: usize,
1802 : /// The number of layers in the most recently seen heatmap
1803 : pub layers_total: usize,
1804 :
1805 : /// The number of layer bytes currently on-disk
1806 : pub bytes_downloaded: u64,
1807 : /// The number of layer bytes in the most recently seen heatmap
1808 : pub bytes_total: u64,
1809 : }
1810 :
1811 0 : #[derive(Serialize, Deserialize, Debug)]
1812 : pub struct TenantScanRemoteStorageShard {
1813 : pub tenant_shard_id: TenantShardId,
1814 : pub generation: Option<u32>,
1815 : pub stripe_size: Option<ShardStripeSize>,
1816 : }
1817 :
1818 0 : #[derive(Serialize, Deserialize, Debug, Default)]
1819 : pub struct TenantScanRemoteStorageResponse {
1820 : pub shards: Vec<TenantScanRemoteStorageShard>,
1821 : }
1822 :
1823 0 : #[derive(Serialize, Deserialize, Debug, Clone)]
1824 : #[serde(rename_all = "snake_case")]
1825 : pub enum TenantSorting {
1826 : /// Total size of layers on local disk for all timelines in a shard.
1827 : ResidentSize,
1828 : /// The logical size of the largest timeline within a _tenant_ (not shard). Only tracked on
1829 : /// shard 0, contains the sum across all shards.
1830 : MaxLogicalSize,
1831 : /// The logical size of the largest timeline within a _tenant_ (not shard), divided by number of
1832 : /// shards. Only tracked on shard 0, and estimates the per-shard logical size.
1833 : MaxLogicalSizePerShard,
1834 : }
1835 :
1836 : impl Default for TenantSorting {
1837 0 : fn default() -> Self {
1838 0 : Self::ResidentSize
1839 0 : }
1840 : }
1841 :
1842 0 : #[derive(Serialize, Deserialize, Debug, Clone)]
1843 : pub struct TopTenantShardsRequest {
1844 : // How would you like to sort the tenants?
1845 : pub order_by: TenantSorting,
1846 :
1847 : // How many results?
1848 : pub limit: usize,
1849 :
1850 : // Omit tenants with more than this many shards (e.g. if this is the max number of shards
1851 : // that the caller would ever split to)
1852 : pub where_shards_lt: Option<ShardCount>,
1853 :
1854 : // Omit tenants where the ordering metric is less than this (this is an optimization to
1855 : // let us quickly exclude numerous tiny shards)
1856 : pub where_gt: Option<u64>,
1857 : }
1858 :
1859 0 : #[derive(Serialize, Deserialize, Debug, PartialEq, Eq)]
1860 : pub struct TopTenantShardItem {
1861 : pub id: TenantShardId,
1862 :
1863 : /// Total size of layers on local disk for all timelines in this shard.
1864 : pub resident_size: u64,
1865 :
1866 : /// Total size of layers in remote storage for all timelines in this shard.
1867 : pub physical_size: u64,
1868 :
1869 : /// The largest logical size of a timeline within this _tenant_ (not shard). This is only
1870 : /// tracked on shard 0, and contains the sum of the logical size across all shards.
1871 : pub max_logical_size: u64,
1872 :
1873 : /// The largest logical size of a timeline within this _tenant_ (not shard) divided by number of
1874 : /// shards. This is only tracked on shard 0, and is only an estimate as we divide it evenly by
1875 : /// shard count, rounded up.
1876 : pub max_logical_size_per_shard: u64,
1877 : }
1878 :
1879 0 : #[derive(Serialize, Deserialize, Debug, Default)]
1880 : pub struct TopTenantShardsResponse {
1881 : pub shards: Vec<TopTenantShardItem>,
1882 : }
1883 :
1884 : pub mod virtual_file {
1885 :
1886 : #[derive(
1887 : Copy,
1888 : Clone,
1889 : PartialEq,
1890 : Eq,
1891 : Hash,
1892 : strum_macros::EnumString,
1893 : strum_macros::Display,
1894 0 : serde_with::DeserializeFromStr,
1895 : serde_with::SerializeDisplay,
1896 : Debug,
1897 : )]
1898 : #[strum(serialize_all = "kebab-case")]
1899 : pub enum IoEngineKind {
1900 : StdFs,
1901 : #[cfg(target_os = "linux")]
1902 : TokioEpollUring,
1903 : }
1904 :
1905 : /// Direct IO modes for a pageserver.
1906 : #[derive(
1907 : Copy,
1908 : Clone,
1909 : PartialEq,
1910 : Eq,
1911 : Hash,
1912 : strum_macros::EnumString,
1913 : strum_macros::EnumIter,
1914 : strum_macros::Display,
1915 0 : serde_with::DeserializeFromStr,
1916 : serde_with::SerializeDisplay,
1917 : Debug,
1918 : )]
1919 : #[strum(serialize_all = "kebab-case")]
1920 : #[repr(u8)]
1921 : pub enum IoMode {
1922 : /// Uses buffered IO.
1923 : Buffered,
1924 : /// Uses direct IO for reads only.
1925 : Direct,
1926 : /// Use direct IO for reads and writes.
1927 : DirectRw,
1928 : }
1929 :
1930 : impl IoMode {
1931 260 : pub fn preferred() -> Self {
1932 260 : IoMode::DirectRw
1933 260 : }
1934 : }
1935 :
1936 : impl TryFrom<u8> for IoMode {
1937 : type Error = u8;
1938 :
1939 2654 : fn try_from(value: u8) -> Result<Self, Self::Error> {
1940 2654 : Ok(match value {
1941 2654 : v if v == (IoMode::Buffered as u8) => IoMode::Buffered,
1942 2654 : v if v == (IoMode::Direct as u8) => IoMode::Direct,
1943 2654 : v if v == (IoMode::DirectRw as u8) => IoMode::DirectRw,
1944 0 : x => return Err(x),
1945 : })
1946 2654 : }
1947 : }
1948 : }
1949 :
1950 0 : #[derive(Debug, Clone, Serialize, Deserialize)]
1951 : pub struct ScanDisposableKeysResponse {
1952 : pub disposable_count: usize,
1953 : pub not_disposable_count: usize,
1954 : }
1955 :
1956 : // This is a cut-down version of TenantHistorySize from the pageserver crate, omitting fields
1957 : // that require pageserver-internal types. It is sufficient to get the total size.
1958 0 : #[derive(Serialize, Deserialize, Debug)]
1959 : pub struct TenantHistorySize {
1960 : pub id: TenantId,
1961 : /// Size is a mixture of WAL and logical size, so the unit is bytes.
1962 : ///
1963 : /// Will be none if `?inputs_only=true` was given.
1964 : pub size: Option<u64>,
1965 : }
1966 :
1967 0 : #[derive(Debug, Serialize, Deserialize)]
1968 : pub struct PageTraceEvent {
1969 : pub key: CompactKey,
1970 : pub effective_lsn: Lsn,
1971 : pub time: SystemTime,
1972 : }
1973 :
1974 : impl Default for PageTraceEvent {
1975 0 : fn default() -> Self {
1976 0 : Self {
1977 0 : key: Default::default(),
1978 0 : effective_lsn: Default::default(),
1979 0 : time: std::time::UNIX_EPOCH,
1980 0 : }
1981 0 : }
1982 : }
1983 :
1984 : #[cfg(test)]
1985 : mod tests {
1986 : use std::str::FromStr;
1987 :
1988 : use serde_json::json;
1989 :
1990 : use super::*;
1991 :
1992 : #[test]
1993 1 : fn test_tenantinfo_serde() {
1994 : // Test serialization/deserialization of TenantInfo
1995 1 : let original_active = TenantInfo {
1996 1 : id: TenantShardId::unsharded(TenantId::generate()),
1997 1 : state: TenantState::Active,
1998 1 : current_physical_size: Some(42),
1999 1 : attachment_status: TenantAttachmentStatus::Attached,
2000 1 : generation: 1,
2001 1 : gc_blocking: None,
2002 1 : };
2003 1 : let expected_active = json!({
2004 1 : "id": original_active.id.to_string(),
2005 1 : "state": {
2006 1 : "slug": "Active",
2007 : },
2008 1 : "current_physical_size": 42,
2009 1 : "attachment_status": {
2010 1 : "slug":"attached",
2011 : },
2012 1 : "generation" : 1
2013 : });
2014 :
2015 1 : let original_broken = TenantInfo {
2016 1 : id: TenantShardId::unsharded(TenantId::generate()),
2017 1 : state: TenantState::Broken {
2018 1 : reason: "reason".into(),
2019 1 : backtrace: "backtrace info".into(),
2020 1 : },
2021 1 : current_physical_size: Some(42),
2022 1 : attachment_status: TenantAttachmentStatus::Attached,
2023 1 : generation: 1,
2024 1 : gc_blocking: None,
2025 1 : };
2026 1 : let expected_broken = json!({
2027 1 : "id": original_broken.id.to_string(),
2028 1 : "state": {
2029 1 : "slug": "Broken",
2030 1 : "data": {
2031 1 : "backtrace": "backtrace info",
2032 1 : "reason": "reason",
2033 : }
2034 : },
2035 1 : "current_physical_size": 42,
2036 1 : "attachment_status": {
2037 1 : "slug":"attached",
2038 : },
2039 1 : "generation" : 1
2040 : });
2041 :
2042 1 : assert_eq!(
2043 1 : serde_json::to_value(&original_active).unwrap(),
2044 : expected_active
2045 : );
2046 :
2047 1 : assert_eq!(
2048 1 : serde_json::to_value(&original_broken).unwrap(),
2049 : expected_broken
2050 : );
2051 1 : assert!(format!("{:?}", &original_broken.state).contains("reason"));
2052 1 : assert!(format!("{:?}", &original_broken.state).contains("backtrace info"));
2053 1 : }
2054 :
2055 : #[test]
2056 1 : fn test_reject_unknown_field() {
2057 1 : let id = TenantId::generate();
2058 1 : let config_request = json!({
2059 1 : "tenant_id": id.to_string(),
2060 1 : "unknown_field": "unknown_value".to_string(),
2061 : });
2062 1 : let err = serde_json::from_value::<TenantConfigRequest>(config_request).unwrap_err();
2063 1 : assert!(
2064 1 : err.to_string().contains("unknown field `unknown_field`"),
2065 0 : "expect unknown field `unknown_field` error, got: {err}"
2066 : );
2067 1 : }
2068 :
2069 : #[test]
2070 1 : fn tenantstatus_activating_serde() {
2071 1 : let states = [TenantState::Activating(ActivatingFrom::Attaching)];
2072 1 : let expected = "[{\"slug\":\"Activating\",\"data\":\"Attaching\"}]";
2073 :
2074 1 : let actual = serde_json::to_string(&states).unwrap();
2075 :
2076 1 : assert_eq!(actual, expected);
2077 :
2078 1 : let parsed = serde_json::from_str::<Vec<TenantState>>(&actual).unwrap();
2079 :
2080 1 : assert_eq!(states.as_slice(), &parsed);
2081 1 : }
2082 :
2083 : #[test]
2084 1 : fn tenantstatus_activating_strum() {
2085 : // tests added, because we use these for metrics
2086 1 : let examples = [
2087 1 : (line!(), TenantState::Attaching, "Attaching"),
2088 1 : (
2089 1 : line!(),
2090 1 : TenantState::Activating(ActivatingFrom::Attaching),
2091 1 : "Activating",
2092 1 : ),
2093 1 : (line!(), TenantState::Active, "Active"),
2094 1 : (
2095 1 : line!(),
2096 1 : TenantState::Stopping { progress: None },
2097 1 : "Stopping",
2098 1 : ),
2099 1 : (
2100 1 : line!(),
2101 1 : TenantState::Stopping {
2102 1 : progress: Some(completion::Barrier::default()),
2103 1 : },
2104 1 : "Stopping",
2105 1 : ),
2106 1 : (
2107 1 : line!(),
2108 1 : TenantState::Broken {
2109 1 : reason: "Example".into(),
2110 1 : backtrace: "Looooong backtrace".into(),
2111 1 : },
2112 1 : "Broken",
2113 1 : ),
2114 1 : ];
2115 :
2116 7 : for (line, rendered, expected) in examples {
2117 6 : let actual: &'static str = rendered.into();
2118 6 : assert_eq!(actual, expected, "example on {line}");
2119 : }
2120 1 : }
2121 :
2122 : #[test]
2123 1 : fn test_image_compression_algorithm_parsing() {
2124 : use ImageCompressionAlgorithm::*;
2125 1 : let cases = [
2126 1 : ("disabled", Disabled),
2127 1 : ("zstd", Zstd { level: None }),
2128 1 : ("zstd(18)", Zstd { level: Some(18) }),
2129 1 : ("zstd(-3)", Zstd { level: Some(-3) }),
2130 1 : ];
2131 :
2132 5 : for (display, expected) in cases {
2133 4 : assert_eq!(
2134 4 : ImageCompressionAlgorithm::from_str(display).unwrap(),
2135 : expected,
2136 0 : "parsing works"
2137 : );
2138 4 : assert_eq!(format!("{expected}"), display, "Display FromStr roundtrip");
2139 :
2140 4 : let ser = serde_json::to_string(&expected).expect("serialization");
2141 4 : assert_eq!(
2142 4 : serde_json::from_str::<ImageCompressionAlgorithm>(&ser).unwrap(),
2143 : expected,
2144 0 : "serde roundtrip"
2145 : );
2146 :
2147 4 : assert_eq!(
2148 4 : serde_json::Value::String(display.to_string()),
2149 4 : serde_json::to_value(expected).unwrap(),
2150 0 : "Display is the serde serialization"
2151 : );
2152 : }
2153 1 : }
2154 :
2155 : #[test]
2156 1 : fn test_tenant_config_patch_request_serde() {
2157 1 : let patch_request = TenantConfigPatchRequest {
2158 1 : tenant_id: TenantId::from_str("17c6d121946a61e5ab0fe5a2fd4d8215").unwrap(),
2159 1 : config: TenantConfigPatch {
2160 1 : checkpoint_distance: FieldPatch::Upsert(42),
2161 1 : gc_horizon: FieldPatch::Remove,
2162 1 : compaction_threshold: FieldPatch::Noop,
2163 1 : ..TenantConfigPatch::default()
2164 1 : },
2165 1 : };
2166 :
2167 1 : let json = serde_json::to_string(&patch_request).unwrap();
2168 :
2169 1 : let expected = r#"{"tenant_id":"17c6d121946a61e5ab0fe5a2fd4d8215","checkpoint_distance":42,"gc_horizon":null}"#;
2170 1 : assert_eq!(json, expected);
2171 :
2172 1 : let decoded: TenantConfigPatchRequest = serde_json::from_str(&json).unwrap();
2173 1 : assert_eq!(decoded.tenant_id, patch_request.tenant_id);
2174 1 : assert_eq!(decoded.config, patch_request.config);
2175 :
2176 : // Now apply the patch to a config to demonstrate semantics
2177 :
2178 1 : let base = TenantConfig {
2179 1 : checkpoint_distance: Some(28),
2180 1 : gc_horizon: Some(100),
2181 1 : compaction_target_size: Some(1024),
2182 1 : ..Default::default()
2183 1 : };
2184 :
2185 1 : let expected = TenantConfig {
2186 1 : checkpoint_distance: Some(42),
2187 1 : gc_horizon: None,
2188 1 : ..base.clone()
2189 1 : };
2190 :
2191 1 : let patched = base.apply_patch(decoded.config).unwrap();
2192 :
2193 1 : assert_eq!(patched, expected);
2194 1 : }
2195 : }
|
