Line data Source code
1 : //! In-memory index to track the tenant files on the remote storage.
2 : //!
3 : //! Able to restore itself from the storage index parts, that are located in every timeline's remote directory and contain all data about
4 : //! remote timeline layers and its metadata.
5 :
6 : use std::collections::HashMap;
7 :
8 : use chrono::NaiveDateTime;
9 : use pageserver_api::models::AuxFilePolicy;
10 : use pageserver_api::models::RelSizeMigration;
11 : use pageserver_api::shard::ShardIndex;
12 : use serde::{Deserialize, Serialize};
13 : use utils::id::TimelineId;
14 : use utils::lsn::Lsn;
15 :
16 : use super::is_same_remote_layer_path;
17 : use crate::tenant::Generation;
18 : use crate::tenant::metadata::TimelineMetadata;
19 : use crate::tenant::storage_layer::LayerName;
20 : use crate::tenant::timeline::import_pgdata;
21 :
22 : /// In-memory representation of an `index_part.json` file
23 : ///
24 : /// Contains the data about all files in the timeline, present remotely and its metadata.
25 : ///
26 : /// This type needs to be backwards and forwards compatible. When changing the fields,
27 : /// remember to add a test case for the changed version.
28 : #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
29 : pub struct IndexPart {
30 : /// Debugging aid describing the version of this type.
31 : #[serde(default)]
32 : version: usize,
33 :
34 : #[serde(default)]
35 : #[serde(skip_serializing_if = "Option::is_none")]
36 : pub deleted_at: Option<NaiveDateTime>,
37 :
38 : #[serde(default)]
39 : #[serde(skip_serializing_if = "Option::is_none")]
40 : pub archived_at: Option<NaiveDateTime>,
41 :
42 : /// This field supports import-from-pgdata ("fast imports" platform feature).
43 : /// We don't currently use fast imports, so, this field is None for all production timelines.
44 : /// See <https://github.com/neondatabase/neon/pull/9218> for more information.
45 : #[serde(default)]
46 : #[serde(skip_serializing_if = "Option::is_none")]
47 : pub import_pgdata: Option<import_pgdata::index_part_format::Root>,
48 :
49 : /// Layer filenames and metadata. For an index persisted in remote storage, all layers must
50 : /// exist in remote storage.
51 : pub layer_metadata: HashMap<LayerName, LayerFileMetadata>,
52 :
53 : /// Because of the trouble of eyeballing the legacy "metadata" field, we copied the
54 : /// "disk_consistent_lsn" out. After version 7 this is no longer needed, but the name cannot be
55 : /// reused.
56 : pub(super) disk_consistent_lsn: Lsn,
57 :
58 : // TODO: rename as "metadata" next week, keep the alias = "metadata_bytes", bump version Adding
59 : // the "alias = metadata" was forgotten in #7693, so we have to use "rewrite = metadata_bytes"
60 : // for backwards compatibility.
61 : #[serde(
62 : rename = "metadata_bytes",
63 : alias = "metadata",
64 : with = "crate::tenant::metadata::modern_serde"
65 : )]
66 : pub metadata: TimelineMetadata,
67 :
68 : #[serde(default)]
69 : pub(crate) lineage: Lineage,
70 :
71 : #[serde(skip_serializing_if = "Option::is_none", default)]
72 : pub(crate) gc_blocking: Option<GcBlocking>,
73 :
74 : /// Describes the kind of aux files stored in the timeline.
75 : ///
76 : /// The value is modified during file ingestion when the latest wanted value communicated via tenant config is applied if it is acceptable.
77 : /// A V1 setting after V2 files have been committed is not accepted.
78 : ///
79 : /// None means no aux files have been written to the storage before the point
80 : /// when this flag is introduced.
81 : ///
82 : /// This flag is not used any more as all tenants have been transitioned to the new aux file policy.
83 : #[serde(skip_serializing_if = "Option::is_none", default)]
84 : pub(crate) last_aux_file_policy: Option<AuxFilePolicy>,
85 :
86 : #[serde(skip_serializing_if = "Option::is_none", default)]
87 : pub(crate) rel_size_migration: Option<RelSizeMigration>,
88 :
89 : /// Not used anymore -- kept here for backwards compatibility. Merged into the `gc_compaction` field.
90 : #[serde(skip_serializing_if = "Option::is_none", default)]
91 : l2_lsn: Option<Lsn>,
92 :
93 : /// State for the garbage-collecting compaction pass.
94 : ///
95 : /// Garbage-collecting compaction (gc-compaction) prunes `Value`s that are outside
96 : /// the PITR window and not needed by child timelines.
97 : ///
98 : /// A commonly used synonym for this compaction pass is
99 : /// "bottommost-compaction" because the affected LSN range
100 : /// is the "bottom" of the (key,lsn) map.
101 : ///
102 : /// Gc-compaction is a quite expensive operation; that's why we use
103 : /// trigger condition.
104 : /// This field here holds the state pertaining to that trigger condition
105 : /// and (in future) to the progress of the gc-compaction, so that it's
106 : /// resumable across restarts & migrations.
107 : ///
108 : /// Note that the underlying algorithm is _also_ called `gc-compaction`
109 : /// in most places & design docs; but in fact it is more flexible than
110 : /// just the specific use case here; it needs a new name.
111 : #[serde(skip_serializing_if = "Option::is_none", default)]
112 : pub(crate) gc_compaction: Option<GcCompactionState>,
113 :
114 : /// The timestamp when the timeline was marked invisible in synthetic size calculations.
115 : #[serde(skip_serializing_if = "Option::is_none", default)]
116 : pub(crate) marked_invisible_at: Option<NaiveDateTime>,
117 :
118 : /// The LSN at which we started the rel size migration. Accesses below this LSN should be
119 : /// processed with the v1 read path. Usually this LSN should be set together with `rel_size_migration`.
120 : #[serde(skip_serializing_if = "Option::is_none", default)]
121 : pub(crate) rel_size_migrated_at: Option<Lsn>,
122 : }
123 :
124 0 : #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
125 : pub struct GcCompactionState {
126 : /// The upper bound of the last completed garbage-collecting compaction, aka. L2 LSN.
127 : pub(crate) last_completed_lsn: Lsn,
128 : }
129 :
130 : impl IndexPart {
131 : /// When adding or modifying any parts of `IndexPart`, increment the version so that it can be
132 : /// used to understand later versions.
133 : ///
134 : /// Version is currently informative only.
135 : /// Version history
136 : /// - 2: added `deleted_at`
137 : /// - 3: no longer deserialize `timeline_layers` (serialized format is the same, but timeline_layers
138 : /// is always generated from the keys of `layer_metadata`)
139 : /// - 4: timeline_layers is fully removed.
140 : /// - 5: lineage was added
141 : /// - 6: last_aux_file_policy is added.
142 : /// - 7: metadata_bytes is no longer written, but still read
143 : /// - 8: added `archived_at`
144 : /// - 9: +gc_blocking
145 : /// - 10: +import_pgdata
146 : /// - 11: +rel_size_migration
147 : /// - 12: +l2_lsn
148 : /// - 13: +gc_compaction
149 : /// - 14: +marked_invisible_at
150 : /// - 15: +rel_size_migrated_at
151 : const LATEST_VERSION: usize = 15;
152 :
153 : // Versions we may see when reading from a bucket.
154 : pub const KNOWN_VERSIONS: &'static [usize] =
155 : &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15];
156 :
157 : pub const FILE_NAME: &'static str = "index_part.json";
158 :
159 274 : pub fn empty(metadata: TimelineMetadata) -> Self {
160 274 : IndexPart {
161 274 : version: Self::LATEST_VERSION,
162 274 : layer_metadata: Default::default(),
163 274 : disk_consistent_lsn: metadata.disk_consistent_lsn(),
164 274 : metadata,
165 274 : deleted_at: None,
166 274 : archived_at: None,
167 274 : lineage: Default::default(),
168 274 : gc_blocking: None,
169 274 : last_aux_file_policy: None,
170 274 : import_pgdata: None,
171 274 : rel_size_migration: None,
172 274 : l2_lsn: None,
173 274 : gc_compaction: None,
174 274 : marked_invisible_at: None,
175 274 : rel_size_migrated_at: None,
176 274 : }
177 274 : }
178 :
179 0 : pub fn version(&self) -> usize {
180 0 : self.version
181 0 : }
182 :
183 : /// If you want this under normal operations, read it from self.metadata:
184 : /// this method is just for the scrubber to use when validating an index.
185 0 : pub fn duplicated_disk_consistent_lsn(&self) -> Lsn {
186 0 : self.disk_consistent_lsn
187 0 : }
188 :
189 15 : pub fn from_json_bytes(bytes: &[u8]) -> Result<Self, serde_json::Error> {
190 15 : serde_json::from_slice::<IndexPart>(bytes)
191 15 : }
192 :
193 772 : pub fn to_json_bytes(&self) -> serde_json::Result<Vec<u8>> {
194 772 : serde_json::to_vec(self)
195 772 : }
196 :
197 : #[cfg(test)]
198 14 : pub(crate) fn example() -> Self {
199 14 : Self::empty(TimelineMetadata::example())
200 14 : }
201 :
202 : /// Returns true if the index contains a reference to the given layer (i.e. file path).
203 : ///
204 : /// TODO: there should be a variant of LayerName for the physical remote path that contains
205 : /// information about the shard and generation, to avoid passing in metadata.
206 25680 : pub fn references(&self, name: &LayerName, metadata: &LayerFileMetadata) -> bool {
207 25680 : let Some(index_metadata) = self.layer_metadata.get(name) else {
208 12775 : return false;
209 : };
210 12905 : is_same_remote_layer_path(name, metadata, name, index_metadata)
211 25680 : }
212 :
213 : /// Check for invariants in the index: this is useful when uploading an index to ensure that if
214 : /// we encounter a bug, we do not persist buggy metadata.
215 1395 : pub(crate) fn validate(&self) -> Result<(), String> {
216 1395 : if self.import_pgdata.is_none()
217 1395 : && self.metadata.ancestor_timeline().is_none()
218 1039 : && self.layer_metadata.is_empty()
219 : {
220 : // Unless we're in the middle of a raw pgdata import, or this is a child timeline,the index must
221 : // always have at least one layer.
222 0 : return Err("Index has no ancestor and no layers".to_string());
223 1395 : }
224 :
225 1395 : Ok(())
226 1395 : }
227 : }
228 :
229 : /// Metadata gathered for each of the layer files.
230 : ///
231 : /// Fields have to be `Option`s because remote [`IndexPart`]'s can be from different version, which
232 : /// might have less or more metadata depending if upgrading or rolling back an upgrade.
233 0 : #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
234 : pub struct LayerFileMetadata {
235 : pub file_size: u64,
236 :
237 : #[serde(default = "Generation::none")]
238 : #[serde(skip_serializing_if = "Generation::is_none")]
239 : pub generation: Generation,
240 :
241 : #[serde(default = "ShardIndex::unsharded")]
242 : #[serde(skip_serializing_if = "ShardIndex::is_unsharded")]
243 : pub shard: ShardIndex,
244 : }
245 :
246 : impl LayerFileMetadata {
247 1612 : pub fn new(file_size: u64, generation: Generation, shard: ShardIndex) -> Self {
248 1612 : LayerFileMetadata {
249 1612 : file_size,
250 1612 : generation,
251 1612 : shard,
252 1612 : }
253 1612 : }
254 : /// Helper to get both generation and file size in a tuple
255 0 : pub fn generation_file_size(&self) -> (Generation, u64) {
256 0 : (self.generation, self.file_size)
257 0 : }
258 : }
259 :
260 : /// Limited history of earlier ancestors.
261 : ///
262 : /// A timeline can have more than 1 earlier ancestor, in the rare case that it was repeatedly
263 : /// reparented by having an later timeline be detached from it's ancestor.
264 0 : #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Default)]
265 : pub(crate) struct Lineage {
266 : /// Has the `reparenting_history` been truncated to [`Lineage::REMEMBER_AT_MOST`].
267 : #[serde(skip_serializing_if = "is_false", default)]
268 : reparenting_history_truncated: bool,
269 :
270 : /// Earlier ancestors, truncated when [`Self::reparenting_history_truncated`]
271 : ///
272 : /// These are stored in case we want to support WAL based DR on the timeline. There can be many
273 : /// of these and at most one [`Self::original_ancestor`]. There cannot be more reparentings
274 : /// after [`Self::original_ancestor`] has been set.
275 : #[serde(skip_serializing_if = "Vec::is_empty", default)]
276 : reparenting_history: Vec<TimelineId>,
277 :
278 : /// The ancestor from which this timeline has been detached from and when.
279 : ///
280 : /// If you are adding support for detaching from a hierarchy, consider changing the ancestry
281 : /// into a `Vec<(TimelineId, Lsn)>` to be a path instead.
282 : // FIXME: this is insufficient even for path of two timelines for future wal recovery
283 : // purposes:
284 : //
285 : // assuming a "old main" which has received most of the WAL, and has a branch "new main",
286 : // starting a bit before "old main" last_record_lsn. the current version works fine,
287 : // because we will know to replay wal and branch at the recorded Lsn to do wal recovery.
288 : //
289 : // then assuming "new main" would similarly receive a branch right before its last_record_lsn,
290 : // "new new main". the current implementation would just store ("new main", ancestor_lsn, _)
291 : // here. however, we cannot recover from WAL using only that information, we would need the
292 : // whole ancestry here:
293 : //
294 : // ```json
295 : // [
296 : // ["old main", ancestor_lsn("new main"), _],
297 : // ["new main", ancestor_lsn("new new main"), _]
298 : // ]
299 : // ```
300 : #[serde(skip_serializing_if = "Option::is_none", default)]
301 : original_ancestor: Option<(TimelineId, Lsn, NaiveDateTime)>,
302 : }
303 :
304 3150 : fn is_false(b: &bool) -> bool {
305 3150 : !b
306 3150 : }
307 :
308 : impl Lineage {
309 : const REMEMBER_AT_MOST: usize = 100;
310 :
311 0 : pub(crate) fn record_previous_ancestor(&mut self, old_ancestor: &TimelineId) -> bool {
312 0 : if self.reparenting_history.last() == Some(old_ancestor) {
313 : // do not re-record it
314 0 : false
315 : } else {
316 : #[cfg(feature = "testing")]
317 : {
318 0 : let existing = self
319 0 : .reparenting_history
320 0 : .iter()
321 0 : .position(|x| x == old_ancestor);
322 0 : assert_eq!(
323 : existing, None,
324 0 : "we cannot reparent onto and off and onto the same timeline twice"
325 : );
326 : }
327 0 : let drop_oldest = self.reparenting_history.len() + 1 >= Self::REMEMBER_AT_MOST;
328 :
329 0 : self.reparenting_history_truncated |= drop_oldest;
330 0 : if drop_oldest {
331 0 : self.reparenting_history.remove(0);
332 0 : }
333 0 : self.reparenting_history.push(*old_ancestor);
334 0 : true
335 : }
336 0 : }
337 :
338 : /// Returns true if anything changed.
339 0 : pub(crate) fn record_detaching(&mut self, branchpoint: &(TimelineId, Lsn)) -> bool {
340 0 : if let Some((id, lsn, _)) = self.original_ancestor {
341 0 : assert_eq!(
342 0 : &(id, lsn),
343 : branchpoint,
344 0 : "detaching attempt has to be for the same ancestor we are already detached from"
345 : );
346 0 : false
347 : } else {
348 0 : self.original_ancestor =
349 0 : Some((branchpoint.0, branchpoint.1, chrono::Utc::now().naive_utc()));
350 0 : true
351 : }
352 0 : }
353 :
354 : /// The queried lsn is most likely the basebackup lsn, and this answers question "is it allowed
355 : /// to start a read/write primary at this lsn".
356 : ///
357 : /// Returns true if the Lsn was previously our branch point.
358 0 : pub(crate) fn is_previous_ancestor_lsn(&self, lsn: Lsn) -> bool {
359 0 : self.original_ancestor
360 0 : .is_some_and(|(_, ancestor_lsn, _)| ancestor_lsn == lsn)
361 0 : }
362 :
363 : /// Returns true if the timeline originally had an ancestor, and no longer has one.
364 0 : pub(crate) fn is_detached_from_ancestor(&self) -> bool {
365 0 : self.original_ancestor.is_some()
366 0 : }
367 :
368 : /// Returns original ancestor timeline id and lsn that this timeline has been detached from.
369 0 : pub(crate) fn detached_previous_ancestor(&self) -> Option<(TimelineId, Lsn)> {
370 0 : self.original_ancestor.map(|(id, lsn, _)| (id, lsn))
371 0 : }
372 :
373 0 : pub(crate) fn is_reparented(&self) -> bool {
374 0 : !self.reparenting_history.is_empty()
375 0 : }
376 : }
377 :
378 0 : #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
379 : pub(crate) struct GcBlocking {
380 : pub(crate) started_at: NaiveDateTime,
381 : pub(crate) reasons: enumset::EnumSet<GcBlockingReason>,
382 : }
383 :
384 0 : #[derive(Debug, enumset::EnumSetType, serde::Serialize, serde::Deserialize)]
385 : #[enumset(serialize_repr = "list")]
386 : pub(crate) enum GcBlockingReason {
387 : Manual,
388 : DetachAncestor,
389 : }
390 :
391 : impl GcBlocking {
392 0 : pub(super) fn started_now_for(reason: GcBlockingReason) -> Self {
393 0 : GcBlocking {
394 0 : started_at: chrono::Utc::now().naive_utc(),
395 0 : reasons: enumset::EnumSet::only(reason),
396 0 : }
397 0 : }
398 :
399 : /// Returns true if the given reason is one of the reasons why the gc is blocked.
400 0 : pub(crate) fn blocked_by(&self, reason: GcBlockingReason) -> bool {
401 0 : self.reasons.contains(reason)
402 0 : }
403 :
404 : /// Returns a version of self with the given reason.
405 0 : pub(super) fn with_reason(&self, reason: GcBlockingReason) -> Self {
406 0 : assert!(!self.blocked_by(reason));
407 0 : let mut reasons = self.reasons;
408 0 : reasons.insert(reason);
409 :
410 0 : Self {
411 0 : started_at: self.started_at,
412 0 : reasons,
413 0 : }
414 0 : }
415 :
416 : /// Returns a version of self without the given reason. Assumption is that if
417 : /// there are no more reasons, we can unblock the gc by returning `None`.
418 0 : pub(super) fn without_reason(&self, reason: GcBlockingReason) -> Option<Self> {
419 0 : assert!(self.blocked_by(reason));
420 :
421 0 : if self.reasons.len() == 1 {
422 0 : None
423 : } else {
424 0 : let mut reasons = self.reasons;
425 0 : assert!(reasons.remove(reason));
426 0 : assert!(!reasons.is_empty());
427 :
428 0 : Some(Self {
429 0 : started_at: self.started_at,
430 0 : reasons,
431 0 : })
432 : }
433 0 : }
434 : }
435 :
436 : #[cfg(test)]
437 : mod tests {
438 : use postgres_ffi::PgMajorVersion;
439 : use std::str::FromStr;
440 : use utils::id::TimelineId;
441 :
442 : use super::*;
443 :
444 : #[test]
445 1 : fn v1_indexpart_is_parsed() {
446 1 : let example = r#"{
447 1 : "version":1,
448 1 : "timeline_layers":["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"],
449 1 : "layer_metadata":{
450 1 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
451 1 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
452 1 : },
453 1 : "disk_consistent_lsn":"0/16960E8",
454 1 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
455 1 : }"#;
456 :
457 1 : let expected = IndexPart {
458 1 : // note this is not verified, could be anything, but exists for humans debugging.. could be the git version instead?
459 1 : version: 1,
460 1 : layer_metadata: HashMap::from([
461 1 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
462 1 : file_size: 25600000,
463 1 : generation: Generation::none(),
464 1 : shard: ShardIndex::unsharded()
465 1 : }),
466 1 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
467 1 : // serde_json should always parse this but this might be a double with jq for
468 1 : // example.
469 1 : file_size: 9007199254741001,
470 1 : generation: Generation::none(),
471 1 : shard: ShardIndex::unsharded()
472 1 : })
473 1 : ]),
474 1 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
475 1 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
476 1 : deleted_at: None,
477 1 : archived_at: None,
478 1 : lineage: Lineage::default(),
479 1 : gc_blocking: None,
480 1 : last_aux_file_policy: None,
481 1 : import_pgdata: None,
482 1 : rel_size_migration: None,
483 1 : l2_lsn: None,
484 1 : gc_compaction: None,
485 1 : marked_invisible_at: None,
486 1 : rel_size_migrated_at: None,
487 1 : };
488 :
489 1 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
490 1 : assert_eq!(part, expected);
491 1 : }
492 :
493 : #[test]
494 1 : fn v1_indexpart_is_parsed_with_optional_missing_layers() {
495 1 : let example = r#"{
496 1 : "version":1,
497 1 : "timeline_layers":["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"],
498 1 : "missing_layers":["This shouldn't fail deserialization"],
499 1 : "layer_metadata":{
500 1 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
501 1 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
502 1 : },
503 1 : "disk_consistent_lsn":"0/16960E8",
504 1 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
505 1 : }"#;
506 :
507 1 : let expected = IndexPart {
508 1 : // note this is not verified, could be anything, but exists for humans debugging.. could be the git version instead?
509 1 : version: 1,
510 1 : layer_metadata: HashMap::from([
511 1 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
512 1 : file_size: 25600000,
513 1 : generation: Generation::none(),
514 1 : shard: ShardIndex::unsharded()
515 1 : }),
516 1 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
517 1 : // serde_json should always parse this but this might be a double with jq for
518 1 : // example.
519 1 : file_size: 9007199254741001,
520 1 : generation: Generation::none(),
521 1 : shard: ShardIndex::unsharded()
522 1 : })
523 1 : ]),
524 1 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
525 1 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
526 1 : deleted_at: None,
527 1 : archived_at: None,
528 1 : lineage: Lineage::default(),
529 1 : gc_blocking: None,
530 1 : last_aux_file_policy: None,
531 1 : import_pgdata: None,
532 1 : rel_size_migration: None,
533 1 : l2_lsn: None,
534 1 : gc_compaction: None,
535 1 : marked_invisible_at: None,
536 1 : rel_size_migrated_at: None,
537 1 : };
538 :
539 1 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
540 1 : assert_eq!(part, expected);
541 1 : }
542 :
543 : #[test]
544 1 : fn v2_indexpart_is_parsed_with_deleted_at() {
545 1 : let example = r#"{
546 1 : "version":2,
547 1 : "timeline_layers":["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"],
548 1 : "missing_layers":["This shouldn't fail deserialization"],
549 1 : "layer_metadata":{
550 1 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
551 1 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
552 1 : },
553 1 : "disk_consistent_lsn":"0/16960E8",
554 1 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
555 1 : "deleted_at": "2023-07-31T09:00:00.123"
556 1 : }"#;
557 :
558 1 : let expected = IndexPart {
559 1 : // note this is not verified, could be anything, but exists for humans debugging.. could be the git version instead?
560 1 : version: 2,
561 1 : layer_metadata: HashMap::from([
562 1 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
563 1 : file_size: 25600000,
564 1 : generation: Generation::none(),
565 1 : shard: ShardIndex::unsharded()
566 1 : }),
567 1 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
568 1 : // serde_json should always parse this but this might be a double with jq for
569 1 : // example.
570 1 : file_size: 9007199254741001,
571 1 : generation: Generation::none(),
572 1 : shard: ShardIndex::unsharded()
573 1 : })
574 1 : ]),
575 1 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
576 1 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
577 1 : deleted_at: Some(parse_naive_datetime("2023-07-31T09:00:00.123000000")),
578 1 : archived_at: None,
579 1 : lineage: Lineage::default(),
580 1 : gc_blocking: None,
581 1 : last_aux_file_policy: None,
582 1 : import_pgdata: None,
583 1 : rel_size_migration: None,
584 1 : l2_lsn: None,
585 1 : gc_compaction: None,
586 1 : marked_invisible_at: None,
587 1 : rel_size_migrated_at: None,
588 1 : };
589 :
590 1 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
591 1 : assert_eq!(part, expected);
592 1 : }
593 :
594 : #[test]
595 1 : fn empty_layers_are_parsed() {
596 1 : let empty_layers_json = r#"{
597 1 : "version":1,
598 1 : "timeline_layers":[],
599 1 : "layer_metadata":{},
600 1 : "disk_consistent_lsn":"0/2532648",
601 1 : "metadata_bytes":[136,151,49,208,0,70,0,4,0,0,0,0,2,83,38,72,1,0,0,0,0,2,83,38,32,1,87,198,240,135,97,119,45,125,38,29,155,161,140,141,255,210,0,0,0,0,2,83,38,72,0,0,0,0,1,73,240,192,0,0,0,0,1,73,240,192,0,0,0,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
602 1 : }"#;
603 :
604 1 : let expected = IndexPart {
605 1 : version: 1,
606 1 : layer_metadata: HashMap::new(),
607 1 : disk_consistent_lsn: "0/2532648".parse::<Lsn>().unwrap(),
608 1 : metadata: TimelineMetadata::from_bytes(&[
609 1 : 136, 151, 49, 208, 0, 70, 0, 4, 0, 0, 0, 0, 2, 83, 38, 72, 1, 0, 0, 0, 0, 2, 83,
610 1 : 38, 32, 1, 87, 198, 240, 135, 97, 119, 45, 125, 38, 29, 155, 161, 140, 141, 255,
611 1 : 210, 0, 0, 0, 0, 2, 83, 38, 72, 0, 0, 0, 0, 1, 73, 240, 192, 0, 0, 0, 0, 1, 73,
612 1 : 240, 192, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
613 1 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
614 1 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
615 1 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
616 1 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
617 1 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
618 1 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
619 1 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
620 1 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
621 1 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
622 1 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
623 1 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
624 1 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
625 1 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
626 1 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
627 1 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
628 1 : 0, 0,
629 1 : ])
630 1 : .unwrap(),
631 1 : deleted_at: None,
632 1 : archived_at: None,
633 1 : lineage: Lineage::default(),
634 1 : gc_blocking: None,
635 1 : last_aux_file_policy: None,
636 1 : import_pgdata: None,
637 1 : rel_size_migration: None,
638 1 : l2_lsn: None,
639 1 : gc_compaction: None,
640 1 : marked_invisible_at: None,
641 1 : rel_size_migrated_at: None,
642 1 : };
643 :
644 1 : let empty_layers_parsed = IndexPart::from_json_bytes(empty_layers_json.as_bytes()).unwrap();
645 :
646 1 : assert_eq!(empty_layers_parsed, expected);
647 1 : }
648 :
649 : #[test]
650 1 : fn v4_indexpart_is_parsed() {
651 1 : let example = r#"{
652 1 : "version":4,
653 1 : "layer_metadata":{
654 1 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
655 1 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
656 1 : },
657 1 : "disk_consistent_lsn":"0/16960E8",
658 1 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
659 1 : "deleted_at": "2023-07-31T09:00:00.123"
660 1 : }"#;
661 :
662 1 : let expected = IndexPart {
663 1 : version: 4,
664 1 : layer_metadata: HashMap::from([
665 1 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
666 1 : file_size: 25600000,
667 1 : generation: Generation::none(),
668 1 : shard: ShardIndex::unsharded()
669 1 : }),
670 1 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
671 1 : // serde_json should always parse this but this might be a double with jq for
672 1 : // example.
673 1 : file_size: 9007199254741001,
674 1 : generation: Generation::none(),
675 1 : shard: ShardIndex::unsharded()
676 1 : })
677 1 : ]),
678 1 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
679 1 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
680 1 : deleted_at: Some(parse_naive_datetime("2023-07-31T09:00:00.123000000")),
681 1 : archived_at: None,
682 1 : lineage: Lineage::default(),
683 1 : gc_blocking: None,
684 1 : last_aux_file_policy: None,
685 1 : import_pgdata: None,
686 1 : rel_size_migration: None,
687 1 : l2_lsn: None,
688 1 : gc_compaction: None,
689 1 : marked_invisible_at: None,
690 1 : rel_size_migrated_at: None,
691 1 : };
692 :
693 1 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
694 1 : assert_eq!(part, expected);
695 1 : }
696 :
697 : #[test]
698 1 : fn v5_indexpart_is_parsed() {
699 1 : let example = r#"{
700 1 : "version":5,
701 1 : "layer_metadata":{
702 1 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000014EF420-00000000014EF499":{"file_size":23289856,"generation":1},
703 1 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000014EF499-00000000015A7619":{"file_size":1015808,"generation":1}},
704 1 : "disk_consistent_lsn":"0/15A7618",
705 1 : "metadata_bytes":[226,88,25,241,0,46,0,4,0,0,0,0,1,90,118,24,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,78,244,32,0,0,0,0,1,78,244,32,0,0,0,16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
706 1 : "lineage":{
707 1 : "original_ancestor":["e2bfd8c633d713d279e6fcd2bcc15b6d","0/15A7618","2024-05-07T18:52:36.322426563"],
708 1 : "reparenting_history":["e1bfd8c633d713d279e6fcd2bcc15b6d"]
709 1 : }
710 1 : }"#;
711 :
712 1 : let expected = IndexPart {
713 1 : version: 5,
714 1 : layer_metadata: HashMap::from([
715 1 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000014EF420-00000000014EF499".parse().unwrap(), LayerFileMetadata {
716 1 : file_size: 23289856,
717 1 : generation: Generation::new(1),
718 1 : shard: ShardIndex::unsharded(),
719 1 : }),
720 1 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000014EF499-00000000015A7619".parse().unwrap(), LayerFileMetadata {
721 1 : file_size: 1015808,
722 1 : generation: Generation::new(1),
723 1 : shard: ShardIndex::unsharded(),
724 1 : })
725 1 : ]),
726 1 : disk_consistent_lsn: Lsn::from_str("0/15A7618").unwrap(),
727 1 : metadata: TimelineMetadata::from_bytes(&[226,88,25,241,0,46,0,4,0,0,0,0,1,90,118,24,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,78,244,32,0,0,0,0,1,78,244,32,0,0,0,16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
728 1 : deleted_at: None,
729 1 : archived_at: None,
730 1 : lineage: Lineage {
731 1 : reparenting_history_truncated: false,
732 1 : reparenting_history: vec![TimelineId::from_str("e1bfd8c633d713d279e6fcd2bcc15b6d").unwrap()],
733 1 : original_ancestor: Some((TimelineId::from_str("e2bfd8c633d713d279e6fcd2bcc15b6d").unwrap(), Lsn::from_str("0/15A7618").unwrap(), parse_naive_datetime("2024-05-07T18:52:36.322426563"))),
734 1 : },
735 1 : gc_blocking: None,
736 1 : last_aux_file_policy: None,
737 1 : import_pgdata: None,
738 1 : rel_size_migration: None,
739 1 : l2_lsn: None,
740 1 : gc_compaction: None,
741 1 : marked_invisible_at: None,
742 1 : rel_size_migrated_at: None,
743 1 : };
744 :
745 1 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
746 1 : assert_eq!(part, expected);
747 1 : }
748 :
749 : #[test]
750 1 : fn v6_indexpart_is_parsed() {
751 1 : let example = r#"{
752 1 : "version":6,
753 1 : "layer_metadata":{
754 1 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
755 1 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
756 1 : },
757 1 : "disk_consistent_lsn":"0/16960E8",
758 1 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
759 1 : "deleted_at": "2023-07-31T09:00:00.123",
760 1 : "lineage":{
761 1 : "original_ancestor":["e2bfd8c633d713d279e6fcd2bcc15b6d","0/15A7618","2024-05-07T18:52:36.322426563"],
762 1 : "reparenting_history":["e1bfd8c633d713d279e6fcd2bcc15b6d"]
763 1 : },
764 1 : "last_aux_file_policy": "V2"
765 1 : }"#;
766 :
767 1 : let expected = IndexPart {
768 1 : version: 6,
769 1 : layer_metadata: HashMap::from([
770 1 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
771 1 : file_size: 25600000,
772 1 : generation: Generation::none(),
773 1 : shard: ShardIndex::unsharded()
774 1 : }),
775 1 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
776 1 : // serde_json should always parse this but this might be a double with jq for
777 1 : // example.
778 1 : file_size: 9007199254741001,
779 1 : generation: Generation::none(),
780 1 : shard: ShardIndex::unsharded()
781 1 : })
782 1 : ]),
783 1 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
784 1 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
785 1 : deleted_at: Some(parse_naive_datetime("2023-07-31T09:00:00.123000000")),
786 1 : archived_at: None,
787 1 : lineage: Lineage {
788 1 : reparenting_history_truncated: false,
789 1 : reparenting_history: vec![TimelineId::from_str("e1bfd8c633d713d279e6fcd2bcc15b6d").unwrap()],
790 1 : original_ancestor: Some((TimelineId::from_str("e2bfd8c633d713d279e6fcd2bcc15b6d").unwrap(), Lsn::from_str("0/15A7618").unwrap(), parse_naive_datetime("2024-05-07T18:52:36.322426563"))),
791 1 : },
792 1 : gc_blocking: None,
793 1 : last_aux_file_policy: Some(AuxFilePolicy::V2),
794 1 : import_pgdata: None,
795 1 : rel_size_migration: None,
796 1 : l2_lsn: None,
797 1 : gc_compaction: None,
798 1 : marked_invisible_at: None,
799 1 : rel_size_migrated_at: None,
800 1 : };
801 :
802 1 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
803 1 : assert_eq!(part, expected);
804 1 : }
805 :
806 : #[test]
807 1 : fn v7_indexpart_is_parsed() {
808 1 : let example = r#"{
809 1 : "version": 7,
810 1 : "layer_metadata":{
811 1 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
812 1 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
813 1 : },
814 1 : "disk_consistent_lsn":"0/16960E8",
815 1 : "metadata": {
816 1 : "disk_consistent_lsn": "0/16960E8",
817 1 : "prev_record_lsn": "0/1696070",
818 1 : "ancestor_timeline": "e45a7f37d3ee2ff17dc14bf4f4e3f52e",
819 1 : "ancestor_lsn": "0/0",
820 1 : "latest_gc_cutoff_lsn": "0/1696070",
821 1 : "initdb_lsn": "0/1696070",
822 1 : "pg_version": 14
823 1 : },
824 1 : "deleted_at": "2023-07-31T09:00:00.123"
825 1 : }"#;
826 :
827 1 : let expected = IndexPart {
828 1 : version: 7,
829 1 : layer_metadata: HashMap::from([
830 1 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
831 1 : file_size: 25600000,
832 1 : generation: Generation::none(),
833 1 : shard: ShardIndex::unsharded()
834 1 : }),
835 1 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
836 1 : file_size: 9007199254741001,
837 1 : generation: Generation::none(),
838 1 : shard: ShardIndex::unsharded()
839 1 : })
840 1 : ]),
841 1 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
842 1 : metadata: TimelineMetadata::new(
843 1 : Lsn::from_str("0/16960E8").unwrap(),
844 1 : Some(Lsn::from_str("0/1696070").unwrap()),
845 1 : Some(TimelineId::from_str("e45a7f37d3ee2ff17dc14bf4f4e3f52e").unwrap()),
846 1 : Lsn::INVALID,
847 1 : Lsn::from_str("0/1696070").unwrap(),
848 1 : Lsn::from_str("0/1696070").unwrap(),
849 1 : PgMajorVersion::PG14,
850 1 : ).with_recalculated_checksum().unwrap(),
851 1 : deleted_at: Some(parse_naive_datetime("2023-07-31T09:00:00.123000000")),
852 1 : archived_at: None,
853 1 : lineage: Default::default(),
854 1 : gc_blocking: None,
855 1 : last_aux_file_policy: Default::default(),
856 1 : import_pgdata: None,
857 1 : rel_size_migration: None,
858 1 : l2_lsn: None,
859 1 : gc_compaction: None,
860 1 : marked_invisible_at: None,
861 1 : rel_size_migrated_at: None,
862 1 : };
863 :
864 1 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
865 1 : assert_eq!(part, expected);
866 1 : }
867 :
868 : #[test]
869 1 : fn v8_indexpart_is_parsed() {
870 1 : let example = r#"{
871 1 : "version": 8,
872 1 : "layer_metadata":{
873 1 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
874 1 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
875 1 : },
876 1 : "disk_consistent_lsn":"0/16960E8",
877 1 : "metadata": {
878 1 : "disk_consistent_lsn": "0/16960E8",
879 1 : "prev_record_lsn": "0/1696070",
880 1 : "ancestor_timeline": "e45a7f37d3ee2ff17dc14bf4f4e3f52e",
881 1 : "ancestor_lsn": "0/0",
882 1 : "latest_gc_cutoff_lsn": "0/1696070",
883 1 : "initdb_lsn": "0/1696070",
884 1 : "pg_version": 14
885 1 : },
886 1 : "deleted_at": "2023-07-31T09:00:00.123",
887 1 : "archived_at": "2023-04-29T09:00:00.123"
888 1 : }"#;
889 :
890 1 : let expected = IndexPart {
891 1 : version: 8,
892 1 : layer_metadata: HashMap::from([
893 1 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
894 1 : file_size: 25600000,
895 1 : generation: Generation::none(),
896 1 : shard: ShardIndex::unsharded()
897 1 : }),
898 1 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
899 1 : file_size: 9007199254741001,
900 1 : generation: Generation::none(),
901 1 : shard: ShardIndex::unsharded()
902 1 : })
903 1 : ]),
904 1 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
905 1 : metadata: TimelineMetadata::new(
906 1 : Lsn::from_str("0/16960E8").unwrap(),
907 1 : Some(Lsn::from_str("0/1696070").unwrap()),
908 1 : Some(TimelineId::from_str("e45a7f37d3ee2ff17dc14bf4f4e3f52e").unwrap()),
909 1 : Lsn::INVALID,
910 1 : Lsn::from_str("0/1696070").unwrap(),
911 1 : Lsn::from_str("0/1696070").unwrap(),
912 1 : PgMajorVersion::PG14,
913 1 : ).with_recalculated_checksum().unwrap(),
914 1 : deleted_at: Some(parse_naive_datetime("2023-07-31T09:00:00.123000000")),
915 1 : archived_at: Some(parse_naive_datetime("2023-04-29T09:00:00.123000000")),
916 1 : lineage: Default::default(),
917 1 : gc_blocking: None,
918 1 : last_aux_file_policy: Default::default(),
919 1 : import_pgdata: None,
920 1 : rel_size_migration: None,
921 1 : l2_lsn: None,
922 1 : gc_compaction: None,
923 1 : marked_invisible_at: None,
924 1 : rel_size_migrated_at: None,
925 1 : };
926 :
927 1 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
928 1 : assert_eq!(part, expected);
929 1 : }
930 :
931 : #[test]
932 1 : fn v9_indexpart_is_parsed() {
933 1 : let example = r#"{
934 1 : "version": 9,
935 1 : "layer_metadata":{
936 1 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
937 1 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
938 1 : },
939 1 : "disk_consistent_lsn":"0/16960E8",
940 1 : "metadata": {
941 1 : "disk_consistent_lsn": "0/16960E8",
942 1 : "prev_record_lsn": "0/1696070",
943 1 : "ancestor_timeline": "e45a7f37d3ee2ff17dc14bf4f4e3f52e",
944 1 : "ancestor_lsn": "0/0",
945 1 : "latest_gc_cutoff_lsn": "0/1696070",
946 1 : "initdb_lsn": "0/1696070",
947 1 : "pg_version": 14
948 1 : },
949 1 : "gc_blocking": {
950 1 : "started_at": "2024-07-19T09:00:00.123",
951 1 : "reasons": ["DetachAncestor"]
952 1 : }
953 1 : }"#;
954 :
955 1 : let expected = IndexPart {
956 1 : version: 9,
957 1 : layer_metadata: HashMap::from([
958 1 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
959 1 : file_size: 25600000,
960 1 : generation: Generation::none(),
961 1 : shard: ShardIndex::unsharded()
962 1 : }),
963 1 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
964 1 : file_size: 9007199254741001,
965 1 : generation: Generation::none(),
966 1 : shard: ShardIndex::unsharded()
967 1 : })
968 1 : ]),
969 1 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
970 1 : metadata: TimelineMetadata::new(
971 1 : Lsn::from_str("0/16960E8").unwrap(),
972 1 : Some(Lsn::from_str("0/1696070").unwrap()),
973 1 : Some(TimelineId::from_str("e45a7f37d3ee2ff17dc14bf4f4e3f52e").unwrap()),
974 1 : Lsn::INVALID,
975 1 : Lsn::from_str("0/1696070").unwrap(),
976 1 : Lsn::from_str("0/1696070").unwrap(),
977 1 : PgMajorVersion::PG14,
978 1 : ).with_recalculated_checksum().unwrap(),
979 1 : deleted_at: None,
980 1 : lineage: Default::default(),
981 1 : gc_blocking: Some(GcBlocking {
982 1 : started_at: parse_naive_datetime("2024-07-19T09:00:00.123000000"),
983 1 : reasons: enumset::EnumSet::from_iter([GcBlockingReason::DetachAncestor]),
984 1 : }),
985 1 : last_aux_file_policy: Default::default(),
986 1 : archived_at: None,
987 1 : import_pgdata: None,
988 1 : rel_size_migration: None,
989 1 : l2_lsn: None,
990 1 : gc_compaction: None,
991 1 : marked_invisible_at: None,
992 1 : rel_size_migrated_at: None,
993 1 : };
994 :
995 1 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
996 1 : assert_eq!(part, expected);
997 1 : }
998 :
999 : #[test]
1000 1 : fn v10_importpgdata_is_parsed() {
1001 1 : let example = r#"{
1002 1 : "version": 10,
1003 1 : "layer_metadata":{
1004 1 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
1005 1 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
1006 1 : },
1007 1 : "disk_consistent_lsn":"0/16960E8",
1008 1 : "metadata": {
1009 1 : "disk_consistent_lsn": "0/16960E8",
1010 1 : "prev_record_lsn": "0/1696070",
1011 1 : "ancestor_timeline": "e45a7f37d3ee2ff17dc14bf4f4e3f52e",
1012 1 : "ancestor_lsn": "0/0",
1013 1 : "latest_gc_cutoff_lsn": "0/1696070",
1014 1 : "initdb_lsn": "0/1696070",
1015 1 : "pg_version": 14
1016 1 : },
1017 1 : "gc_blocking": {
1018 1 : "started_at": "2024-07-19T09:00:00.123",
1019 1 : "reasons": ["DetachAncestor"]
1020 1 : },
1021 1 : "import_pgdata": {
1022 1 : "V1": {
1023 1 : "Done": {
1024 1 : "idempotency_key": "specified-by-client-218a5213-5044-4562-a28d-d024c5f057f5",
1025 1 : "started_at": "2024-11-13T09:23:42.123",
1026 1 : "finished_at": "2024-11-13T09:42:23.123"
1027 1 : }
1028 1 : }
1029 1 : }
1030 1 : }"#;
1031 :
1032 1 : let expected = IndexPart {
1033 1 : version: 10,
1034 1 : layer_metadata: HashMap::from([
1035 1 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
1036 1 : file_size: 25600000,
1037 1 : generation: Generation::none(),
1038 1 : shard: ShardIndex::unsharded()
1039 1 : }),
1040 1 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
1041 1 : file_size: 9007199254741001,
1042 1 : generation: Generation::none(),
1043 1 : shard: ShardIndex::unsharded()
1044 1 : })
1045 1 : ]),
1046 1 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
1047 1 : metadata: TimelineMetadata::new(
1048 1 : Lsn::from_str("0/16960E8").unwrap(),
1049 1 : Some(Lsn::from_str("0/1696070").unwrap()),
1050 1 : Some(TimelineId::from_str("e45a7f37d3ee2ff17dc14bf4f4e3f52e").unwrap()),
1051 1 : Lsn::INVALID,
1052 1 : Lsn::from_str("0/1696070").unwrap(),
1053 1 : Lsn::from_str("0/1696070").unwrap(),
1054 1 : PgMajorVersion::PG14,
1055 1 : ).with_recalculated_checksum().unwrap(),
1056 1 : deleted_at: None,
1057 1 : lineage: Default::default(),
1058 1 : gc_blocking: Some(GcBlocking {
1059 1 : started_at: parse_naive_datetime("2024-07-19T09:00:00.123000000"),
1060 1 : reasons: enumset::EnumSet::from_iter([GcBlockingReason::DetachAncestor]),
1061 1 : }),
1062 1 : last_aux_file_policy: Default::default(),
1063 1 : archived_at: None,
1064 1 : import_pgdata: Some(import_pgdata::index_part_format::Root::V1(import_pgdata::index_part_format::V1::Done(import_pgdata::index_part_format::Done{
1065 1 : started_at: parse_naive_datetime("2024-11-13T09:23:42.123000000"),
1066 1 : finished_at: parse_naive_datetime("2024-11-13T09:42:23.123000000"),
1067 1 : idempotency_key: import_pgdata::index_part_format::IdempotencyKey::new("specified-by-client-218a5213-5044-4562-a28d-d024c5f057f5".to_string()),
1068 1 : }))),
1069 1 : rel_size_migration: None,
1070 1 : l2_lsn: None,
1071 1 : gc_compaction: None,
1072 1 : marked_invisible_at: None,
1073 1 : rel_size_migrated_at: None,
1074 1 : };
1075 :
1076 1 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
1077 1 : assert_eq!(part, expected);
1078 1 : }
1079 :
1080 : #[test]
1081 1 : fn v11_rel_size_migration_is_parsed() {
1082 1 : let example = r#"{
1083 1 : "version": 11,
1084 1 : "layer_metadata":{
1085 1 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
1086 1 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
1087 1 : },
1088 1 : "disk_consistent_lsn":"0/16960E8",
1089 1 : "metadata": {
1090 1 : "disk_consistent_lsn": "0/16960E8",
1091 1 : "prev_record_lsn": "0/1696070",
1092 1 : "ancestor_timeline": "e45a7f37d3ee2ff17dc14bf4f4e3f52e",
1093 1 : "ancestor_lsn": "0/0",
1094 1 : "latest_gc_cutoff_lsn": "0/1696070",
1095 1 : "initdb_lsn": "0/1696070",
1096 1 : "pg_version": 14
1097 1 : },
1098 1 : "gc_blocking": {
1099 1 : "started_at": "2024-07-19T09:00:00.123",
1100 1 : "reasons": ["DetachAncestor"]
1101 1 : },
1102 1 : "import_pgdata": {
1103 1 : "V1": {
1104 1 : "Done": {
1105 1 : "idempotency_key": "specified-by-client-218a5213-5044-4562-a28d-d024c5f057f5",
1106 1 : "started_at": "2024-11-13T09:23:42.123",
1107 1 : "finished_at": "2024-11-13T09:42:23.123"
1108 1 : }
1109 1 : }
1110 1 : },
1111 1 : "rel_size_migration": "legacy"
1112 1 : }"#;
1113 :
1114 1 : let expected = IndexPart {
1115 1 : version: 11,
1116 1 : layer_metadata: HashMap::from([
1117 1 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
1118 1 : file_size: 25600000,
1119 1 : generation: Generation::none(),
1120 1 : shard: ShardIndex::unsharded()
1121 1 : }),
1122 1 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
1123 1 : file_size: 9007199254741001,
1124 1 : generation: Generation::none(),
1125 1 : shard: ShardIndex::unsharded()
1126 1 : })
1127 1 : ]),
1128 1 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
1129 1 : metadata: TimelineMetadata::new(
1130 1 : Lsn::from_str("0/16960E8").unwrap(),
1131 1 : Some(Lsn::from_str("0/1696070").unwrap()),
1132 1 : Some(TimelineId::from_str("e45a7f37d3ee2ff17dc14bf4f4e3f52e").unwrap()),
1133 1 : Lsn::INVALID,
1134 1 : Lsn::from_str("0/1696070").unwrap(),
1135 1 : Lsn::from_str("0/1696070").unwrap(),
1136 1 : PgMajorVersion::PG14,
1137 1 : ).with_recalculated_checksum().unwrap(),
1138 1 : deleted_at: None,
1139 1 : lineage: Default::default(),
1140 1 : gc_blocking: Some(GcBlocking {
1141 1 : started_at: parse_naive_datetime("2024-07-19T09:00:00.123000000"),
1142 1 : reasons: enumset::EnumSet::from_iter([GcBlockingReason::DetachAncestor]),
1143 1 : }),
1144 1 : last_aux_file_policy: Default::default(),
1145 1 : archived_at: None,
1146 1 : import_pgdata: Some(import_pgdata::index_part_format::Root::V1(import_pgdata::index_part_format::V1::Done(import_pgdata::index_part_format::Done{
1147 1 : started_at: parse_naive_datetime("2024-11-13T09:23:42.123000000"),
1148 1 : finished_at: parse_naive_datetime("2024-11-13T09:42:23.123000000"),
1149 1 : idempotency_key: import_pgdata::index_part_format::IdempotencyKey::new("specified-by-client-218a5213-5044-4562-a28d-d024c5f057f5".to_string()),
1150 1 : }))),
1151 1 : rel_size_migration: Some(RelSizeMigration::Legacy),
1152 1 : l2_lsn: None,
1153 1 : gc_compaction: None,
1154 1 : marked_invisible_at: None,
1155 1 : rel_size_migrated_at: None,
1156 1 : };
1157 :
1158 1 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
1159 1 : assert_eq!(part, expected);
1160 1 : }
1161 :
1162 : #[test]
1163 1 : fn v12_v13_l2_gc_ompaction_is_parsed() {
1164 1 : let example = r#"{
1165 1 : "version": 13,
1166 1 : "layer_metadata":{
1167 1 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
1168 1 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
1169 1 : },
1170 1 : "disk_consistent_lsn":"0/16960E8",
1171 1 : "metadata": {
1172 1 : "disk_consistent_lsn": "0/16960E8",
1173 1 : "prev_record_lsn": "0/1696070",
1174 1 : "ancestor_timeline": "e45a7f37d3ee2ff17dc14bf4f4e3f52e",
1175 1 : "ancestor_lsn": "0/0",
1176 1 : "latest_gc_cutoff_lsn": "0/1696070",
1177 1 : "initdb_lsn": "0/1696070",
1178 1 : "pg_version": 14
1179 1 : },
1180 1 : "gc_blocking": {
1181 1 : "started_at": "2024-07-19T09:00:00.123",
1182 1 : "reasons": ["DetachAncestor"]
1183 1 : },
1184 1 : "import_pgdata": {
1185 1 : "V1": {
1186 1 : "Done": {
1187 1 : "idempotency_key": "specified-by-client-218a5213-5044-4562-a28d-d024c5f057f5",
1188 1 : "started_at": "2024-11-13T09:23:42.123",
1189 1 : "finished_at": "2024-11-13T09:42:23.123"
1190 1 : }
1191 1 : }
1192 1 : },
1193 1 : "rel_size_migration": "legacy",
1194 1 : "l2_lsn": "0/16960E8",
1195 1 : "gc_compaction": {
1196 1 : "last_completed_lsn": "0/16960E8"
1197 1 : }
1198 1 : }"#;
1199 :
1200 1 : let expected = IndexPart {
1201 1 : version: 13,
1202 1 : layer_metadata: HashMap::from([
1203 1 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
1204 1 : file_size: 25600000,
1205 1 : generation: Generation::none(),
1206 1 : shard: ShardIndex::unsharded()
1207 1 : }),
1208 1 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
1209 1 : file_size: 9007199254741001,
1210 1 : generation: Generation::none(),
1211 1 : shard: ShardIndex::unsharded()
1212 1 : })
1213 1 : ]),
1214 1 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
1215 1 : metadata: TimelineMetadata::new(
1216 1 : Lsn::from_str("0/16960E8").unwrap(),
1217 1 : Some(Lsn::from_str("0/1696070").unwrap()),
1218 1 : Some(TimelineId::from_str("e45a7f37d3ee2ff17dc14bf4f4e3f52e").unwrap()),
1219 1 : Lsn::INVALID,
1220 1 : Lsn::from_str("0/1696070").unwrap(),
1221 1 : Lsn::from_str("0/1696070").unwrap(),
1222 1 : PgMajorVersion::PG14,
1223 1 : ).with_recalculated_checksum().unwrap(),
1224 1 : deleted_at: None,
1225 1 : lineage: Default::default(),
1226 1 : gc_blocking: Some(GcBlocking {
1227 1 : started_at: parse_naive_datetime("2024-07-19T09:00:00.123000000"),
1228 1 : reasons: enumset::EnumSet::from_iter([GcBlockingReason::DetachAncestor]),
1229 1 : }),
1230 1 : last_aux_file_policy: Default::default(),
1231 1 : archived_at: None,
1232 1 : import_pgdata: Some(import_pgdata::index_part_format::Root::V1(import_pgdata::index_part_format::V1::Done(import_pgdata::index_part_format::Done{
1233 1 : started_at: parse_naive_datetime("2024-11-13T09:23:42.123000000"),
1234 1 : finished_at: parse_naive_datetime("2024-11-13T09:42:23.123000000"),
1235 1 : idempotency_key: import_pgdata::index_part_format::IdempotencyKey::new("specified-by-client-218a5213-5044-4562-a28d-d024c5f057f5".to_string()),
1236 1 : }))),
1237 1 : rel_size_migration: Some(RelSizeMigration::Legacy),
1238 1 : l2_lsn: Some("0/16960E8".parse::<Lsn>().unwrap()),
1239 1 : gc_compaction: Some(GcCompactionState {
1240 1 : last_completed_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
1241 1 : }),
1242 1 : marked_invisible_at: None,
1243 1 : rel_size_migrated_at: None,
1244 1 : };
1245 :
1246 1 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
1247 1 : assert_eq!(part, expected);
1248 1 : }
1249 :
1250 : #[test]
1251 1 : fn v14_marked_invisible_at_is_parsed() {
1252 1 : let example = r#"{
1253 1 : "version": 14,
1254 1 : "layer_metadata":{
1255 1 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
1256 1 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
1257 1 : },
1258 1 : "disk_consistent_lsn":"0/16960E8",
1259 1 : "metadata": {
1260 1 : "disk_consistent_lsn": "0/16960E8",
1261 1 : "prev_record_lsn": "0/1696070",
1262 1 : "ancestor_timeline": "e45a7f37d3ee2ff17dc14bf4f4e3f52e",
1263 1 : "ancestor_lsn": "0/0",
1264 1 : "latest_gc_cutoff_lsn": "0/1696070",
1265 1 : "initdb_lsn": "0/1696070",
1266 1 : "pg_version": 14
1267 1 : },
1268 1 : "gc_blocking": {
1269 1 : "started_at": "2024-07-19T09:00:00.123",
1270 1 : "reasons": ["DetachAncestor"]
1271 1 : },
1272 1 : "import_pgdata": {
1273 1 : "V1": {
1274 1 : "Done": {
1275 1 : "idempotency_key": "specified-by-client-218a5213-5044-4562-a28d-d024c5f057f5",
1276 1 : "started_at": "2024-11-13T09:23:42.123",
1277 1 : "finished_at": "2024-11-13T09:42:23.123"
1278 1 : }
1279 1 : }
1280 1 : },
1281 1 : "rel_size_migration": "legacy",
1282 1 : "l2_lsn": "0/16960E8",
1283 1 : "gc_compaction": {
1284 1 : "last_completed_lsn": "0/16960E8"
1285 1 : },
1286 1 : "marked_invisible_at": "2023-07-31T09:00:00.123"
1287 1 : }"#;
1288 :
1289 1 : let expected = IndexPart {
1290 1 : version: 14,
1291 1 : layer_metadata: HashMap::from([
1292 1 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
1293 1 : file_size: 25600000,
1294 1 : generation: Generation::none(),
1295 1 : shard: ShardIndex::unsharded()
1296 1 : }),
1297 1 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
1298 1 : file_size: 9007199254741001,
1299 1 : generation: Generation::none(),
1300 1 : shard: ShardIndex::unsharded()
1301 1 : })
1302 1 : ]),
1303 1 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
1304 1 : metadata: TimelineMetadata::new(
1305 1 : Lsn::from_str("0/16960E8").unwrap(),
1306 1 : Some(Lsn::from_str("0/1696070").unwrap()),
1307 1 : Some(TimelineId::from_str("e45a7f37d3ee2ff17dc14bf4f4e3f52e").unwrap()),
1308 1 : Lsn::INVALID,
1309 1 : Lsn::from_str("0/1696070").unwrap(),
1310 1 : Lsn::from_str("0/1696070").unwrap(),
1311 1 : PgMajorVersion::PG14,
1312 1 : ).with_recalculated_checksum().unwrap(),
1313 1 : deleted_at: None,
1314 1 : lineage: Default::default(),
1315 1 : gc_blocking: Some(GcBlocking {
1316 1 : started_at: parse_naive_datetime("2024-07-19T09:00:00.123000000"),
1317 1 : reasons: enumset::EnumSet::from_iter([GcBlockingReason::DetachAncestor]),
1318 1 : }),
1319 1 : last_aux_file_policy: Default::default(),
1320 1 : archived_at: None,
1321 1 : import_pgdata: Some(import_pgdata::index_part_format::Root::V1(import_pgdata::index_part_format::V1::Done(import_pgdata::index_part_format::Done{
1322 1 : started_at: parse_naive_datetime("2024-11-13T09:23:42.123000000"),
1323 1 : finished_at: parse_naive_datetime("2024-11-13T09:42:23.123000000"),
1324 1 : idempotency_key: import_pgdata::index_part_format::IdempotencyKey::new("specified-by-client-218a5213-5044-4562-a28d-d024c5f057f5".to_string()),
1325 1 : }))),
1326 1 : rel_size_migration: Some(RelSizeMigration::Legacy),
1327 1 : l2_lsn: Some("0/16960E8".parse::<Lsn>().unwrap()),
1328 1 : gc_compaction: Some(GcCompactionState {
1329 1 : last_completed_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
1330 1 : }),
1331 1 : marked_invisible_at: Some(parse_naive_datetime("2023-07-31T09:00:00.123000000")),
1332 1 : rel_size_migrated_at: None,
1333 1 : };
1334 :
1335 1 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
1336 1 : assert_eq!(part, expected);
1337 1 : }
1338 :
1339 : #[test]
1340 1 : fn v15_rel_size_migrated_at_is_parsed() {
1341 1 : let example = r#"{
1342 1 : "version": 15,
1343 1 : "layer_metadata":{
1344 1 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
1345 1 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
1346 1 : },
1347 1 : "disk_consistent_lsn":"0/16960E8",
1348 1 : "metadata": {
1349 1 : "disk_consistent_lsn": "0/16960E8",
1350 1 : "prev_record_lsn": "0/1696070",
1351 1 : "ancestor_timeline": "e45a7f37d3ee2ff17dc14bf4f4e3f52e",
1352 1 : "ancestor_lsn": "0/0",
1353 1 : "latest_gc_cutoff_lsn": "0/1696070",
1354 1 : "initdb_lsn": "0/1696070",
1355 1 : "pg_version": 14
1356 1 : },
1357 1 : "gc_blocking": {
1358 1 : "started_at": "2024-07-19T09:00:00.123",
1359 1 : "reasons": ["DetachAncestor"]
1360 1 : },
1361 1 : "import_pgdata": {
1362 1 : "V1": {
1363 1 : "Done": {
1364 1 : "idempotency_key": "specified-by-client-218a5213-5044-4562-a28d-d024c5f057f5",
1365 1 : "started_at": "2024-11-13T09:23:42.123",
1366 1 : "finished_at": "2024-11-13T09:42:23.123"
1367 1 : }
1368 1 : }
1369 1 : },
1370 1 : "rel_size_migration": "legacy",
1371 1 : "l2_lsn": "0/16960E8",
1372 1 : "gc_compaction": {
1373 1 : "last_completed_lsn": "0/16960E8"
1374 1 : },
1375 1 : "marked_invisible_at": "2023-07-31T09:00:00.123",
1376 1 : "rel_size_migrated_at": "0/16960E8"
1377 1 : }"#;
1378 :
1379 1 : let expected = IndexPart {
1380 1 : version: 15,
1381 1 : layer_metadata: HashMap::from([
1382 1 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
1383 1 : file_size: 25600000,
1384 1 : generation: Generation::none(),
1385 1 : shard: ShardIndex::unsharded()
1386 1 : }),
1387 1 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
1388 1 : file_size: 9007199254741001,
1389 1 : generation: Generation::none(),
1390 1 : shard: ShardIndex::unsharded()
1391 1 : })
1392 1 : ]),
1393 1 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
1394 1 : metadata: TimelineMetadata::new(
1395 1 : Lsn::from_str("0/16960E8").unwrap(),
1396 1 : Some(Lsn::from_str("0/1696070").unwrap()),
1397 1 : Some(TimelineId::from_str("e45a7f37d3ee2ff17dc14bf4f4e3f52e").unwrap()),
1398 1 : Lsn::INVALID,
1399 1 : Lsn::from_str("0/1696070").unwrap(),
1400 1 : Lsn::from_str("0/1696070").unwrap(),
1401 1 : PgMajorVersion::PG14,
1402 1 : ).with_recalculated_checksum().unwrap(),
1403 1 : deleted_at: None,
1404 1 : lineage: Default::default(),
1405 1 : gc_blocking: Some(GcBlocking {
1406 1 : started_at: parse_naive_datetime("2024-07-19T09:00:00.123000000"),
1407 1 : reasons: enumset::EnumSet::from_iter([GcBlockingReason::DetachAncestor]),
1408 1 : }),
1409 1 : last_aux_file_policy: Default::default(),
1410 1 : archived_at: None,
1411 1 : import_pgdata: Some(import_pgdata::index_part_format::Root::V1(import_pgdata::index_part_format::V1::Done(import_pgdata::index_part_format::Done{
1412 1 : started_at: parse_naive_datetime("2024-11-13T09:23:42.123000000"),
1413 1 : finished_at: parse_naive_datetime("2024-11-13T09:42:23.123000000"),
1414 1 : idempotency_key: import_pgdata::index_part_format::IdempotencyKey::new("specified-by-client-218a5213-5044-4562-a28d-d024c5f057f5".to_string()),
1415 1 : }))),
1416 1 : rel_size_migration: Some(RelSizeMigration::Legacy),
1417 1 : l2_lsn: Some("0/16960E8".parse::<Lsn>().unwrap()),
1418 1 : gc_compaction: Some(GcCompactionState {
1419 1 : last_completed_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
1420 1 : }),
1421 1 : marked_invisible_at: Some(parse_naive_datetime("2023-07-31T09:00:00.123000000")),
1422 1 : rel_size_migrated_at: Some("0/16960E8".parse::<Lsn>().unwrap()),
1423 1 : };
1424 :
1425 1 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
1426 1 : assert_eq!(part, expected);
1427 1 : }
1428 :
1429 26 : fn parse_naive_datetime(s: &str) -> NaiveDateTime {
1430 26 : chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S.%f").unwrap()
1431 26 : }
1432 : }
|
