Line data Source code
1 : //! In-memory index to track the tenant files on the remote storage.
2 : //!
3 : //! Able to restore itself from the storage index parts, that are located in every timeline's remote directory and contain all data about
4 : //! remote timeline layers and its metadata.
5 :
6 : use std::collections::HashMap;
7 :
8 : use chrono::NaiveDateTime;
9 : use pageserver_api::models::AuxFilePolicy;
10 : use pageserver_api::models::RelSizeMigration;
11 : use pageserver_api::shard::ShardIndex;
12 : use serde::{Deserialize, Serialize};
13 : use utils::id::TimelineId;
14 : use utils::lsn::Lsn;
15 :
16 : use super::is_same_remote_layer_path;
17 : use crate::tenant::Generation;
18 : use crate::tenant::metadata::TimelineMetadata;
19 : use crate::tenant::storage_layer::LayerName;
20 : use crate::tenant::timeline::import_pgdata;
21 :
22 : /// In-memory representation of an `index_part.json` file
23 : ///
24 : /// Contains the data about all files in the timeline, present remotely and its metadata.
25 : ///
26 : /// This type needs to be backwards and forwards compatible. When changing the fields,
27 : /// remember to add a test case for the changed version.
28 18438 : #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
29 : pub struct IndexPart {
30 : /// Debugging aid describing the version of this type.
31 : #[serde(default)]
32 : version: usize,
33 :
34 : #[serde(default)]
35 : #[serde(skip_serializing_if = "Option::is_none")]
36 : pub deleted_at: Option<NaiveDateTime>,
37 :
38 : #[serde(default)]
39 : #[serde(skip_serializing_if = "Option::is_none")]
40 : pub archived_at: Option<NaiveDateTime>,
41 :
42 : /// This field supports import-from-pgdata ("fast imports" platform feature).
43 : /// We don't currently use fast imports, so, this field is None for all production timelines.
44 : /// See <https://github.com/neondatabase/neon/pull/9218> for more information.
45 : #[serde(default)]
46 : #[serde(skip_serializing_if = "Option::is_none")]
47 : pub import_pgdata: Option<import_pgdata::index_part_format::Root>,
48 :
49 : /// Layer filenames and metadata. For an index persisted in remote storage, all layers must
50 : /// exist in remote storage.
51 : pub layer_metadata: HashMap<LayerName, LayerFileMetadata>,
52 :
53 : /// Because of the trouble of eyeballing the legacy "metadata" field, we copied the
54 : /// "disk_consistent_lsn" out. After version 7 this is no longer needed, but the name cannot be
55 : /// reused.
56 : pub(super) disk_consistent_lsn: Lsn,
57 :
58 : // TODO: rename as "metadata" next week, keep the alias = "metadata_bytes", bump version Adding
59 : // the "alias = metadata" was forgotten in #7693, so we have to use "rewrite = metadata_bytes"
60 : // for backwards compatibility.
61 : #[serde(
62 : rename = "metadata_bytes",
63 : alias = "metadata",
64 : with = "crate::tenant::metadata::modern_serde"
65 : )]
66 : pub metadata: TimelineMetadata,
67 :
68 : #[serde(default)]
69 : pub(crate) lineage: Lineage,
70 :
71 : #[serde(skip_serializing_if = "Option::is_none", default)]
72 : pub(crate) gc_blocking: Option<GcBlocking>,
73 :
74 : /// Describes the kind of aux files stored in the timeline.
75 : ///
76 : /// The value is modified during file ingestion when the latest wanted value communicated via tenant config is applied if it is acceptable.
77 : /// A V1 setting after V2 files have been committed is not accepted.
78 : ///
79 : /// None means no aux files have been written to the storage before the point
80 : /// when this flag is introduced.
81 : ///
82 : /// This flag is not used any more as all tenants have been transitioned to the new aux file policy.
83 : #[serde(skip_serializing_if = "Option::is_none", default)]
84 : pub(crate) last_aux_file_policy: Option<AuxFilePolicy>,
85 :
86 : #[serde(skip_serializing_if = "Option::is_none", default)]
87 : pub(crate) rel_size_migration: Option<RelSizeMigration>,
88 :
89 : /// Not used anymore -- kept here for backwards compatibility. Merged into the `gc_compaction` field.
90 : #[serde(skip_serializing_if = "Option::is_none", default)]
91 : l2_lsn: Option<Lsn>,
92 :
93 : /// State for the garbage-collecting compaction pass.
94 : ///
95 : /// Garbage-collecting compaction (gc-compaction) prunes `Value`s that are outside
96 : /// the PITR window and not needed by child timelines.
97 : ///
98 : /// A commonly used synonym for this compaction pass is
99 : /// "bottommost-compaction" because the affected LSN range
100 : /// is the "bottom" of the (key,lsn) map.
101 : ///
102 : /// Gc-compaction is a quite expensive operation; that's why we use
103 : /// trigger condition.
104 : /// This field here holds the state pertaining to that trigger condition
105 : /// and (in future) to the progress of the gc-compaction, so that it's
106 : /// resumable across restarts & migrations.
107 : ///
108 : /// Note that the underlying algorithm is _also_ called `gc-compaction`
109 : /// in most places & design docs; but in fact it is more flexible than
110 : /// just the specific use case here; it needs a new name.
111 : #[serde(skip_serializing_if = "Option::is_none", default)]
112 : pub(crate) gc_compaction: Option<GcCompactionState>,
113 :
114 : /// The timestamp when the timeline was marked invisible in synthetic size calculations.
115 : #[serde(skip_serializing_if = "Option::is_none", default)]
116 : pub(crate) marked_invisible_at: Option<NaiveDateTime>,
117 : }
118 :
119 24 : #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
120 : pub struct GcCompactionState {
121 : /// The upper bound of the last completed garbage-collecting compaction, aka. L2 LSN.
122 : pub(crate) last_completed_lsn: Lsn,
123 : }
124 :
125 : impl IndexPart {
126 : /// When adding or modifying any parts of `IndexPart`, increment the version so that it can be
127 : /// used to understand later versions.
128 : ///
129 : /// Version is currently informative only.
130 : /// Version history
131 : /// - 2: added `deleted_at`
132 : /// - 3: no longer deserialize `timeline_layers` (serialized format is the same, but timeline_layers
133 : /// is always generated from the keys of `layer_metadata`)
134 : /// - 4: timeline_layers is fully removed.
135 : /// - 5: lineage was added
136 : /// - 6: last_aux_file_policy is added.
137 : /// - 7: metadata_bytes is no longer written, but still read
138 : /// - 8: added `archived_at`
139 : /// - 9: +gc_blocking
140 : /// - 10: +import_pgdata
141 : /// - 11: +rel_size_migration
142 : /// - 12: +l2_lsn
143 : /// - 13: +gc_compaction
144 : /// - 14: +marked_invisible_at
145 : const LATEST_VERSION: usize = 14;
146 :
147 : // Versions we may see when reading from a bucket.
148 : pub const KNOWN_VERSIONS: &'static [usize] = &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14];
149 :
150 : pub const FILE_NAME: &'static str = "index_part.json";
151 :
152 3264 : pub fn empty(metadata: TimelineMetadata) -> Self {
153 3264 : IndexPart {
154 3264 : version: Self::LATEST_VERSION,
155 3264 : layer_metadata: Default::default(),
156 3264 : disk_consistent_lsn: metadata.disk_consistent_lsn(),
157 3264 : metadata,
158 3264 : deleted_at: None,
159 3264 : archived_at: None,
160 3264 : lineage: Default::default(),
161 3264 : gc_blocking: None,
162 3264 : last_aux_file_policy: None,
163 3264 : import_pgdata: None,
164 3264 : rel_size_migration: None,
165 3264 : l2_lsn: None,
166 3264 : gc_compaction: None,
167 3264 : marked_invisible_at: None,
168 3264 : }
169 3264 : }
170 :
171 0 : pub fn version(&self) -> usize {
172 0 : self.version
173 0 : }
174 :
175 : /// If you want this under normal operations, read it from self.metadata:
176 : /// this method is just for the scrubber to use when validating an index.
177 0 : pub fn duplicated_disk_consistent_lsn(&self) -> Lsn {
178 0 : self.disk_consistent_lsn
179 0 : }
180 :
181 168 : pub fn from_json_bytes(bytes: &[u8]) -> Result<Self, serde_json::Error> {
182 168 : serde_json::from_slice::<IndexPart>(bytes)
183 168 : }
184 :
185 8994 : pub fn to_json_bytes(&self) -> serde_json::Result<Vec<u8>> {
186 8994 : serde_json::to_vec(self)
187 8994 : }
188 :
189 : #[cfg(test)]
190 168 : pub(crate) fn example() -> Self {
191 168 : Self::empty(TimelineMetadata::example())
192 168 : }
193 :
194 : /// Returns true if the index contains a reference to the given layer (i.e. file path).
195 : ///
196 : /// TODO: there should be a variant of LayerName for the physical remote path that contains
197 : /// information about the shard and generation, to avoid passing in metadata.
198 206475 : pub fn references(&self, name: &LayerName, metadata: &LayerFileMetadata) -> bool {
199 206475 : let Some(index_metadata) = self.layer_metadata.get(name) else {
200 102563 : return false;
201 : };
202 103912 : is_same_remote_layer_path(name, metadata, name, index_metadata)
203 206475 : }
204 :
205 : /// Check for invariants in the index: this is useful when uploading an index to ensure that if
206 : /// we encounter a bug, we do not persist buggy metadata.
207 16410 : pub(crate) fn validate(&self) -> Result<(), String> {
208 16410 : if self.import_pgdata.is_none()
209 16410 : && self.metadata.ancestor_timeline().is_none()
210 12183 : && self.layer_metadata.is_empty()
211 : {
212 : // Unless we're in the middle of a raw pgdata import, or this is a child timeline,the index must
213 : // always have at least one layer.
214 0 : return Err("Index has no ancestor and no layers".to_string());
215 16410 : }
216 16410 :
217 16410 : Ok(())
218 16410 : }
219 : }
220 :
221 : /// Metadata gathered for each of the layer files.
222 : ///
223 : /// Fields have to be `Option`s because remote [`IndexPart`]'s can be from different version, which
224 : /// might have less or more metadata depending if upgrading or rolling back an upgrade.
225 56928 : #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
226 : pub struct LayerFileMetadata {
227 : pub file_size: u64,
228 :
229 : #[serde(default = "Generation::none")]
230 : #[serde(skip_serializing_if = "Generation::is_none")]
231 : pub generation: Generation,
232 :
233 : #[serde(default = "ShardIndex::unsharded")]
234 : #[serde(skip_serializing_if = "ShardIndex::is_unsharded")]
235 : pub shard: ShardIndex,
236 : }
237 :
238 : impl LayerFileMetadata {
239 19225 : pub fn new(file_size: u64, generation: Generation, shard: ShardIndex) -> Self {
240 19225 : LayerFileMetadata {
241 19225 : file_size,
242 19225 : generation,
243 19225 : shard,
244 19225 : }
245 19225 : }
246 : /// Helper to get both generation and file size in a tuple
247 0 : pub fn generation_file_size(&self) -> (Generation, u64) {
248 0 : (self.generation, self.file_size)
249 0 : }
250 : }
251 :
252 : /// Limited history of earlier ancestors.
253 : ///
254 : /// A timeline can have more than 1 earlier ancestor, in the rare case that it was repeatedly
255 : /// reparented by having an later timeline be detached from it's ancestor.
256 48 : #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Default)]
257 : pub(crate) struct Lineage {
258 : /// Has the `reparenting_history` been truncated to [`Lineage::REMEMBER_AT_MOST`].
259 : #[serde(skip_serializing_if = "is_false", default)]
260 : reparenting_history_truncated: bool,
261 :
262 : /// Earlier ancestors, truncated when [`Self::reparenting_history_truncated`]
263 : ///
264 : /// These are stored in case we want to support WAL based DR on the timeline. There can be many
265 : /// of these and at most one [`Self::original_ancestor`]. There cannot be more reparentings
266 : /// after [`Self::original_ancestor`] has been set.
267 : #[serde(skip_serializing_if = "Vec::is_empty", default)]
268 : reparenting_history: Vec<TimelineId>,
269 :
270 : /// The ancestor from which this timeline has been detached from and when.
271 : ///
272 : /// If you are adding support for detaching from a hierarchy, consider changing the ancestry
273 : /// into a `Vec<(TimelineId, Lsn)>` to be a path instead.
274 : // FIXME: this is insufficient even for path of two timelines for future wal recovery
275 : // purposes:
276 : //
277 : // assuming a "old main" which has received most of the WAL, and has a branch "new main",
278 : // starting a bit before "old main" last_record_lsn. the current version works fine,
279 : // because we will know to replay wal and branch at the recorded Lsn to do wal recovery.
280 : //
281 : // then assuming "new main" would similarly receive a branch right before its last_record_lsn,
282 : // "new new main". the current implementation would just store ("new main", ancestor_lsn, _)
283 : // here. however, we cannot recover from WAL using only that information, we would need the
284 : // whole ancestry here:
285 : //
286 : // ```json
287 : // [
288 : // ["old main", ancestor_lsn("new main"), _],
289 : // ["new main", ancestor_lsn("new new main"), _]
290 : // ]
291 : // ```
292 : #[serde(skip_serializing_if = "Option::is_none", default)]
293 : original_ancestor: Option<(TimelineId, Lsn, NaiveDateTime)>,
294 : }
295 :
296 36876 : fn is_false(b: &bool) -> bool {
297 36876 : !b
298 36876 : }
299 :
300 : impl Lineage {
301 : const REMEMBER_AT_MOST: usize = 100;
302 :
303 0 : pub(crate) fn record_previous_ancestor(&mut self, old_ancestor: &TimelineId) -> bool {
304 0 : if self.reparenting_history.last() == Some(old_ancestor) {
305 : // do not re-record it
306 0 : false
307 : } else {
308 : #[cfg(feature = "testing")]
309 : {
310 0 : let existing = self
311 0 : .reparenting_history
312 0 : .iter()
313 0 : .position(|x| x == old_ancestor);
314 0 : assert_eq!(
315 : existing, None,
316 0 : "we cannot reparent onto and off and onto the same timeline twice"
317 : );
318 : }
319 0 : let drop_oldest = self.reparenting_history.len() + 1 >= Self::REMEMBER_AT_MOST;
320 0 :
321 0 : self.reparenting_history_truncated |= drop_oldest;
322 0 : if drop_oldest {
323 0 : self.reparenting_history.remove(0);
324 0 : }
325 0 : self.reparenting_history.push(*old_ancestor);
326 0 : true
327 : }
328 0 : }
329 :
330 : /// Returns true if anything changed.
331 0 : pub(crate) fn record_detaching(&mut self, branchpoint: &(TimelineId, Lsn)) -> bool {
332 0 : if let Some((id, lsn, _)) = self.original_ancestor {
333 0 : assert_eq!(
334 0 : &(id, lsn),
335 : branchpoint,
336 0 : "detaching attempt has to be for the same ancestor we are already detached from"
337 : );
338 0 : false
339 : } else {
340 0 : self.original_ancestor =
341 0 : Some((branchpoint.0, branchpoint.1, chrono::Utc::now().naive_utc()));
342 0 : true
343 : }
344 0 : }
345 :
346 : /// The queried lsn is most likely the basebackup lsn, and this answers question "is it allowed
347 : /// to start a read/write primary at this lsn".
348 : ///
349 : /// Returns true if the Lsn was previously our branch point.
350 0 : pub(crate) fn is_previous_ancestor_lsn(&self, lsn: Lsn) -> bool {
351 0 : self.original_ancestor
352 0 : .is_some_and(|(_, ancestor_lsn, _)| ancestor_lsn == lsn)
353 0 : }
354 :
355 : /// Returns true if the timeline originally had an ancestor, and no longer has one.
356 0 : pub(crate) fn is_detached_from_ancestor(&self) -> bool {
357 0 : self.original_ancestor.is_some()
358 0 : }
359 :
360 : /// Returns original ancestor timeline id and lsn that this timeline has been detached from.
361 0 : pub(crate) fn detached_previous_ancestor(&self) -> Option<(TimelineId, Lsn)> {
362 0 : self.original_ancestor.map(|(id, lsn, _)| (id, lsn))
363 0 : }
364 :
365 0 : pub(crate) fn is_reparented(&self) -> bool {
366 0 : !self.reparenting_history.is_empty()
367 0 : }
368 : }
369 :
370 120 : #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
371 : pub(crate) struct GcBlocking {
372 : pub(crate) started_at: NaiveDateTime,
373 : pub(crate) reasons: enumset::EnumSet<GcBlockingReason>,
374 : }
375 :
376 60 : #[derive(Debug, enumset::EnumSetType, serde::Serialize, serde::Deserialize)]
377 : #[enumset(serialize_repr = "list")]
378 : pub(crate) enum GcBlockingReason {
379 : Manual,
380 : DetachAncestor,
381 : }
382 :
383 : impl GcBlocking {
384 0 : pub(super) fn started_now_for(reason: GcBlockingReason) -> Self {
385 0 : GcBlocking {
386 0 : started_at: chrono::Utc::now().naive_utc(),
387 0 : reasons: enumset::EnumSet::only(reason),
388 0 : }
389 0 : }
390 :
391 : /// Returns true if the given reason is one of the reasons why the gc is blocked.
392 0 : pub(crate) fn blocked_by(&self, reason: GcBlockingReason) -> bool {
393 0 : self.reasons.contains(reason)
394 0 : }
395 :
396 : /// Returns a version of self with the given reason.
397 0 : pub(super) fn with_reason(&self, reason: GcBlockingReason) -> Self {
398 0 : assert!(!self.blocked_by(reason));
399 0 : let mut reasons = self.reasons;
400 0 : reasons.insert(reason);
401 0 :
402 0 : Self {
403 0 : started_at: self.started_at,
404 0 : reasons,
405 0 : }
406 0 : }
407 :
408 : /// Returns a version of self without the given reason. Assumption is that if
409 : /// there are no more reasons, we can unblock the gc by returning `None`.
410 0 : pub(super) fn without_reason(&self, reason: GcBlockingReason) -> Option<Self> {
411 0 : assert!(self.blocked_by(reason));
412 :
413 0 : if self.reasons.len() == 1 {
414 0 : None
415 : } else {
416 0 : let mut reasons = self.reasons;
417 0 : assert!(reasons.remove(reason));
418 0 : assert!(!reasons.is_empty());
419 :
420 0 : Some(Self {
421 0 : started_at: self.started_at,
422 0 : reasons,
423 0 : })
424 : }
425 0 : }
426 : }
427 :
428 : #[cfg(test)]
429 : mod tests {
430 : use std::str::FromStr;
431 :
432 : use utils::id::TimelineId;
433 :
434 : use super::*;
435 :
436 : #[test]
437 12 : fn v1_indexpart_is_parsed() {
438 12 : let example = r#"{
439 12 : "version":1,
440 12 : "timeline_layers":["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"],
441 12 : "layer_metadata":{
442 12 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
443 12 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
444 12 : },
445 12 : "disk_consistent_lsn":"0/16960E8",
446 12 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
447 12 : }"#;
448 12 :
449 12 : let expected = IndexPart {
450 12 : // note this is not verified, could be anything, but exists for humans debugging.. could be the git version instead?
451 12 : version: 1,
452 12 : layer_metadata: HashMap::from([
453 12 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
454 12 : file_size: 25600000,
455 12 : generation: Generation::none(),
456 12 : shard: ShardIndex::unsharded()
457 12 : }),
458 12 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
459 12 : // serde_json should always parse this but this might be a double with jq for
460 12 : // example.
461 12 : file_size: 9007199254741001,
462 12 : generation: Generation::none(),
463 12 : shard: ShardIndex::unsharded()
464 12 : })
465 12 : ]),
466 12 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
467 12 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
468 12 : deleted_at: None,
469 12 : archived_at: None,
470 12 : lineage: Lineage::default(),
471 12 : gc_blocking: None,
472 12 : last_aux_file_policy: None,
473 12 : import_pgdata: None,
474 12 : rel_size_migration: None,
475 12 : l2_lsn: None,
476 12 : gc_compaction: None,
477 12 : marked_invisible_at: None,
478 12 : };
479 12 :
480 12 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
481 12 : assert_eq!(part, expected);
482 12 : }
483 :
484 : #[test]
485 12 : fn v1_indexpart_is_parsed_with_optional_missing_layers() {
486 12 : let example = r#"{
487 12 : "version":1,
488 12 : "timeline_layers":["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"],
489 12 : "missing_layers":["This shouldn't fail deserialization"],
490 12 : "layer_metadata":{
491 12 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
492 12 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
493 12 : },
494 12 : "disk_consistent_lsn":"0/16960E8",
495 12 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
496 12 : }"#;
497 12 :
498 12 : let expected = IndexPart {
499 12 : // note this is not verified, could be anything, but exists for humans debugging.. could be the git version instead?
500 12 : version: 1,
501 12 : layer_metadata: HashMap::from([
502 12 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
503 12 : file_size: 25600000,
504 12 : generation: Generation::none(),
505 12 : shard: ShardIndex::unsharded()
506 12 : }),
507 12 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
508 12 : // serde_json should always parse this but this might be a double with jq for
509 12 : // example.
510 12 : file_size: 9007199254741001,
511 12 : generation: Generation::none(),
512 12 : shard: ShardIndex::unsharded()
513 12 : })
514 12 : ]),
515 12 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
516 12 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
517 12 : deleted_at: None,
518 12 : archived_at: None,
519 12 : lineage: Lineage::default(),
520 12 : gc_blocking: None,
521 12 : last_aux_file_policy: None,
522 12 : import_pgdata: None,
523 12 : rel_size_migration: None,
524 12 : l2_lsn: None,
525 12 : gc_compaction: None,
526 12 : marked_invisible_at: None,
527 12 : };
528 12 :
529 12 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
530 12 : assert_eq!(part, expected);
531 12 : }
532 :
533 : #[test]
534 12 : fn v2_indexpart_is_parsed_with_deleted_at() {
535 12 : let example = r#"{
536 12 : "version":2,
537 12 : "timeline_layers":["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"],
538 12 : "missing_layers":["This shouldn't fail deserialization"],
539 12 : "layer_metadata":{
540 12 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
541 12 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
542 12 : },
543 12 : "disk_consistent_lsn":"0/16960E8",
544 12 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
545 12 : "deleted_at": "2023-07-31T09:00:00.123"
546 12 : }"#;
547 12 :
548 12 : let expected = IndexPart {
549 12 : // note this is not verified, could be anything, but exists for humans debugging.. could be the git version instead?
550 12 : version: 2,
551 12 : layer_metadata: HashMap::from([
552 12 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
553 12 : file_size: 25600000,
554 12 : generation: Generation::none(),
555 12 : shard: ShardIndex::unsharded()
556 12 : }),
557 12 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
558 12 : // serde_json should always parse this but this might be a double with jq for
559 12 : // example.
560 12 : file_size: 9007199254741001,
561 12 : generation: Generation::none(),
562 12 : shard: ShardIndex::unsharded()
563 12 : })
564 12 : ]),
565 12 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
566 12 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
567 12 : deleted_at: Some(parse_naive_datetime("2023-07-31T09:00:00.123000000")),
568 12 : archived_at: None,
569 12 : lineage: Lineage::default(),
570 12 : gc_blocking: None,
571 12 : last_aux_file_policy: None,
572 12 : import_pgdata: None,
573 12 : rel_size_migration: None,
574 12 : l2_lsn: None,
575 12 : gc_compaction: None,
576 12 : marked_invisible_at: None,
577 12 : };
578 12 :
579 12 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
580 12 : assert_eq!(part, expected);
581 12 : }
582 :
583 : #[test]
584 12 : fn empty_layers_are_parsed() {
585 12 : let empty_layers_json = r#"{
586 12 : "version":1,
587 12 : "timeline_layers":[],
588 12 : "layer_metadata":{},
589 12 : "disk_consistent_lsn":"0/2532648",
590 12 : "metadata_bytes":[136,151,49,208,0,70,0,4,0,0,0,0,2,83,38,72,1,0,0,0,0,2,83,38,32,1,87,198,240,135,97,119,45,125,38,29,155,161,140,141,255,210,0,0,0,0,2,83,38,72,0,0,0,0,1,73,240,192,0,0,0,0,1,73,240,192,0,0,0,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
591 12 : }"#;
592 12 :
593 12 : let expected = IndexPart {
594 12 : version: 1,
595 12 : layer_metadata: HashMap::new(),
596 12 : disk_consistent_lsn: "0/2532648".parse::<Lsn>().unwrap(),
597 12 : metadata: TimelineMetadata::from_bytes(&[
598 12 : 136, 151, 49, 208, 0, 70, 0, 4, 0, 0, 0, 0, 2, 83, 38, 72, 1, 0, 0, 0, 0, 2, 83,
599 12 : 38, 32, 1, 87, 198, 240, 135, 97, 119, 45, 125, 38, 29, 155, 161, 140, 141, 255,
600 12 : 210, 0, 0, 0, 0, 2, 83, 38, 72, 0, 0, 0, 0, 1, 73, 240, 192, 0, 0, 0, 0, 1, 73,
601 12 : 240, 192, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
602 12 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
603 12 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
604 12 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
605 12 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
606 12 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
607 12 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
608 12 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
609 12 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
610 12 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
611 12 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
612 12 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
613 12 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
614 12 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
615 12 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
616 12 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
617 12 : 0, 0,
618 12 : ])
619 12 : .unwrap(),
620 12 : deleted_at: None,
621 12 : archived_at: None,
622 12 : lineage: Lineage::default(),
623 12 : gc_blocking: None,
624 12 : last_aux_file_policy: None,
625 12 : import_pgdata: None,
626 12 : rel_size_migration: None,
627 12 : l2_lsn: None,
628 12 : gc_compaction: None,
629 12 : marked_invisible_at: None,
630 12 : };
631 12 :
632 12 : let empty_layers_parsed = IndexPart::from_json_bytes(empty_layers_json.as_bytes()).unwrap();
633 12 :
634 12 : assert_eq!(empty_layers_parsed, expected);
635 12 : }
636 :
637 : #[test]
638 12 : fn v4_indexpart_is_parsed() {
639 12 : let example = r#"{
640 12 : "version":4,
641 12 : "layer_metadata":{
642 12 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
643 12 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
644 12 : },
645 12 : "disk_consistent_lsn":"0/16960E8",
646 12 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
647 12 : "deleted_at": "2023-07-31T09:00:00.123"
648 12 : }"#;
649 12 :
650 12 : let expected = IndexPart {
651 12 : version: 4,
652 12 : layer_metadata: HashMap::from([
653 12 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
654 12 : file_size: 25600000,
655 12 : generation: Generation::none(),
656 12 : shard: ShardIndex::unsharded()
657 12 : }),
658 12 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
659 12 : // serde_json should always parse this but this might be a double with jq for
660 12 : // example.
661 12 : file_size: 9007199254741001,
662 12 : generation: Generation::none(),
663 12 : shard: ShardIndex::unsharded()
664 12 : })
665 12 : ]),
666 12 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
667 12 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
668 12 : deleted_at: Some(parse_naive_datetime("2023-07-31T09:00:00.123000000")),
669 12 : archived_at: None,
670 12 : lineage: Lineage::default(),
671 12 : gc_blocking: None,
672 12 : last_aux_file_policy: None,
673 12 : import_pgdata: None,
674 12 : rel_size_migration: None,
675 12 : l2_lsn: None,
676 12 : gc_compaction: None,
677 12 : marked_invisible_at: None,
678 12 : };
679 12 :
680 12 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
681 12 : assert_eq!(part, expected);
682 12 : }
683 :
684 : #[test]
685 12 : fn v5_indexpart_is_parsed() {
686 12 : let example = r#"{
687 12 : "version":5,
688 12 : "layer_metadata":{
689 12 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000014EF420-00000000014EF499":{"file_size":23289856,"generation":1},
690 12 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000014EF499-00000000015A7619":{"file_size":1015808,"generation":1}},
691 12 : "disk_consistent_lsn":"0/15A7618",
692 12 : "metadata_bytes":[226,88,25,241,0,46,0,4,0,0,0,0,1,90,118,24,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,78,244,32,0,0,0,0,1,78,244,32,0,0,0,16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
693 12 : "lineage":{
694 12 : "original_ancestor":["e2bfd8c633d713d279e6fcd2bcc15b6d","0/15A7618","2024-05-07T18:52:36.322426563"],
695 12 : "reparenting_history":["e1bfd8c633d713d279e6fcd2bcc15b6d"]
696 12 : }
697 12 : }"#;
698 12 :
699 12 : let expected = IndexPart {
700 12 : version: 5,
701 12 : layer_metadata: HashMap::from([
702 12 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000014EF420-00000000014EF499".parse().unwrap(), LayerFileMetadata {
703 12 : file_size: 23289856,
704 12 : generation: Generation::new(1),
705 12 : shard: ShardIndex::unsharded(),
706 12 : }),
707 12 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000014EF499-00000000015A7619".parse().unwrap(), LayerFileMetadata {
708 12 : file_size: 1015808,
709 12 : generation: Generation::new(1),
710 12 : shard: ShardIndex::unsharded(),
711 12 : })
712 12 : ]),
713 12 : disk_consistent_lsn: Lsn::from_str("0/15A7618").unwrap(),
714 12 : metadata: TimelineMetadata::from_bytes(&[226,88,25,241,0,46,0,4,0,0,0,0,1,90,118,24,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,78,244,32,0,0,0,0,1,78,244,32,0,0,0,16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
715 12 : deleted_at: None,
716 12 : archived_at: None,
717 12 : lineage: Lineage {
718 12 : reparenting_history_truncated: false,
719 12 : reparenting_history: vec![TimelineId::from_str("e1bfd8c633d713d279e6fcd2bcc15b6d").unwrap()],
720 12 : original_ancestor: Some((TimelineId::from_str("e2bfd8c633d713d279e6fcd2bcc15b6d").unwrap(), Lsn::from_str("0/15A7618").unwrap(), parse_naive_datetime("2024-05-07T18:52:36.322426563"))),
721 12 : },
722 12 : gc_blocking: None,
723 12 : last_aux_file_policy: None,
724 12 : import_pgdata: None,
725 12 : rel_size_migration: None,
726 12 : l2_lsn: None,
727 12 : gc_compaction: None,
728 12 : marked_invisible_at: None,
729 12 : };
730 12 :
731 12 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
732 12 : assert_eq!(part, expected);
733 12 : }
734 :
735 : #[test]
736 12 : fn v6_indexpart_is_parsed() {
737 12 : let example = r#"{
738 12 : "version":6,
739 12 : "layer_metadata":{
740 12 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
741 12 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
742 12 : },
743 12 : "disk_consistent_lsn":"0/16960E8",
744 12 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
745 12 : "deleted_at": "2023-07-31T09:00:00.123",
746 12 : "lineage":{
747 12 : "original_ancestor":["e2bfd8c633d713d279e6fcd2bcc15b6d","0/15A7618","2024-05-07T18:52:36.322426563"],
748 12 : "reparenting_history":["e1bfd8c633d713d279e6fcd2bcc15b6d"]
749 12 : },
750 12 : "last_aux_file_policy": "V2"
751 12 : }"#;
752 12 :
753 12 : let expected = IndexPart {
754 12 : version: 6,
755 12 : layer_metadata: HashMap::from([
756 12 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
757 12 : file_size: 25600000,
758 12 : generation: Generation::none(),
759 12 : shard: ShardIndex::unsharded()
760 12 : }),
761 12 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
762 12 : // serde_json should always parse this but this might be a double with jq for
763 12 : // example.
764 12 : file_size: 9007199254741001,
765 12 : generation: Generation::none(),
766 12 : shard: ShardIndex::unsharded()
767 12 : })
768 12 : ]),
769 12 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
770 12 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
771 12 : deleted_at: Some(parse_naive_datetime("2023-07-31T09:00:00.123000000")),
772 12 : archived_at: None,
773 12 : lineage: Lineage {
774 12 : reparenting_history_truncated: false,
775 12 : reparenting_history: vec![TimelineId::from_str("e1bfd8c633d713d279e6fcd2bcc15b6d").unwrap()],
776 12 : original_ancestor: Some((TimelineId::from_str("e2bfd8c633d713d279e6fcd2bcc15b6d").unwrap(), Lsn::from_str("0/15A7618").unwrap(), parse_naive_datetime("2024-05-07T18:52:36.322426563"))),
777 12 : },
778 12 : gc_blocking: None,
779 12 : last_aux_file_policy: Some(AuxFilePolicy::V2),
780 12 : import_pgdata: None,
781 12 : rel_size_migration: None,
782 12 : l2_lsn: None,
783 12 : gc_compaction: None,
784 12 : marked_invisible_at: None,
785 12 : };
786 12 :
787 12 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
788 12 : assert_eq!(part, expected);
789 12 : }
790 :
791 : #[test]
792 12 : fn v7_indexpart_is_parsed() {
793 12 : let example = r#"{
794 12 : "version": 7,
795 12 : "layer_metadata":{
796 12 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
797 12 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
798 12 : },
799 12 : "disk_consistent_lsn":"0/16960E8",
800 12 : "metadata": {
801 12 : "disk_consistent_lsn": "0/16960E8",
802 12 : "prev_record_lsn": "0/1696070",
803 12 : "ancestor_timeline": "e45a7f37d3ee2ff17dc14bf4f4e3f52e",
804 12 : "ancestor_lsn": "0/0",
805 12 : "latest_gc_cutoff_lsn": "0/1696070",
806 12 : "initdb_lsn": "0/1696070",
807 12 : "pg_version": 14
808 12 : },
809 12 : "deleted_at": "2023-07-31T09:00:00.123"
810 12 : }"#;
811 12 :
812 12 : let expected = IndexPart {
813 12 : version: 7,
814 12 : layer_metadata: HashMap::from([
815 12 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
816 12 : file_size: 25600000,
817 12 : generation: Generation::none(),
818 12 : shard: ShardIndex::unsharded()
819 12 : }),
820 12 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
821 12 : file_size: 9007199254741001,
822 12 : generation: Generation::none(),
823 12 : shard: ShardIndex::unsharded()
824 12 : })
825 12 : ]),
826 12 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
827 12 : metadata: TimelineMetadata::new(
828 12 : Lsn::from_str("0/16960E8").unwrap(),
829 12 : Some(Lsn::from_str("0/1696070").unwrap()),
830 12 : Some(TimelineId::from_str("e45a7f37d3ee2ff17dc14bf4f4e3f52e").unwrap()),
831 12 : Lsn::INVALID,
832 12 : Lsn::from_str("0/1696070").unwrap(),
833 12 : Lsn::from_str("0/1696070").unwrap(),
834 12 : 14,
835 12 : ).with_recalculated_checksum().unwrap(),
836 12 : deleted_at: Some(parse_naive_datetime("2023-07-31T09:00:00.123000000")),
837 12 : archived_at: None,
838 12 : lineage: Default::default(),
839 12 : gc_blocking: None,
840 12 : last_aux_file_policy: Default::default(),
841 12 : import_pgdata: None,
842 12 : rel_size_migration: None,
843 12 : l2_lsn: None,
844 12 : gc_compaction: None,
845 12 : marked_invisible_at: None,
846 12 : };
847 12 :
848 12 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
849 12 : assert_eq!(part, expected);
850 12 : }
851 :
852 : #[test]
853 12 : fn v8_indexpart_is_parsed() {
854 12 : let example = r#"{
855 12 : "version": 8,
856 12 : "layer_metadata":{
857 12 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
858 12 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
859 12 : },
860 12 : "disk_consistent_lsn":"0/16960E8",
861 12 : "metadata": {
862 12 : "disk_consistent_lsn": "0/16960E8",
863 12 : "prev_record_lsn": "0/1696070",
864 12 : "ancestor_timeline": "e45a7f37d3ee2ff17dc14bf4f4e3f52e",
865 12 : "ancestor_lsn": "0/0",
866 12 : "latest_gc_cutoff_lsn": "0/1696070",
867 12 : "initdb_lsn": "0/1696070",
868 12 : "pg_version": 14
869 12 : },
870 12 : "deleted_at": "2023-07-31T09:00:00.123",
871 12 : "archived_at": "2023-04-29T09:00:00.123"
872 12 : }"#;
873 12 :
874 12 : let expected = IndexPart {
875 12 : version: 8,
876 12 : layer_metadata: HashMap::from([
877 12 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
878 12 : file_size: 25600000,
879 12 : generation: Generation::none(),
880 12 : shard: ShardIndex::unsharded()
881 12 : }),
882 12 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
883 12 : file_size: 9007199254741001,
884 12 : generation: Generation::none(),
885 12 : shard: ShardIndex::unsharded()
886 12 : })
887 12 : ]),
888 12 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
889 12 : metadata: TimelineMetadata::new(
890 12 : Lsn::from_str("0/16960E8").unwrap(),
891 12 : Some(Lsn::from_str("0/1696070").unwrap()),
892 12 : Some(TimelineId::from_str("e45a7f37d3ee2ff17dc14bf4f4e3f52e").unwrap()),
893 12 : Lsn::INVALID,
894 12 : Lsn::from_str("0/1696070").unwrap(),
895 12 : Lsn::from_str("0/1696070").unwrap(),
896 12 : 14,
897 12 : ).with_recalculated_checksum().unwrap(),
898 12 : deleted_at: Some(parse_naive_datetime("2023-07-31T09:00:00.123000000")),
899 12 : archived_at: Some(parse_naive_datetime("2023-04-29T09:00:00.123000000")),
900 12 : lineage: Default::default(),
901 12 : gc_blocking: None,
902 12 : last_aux_file_policy: Default::default(),
903 12 : import_pgdata: None,
904 12 : rel_size_migration: None,
905 12 : l2_lsn: None,
906 12 : gc_compaction: None,
907 12 : marked_invisible_at: None,
908 12 : };
909 12 :
910 12 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
911 12 : assert_eq!(part, expected);
912 12 : }
913 :
914 : #[test]
915 12 : fn v9_indexpart_is_parsed() {
916 12 : let example = r#"{
917 12 : "version": 9,
918 12 : "layer_metadata":{
919 12 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
920 12 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
921 12 : },
922 12 : "disk_consistent_lsn":"0/16960E8",
923 12 : "metadata": {
924 12 : "disk_consistent_lsn": "0/16960E8",
925 12 : "prev_record_lsn": "0/1696070",
926 12 : "ancestor_timeline": "e45a7f37d3ee2ff17dc14bf4f4e3f52e",
927 12 : "ancestor_lsn": "0/0",
928 12 : "latest_gc_cutoff_lsn": "0/1696070",
929 12 : "initdb_lsn": "0/1696070",
930 12 : "pg_version": 14
931 12 : },
932 12 : "gc_blocking": {
933 12 : "started_at": "2024-07-19T09:00:00.123",
934 12 : "reasons": ["DetachAncestor"]
935 12 : }
936 12 : }"#;
937 12 :
938 12 : let expected = IndexPart {
939 12 : version: 9,
940 12 : layer_metadata: HashMap::from([
941 12 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
942 12 : file_size: 25600000,
943 12 : generation: Generation::none(),
944 12 : shard: ShardIndex::unsharded()
945 12 : }),
946 12 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
947 12 : file_size: 9007199254741001,
948 12 : generation: Generation::none(),
949 12 : shard: ShardIndex::unsharded()
950 12 : })
951 12 : ]),
952 12 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
953 12 : metadata: TimelineMetadata::new(
954 12 : Lsn::from_str("0/16960E8").unwrap(),
955 12 : Some(Lsn::from_str("0/1696070").unwrap()),
956 12 : Some(TimelineId::from_str("e45a7f37d3ee2ff17dc14bf4f4e3f52e").unwrap()),
957 12 : Lsn::INVALID,
958 12 : Lsn::from_str("0/1696070").unwrap(),
959 12 : Lsn::from_str("0/1696070").unwrap(),
960 12 : 14,
961 12 : ).with_recalculated_checksum().unwrap(),
962 12 : deleted_at: None,
963 12 : lineage: Default::default(),
964 12 : gc_blocking: Some(GcBlocking {
965 12 : started_at: parse_naive_datetime("2024-07-19T09:00:00.123000000"),
966 12 : reasons: enumset::EnumSet::from_iter([GcBlockingReason::DetachAncestor]),
967 12 : }),
968 12 : last_aux_file_policy: Default::default(),
969 12 : archived_at: None,
970 12 : import_pgdata: None,
971 12 : rel_size_migration: None,
972 12 : l2_lsn: None,
973 12 : gc_compaction: None,
974 12 : marked_invisible_at: None,
975 12 : };
976 12 :
977 12 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
978 12 : assert_eq!(part, expected);
979 12 : }
980 :
981 : #[test]
982 12 : fn v10_importpgdata_is_parsed() {
983 12 : let example = r#"{
984 12 : "version": 10,
985 12 : "layer_metadata":{
986 12 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
987 12 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
988 12 : },
989 12 : "disk_consistent_lsn":"0/16960E8",
990 12 : "metadata": {
991 12 : "disk_consistent_lsn": "0/16960E8",
992 12 : "prev_record_lsn": "0/1696070",
993 12 : "ancestor_timeline": "e45a7f37d3ee2ff17dc14bf4f4e3f52e",
994 12 : "ancestor_lsn": "0/0",
995 12 : "latest_gc_cutoff_lsn": "0/1696070",
996 12 : "initdb_lsn": "0/1696070",
997 12 : "pg_version": 14
998 12 : },
999 12 : "gc_blocking": {
1000 12 : "started_at": "2024-07-19T09:00:00.123",
1001 12 : "reasons": ["DetachAncestor"]
1002 12 : },
1003 12 : "import_pgdata": {
1004 12 : "V1": {
1005 12 : "Done": {
1006 12 : "idempotency_key": "specified-by-client-218a5213-5044-4562-a28d-d024c5f057f5",
1007 12 : "started_at": "2024-11-13T09:23:42.123",
1008 12 : "finished_at": "2024-11-13T09:42:23.123"
1009 12 : }
1010 12 : }
1011 12 : }
1012 12 : }"#;
1013 12 :
1014 12 : let expected = IndexPart {
1015 12 : version: 10,
1016 12 : layer_metadata: HashMap::from([
1017 12 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
1018 12 : file_size: 25600000,
1019 12 : generation: Generation::none(),
1020 12 : shard: ShardIndex::unsharded()
1021 12 : }),
1022 12 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
1023 12 : file_size: 9007199254741001,
1024 12 : generation: Generation::none(),
1025 12 : shard: ShardIndex::unsharded()
1026 12 : })
1027 12 : ]),
1028 12 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
1029 12 : metadata: TimelineMetadata::new(
1030 12 : Lsn::from_str("0/16960E8").unwrap(),
1031 12 : Some(Lsn::from_str("0/1696070").unwrap()),
1032 12 : Some(TimelineId::from_str("e45a7f37d3ee2ff17dc14bf4f4e3f52e").unwrap()),
1033 12 : Lsn::INVALID,
1034 12 : Lsn::from_str("0/1696070").unwrap(),
1035 12 : Lsn::from_str("0/1696070").unwrap(),
1036 12 : 14,
1037 12 : ).with_recalculated_checksum().unwrap(),
1038 12 : deleted_at: None,
1039 12 : lineage: Default::default(),
1040 12 : gc_blocking: Some(GcBlocking {
1041 12 : started_at: parse_naive_datetime("2024-07-19T09:00:00.123000000"),
1042 12 : reasons: enumset::EnumSet::from_iter([GcBlockingReason::DetachAncestor]),
1043 12 : }),
1044 12 : last_aux_file_policy: Default::default(),
1045 12 : archived_at: None,
1046 12 : import_pgdata: Some(import_pgdata::index_part_format::Root::V1(import_pgdata::index_part_format::V1::Done(import_pgdata::index_part_format::Done{
1047 12 : started_at: parse_naive_datetime("2024-11-13T09:23:42.123000000"),
1048 12 : finished_at: parse_naive_datetime("2024-11-13T09:42:23.123000000"),
1049 12 : idempotency_key: import_pgdata::index_part_format::IdempotencyKey::new("specified-by-client-218a5213-5044-4562-a28d-d024c5f057f5".to_string()),
1050 12 : }))),
1051 12 : rel_size_migration: None,
1052 12 : l2_lsn: None,
1053 12 : gc_compaction: None,
1054 12 : marked_invisible_at: None,
1055 12 : };
1056 12 :
1057 12 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
1058 12 : assert_eq!(part, expected);
1059 12 : }
1060 :
1061 : #[test]
1062 12 : fn v11_rel_size_migration_is_parsed() {
1063 12 : let example = r#"{
1064 12 : "version": 11,
1065 12 : "layer_metadata":{
1066 12 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
1067 12 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
1068 12 : },
1069 12 : "disk_consistent_lsn":"0/16960E8",
1070 12 : "metadata": {
1071 12 : "disk_consistent_lsn": "0/16960E8",
1072 12 : "prev_record_lsn": "0/1696070",
1073 12 : "ancestor_timeline": "e45a7f37d3ee2ff17dc14bf4f4e3f52e",
1074 12 : "ancestor_lsn": "0/0",
1075 12 : "latest_gc_cutoff_lsn": "0/1696070",
1076 12 : "initdb_lsn": "0/1696070",
1077 12 : "pg_version": 14
1078 12 : },
1079 12 : "gc_blocking": {
1080 12 : "started_at": "2024-07-19T09:00:00.123",
1081 12 : "reasons": ["DetachAncestor"]
1082 12 : },
1083 12 : "import_pgdata": {
1084 12 : "V1": {
1085 12 : "Done": {
1086 12 : "idempotency_key": "specified-by-client-218a5213-5044-4562-a28d-d024c5f057f5",
1087 12 : "started_at": "2024-11-13T09:23:42.123",
1088 12 : "finished_at": "2024-11-13T09:42:23.123"
1089 12 : }
1090 12 : }
1091 12 : },
1092 12 : "rel_size_migration": "legacy"
1093 12 : }"#;
1094 12 :
1095 12 : let expected = IndexPart {
1096 12 : version: 11,
1097 12 : layer_metadata: HashMap::from([
1098 12 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
1099 12 : file_size: 25600000,
1100 12 : generation: Generation::none(),
1101 12 : shard: ShardIndex::unsharded()
1102 12 : }),
1103 12 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
1104 12 : file_size: 9007199254741001,
1105 12 : generation: Generation::none(),
1106 12 : shard: ShardIndex::unsharded()
1107 12 : })
1108 12 : ]),
1109 12 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
1110 12 : metadata: TimelineMetadata::new(
1111 12 : Lsn::from_str("0/16960E8").unwrap(),
1112 12 : Some(Lsn::from_str("0/1696070").unwrap()),
1113 12 : Some(TimelineId::from_str("e45a7f37d3ee2ff17dc14bf4f4e3f52e").unwrap()),
1114 12 : Lsn::INVALID,
1115 12 : Lsn::from_str("0/1696070").unwrap(),
1116 12 : Lsn::from_str("0/1696070").unwrap(),
1117 12 : 14,
1118 12 : ).with_recalculated_checksum().unwrap(),
1119 12 : deleted_at: None,
1120 12 : lineage: Default::default(),
1121 12 : gc_blocking: Some(GcBlocking {
1122 12 : started_at: parse_naive_datetime("2024-07-19T09:00:00.123000000"),
1123 12 : reasons: enumset::EnumSet::from_iter([GcBlockingReason::DetachAncestor]),
1124 12 : }),
1125 12 : last_aux_file_policy: Default::default(),
1126 12 : archived_at: None,
1127 12 : import_pgdata: Some(import_pgdata::index_part_format::Root::V1(import_pgdata::index_part_format::V1::Done(import_pgdata::index_part_format::Done{
1128 12 : started_at: parse_naive_datetime("2024-11-13T09:23:42.123000000"),
1129 12 : finished_at: parse_naive_datetime("2024-11-13T09:42:23.123000000"),
1130 12 : idempotency_key: import_pgdata::index_part_format::IdempotencyKey::new("specified-by-client-218a5213-5044-4562-a28d-d024c5f057f5".to_string()),
1131 12 : }))),
1132 12 : rel_size_migration: Some(RelSizeMigration::Legacy),
1133 12 : l2_lsn: None,
1134 12 : gc_compaction: None,
1135 12 : marked_invisible_at: None,
1136 12 : };
1137 12 :
1138 12 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
1139 12 : assert_eq!(part, expected);
1140 12 : }
1141 :
1142 : #[test]
1143 12 : fn v12_v13_l2_gc_ompaction_is_parsed() {
1144 12 : let example = r#"{
1145 12 : "version": 13,
1146 12 : "layer_metadata":{
1147 12 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
1148 12 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
1149 12 : },
1150 12 : "disk_consistent_lsn":"0/16960E8",
1151 12 : "metadata": {
1152 12 : "disk_consistent_lsn": "0/16960E8",
1153 12 : "prev_record_lsn": "0/1696070",
1154 12 : "ancestor_timeline": "e45a7f37d3ee2ff17dc14bf4f4e3f52e",
1155 12 : "ancestor_lsn": "0/0",
1156 12 : "latest_gc_cutoff_lsn": "0/1696070",
1157 12 : "initdb_lsn": "0/1696070",
1158 12 : "pg_version": 14
1159 12 : },
1160 12 : "gc_blocking": {
1161 12 : "started_at": "2024-07-19T09:00:00.123",
1162 12 : "reasons": ["DetachAncestor"]
1163 12 : },
1164 12 : "import_pgdata": {
1165 12 : "V1": {
1166 12 : "Done": {
1167 12 : "idempotency_key": "specified-by-client-218a5213-5044-4562-a28d-d024c5f057f5",
1168 12 : "started_at": "2024-11-13T09:23:42.123",
1169 12 : "finished_at": "2024-11-13T09:42:23.123"
1170 12 : }
1171 12 : }
1172 12 : },
1173 12 : "rel_size_migration": "legacy",
1174 12 : "l2_lsn": "0/16960E8",
1175 12 : "gc_compaction": {
1176 12 : "last_completed_lsn": "0/16960E8"
1177 12 : }
1178 12 : }"#;
1179 12 :
1180 12 : let expected = IndexPart {
1181 12 : version: 13,
1182 12 : layer_metadata: HashMap::from([
1183 12 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
1184 12 : file_size: 25600000,
1185 12 : generation: Generation::none(),
1186 12 : shard: ShardIndex::unsharded()
1187 12 : }),
1188 12 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
1189 12 : file_size: 9007199254741001,
1190 12 : generation: Generation::none(),
1191 12 : shard: ShardIndex::unsharded()
1192 12 : })
1193 12 : ]),
1194 12 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
1195 12 : metadata: TimelineMetadata::new(
1196 12 : Lsn::from_str("0/16960E8").unwrap(),
1197 12 : Some(Lsn::from_str("0/1696070").unwrap()),
1198 12 : Some(TimelineId::from_str("e45a7f37d3ee2ff17dc14bf4f4e3f52e").unwrap()),
1199 12 : Lsn::INVALID,
1200 12 : Lsn::from_str("0/1696070").unwrap(),
1201 12 : Lsn::from_str("0/1696070").unwrap(),
1202 12 : 14,
1203 12 : ).with_recalculated_checksum().unwrap(),
1204 12 : deleted_at: None,
1205 12 : lineage: Default::default(),
1206 12 : gc_blocking: Some(GcBlocking {
1207 12 : started_at: parse_naive_datetime("2024-07-19T09:00:00.123000000"),
1208 12 : reasons: enumset::EnumSet::from_iter([GcBlockingReason::DetachAncestor]),
1209 12 : }),
1210 12 : last_aux_file_policy: Default::default(),
1211 12 : archived_at: None,
1212 12 : import_pgdata: Some(import_pgdata::index_part_format::Root::V1(import_pgdata::index_part_format::V1::Done(import_pgdata::index_part_format::Done{
1213 12 : started_at: parse_naive_datetime("2024-11-13T09:23:42.123000000"),
1214 12 : finished_at: parse_naive_datetime("2024-11-13T09:42:23.123000000"),
1215 12 : idempotency_key: import_pgdata::index_part_format::IdempotencyKey::new("specified-by-client-218a5213-5044-4562-a28d-d024c5f057f5".to_string()),
1216 12 : }))),
1217 12 : rel_size_migration: Some(RelSizeMigration::Legacy),
1218 12 : l2_lsn: Some("0/16960E8".parse::<Lsn>().unwrap()),
1219 12 : gc_compaction: Some(GcCompactionState {
1220 12 : last_completed_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
1221 12 : }),
1222 12 : marked_invisible_at: None,
1223 12 : };
1224 12 :
1225 12 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
1226 12 : assert_eq!(part, expected);
1227 12 : }
1228 :
1229 : #[test]
1230 12 : fn v14_marked_invisible_at_is_parsed() {
1231 12 : let example = r#"{
1232 12 : "version": 14,
1233 12 : "layer_metadata":{
1234 12 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
1235 12 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
1236 12 : },
1237 12 : "disk_consistent_lsn":"0/16960E8",
1238 12 : "metadata": {
1239 12 : "disk_consistent_lsn": "0/16960E8",
1240 12 : "prev_record_lsn": "0/1696070",
1241 12 : "ancestor_timeline": "e45a7f37d3ee2ff17dc14bf4f4e3f52e",
1242 12 : "ancestor_lsn": "0/0",
1243 12 : "latest_gc_cutoff_lsn": "0/1696070",
1244 12 : "initdb_lsn": "0/1696070",
1245 12 : "pg_version": 14
1246 12 : },
1247 12 : "gc_blocking": {
1248 12 : "started_at": "2024-07-19T09:00:00.123",
1249 12 : "reasons": ["DetachAncestor"]
1250 12 : },
1251 12 : "import_pgdata": {
1252 12 : "V1": {
1253 12 : "Done": {
1254 12 : "idempotency_key": "specified-by-client-218a5213-5044-4562-a28d-d024c5f057f5",
1255 12 : "started_at": "2024-11-13T09:23:42.123",
1256 12 : "finished_at": "2024-11-13T09:42:23.123"
1257 12 : }
1258 12 : }
1259 12 : },
1260 12 : "rel_size_migration": "legacy",
1261 12 : "l2_lsn": "0/16960E8",
1262 12 : "gc_compaction": {
1263 12 : "last_completed_lsn": "0/16960E8"
1264 12 : },
1265 12 : "marked_invisible_at": "2023-07-31T09:00:00.123"
1266 12 : }"#;
1267 12 :
1268 12 : let expected = IndexPart {
1269 12 : version: 14,
1270 12 : layer_metadata: HashMap::from([
1271 12 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
1272 12 : file_size: 25600000,
1273 12 : generation: Generation::none(),
1274 12 : shard: ShardIndex::unsharded()
1275 12 : }),
1276 12 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
1277 12 : file_size: 9007199254741001,
1278 12 : generation: Generation::none(),
1279 12 : shard: ShardIndex::unsharded()
1280 12 : })
1281 12 : ]),
1282 12 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
1283 12 : metadata: TimelineMetadata::new(
1284 12 : Lsn::from_str("0/16960E8").unwrap(),
1285 12 : Some(Lsn::from_str("0/1696070").unwrap()),
1286 12 : Some(TimelineId::from_str("e45a7f37d3ee2ff17dc14bf4f4e3f52e").unwrap()),
1287 12 : Lsn::INVALID,
1288 12 : Lsn::from_str("0/1696070").unwrap(),
1289 12 : Lsn::from_str("0/1696070").unwrap(),
1290 12 : 14,
1291 12 : ).with_recalculated_checksum().unwrap(),
1292 12 : deleted_at: None,
1293 12 : lineage: Default::default(),
1294 12 : gc_blocking: Some(GcBlocking {
1295 12 : started_at: parse_naive_datetime("2024-07-19T09:00:00.123000000"),
1296 12 : reasons: enumset::EnumSet::from_iter([GcBlockingReason::DetachAncestor]),
1297 12 : }),
1298 12 : last_aux_file_policy: Default::default(),
1299 12 : archived_at: None,
1300 12 : import_pgdata: Some(import_pgdata::index_part_format::Root::V1(import_pgdata::index_part_format::V1::Done(import_pgdata::index_part_format::Done{
1301 12 : started_at: parse_naive_datetime("2024-11-13T09:23:42.123000000"),
1302 12 : finished_at: parse_naive_datetime("2024-11-13T09:42:23.123000000"),
1303 12 : idempotency_key: import_pgdata::index_part_format::IdempotencyKey::new("specified-by-client-218a5213-5044-4562-a28d-d024c5f057f5".to_string()),
1304 12 : }))),
1305 12 : rel_size_migration: Some(RelSizeMigration::Legacy),
1306 12 : l2_lsn: Some("0/16960E8".parse::<Lsn>().unwrap()),
1307 12 : gc_compaction: Some(GcCompactionState {
1308 12 : last_completed_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
1309 12 : }),
1310 12 : marked_invisible_at: Some(parse_naive_datetime("2023-07-31T09:00:00.123000000")),
1311 12 : };
1312 12 :
1313 12 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
1314 12 : assert_eq!(part, expected);
1315 12 : }
1316 :
1317 264 : fn parse_naive_datetime(s: &str) -> NaiveDateTime {
1318 264 : chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S.%f").unwrap()
1319 264 : }
1320 : }
|
