Line data Source code
1 : //! In-memory index to track the tenant files on the remote storage.
2 : //!
3 : //! Able to restore itself from the storage index parts, that are located in every timeline's remote directory and contain all data about
4 : //! remote timeline layers and its metadata.
5 :
6 : use std::collections::HashMap;
7 :
8 : use chrono::NaiveDateTime;
9 : use pageserver_api::models::AuxFilePolicy;
10 : use serde::{Deserialize, Serialize};
11 :
12 : use super::is_same_remote_layer_path;
13 : use crate::tenant::metadata::TimelineMetadata;
14 : use crate::tenant::storage_layer::LayerName;
15 : use crate::tenant::timeline::import_pgdata;
16 : use crate::tenant::Generation;
17 : use pageserver_api::shard::ShardIndex;
18 : use utils::id::TimelineId;
19 : use utils::lsn::Lsn;
20 :
21 : /// In-memory representation of an `index_part.json` file
22 : ///
23 : /// Contains the data about all files in the timeline, present remotely and its metadata.
24 : ///
25 : /// This type needs to be backwards and forwards compatible. When changing the fields,
26 : /// remember to add a test case for the changed version.
27 6090 : #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
28 : pub struct IndexPart {
29 : /// Debugging aid describing the version of this type.
30 : #[serde(default)]
31 : version: usize,
32 :
33 : #[serde(default)]
34 : #[serde(skip_serializing_if = "Option::is_none")]
35 : pub deleted_at: Option<NaiveDateTime>,
36 :
37 : #[serde(default)]
38 : #[serde(skip_serializing_if = "Option::is_none")]
39 : pub archived_at: Option<NaiveDateTime>,
40 :
41 : /// This field supports import-from-pgdata ("fast imports" platform feature).
42 : /// We don't currently use fast imports, so, this field is None for all production timelines.
43 : /// See <https://github.com/neondatabase/neon/pull/9218> for more information.
44 : #[serde(default)]
45 : #[serde(skip_serializing_if = "Option::is_none")]
46 : pub import_pgdata: Option<import_pgdata::index_part_format::Root>,
47 :
48 : /// Layer filenames and metadata. For an index persisted in remote storage, all layers must
49 : /// exist in remote storage.
50 : pub layer_metadata: HashMap<LayerName, LayerFileMetadata>,
51 :
52 : /// Because of the trouble of eyeballing the legacy "metadata" field, we copied the
53 : /// "disk_consistent_lsn" out. After version 7 this is no longer needed, but the name cannot be
54 : /// reused.
55 : pub(super) disk_consistent_lsn: Lsn,
56 :
57 : // TODO: rename as "metadata" next week, keep the alias = "metadata_bytes", bump version Adding
58 : // the "alias = metadata" was forgotten in #7693, so we have to use "rewrite = metadata_bytes"
59 : // for backwards compatibility.
60 : #[serde(
61 : rename = "metadata_bytes",
62 : alias = "metadata",
63 : with = "crate::tenant::metadata::modern_serde"
64 : )]
65 : pub metadata: TimelineMetadata,
66 :
67 : #[serde(default)]
68 : pub(crate) lineage: Lineage,
69 :
70 : #[serde(skip_serializing_if = "Option::is_none", default)]
71 : pub(crate) gc_blocking: Option<GcBlocking>,
72 :
73 : /// Describes the kind of aux files stored in the timeline.
74 : ///
75 : /// The value is modified during file ingestion when the latest wanted value communicated via tenant config is applied if it is acceptable.
76 : /// A V1 setting after V2 files have been committed is not accepted.
77 : ///
78 : /// None means no aux files have been written to the storage before the point
79 : /// when this flag is introduced.
80 : ///
81 : /// This flag is not used any more as all tenants have been transitioned to the new aux file policy.
82 : #[serde(skip_serializing_if = "Option::is_none", default)]
83 : pub(crate) last_aux_file_policy: Option<AuxFilePolicy>,
84 :
85 : #[serde(skip_serializing_if = "Option::is_none", default)]
86 : pub(crate) rel_size_migration: Option<RelSizeMigration>,
87 :
88 : /// The LSN of gc-compaction horizon. Once gc-compaction is finished for all layer files below an LSN, this LSN will be updated.
89 : #[serde(skip_serializing_if = "Option::is_none", default)]
90 : pub(crate) l2_lsn: Option<Lsn>,
91 : }
92 :
93 8 : #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
94 : #[serde(rename_all = "camelCase")]
95 : pub enum RelSizeMigration {
96 : /// The tenant is using the old rel_size format.
97 : /// Note that this enum is persisted as `Option<RelSizeMigration>` in the index part, so
98 : /// `None` is the same as `Some(RelSizeMigration::Legacy)`.
99 : Legacy,
100 : /// The tenant is migrating to the new rel_size format. Both old and new rel_size format are
101 : /// persisted in the index part. The read path will read both formats and merge them.
102 : Migrating,
103 : /// The tenant has migrated to the new rel_size format. Only the new rel_size format is persisted
104 : /// in the index part, and the read path will not read the old format.
105 : Migrated,
106 : }
107 :
108 : impl IndexPart {
109 : /// When adding or modifying any parts of `IndexPart`, increment the version so that it can be
110 : /// used to understand later versions.
111 : ///
112 : /// Version is currently informative only.
113 : /// Version history
114 : /// - 2: added `deleted_at`
115 : /// - 3: no longer deserialize `timeline_layers` (serialized format is the same, but timeline_layers
116 : /// is always generated from the keys of `layer_metadata`)
117 : /// - 4: timeline_layers is fully removed.
118 : /// - 5: lineage was added
119 : /// - 6: last_aux_file_policy is added.
120 : /// - 7: metadata_bytes is no longer written, but still read
121 : /// - 8: added `archived_at`
122 : /// - 9: +gc_blocking
123 : /// - 10: +import_pgdata
124 : /// - 11: +rel_size_migration
125 : /// - 12: +l2_lsn
126 : const LATEST_VERSION: usize = 12;
127 :
128 : // Versions we may see when reading from a bucket.
129 : pub const KNOWN_VERSIONS: &'static [usize] = &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12];
130 :
131 : pub const FILE_NAME: &'static str = "index_part.json";
132 :
133 1052 : pub fn empty(metadata: TimelineMetadata) -> Self {
134 1052 : IndexPart {
135 1052 : version: Self::LATEST_VERSION,
136 1052 : layer_metadata: Default::default(),
137 1052 : disk_consistent_lsn: metadata.disk_consistent_lsn(),
138 1052 : metadata,
139 1052 : deleted_at: None,
140 1052 : archived_at: None,
141 1052 : lineage: Default::default(),
142 1052 : gc_blocking: None,
143 1052 : last_aux_file_policy: None,
144 1052 : import_pgdata: None,
145 1052 : rel_size_migration: None,
146 1052 : l2_lsn: None,
147 1052 : }
148 1052 : }
149 :
150 0 : pub fn version(&self) -> usize {
151 0 : self.version
152 0 : }
153 :
154 : /// If you want this under normal operations, read it from self.metadata:
155 : /// this method is just for the scrubber to use when validating an index.
156 0 : pub fn duplicated_disk_consistent_lsn(&self) -> Lsn {
157 0 : self.disk_consistent_lsn
158 0 : }
159 :
160 52 : pub fn from_json_bytes(bytes: &[u8]) -> Result<Self, serde_json::Error> {
161 52 : serde_json::from_slice::<IndexPart>(bytes)
162 52 : }
163 :
164 2982 : pub fn to_json_bytes(&self) -> serde_json::Result<Vec<u8>> {
165 2982 : serde_json::to_vec(self)
166 2982 : }
167 :
168 : #[cfg(test)]
169 56 : pub(crate) fn example() -> Self {
170 56 : Self::empty(TimelineMetadata::example())
171 56 : }
172 :
173 : /// Returns true if the index contains a reference to the given layer (i.e. file path).
174 : ///
175 : /// TODO: there should be a variant of LayerName for the physical remote path that contains
176 : /// information about the shard and generation, to avoid passing in metadata.
177 69551 : pub fn references(&self, name: &LayerName, metadata: &LayerFileMetadata) -> bool {
178 69551 : let Some(index_metadata) = self.layer_metadata.get(name) else {
179 33682 : return false;
180 : };
181 35869 : is_same_remote_layer_path(name, metadata, name, index_metadata)
182 69551 : }
183 :
184 : /// Check for invariants in the index: this is useful when uploading an index to ensure that if
185 : /// we encounter a bug, we do not persist buggy metadata.
186 5430 : pub(crate) fn validate(&self) -> Result<(), String> {
187 5430 : if self.import_pgdata.is_none()
188 5430 : && self.metadata.ancestor_timeline().is_none()
189 4020 : && self.layer_metadata.is_empty()
190 : {
191 : // Unless we're in the middle of a raw pgdata import, or this is a child timeline,the index must
192 : // always have at least one layer.
193 0 : return Err("Index has no ancestor and no layers".to_string());
194 5430 : }
195 5430 :
196 5430 : Ok(())
197 5430 : }
198 : }
199 :
200 : /// Metadata gathered for each of the layer files.
201 : ///
202 : /// Fields have to be `Option`s because remote [`IndexPart`]'s can be from different version, which
203 : /// might have less or more metadata depending if upgrading or rolling back an upgrade.
204 18968 : #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
205 : pub struct LayerFileMetadata {
206 : pub file_size: u64,
207 :
208 : #[serde(default = "Generation::none")]
209 : #[serde(skip_serializing_if = "Generation::is_none")]
210 : pub generation: Generation,
211 :
212 : #[serde(default = "ShardIndex::unsharded")]
213 : #[serde(skip_serializing_if = "ShardIndex::is_unsharded")]
214 : pub shard: ShardIndex,
215 : }
216 :
217 : impl LayerFileMetadata {
218 6318 : pub fn new(file_size: u64, generation: Generation, shard: ShardIndex) -> Self {
219 6318 : LayerFileMetadata {
220 6318 : file_size,
221 6318 : generation,
222 6318 : shard,
223 6318 : }
224 6318 : }
225 : /// Helper to get both generation and file size in a tuple
226 0 : pub fn generation_file_size(&self) -> (Generation, u64) {
227 0 : (self.generation, self.file_size)
228 0 : }
229 : }
230 :
231 : /// Limited history of earlier ancestors.
232 : ///
233 : /// A timeline can have more than 1 earlier ancestor, in the rare case that it was repeatedly
234 : /// reparented by having an later timeline be detached from it's ancestor.
235 16 : #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Default)]
236 : pub(crate) struct Lineage {
237 : /// Has the `reparenting_history` been truncated to [`Lineage::REMEMBER_AT_MOST`].
238 : #[serde(skip_serializing_if = "is_false", default)]
239 : reparenting_history_truncated: bool,
240 :
241 : /// Earlier ancestors, truncated when [`Self::reparenting_history_truncated`]
242 : ///
243 : /// These are stored in case we want to support WAL based DR on the timeline. There can be many
244 : /// of these and at most one [`Self::original_ancestor`]. There cannot be more reparentings
245 : /// after [`Self::original_ancestor`] has been set.
246 : #[serde(skip_serializing_if = "Vec::is_empty", default)]
247 : reparenting_history: Vec<TimelineId>,
248 :
249 : /// The ancestor from which this timeline has been detached from and when.
250 : ///
251 : /// If you are adding support for detaching from a hierarchy, consider changing the ancestry
252 : /// into a `Vec<(TimelineId, Lsn)>` to be a path instead.
253 : // FIXME: this is insufficient even for path of two timelines for future wal recovery
254 : // purposes:
255 : //
256 : // assuming a "old main" which has received most of the WAL, and has a branch "new main",
257 : // starting a bit before "old main" last_record_lsn. the current version works fine,
258 : // because we will know to replay wal and branch at the recorded Lsn to do wal recovery.
259 : //
260 : // then assuming "new main" would similarly receive a branch right before its last_record_lsn,
261 : // "new new main". the current implementation would just store ("new main", ancestor_lsn, _)
262 : // here. however, we cannot recover from WAL using only that information, we would need the
263 : // whole ancestry here:
264 : //
265 : // ```json
266 : // [
267 : // ["old main", ancestor_lsn("new main"), _],
268 : // ["new main", ancestor_lsn("new new main"), _]
269 : // ]
270 : // ```
271 : #[serde(skip_serializing_if = "Option::is_none", default)]
272 : original_ancestor: Option<(TimelineId, Lsn, NaiveDateTime)>,
273 : }
274 :
275 12180 : fn is_false(b: &bool) -> bool {
276 12180 : !b
277 12180 : }
278 :
279 : impl Lineage {
280 : const REMEMBER_AT_MOST: usize = 100;
281 :
282 0 : pub(crate) fn record_previous_ancestor(&mut self, old_ancestor: &TimelineId) -> bool {
283 0 : if self.reparenting_history.last() == Some(old_ancestor) {
284 : // do not re-record it
285 0 : false
286 : } else {
287 : #[cfg(feature = "testing")]
288 : {
289 0 : let existing = self
290 0 : .reparenting_history
291 0 : .iter()
292 0 : .position(|x| x == old_ancestor);
293 0 : assert_eq!(
294 : existing, None,
295 0 : "we cannot reparent onto and off and onto the same timeline twice"
296 : );
297 : }
298 0 : let drop_oldest = self.reparenting_history.len() + 1 >= Self::REMEMBER_AT_MOST;
299 0 :
300 0 : self.reparenting_history_truncated |= drop_oldest;
301 0 : if drop_oldest {
302 0 : self.reparenting_history.remove(0);
303 0 : }
304 0 : self.reparenting_history.push(*old_ancestor);
305 0 : true
306 : }
307 0 : }
308 :
309 : /// Returns true if anything changed.
310 0 : pub(crate) fn record_detaching(&mut self, branchpoint: &(TimelineId, Lsn)) -> bool {
311 0 : if let Some((id, lsn, _)) = self.original_ancestor {
312 0 : assert_eq!(
313 0 : &(id, lsn),
314 : branchpoint,
315 0 : "detaching attempt has to be for the same ancestor we are already detached from"
316 : );
317 0 : false
318 : } else {
319 0 : self.original_ancestor =
320 0 : Some((branchpoint.0, branchpoint.1, chrono::Utc::now().naive_utc()));
321 0 : true
322 : }
323 0 : }
324 :
325 : /// The queried lsn is most likely the basebackup lsn, and this answers question "is it allowed
326 : /// to start a read/write primary at this lsn".
327 : ///
328 : /// Returns true if the Lsn was previously our branch point.
329 0 : pub(crate) fn is_previous_ancestor_lsn(&self, lsn: Lsn) -> bool {
330 0 : self.original_ancestor
331 0 : .is_some_and(|(_, ancestor_lsn, _)| ancestor_lsn == lsn)
332 0 : }
333 :
334 : /// Returns true if the timeline originally had an ancestor, and no longer has one.
335 0 : pub(crate) fn is_detached_from_ancestor(&self) -> bool {
336 0 : self.original_ancestor.is_some()
337 0 : }
338 :
339 : /// Returns original ancestor timeline id and lsn that this timeline has been detached from.
340 0 : pub(crate) fn detached_previous_ancestor(&self) -> Option<(TimelineId, Lsn)> {
341 0 : self.original_ancestor.map(|(id, lsn, _)| (id, lsn))
342 0 : }
343 :
344 0 : pub(crate) fn is_reparented(&self) -> bool {
345 0 : !self.reparenting_history.is_empty()
346 0 : }
347 : }
348 :
349 32 : #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
350 : pub(crate) struct GcBlocking {
351 : pub(crate) started_at: NaiveDateTime,
352 : pub(crate) reasons: enumset::EnumSet<GcBlockingReason>,
353 : }
354 :
355 16 : #[derive(Debug, enumset::EnumSetType, serde::Serialize, serde::Deserialize)]
356 : #[enumset(serialize_repr = "list")]
357 : pub(crate) enum GcBlockingReason {
358 : Manual,
359 : DetachAncestor,
360 : }
361 :
362 : impl GcBlocking {
363 0 : pub(super) fn started_now_for(reason: GcBlockingReason) -> Self {
364 0 : GcBlocking {
365 0 : started_at: chrono::Utc::now().naive_utc(),
366 0 : reasons: enumset::EnumSet::only(reason),
367 0 : }
368 0 : }
369 :
370 : /// Returns true if the given reason is one of the reasons why the gc is blocked.
371 0 : pub(crate) fn blocked_by(&self, reason: GcBlockingReason) -> bool {
372 0 : self.reasons.contains(reason)
373 0 : }
374 :
375 : /// Returns a version of self with the given reason.
376 0 : pub(super) fn with_reason(&self, reason: GcBlockingReason) -> Self {
377 0 : assert!(!self.blocked_by(reason));
378 0 : let mut reasons = self.reasons;
379 0 : reasons.insert(reason);
380 0 :
381 0 : Self {
382 0 : started_at: self.started_at,
383 0 : reasons,
384 0 : }
385 0 : }
386 :
387 : /// Returns a version of self without the given reason. Assumption is that if
388 : /// there are no more reasons, we can unblock the gc by returning `None`.
389 0 : pub(super) fn without_reason(&self, reason: GcBlockingReason) -> Option<Self> {
390 0 : assert!(self.blocked_by(reason));
391 :
392 0 : if self.reasons.len() == 1 {
393 0 : None
394 : } else {
395 0 : let mut reasons = self.reasons;
396 0 : assert!(reasons.remove(reason));
397 0 : assert!(!reasons.is_empty());
398 :
399 0 : Some(Self {
400 0 : started_at: self.started_at,
401 0 : reasons,
402 0 : })
403 : }
404 0 : }
405 : }
406 :
407 : #[cfg(test)]
408 : mod tests {
409 : use super::*;
410 : use std::str::FromStr;
411 : use utils::id::TimelineId;
412 :
413 : #[test]
414 4 : fn v1_indexpart_is_parsed() {
415 4 : let example = r#"{
416 4 : "version":1,
417 4 : "timeline_layers":["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"],
418 4 : "layer_metadata":{
419 4 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
420 4 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
421 4 : },
422 4 : "disk_consistent_lsn":"0/16960E8",
423 4 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
424 4 : }"#;
425 4 :
426 4 : let expected = IndexPart {
427 4 : // note this is not verified, could be anything, but exists for humans debugging.. could be the git version instead?
428 4 : version: 1,
429 4 : layer_metadata: HashMap::from([
430 4 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
431 4 : file_size: 25600000,
432 4 : generation: Generation::none(),
433 4 : shard: ShardIndex::unsharded()
434 4 : }),
435 4 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
436 4 : // serde_json should always parse this but this might be a double with jq for
437 4 : // example.
438 4 : file_size: 9007199254741001,
439 4 : generation: Generation::none(),
440 4 : shard: ShardIndex::unsharded()
441 4 : })
442 4 : ]),
443 4 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
444 4 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
445 4 : deleted_at: None,
446 4 : archived_at: None,
447 4 : lineage: Lineage::default(),
448 4 : gc_blocking: None,
449 4 : last_aux_file_policy: None,
450 4 : import_pgdata: None,
451 4 : rel_size_migration: None,
452 4 : l2_lsn: None,
453 4 : };
454 4 :
455 4 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
456 4 : assert_eq!(part, expected);
457 4 : }
458 :
459 : #[test]
460 4 : fn v1_indexpart_is_parsed_with_optional_missing_layers() {
461 4 : let example = r#"{
462 4 : "version":1,
463 4 : "timeline_layers":["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"],
464 4 : "missing_layers":["This shouldn't fail deserialization"],
465 4 : "layer_metadata":{
466 4 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
467 4 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
468 4 : },
469 4 : "disk_consistent_lsn":"0/16960E8",
470 4 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
471 4 : }"#;
472 4 :
473 4 : let expected = IndexPart {
474 4 : // note this is not verified, could be anything, but exists for humans debugging.. could be the git version instead?
475 4 : version: 1,
476 4 : layer_metadata: HashMap::from([
477 4 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
478 4 : file_size: 25600000,
479 4 : generation: Generation::none(),
480 4 : shard: ShardIndex::unsharded()
481 4 : }),
482 4 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
483 4 : // serde_json should always parse this but this might be a double with jq for
484 4 : // example.
485 4 : file_size: 9007199254741001,
486 4 : generation: Generation::none(),
487 4 : shard: ShardIndex::unsharded()
488 4 : })
489 4 : ]),
490 4 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
491 4 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
492 4 : deleted_at: None,
493 4 : archived_at: None,
494 4 : lineage: Lineage::default(),
495 4 : gc_blocking: None,
496 4 : last_aux_file_policy: None,
497 4 : import_pgdata: None,
498 4 : rel_size_migration: None,
499 4 : l2_lsn: None,
500 4 : };
501 4 :
502 4 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
503 4 : assert_eq!(part, expected);
504 4 : }
505 :
506 : #[test]
507 4 : fn v2_indexpart_is_parsed_with_deleted_at() {
508 4 : let example = r#"{
509 4 : "version":2,
510 4 : "timeline_layers":["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"],
511 4 : "missing_layers":["This shouldn't fail deserialization"],
512 4 : "layer_metadata":{
513 4 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
514 4 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
515 4 : },
516 4 : "disk_consistent_lsn":"0/16960E8",
517 4 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
518 4 : "deleted_at": "2023-07-31T09:00:00.123"
519 4 : }"#;
520 4 :
521 4 : let expected = IndexPart {
522 4 : // note this is not verified, could be anything, but exists for humans debugging.. could be the git version instead?
523 4 : version: 2,
524 4 : layer_metadata: HashMap::from([
525 4 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
526 4 : file_size: 25600000,
527 4 : generation: Generation::none(),
528 4 : shard: ShardIndex::unsharded()
529 4 : }),
530 4 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
531 4 : // serde_json should always parse this but this might be a double with jq for
532 4 : // example.
533 4 : file_size: 9007199254741001,
534 4 : generation: Generation::none(),
535 4 : shard: ShardIndex::unsharded()
536 4 : })
537 4 : ]),
538 4 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
539 4 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
540 4 : deleted_at: Some(parse_naive_datetime("2023-07-31T09:00:00.123000000")),
541 4 : archived_at: None,
542 4 : lineage: Lineage::default(),
543 4 : gc_blocking: None,
544 4 : last_aux_file_policy: None,
545 4 : import_pgdata: None,
546 4 : rel_size_migration: None,
547 4 : l2_lsn: None,
548 4 : };
549 4 :
550 4 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
551 4 : assert_eq!(part, expected);
552 4 : }
553 :
554 : #[test]
555 4 : fn empty_layers_are_parsed() {
556 4 : let empty_layers_json = r#"{
557 4 : "version":1,
558 4 : "timeline_layers":[],
559 4 : "layer_metadata":{},
560 4 : "disk_consistent_lsn":"0/2532648",
561 4 : "metadata_bytes":[136,151,49,208,0,70,0,4,0,0,0,0,2,83,38,72,1,0,0,0,0,2,83,38,32,1,87,198,240,135,97,119,45,125,38,29,155,161,140,141,255,210,0,0,0,0,2,83,38,72,0,0,0,0,1,73,240,192,0,0,0,0,1,73,240,192,0,0,0,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
562 4 : }"#;
563 4 :
564 4 : let expected = IndexPart {
565 4 : version: 1,
566 4 : layer_metadata: HashMap::new(),
567 4 : disk_consistent_lsn: "0/2532648".parse::<Lsn>().unwrap(),
568 4 : metadata: TimelineMetadata::from_bytes(&[
569 4 : 136, 151, 49, 208, 0, 70, 0, 4, 0, 0, 0, 0, 2, 83, 38, 72, 1, 0, 0, 0, 0, 2, 83,
570 4 : 38, 32, 1, 87, 198, 240, 135, 97, 119, 45, 125, 38, 29, 155, 161, 140, 141, 255,
571 4 : 210, 0, 0, 0, 0, 2, 83, 38, 72, 0, 0, 0, 0, 1, 73, 240, 192, 0, 0, 0, 0, 1, 73,
572 4 : 240, 192, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
573 4 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
574 4 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
575 4 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
576 4 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
577 4 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
578 4 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
579 4 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
580 4 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
581 4 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
582 4 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
583 4 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
584 4 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
585 4 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
586 4 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
587 4 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
588 4 : 0, 0,
589 4 : ])
590 4 : .unwrap(),
591 4 : deleted_at: None,
592 4 : archived_at: None,
593 4 : lineage: Lineage::default(),
594 4 : gc_blocking: None,
595 4 : last_aux_file_policy: None,
596 4 : import_pgdata: None,
597 4 : rel_size_migration: None,
598 4 : l2_lsn: None,
599 4 : };
600 4 :
601 4 : let empty_layers_parsed = IndexPart::from_json_bytes(empty_layers_json.as_bytes()).unwrap();
602 4 :
603 4 : assert_eq!(empty_layers_parsed, expected);
604 4 : }
605 :
606 : #[test]
607 4 : fn v4_indexpart_is_parsed() {
608 4 : let example = r#"{
609 4 : "version":4,
610 4 : "layer_metadata":{
611 4 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
612 4 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
613 4 : },
614 4 : "disk_consistent_lsn":"0/16960E8",
615 4 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
616 4 : "deleted_at": "2023-07-31T09:00:00.123"
617 4 : }"#;
618 4 :
619 4 : let expected = IndexPart {
620 4 : version: 4,
621 4 : layer_metadata: HashMap::from([
622 4 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
623 4 : file_size: 25600000,
624 4 : generation: Generation::none(),
625 4 : shard: ShardIndex::unsharded()
626 4 : }),
627 4 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
628 4 : // serde_json should always parse this but this might be a double with jq for
629 4 : // example.
630 4 : file_size: 9007199254741001,
631 4 : generation: Generation::none(),
632 4 : shard: ShardIndex::unsharded()
633 4 : })
634 4 : ]),
635 4 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
636 4 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
637 4 : deleted_at: Some(parse_naive_datetime("2023-07-31T09:00:00.123000000")),
638 4 : archived_at: None,
639 4 : lineage: Lineage::default(),
640 4 : gc_blocking: None,
641 4 : last_aux_file_policy: None,
642 4 : import_pgdata: None,
643 4 : rel_size_migration: None,
644 4 : l2_lsn: None,
645 4 : };
646 4 :
647 4 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
648 4 : assert_eq!(part, expected);
649 4 : }
650 :
651 : #[test]
652 4 : fn v5_indexpart_is_parsed() {
653 4 : let example = r#"{
654 4 : "version":5,
655 4 : "layer_metadata":{
656 4 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000014EF420-00000000014EF499":{"file_size":23289856,"generation":1},
657 4 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000014EF499-00000000015A7619":{"file_size":1015808,"generation":1}},
658 4 : "disk_consistent_lsn":"0/15A7618",
659 4 : "metadata_bytes":[226,88,25,241,0,46,0,4,0,0,0,0,1,90,118,24,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,78,244,32,0,0,0,0,1,78,244,32,0,0,0,16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
660 4 : "lineage":{
661 4 : "original_ancestor":["e2bfd8c633d713d279e6fcd2bcc15b6d","0/15A7618","2024-05-07T18:52:36.322426563"],
662 4 : "reparenting_history":["e1bfd8c633d713d279e6fcd2bcc15b6d"]
663 4 : }
664 4 : }"#;
665 4 :
666 4 : let expected = IndexPart {
667 4 : version: 5,
668 4 : layer_metadata: HashMap::from([
669 4 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000014EF420-00000000014EF499".parse().unwrap(), LayerFileMetadata {
670 4 : file_size: 23289856,
671 4 : generation: Generation::new(1),
672 4 : shard: ShardIndex::unsharded(),
673 4 : }),
674 4 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000014EF499-00000000015A7619".parse().unwrap(), LayerFileMetadata {
675 4 : file_size: 1015808,
676 4 : generation: Generation::new(1),
677 4 : shard: ShardIndex::unsharded(),
678 4 : })
679 4 : ]),
680 4 : disk_consistent_lsn: Lsn::from_str("0/15A7618").unwrap(),
681 4 : metadata: TimelineMetadata::from_bytes(&[226,88,25,241,0,46,0,4,0,0,0,0,1,90,118,24,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,78,244,32,0,0,0,0,1,78,244,32,0,0,0,16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
682 4 : deleted_at: None,
683 4 : archived_at: None,
684 4 : lineage: Lineage {
685 4 : reparenting_history_truncated: false,
686 4 : reparenting_history: vec![TimelineId::from_str("e1bfd8c633d713d279e6fcd2bcc15b6d").unwrap()],
687 4 : original_ancestor: Some((TimelineId::from_str("e2bfd8c633d713d279e6fcd2bcc15b6d").unwrap(), Lsn::from_str("0/15A7618").unwrap(), parse_naive_datetime("2024-05-07T18:52:36.322426563"))),
688 4 : },
689 4 : gc_blocking: None,
690 4 : last_aux_file_policy: None,
691 4 : import_pgdata: None,
692 4 : rel_size_migration: None,
693 4 : l2_lsn: None,
694 4 : };
695 4 :
696 4 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
697 4 : assert_eq!(part, expected);
698 4 : }
699 :
700 : #[test]
701 4 : fn v6_indexpart_is_parsed() {
702 4 : let example = r#"{
703 4 : "version":6,
704 4 : "layer_metadata":{
705 4 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
706 4 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
707 4 : },
708 4 : "disk_consistent_lsn":"0/16960E8",
709 4 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
710 4 : "deleted_at": "2023-07-31T09:00:00.123",
711 4 : "lineage":{
712 4 : "original_ancestor":["e2bfd8c633d713d279e6fcd2bcc15b6d","0/15A7618","2024-05-07T18:52:36.322426563"],
713 4 : "reparenting_history":["e1bfd8c633d713d279e6fcd2bcc15b6d"]
714 4 : },
715 4 : "last_aux_file_policy": "V2"
716 4 : }"#;
717 4 :
718 4 : let expected = IndexPart {
719 4 : version: 6,
720 4 : layer_metadata: HashMap::from([
721 4 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
722 4 : file_size: 25600000,
723 4 : generation: Generation::none(),
724 4 : shard: ShardIndex::unsharded()
725 4 : }),
726 4 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
727 4 : // serde_json should always parse this but this might be a double with jq for
728 4 : // example.
729 4 : file_size: 9007199254741001,
730 4 : generation: Generation::none(),
731 4 : shard: ShardIndex::unsharded()
732 4 : })
733 4 : ]),
734 4 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
735 4 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
736 4 : deleted_at: Some(parse_naive_datetime("2023-07-31T09:00:00.123000000")),
737 4 : archived_at: None,
738 4 : lineage: Lineage {
739 4 : reparenting_history_truncated: false,
740 4 : reparenting_history: vec![TimelineId::from_str("e1bfd8c633d713d279e6fcd2bcc15b6d").unwrap()],
741 4 : original_ancestor: Some((TimelineId::from_str("e2bfd8c633d713d279e6fcd2bcc15b6d").unwrap(), Lsn::from_str("0/15A7618").unwrap(), parse_naive_datetime("2024-05-07T18:52:36.322426563"))),
742 4 : },
743 4 : gc_blocking: None,
744 4 : last_aux_file_policy: Some(AuxFilePolicy::V2),
745 4 : import_pgdata: None,
746 4 : rel_size_migration: None,
747 4 : l2_lsn: None,
748 4 : };
749 4 :
750 4 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
751 4 : assert_eq!(part, expected);
752 4 : }
753 :
754 : #[test]
755 4 : fn v7_indexpart_is_parsed() {
756 4 : let example = r#"{
757 4 : "version": 7,
758 4 : "layer_metadata":{
759 4 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
760 4 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
761 4 : },
762 4 : "disk_consistent_lsn":"0/16960E8",
763 4 : "metadata": {
764 4 : "disk_consistent_lsn": "0/16960E8",
765 4 : "prev_record_lsn": "0/1696070",
766 4 : "ancestor_timeline": "e45a7f37d3ee2ff17dc14bf4f4e3f52e",
767 4 : "ancestor_lsn": "0/0",
768 4 : "latest_gc_cutoff_lsn": "0/1696070",
769 4 : "initdb_lsn": "0/1696070",
770 4 : "pg_version": 14
771 4 : },
772 4 : "deleted_at": "2023-07-31T09:00:00.123"
773 4 : }"#;
774 4 :
775 4 : let expected = IndexPart {
776 4 : version: 7,
777 4 : layer_metadata: HashMap::from([
778 4 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
779 4 : file_size: 25600000,
780 4 : generation: Generation::none(),
781 4 : shard: ShardIndex::unsharded()
782 4 : }),
783 4 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
784 4 : file_size: 9007199254741001,
785 4 : generation: Generation::none(),
786 4 : shard: ShardIndex::unsharded()
787 4 : })
788 4 : ]),
789 4 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
790 4 : metadata: TimelineMetadata::new(
791 4 : Lsn::from_str("0/16960E8").unwrap(),
792 4 : Some(Lsn::from_str("0/1696070").unwrap()),
793 4 : Some(TimelineId::from_str("e45a7f37d3ee2ff17dc14bf4f4e3f52e").unwrap()),
794 4 : Lsn::INVALID,
795 4 : Lsn::from_str("0/1696070").unwrap(),
796 4 : Lsn::from_str("0/1696070").unwrap(),
797 4 : 14,
798 4 : ).with_recalculated_checksum().unwrap(),
799 4 : deleted_at: Some(parse_naive_datetime("2023-07-31T09:00:00.123000000")),
800 4 : archived_at: None,
801 4 : lineage: Default::default(),
802 4 : gc_blocking: None,
803 4 : last_aux_file_policy: Default::default(),
804 4 : import_pgdata: None,
805 4 : rel_size_migration: None,
806 4 : l2_lsn: None,
807 4 : };
808 4 :
809 4 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
810 4 : assert_eq!(part, expected);
811 4 : }
812 :
813 : #[test]
814 4 : fn v8_indexpart_is_parsed() {
815 4 : let example = r#"{
816 4 : "version": 8,
817 4 : "layer_metadata":{
818 4 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
819 4 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
820 4 : },
821 4 : "disk_consistent_lsn":"0/16960E8",
822 4 : "metadata": {
823 4 : "disk_consistent_lsn": "0/16960E8",
824 4 : "prev_record_lsn": "0/1696070",
825 4 : "ancestor_timeline": "e45a7f37d3ee2ff17dc14bf4f4e3f52e",
826 4 : "ancestor_lsn": "0/0",
827 4 : "latest_gc_cutoff_lsn": "0/1696070",
828 4 : "initdb_lsn": "0/1696070",
829 4 : "pg_version": 14
830 4 : },
831 4 : "deleted_at": "2023-07-31T09:00:00.123",
832 4 : "archived_at": "2023-04-29T09:00:00.123"
833 4 : }"#;
834 4 :
835 4 : let expected = IndexPart {
836 4 : version: 8,
837 4 : layer_metadata: HashMap::from([
838 4 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
839 4 : file_size: 25600000,
840 4 : generation: Generation::none(),
841 4 : shard: ShardIndex::unsharded()
842 4 : }),
843 4 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
844 4 : file_size: 9007199254741001,
845 4 : generation: Generation::none(),
846 4 : shard: ShardIndex::unsharded()
847 4 : })
848 4 : ]),
849 4 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
850 4 : metadata: TimelineMetadata::new(
851 4 : Lsn::from_str("0/16960E8").unwrap(),
852 4 : Some(Lsn::from_str("0/1696070").unwrap()),
853 4 : Some(TimelineId::from_str("e45a7f37d3ee2ff17dc14bf4f4e3f52e").unwrap()),
854 4 : Lsn::INVALID,
855 4 : Lsn::from_str("0/1696070").unwrap(),
856 4 : Lsn::from_str("0/1696070").unwrap(),
857 4 : 14,
858 4 : ).with_recalculated_checksum().unwrap(),
859 4 : deleted_at: Some(parse_naive_datetime("2023-07-31T09:00:00.123000000")),
860 4 : archived_at: Some(parse_naive_datetime("2023-04-29T09:00:00.123000000")),
861 4 : lineage: Default::default(),
862 4 : gc_blocking: None,
863 4 : last_aux_file_policy: Default::default(),
864 4 : import_pgdata: None,
865 4 : rel_size_migration: None,
866 4 : l2_lsn: None,
867 4 : };
868 4 :
869 4 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
870 4 : assert_eq!(part, expected);
871 4 : }
872 :
873 : #[test]
874 4 : fn v9_indexpart_is_parsed() {
875 4 : let example = r#"{
876 4 : "version": 9,
877 4 : "layer_metadata":{
878 4 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
879 4 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
880 4 : },
881 4 : "disk_consistent_lsn":"0/16960E8",
882 4 : "metadata": {
883 4 : "disk_consistent_lsn": "0/16960E8",
884 4 : "prev_record_lsn": "0/1696070",
885 4 : "ancestor_timeline": "e45a7f37d3ee2ff17dc14bf4f4e3f52e",
886 4 : "ancestor_lsn": "0/0",
887 4 : "latest_gc_cutoff_lsn": "0/1696070",
888 4 : "initdb_lsn": "0/1696070",
889 4 : "pg_version": 14
890 4 : },
891 4 : "gc_blocking": {
892 4 : "started_at": "2024-07-19T09:00:00.123",
893 4 : "reasons": ["DetachAncestor"]
894 4 : }
895 4 : }"#;
896 4 :
897 4 : let expected = IndexPart {
898 4 : version: 9,
899 4 : layer_metadata: HashMap::from([
900 4 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
901 4 : file_size: 25600000,
902 4 : generation: Generation::none(),
903 4 : shard: ShardIndex::unsharded()
904 4 : }),
905 4 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
906 4 : file_size: 9007199254741001,
907 4 : generation: Generation::none(),
908 4 : shard: ShardIndex::unsharded()
909 4 : })
910 4 : ]),
911 4 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
912 4 : metadata: TimelineMetadata::new(
913 4 : Lsn::from_str("0/16960E8").unwrap(),
914 4 : Some(Lsn::from_str("0/1696070").unwrap()),
915 4 : Some(TimelineId::from_str("e45a7f37d3ee2ff17dc14bf4f4e3f52e").unwrap()),
916 4 : Lsn::INVALID,
917 4 : Lsn::from_str("0/1696070").unwrap(),
918 4 : Lsn::from_str("0/1696070").unwrap(),
919 4 : 14,
920 4 : ).with_recalculated_checksum().unwrap(),
921 4 : deleted_at: None,
922 4 : lineage: Default::default(),
923 4 : gc_blocking: Some(GcBlocking {
924 4 : started_at: parse_naive_datetime("2024-07-19T09:00:00.123000000"),
925 4 : reasons: enumset::EnumSet::from_iter([GcBlockingReason::DetachAncestor]),
926 4 : }),
927 4 : last_aux_file_policy: Default::default(),
928 4 : archived_at: None,
929 4 : import_pgdata: None,
930 4 : rel_size_migration: None,
931 4 : l2_lsn: None,
932 4 : };
933 4 :
934 4 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
935 4 : assert_eq!(part, expected);
936 4 : }
937 :
938 : #[test]
939 4 : fn v10_importpgdata_is_parsed() {
940 4 : let example = r#"{
941 4 : "version": 10,
942 4 : "layer_metadata":{
943 4 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
944 4 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
945 4 : },
946 4 : "disk_consistent_lsn":"0/16960E8",
947 4 : "metadata": {
948 4 : "disk_consistent_lsn": "0/16960E8",
949 4 : "prev_record_lsn": "0/1696070",
950 4 : "ancestor_timeline": "e45a7f37d3ee2ff17dc14bf4f4e3f52e",
951 4 : "ancestor_lsn": "0/0",
952 4 : "latest_gc_cutoff_lsn": "0/1696070",
953 4 : "initdb_lsn": "0/1696070",
954 4 : "pg_version": 14
955 4 : },
956 4 : "gc_blocking": {
957 4 : "started_at": "2024-07-19T09:00:00.123",
958 4 : "reasons": ["DetachAncestor"]
959 4 : },
960 4 : "import_pgdata": {
961 4 : "V1": {
962 4 : "Done": {
963 4 : "idempotency_key": "specified-by-client-218a5213-5044-4562-a28d-d024c5f057f5",
964 4 : "started_at": "2024-11-13T09:23:42.123",
965 4 : "finished_at": "2024-11-13T09:42:23.123"
966 4 : }
967 4 : }
968 4 : }
969 4 : }"#;
970 4 :
971 4 : let expected = IndexPart {
972 4 : version: 10,
973 4 : layer_metadata: HashMap::from([
974 4 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
975 4 : file_size: 25600000,
976 4 : generation: Generation::none(),
977 4 : shard: ShardIndex::unsharded()
978 4 : }),
979 4 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
980 4 : file_size: 9007199254741001,
981 4 : generation: Generation::none(),
982 4 : shard: ShardIndex::unsharded()
983 4 : })
984 4 : ]),
985 4 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
986 4 : metadata: TimelineMetadata::new(
987 4 : Lsn::from_str("0/16960E8").unwrap(),
988 4 : Some(Lsn::from_str("0/1696070").unwrap()),
989 4 : Some(TimelineId::from_str("e45a7f37d3ee2ff17dc14bf4f4e3f52e").unwrap()),
990 4 : Lsn::INVALID,
991 4 : Lsn::from_str("0/1696070").unwrap(),
992 4 : Lsn::from_str("0/1696070").unwrap(),
993 4 : 14,
994 4 : ).with_recalculated_checksum().unwrap(),
995 4 : deleted_at: None,
996 4 : lineage: Default::default(),
997 4 : gc_blocking: Some(GcBlocking {
998 4 : started_at: parse_naive_datetime("2024-07-19T09:00:00.123000000"),
999 4 : reasons: enumset::EnumSet::from_iter([GcBlockingReason::DetachAncestor]),
1000 4 : }),
1001 4 : last_aux_file_policy: Default::default(),
1002 4 : archived_at: None,
1003 4 : import_pgdata: Some(import_pgdata::index_part_format::Root::V1(import_pgdata::index_part_format::V1::Done(import_pgdata::index_part_format::Done{
1004 4 : started_at: parse_naive_datetime("2024-11-13T09:23:42.123000000"),
1005 4 : finished_at: parse_naive_datetime("2024-11-13T09:42:23.123000000"),
1006 4 : idempotency_key: import_pgdata::index_part_format::IdempotencyKey::new("specified-by-client-218a5213-5044-4562-a28d-d024c5f057f5".to_string()),
1007 4 : }))),
1008 4 : rel_size_migration: None,
1009 4 : l2_lsn: None,
1010 4 : };
1011 4 :
1012 4 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
1013 4 : assert_eq!(part, expected);
1014 4 : }
1015 :
1016 : #[test]
1017 4 : fn v11_rel_size_migration_is_parsed() {
1018 4 : let example = r#"{
1019 4 : "version": 11,
1020 4 : "layer_metadata":{
1021 4 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
1022 4 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
1023 4 : },
1024 4 : "disk_consistent_lsn":"0/16960E8",
1025 4 : "metadata": {
1026 4 : "disk_consistent_lsn": "0/16960E8",
1027 4 : "prev_record_lsn": "0/1696070",
1028 4 : "ancestor_timeline": "e45a7f37d3ee2ff17dc14bf4f4e3f52e",
1029 4 : "ancestor_lsn": "0/0",
1030 4 : "latest_gc_cutoff_lsn": "0/1696070",
1031 4 : "initdb_lsn": "0/1696070",
1032 4 : "pg_version": 14
1033 4 : },
1034 4 : "gc_blocking": {
1035 4 : "started_at": "2024-07-19T09:00:00.123",
1036 4 : "reasons": ["DetachAncestor"]
1037 4 : },
1038 4 : "import_pgdata": {
1039 4 : "V1": {
1040 4 : "Done": {
1041 4 : "idempotency_key": "specified-by-client-218a5213-5044-4562-a28d-d024c5f057f5",
1042 4 : "started_at": "2024-11-13T09:23:42.123",
1043 4 : "finished_at": "2024-11-13T09:42:23.123"
1044 4 : }
1045 4 : }
1046 4 : },
1047 4 : "rel_size_migration": "legacy"
1048 4 : }"#;
1049 4 :
1050 4 : let expected = IndexPart {
1051 4 : version: 11,
1052 4 : layer_metadata: HashMap::from([
1053 4 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
1054 4 : file_size: 25600000,
1055 4 : generation: Generation::none(),
1056 4 : shard: ShardIndex::unsharded()
1057 4 : }),
1058 4 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
1059 4 : file_size: 9007199254741001,
1060 4 : generation: Generation::none(),
1061 4 : shard: ShardIndex::unsharded()
1062 4 : })
1063 4 : ]),
1064 4 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
1065 4 : metadata: TimelineMetadata::new(
1066 4 : Lsn::from_str("0/16960E8").unwrap(),
1067 4 : Some(Lsn::from_str("0/1696070").unwrap()),
1068 4 : Some(TimelineId::from_str("e45a7f37d3ee2ff17dc14bf4f4e3f52e").unwrap()),
1069 4 : Lsn::INVALID,
1070 4 : Lsn::from_str("0/1696070").unwrap(),
1071 4 : Lsn::from_str("0/1696070").unwrap(),
1072 4 : 14,
1073 4 : ).with_recalculated_checksum().unwrap(),
1074 4 : deleted_at: None,
1075 4 : lineage: Default::default(),
1076 4 : gc_blocking: Some(GcBlocking {
1077 4 : started_at: parse_naive_datetime("2024-07-19T09:00:00.123000000"),
1078 4 : reasons: enumset::EnumSet::from_iter([GcBlockingReason::DetachAncestor]),
1079 4 : }),
1080 4 : last_aux_file_policy: Default::default(),
1081 4 : archived_at: None,
1082 4 : import_pgdata: Some(import_pgdata::index_part_format::Root::V1(import_pgdata::index_part_format::V1::Done(import_pgdata::index_part_format::Done{
1083 4 : started_at: parse_naive_datetime("2024-11-13T09:23:42.123000000"),
1084 4 : finished_at: parse_naive_datetime("2024-11-13T09:42:23.123000000"),
1085 4 : idempotency_key: import_pgdata::index_part_format::IdempotencyKey::new("specified-by-client-218a5213-5044-4562-a28d-d024c5f057f5".to_string()),
1086 4 : }))),
1087 4 : rel_size_migration: Some(RelSizeMigration::Legacy),
1088 4 : l2_lsn: None,
1089 4 : };
1090 4 :
1091 4 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
1092 4 : assert_eq!(part, expected);
1093 4 : }
1094 :
1095 : #[test]
1096 4 : fn v12_l2_lsn_is_parsed() {
1097 4 : let example = r#"{
1098 4 : "version": 12,
1099 4 : "layer_metadata":{
1100 4 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
1101 4 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
1102 4 : },
1103 4 : "disk_consistent_lsn":"0/16960E8",
1104 4 : "metadata": {
1105 4 : "disk_consistent_lsn": "0/16960E8",
1106 4 : "prev_record_lsn": "0/1696070",
1107 4 : "ancestor_timeline": "e45a7f37d3ee2ff17dc14bf4f4e3f52e",
1108 4 : "ancestor_lsn": "0/0",
1109 4 : "latest_gc_cutoff_lsn": "0/1696070",
1110 4 : "initdb_lsn": "0/1696070",
1111 4 : "pg_version": 14
1112 4 : },
1113 4 : "gc_blocking": {
1114 4 : "started_at": "2024-07-19T09:00:00.123",
1115 4 : "reasons": ["DetachAncestor"]
1116 4 : },
1117 4 : "import_pgdata": {
1118 4 : "V1": {
1119 4 : "Done": {
1120 4 : "idempotency_key": "specified-by-client-218a5213-5044-4562-a28d-d024c5f057f5",
1121 4 : "started_at": "2024-11-13T09:23:42.123",
1122 4 : "finished_at": "2024-11-13T09:42:23.123"
1123 4 : }
1124 4 : }
1125 4 : },
1126 4 : "rel_size_migration": "legacy",
1127 4 : "l2_lsn": "0/16960E8"
1128 4 : }"#;
1129 4 :
1130 4 : let expected = IndexPart {
1131 4 : version: 12,
1132 4 : layer_metadata: HashMap::from([
1133 4 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
1134 4 : file_size: 25600000,
1135 4 : generation: Generation::none(),
1136 4 : shard: ShardIndex::unsharded()
1137 4 : }),
1138 4 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
1139 4 : file_size: 9007199254741001,
1140 4 : generation: Generation::none(),
1141 4 : shard: ShardIndex::unsharded()
1142 4 : })
1143 4 : ]),
1144 4 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
1145 4 : metadata: TimelineMetadata::new(
1146 4 : Lsn::from_str("0/16960E8").unwrap(),
1147 4 : Some(Lsn::from_str("0/1696070").unwrap()),
1148 4 : Some(TimelineId::from_str("e45a7f37d3ee2ff17dc14bf4f4e3f52e").unwrap()),
1149 4 : Lsn::INVALID,
1150 4 : Lsn::from_str("0/1696070").unwrap(),
1151 4 : Lsn::from_str("0/1696070").unwrap(),
1152 4 : 14,
1153 4 : ).with_recalculated_checksum().unwrap(),
1154 4 : deleted_at: None,
1155 4 : lineage: Default::default(),
1156 4 : gc_blocking: Some(GcBlocking {
1157 4 : started_at: parse_naive_datetime("2024-07-19T09:00:00.123000000"),
1158 4 : reasons: enumset::EnumSet::from_iter([GcBlockingReason::DetachAncestor]),
1159 4 : }),
1160 4 : last_aux_file_policy: Default::default(),
1161 4 : archived_at: None,
1162 4 : import_pgdata: Some(import_pgdata::index_part_format::Root::V1(import_pgdata::index_part_format::V1::Done(import_pgdata::index_part_format::Done{
1163 4 : started_at: parse_naive_datetime("2024-11-13T09:23:42.123000000"),
1164 4 : finished_at: parse_naive_datetime("2024-11-13T09:42:23.123000000"),
1165 4 : idempotency_key: import_pgdata::index_part_format::IdempotencyKey::new("specified-by-client-218a5213-5044-4562-a28d-d024c5f057f5".to_string()),
1166 4 : }))),
1167 4 : rel_size_migration: Some(RelSizeMigration::Legacy),
1168 4 : l2_lsn: Some("0/16960E8".parse::<Lsn>().unwrap()),
1169 4 : };
1170 4 :
1171 4 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
1172 4 : assert_eq!(part, expected);
1173 4 : }
1174 :
1175 72 : fn parse_naive_datetime(s: &str) -> NaiveDateTime {
1176 72 : chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S.%f").unwrap()
1177 72 : }
1178 : }
|
