Line data Source code
1 : //! In-memory index to track the tenant files on the remote storage.
2 : //!
3 : //! Able to restore itself from the storage index parts, that are located in every timeline's remote directory and contain all data about
4 : //! remote timeline layers and its metadata.
5 :
6 : use std::collections::HashMap;
7 :
8 : use chrono::NaiveDateTime;
9 : use pageserver_api::models::AuxFilePolicy;
10 : use serde::{Deserialize, Serialize};
11 :
12 : use super::is_same_remote_layer_path;
13 : use crate::tenant::metadata::TimelineMetadata;
14 : use crate::tenant::storage_layer::LayerName;
15 : use crate::tenant::timeline::import_pgdata;
16 : use crate::tenant::Generation;
17 : use pageserver_api::shard::ShardIndex;
18 : use utils::id::TimelineId;
19 : use utils::lsn::Lsn;
20 :
21 : /// In-memory representation of an `index_part.json` file
22 : ///
23 : /// Contains the data about all files in the timeline, present remotely and its metadata.
24 : ///
25 : /// This type needs to be backwards and forwards compatible. When changing the fields,
26 : /// remember to add a test case for the changed version.
27 2986 : #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
28 : pub struct IndexPart {
29 : /// Debugging aid describing the version of this type.
30 : #[serde(default)]
31 : version: usize,
32 :
33 : #[serde(default)]
34 : #[serde(skip_serializing_if = "Option::is_none")]
35 : pub deleted_at: Option<NaiveDateTime>,
36 :
37 : #[serde(default)]
38 : #[serde(skip_serializing_if = "Option::is_none")]
39 : pub archived_at: Option<NaiveDateTime>,
40 :
41 : /// This field supports import-from-pgdata ("fast imports" platform feature).
42 : /// We don't currently use fast imports, so, this field is None for all production timelines.
43 : /// See <https://github.com/neondatabase/neon/pull/9218> for more information.
44 : #[serde(default)]
45 : #[serde(skip_serializing_if = "Option::is_none")]
46 : pub import_pgdata: Option<import_pgdata::index_part_format::Root>,
47 :
48 : /// Layer filenames and metadata. For an index persisted in remote storage, all layers must
49 : /// exist in remote storage.
50 : pub layer_metadata: HashMap<LayerName, LayerFileMetadata>,
51 :
52 : /// Because of the trouble of eyeballing the legacy "metadata" field, we copied the
53 : /// "disk_consistent_lsn" out. After version 7 this is no longer needed, but the name cannot be
54 : /// reused.
55 : pub(super) disk_consistent_lsn: Lsn,
56 :
57 : // TODO: rename as "metadata" next week, keep the alias = "metadata_bytes", bump version Adding
58 : // the "alias = metadata" was forgotten in #7693, so we have to use "rewrite = metadata_bytes"
59 : // for backwards compatibility.
60 : #[serde(
61 : rename = "metadata_bytes",
62 : alias = "metadata",
63 : with = "crate::tenant::metadata::modern_serde"
64 : )]
65 : pub metadata: TimelineMetadata,
66 :
67 : #[serde(default)]
68 : pub(crate) lineage: Lineage,
69 :
70 : #[serde(skip_serializing_if = "Option::is_none", default)]
71 : pub(crate) gc_blocking: Option<GcBlocking>,
72 :
73 : /// Describes the kind of aux files stored in the timeline.
74 : ///
75 : /// The value is modified during file ingestion when the latest wanted value communicated via tenant config is applied if it is acceptable.
76 : /// A V1 setting after V2 files have been committed is not accepted.
77 : ///
78 : /// None means no aux files have been written to the storage before the point
79 : /// when this flag is introduced.
80 : #[serde(skip_serializing_if = "Option::is_none", default)]
81 : pub(crate) last_aux_file_policy: Option<AuxFilePolicy>,
82 : }
83 :
84 : impl IndexPart {
85 : /// When adding or modifying any parts of `IndexPart`, increment the version so that it can be
86 : /// used to understand later versions.
87 : ///
88 : /// Version is currently informative only.
89 : /// Version history
90 : /// - 2: added `deleted_at`
91 : /// - 3: no longer deserialize `timeline_layers` (serialized format is the same, but timeline_layers
92 : /// is always generated from the keys of `layer_metadata`)
93 : /// - 4: timeline_layers is fully removed.
94 : /// - 5: lineage was added
95 : /// - 6: last_aux_file_policy is added.
96 : /// - 7: metadata_bytes is no longer written, but still read
97 : /// - 8: added `archived_at`
98 : /// - 9: +gc_blocking
99 : /// - 10: +import_pgdata
100 : const LATEST_VERSION: usize = 10;
101 :
102 : // Versions we may see when reading from a bucket.
103 : pub const KNOWN_VERSIONS: &'static [usize] = &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
104 :
105 : pub const FILE_NAME: &'static str = "index_part.json";
106 :
107 428 : pub(crate) fn empty(metadata: TimelineMetadata) -> Self {
108 428 : IndexPart {
109 428 : version: Self::LATEST_VERSION,
110 428 : layer_metadata: Default::default(),
111 428 : disk_consistent_lsn: metadata.disk_consistent_lsn(),
112 428 : metadata,
113 428 : deleted_at: None,
114 428 : archived_at: None,
115 428 : lineage: Default::default(),
116 428 : gc_blocking: None,
117 428 : last_aux_file_policy: None,
118 428 : import_pgdata: None,
119 428 : }
120 428 : }
121 :
122 0 : pub fn version(&self) -> usize {
123 0 : self.version
124 0 : }
125 :
126 : /// If you want this under normal operations, read it from self.metadata:
127 : /// this method is just for the scrubber to use when validating an index.
128 0 : pub fn duplicated_disk_consistent_lsn(&self) -> Lsn {
129 0 : self.disk_consistent_lsn
130 0 : }
131 :
132 22 : pub fn from_json_bytes(bytes: &[u8]) -> Result<Self, serde_json::Error> {
133 22 : serde_json::from_slice::<IndexPart>(bytes)
134 22 : }
135 :
136 1460 : pub fn to_json_bytes(&self) -> serde_json::Result<Vec<u8>> {
137 1460 : serde_json::to_vec(self)
138 1460 : }
139 :
140 : #[cfg(test)]
141 12 : pub(crate) fn example() -> Self {
142 12 : Self::empty(TimelineMetadata::example())
143 12 : }
144 :
145 : /// Returns true if the index contains a reference to the given layer (i.e. file path).
146 : ///
147 : /// TODO: there should be a variant of LayerName for the physical remote path that contains
148 : /// information about the shard and generation, to avoid passing in metadata.
149 1711 : pub fn references(&self, name: &LayerName, metadata: &LayerFileMetadata) -> bool {
150 1711 : let Some(index_metadata) = self.layer_metadata.get(name) else {
151 1707 : return false;
152 : };
153 4 : is_same_remote_layer_path(name, metadata, name, index_metadata)
154 1711 : }
155 : }
156 :
157 : /// Metadata gathered for each of the layer files.
158 : ///
159 : /// Fields have to be `Option`s because remote [`IndexPart`]'s can be from different version, which
160 : /// might have less or more metadata depending if upgrading or rolling back an upgrade.
161 12668 : #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
162 : pub struct LayerFileMetadata {
163 : pub file_size: u64,
164 :
165 : #[serde(default = "Generation::none")]
166 : #[serde(skip_serializing_if = "Generation::is_none")]
167 : pub generation: Generation,
168 :
169 : #[serde(default = "ShardIndex::unsharded")]
170 : #[serde(skip_serializing_if = "ShardIndex::is_unsharded")]
171 : pub shard: ShardIndex,
172 : }
173 :
174 : impl LayerFileMetadata {
175 2512 : pub fn new(file_size: u64, generation: Generation, shard: ShardIndex) -> Self {
176 2512 : LayerFileMetadata {
177 2512 : file_size,
178 2512 : generation,
179 2512 : shard,
180 2512 : }
181 2512 : }
182 : }
183 :
184 : /// Limited history of earlier ancestors.
185 : ///
186 : /// A timeline can have more than 1 earlier ancestor, in the rare case that it was repeatedly
187 : /// reparented by having an later timeline be detached from it's ancestor.
188 34 : #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Default)]
189 : pub(crate) struct Lineage {
190 : /// Has the `reparenting_history` been truncated to [`Lineage::REMEMBER_AT_MOST`].
191 : #[serde(skip_serializing_if = "is_false", default)]
192 : reparenting_history_truncated: bool,
193 :
194 : /// Earlier ancestors, truncated when [`Self::reparenting_history_truncated`]
195 : ///
196 : /// These are stored in case we want to support WAL based DR on the timeline. There can be many
197 : /// of these and at most one [`Self::original_ancestor`]. There cannot be more reparentings
198 : /// after [`Self::original_ancestor`] has been set.
199 : #[serde(skip_serializing_if = "Vec::is_empty", default)]
200 : reparenting_history: Vec<TimelineId>,
201 :
202 : /// The ancestor from which this timeline has been detached from and when.
203 : ///
204 : /// If you are adding support for detaching from a hierarchy, consider changing the ancestry
205 : /// into a `Vec<(TimelineId, Lsn)>` to be a path instead.
206 : // FIXME: this is insufficient even for path of two timelines for future wal recovery
207 : // purposes:
208 : //
209 : // assuming a "old main" which has received most of the WAL, and has a branch "new main",
210 : // starting a bit before "old main" last_record_lsn. the current version works fine,
211 : // because we will know to replay wal and branch at the recorded Lsn to do wal recovery.
212 : //
213 : // then assuming "new main" would similarly receive a branch right before its last_record_lsn,
214 : // "new new main". the current implementation would just store ("new main", ancestor_lsn, _)
215 : // here. however, we cannot recover from WAL using only that information, we would need the
216 : // whole ancestry here:
217 : //
218 : // ```json
219 : // [
220 : // ["old main", ancestor_lsn("new main"), _],
221 : // ["new main", ancestor_lsn("new new main"), _]
222 : // ]
223 : // ```
224 : #[serde(skip_serializing_if = "Option::is_none", default)]
225 : original_ancestor: Option<(TimelineId, Lsn, NaiveDateTime)>,
226 : }
227 :
228 5972 : fn is_false(b: &bool) -> bool {
229 5972 : !b
230 5972 : }
231 :
232 : impl Lineage {
233 : const REMEMBER_AT_MOST: usize = 100;
234 :
235 0 : pub(crate) fn record_previous_ancestor(&mut self, old_ancestor: &TimelineId) -> bool {
236 0 : if self.reparenting_history.last() == Some(old_ancestor) {
237 : // do not re-record it
238 0 : false
239 : } else {
240 : #[cfg(feature = "testing")]
241 : {
242 0 : let existing = self
243 0 : .reparenting_history
244 0 : .iter()
245 0 : .position(|x| x == old_ancestor);
246 0 : assert_eq!(
247 : existing, None,
248 0 : "we cannot reparent onto and off and onto the same timeline twice"
249 : );
250 : }
251 0 : let drop_oldest = self.reparenting_history.len() + 1 >= Self::REMEMBER_AT_MOST;
252 0 :
253 0 : self.reparenting_history_truncated |= drop_oldest;
254 0 : if drop_oldest {
255 0 : self.reparenting_history.remove(0);
256 0 : }
257 0 : self.reparenting_history.push(*old_ancestor);
258 0 : true
259 : }
260 0 : }
261 :
262 : /// Returns true if anything changed.
263 0 : pub(crate) fn record_detaching(&mut self, branchpoint: &(TimelineId, Lsn)) -> bool {
264 0 : if let Some((id, lsn, _)) = self.original_ancestor {
265 0 : assert_eq!(
266 0 : &(id, lsn),
267 : branchpoint,
268 0 : "detaching attempt has to be for the same ancestor we are already detached from"
269 : );
270 0 : false
271 : } else {
272 0 : self.original_ancestor =
273 0 : Some((branchpoint.0, branchpoint.1, chrono::Utc::now().naive_utc()));
274 0 : true
275 : }
276 0 : }
277 :
278 : /// The queried lsn is most likely the basebackup lsn, and this answers question "is it allowed
279 : /// to start a read/write primary at this lsn".
280 : ///
281 : /// Returns true if the Lsn was previously our branch point.
282 0 : pub(crate) fn is_previous_ancestor_lsn(&self, lsn: Lsn) -> bool {
283 0 : self.original_ancestor
284 0 : .is_some_and(|(_, ancestor_lsn, _)| ancestor_lsn == lsn)
285 0 : }
286 :
287 : /// Returns true if the timeline originally had an ancestor, and no longer has one.
288 0 : pub(crate) fn is_detached_from_ancestor(&self) -> bool {
289 0 : self.original_ancestor.is_some()
290 0 : }
291 :
292 : /// Returns original ancestor timeline id and lsn that this timeline has been detached from.
293 0 : pub(crate) fn detached_previous_ancestor(&self) -> Option<(TimelineId, Lsn)> {
294 0 : self.original_ancestor.map(|(id, lsn, _)| (id, lsn))
295 0 : }
296 :
297 0 : pub(crate) fn is_reparented(&self) -> bool {
298 0 : !self.reparenting_history.is_empty()
299 0 : }
300 : }
301 :
302 12 : #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
303 : pub(crate) struct GcBlocking {
304 : pub(crate) started_at: NaiveDateTime,
305 : pub(crate) reasons: enumset::EnumSet<GcBlockingReason>,
306 : }
307 :
308 8 : #[derive(Debug, enumset::EnumSetType, serde::Serialize, serde::Deserialize)]
309 : #[enumset(serialize_repr = "list")]
310 : pub(crate) enum GcBlockingReason {
311 : Manual,
312 : DetachAncestor,
313 : }
314 :
315 : impl GcBlocking {
316 0 : pub(super) fn started_now_for(reason: GcBlockingReason) -> Self {
317 0 : GcBlocking {
318 0 : started_at: chrono::Utc::now().naive_utc(),
319 0 : reasons: enumset::EnumSet::only(reason),
320 0 : }
321 0 : }
322 :
323 : /// Returns true if the given reason is one of the reasons why the gc is blocked.
324 0 : pub(crate) fn blocked_by(&self, reason: GcBlockingReason) -> bool {
325 0 : self.reasons.contains(reason)
326 0 : }
327 :
328 : /// Returns a version of self with the given reason.
329 0 : pub(super) fn with_reason(&self, reason: GcBlockingReason) -> Self {
330 0 : assert!(!self.blocked_by(reason));
331 0 : let mut reasons = self.reasons;
332 0 : reasons.insert(reason);
333 0 :
334 0 : Self {
335 0 : started_at: self.started_at,
336 0 : reasons,
337 0 : }
338 0 : }
339 :
340 : /// Returns a version of self without the given reason. Assumption is that if
341 : /// there are no more reasons, we can unblock the gc by returning `None`.
342 0 : pub(super) fn without_reason(&self, reason: GcBlockingReason) -> Option<Self> {
343 0 : assert!(self.blocked_by(reason));
344 :
345 0 : if self.reasons.len() == 1 {
346 0 : None
347 : } else {
348 0 : let mut reasons = self.reasons;
349 0 : assert!(reasons.remove(reason));
350 0 : assert!(!reasons.is_empty());
351 :
352 0 : Some(Self {
353 0 : started_at: self.started_at,
354 0 : reasons,
355 0 : })
356 : }
357 0 : }
358 : }
359 :
360 : #[cfg(test)]
361 : mod tests {
362 : use super::*;
363 : use std::str::FromStr;
364 : use utils::id::TimelineId;
365 :
366 : #[test]
367 2 : fn v1_indexpart_is_parsed() {
368 2 : let example = r#"{
369 2 : "version":1,
370 2 : "timeline_layers":["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"],
371 2 : "layer_metadata":{
372 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
373 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
374 2 : },
375 2 : "disk_consistent_lsn":"0/16960E8",
376 2 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
377 2 : }"#;
378 2 :
379 2 : let expected = IndexPart {
380 2 : // note this is not verified, could be anything, but exists for humans debugging.. could be the git version instead?
381 2 : version: 1,
382 2 : layer_metadata: HashMap::from([
383 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
384 2 : file_size: 25600000,
385 2 : generation: Generation::none(),
386 2 : shard: ShardIndex::unsharded()
387 2 : }),
388 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
389 2 : // serde_json should always parse this but this might be a double with jq for
390 2 : // example.
391 2 : file_size: 9007199254741001,
392 2 : generation: Generation::none(),
393 2 : shard: ShardIndex::unsharded()
394 2 : })
395 2 : ]),
396 2 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
397 2 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
398 2 : deleted_at: None,
399 2 : archived_at: None,
400 2 : lineage: Lineage::default(),
401 2 : gc_blocking: None,
402 2 : last_aux_file_policy: None,
403 2 : import_pgdata: None,
404 2 : };
405 2 :
406 2 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
407 2 : assert_eq!(part, expected);
408 2 : }
409 :
410 : #[test]
411 2 : fn v1_indexpart_is_parsed_with_optional_missing_layers() {
412 2 : let example = r#"{
413 2 : "version":1,
414 2 : "timeline_layers":["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"],
415 2 : "missing_layers":["This shouldn't fail deserialization"],
416 2 : "layer_metadata":{
417 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
418 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
419 2 : },
420 2 : "disk_consistent_lsn":"0/16960E8",
421 2 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
422 2 : }"#;
423 2 :
424 2 : let expected = IndexPart {
425 2 : // note this is not verified, could be anything, but exists for humans debugging.. could be the git version instead?
426 2 : version: 1,
427 2 : layer_metadata: HashMap::from([
428 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
429 2 : file_size: 25600000,
430 2 : generation: Generation::none(),
431 2 : shard: ShardIndex::unsharded()
432 2 : }),
433 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
434 2 : // serde_json should always parse this but this might be a double with jq for
435 2 : // example.
436 2 : file_size: 9007199254741001,
437 2 : generation: Generation::none(),
438 2 : shard: ShardIndex::unsharded()
439 2 : })
440 2 : ]),
441 2 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
442 2 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
443 2 : deleted_at: None,
444 2 : archived_at: None,
445 2 : lineage: Lineage::default(),
446 2 : gc_blocking: None,
447 2 : last_aux_file_policy: None,
448 2 : import_pgdata: None,
449 2 : };
450 2 :
451 2 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
452 2 : assert_eq!(part, expected);
453 2 : }
454 :
455 : #[test]
456 2 : fn v2_indexpart_is_parsed_with_deleted_at() {
457 2 : let example = r#"{
458 2 : "version":2,
459 2 : "timeline_layers":["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"],
460 2 : "missing_layers":["This shouldn't fail deserialization"],
461 2 : "layer_metadata":{
462 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
463 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
464 2 : },
465 2 : "disk_consistent_lsn":"0/16960E8",
466 2 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
467 2 : "deleted_at": "2023-07-31T09:00:00.123"
468 2 : }"#;
469 2 :
470 2 : let expected = IndexPart {
471 2 : // note this is not verified, could be anything, but exists for humans debugging.. could be the git version instead?
472 2 : version: 2,
473 2 : layer_metadata: HashMap::from([
474 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
475 2 : file_size: 25600000,
476 2 : generation: Generation::none(),
477 2 : shard: ShardIndex::unsharded()
478 2 : }),
479 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
480 2 : // serde_json should always parse this but this might be a double with jq for
481 2 : // example.
482 2 : file_size: 9007199254741001,
483 2 : generation: Generation::none(),
484 2 : shard: ShardIndex::unsharded()
485 2 : })
486 2 : ]),
487 2 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
488 2 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
489 2 : deleted_at: Some(parse_naive_datetime("2023-07-31T09:00:00.123000000")),
490 2 : archived_at: None,
491 2 : lineage: Lineage::default(),
492 2 : gc_blocking: None,
493 2 : last_aux_file_policy: None,
494 2 : import_pgdata: None,
495 2 : };
496 2 :
497 2 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
498 2 : assert_eq!(part, expected);
499 2 : }
500 :
501 : #[test]
502 2 : fn empty_layers_are_parsed() {
503 2 : let empty_layers_json = r#"{
504 2 : "version":1,
505 2 : "timeline_layers":[],
506 2 : "layer_metadata":{},
507 2 : "disk_consistent_lsn":"0/2532648",
508 2 : "metadata_bytes":[136,151,49,208,0,70,0,4,0,0,0,0,2,83,38,72,1,0,0,0,0,2,83,38,32,1,87,198,240,135,97,119,45,125,38,29,155,161,140,141,255,210,0,0,0,0,2,83,38,72,0,0,0,0,1,73,240,192,0,0,0,0,1,73,240,192,0,0,0,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
509 2 : }"#;
510 2 :
511 2 : let expected = IndexPart {
512 2 : version: 1,
513 2 : layer_metadata: HashMap::new(),
514 2 : disk_consistent_lsn: "0/2532648".parse::<Lsn>().unwrap(),
515 2 : metadata: TimelineMetadata::from_bytes(&[
516 2 : 136, 151, 49, 208, 0, 70, 0, 4, 0, 0, 0, 0, 2, 83, 38, 72, 1, 0, 0, 0, 0, 2, 83,
517 2 : 38, 32, 1, 87, 198, 240, 135, 97, 119, 45, 125, 38, 29, 155, 161, 140, 141, 255,
518 2 : 210, 0, 0, 0, 0, 2, 83, 38, 72, 0, 0, 0, 0, 1, 73, 240, 192, 0, 0, 0, 0, 1, 73,
519 2 : 240, 192, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
520 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
521 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
522 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
523 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
524 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
525 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
526 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
527 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
528 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
529 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
530 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
531 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
532 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
533 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
534 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
535 2 : 0, 0,
536 2 : ])
537 2 : .unwrap(),
538 2 : deleted_at: None,
539 2 : archived_at: None,
540 2 : lineage: Lineage::default(),
541 2 : gc_blocking: None,
542 2 : last_aux_file_policy: None,
543 2 : import_pgdata: None,
544 2 : };
545 2 :
546 2 : let empty_layers_parsed = IndexPart::from_json_bytes(empty_layers_json.as_bytes()).unwrap();
547 2 :
548 2 : assert_eq!(empty_layers_parsed, expected);
549 2 : }
550 :
551 : #[test]
552 2 : fn v4_indexpart_is_parsed() {
553 2 : let example = r#"{
554 2 : "version":4,
555 2 : "layer_metadata":{
556 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
557 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
558 2 : },
559 2 : "disk_consistent_lsn":"0/16960E8",
560 2 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
561 2 : "deleted_at": "2023-07-31T09:00:00.123"
562 2 : }"#;
563 2 :
564 2 : let expected = IndexPart {
565 2 : version: 4,
566 2 : layer_metadata: HashMap::from([
567 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
568 2 : file_size: 25600000,
569 2 : generation: Generation::none(),
570 2 : shard: ShardIndex::unsharded()
571 2 : }),
572 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
573 2 : // serde_json should always parse this but this might be a double with jq for
574 2 : // example.
575 2 : file_size: 9007199254741001,
576 2 : generation: Generation::none(),
577 2 : shard: ShardIndex::unsharded()
578 2 : })
579 2 : ]),
580 2 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
581 2 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
582 2 : deleted_at: Some(parse_naive_datetime("2023-07-31T09:00:00.123000000")),
583 2 : archived_at: None,
584 2 : lineage: Lineage::default(),
585 2 : gc_blocking: None,
586 2 : last_aux_file_policy: None,
587 2 : import_pgdata: None,
588 2 : };
589 2 :
590 2 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
591 2 : assert_eq!(part, expected);
592 2 : }
593 :
594 : #[test]
595 2 : fn v5_indexpart_is_parsed() {
596 2 : let example = r#"{
597 2 : "version":5,
598 2 : "layer_metadata":{
599 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000014EF420-00000000014EF499":{"file_size":23289856,"generation":1},
600 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000014EF499-00000000015A7619":{"file_size":1015808,"generation":1}},
601 2 : "disk_consistent_lsn":"0/15A7618",
602 2 : "metadata_bytes":[226,88,25,241,0,46,0,4,0,0,0,0,1,90,118,24,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,78,244,32,0,0,0,0,1,78,244,32,0,0,0,16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
603 2 : "lineage":{
604 2 : "original_ancestor":["e2bfd8c633d713d279e6fcd2bcc15b6d","0/15A7618","2024-05-07T18:52:36.322426563"],
605 2 : "reparenting_history":["e1bfd8c633d713d279e6fcd2bcc15b6d"]
606 2 : }
607 2 : }"#;
608 2 :
609 2 : let expected = IndexPart {
610 2 : version: 5,
611 2 : layer_metadata: HashMap::from([
612 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000014EF420-00000000014EF499".parse().unwrap(), LayerFileMetadata {
613 2 : file_size: 23289856,
614 2 : generation: Generation::new(1),
615 2 : shard: ShardIndex::unsharded(),
616 2 : }),
617 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000014EF499-00000000015A7619".parse().unwrap(), LayerFileMetadata {
618 2 : file_size: 1015808,
619 2 : generation: Generation::new(1),
620 2 : shard: ShardIndex::unsharded(),
621 2 : })
622 2 : ]),
623 2 : disk_consistent_lsn: Lsn::from_str("0/15A7618").unwrap(),
624 2 : metadata: TimelineMetadata::from_bytes(&[226,88,25,241,0,46,0,4,0,0,0,0,1,90,118,24,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,78,244,32,0,0,0,0,1,78,244,32,0,0,0,16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
625 2 : deleted_at: None,
626 2 : archived_at: None,
627 2 : lineage: Lineage {
628 2 : reparenting_history_truncated: false,
629 2 : reparenting_history: vec![TimelineId::from_str("e1bfd8c633d713d279e6fcd2bcc15b6d").unwrap()],
630 2 : original_ancestor: Some((TimelineId::from_str("e2bfd8c633d713d279e6fcd2bcc15b6d").unwrap(), Lsn::from_str("0/15A7618").unwrap(), parse_naive_datetime("2024-05-07T18:52:36.322426563"))),
631 2 : },
632 2 : gc_blocking: None,
633 2 : last_aux_file_policy: None,
634 2 : import_pgdata: None,
635 2 : };
636 2 :
637 2 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
638 2 : assert_eq!(part, expected);
639 2 : }
640 :
641 : #[test]
642 2 : fn v6_indexpart_is_parsed() {
643 2 : let example = r#"{
644 2 : "version":6,
645 2 : "layer_metadata":{
646 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
647 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
648 2 : },
649 2 : "disk_consistent_lsn":"0/16960E8",
650 2 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
651 2 : "deleted_at": "2023-07-31T09:00:00.123",
652 2 : "lineage":{
653 2 : "original_ancestor":["e2bfd8c633d713d279e6fcd2bcc15b6d","0/15A7618","2024-05-07T18:52:36.322426563"],
654 2 : "reparenting_history":["e1bfd8c633d713d279e6fcd2bcc15b6d"]
655 2 : },
656 2 : "last_aux_file_policy": "V2"
657 2 : }"#;
658 2 :
659 2 : let expected = IndexPart {
660 2 : version: 6,
661 2 : layer_metadata: HashMap::from([
662 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
663 2 : file_size: 25600000,
664 2 : generation: Generation::none(),
665 2 : shard: ShardIndex::unsharded()
666 2 : }),
667 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
668 2 : // serde_json should always parse this but this might be a double with jq for
669 2 : // example.
670 2 : file_size: 9007199254741001,
671 2 : generation: Generation::none(),
672 2 : shard: ShardIndex::unsharded()
673 2 : })
674 2 : ]),
675 2 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
676 2 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
677 2 : deleted_at: Some(parse_naive_datetime("2023-07-31T09:00:00.123000000")),
678 2 : archived_at: None,
679 2 : lineage: Lineage {
680 2 : reparenting_history_truncated: false,
681 2 : reparenting_history: vec![TimelineId::from_str("e1bfd8c633d713d279e6fcd2bcc15b6d").unwrap()],
682 2 : original_ancestor: Some((TimelineId::from_str("e2bfd8c633d713d279e6fcd2bcc15b6d").unwrap(), Lsn::from_str("0/15A7618").unwrap(), parse_naive_datetime("2024-05-07T18:52:36.322426563"))),
683 2 : },
684 2 : gc_blocking: None,
685 2 : last_aux_file_policy: Some(AuxFilePolicy::V2),
686 2 : import_pgdata: None,
687 2 : };
688 2 :
689 2 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
690 2 : assert_eq!(part, expected);
691 2 : }
692 :
693 : #[test]
694 2 : fn v7_indexpart_is_parsed() {
695 2 : let example = r#"{
696 2 : "version": 7,
697 2 : "layer_metadata":{
698 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
699 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
700 2 : },
701 2 : "disk_consistent_lsn":"0/16960E8",
702 2 : "metadata": {
703 2 : "disk_consistent_lsn": "0/16960E8",
704 2 : "prev_record_lsn": "0/1696070",
705 2 : "ancestor_timeline": "e45a7f37d3ee2ff17dc14bf4f4e3f52e",
706 2 : "ancestor_lsn": "0/0",
707 2 : "latest_gc_cutoff_lsn": "0/1696070",
708 2 : "initdb_lsn": "0/1696070",
709 2 : "pg_version": 14
710 2 : },
711 2 : "deleted_at": "2023-07-31T09:00:00.123"
712 2 : }"#;
713 2 :
714 2 : let expected = IndexPart {
715 2 : version: 7,
716 2 : layer_metadata: HashMap::from([
717 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
718 2 : file_size: 25600000,
719 2 : generation: Generation::none(),
720 2 : shard: ShardIndex::unsharded()
721 2 : }),
722 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
723 2 : file_size: 9007199254741001,
724 2 : generation: Generation::none(),
725 2 : shard: ShardIndex::unsharded()
726 2 : })
727 2 : ]),
728 2 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
729 2 : metadata: TimelineMetadata::new(
730 2 : Lsn::from_str("0/16960E8").unwrap(),
731 2 : Some(Lsn::from_str("0/1696070").unwrap()),
732 2 : Some(TimelineId::from_str("e45a7f37d3ee2ff17dc14bf4f4e3f52e").unwrap()),
733 2 : Lsn::INVALID,
734 2 : Lsn::from_str("0/1696070").unwrap(),
735 2 : Lsn::from_str("0/1696070").unwrap(),
736 2 : 14,
737 2 : ).with_recalculated_checksum().unwrap(),
738 2 : deleted_at: Some(parse_naive_datetime("2023-07-31T09:00:00.123000000")),
739 2 : archived_at: None,
740 2 : lineage: Default::default(),
741 2 : gc_blocking: None,
742 2 : last_aux_file_policy: Default::default(),
743 2 : import_pgdata: None,
744 2 : };
745 2 :
746 2 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
747 2 : assert_eq!(part, expected);
748 2 : }
749 :
750 : #[test]
751 2 : fn v8_indexpart_is_parsed() {
752 2 : let example = r#"{
753 2 : "version": 8,
754 2 : "layer_metadata":{
755 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
756 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
757 2 : },
758 2 : "disk_consistent_lsn":"0/16960E8",
759 2 : "metadata": {
760 2 : "disk_consistent_lsn": "0/16960E8",
761 2 : "prev_record_lsn": "0/1696070",
762 2 : "ancestor_timeline": "e45a7f37d3ee2ff17dc14bf4f4e3f52e",
763 2 : "ancestor_lsn": "0/0",
764 2 : "latest_gc_cutoff_lsn": "0/1696070",
765 2 : "initdb_lsn": "0/1696070",
766 2 : "pg_version": 14
767 2 : },
768 2 : "deleted_at": "2023-07-31T09:00:00.123",
769 2 : "archived_at": "2023-04-29T09:00:00.123"
770 2 : }"#;
771 2 :
772 2 : let expected = IndexPart {
773 2 : version: 8,
774 2 : layer_metadata: HashMap::from([
775 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
776 2 : file_size: 25600000,
777 2 : generation: Generation::none(),
778 2 : shard: ShardIndex::unsharded()
779 2 : }),
780 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
781 2 : file_size: 9007199254741001,
782 2 : generation: Generation::none(),
783 2 : shard: ShardIndex::unsharded()
784 2 : })
785 2 : ]),
786 2 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
787 2 : metadata: TimelineMetadata::new(
788 2 : Lsn::from_str("0/16960E8").unwrap(),
789 2 : Some(Lsn::from_str("0/1696070").unwrap()),
790 2 : Some(TimelineId::from_str("e45a7f37d3ee2ff17dc14bf4f4e3f52e").unwrap()),
791 2 : Lsn::INVALID,
792 2 : Lsn::from_str("0/1696070").unwrap(),
793 2 : Lsn::from_str("0/1696070").unwrap(),
794 2 : 14,
795 2 : ).with_recalculated_checksum().unwrap(),
796 2 : deleted_at: Some(parse_naive_datetime("2023-07-31T09:00:00.123000000")),
797 2 : archived_at: Some(parse_naive_datetime("2023-04-29T09:00:00.123000000")),
798 2 : lineage: Default::default(),
799 2 : gc_blocking: None,
800 2 : last_aux_file_policy: Default::default(),
801 2 : import_pgdata: None,
802 2 : };
803 2 :
804 2 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
805 2 : assert_eq!(part, expected);
806 2 : }
807 :
808 : #[test]
809 2 : fn v9_indexpart_is_parsed() {
810 2 : let example = r#"{
811 2 : "version": 9,
812 2 : "layer_metadata":{
813 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
814 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
815 2 : },
816 2 : "disk_consistent_lsn":"0/16960E8",
817 2 : "metadata": {
818 2 : "disk_consistent_lsn": "0/16960E8",
819 2 : "prev_record_lsn": "0/1696070",
820 2 : "ancestor_timeline": "e45a7f37d3ee2ff17dc14bf4f4e3f52e",
821 2 : "ancestor_lsn": "0/0",
822 2 : "latest_gc_cutoff_lsn": "0/1696070",
823 2 : "initdb_lsn": "0/1696070",
824 2 : "pg_version": 14
825 2 : },
826 2 : "gc_blocking": {
827 2 : "started_at": "2024-07-19T09:00:00.123",
828 2 : "reasons": ["DetachAncestor"]
829 2 : }
830 2 : }"#;
831 2 :
832 2 : let expected = IndexPart {
833 2 : version: 9,
834 2 : layer_metadata: HashMap::from([
835 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
836 2 : file_size: 25600000,
837 2 : generation: Generation::none(),
838 2 : shard: ShardIndex::unsharded()
839 2 : }),
840 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
841 2 : file_size: 9007199254741001,
842 2 : generation: Generation::none(),
843 2 : shard: ShardIndex::unsharded()
844 2 : })
845 2 : ]),
846 2 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
847 2 : metadata: TimelineMetadata::new(
848 2 : Lsn::from_str("0/16960E8").unwrap(),
849 2 : Some(Lsn::from_str("0/1696070").unwrap()),
850 2 : Some(TimelineId::from_str("e45a7f37d3ee2ff17dc14bf4f4e3f52e").unwrap()),
851 2 : Lsn::INVALID,
852 2 : Lsn::from_str("0/1696070").unwrap(),
853 2 : Lsn::from_str("0/1696070").unwrap(),
854 2 : 14,
855 2 : ).with_recalculated_checksum().unwrap(),
856 2 : deleted_at: None,
857 2 : lineage: Default::default(),
858 2 : gc_blocking: Some(GcBlocking {
859 2 : started_at: parse_naive_datetime("2024-07-19T09:00:00.123000000"),
860 2 : reasons: enumset::EnumSet::from_iter([GcBlockingReason::DetachAncestor]),
861 2 : }),
862 2 : last_aux_file_policy: Default::default(),
863 2 : archived_at: None,
864 2 : import_pgdata: None,
865 2 : };
866 2 :
867 2 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
868 2 : assert_eq!(part, expected);
869 2 : }
870 :
871 : #[test]
872 2 : fn v10_importpgdata_is_parsed() {
873 2 : let example = r#"{
874 2 : "version": 10,
875 2 : "layer_metadata":{
876 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
877 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
878 2 : },
879 2 : "disk_consistent_lsn":"0/16960E8",
880 2 : "metadata": {
881 2 : "disk_consistent_lsn": "0/16960E8",
882 2 : "prev_record_lsn": "0/1696070",
883 2 : "ancestor_timeline": "e45a7f37d3ee2ff17dc14bf4f4e3f52e",
884 2 : "ancestor_lsn": "0/0",
885 2 : "latest_gc_cutoff_lsn": "0/1696070",
886 2 : "initdb_lsn": "0/1696070",
887 2 : "pg_version": 14
888 2 : },
889 2 : "gc_blocking": {
890 2 : "started_at": "2024-07-19T09:00:00.123",
891 2 : "reasons": ["DetachAncestor"]
892 2 : },
893 2 : "import_pgdata": {
894 2 : "V1": {
895 2 : "Done": {
896 2 : "idempotency_key": "specified-by-client-218a5213-5044-4562-a28d-d024c5f057f5",
897 2 : "started_at": "2024-11-13T09:23:42.123",
898 2 : "finished_at": "2024-11-13T09:42:23.123"
899 2 : }
900 2 : }
901 2 : }
902 2 : }"#;
903 2 :
904 2 : let expected = IndexPart {
905 2 : version: 10,
906 2 : layer_metadata: HashMap::from([
907 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
908 2 : file_size: 25600000,
909 2 : generation: Generation::none(),
910 2 : shard: ShardIndex::unsharded()
911 2 : }),
912 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
913 2 : file_size: 9007199254741001,
914 2 : generation: Generation::none(),
915 2 : shard: ShardIndex::unsharded()
916 2 : })
917 2 : ]),
918 2 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
919 2 : metadata: TimelineMetadata::new(
920 2 : Lsn::from_str("0/16960E8").unwrap(),
921 2 : Some(Lsn::from_str("0/1696070").unwrap()),
922 2 : Some(TimelineId::from_str("e45a7f37d3ee2ff17dc14bf4f4e3f52e").unwrap()),
923 2 : Lsn::INVALID,
924 2 : Lsn::from_str("0/1696070").unwrap(),
925 2 : Lsn::from_str("0/1696070").unwrap(),
926 2 : 14,
927 2 : ).with_recalculated_checksum().unwrap(),
928 2 : deleted_at: None,
929 2 : lineage: Default::default(),
930 2 : gc_blocking: Some(GcBlocking {
931 2 : started_at: parse_naive_datetime("2024-07-19T09:00:00.123000000"),
932 2 : reasons: enumset::EnumSet::from_iter([GcBlockingReason::DetachAncestor]),
933 2 : }),
934 2 : last_aux_file_policy: Default::default(),
935 2 : archived_at: None,
936 2 : import_pgdata: Some(import_pgdata::index_part_format::Root::V1(import_pgdata::index_part_format::V1::Done(import_pgdata::index_part_format::Done{
937 2 : started_at: parse_naive_datetime("2024-11-13T09:23:42.123000000"),
938 2 : finished_at: parse_naive_datetime("2024-11-13T09:42:23.123000000"),
939 2 : idempotency_key: import_pgdata::index_part_format::IdempotencyKey::new("specified-by-client-218a5213-5044-4562-a28d-d024c5f057f5".to_string()),
940 2 : })))
941 2 : };
942 2 :
943 2 : let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
944 2 : assert_eq!(part, expected);
945 2 : }
946 :
947 24 : fn parse_naive_datetime(s: &str) -> NaiveDateTime {
948 24 : chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S.%f").unwrap()
949 24 : }
950 : }
|
