Line data Source code
1 : //! In-memory index to track the tenant files on the remote storage.
2 : //!
3 : //! Able to restore itself from the storage index parts, that are located in every timeline's remote directory and contain all data about
4 : //! remote timeline layers and its metadata.
5 :
6 : use std::collections::HashMap;
7 :
8 : use chrono::NaiveDateTime;
9 : use pageserver_api::models::AuxFilePolicy;
10 : use serde::{Deserialize, Serialize};
11 : use utils::id::TimelineId;
12 :
13 : use crate::tenant::metadata::TimelineMetadata;
14 : use crate::tenant::storage_layer::LayerName;
15 : use crate::tenant::Generation;
16 : use pageserver_api::shard::ShardIndex;
17 :
18 : use utils::lsn::Lsn;
19 :
20 : /// In-memory representation of an `index_part.json` file
21 : ///
22 : /// Contains the data about all files in the timeline, present remotely and its metadata.
23 : ///
24 : /// This type needs to be backwards and forwards compatible. When changing the fields,
25 : /// remember to add a test case for the changed version.
26 2839 : #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
27 : pub struct IndexPart {
28 : /// Debugging aid describing the version of this type.
29 : #[serde(default)]
30 : version: usize,
31 :
32 : #[serde(default)]
33 : #[serde(skip_serializing_if = "Option::is_none")]
34 : pub deleted_at: Option<NaiveDateTime>,
35 :
36 : #[serde(default)]
37 : #[serde(skip_serializing_if = "Option::is_none")]
38 : pub archived_at: Option<NaiveDateTime>,
39 :
40 : /// Per layer file name metadata, which can be present for a present or missing layer file.
41 : ///
42 : /// Older versions of `IndexPart` will not have this property or have only a part of metadata
43 : /// that latest version stores.
44 : pub layer_metadata: HashMap<LayerName, LayerFileMetadata>,
45 :
46 : /// Because of the trouble of eyeballing the legacy "metadata" field, we copied the
47 : /// "disk_consistent_lsn" out. After version 7 this is no longer needed, but the name cannot be
48 : /// reused.
49 : pub(super) disk_consistent_lsn: Lsn,
50 :
51 : // TODO: rename as "metadata" next week, keep the alias = "metadata_bytes", bump version Adding
52 : // the "alias = metadata" was forgotten in #7693, so we have to use "rewrite = metadata_bytes"
53 : // for backwards compatibility.
54 : #[serde(
55 : rename = "metadata_bytes",
56 : alias = "metadata",
57 : with = "crate::tenant::metadata::modern_serde"
58 : )]
59 : pub metadata: TimelineMetadata,
60 :
61 : #[serde(default)]
62 : pub(crate) lineage: Lineage,
63 :
64 : #[serde(skip_serializing_if = "Option::is_none", default)]
65 : pub(crate) gc_blocking: Option<GcBlocking>,
66 :
67 : /// Describes the kind of aux files stored in the timeline.
68 : ///
69 : /// The value is modified during file ingestion when the latest wanted value communicated via tenant config is applied if it is acceptable.
70 : /// A V1 setting after V2 files have been committed is not accepted.
71 : ///
72 : /// None means no aux files have been written to the storage before the point
73 : /// when this flag is introduced.
74 : #[serde(skip_serializing_if = "Option::is_none", default)]
75 : pub(crate) last_aux_file_policy: Option<AuxFilePolicy>,
76 : }
77 :
78 : impl IndexPart {
79 : /// When adding or modifying any parts of `IndexPart`, increment the version so that it can be
80 : /// used to understand later versions.
81 : ///
82 : /// Version is currently informative only.
83 : /// Version history
84 : /// - 2: added `deleted_at`
85 : /// - 3: no longer deserialize `timeline_layers` (serialized format is the same, but timeline_layers
86 : /// is always generated from the keys of `layer_metadata`)
87 : /// - 4: timeline_layers is fully removed.
88 : /// - 5: lineage was added
89 : /// - 6: last_aux_file_policy is added.
90 : /// - 7: metadata_bytes is no longer written, but still read
91 : /// - 8: added `archived_at`
92 : /// - 9: +gc_blocking
93 : const LATEST_VERSION: usize = 9;
94 :
95 : // Versions we may see when reading from a bucket.
96 : pub const KNOWN_VERSIONS: &'static [usize] = &[1, 2, 3, 4, 5, 6, 7, 8, 9];
97 :
98 : pub const FILE_NAME: &'static str = "index_part.json";
99 :
100 416 : pub(crate) fn empty(metadata: TimelineMetadata) -> Self {
101 416 : IndexPart {
102 416 : version: Self::LATEST_VERSION,
103 416 : layer_metadata: Default::default(),
104 416 : disk_consistent_lsn: metadata.disk_consistent_lsn(),
105 416 : metadata,
106 416 : deleted_at: None,
107 416 : archived_at: None,
108 416 : lineage: Default::default(),
109 416 : gc_blocking: None,
110 416 : last_aux_file_policy: None,
111 416 : }
112 416 : }
113 :
114 0 : pub fn version(&self) -> usize {
115 0 : self.version
116 0 : }
117 :
118 : /// If you want this under normal operations, read it from self.metadata:
119 : /// this method is just for the scrubber to use when validating an index.
120 0 : pub fn duplicated_disk_consistent_lsn(&self) -> Lsn {
121 0 : self.disk_consistent_lsn
122 0 : }
123 :
124 20 : pub fn from_s3_bytes(bytes: &[u8]) -> Result<Self, serde_json::Error> {
125 20 : serde_json::from_slice::<IndexPart>(bytes)
126 20 : }
127 :
128 1399 : pub fn to_s3_bytes(&self) -> serde_json::Result<Vec<u8>> {
129 1399 : serde_json::to_vec(self)
130 1399 : }
131 :
132 : #[cfg(test)]
133 12 : pub(crate) fn example() -> Self {
134 12 : Self::empty(TimelineMetadata::example())
135 12 : }
136 : }
137 :
138 : /// Metadata gathered for each of the layer files.
139 : ///
140 : /// Fields have to be `Option`s because remote [`IndexPart`]'s can be from different version, which
141 : /// might have less or more metadata depending if upgrading or rolling back an upgrade.
142 12660 : #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
143 : pub struct LayerFileMetadata {
144 : pub file_size: u64,
145 :
146 : #[serde(default = "Generation::none")]
147 : #[serde(skip_serializing_if = "Generation::is_none")]
148 : pub generation: Generation,
149 :
150 : #[serde(default = "ShardIndex::unsharded")]
151 : #[serde(skip_serializing_if = "ShardIndex::is_unsharded")]
152 : pub shard: ShardIndex,
153 : }
154 :
155 : impl LayerFileMetadata {
156 2410 : pub fn new(file_size: u64, generation: Generation, shard: ShardIndex) -> Self {
157 2410 : LayerFileMetadata {
158 2410 : file_size,
159 2410 : generation,
160 2410 : shard,
161 2410 : }
162 2410 : }
163 : }
164 :
165 : /// Limited history of earlier ancestors.
166 : ///
167 : /// A timeline can have more than 1 earlier ancestor, in the rare case that it was repeatedly
168 : /// reparented by having an later timeline be detached from it's ancestor.
169 34 : #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Default)]
170 : pub(crate) struct Lineage {
171 : /// Has the `reparenting_history` been truncated to [`Lineage::REMEMBER_AT_MOST`].
172 : #[serde(skip_serializing_if = "is_false", default)]
173 : reparenting_history_truncated: bool,
174 :
175 : /// Earlier ancestors, truncated when [`Self::reparenting_history_truncated`]
176 : ///
177 : /// These are stored in case we want to support WAL based DR on the timeline. There can be many
178 : /// of these and at most one [`Self::original_ancestor`]. There cannot be more reparentings
179 : /// after [`Self::original_ancestor`] has been set.
180 : #[serde(skip_serializing_if = "Vec::is_empty", default)]
181 : reparenting_history: Vec<TimelineId>,
182 :
183 : /// The ancestor from which this timeline has been detached from and when.
184 : ///
185 : /// If you are adding support for detaching from a hierarchy, consider changing the ancestry
186 : /// into a `Vec<(TimelineId, Lsn)>` to be a path instead.
187 : // FIXME: this is insufficient even for path of two timelines for future wal recovery
188 : // purposes:
189 : //
190 : // assuming a "old main" which has received most of the WAL, and has a branch "new main",
191 : // starting a bit before "old main" last_record_lsn. the current version works fine,
192 : // because we will know to replay wal and branch at the recorded Lsn to do wal recovery.
193 : //
194 : // then assuming "new main" would similarly receive a branch right before its last_record_lsn,
195 : // "new new main". the current implementation would just store ("new main", ancestor_lsn, _)
196 : // here. however, we cannot recover from WAL using only that information, we would need the
197 : // whole ancestry here:
198 : //
199 : // ```json
200 : // [
201 : // ["old main", ancestor_lsn("new main"), _],
202 : // ["new main", ancestor_lsn("new new main"), _]
203 : // ]
204 : // ```
205 : #[serde(skip_serializing_if = "Option::is_none", default)]
206 : original_ancestor: Option<(TimelineId, Lsn, NaiveDateTime)>,
207 : }
208 :
209 5678 : fn is_false(b: &bool) -> bool {
210 5678 : !b
211 5678 : }
212 :
213 : impl Lineage {
214 : const REMEMBER_AT_MOST: usize = 100;
215 :
216 0 : pub(crate) fn record_previous_ancestor(&mut self, old_ancestor: &TimelineId) -> bool {
217 0 : if self.reparenting_history.last() == Some(old_ancestor) {
218 : // do not re-record it
219 0 : false
220 : } else {
221 : #[cfg(feature = "testing")]
222 : {
223 0 : let existing = self
224 0 : .reparenting_history
225 0 : .iter()
226 0 : .position(|x| x == old_ancestor);
227 0 : assert_eq!(
228 : existing, None,
229 0 : "we cannot reparent onto and off and onto the same timeline twice"
230 : );
231 : }
232 0 : let drop_oldest = self.reparenting_history.len() + 1 >= Self::REMEMBER_AT_MOST;
233 0 :
234 0 : self.reparenting_history_truncated |= drop_oldest;
235 0 : if drop_oldest {
236 0 : self.reparenting_history.remove(0);
237 0 : }
238 0 : self.reparenting_history.push(*old_ancestor);
239 0 : true
240 : }
241 0 : }
242 :
243 : /// Returns true if anything changed.
244 0 : pub(crate) fn record_detaching(&mut self, branchpoint: &(TimelineId, Lsn)) -> bool {
245 0 : if let Some((id, lsn, _)) = self.original_ancestor {
246 0 : assert_eq!(
247 0 : &(id, lsn),
248 : branchpoint,
249 0 : "detaching attempt has to be for the same ancestor we are already detached from"
250 : );
251 0 : false
252 : } else {
253 0 : self.original_ancestor =
254 0 : Some((branchpoint.0, branchpoint.1, chrono::Utc::now().naive_utc()));
255 0 : true
256 : }
257 0 : }
258 :
259 : /// The queried lsn is most likely the basebackup lsn, and this answers question "is it allowed
260 : /// to start a read/write primary at this lsn".
261 : ///
262 : /// Returns true if the Lsn was previously our branch point.
263 0 : pub(crate) fn is_previous_ancestor_lsn(&self, lsn: Lsn) -> bool {
264 0 : self.original_ancestor
265 0 : .is_some_and(|(_, ancestor_lsn, _)| ancestor_lsn == lsn)
266 0 : }
267 :
268 : /// Returns true if the timeline originally had an ancestor, and no longer has one.
269 0 : pub(crate) fn is_detached_from_ancestor(&self) -> bool {
270 0 : self.original_ancestor.is_some()
271 0 : }
272 :
273 : /// Returns original ancestor timeline id and lsn that this timeline has been detached from.
274 0 : pub(crate) fn detached_previous_ancestor(&self) -> Option<(TimelineId, Lsn)> {
275 0 : self.original_ancestor.map(|(id, lsn, _)| (id, lsn))
276 0 : }
277 :
278 0 : pub(crate) fn is_reparented(&self) -> bool {
279 0 : !self.reparenting_history.is_empty()
280 0 : }
281 : }
282 :
283 6 : #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
284 : pub(crate) struct GcBlocking {
285 : pub(crate) started_at: NaiveDateTime,
286 : pub(crate) reasons: enumset::EnumSet<GcBlockingReason>,
287 : }
288 :
289 4 : #[derive(Debug, enumset::EnumSetType, serde::Serialize, serde::Deserialize)]
290 : #[enumset(serialize_repr = "list")]
291 : pub(crate) enum GcBlockingReason {
292 : Manual,
293 : DetachAncestor,
294 : }
295 :
296 : impl GcBlocking {
297 0 : pub(super) fn started_now_for(reason: GcBlockingReason) -> Self {
298 0 : GcBlocking {
299 0 : started_at: chrono::Utc::now().naive_utc(),
300 0 : reasons: enumset::EnumSet::only(reason),
301 0 : }
302 0 : }
303 :
304 : /// Returns true if the given reason is one of the reasons why the gc is blocked.
305 0 : pub(crate) fn blocked_by(&self, reason: GcBlockingReason) -> bool {
306 0 : self.reasons.contains(reason)
307 0 : }
308 :
309 : /// Returns a version of self with the given reason.
310 0 : pub(super) fn with_reason(&self, reason: GcBlockingReason) -> Self {
311 0 : assert!(!self.blocked_by(reason));
312 0 : let mut reasons = self.reasons;
313 0 : reasons.insert(reason);
314 0 :
315 0 : Self {
316 0 : started_at: self.started_at,
317 0 : reasons,
318 0 : }
319 0 : }
320 :
321 : /// Returns a version of self without the given reason. Assumption is that if
322 : /// there are no more reasons, we can unblock the gc by returning `None`.
323 0 : pub(super) fn without_reason(&self, reason: GcBlockingReason) -> Option<Self> {
324 0 : assert!(self.blocked_by(reason));
325 :
326 0 : if self.reasons.len() == 1 {
327 0 : None
328 : } else {
329 0 : let mut reasons = self.reasons;
330 0 : assert!(reasons.remove(reason));
331 0 : assert!(!reasons.is_empty());
332 :
333 0 : Some(Self {
334 0 : started_at: self.started_at,
335 0 : reasons,
336 0 : })
337 : }
338 0 : }
339 : }
340 :
341 : #[cfg(test)]
342 : mod tests {
343 : use super::*;
344 : use std::str::FromStr;
345 : use utils::id::TimelineId;
346 :
347 : #[test]
348 2 : fn v1_indexpart_is_parsed() {
349 2 : let example = r#"{
350 2 : "version":1,
351 2 : "timeline_layers":["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"],
352 2 : "layer_metadata":{
353 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
354 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
355 2 : },
356 2 : "disk_consistent_lsn":"0/16960E8",
357 2 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
358 2 : }"#;
359 2 :
360 2 : let expected = IndexPart {
361 2 : // note this is not verified, could be anything, but exists for humans debugging.. could be the git version instead?
362 2 : version: 1,
363 2 : layer_metadata: HashMap::from([
364 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
365 2 : file_size: 25600000,
366 2 : generation: Generation::none(),
367 2 : shard: ShardIndex::unsharded()
368 2 : }),
369 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
370 2 : // serde_json should always parse this but this might be a double with jq for
371 2 : // example.
372 2 : file_size: 9007199254741001,
373 2 : generation: Generation::none(),
374 2 : shard: ShardIndex::unsharded()
375 2 : })
376 2 : ]),
377 2 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
378 2 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
379 2 : deleted_at: None,
380 2 : archived_at: None,
381 2 : lineage: Lineage::default(),
382 2 : gc_blocking: None,
383 2 : last_aux_file_policy: None,
384 2 : };
385 2 :
386 2 : let part = IndexPart::from_s3_bytes(example.as_bytes()).unwrap();
387 2 : assert_eq!(part, expected);
388 2 : }
389 :
390 : #[test]
391 2 : fn v1_indexpart_is_parsed_with_optional_missing_layers() {
392 2 : let example = r#"{
393 2 : "version":1,
394 2 : "timeline_layers":["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"],
395 2 : "missing_layers":["This shouldn't fail deserialization"],
396 2 : "layer_metadata":{
397 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
398 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
399 2 : },
400 2 : "disk_consistent_lsn":"0/16960E8",
401 2 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
402 2 : }"#;
403 2 :
404 2 : let expected = IndexPart {
405 2 : // note this is not verified, could be anything, but exists for humans debugging.. could be the git version instead?
406 2 : version: 1,
407 2 : layer_metadata: HashMap::from([
408 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
409 2 : file_size: 25600000,
410 2 : generation: Generation::none(),
411 2 : shard: ShardIndex::unsharded()
412 2 : }),
413 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
414 2 : // serde_json should always parse this but this might be a double with jq for
415 2 : // example.
416 2 : file_size: 9007199254741001,
417 2 : generation: Generation::none(),
418 2 : shard: ShardIndex::unsharded()
419 2 : })
420 2 : ]),
421 2 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
422 2 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
423 2 : deleted_at: None,
424 2 : archived_at: None,
425 2 : lineage: Lineage::default(),
426 2 : gc_blocking: None,
427 2 : last_aux_file_policy: None,
428 2 : };
429 2 :
430 2 : let part = IndexPart::from_s3_bytes(example.as_bytes()).unwrap();
431 2 : assert_eq!(part, expected);
432 2 : }
433 :
434 : #[test]
435 2 : fn v2_indexpart_is_parsed_with_deleted_at() {
436 2 : let example = r#"{
437 2 : "version":2,
438 2 : "timeline_layers":["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"],
439 2 : "missing_layers":["This shouldn't fail deserialization"],
440 2 : "layer_metadata":{
441 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
442 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
443 2 : },
444 2 : "disk_consistent_lsn":"0/16960E8",
445 2 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
446 2 : "deleted_at": "2023-07-31T09:00:00.123"
447 2 : }"#;
448 2 :
449 2 : let expected = IndexPart {
450 2 : // note this is not verified, could be anything, but exists for humans debugging.. could be the git version instead?
451 2 : version: 2,
452 2 : layer_metadata: HashMap::from([
453 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
454 2 : file_size: 25600000,
455 2 : generation: Generation::none(),
456 2 : shard: ShardIndex::unsharded()
457 2 : }),
458 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
459 2 : // serde_json should always parse this but this might be a double with jq for
460 2 : // example.
461 2 : file_size: 9007199254741001,
462 2 : generation: Generation::none(),
463 2 : shard: ShardIndex::unsharded()
464 2 : })
465 2 : ]),
466 2 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
467 2 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
468 2 : deleted_at: Some(parse_naive_datetime("2023-07-31T09:00:00.123000000")),
469 2 : archived_at: None,
470 2 : lineage: Lineage::default(),
471 2 : gc_blocking: None,
472 2 : last_aux_file_policy: None,
473 2 : };
474 2 :
475 2 : let part = IndexPart::from_s3_bytes(example.as_bytes()).unwrap();
476 2 : assert_eq!(part, expected);
477 2 : }
478 :
479 : #[test]
480 2 : fn empty_layers_are_parsed() {
481 2 : let empty_layers_json = r#"{
482 2 : "version":1,
483 2 : "timeline_layers":[],
484 2 : "layer_metadata":{},
485 2 : "disk_consistent_lsn":"0/2532648",
486 2 : "metadata_bytes":[136,151,49,208,0,70,0,4,0,0,0,0,2,83,38,72,1,0,0,0,0,2,83,38,32,1,87,198,240,135,97,119,45,125,38,29,155,161,140,141,255,210,0,0,0,0,2,83,38,72,0,0,0,0,1,73,240,192,0,0,0,0,1,73,240,192,0,0,0,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
487 2 : }"#;
488 2 :
489 2 : let expected = IndexPart {
490 2 : version: 1,
491 2 : layer_metadata: HashMap::new(),
492 2 : disk_consistent_lsn: "0/2532648".parse::<Lsn>().unwrap(),
493 2 : metadata: TimelineMetadata::from_bytes(&[
494 2 : 136, 151, 49, 208, 0, 70, 0, 4, 0, 0, 0, 0, 2, 83, 38, 72, 1, 0, 0, 0, 0, 2, 83,
495 2 : 38, 32, 1, 87, 198, 240, 135, 97, 119, 45, 125, 38, 29, 155, 161, 140, 141, 255,
496 2 : 210, 0, 0, 0, 0, 2, 83, 38, 72, 0, 0, 0, 0, 1, 73, 240, 192, 0, 0, 0, 0, 1, 73,
497 2 : 240, 192, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
498 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
499 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
500 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
501 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
502 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
503 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
504 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
505 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
506 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
507 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
508 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
509 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
510 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
511 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
512 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
513 2 : 0, 0,
514 2 : ])
515 2 : .unwrap(),
516 2 : deleted_at: None,
517 2 : archived_at: None,
518 2 : lineage: Lineage::default(),
519 2 : gc_blocking: None,
520 2 : last_aux_file_policy: None,
521 2 : };
522 2 :
523 2 : let empty_layers_parsed = IndexPart::from_s3_bytes(empty_layers_json.as_bytes()).unwrap();
524 2 :
525 2 : assert_eq!(empty_layers_parsed, expected);
526 2 : }
527 :
528 : #[test]
529 2 : fn v4_indexpart_is_parsed() {
530 2 : let example = r#"{
531 2 : "version":4,
532 2 : "layer_metadata":{
533 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
534 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
535 2 : },
536 2 : "disk_consistent_lsn":"0/16960E8",
537 2 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
538 2 : "deleted_at": "2023-07-31T09:00:00.123"
539 2 : }"#;
540 2 :
541 2 : let expected = IndexPart {
542 2 : version: 4,
543 2 : layer_metadata: HashMap::from([
544 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
545 2 : file_size: 25600000,
546 2 : generation: Generation::none(),
547 2 : shard: ShardIndex::unsharded()
548 2 : }),
549 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
550 2 : // serde_json should always parse this but this might be a double with jq for
551 2 : // example.
552 2 : file_size: 9007199254741001,
553 2 : generation: Generation::none(),
554 2 : shard: ShardIndex::unsharded()
555 2 : })
556 2 : ]),
557 2 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
558 2 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
559 2 : deleted_at: Some(parse_naive_datetime("2023-07-31T09:00:00.123000000")),
560 2 : archived_at: None,
561 2 : lineage: Lineage::default(),
562 2 : gc_blocking: None,
563 2 : last_aux_file_policy: None,
564 2 : };
565 2 :
566 2 : let part = IndexPart::from_s3_bytes(example.as_bytes()).unwrap();
567 2 : assert_eq!(part, expected);
568 2 : }
569 :
570 : #[test]
571 2 : fn v5_indexpart_is_parsed() {
572 2 : let example = r#"{
573 2 : "version":5,
574 2 : "layer_metadata":{
575 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000014EF420-00000000014EF499":{"file_size":23289856,"generation":1},
576 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000014EF499-00000000015A7619":{"file_size":1015808,"generation":1}},
577 2 : "disk_consistent_lsn":"0/15A7618",
578 2 : "metadata_bytes":[226,88,25,241,0,46,0,4,0,0,0,0,1,90,118,24,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,78,244,32,0,0,0,0,1,78,244,32,0,0,0,16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
579 2 : "lineage":{
580 2 : "original_ancestor":["e2bfd8c633d713d279e6fcd2bcc15b6d","0/15A7618","2024-05-07T18:52:36.322426563"],
581 2 : "reparenting_history":["e1bfd8c633d713d279e6fcd2bcc15b6d"]
582 2 : }
583 2 : }"#;
584 2 :
585 2 : let expected = IndexPart {
586 2 : version: 5,
587 2 : layer_metadata: HashMap::from([
588 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000014EF420-00000000014EF499".parse().unwrap(), LayerFileMetadata {
589 2 : file_size: 23289856,
590 2 : generation: Generation::new(1),
591 2 : shard: ShardIndex::unsharded(),
592 2 : }),
593 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000014EF499-00000000015A7619".parse().unwrap(), LayerFileMetadata {
594 2 : file_size: 1015808,
595 2 : generation: Generation::new(1),
596 2 : shard: ShardIndex::unsharded(),
597 2 : })
598 2 : ]),
599 2 : disk_consistent_lsn: Lsn::from_str("0/15A7618").unwrap(),
600 2 : metadata: TimelineMetadata::from_bytes(&[226,88,25,241,0,46,0,4,0,0,0,0,1,90,118,24,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,78,244,32,0,0,0,0,1,78,244,32,0,0,0,16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
601 2 : deleted_at: None,
602 2 : archived_at: None,
603 2 : lineage: Lineage {
604 2 : reparenting_history_truncated: false,
605 2 : reparenting_history: vec![TimelineId::from_str("e1bfd8c633d713d279e6fcd2bcc15b6d").unwrap()],
606 2 : original_ancestor: Some((TimelineId::from_str("e2bfd8c633d713d279e6fcd2bcc15b6d").unwrap(), Lsn::from_str("0/15A7618").unwrap(), parse_naive_datetime("2024-05-07T18:52:36.322426563"))),
607 2 : },
608 2 : gc_blocking: None,
609 2 : last_aux_file_policy: None,
610 2 : };
611 2 :
612 2 : let part = IndexPart::from_s3_bytes(example.as_bytes()).unwrap();
613 2 : assert_eq!(part, expected);
614 2 : }
615 :
616 : #[test]
617 2 : fn v6_indexpart_is_parsed() {
618 2 : let example = r#"{
619 2 : "version":6,
620 2 : "layer_metadata":{
621 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
622 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
623 2 : },
624 2 : "disk_consistent_lsn":"0/16960E8",
625 2 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
626 2 : "deleted_at": "2023-07-31T09:00:00.123",
627 2 : "lineage":{
628 2 : "original_ancestor":["e2bfd8c633d713d279e6fcd2bcc15b6d","0/15A7618","2024-05-07T18:52:36.322426563"],
629 2 : "reparenting_history":["e1bfd8c633d713d279e6fcd2bcc15b6d"]
630 2 : },
631 2 : "last_aux_file_policy": "V2"
632 2 : }"#;
633 2 :
634 2 : let expected = IndexPart {
635 2 : version: 6,
636 2 : layer_metadata: HashMap::from([
637 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
638 2 : file_size: 25600000,
639 2 : generation: Generation::none(),
640 2 : shard: ShardIndex::unsharded()
641 2 : }),
642 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
643 2 : // serde_json should always parse this but this might be a double with jq for
644 2 : // example.
645 2 : file_size: 9007199254741001,
646 2 : generation: Generation::none(),
647 2 : shard: ShardIndex::unsharded()
648 2 : })
649 2 : ]),
650 2 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
651 2 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
652 2 : deleted_at: Some(parse_naive_datetime("2023-07-31T09:00:00.123000000")),
653 2 : archived_at: None,
654 2 : lineage: Lineage {
655 2 : reparenting_history_truncated: false,
656 2 : reparenting_history: vec![TimelineId::from_str("e1bfd8c633d713d279e6fcd2bcc15b6d").unwrap()],
657 2 : original_ancestor: Some((TimelineId::from_str("e2bfd8c633d713d279e6fcd2bcc15b6d").unwrap(), Lsn::from_str("0/15A7618").unwrap(), parse_naive_datetime("2024-05-07T18:52:36.322426563"))),
658 2 : },
659 2 : gc_blocking: None,
660 2 : last_aux_file_policy: Some(AuxFilePolicy::V2),
661 2 : };
662 2 :
663 2 : let part = IndexPart::from_s3_bytes(example.as_bytes()).unwrap();
664 2 : assert_eq!(part, expected);
665 2 : }
666 :
667 : #[test]
668 2 : fn v7_indexpart_is_parsed() {
669 2 : let example = r#"{
670 2 : "version": 7,
671 2 : "layer_metadata":{
672 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
673 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
674 2 : },
675 2 : "disk_consistent_lsn":"0/16960E8",
676 2 : "metadata": {
677 2 : "disk_consistent_lsn": "0/16960E8",
678 2 : "prev_record_lsn": "0/1696070",
679 2 : "ancestor_timeline": "e45a7f37d3ee2ff17dc14bf4f4e3f52e",
680 2 : "ancestor_lsn": "0/0",
681 2 : "latest_gc_cutoff_lsn": "0/1696070",
682 2 : "initdb_lsn": "0/1696070",
683 2 : "pg_version": 14
684 2 : },
685 2 : "deleted_at": "2023-07-31T09:00:00.123"
686 2 : }"#;
687 2 :
688 2 : let expected = IndexPart {
689 2 : version: 7,
690 2 : layer_metadata: HashMap::from([
691 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
692 2 : file_size: 25600000,
693 2 : generation: Generation::none(),
694 2 : shard: ShardIndex::unsharded()
695 2 : }),
696 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
697 2 : file_size: 9007199254741001,
698 2 : generation: Generation::none(),
699 2 : shard: ShardIndex::unsharded()
700 2 : })
701 2 : ]),
702 2 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
703 2 : metadata: TimelineMetadata::new(
704 2 : Lsn::from_str("0/16960E8").unwrap(),
705 2 : Some(Lsn::from_str("0/1696070").unwrap()),
706 2 : Some(TimelineId::from_str("e45a7f37d3ee2ff17dc14bf4f4e3f52e").unwrap()),
707 2 : Lsn::INVALID,
708 2 : Lsn::from_str("0/1696070").unwrap(),
709 2 : Lsn::from_str("0/1696070").unwrap(),
710 2 : 14,
711 2 : ).with_recalculated_checksum().unwrap(),
712 2 : deleted_at: Some(parse_naive_datetime("2023-07-31T09:00:00.123000000")),
713 2 : archived_at: None,
714 2 : lineage: Default::default(),
715 2 : gc_blocking: None,
716 2 : last_aux_file_policy: Default::default(),
717 2 : };
718 2 :
719 2 : let part = IndexPart::from_s3_bytes(example.as_bytes()).unwrap();
720 2 : assert_eq!(part, expected);
721 2 : }
722 :
723 : #[test]
724 2 : fn v8_indexpart_is_parsed() {
725 2 : let example = r#"{
726 2 : "version": 8,
727 2 : "layer_metadata":{
728 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
729 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
730 2 : },
731 2 : "disk_consistent_lsn":"0/16960E8",
732 2 : "metadata": {
733 2 : "disk_consistent_lsn": "0/16960E8",
734 2 : "prev_record_lsn": "0/1696070",
735 2 : "ancestor_timeline": "e45a7f37d3ee2ff17dc14bf4f4e3f52e",
736 2 : "ancestor_lsn": "0/0",
737 2 : "latest_gc_cutoff_lsn": "0/1696070",
738 2 : "initdb_lsn": "0/1696070",
739 2 : "pg_version": 14
740 2 : },
741 2 : "deleted_at": "2023-07-31T09:00:00.123",
742 2 : "archived_at": "2023-04-29T09:00:00.123"
743 2 : }"#;
744 2 :
745 2 : let expected = IndexPart {
746 2 : version: 8,
747 2 : layer_metadata: HashMap::from([
748 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
749 2 : file_size: 25600000,
750 2 : generation: Generation::none(),
751 2 : shard: ShardIndex::unsharded()
752 2 : }),
753 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
754 2 : file_size: 9007199254741001,
755 2 : generation: Generation::none(),
756 2 : shard: ShardIndex::unsharded()
757 2 : })
758 2 : ]),
759 2 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
760 2 : metadata: TimelineMetadata::new(
761 2 : Lsn::from_str("0/16960E8").unwrap(),
762 2 : Some(Lsn::from_str("0/1696070").unwrap()),
763 2 : Some(TimelineId::from_str("e45a7f37d3ee2ff17dc14bf4f4e3f52e").unwrap()),
764 2 : Lsn::INVALID,
765 2 : Lsn::from_str("0/1696070").unwrap(),
766 2 : Lsn::from_str("0/1696070").unwrap(),
767 2 : 14,
768 2 : ).with_recalculated_checksum().unwrap(),
769 2 : deleted_at: Some(parse_naive_datetime("2023-07-31T09:00:00.123000000")),
770 2 : archived_at: Some(parse_naive_datetime("2023-04-29T09:00:00.123000000")),
771 2 : lineage: Default::default(),
772 2 : gc_blocking: None,
773 2 : last_aux_file_policy: Default::default(),
774 2 : };
775 2 :
776 2 : let part = IndexPart::from_s3_bytes(example.as_bytes()).unwrap();
777 2 : assert_eq!(part, expected);
778 2 : }
779 :
780 : #[test]
781 2 : fn v9_indexpart_is_parsed() {
782 2 : let example = r#"{
783 2 : "version": 9,
784 2 : "layer_metadata":{
785 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
786 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
787 2 : },
788 2 : "disk_consistent_lsn":"0/16960E8",
789 2 : "metadata": {
790 2 : "disk_consistent_lsn": "0/16960E8",
791 2 : "prev_record_lsn": "0/1696070",
792 2 : "ancestor_timeline": "e45a7f37d3ee2ff17dc14bf4f4e3f52e",
793 2 : "ancestor_lsn": "0/0",
794 2 : "latest_gc_cutoff_lsn": "0/1696070",
795 2 : "initdb_lsn": "0/1696070",
796 2 : "pg_version": 14
797 2 : },
798 2 : "gc_blocking": {
799 2 : "started_at": "2024-07-19T09:00:00.123",
800 2 : "reasons": ["DetachAncestor"]
801 2 : }
802 2 : }"#;
803 2 :
804 2 : let expected = IndexPart {
805 2 : version: 9,
806 2 : layer_metadata: HashMap::from([
807 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
808 2 : file_size: 25600000,
809 2 : generation: Generation::none(),
810 2 : shard: ShardIndex::unsharded()
811 2 : }),
812 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
813 2 : file_size: 9007199254741001,
814 2 : generation: Generation::none(),
815 2 : shard: ShardIndex::unsharded()
816 2 : })
817 2 : ]),
818 2 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
819 2 : metadata: TimelineMetadata::new(
820 2 : Lsn::from_str("0/16960E8").unwrap(),
821 2 : Some(Lsn::from_str("0/1696070").unwrap()),
822 2 : Some(TimelineId::from_str("e45a7f37d3ee2ff17dc14bf4f4e3f52e").unwrap()),
823 2 : Lsn::INVALID,
824 2 : Lsn::from_str("0/1696070").unwrap(),
825 2 : Lsn::from_str("0/1696070").unwrap(),
826 2 : 14,
827 2 : ).with_recalculated_checksum().unwrap(),
828 2 : deleted_at: None,
829 2 : lineage: Default::default(),
830 2 : gc_blocking: Some(GcBlocking {
831 2 : started_at: parse_naive_datetime("2024-07-19T09:00:00.123000000"),
832 2 : reasons: enumset::EnumSet::from_iter([GcBlockingReason::DetachAncestor]),
833 2 : }),
834 2 : last_aux_file_policy: Default::default(),
835 2 : archived_at: None,
836 2 : };
837 2 :
838 2 : let part = IndexPart::from_s3_bytes(example.as_bytes()).unwrap();
839 2 : assert_eq!(part, expected);
840 2 : }
841 :
842 18 : fn parse_naive_datetime(s: &str) -> NaiveDateTime {
843 18 : chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S.%f").unwrap()
844 18 : }
845 : }
|
