Line data Source code
1 : //! In-memory index to track the tenant files on the remote storage.
2 : //! Able to restore itself from the storage index parts, that are located in every timeline's remote directory and contain all data about
3 : //! remote timeline layers and its metadata.
4 :
5 : use std::collections::HashMap;
6 :
7 : use chrono::NaiveDateTime;
8 : use pageserver_api::models::AuxFilePolicy;
9 : use serde::{Deserialize, Serialize};
10 : use utils::id::TimelineId;
11 :
12 : use crate::tenant::metadata::TimelineMetadata;
13 : use crate::tenant::storage_layer::LayerName;
14 : use crate::tenant::Generation;
15 : use pageserver_api::shard::ShardIndex;
16 :
17 : use utils::lsn::Lsn;
18 :
19 : /// In-memory representation of an `index_part.json` file
20 : ///
21 : /// Contains the data about all files in the timeline, present remotely and its metadata.
22 : ///
23 : /// This type needs to be backwards and forwards compatible. When changing the fields,
24 : /// remember to add a test case for the changed version.
25 8633 : #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
26 : pub struct IndexPart {
27 : /// Debugging aid describing the version of this type.
28 : #[serde(default)]
29 : version: usize,
30 :
31 : #[serde(default)]
32 : #[serde(skip_serializing_if = "Option::is_none")]
33 : pub deleted_at: Option<NaiveDateTime>,
34 :
35 : #[serde(default)]
36 : #[serde(skip_serializing_if = "Option::is_none")]
37 : pub archived_at: Option<NaiveDateTime>,
38 :
39 : /// Per layer file name metadata, which can be present for a present or missing layer file.
40 : ///
41 : /// Older versions of `IndexPart` will not have this property or have only a part of metadata
42 : /// that latest version stores.
43 : pub layer_metadata: HashMap<LayerName, LayerFileMetadata>,
44 :
45 : /// Because of the trouble of eyeballing the legacy "metadata" field, we copied the
46 : /// "disk_consistent_lsn" out. After version 7 this is no longer needed, but the name cannot be
47 : /// reused.
48 : pub(super) disk_consistent_lsn: Lsn,
49 :
50 : // TODO: rename as "metadata" next week, keep the alias = "metadata_bytes", bump version Adding
51 : // the "alias = metadata" was forgotten in #7693, so we have to use "rewrite = metadata_bytes"
52 : // for backwards compatibility.
53 : #[serde(
54 : rename = "metadata_bytes",
55 : alias = "metadata",
56 : with = "crate::tenant::metadata::modern_serde"
57 : )]
58 : pub metadata: TimelineMetadata,
59 :
60 : #[serde(default)]
61 : pub(crate) lineage: Lineage,
62 :
63 : #[serde(skip_serializing_if = "Option::is_none", default)]
64 : pub(crate) gc_blocking: Option<GcBlocking>,
65 :
66 : /// Describes the kind of aux files stored in the timeline.
67 : ///
68 : /// The value is modified during file ingestion when the latest wanted value communicated via tenant config is applied if it is acceptable.
69 : /// A V1 setting after V2 files have been committed is not accepted.
70 : ///
71 : /// None means no aux files have been written to the storage before the point
72 : /// when this flag is introduced.
73 : #[serde(skip_serializing_if = "Option::is_none", default)]
74 : pub(crate) last_aux_file_policy: Option<AuxFilePolicy>,
75 : }
76 :
77 : impl IndexPart {
78 : /// When adding or modifying any parts of `IndexPart`, increment the version so that it can be
79 : /// used to understand later versions.
80 : ///
81 : /// Version is currently informative only.
82 : /// Version history
83 : /// - 2: added `deleted_at`
84 : /// - 3: no longer deserialize `timeline_layers` (serialized format is the same, but timeline_layers
85 : /// is always generated from the keys of `layer_metadata`)
86 : /// - 4: timeline_layers is fully removed.
87 : /// - 5: lineage was added
88 : /// - 6: last_aux_file_policy is added.
89 : /// - 7: metadata_bytes is no longer written, but still read
90 : /// - 8: added `archived_at`
91 : /// - 9: +gc_blocking
92 : const LATEST_VERSION: usize = 9;
93 :
94 : // Versions we may see when reading from a bucket.
95 : pub const KNOWN_VERSIONS: &'static [usize] = &[1, 2, 3, 4, 5, 6, 7, 8, 9];
96 :
97 : pub const FILE_NAME: &'static str = "index_part.json";
98 :
99 1254 : pub(crate) fn empty(metadata: TimelineMetadata) -> Self {
100 1254 : IndexPart {
101 1254 : version: Self::LATEST_VERSION,
102 1254 : layer_metadata: Default::default(),
103 1254 : disk_consistent_lsn: metadata.disk_consistent_lsn(),
104 1254 : metadata,
105 1254 : deleted_at: None,
106 1254 : archived_at: None,
107 1254 : lineage: Default::default(),
108 1254 : gc_blocking: None,
109 1254 : last_aux_file_policy: None,
110 1254 : }
111 1254 : }
112 :
113 0 : pub fn version(&self) -> usize {
114 0 : self.version
115 0 : }
116 :
117 : /// If you want this under normal operations, read it from self.metadata:
118 : /// this method is just for the scrubber to use when validating an index.
119 0 : pub fn duplicated_disk_consistent_lsn(&self) -> Lsn {
120 0 : self.disk_consistent_lsn
121 0 : }
122 :
123 60 : pub fn from_s3_bytes(bytes: &[u8]) -> Result<Self, serde_json::Error> {
124 60 : serde_json::from_slice::<IndexPart>(bytes)
125 60 : }
126 :
127 4235 : pub fn to_s3_bytes(&self) -> serde_json::Result<Vec<u8>> {
128 4235 : serde_json::to_vec(self)
129 4235 : }
130 :
131 : #[cfg(test)]
132 36 : pub(crate) fn example() -> Self {
133 36 : Self::empty(TimelineMetadata::example())
134 36 : }
135 :
136 36 : pub(crate) fn last_aux_file_policy(&self) -> Option<AuxFilePolicy> {
137 36 : self.last_aux_file_policy
138 36 : }
139 : }
140 :
141 : /// Metadata gathered for each of the layer files.
142 : ///
143 : /// Fields have to be `Option`s because remote [`IndexPart`]'s can be from different version, which
144 : /// might have less or more metadata depending if upgrading or rolling back an upgrade.
145 37980 : #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
146 : pub struct LayerFileMetadata {
147 : pub file_size: u64,
148 :
149 : #[serde(default = "Generation::none")]
150 : #[serde(skip_serializing_if = "Generation::is_none")]
151 : pub generation: Generation,
152 :
153 : #[serde(default = "ShardIndex::unsharded")]
154 : #[serde(skip_serializing_if = "ShardIndex::is_unsharded")]
155 : pub shard: ShardIndex,
156 : }
157 :
158 : impl LayerFileMetadata {
159 7251 : pub fn new(file_size: u64, generation: Generation, shard: ShardIndex) -> Self {
160 7251 : LayerFileMetadata {
161 7251 : file_size,
162 7251 : generation,
163 7251 : shard,
164 7251 : }
165 7251 : }
166 : }
167 :
168 : /// Limited history of earlier ancestors.
169 : ///
170 : /// A timeline can have more than 1 earlier ancestor, in the rare case that it was repeatedly
171 : /// reparented by having an later timeline be detached from it's ancestor.
172 102 : #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Default)]
173 : pub(crate) struct Lineage {
174 : /// Has the `reparenting_history` been truncated to [`Lineage::REMEMBER_AT_MOST`].
175 : #[serde(skip_serializing_if = "is_false", default)]
176 : reparenting_history_truncated: bool,
177 :
178 : /// Earlier ancestors, truncated when [`Self::reparenting_history_truncated`]
179 : ///
180 : /// These are stored in case we want to support WAL based DR on the timeline. There can be many
181 : /// of these and at most one [`Self::original_ancestor`]. There cannot be more reparentings
182 : /// after [`Self::original_ancestor`] has been set.
183 : #[serde(skip_serializing_if = "Vec::is_empty", default)]
184 : reparenting_history: Vec<TimelineId>,
185 :
186 : /// The ancestor from which this timeline has been detached from and when.
187 : ///
188 : /// If you are adding support for detaching from a hierarchy, consider changing the ancestry
189 : /// into a `Vec<(TimelineId, Lsn)>` to be a path instead.
190 : // FIXME: this is insufficient even for path of two timelines for future wal recovery
191 : // purposes:
192 : //
193 : // assuming a "old main" which has received most of the WAL, and has a branch "new main",
194 : // starting a bit before "old main" last_record_lsn. the current version works fine,
195 : // because we will know to replay wal and branch at the recorded Lsn to do wal recovery.
196 : //
197 : // then assuming "new main" would similarly receive a branch right before its last_record_lsn,
198 : // "new new main". the current implementation would just store ("new main", ancestor_lsn, _)
199 : // here. however, we cannot recover from WAL using only that information, we would need the
200 : // whole ancestry here:
201 : //
202 : // ```json
203 : // [
204 : // ["old main", ancestor_lsn("new main"), _],
205 : // ["new main", ancestor_lsn("new new main"), _]
206 : // ]
207 : // ```
208 : #[serde(skip_serializing_if = "Option::is_none", default)]
209 : original_ancestor: Option<(TimelineId, Lsn, NaiveDateTime)>,
210 : }
211 :
212 17266 : fn is_false(b: &bool) -> bool {
213 17266 : !b
214 17266 : }
215 :
216 : impl Lineage {
217 : const REMEMBER_AT_MOST: usize = 100;
218 :
219 0 : pub(crate) fn record_previous_ancestor(&mut self, old_ancestor: &TimelineId) -> bool {
220 0 : if self.reparenting_history.last() == Some(old_ancestor) {
221 : // do not re-record it
222 0 : false
223 : } else {
224 : #[cfg(feature = "testing")]
225 : {
226 0 : let existing = self
227 0 : .reparenting_history
228 0 : .iter()
229 0 : .position(|x| x == old_ancestor);
230 0 : assert_eq!(
231 : existing, None,
232 0 : "we cannot reparent onto and off and onto the same timeline twice"
233 : );
234 : }
235 0 : let drop_oldest = self.reparenting_history.len() + 1 >= Self::REMEMBER_AT_MOST;
236 0 :
237 0 : self.reparenting_history_truncated |= drop_oldest;
238 0 : if drop_oldest {
239 0 : self.reparenting_history.remove(0);
240 0 : }
241 0 : self.reparenting_history.push(*old_ancestor);
242 0 : true
243 : }
244 0 : }
245 :
246 : /// Returns true if anything changed.
247 0 : pub(crate) fn record_detaching(&mut self, branchpoint: &(TimelineId, Lsn)) -> bool {
248 0 : if let Some((id, lsn, _)) = self.original_ancestor {
249 0 : assert_eq!(
250 0 : &(id, lsn),
251 : branchpoint,
252 0 : "detaching attempt has to be for the same ancestor we are already detached from"
253 : );
254 0 : false
255 : } else {
256 0 : self.original_ancestor =
257 0 : Some((branchpoint.0, branchpoint.1, chrono::Utc::now().naive_utc()));
258 0 : true
259 : }
260 0 : }
261 :
262 : /// The queried lsn is most likely the basebackup lsn, and this answers question "is it allowed
263 : /// to start a read/write primary at this lsn".
264 : ///
265 : /// Returns true if the Lsn was previously our branch point.
266 0 : pub(crate) fn is_previous_ancestor_lsn(&self, lsn: Lsn) -> bool {
267 0 : self.original_ancestor
268 0 : .is_some_and(|(_, ancestor_lsn, _)| ancestor_lsn == lsn)
269 0 : }
270 :
271 : /// Returns true if the timeline originally had an ancestor, and no longer has one.
272 0 : pub(crate) fn is_detached_from_ancestor(&self) -> bool {
273 0 : self.original_ancestor.is_some()
274 0 : }
275 :
276 : /// Returns original ancestor timeline id and lsn that this timeline has been detached from.
277 0 : pub(crate) fn detached_previous_ancestor(&self) -> Option<(TimelineId, Lsn)> {
278 0 : self.original_ancestor.map(|(id, lsn, _)| (id, lsn))
279 0 : }
280 :
281 0 : pub(crate) fn is_reparented(&self) -> bool {
282 0 : !self.reparenting_history.is_empty()
283 0 : }
284 : }
285 :
286 18 : #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
287 : pub(crate) struct GcBlocking {
288 : pub(crate) started_at: NaiveDateTime,
289 : pub(crate) reasons: enumset::EnumSet<GcBlockingReason>,
290 : }
291 :
292 12 : #[derive(Debug, enumset::EnumSetType, serde::Serialize, serde::Deserialize)]
293 : #[enumset(serialize_repr = "list")]
294 : pub(crate) enum GcBlockingReason {
295 : Manual,
296 : DetachAncestor,
297 : }
298 :
299 : impl GcBlocking {
300 0 : pub(super) fn started_now_for(reason: GcBlockingReason) -> Self {
301 0 : GcBlocking {
302 0 : started_at: chrono::Utc::now().naive_utc(),
303 0 : reasons: enumset::EnumSet::only(reason),
304 0 : }
305 0 : }
306 :
307 : /// Returns true if the given reason is one of the reasons why the gc is blocked.
308 0 : pub(crate) fn blocked_by(&self, reason: GcBlockingReason) -> bool {
309 0 : self.reasons.contains(reason)
310 0 : }
311 :
312 : /// Returns a version of self with the given reason.
313 0 : pub(super) fn with_reason(&self, reason: GcBlockingReason) -> Self {
314 0 : assert!(!self.blocked_by(reason));
315 0 : let mut reasons = self.reasons;
316 0 : reasons.insert(reason);
317 0 :
318 0 : Self {
319 0 : started_at: self.started_at,
320 0 : reasons,
321 0 : }
322 0 : }
323 :
324 : /// Returns a version of self without the given reason. Assumption is that if
325 : /// there are no more reasons, we can unblock the gc by returning `None`.
326 0 : pub(super) fn without_reason(&self, reason: GcBlockingReason) -> Option<Self> {
327 0 : assert!(self.blocked_by(reason));
328 :
329 0 : if self.reasons.len() == 1 {
330 0 : None
331 : } else {
332 0 : let mut reasons = self.reasons;
333 0 : assert!(reasons.remove(reason));
334 0 : assert!(!reasons.is_empty());
335 :
336 0 : Some(Self {
337 0 : started_at: self.started_at,
338 0 : reasons,
339 0 : })
340 : }
341 0 : }
342 : }
343 :
344 : #[cfg(test)]
345 : mod tests {
346 : use super::*;
347 : use std::str::FromStr;
348 : use utils::id::TimelineId;
349 :
350 : #[test]
351 6 : fn v1_indexpart_is_parsed() {
352 6 : let example = r#"{
353 6 : "version":1,
354 6 : "timeline_layers":["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"],
355 6 : "layer_metadata":{
356 6 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
357 6 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
358 6 : },
359 6 : "disk_consistent_lsn":"0/16960E8",
360 6 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
361 6 : }"#;
362 6 :
363 6 : let expected = IndexPart {
364 6 : // note this is not verified, could be anything, but exists for humans debugging.. could be the git version instead?
365 6 : version: 1,
366 6 : layer_metadata: HashMap::from([
367 6 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
368 6 : file_size: 25600000,
369 6 : generation: Generation::none(),
370 6 : shard: ShardIndex::unsharded()
371 6 : }),
372 6 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
373 6 : // serde_json should always parse this but this might be a double with jq for
374 6 : // example.
375 6 : file_size: 9007199254741001,
376 6 : generation: Generation::none(),
377 6 : shard: ShardIndex::unsharded()
378 6 : })
379 6 : ]),
380 6 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
381 6 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
382 6 : deleted_at: None,
383 6 : archived_at: None,
384 6 : lineage: Lineage::default(),
385 6 : gc_blocking: None,
386 6 : last_aux_file_policy: None,
387 6 : };
388 6 :
389 6 : let part = IndexPart::from_s3_bytes(example.as_bytes()).unwrap();
390 6 : assert_eq!(part, expected);
391 6 : }
392 :
393 : #[test]
394 6 : fn v1_indexpart_is_parsed_with_optional_missing_layers() {
395 6 : let example = r#"{
396 6 : "version":1,
397 6 : "timeline_layers":["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"],
398 6 : "missing_layers":["This shouldn't fail deserialization"],
399 6 : "layer_metadata":{
400 6 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
401 6 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
402 6 : },
403 6 : "disk_consistent_lsn":"0/16960E8",
404 6 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
405 6 : }"#;
406 6 :
407 6 : let expected = IndexPart {
408 6 : // note this is not verified, could be anything, but exists for humans debugging.. could be the git version instead?
409 6 : version: 1,
410 6 : layer_metadata: HashMap::from([
411 6 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
412 6 : file_size: 25600000,
413 6 : generation: Generation::none(),
414 6 : shard: ShardIndex::unsharded()
415 6 : }),
416 6 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
417 6 : // serde_json should always parse this but this might be a double with jq for
418 6 : // example.
419 6 : file_size: 9007199254741001,
420 6 : generation: Generation::none(),
421 6 : shard: ShardIndex::unsharded()
422 6 : })
423 6 : ]),
424 6 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
425 6 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
426 6 : deleted_at: None,
427 6 : archived_at: None,
428 6 : lineage: Lineage::default(),
429 6 : gc_blocking: None,
430 6 : last_aux_file_policy: None,
431 6 : };
432 6 :
433 6 : let part = IndexPart::from_s3_bytes(example.as_bytes()).unwrap();
434 6 : assert_eq!(part, expected);
435 6 : }
436 :
437 : #[test]
438 6 : fn v2_indexpart_is_parsed_with_deleted_at() {
439 6 : let example = r#"{
440 6 : "version":2,
441 6 : "timeline_layers":["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"],
442 6 : "missing_layers":["This shouldn't fail deserialization"],
443 6 : "layer_metadata":{
444 6 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
445 6 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
446 6 : },
447 6 : "disk_consistent_lsn":"0/16960E8",
448 6 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
449 6 : "deleted_at": "2023-07-31T09:00:00.123"
450 6 : }"#;
451 6 :
452 6 : let expected = IndexPart {
453 6 : // note this is not verified, could be anything, but exists for humans debugging.. could be the git version instead?
454 6 : version: 2,
455 6 : layer_metadata: HashMap::from([
456 6 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
457 6 : file_size: 25600000,
458 6 : generation: Generation::none(),
459 6 : shard: ShardIndex::unsharded()
460 6 : }),
461 6 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
462 6 : // serde_json should always parse this but this might be a double with jq for
463 6 : // example.
464 6 : file_size: 9007199254741001,
465 6 : generation: Generation::none(),
466 6 : shard: ShardIndex::unsharded()
467 6 : })
468 6 : ]),
469 6 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
470 6 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
471 6 : deleted_at: Some(parse_naive_datetime("2023-07-31T09:00:00.123000000")),
472 6 : archived_at: None,
473 6 : lineage: Lineage::default(),
474 6 : gc_blocking: None,
475 6 : last_aux_file_policy: None,
476 6 : };
477 6 :
478 6 : let part = IndexPart::from_s3_bytes(example.as_bytes()).unwrap();
479 6 : assert_eq!(part, expected);
480 6 : }
481 :
482 : #[test]
483 6 : fn empty_layers_are_parsed() {
484 6 : let empty_layers_json = r#"{
485 6 : "version":1,
486 6 : "timeline_layers":[],
487 6 : "layer_metadata":{},
488 6 : "disk_consistent_lsn":"0/2532648",
489 6 : "metadata_bytes":[136,151,49,208,0,70,0,4,0,0,0,0,2,83,38,72,1,0,0,0,0,2,83,38,32,1,87,198,240,135,97,119,45,125,38,29,155,161,140,141,255,210,0,0,0,0,2,83,38,72,0,0,0,0,1,73,240,192,0,0,0,0,1,73,240,192,0,0,0,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
490 6 : }"#;
491 6 :
492 6 : let expected = IndexPart {
493 6 : version: 1,
494 6 : layer_metadata: HashMap::new(),
495 6 : disk_consistent_lsn: "0/2532648".parse::<Lsn>().unwrap(),
496 6 : metadata: TimelineMetadata::from_bytes(&[
497 6 : 136, 151, 49, 208, 0, 70, 0, 4, 0, 0, 0, 0, 2, 83, 38, 72, 1, 0, 0, 0, 0, 2, 83,
498 6 : 38, 32, 1, 87, 198, 240, 135, 97, 119, 45, 125, 38, 29, 155, 161, 140, 141, 255,
499 6 : 210, 0, 0, 0, 0, 2, 83, 38, 72, 0, 0, 0, 0, 1, 73, 240, 192, 0, 0, 0, 0, 1, 73,
500 6 : 240, 192, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
501 6 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
502 6 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
503 6 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
504 6 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
505 6 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
506 6 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
507 6 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
508 6 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
509 6 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
510 6 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
511 6 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
512 6 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
513 6 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
514 6 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
515 6 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
516 6 : 0, 0,
517 6 : ])
518 6 : .unwrap(),
519 6 : deleted_at: None,
520 6 : archived_at: None,
521 6 : lineage: Lineage::default(),
522 6 : gc_blocking: None,
523 6 : last_aux_file_policy: None,
524 6 : };
525 6 :
526 6 : let empty_layers_parsed = IndexPart::from_s3_bytes(empty_layers_json.as_bytes()).unwrap();
527 6 :
528 6 : assert_eq!(empty_layers_parsed, expected);
529 6 : }
530 :
531 : #[test]
532 6 : fn v4_indexpart_is_parsed() {
533 6 : let example = r#"{
534 6 : "version":4,
535 6 : "layer_metadata":{
536 6 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
537 6 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
538 6 : },
539 6 : "disk_consistent_lsn":"0/16960E8",
540 6 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
541 6 : "deleted_at": "2023-07-31T09:00:00.123"
542 6 : }"#;
543 6 :
544 6 : let expected = IndexPart {
545 6 : version: 4,
546 6 : layer_metadata: HashMap::from([
547 6 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
548 6 : file_size: 25600000,
549 6 : generation: Generation::none(),
550 6 : shard: ShardIndex::unsharded()
551 6 : }),
552 6 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
553 6 : // serde_json should always parse this but this might be a double with jq for
554 6 : // example.
555 6 : file_size: 9007199254741001,
556 6 : generation: Generation::none(),
557 6 : shard: ShardIndex::unsharded()
558 6 : })
559 6 : ]),
560 6 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
561 6 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
562 6 : deleted_at: Some(parse_naive_datetime("2023-07-31T09:00:00.123000000")),
563 6 : archived_at: None,
564 6 : lineage: Lineage::default(),
565 6 : gc_blocking: None,
566 6 : last_aux_file_policy: None,
567 6 : };
568 6 :
569 6 : let part = IndexPart::from_s3_bytes(example.as_bytes()).unwrap();
570 6 : assert_eq!(part, expected);
571 6 : }
572 :
573 : #[test]
574 6 : fn v5_indexpart_is_parsed() {
575 6 : let example = r#"{
576 6 : "version":5,
577 6 : "layer_metadata":{
578 6 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000014EF420-00000000014EF499":{"file_size":23289856,"generation":1},
579 6 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000014EF499-00000000015A7619":{"file_size":1015808,"generation":1}},
580 6 : "disk_consistent_lsn":"0/15A7618",
581 6 : "metadata_bytes":[226,88,25,241,0,46,0,4,0,0,0,0,1,90,118,24,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,78,244,32,0,0,0,0,1,78,244,32,0,0,0,16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
582 6 : "lineage":{
583 6 : "original_ancestor":["e2bfd8c633d713d279e6fcd2bcc15b6d","0/15A7618","2024-05-07T18:52:36.322426563"],
584 6 : "reparenting_history":["e1bfd8c633d713d279e6fcd2bcc15b6d"]
585 6 : }
586 6 : }"#;
587 6 :
588 6 : let expected = IndexPart {
589 6 : version: 5,
590 6 : layer_metadata: HashMap::from([
591 6 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000014EF420-00000000014EF499".parse().unwrap(), LayerFileMetadata {
592 6 : file_size: 23289856,
593 6 : generation: Generation::new(1),
594 6 : shard: ShardIndex::unsharded(),
595 6 : }),
596 6 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000014EF499-00000000015A7619".parse().unwrap(), LayerFileMetadata {
597 6 : file_size: 1015808,
598 6 : generation: Generation::new(1),
599 6 : shard: ShardIndex::unsharded(),
600 6 : })
601 6 : ]),
602 6 : disk_consistent_lsn: Lsn::from_str("0/15A7618").unwrap(),
603 6 : metadata: TimelineMetadata::from_bytes(&[226,88,25,241,0,46,0,4,0,0,0,0,1,90,118,24,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,78,244,32,0,0,0,0,1,78,244,32,0,0,0,16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
604 6 : deleted_at: None,
605 6 : archived_at: None,
606 6 : lineage: Lineage {
607 6 : reparenting_history_truncated: false,
608 6 : reparenting_history: vec![TimelineId::from_str("e1bfd8c633d713d279e6fcd2bcc15b6d").unwrap()],
609 6 : original_ancestor: Some((TimelineId::from_str("e2bfd8c633d713d279e6fcd2bcc15b6d").unwrap(), Lsn::from_str("0/15A7618").unwrap(), parse_naive_datetime("2024-05-07T18:52:36.322426563"))),
610 6 : },
611 6 : gc_blocking: None,
612 6 : last_aux_file_policy: None,
613 6 : };
614 6 :
615 6 : let part = IndexPart::from_s3_bytes(example.as_bytes()).unwrap();
616 6 : assert_eq!(part, expected);
617 6 : }
618 :
619 : #[test]
620 6 : fn v6_indexpart_is_parsed() {
621 6 : let example = r#"{
622 6 : "version":6,
623 6 : "layer_metadata":{
624 6 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
625 6 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
626 6 : },
627 6 : "disk_consistent_lsn":"0/16960E8",
628 6 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
629 6 : "deleted_at": "2023-07-31T09:00:00.123",
630 6 : "lineage":{
631 6 : "original_ancestor":["e2bfd8c633d713d279e6fcd2bcc15b6d","0/15A7618","2024-05-07T18:52:36.322426563"],
632 6 : "reparenting_history":["e1bfd8c633d713d279e6fcd2bcc15b6d"]
633 6 : },
634 6 : "last_aux_file_policy": "V2"
635 6 : }"#;
636 6 :
637 6 : let expected = IndexPart {
638 6 : version: 6,
639 6 : layer_metadata: HashMap::from([
640 6 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
641 6 : file_size: 25600000,
642 6 : generation: Generation::none(),
643 6 : shard: ShardIndex::unsharded()
644 6 : }),
645 6 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
646 6 : // serde_json should always parse this but this might be a double with jq for
647 6 : // example.
648 6 : file_size: 9007199254741001,
649 6 : generation: Generation::none(),
650 6 : shard: ShardIndex::unsharded()
651 6 : })
652 6 : ]),
653 6 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
654 6 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
655 6 : deleted_at: Some(parse_naive_datetime("2023-07-31T09:00:00.123000000")),
656 6 : archived_at: None,
657 6 : lineage: Lineage {
658 6 : reparenting_history_truncated: false,
659 6 : reparenting_history: vec![TimelineId::from_str("e1bfd8c633d713d279e6fcd2bcc15b6d").unwrap()],
660 6 : original_ancestor: Some((TimelineId::from_str("e2bfd8c633d713d279e6fcd2bcc15b6d").unwrap(), Lsn::from_str("0/15A7618").unwrap(), parse_naive_datetime("2024-05-07T18:52:36.322426563"))),
661 6 : },
662 6 : gc_blocking: None,
663 6 : last_aux_file_policy: Some(AuxFilePolicy::V2),
664 6 : };
665 6 :
666 6 : let part = IndexPart::from_s3_bytes(example.as_bytes()).unwrap();
667 6 : assert_eq!(part, expected);
668 6 : }
669 :
670 : #[test]
671 6 : fn v7_indexpart_is_parsed() {
672 6 : let example = r#"{
673 6 : "version": 7,
674 6 : "layer_metadata":{
675 6 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
676 6 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
677 6 : },
678 6 : "disk_consistent_lsn":"0/16960E8",
679 6 : "metadata": {
680 6 : "disk_consistent_lsn": "0/16960E8",
681 6 : "prev_record_lsn": "0/1696070",
682 6 : "ancestor_timeline": "e45a7f37d3ee2ff17dc14bf4f4e3f52e",
683 6 : "ancestor_lsn": "0/0",
684 6 : "latest_gc_cutoff_lsn": "0/1696070",
685 6 : "initdb_lsn": "0/1696070",
686 6 : "pg_version": 14
687 6 : },
688 6 : "deleted_at": "2023-07-31T09:00:00.123"
689 6 : }"#;
690 6 :
691 6 : let expected = IndexPart {
692 6 : version: 7,
693 6 : layer_metadata: HashMap::from([
694 6 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
695 6 : file_size: 25600000,
696 6 : generation: Generation::none(),
697 6 : shard: ShardIndex::unsharded()
698 6 : }),
699 6 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
700 6 : file_size: 9007199254741001,
701 6 : generation: Generation::none(),
702 6 : shard: ShardIndex::unsharded()
703 6 : })
704 6 : ]),
705 6 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
706 6 : metadata: TimelineMetadata::new(
707 6 : Lsn::from_str("0/16960E8").unwrap(),
708 6 : Some(Lsn::from_str("0/1696070").unwrap()),
709 6 : Some(TimelineId::from_str("e45a7f37d3ee2ff17dc14bf4f4e3f52e").unwrap()),
710 6 : Lsn::INVALID,
711 6 : Lsn::from_str("0/1696070").unwrap(),
712 6 : Lsn::from_str("0/1696070").unwrap(),
713 6 : 14,
714 6 : ).with_recalculated_checksum().unwrap(),
715 6 : deleted_at: Some(parse_naive_datetime("2023-07-31T09:00:00.123000000")),
716 6 : archived_at: None,
717 6 : lineage: Default::default(),
718 6 : gc_blocking: None,
719 6 : last_aux_file_policy: Default::default(),
720 6 : };
721 6 :
722 6 : let part = IndexPart::from_s3_bytes(example.as_bytes()).unwrap();
723 6 : assert_eq!(part, expected);
724 6 : }
725 :
726 : #[test]
727 6 : fn v8_indexpart_is_parsed() {
728 6 : let example = r#"{
729 6 : "version": 8,
730 6 : "layer_metadata":{
731 6 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
732 6 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
733 6 : },
734 6 : "disk_consistent_lsn":"0/16960E8",
735 6 : "metadata": {
736 6 : "disk_consistent_lsn": "0/16960E8",
737 6 : "prev_record_lsn": "0/1696070",
738 6 : "ancestor_timeline": "e45a7f37d3ee2ff17dc14bf4f4e3f52e",
739 6 : "ancestor_lsn": "0/0",
740 6 : "latest_gc_cutoff_lsn": "0/1696070",
741 6 : "initdb_lsn": "0/1696070",
742 6 : "pg_version": 14
743 6 : },
744 6 : "deleted_at": "2023-07-31T09:00:00.123",
745 6 : "archived_at": "2023-04-29T09:00:00.123"
746 6 : }"#;
747 6 :
748 6 : let expected = IndexPart {
749 6 : version: 8,
750 6 : layer_metadata: HashMap::from([
751 6 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
752 6 : file_size: 25600000,
753 6 : generation: Generation::none(),
754 6 : shard: ShardIndex::unsharded()
755 6 : }),
756 6 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
757 6 : file_size: 9007199254741001,
758 6 : generation: Generation::none(),
759 6 : shard: ShardIndex::unsharded()
760 6 : })
761 6 : ]),
762 6 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
763 6 : metadata: TimelineMetadata::new(
764 6 : Lsn::from_str("0/16960E8").unwrap(),
765 6 : Some(Lsn::from_str("0/1696070").unwrap()),
766 6 : Some(TimelineId::from_str("e45a7f37d3ee2ff17dc14bf4f4e3f52e").unwrap()),
767 6 : Lsn::INVALID,
768 6 : Lsn::from_str("0/1696070").unwrap(),
769 6 : Lsn::from_str("0/1696070").unwrap(),
770 6 : 14,
771 6 : ).with_recalculated_checksum().unwrap(),
772 6 : deleted_at: Some(parse_naive_datetime("2023-07-31T09:00:00.123000000")),
773 6 : archived_at: Some(parse_naive_datetime("2023-04-29T09:00:00.123000000")),
774 6 : lineage: Default::default(),
775 6 : gc_blocking: None,
776 6 : last_aux_file_policy: Default::default(),
777 6 : };
778 6 :
779 6 : let part = IndexPart::from_s3_bytes(example.as_bytes()).unwrap();
780 6 : assert_eq!(part, expected);
781 6 : }
782 :
783 : #[test]
784 6 : fn v9_indexpart_is_parsed() {
785 6 : let example = r#"{
786 6 : "version": 9,
787 6 : "layer_metadata":{
788 6 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
789 6 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
790 6 : },
791 6 : "disk_consistent_lsn":"0/16960E8",
792 6 : "metadata": {
793 6 : "disk_consistent_lsn": "0/16960E8",
794 6 : "prev_record_lsn": "0/1696070",
795 6 : "ancestor_timeline": "e45a7f37d3ee2ff17dc14bf4f4e3f52e",
796 6 : "ancestor_lsn": "0/0",
797 6 : "latest_gc_cutoff_lsn": "0/1696070",
798 6 : "initdb_lsn": "0/1696070",
799 6 : "pg_version": 14
800 6 : },
801 6 : "gc_blocking": {
802 6 : "started_at": "2024-07-19T09:00:00.123",
803 6 : "reasons": ["DetachAncestor"]
804 6 : }
805 6 : }"#;
806 6 :
807 6 : let expected = IndexPart {
808 6 : version: 9,
809 6 : layer_metadata: HashMap::from([
810 6 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
811 6 : file_size: 25600000,
812 6 : generation: Generation::none(),
813 6 : shard: ShardIndex::unsharded()
814 6 : }),
815 6 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
816 6 : file_size: 9007199254741001,
817 6 : generation: Generation::none(),
818 6 : shard: ShardIndex::unsharded()
819 6 : })
820 6 : ]),
821 6 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
822 6 : metadata: TimelineMetadata::new(
823 6 : Lsn::from_str("0/16960E8").unwrap(),
824 6 : Some(Lsn::from_str("0/1696070").unwrap()),
825 6 : Some(TimelineId::from_str("e45a7f37d3ee2ff17dc14bf4f4e3f52e").unwrap()),
826 6 : Lsn::INVALID,
827 6 : Lsn::from_str("0/1696070").unwrap(),
828 6 : Lsn::from_str("0/1696070").unwrap(),
829 6 : 14,
830 6 : ).with_recalculated_checksum().unwrap(),
831 6 : deleted_at: None,
832 6 : lineage: Default::default(),
833 6 : gc_blocking: Some(GcBlocking {
834 6 : started_at: parse_naive_datetime("2024-07-19T09:00:00.123000000"),
835 6 : reasons: enumset::EnumSet::from_iter([GcBlockingReason::DetachAncestor]),
836 6 : }),
837 6 : last_aux_file_policy: Default::default(),
838 6 : archived_at: None,
839 6 : };
840 6 :
841 6 : let part = IndexPart::from_s3_bytes(example.as_bytes()).unwrap();
842 6 : assert_eq!(part, expected);
843 6 : }
844 :
845 54 : fn parse_naive_datetime(s: &str) -> NaiveDateTime {
846 54 : chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S.%f").unwrap()
847 54 : }
848 : }
|
