Line data Source code
1 : //! In-memory index to track the tenant files on the remote storage.
2 : //! Able to restore itself from the storage index parts, that are located in every timeline's remote directory and contain all data about
3 : //! remote timeline layers and its metadata.
4 :
5 : use std::collections::HashMap;
6 :
7 : use chrono::NaiveDateTime;
8 : use serde::{Deserialize, Serialize};
9 : use utils::bin_ser::SerializeError;
10 :
11 : use crate::tenant::metadata::TimelineMetadata;
12 : use crate::tenant::storage_layer::LayerFileName;
13 : use crate::tenant::upload_queue::UploadQueueInitialized;
14 : use crate::tenant::Generation;
15 : use pageserver_api::shard::ShardIndex;
16 :
17 : use utils::lsn::Lsn;
18 :
19 : /// Metadata gathered for each of the layer files.
20 : ///
21 : /// Fields have to be `Option`s because remote [`IndexPart`]'s can be from different version, which
22 : /// might have less or more metadata depending if upgrading or rolling back an upgrade.
23 703686 : #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
24 : //#[cfg_attr(test, derive(Default))]
25 : pub struct LayerFileMetadata {
26 : file_size: u64,
27 :
28 : pub(crate) generation: Generation,
29 :
30 : pub(crate) shard: ShardIndex,
31 : }
32 :
33 : impl From<&'_ IndexLayerMetadata> for LayerFileMetadata {
34 117832 : fn from(other: &IndexLayerMetadata) -> Self {
35 117832 : LayerFileMetadata {
36 117832 : file_size: other.file_size,
37 117832 : generation: other.generation,
38 117832 : shard: other.shard,
39 117832 : }
40 117832 : }
41 : }
42 :
43 : impl LayerFileMetadata {
44 50546 : pub fn new(file_size: u64, generation: Generation, shard: ShardIndex) -> Self {
45 50546 : LayerFileMetadata {
46 50546 : file_size,
47 50546 : generation,
48 50546 : shard,
49 50546 : }
50 50546 : }
51 :
52 185666 : pub fn file_size(&self) -> u64 {
53 185666 : self.file_size
54 185666 : }
55 : }
56 :
57 : // TODO seems like another part of the remote storage file format
58 : // compatibility issue, see https://github.com/neondatabase/neon/issues/3072
59 : /// In-memory representation of an `index_part.json` file
60 : ///
61 : /// Contains the data about all files in the timeline, present remotely and its metadata.
62 : ///
63 : /// This type needs to be backwards and forwards compatible. When changing the fields,
64 : /// remember to add a test case for the changed version.
65 6960 : #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
66 : pub struct IndexPart {
67 : /// Debugging aid describing the version of this type.
68 : #[serde(default)]
69 : version: usize,
70 :
71 : #[serde(default)]
72 : #[serde(skip_serializing_if = "Option::is_none")]
73 : pub deleted_at: Option<NaiveDateTime>,
74 :
75 : /// Per layer file name metadata, which can be present for a present or missing layer file.
76 : ///
77 : /// Older versions of `IndexPart` will not have this property or have only a part of metadata
78 : /// that latest version stores.
79 : pub layer_metadata: HashMap<LayerFileName, IndexLayerMetadata>,
80 :
81 : // 'disk_consistent_lsn' is a copy of the 'disk_consistent_lsn' in the metadata.
82 : // It's duplicated for convenience when reading the serialized structure, but is
83 : // private because internally we would read from metadata instead.
84 : disk_consistent_lsn: Lsn,
85 :
86 : #[serde(rename = "metadata_bytes")]
87 : pub metadata: TimelineMetadata,
88 : }
89 :
90 : impl IndexPart {
91 : /// When adding or modifying any parts of `IndexPart`, increment the version so that it can be
92 : /// used to understand later versions.
93 : ///
94 : /// Version is currently informative only.
95 : /// Version history
96 : /// - 2: added `deleted_at`
97 : /// - 3: no longer deserialize `timeline_layers` (serialized format is the same, but timeline_layers
98 : /// is always generated from the keys of `layer_metadata`)
99 : /// - 4: timeline_layers is fully removed.
100 : const LATEST_VERSION: usize = 4;
101 :
102 : // Versions we may see when reading from a bucket.
103 : pub const KNOWN_VERSIONS: &'static [usize] = &[1, 2, 3, 4];
104 :
105 : pub const FILE_NAME: &'static str = "index_part.json";
106 :
107 6351 : pub fn new(
108 6351 : layers_and_metadata: HashMap<LayerFileName, LayerFileMetadata>,
109 6351 : disk_consistent_lsn: Lsn,
110 6351 : metadata: TimelineMetadata,
111 6351 : ) -> Self {
112 6351 : // Transform LayerFileMetadata into IndexLayerMetadata
113 6351 : let layer_metadata = layers_and_metadata
114 6351 : .into_iter()
115 613739 : .map(|(k, v)| (k, IndexLayerMetadata::from(v)))
116 6351 : .collect();
117 6351 :
118 6351 : Self {
119 6351 : version: Self::LATEST_VERSION,
120 6351 : layer_metadata,
121 6351 : disk_consistent_lsn,
122 6351 : metadata,
123 6351 : deleted_at: None,
124 6351 : }
125 6351 : }
126 :
127 0 : pub fn get_version(&self) -> usize {
128 0 : self.version
129 0 : }
130 :
131 : /// If you want this under normal operations, read it from self.metadata:
132 : /// this method is just for the scrubber to use when validating an index.
133 6 : pub fn get_disk_consistent_lsn(&self) -> Lsn {
134 6 : self.disk_consistent_lsn
135 6 : }
136 :
137 16 : pub fn from_s3_bytes(bytes: &[u8]) -> Result<Self, serde_json::Error> {
138 16 : serde_json::from_slice::<IndexPart>(bytes)
139 16 : }
140 :
141 6948 : pub fn to_s3_bytes(&self) -> serde_json::Result<Vec<u8>> {
142 6948 : serde_json::to_vec(self)
143 6948 : }
144 : }
145 :
146 : impl TryFrom<&UploadQueueInitialized> for IndexPart {
147 : type Error = SerializeError;
148 :
149 177 : fn try_from(upload_queue: &UploadQueueInitialized) -> Result<Self, Self::Error> {
150 177 : let disk_consistent_lsn = upload_queue.latest_metadata.disk_consistent_lsn();
151 177 : let metadata = upload_queue.latest_metadata.clone();
152 177 :
153 177 : Ok(Self::new(
154 177 : upload_queue.latest_files.clone(),
155 177 : disk_consistent_lsn,
156 177 : metadata,
157 177 : ))
158 177 : }
159 : }
160 :
161 : /// Serialized form of [`LayerFileMetadata`].
162 619973 : #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
163 : pub struct IndexLayerMetadata {
164 : pub file_size: u64,
165 :
166 : #[serde(default = "Generation::none")]
167 : #[serde(skip_serializing_if = "Generation::is_none")]
168 : pub generation: Generation,
169 :
170 : #[serde(default = "ShardIndex::unsharded")]
171 : #[serde(skip_serializing_if = "ShardIndex::is_unsharded")]
172 : pub shard: ShardIndex,
173 : }
174 :
175 : impl From<LayerFileMetadata> for IndexLayerMetadata {
176 616232 : fn from(other: LayerFileMetadata) -> Self {
177 616232 : IndexLayerMetadata {
178 616232 : file_size: other.file_size,
179 616232 : generation: other.generation,
180 616232 : shard: other.shard,
181 616232 : }
182 616232 : }
183 : }
184 :
185 : #[cfg(test)]
186 : mod tests {
187 : use super::*;
188 :
189 2 : #[test]
190 2 : fn v1_indexpart_is_parsed() {
191 2 : let example = r#"{
192 2 : "version":1,
193 2 : "timeline_layers":["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"],
194 2 : "layer_metadata":{
195 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
196 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
197 2 : },
198 2 : "disk_consistent_lsn":"0/16960E8",
199 2 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
200 2 : }"#;
201 2 :
202 2 : let expected = IndexPart {
203 2 : // note this is not verified, could be anything, but exists for humans debugging.. could be the git version instead?
204 2 : version: 1,
205 2 : layer_metadata: HashMap::from([
206 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), IndexLayerMetadata {
207 2 : file_size: 25600000,
208 2 : generation: Generation::none(),
209 2 : shard: ShardIndex::unsharded()
210 2 : }),
211 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), IndexLayerMetadata {
212 2 : // serde_json should always parse this but this might be a double with jq for
213 2 : // example.
214 2 : file_size: 9007199254741001,
215 2 : generation: Generation::none(),
216 2 : shard: ShardIndex::unsharded()
217 2 : })
218 2 : ]),
219 2 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
220 2 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
221 2 : deleted_at: None,
222 2 : };
223 2 :
224 2 : let part = IndexPart::from_s3_bytes(example.as_bytes()).unwrap();
225 2 : assert_eq!(part, expected);
226 2 : }
227 :
228 2 : #[test]
229 2 : fn v1_indexpart_is_parsed_with_optional_missing_layers() {
230 2 : let example = r#"{
231 2 : "version":1,
232 2 : "timeline_layers":["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"],
233 2 : "missing_layers":["This shouldn't fail deserialization"],
234 2 : "layer_metadata":{
235 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
236 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
237 2 : },
238 2 : "disk_consistent_lsn":"0/16960E8",
239 2 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
240 2 : }"#;
241 2 :
242 2 : let expected = IndexPart {
243 2 : // note this is not verified, could be anything, but exists for humans debugging.. could be the git version instead?
244 2 : version: 1,
245 2 : layer_metadata: HashMap::from([
246 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), IndexLayerMetadata {
247 2 : file_size: 25600000,
248 2 : generation: Generation::none(),
249 2 : shard: ShardIndex::unsharded()
250 2 : }),
251 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), IndexLayerMetadata {
252 2 : // serde_json should always parse this but this might be a double with jq for
253 2 : // example.
254 2 : file_size: 9007199254741001,
255 2 : generation: Generation::none(),
256 2 : shard: ShardIndex::unsharded()
257 2 : })
258 2 : ]),
259 2 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
260 2 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
261 2 : deleted_at: None,
262 2 : };
263 2 :
264 2 : let part = IndexPart::from_s3_bytes(example.as_bytes()).unwrap();
265 2 : assert_eq!(part, expected);
266 2 : }
267 :
268 2 : #[test]
269 2 : fn v2_indexpart_is_parsed_with_deleted_at() {
270 2 : let example = r#"{
271 2 : "version":2,
272 2 : "timeline_layers":["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"],
273 2 : "missing_layers":["This shouldn't fail deserialization"],
274 2 : "layer_metadata":{
275 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
276 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
277 2 : },
278 2 : "disk_consistent_lsn":"0/16960E8",
279 2 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
280 2 : "deleted_at": "2023-07-31T09:00:00.123"
281 2 : }"#;
282 2 :
283 2 : let expected = IndexPart {
284 2 : // note this is not verified, could be anything, but exists for humans debugging.. could be the git version instead?
285 2 : version: 2,
286 2 : layer_metadata: HashMap::from([
287 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), IndexLayerMetadata {
288 2 : file_size: 25600000,
289 2 : generation: Generation::none(),
290 2 : shard: ShardIndex::unsharded()
291 2 : }),
292 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), IndexLayerMetadata {
293 2 : // serde_json should always parse this but this might be a double with jq for
294 2 : // example.
295 2 : file_size: 9007199254741001,
296 2 : generation: Generation::none(),
297 2 : shard: ShardIndex::unsharded()
298 2 : })
299 2 : ]),
300 2 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
301 2 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
302 2 : deleted_at: Some(chrono::NaiveDateTime::parse_from_str(
303 2 : "2023-07-31T09:00:00.123000000", "%Y-%m-%dT%H:%M:%S.%f").unwrap())
304 2 : };
305 2 :
306 2 : let part = IndexPart::from_s3_bytes(example.as_bytes()).unwrap();
307 2 : assert_eq!(part, expected);
308 2 : }
309 :
310 2 : #[test]
311 2 : fn empty_layers_are_parsed() {
312 2 : let empty_layers_json = r#"{
313 2 : "version":1,
314 2 : "timeline_layers":[],
315 2 : "layer_metadata":{},
316 2 : "disk_consistent_lsn":"0/2532648",
317 2 : "metadata_bytes":[136,151,49,208,0,70,0,4,0,0,0,0,2,83,38,72,1,0,0,0,0,2,83,38,32,1,87,198,240,135,97,119,45,125,38,29,155,161,140,141,255,210,0,0,0,0,2,83,38,72,0,0,0,0,1,73,240,192,0,0,0,0,1,73,240,192,0,0,0,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
318 2 : }"#;
319 2 :
320 2 : let expected = IndexPart {
321 2 : version: 1,
322 2 : layer_metadata: HashMap::new(),
323 2 : disk_consistent_lsn: "0/2532648".parse::<Lsn>().unwrap(),
324 2 : metadata: TimelineMetadata::from_bytes(&[
325 2 : 136, 151, 49, 208, 0, 70, 0, 4, 0, 0, 0, 0, 2, 83, 38, 72, 1, 0, 0, 0, 0, 2, 83,
326 2 : 38, 32, 1, 87, 198, 240, 135, 97, 119, 45, 125, 38, 29, 155, 161, 140, 141, 255,
327 2 : 210, 0, 0, 0, 0, 2, 83, 38, 72, 0, 0, 0, 0, 1, 73, 240, 192, 0, 0, 0, 0, 1, 73,
328 2 : 240, 192, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
329 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
330 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
331 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
332 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
333 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
334 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
335 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
336 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
337 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
338 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
339 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
340 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
341 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
342 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
343 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
344 2 : 0, 0,
345 2 : ])
346 2 : .unwrap(),
347 2 : deleted_at: None,
348 2 : };
349 2 :
350 2 : let empty_layers_parsed = IndexPart::from_s3_bytes(empty_layers_json.as_bytes()).unwrap();
351 2 :
352 2 : assert_eq!(empty_layers_parsed, expected);
353 2 : }
354 :
355 2 : #[test]
356 2 : fn v4_indexpart_is_parsed() {
357 2 : let example = r#"{
358 2 : "version":4,
359 2 : "layer_metadata":{
360 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
361 2 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
362 2 : },
363 2 : "disk_consistent_lsn":"0/16960E8",
364 2 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
365 2 : "deleted_at": "2023-07-31T09:00:00.123"
366 2 : }"#;
367 2 :
368 2 : let expected = IndexPart {
369 2 : version: 4,
370 2 : layer_metadata: HashMap::from([
371 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), IndexLayerMetadata {
372 2 : file_size: 25600000,
373 2 : generation: Generation::none(),
374 2 : shard: ShardIndex::unsharded()
375 2 : }),
376 2 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), IndexLayerMetadata {
377 2 : // serde_json should always parse this but this might be a double with jq for
378 2 : // example.
379 2 : file_size: 9007199254741001,
380 2 : generation: Generation::none(),
381 2 : shard: ShardIndex::unsharded()
382 2 : })
383 2 : ]),
384 2 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
385 2 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
386 2 : deleted_at: Some(chrono::NaiveDateTime::parse_from_str(
387 2 : "2023-07-31T09:00:00.123000000", "%Y-%m-%dT%H:%M:%S.%f").unwrap()),
388 2 : };
389 2 :
390 2 : let part = IndexPart::from_s3_bytes(example.as_bytes()).unwrap();
391 2 : assert_eq!(part, expected);
392 2 : }
393 : }
|
