TLA Line data Source code
1 : //! In-memory index to track the tenant files on the remote storage.
2 : //! Able to restore itself from the storage index parts, that are located in every timeline's remote directory and contain all data about
3 : //! remote timeline layers and its metadata.
4 :
5 : use std::collections::HashMap;
6 :
7 : use chrono::NaiveDateTime;
8 : use serde::{Deserialize, Serialize};
9 : use utils::bin_ser::SerializeError;
10 :
11 : use crate::tenant::metadata::TimelineMetadata;
12 : use crate::tenant::storage_layer::LayerFileName;
13 : use crate::tenant::upload_queue::UploadQueueInitialized;
14 : use crate::tenant::Generation;
15 : use pageserver_api::shard::ShardIndex;
16 :
17 : use utils::lsn::Lsn;
18 :
19 : /// Metadata gathered for each of the layer files.
20 : ///
21 : /// Fields have to be `Option`s because remote [`IndexPart`]'s can be from different version, which
22 : /// might have less or more metadata depending if upgrading or rolling back an upgrade.
23 CBC 569516 : #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
24 : //#[cfg_attr(test, derive(Default))]
25 : pub struct LayerFileMetadata {
26 : file_size: u64,
27 :
28 : pub(crate) generation: Generation,
29 :
30 : pub(crate) shard: ShardIndex,
31 : }
32 :
33 : impl From<&'_ IndexLayerMetadata> for LayerFileMetadata {
34 120710 : fn from(other: &IndexLayerMetadata) -> Self {
35 120710 : LayerFileMetadata {
36 120710 : file_size: other.file_size,
37 120710 : generation: other.generation,
38 120710 : shard: other.shard,
39 120710 : }
40 120710 : }
41 : }
42 :
43 : impl LayerFileMetadata {
44 48903 : pub fn new(file_size: u64, generation: Generation, shard: ShardIndex) -> Self {
45 48903 : LayerFileMetadata {
46 48903 : file_size,
47 48903 : generation,
48 48903 : shard,
49 48903 : }
50 48903 : }
51 :
52 186542 : pub fn file_size(&self) -> u64 {
53 186542 : self.file_size
54 186542 : }
55 : }
56 :
57 : // TODO seems like another part of the remote storage file format
58 : // compatibility issue, see https://github.com/neondatabase/neon/issues/3072
59 : /// In-memory representation of an `index_part.json` file
60 : ///
61 : /// Contains the data about all files in the timeline, present remotely and its metadata.
62 : ///
63 : /// This type needs to be backwards and forwards compatible. When changing the fields,
64 : /// remember to add a test case for the changed version.
65 5958 : #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
66 : pub struct IndexPart {
67 : /// Debugging aid describing the version of this type.
68 : #[serde(default)]
69 : version: usize,
70 :
71 : #[serde(default)]
72 : #[serde(skip_serializing_if = "Option::is_none")]
73 : pub deleted_at: Option<NaiveDateTime>,
74 :
75 : /// Per layer file name metadata, which can be present for a present or missing layer file.
76 : ///
77 : /// Older versions of `IndexPart` will not have this property or have only a part of metadata
78 : /// that latest version stores.
79 : pub layer_metadata: HashMap<LayerFileName, IndexLayerMetadata>,
80 :
81 : // 'disk_consistent_lsn' is a copy of the 'disk_consistent_lsn' in the metadata.
82 : // It's duplicated for convenience when reading the serialized structure, but is
83 : // private because internally we would read from metadata instead.
84 : disk_consistent_lsn: Lsn,
85 :
86 : #[serde(rename = "metadata_bytes")]
87 : pub metadata: TimelineMetadata,
88 : }
89 :
90 : impl IndexPart {
91 : /// When adding or modifying any parts of `IndexPart`, increment the version so that it can be
92 : /// used to understand later versions.
93 : ///
94 : /// Version is currently informative only.
95 : /// Version history
96 : /// - 2: added `deleted_at`
97 : /// - 3: no longer deserialize `timeline_layers` (serialized format is the same, but timeline_layers
98 : /// is always generated from the keys of `layer_metadata`)
99 : /// - 4: timeline_layers is fully removed.
100 : const LATEST_VERSION: usize = 4;
101 :
102 : // Versions we may see when reading from a bucket.
103 : pub const KNOWN_VERSIONS: &'static [usize] = &[1, 2, 3, 4];
104 :
105 : pub const FILE_NAME: &'static str = "index_part.json";
106 :
107 5377 : pub fn new(
108 5377 : layers_and_metadata: HashMap<LayerFileName, LayerFileMetadata>,
109 5377 : disk_consistent_lsn: Lsn,
110 5377 : metadata: TimelineMetadata,
111 5377 : ) -> Self {
112 5377 : // Transform LayerFileMetadata into IndexLayerMetadata
113 5377 : let layer_metadata = layers_and_metadata
114 5377 : .into_iter()
115 481631 : .map(|(k, v)| (k, IndexLayerMetadata::from(v)))
116 5377 : .collect();
117 5377 :
118 5377 : Self {
119 5377 : version: Self::LATEST_VERSION,
120 5377 : layer_metadata,
121 5377 : disk_consistent_lsn,
122 5377 : metadata,
123 5377 : deleted_at: None,
124 5377 : }
125 5377 : }
126 :
127 UBC 0 : pub fn get_version(&self) -> usize {
128 0 : self.version
129 0 : }
130 :
131 : /// If you want this under normal operations, read it from self.metadata:
132 : /// this method is just for the scrubber to use when validating an index.
133 CBC 6 : pub fn get_disk_consistent_lsn(&self) -> Lsn {
134 6 : self.disk_consistent_lsn
135 6 : }
136 :
137 11 : pub fn from_s3_bytes(bytes: &[u8]) -> Result<Self, serde_json::Error> {
138 11 : serde_json::from_slice::<IndexPart>(bytes)
139 11 : }
140 :
141 5952 : pub fn to_s3_bytes(&self) -> serde_json::Result<Vec<u8>> {
142 5952 : serde_json::to_vec(self)
143 5952 : }
144 : }
145 :
146 : impl TryFrom<&UploadQueueInitialized> for IndexPart {
147 : type Error = SerializeError;
148 :
149 159 : fn try_from(upload_queue: &UploadQueueInitialized) -> Result<Self, Self::Error> {
150 159 : let disk_consistent_lsn = upload_queue.latest_metadata.disk_consistent_lsn();
151 159 : let metadata = upload_queue.latest_metadata.clone();
152 159 :
153 159 : Ok(Self::new(
154 159 : upload_queue.latest_files.clone(),
155 159 : disk_consistent_lsn,
156 159 : metadata,
157 159 : ))
158 159 : }
159 : }
160 :
161 : /// Serialized form of [`LayerFileMetadata`].
162 486263 : #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
163 : pub struct IndexLayerMetadata {
164 : pub file_size: u64,
165 :
166 : #[serde(default = "Generation::none")]
167 : #[serde(skip_serializing_if = "Generation::is_none")]
168 : pub generation: Generation,
169 :
170 : #[serde(default = "ShardIndex::unsharded")]
171 : #[serde(skip_serializing_if = "ShardIndex::is_unsharded")]
172 : pub shard: ShardIndex,
173 : }
174 :
175 : impl From<LayerFileMetadata> for IndexLayerMetadata {
176 484084 : fn from(other: LayerFileMetadata) -> Self {
177 484084 : IndexLayerMetadata {
178 484084 : file_size: other.file_size,
179 484084 : generation: other.generation,
180 484084 : shard: other.shard,
181 484084 : }
182 484084 : }
183 : }
184 :
185 : #[cfg(test)]
186 : mod tests {
187 : use super::*;
188 :
189 1 : #[test]
190 1 : fn v1_indexpart_is_parsed() {
191 1 : let example = r#"{
192 1 : "version":1,
193 1 : "timeline_layers":["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"],
194 1 : "layer_metadata":{
195 1 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
196 1 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
197 1 : },
198 1 : "disk_consistent_lsn":"0/16960E8",
199 1 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
200 1 : }"#;
201 1 :
202 1 : let expected = IndexPart {
203 1 : // note this is not verified, could be anything, but exists for humans debugging.. could be the git version instead?
204 1 : version: 1,
205 1 : layer_metadata: HashMap::from([
206 1 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), IndexLayerMetadata {
207 1 : file_size: 25600000,
208 1 : generation: Generation::none(),
209 1 : shard: ShardIndex::unsharded()
210 1 : }),
211 1 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), IndexLayerMetadata {
212 1 : // serde_json should always parse this but this might be a double with jq for
213 1 : // example.
214 1 : file_size: 9007199254741001,
215 1 : generation: Generation::none(),
216 1 : shard: ShardIndex::unsharded()
217 1 : })
218 1 : ]),
219 1 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
220 1 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
221 1 : deleted_at: None,
222 1 : };
223 1 :
224 1 : let part = IndexPart::from_s3_bytes(example.as_bytes()).unwrap();
225 1 : assert_eq!(part, expected);
226 1 : }
227 :
228 1 : #[test]
229 1 : fn v1_indexpart_is_parsed_with_optional_missing_layers() {
230 1 : let example = r#"{
231 1 : "version":1,
232 1 : "timeline_layers":["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"],
233 1 : "missing_layers":["This shouldn't fail deserialization"],
234 1 : "layer_metadata":{
235 1 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
236 1 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
237 1 : },
238 1 : "disk_consistent_lsn":"0/16960E8",
239 1 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
240 1 : }"#;
241 1 :
242 1 : let expected = IndexPart {
243 1 : // note this is not verified, could be anything, but exists for humans debugging.. could be the git version instead?
244 1 : version: 1,
245 1 : layer_metadata: HashMap::from([
246 1 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), IndexLayerMetadata {
247 1 : file_size: 25600000,
248 1 : generation: Generation::none(),
249 1 : shard: ShardIndex::unsharded()
250 1 : }),
251 1 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), IndexLayerMetadata {
252 1 : // serde_json should always parse this but this might be a double with jq for
253 1 : // example.
254 1 : file_size: 9007199254741001,
255 1 : generation: Generation::none(),
256 1 : shard: ShardIndex::unsharded()
257 1 : })
258 1 : ]),
259 1 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
260 1 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
261 1 : deleted_at: None,
262 1 : };
263 1 :
264 1 : let part = IndexPart::from_s3_bytes(example.as_bytes()).unwrap();
265 1 : assert_eq!(part, expected);
266 1 : }
267 :
268 1 : #[test]
269 1 : fn v2_indexpart_is_parsed_with_deleted_at() {
270 1 : let example = r#"{
271 1 : "version":2,
272 1 : "timeline_layers":["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"],
273 1 : "missing_layers":["This shouldn't fail deserialization"],
274 1 : "layer_metadata":{
275 1 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
276 1 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
277 1 : },
278 1 : "disk_consistent_lsn":"0/16960E8",
279 1 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
280 1 : "deleted_at": "2023-07-31T09:00:00.123"
281 1 : }"#;
282 1 :
283 1 : let expected = IndexPart {
284 1 : // note this is not verified, could be anything, but exists for humans debugging.. could be the git version instead?
285 1 : version: 2,
286 1 : layer_metadata: HashMap::from([
287 1 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), IndexLayerMetadata {
288 1 : file_size: 25600000,
289 1 : generation: Generation::none(),
290 1 : shard: ShardIndex::unsharded()
291 1 : }),
292 1 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), IndexLayerMetadata {
293 1 : // serde_json should always parse this but this might be a double with jq for
294 1 : // example.
295 1 : file_size: 9007199254741001,
296 1 : generation: Generation::none(),
297 1 : shard: ShardIndex::unsharded()
298 1 : })
299 1 : ]),
300 1 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
301 1 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
302 1 : deleted_at: Some(chrono::NaiveDateTime::parse_from_str(
303 1 : "2023-07-31T09:00:00.123000000", "%Y-%m-%dT%H:%M:%S.%f").unwrap())
304 1 : };
305 1 :
306 1 : let part = IndexPart::from_s3_bytes(example.as_bytes()).unwrap();
307 1 : assert_eq!(part, expected);
308 1 : }
309 :
310 1 : #[test]
311 1 : fn empty_layers_are_parsed() {
312 1 : let empty_layers_json = r#"{
313 1 : "version":1,
314 1 : "timeline_layers":[],
315 1 : "layer_metadata":{},
316 1 : "disk_consistent_lsn":"0/2532648",
317 1 : "metadata_bytes":[136,151,49,208,0,70,0,4,0,0,0,0,2,83,38,72,1,0,0,0,0,2,83,38,32,1,87,198,240,135,97,119,45,125,38,29,155,161,140,141,255,210,0,0,0,0,2,83,38,72,0,0,0,0,1,73,240,192,0,0,0,0,1,73,240,192,0,0,0,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
318 1 : }"#;
319 1 :
320 1 : let expected = IndexPart {
321 1 : version: 1,
322 1 : layer_metadata: HashMap::new(),
323 1 : disk_consistent_lsn: "0/2532648".parse::<Lsn>().unwrap(),
324 1 : metadata: TimelineMetadata::from_bytes(&[
325 1 : 136, 151, 49, 208, 0, 70, 0, 4, 0, 0, 0, 0, 2, 83, 38, 72, 1, 0, 0, 0, 0, 2, 83,
326 1 : 38, 32, 1, 87, 198, 240, 135, 97, 119, 45, 125, 38, 29, 155, 161, 140, 141, 255,
327 1 : 210, 0, 0, 0, 0, 2, 83, 38, 72, 0, 0, 0, 0, 1, 73, 240, 192, 0, 0, 0, 0, 1, 73,
328 1 : 240, 192, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
329 1 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
330 1 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
331 1 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
332 1 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
333 1 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
334 1 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
335 1 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
336 1 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
337 1 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
338 1 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
339 1 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
340 1 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
341 1 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
342 1 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
343 1 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
344 1 : 0, 0,
345 1 : ])
346 1 : .unwrap(),
347 1 : deleted_at: None,
348 1 : };
349 1 :
350 1 : let empty_layers_parsed = IndexPart::from_s3_bytes(empty_layers_json.as_bytes()).unwrap();
351 1 :
352 1 : assert_eq!(empty_layers_parsed, expected);
353 1 : }
354 :
355 1 : #[test]
356 1 : fn v4_indexpart_is_parsed() {
357 1 : let example = r#"{
358 1 : "version":4,
359 1 : "layer_metadata":{
360 1 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
361 1 : "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
362 1 : },
363 1 : "disk_consistent_lsn":"0/16960E8",
364 1 : "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
365 1 : "deleted_at": "2023-07-31T09:00:00.123"
366 1 : }"#;
367 1 :
368 1 : let expected = IndexPart {
369 1 : version: 4,
370 1 : layer_metadata: HashMap::from([
371 1 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), IndexLayerMetadata {
372 1 : file_size: 25600000,
373 1 : generation: Generation::none(),
374 1 : shard: ShardIndex::unsharded()
375 1 : }),
376 1 : ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), IndexLayerMetadata {
377 1 : // serde_json should always parse this but this might be a double with jq for
378 1 : // example.
379 1 : file_size: 9007199254741001,
380 1 : generation: Generation::none(),
381 1 : shard: ShardIndex::unsharded()
382 1 : })
383 1 : ]),
384 1 : disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
385 1 : metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(),
386 1 : deleted_at: Some(chrono::NaiveDateTime::parse_from_str(
387 1 : "2023-07-31T09:00:00.123000000", "%Y-%m-%dT%H:%M:%S.%f").unwrap()),
388 1 : };
389 1 :
390 1 : let part = IndexPart::from_s3_bytes(example.as_bytes()).unwrap();
391 1 : assert_eq!(part, expected);
392 1 : }
393 : }
|
