Line data Source code
1 : //! Describes the legacy now hopefully no longer modified per-timeline metadata.
2 : //!
3 : //! It is stored in `index_part.json` managed by [`remote_timeline_client`]. For many tenants and
4 : //! their timelines, this struct and its original serialization format is still needed because
5 : //! they were written a long time ago.
6 : //!
7 : //! Instead of changing and adding versioning to this, just change [`IndexPart`] with soft json
8 : //! versioning.
9 : //!
10 : //! To clean up this module we need to migrate all index_part.json files to a later version.
11 : //! While doing this, we need to be mindful about s3 based recovery as well, so it might take
12 : //! however long we keep the old versions to be able to delete the old code. After that, we can
13 : //! remove everything else than [`TimelineMetadataBodyV2`], rename it as `TimelineMetadata` and
14 : //! move it to `index.rs`. Before doing all of this, we need to keep the structures for backwards
15 : //! compatibility.
16 : //!
17 : //! [`remote_timeline_client`]: super::remote_timeline_client
18 : //! [`IndexPart`]: super::remote_timeline_client::index::IndexPart
19 :
20 : use anyhow::ensure;
21 : use serde::{Deserialize, Serialize};
22 : use utils::bin_ser::SerializeError;
23 : use utils::{bin_ser::BeSer, id::TimelineId, lsn::Lsn};
24 :
25 : /// Use special format number to enable backward compatibility.
26 : const METADATA_FORMAT_VERSION: u16 = 4;
27 :
28 : /// Previous supported format versions.
29 : ///
30 : /// In practice, none of these should remain, all are [`METADATA_FORMAT_VERSION`], but confirming
31 : /// that requires a scrubber run which is yet to be done.
32 : const METADATA_OLD_FORMAT_VERSION: u16 = 3;
33 :
34 : /// When the file existed on disk we assumed that a write of up to METADATA_MAX_SIZE bytes is atomic.
35 : ///
36 : /// This is the same assumption that PostgreSQL makes with the control file,
37 : ///
38 : /// see PG_CONTROL_MAX_SAFE_SIZE
39 : const METADATA_MAX_SIZE: usize = 512;
40 :
41 : /// Legacy metadata stored as a component of `index_part.json` per timeline.
42 : ///
43 : /// Do not make new changes to this type or the module. In production, we have two different kinds
44 : /// of serializations of this type: bincode and json. Bincode version reflects what used to be
45 : /// stored on disk in earlier versions and does internal crc32 checksumming.
46 : ///
47 : /// This type should not implement `serde::Serialize` or `serde::Deserialize` because there would
48 : /// be a confusion whether you want the old version ([`TimelineMetadata::from_bytes`]) or the modern
49 : /// as-exists in `index_part.json` ([`self::modern_serde`]).
50 : ///
51 : /// ```compile_fail
52 : /// #[derive(serde::Serialize)]
53 : /// struct DoNotDoThis(pageserver::tenant::metadata::TimelineMetadata);
54 : /// ```
55 : ///
56 : /// ```compile_fail
57 : /// #[derive(serde::Deserialize)]
58 : /// struct NeitherDoThis(pageserver::tenant::metadata::TimelineMetadata);
59 : /// ```
60 : #[derive(Debug, Clone, PartialEq, Eq)]
61 : pub struct TimelineMetadata {
62 : hdr: TimelineMetadataHeader,
63 : body: TimelineMetadataBodyV2,
64 : }
65 :
66 156 : #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
67 : struct TimelineMetadataHeader {
68 : checksum: u32, // CRC of serialized metadata body
69 : size: u16, // size of serialized metadata
70 : format_version: u16, // metadata format version (used for compatibility checks)
71 : }
72 :
73 : impl TryFrom<&TimelineMetadataBodyV2> for TimelineMetadataHeader {
74 : type Error = Crc32CalculationFailed;
75 :
76 108 : fn try_from(value: &TimelineMetadataBodyV2) -> Result<Self, Self::Error> {
77 : #[derive(Default)]
78 : struct Crc32Sink {
79 : crc: u32,
80 : count: usize,
81 : }
82 :
83 : impl std::io::Write for Crc32Sink {
84 1500 : fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
85 1500 : self.crc = crc32c::crc32c_append(self.crc, buf);
86 1500 : self.count += buf.len();
87 1500 : Ok(buf.len())
88 1500 : }
89 :
90 0 : fn flush(&mut self) -> std::io::Result<()> {
91 0 : Ok(())
92 0 : }
93 : }
94 :
95 : // jump through hoops to calculate the crc32 so that TimelineMetadata::ne works
96 : // across serialization versions
97 108 : let mut sink = Crc32Sink::default();
98 108 : <TimelineMetadataBodyV2 as utils::bin_ser::BeSer>::ser_into(value, &mut sink)
99 108 : .map_err(Crc32CalculationFailed)?;
100 :
101 108 : let size = METADATA_HDR_SIZE + sink.count;
102 108 :
103 108 : Ok(TimelineMetadataHeader {
104 108 : checksum: sink.crc,
105 108 : size: size as u16,
106 108 : format_version: METADATA_FORMAT_VERSION,
107 108 : })
108 108 : }
109 : }
110 :
111 0 : #[derive(thiserror::Error, Debug)]
112 : #[error("re-serializing for crc32 failed")]
113 : struct Crc32CalculationFailed(#[source] utils::bin_ser::SerializeError);
114 :
115 : const METADATA_HDR_SIZE: usize = size_of::<TimelineMetadataHeader>();
116 :
117 864 : #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
118 : struct TimelineMetadataBodyV2 {
119 : disk_consistent_lsn: Lsn,
120 : // This is only set if we know it. We track it in memory when the page
121 : // server is running, but we only track the value corresponding to
122 : // 'last_record_lsn', not 'disk_consistent_lsn' which can lag behind by a
123 : // lot. We only store it in the metadata file when we flush *all* the
124 : // in-memory data so that 'last_record_lsn' is the same as
125 : // 'disk_consistent_lsn'. That's OK, because after page server restart, as
126 : // soon as we reprocess at least one record, we will have a valid
127 : // 'prev_record_lsn' value in memory again. This is only really needed when
128 : // doing a clean shutdown, so that there is no more WAL beyond
129 : // 'disk_consistent_lsn'
130 : prev_record_lsn: Option<Lsn>,
131 : ancestor_timeline: Option<TimelineId>,
132 : ancestor_lsn: Lsn,
133 : latest_gc_cutoff_lsn: Lsn,
134 : initdb_lsn: Lsn,
135 : pg_version: u32,
136 : }
137 :
138 6 : #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
139 : struct TimelineMetadataBodyV1 {
140 : disk_consistent_lsn: Lsn,
141 : // This is only set if we know it. We track it in memory when the page
142 : // server is running, but we only track the value corresponding to
143 : // 'last_record_lsn', not 'disk_consistent_lsn' which can lag behind by a
144 : // lot. We only store it in the metadata file when we flush *all* the
145 : // in-memory data so that 'last_record_lsn' is the same as
146 : // 'disk_consistent_lsn'. That's OK, because after page server restart, as
147 : // soon as we reprocess at least one record, we will have a valid
148 : // 'prev_record_lsn' value in memory again. This is only really needed when
149 : // doing a clean shutdown, so that there is no more WAL beyond
150 : // 'disk_consistent_lsn'
151 : prev_record_lsn: Option<Lsn>,
152 : ancestor_timeline: Option<TimelineId>,
153 : ancestor_lsn: Lsn,
154 : latest_gc_cutoff_lsn: Lsn,
155 : initdb_lsn: Lsn,
156 : }
157 :
158 : impl TimelineMetadata {
159 1302 : pub fn new(
160 1302 : disk_consistent_lsn: Lsn,
161 1302 : prev_record_lsn: Option<Lsn>,
162 1302 : ancestor_timeline: Option<TimelineId>,
163 1302 : ancestor_lsn: Lsn,
164 1302 : latest_gc_cutoff_lsn: Lsn,
165 1302 : initdb_lsn: Lsn,
166 1302 : pg_version: u32,
167 1302 : ) -> Self {
168 1302 : Self {
169 1302 : hdr: TimelineMetadataHeader {
170 1302 : checksum: 0,
171 1302 : size: 0,
172 1302 : format_version: METADATA_FORMAT_VERSION,
173 1302 : },
174 1302 : body: TimelineMetadataBodyV2 {
175 1302 : disk_consistent_lsn,
176 1302 : prev_record_lsn,
177 1302 : ancestor_timeline,
178 1302 : ancestor_lsn,
179 1302 : latest_gc_cutoff_lsn,
180 1302 : initdb_lsn,
181 1302 : pg_version,
182 1302 : },
183 1302 : }
184 1302 : }
185 :
186 : #[cfg(test)]
187 18 : pub(crate) fn with_recalculated_checksum(mut self) -> anyhow::Result<Self> {
188 18 : self.hdr = TimelineMetadataHeader::try_from(&self.body)?;
189 18 : Ok(self)
190 18 : }
191 :
192 6 : fn upgrade_timeline_metadata(metadata_bytes: &[u8]) -> anyhow::Result<Self> {
193 6 : let mut hdr = TimelineMetadataHeader::des(&metadata_bytes[0..METADATA_HDR_SIZE])?;
194 :
195 : // backward compatible only up to this version
196 6 : ensure!(
197 6 : hdr.format_version == METADATA_OLD_FORMAT_VERSION,
198 0 : "unsupported metadata format version {}",
199 : hdr.format_version
200 : );
201 :
202 6 : let metadata_size = hdr.size as usize;
203 :
204 6 : let body: TimelineMetadataBodyV1 =
205 6 : TimelineMetadataBodyV1::des(&metadata_bytes[METADATA_HDR_SIZE..metadata_size])?;
206 :
207 6 : let body = TimelineMetadataBodyV2 {
208 6 : disk_consistent_lsn: body.disk_consistent_lsn,
209 6 : prev_record_lsn: body.prev_record_lsn,
210 6 : ancestor_timeline: body.ancestor_timeline,
211 6 : ancestor_lsn: body.ancestor_lsn,
212 6 : latest_gc_cutoff_lsn: body.latest_gc_cutoff_lsn,
213 6 : initdb_lsn: body.initdb_lsn,
214 6 : pg_version: 14, // All timelines created before this version had pg_version 14
215 6 : };
216 6 :
217 6 : hdr.format_version = METADATA_FORMAT_VERSION;
218 6 :
219 6 : Ok(Self { hdr, body })
220 6 : }
221 :
222 150 : pub fn from_bytes(metadata_bytes: &[u8]) -> anyhow::Result<Self> {
223 150 : ensure!(
224 150 : metadata_bytes.len() == METADATA_MAX_SIZE,
225 0 : "metadata bytes size is wrong"
226 : );
227 150 : let hdr = TimelineMetadataHeader::des(&metadata_bytes[0..METADATA_HDR_SIZE])?;
228 :
229 150 : let metadata_size = hdr.size as usize;
230 150 : ensure!(
231 150 : metadata_size <= METADATA_MAX_SIZE,
232 0 : "corrupted metadata file"
233 : );
234 150 : let calculated_checksum = crc32c::crc32c(&metadata_bytes[METADATA_HDR_SIZE..metadata_size]);
235 150 : ensure!(
236 150 : hdr.checksum == calculated_checksum,
237 0 : "metadata checksum mismatch"
238 : );
239 :
240 150 : if hdr.format_version != METADATA_FORMAT_VERSION {
241 : // If metadata has the old format,
242 : // upgrade it and return the result
243 6 : TimelineMetadata::upgrade_timeline_metadata(metadata_bytes)
244 : } else {
245 144 : let body =
246 144 : TimelineMetadataBodyV2::des(&metadata_bytes[METADATA_HDR_SIZE..metadata_size])?;
247 144 : ensure!(
248 144 : body.disk_consistent_lsn.is_aligned(),
249 0 : "disk_consistent_lsn is not aligned"
250 : );
251 144 : Ok(TimelineMetadata { hdr, body })
252 : }
253 150 : }
254 :
255 54 : pub fn to_bytes(&self) -> Result<Vec<u8>, SerializeError> {
256 54 : let body_bytes = self.body.ser()?;
257 54 : let metadata_size = METADATA_HDR_SIZE + body_bytes.len();
258 54 : let hdr = TimelineMetadataHeader {
259 54 : size: metadata_size as u16,
260 54 : format_version: METADATA_FORMAT_VERSION,
261 54 : checksum: crc32c::crc32c(&body_bytes),
262 54 : };
263 54 : let hdr_bytes = hdr.ser()?;
264 54 : let mut metadata_bytes = vec![0u8; METADATA_MAX_SIZE];
265 54 : metadata_bytes[0..METADATA_HDR_SIZE].copy_from_slice(&hdr_bytes);
266 54 : metadata_bytes[METADATA_HDR_SIZE..metadata_size].copy_from_slice(&body_bytes);
267 54 : Ok(metadata_bytes)
268 54 : }
269 :
270 : /// [`Lsn`] that corresponds to the corresponding timeline directory
271 : /// contents, stored locally in the pageserver workdir.
272 63791 : pub fn disk_consistent_lsn(&self) -> Lsn {
273 63791 : self.body.disk_consistent_lsn
274 63791 : }
275 :
276 1242 : pub fn prev_record_lsn(&self) -> Option<Lsn> {
277 1242 : self.body.prev_record_lsn
278 1242 : }
279 :
280 1278 : pub fn ancestor_timeline(&self) -> Option<TimelineId> {
281 1278 : self.body.ancestor_timeline
282 1278 : }
283 :
284 1926 : pub fn ancestor_lsn(&self) -> Lsn {
285 1926 : self.body.ancestor_lsn
286 1926 : }
287 :
288 : /// When reparenting, the `ancestor_lsn` does not change.
289 : ///
290 : /// Returns true if anything was changed.
291 0 : pub fn reparent(&mut self, timeline: &TimelineId) {
292 0 : assert!(self.body.ancestor_timeline.is_some());
293 : // no assertion for redoing this: it's fine, we may have to repeat this multiple times over
294 0 : self.body.ancestor_timeline = Some(*timeline);
295 0 : }
296 :
297 : /// Returns true if anything was changed
298 0 : pub fn detach_from_ancestor(&mut self, branchpoint: &(TimelineId, Lsn)) {
299 0 : if let Some(ancestor) = self.body.ancestor_timeline {
300 0 : assert_eq!(ancestor, branchpoint.0);
301 0 : }
302 0 : if self.body.ancestor_lsn != Lsn(0) {
303 0 : assert_eq!(self.body.ancestor_lsn, branchpoint.1);
304 0 : }
305 0 : self.body.ancestor_timeline = None;
306 0 : self.body.ancestor_lsn = Lsn(0);
307 0 : }
308 :
309 1242 : pub fn latest_gc_cutoff_lsn(&self) -> Lsn {
310 1242 : self.body.latest_gc_cutoff_lsn
311 1242 : }
312 :
313 1242 : pub fn initdb_lsn(&self) -> Lsn {
314 1242 : self.body.initdb_lsn
315 1242 : }
316 :
317 1242 : pub fn pg_version(&self) -> u32 {
318 1242 : self.body.pg_version
319 1242 : }
320 :
321 : // Checksums make it awkward to build a valid instance by hand. This helper
322 : // provides a TimelineMetadata with a valid checksum in its header.
323 : #[cfg(test)]
324 36 : pub fn example() -> Self {
325 36 : let instance = Self::new(
326 36 : "0/16960E8".parse::<Lsn>().unwrap(),
327 36 : None,
328 36 : None,
329 36 : Lsn::from_hex("00000000").unwrap(),
330 36 : Lsn::from_hex("00000000").unwrap(),
331 36 : Lsn::from_hex("00000000").unwrap(),
332 36 : 0,
333 36 : );
334 36 : let bytes = instance.to_bytes().unwrap();
335 36 : Self::from_bytes(&bytes).unwrap()
336 36 : }
337 :
338 3444 : pub(crate) fn apply(&mut self, update: &MetadataUpdate) {
339 3444 : self.body.disk_consistent_lsn = update.disk_consistent_lsn;
340 3444 : self.body.prev_record_lsn = update.prev_record_lsn;
341 3444 : self.body.latest_gc_cutoff_lsn = update.latest_gc_cutoff_lsn;
342 3444 : }
343 : }
344 :
345 : pub(crate) mod modern_serde {
346 : use super::{TimelineMetadata, TimelineMetadataBodyV2, TimelineMetadataHeader};
347 : use serde::{Deserialize, Serialize};
348 :
349 138 : pub(crate) fn deserialize<'de, D>(deserializer: D) -> Result<TimelineMetadata, D::Error>
350 138 : where
351 138 : D: serde::de::Deserializer<'de>,
352 138 : {
353 : // for legacy reasons versions 1-5 had TimelineMetadata serialized as a Vec<u8> field with
354 : // BeSer.
355 : struct Visitor;
356 :
357 : impl<'d> serde::de::Visitor<'d> for Visitor {
358 : type Value = TimelineMetadata;
359 :
360 0 : fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
361 0 : f.write_str("BeSer bytes or json structure")
362 0 : }
363 :
364 48 : fn visit_seq<A>(self, seq: A) -> Result<Self::Value, A::Error>
365 48 : where
366 48 : A: serde::de::SeqAccess<'d>,
367 48 : {
368 : use serde::de::Error;
369 48 : let de = serde::de::value::SeqAccessDeserializer::new(seq);
370 48 : Vec::<u8>::deserialize(de)
371 48 : .map(|v| TimelineMetadata::from_bytes(&v).map_err(A::Error::custom))?
372 48 : }
373 :
374 90 : fn visit_map<A>(self, map: A) -> Result<Self::Value, A::Error>
375 90 : where
376 90 : A: serde::de::MapAccess<'d>,
377 90 : {
378 : use serde::de::Error;
379 :
380 90 : let de = serde::de::value::MapAccessDeserializer::new(map);
381 90 : let body = TimelineMetadataBodyV2::deserialize(de)?;
382 90 : let hdr = TimelineMetadataHeader::try_from(&body).map_err(A::Error::custom)?;
383 :
384 90 : Ok(TimelineMetadata { hdr, body })
385 90 : }
386 : }
387 :
388 138 : deserializer.deserialize_any(Visitor)
389 138 : }
390 :
391 8645 : pub(crate) fn serialize<S>(
392 8645 : metadata: &TimelineMetadata,
393 8645 : serializer: S,
394 8645 : ) -> Result<S::Ok, S::Error>
395 8645 : where
396 8645 : S: serde::Serializer,
397 8645 : {
398 8645 : // header is not needed, upon reading we've upgraded all v1 to v2
399 8645 : metadata.body.serialize(serializer)
400 8645 : }
401 :
402 : #[test]
403 6 : fn deserializes_bytes_as_well_as_equivalent_body_v2() {
404 12 : #[derive(serde::Deserialize, serde::Serialize)]
405 : struct Wrapper(
406 : #[serde(deserialize_with = "deserialize", serialize_with = "serialize")]
407 : TimelineMetadata,
408 : );
409 :
410 6 : let too_many_bytes = "[216,111,252,208,0,54,0,4,0,0,0,0,1,73,253,144,1,0,0,0,0,1,73,253,24,0,0,0,0,0,0,0,0,0,0,0,0,0,1,73,253,24,0,0,0,0,1,73,253,24,0,0,0,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]";
411 6 :
412 6 : let wrapper_from_bytes = serde_json::from_str::<Wrapper>(too_many_bytes).unwrap();
413 6 :
414 6 : let serialized = serde_json::to_value(&wrapper_from_bytes).unwrap();
415 6 :
416 6 : assert_eq!(
417 6 : serialized,
418 6 : serde_json::json! {{
419 6 : "disk_consistent_lsn": "0/149FD90",
420 6 : "prev_record_lsn": "0/149FD18",
421 6 : "ancestor_timeline": null,
422 6 : "ancestor_lsn": "0/0",
423 6 : "latest_gc_cutoff_lsn": "0/149FD18",
424 6 : "initdb_lsn": "0/149FD18",
425 6 : "pg_version": 15
426 6 : }}
427 6 : );
428 :
429 6 : let wrapper_from_json = serde_json::value::from_value::<Wrapper>(serialized).unwrap();
430 6 :
431 6 : assert_eq!(wrapper_from_bytes.0, wrapper_from_json.0);
432 6 : }
433 : }
434 :
435 : /// Parts of the metadata which are regularly modified.
436 : pub(crate) struct MetadataUpdate {
437 : disk_consistent_lsn: Lsn,
438 : prev_record_lsn: Option<Lsn>,
439 : latest_gc_cutoff_lsn: Lsn,
440 : }
441 :
442 : impl MetadataUpdate {
443 3444 : pub(crate) fn new(
444 3444 : disk_consistent_lsn: Lsn,
445 3444 : prev_record_lsn: Option<Lsn>,
446 3444 : latest_gc_cutoff_lsn: Lsn,
447 3444 : ) -> Self {
448 3444 : Self {
449 3444 : disk_consistent_lsn,
450 3444 : prev_record_lsn,
451 3444 : latest_gc_cutoff_lsn,
452 3444 : }
453 3444 : }
454 : }
455 :
456 : #[cfg(test)]
457 : mod tests {
458 : use super::*;
459 : use crate::tenant::harness::TIMELINE_ID;
460 :
461 : #[test]
462 6 : fn metadata_serializes_correctly() {
463 6 : let original_metadata = TimelineMetadata::new(
464 6 : Lsn(0x200),
465 6 : Some(Lsn(0x100)),
466 6 : Some(TIMELINE_ID),
467 6 : Lsn(0),
468 6 : Lsn(0),
469 6 : Lsn(0),
470 6 : // Any version will do here, so use the default
471 6 : crate::DEFAULT_PG_VERSION,
472 6 : );
473 6 :
474 6 : let metadata_bytes = original_metadata
475 6 : .to_bytes()
476 6 : .expect("Should serialize correct metadata to bytes");
477 6 :
478 6 : let deserialized_metadata = TimelineMetadata::from_bytes(&metadata_bytes)
479 6 : .expect("Should deserialize its own bytes");
480 6 :
481 6 : assert_eq!(
482 : deserialized_metadata.body, original_metadata.body,
483 0 : "Metadata that was serialized to bytes and deserialized back should not change"
484 : );
485 6 : }
486 :
487 : // Generate old version metadata and read it with current code.
488 : // Ensure that it is upgraded correctly
489 : #[test]
490 6 : fn test_metadata_upgrade() {
491 : #[derive(Debug, Clone, PartialEq, Eq)]
492 : struct TimelineMetadataV1 {
493 : hdr: TimelineMetadataHeader,
494 : body: TimelineMetadataBodyV1,
495 : }
496 :
497 6 : let metadata_v1 = TimelineMetadataV1 {
498 6 : hdr: TimelineMetadataHeader {
499 6 : checksum: 0,
500 6 : size: 0,
501 6 : format_version: METADATA_OLD_FORMAT_VERSION,
502 6 : },
503 6 : body: TimelineMetadataBodyV1 {
504 6 : disk_consistent_lsn: Lsn(0x200),
505 6 : prev_record_lsn: Some(Lsn(0x100)),
506 6 : ancestor_timeline: Some(TIMELINE_ID),
507 6 : ancestor_lsn: Lsn(0),
508 6 : latest_gc_cutoff_lsn: Lsn(0),
509 6 : initdb_lsn: Lsn(0),
510 6 : },
511 6 : };
512 :
513 : impl TimelineMetadataV1 {
514 6 : pub fn to_bytes(&self) -> anyhow::Result<Vec<u8>> {
515 6 : let body_bytes = self.body.ser()?;
516 6 : let metadata_size = METADATA_HDR_SIZE + body_bytes.len();
517 6 : let hdr = TimelineMetadataHeader {
518 6 : size: metadata_size as u16,
519 6 : format_version: METADATA_OLD_FORMAT_VERSION,
520 6 : checksum: crc32c::crc32c(&body_bytes),
521 6 : };
522 6 : let hdr_bytes = hdr.ser()?;
523 6 : let mut metadata_bytes = vec![0u8; METADATA_MAX_SIZE];
524 6 : metadata_bytes[0..METADATA_HDR_SIZE].copy_from_slice(&hdr_bytes);
525 6 : metadata_bytes[METADATA_HDR_SIZE..metadata_size].copy_from_slice(&body_bytes);
526 6 : Ok(metadata_bytes)
527 6 : }
528 : }
529 :
530 6 : let metadata_bytes = metadata_v1
531 6 : .to_bytes()
532 6 : .expect("Should serialize correct metadata to bytes");
533 6 :
534 6 : // This should deserialize to the latest version format
535 6 : let deserialized_metadata = TimelineMetadata::from_bytes(&metadata_bytes)
536 6 : .expect("Should deserialize its own bytes");
537 6 :
538 6 : let expected_metadata = TimelineMetadata::new(
539 6 : Lsn(0x200),
540 6 : Some(Lsn(0x100)),
541 6 : Some(TIMELINE_ID),
542 6 : Lsn(0),
543 6 : Lsn(0),
544 6 : Lsn(0),
545 6 : 14, // All timelines created before this version had pg_version 14
546 6 : );
547 6 :
548 6 : assert_eq!(
549 : deserialized_metadata.body, expected_metadata.body,
550 0 : "Metadata of the old version {} should be upgraded to the latest version {}",
551 : METADATA_OLD_FORMAT_VERSION, METADATA_FORMAT_VERSION
552 : );
553 6 : }
554 :
555 : #[test]
556 6 : fn test_metadata_bincode_serde_ensure_roundtrip() {
557 6 : let original_metadata = TimelineMetadata::new(
558 6 : Lsn(0x200),
559 6 : Some(Lsn(0x100)),
560 6 : Some(TIMELINE_ID),
561 6 : Lsn(0),
562 6 : Lsn(0),
563 6 : Lsn(0),
564 6 : // Any version will do here, so use the default
565 6 : crate::DEFAULT_PG_VERSION,
566 6 : );
567 6 : let expected_bytes = vec![
568 6 : /* TimelineMetadataHeader */
569 6 : 74, 104, 158, 105, 0, 70, 0, 4, // checksum, size, format_version (4 + 2 + 2)
570 6 : /* TimelineMetadataBodyV2 */
571 6 : 0, 0, 0, 0, 0, 0, 2, 0, // disk_consistent_lsn (8 bytes)
572 6 : 1, 0, 0, 0, 0, 0, 0, 1, 0, // prev_record_lsn (9 bytes)
573 6 : 1, 17, 34, 51, 68, 85, 102, 119, 136, 17, 34, 51, 68, 85, 102, 119,
574 6 : 136, // ancestor_timeline (17 bytes)
575 6 : 0, 0, 0, 0, 0, 0, 0, 0, // ancestor_lsn (8 bytes)
576 6 : 0, 0, 0, 0, 0, 0, 0, 0, // latest_gc_cutoff_lsn (8 bytes)
577 6 : 0, 0, 0, 0, 0, 0, 0, 0, // initdb_lsn (8 bytes)
578 6 : 0, 0, 0, 16, // pg_version (4 bytes)
579 6 : /* padding bytes */
580 6 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
581 6 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
582 6 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
583 6 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
584 6 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
585 6 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
586 6 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
587 6 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
588 6 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
589 6 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
590 6 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
591 6 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
592 6 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
593 6 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
594 6 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
595 6 : 0, 0, 0, 0, 0, 0, 0,
596 6 : ];
597 6 : let metadata_ser_bytes = original_metadata.to_bytes().unwrap();
598 6 : assert_eq!(metadata_ser_bytes, expected_bytes);
599 :
600 6 : let expected_metadata = {
601 6 : let mut temp_metadata = original_metadata;
602 6 : let body_bytes = temp_metadata
603 6 : .body
604 6 : .ser()
605 6 : .expect("Cannot serialize the metadata body");
606 6 : let metadata_size = METADATA_HDR_SIZE + body_bytes.len();
607 6 : let hdr = TimelineMetadataHeader {
608 6 : size: metadata_size as u16,
609 6 : format_version: METADATA_FORMAT_VERSION,
610 6 : checksum: crc32c::crc32c(&body_bytes),
611 6 : };
612 6 : temp_metadata.hdr = hdr;
613 6 : temp_metadata
614 6 : };
615 6 : let des_metadata = TimelineMetadata::from_bytes(&metadata_ser_bytes).unwrap();
616 6 : assert_eq!(des_metadata, expected_metadata);
617 6 : }
618 : }
|
