Line data Source code
1 : //! Describes the legacy now hopefully no longer modified per-timeline metadata.
2 : //!
3 : //! It is stored in `index_part.json` managed by [`remote_timeline_client`]. For many tenants and
4 : //! their timelines, this struct and its original serialization format is still needed because
5 : //! they were written a long time ago.
6 : //!
7 : //! Instead of changing and adding versioning to this, just change [`IndexPart`] with soft json
8 : //! versioning.
9 : //!
10 : //! To clean up this module we need to migrate all index_part.json files to a later version.
11 : //! While doing this, we need to be mindful about s3 based recovery as well, so it might take
12 : //! however long we keep the old versions to be able to delete the old code. After that, we can
13 : //! remove everything else than [`TimelineMetadataBodyV2`], rename it as `TimelineMetadata` and
14 : //! move it to `index.rs`. Before doing all of this, we need to keep the structures for backwards
15 : //! compatibility.
16 : //!
17 : //! [`remote_timeline_client`]: super::remote_timeline_client
18 : //! [`IndexPart`]: super::remote_timeline_client::index::IndexPart
19 :
20 : use anyhow::ensure;
21 : use serde::{Deserialize, Serialize};
22 : use utils::bin_ser::SerializeError;
23 : use utils::{bin_ser::BeSer, id::TimelineId, lsn::Lsn};
24 :
25 : /// Use special format number to enable backward compatibility.
26 : const METADATA_FORMAT_VERSION: u16 = 4;
27 :
28 : /// Previous supported format versions.
29 : ///
30 : /// In practice, none of these should remain, all are [`METADATA_FORMAT_VERSION`], but confirming
31 : /// that requires a scrubber run which is yet to be done.
32 : const METADATA_OLD_FORMAT_VERSION: u16 = 3;
33 :
34 : /// When the file existed on disk we assumed that a write of up to METADATA_MAX_SIZE bytes is atomic.
35 : ///
36 : /// This is the same assumption that PostgreSQL makes with the control file,
37 : ///
38 : /// see PG_CONTROL_MAX_SAFE_SIZE
39 : const METADATA_MAX_SIZE: usize = 512;
40 :
41 : /// Legacy metadata stored as a component of `index_part.json` per timeline.
42 : ///
43 : /// Do not make new changes to this type or the module. In production, we have two different kinds
44 : /// of serializations of this type: bincode and json. Bincode version reflects what used to be
45 : /// stored on disk in earlier versions and does internal crc32 checksumming.
46 : ///
47 : /// This type should not implement `serde::Serialize` or `serde::Deserialize` because there would
48 : /// be a confusion whether you want the old version ([`TimelineMetadata::from_bytes`]) or the modern
49 : /// as-exists in `index_part.json` ([`self::modern_serde`]).
50 : ///
51 : /// ```compile_fail
52 : /// #[derive(serde::Serialize)]
53 : /// struct DoNotDoThis(pageserver::tenant::metadata::TimelineMetadata);
54 : /// ```
55 : ///
56 : /// ```compile_fail
57 : /// #[derive(serde::Deserialize)]
58 : /// struct NeitherDoThis(pageserver::tenant::metadata::TimelineMetadata);
59 : /// ```
60 : #[derive(Debug, Clone, PartialEq, Eq)]
61 : pub struct TimelineMetadata {
62 : hdr: TimelineMetadataHeader,
63 : body: TimelineMetadataBodyV2,
64 : }
65 :
66 0 : #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
67 : struct TimelineMetadataHeader {
68 : checksum: u32, // CRC of serialized metadata body
69 : size: u16, // size of serialized metadata
70 : format_version: u16, // metadata format version (used for compatibility checks)
71 : }
72 :
73 : impl TryFrom<&TimelineMetadataBodyV2> for TimelineMetadataHeader {
74 : type Error = Crc32CalculationFailed;
75 :
76 96 : fn try_from(value: &TimelineMetadataBodyV2) -> Result<Self, Self::Error> {
77 : #[derive(Default)]
78 : struct Crc32Sink {
79 : crc: u32,
80 : count: usize,
81 : }
82 :
83 : impl std::io::Write for Crc32Sink {
84 1576 : fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
85 1576 : self.crc = crc32c::crc32c_append(self.crc, buf);
86 1576 : self.count += buf.len();
87 1576 : Ok(buf.len())
88 1576 : }
89 :
90 0 : fn flush(&mut self) -> std::io::Result<()> {
91 0 : Ok(())
92 0 : }
93 : }
94 :
95 : // jump through hoops to calculate the crc32 so that TimelineMetadata::ne works
96 : // across serialization versions
97 96 : let mut sink = Crc32Sink::default();
98 96 : <TimelineMetadataBodyV2 as utils::bin_ser::BeSer>::ser_into(value, &mut sink)
99 96 : .map_err(Crc32CalculationFailed)?;
100 :
101 96 : let size = METADATA_HDR_SIZE + sink.count;
102 96 :
103 96 : Ok(TimelineMetadataHeader {
104 96 : checksum: sink.crc,
105 96 : size: size as u16,
106 96 : format_version: METADATA_FORMAT_VERSION,
107 96 : })
108 96 : }
109 : }
110 :
111 : #[derive(thiserror::Error, Debug)]
112 : #[error("re-serializing for crc32 failed")]
113 : struct Crc32CalculationFailed(#[source] utils::bin_ser::SerializeError);
114 :
115 : const METADATA_HDR_SIZE: usize = size_of::<TimelineMetadataHeader>();
116 :
117 504 : #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
118 : struct TimelineMetadataBodyV2 {
119 : disk_consistent_lsn: Lsn,
120 : // This is only set if we know it. We track it in memory when the page
121 : // server is running, but we only track the value corresponding to
122 : // 'last_record_lsn', not 'disk_consistent_lsn' which can lag behind by a
123 : // lot. We only store it in the metadata file when we flush *all* the
124 : // in-memory data so that 'last_record_lsn' is the same as
125 : // 'disk_consistent_lsn'. That's OK, because after page server restart, as
126 : // soon as we reprocess at least one record, we will have a valid
127 : // 'prev_record_lsn' value in memory again. This is only really needed when
128 : // doing a clean shutdown, so that there is no more WAL beyond
129 : // 'disk_consistent_lsn'
130 : prev_record_lsn: Option<Lsn>,
131 : ancestor_timeline: Option<TimelineId>,
132 : ancestor_lsn: Lsn,
133 :
134 : // The LSN at which GC was last executed. Synonym of [`Timeline::applied_gc_cutoff_lsn`].
135 : latest_gc_cutoff_lsn: Lsn,
136 :
137 : initdb_lsn: Lsn,
138 : pg_version: u32,
139 : }
140 :
141 0 : #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
142 : struct TimelineMetadataBodyV1 {
143 : disk_consistent_lsn: Lsn,
144 : // This is only set if we know it. We track it in memory when the page
145 : // server is running, but we only track the value corresponding to
146 : // 'last_record_lsn', not 'disk_consistent_lsn' which can lag behind by a
147 : // lot. We only store it in the metadata file when we flush *all* the
148 : // in-memory data so that 'last_record_lsn' is the same as
149 : // 'disk_consistent_lsn'. That's OK, because after page server restart, as
150 : // soon as we reprocess at least one record, we will have a valid
151 : // 'prev_record_lsn' value in memory again. This is only really needed when
152 : // doing a clean shutdown, so that there is no more WAL beyond
153 : // 'disk_consistent_lsn'
154 : prev_record_lsn: Option<Lsn>,
155 : ancestor_timeline: Option<TimelineId>,
156 : ancestor_lsn: Lsn,
157 : latest_gc_cutoff_lsn: Lsn,
158 : initdb_lsn: Lsn,
159 : }
160 :
161 : impl TimelineMetadata {
162 1092 : pub fn new(
163 1092 : disk_consistent_lsn: Lsn,
164 1092 : prev_record_lsn: Option<Lsn>,
165 1092 : ancestor_timeline: Option<TimelineId>,
166 1092 : ancestor_lsn: Lsn,
167 1092 : latest_gc_cutoff_lsn: Lsn,
168 1092 : initdb_lsn: Lsn,
169 1092 : pg_version: u32,
170 1092 : ) -> Self {
171 1092 : Self {
172 1092 : hdr: TimelineMetadataHeader {
173 1092 : checksum: 0,
174 1092 : size: 0,
175 1092 : format_version: METADATA_FORMAT_VERSION,
176 1092 : },
177 1092 : body: TimelineMetadataBodyV2 {
178 1092 : disk_consistent_lsn,
179 1092 : prev_record_lsn,
180 1092 : ancestor_timeline,
181 1092 : ancestor_lsn,
182 1092 : latest_gc_cutoff_lsn,
183 1092 : initdb_lsn,
184 1092 : pg_version,
185 1092 : },
186 1092 : }
187 1092 : }
188 :
189 : #[cfg(test)]
190 24 : pub(crate) fn with_recalculated_checksum(mut self) -> anyhow::Result<Self> {
191 24 : self.hdr = TimelineMetadataHeader::try_from(&self.body)?;
192 24 : Ok(self)
193 24 : }
194 :
195 4 : fn upgrade_timeline_metadata(metadata_bytes: &[u8]) -> anyhow::Result<Self> {
196 4 : let mut hdr = TimelineMetadataHeader::des(&metadata_bytes[0..METADATA_HDR_SIZE])?;
197 :
198 : // backward compatible only up to this version
199 4 : ensure!(
200 4 : hdr.format_version == METADATA_OLD_FORMAT_VERSION,
201 0 : "unsupported metadata format version {}",
202 : hdr.format_version
203 : );
204 :
205 4 : let metadata_size = hdr.size as usize;
206 :
207 4 : let body: TimelineMetadataBodyV1 =
208 4 : TimelineMetadataBodyV1::des(&metadata_bytes[METADATA_HDR_SIZE..metadata_size])?;
209 :
210 4 : let body = TimelineMetadataBodyV2 {
211 4 : disk_consistent_lsn: body.disk_consistent_lsn,
212 4 : prev_record_lsn: body.prev_record_lsn,
213 4 : ancestor_timeline: body.ancestor_timeline,
214 4 : ancestor_lsn: body.ancestor_lsn,
215 4 : latest_gc_cutoff_lsn: body.latest_gc_cutoff_lsn,
216 4 : initdb_lsn: body.initdb_lsn,
217 4 : pg_version: 14, // All timelines created before this version had pg_version 14
218 4 : };
219 4 :
220 4 : hdr.format_version = METADATA_FORMAT_VERSION;
221 4 :
222 4 : Ok(Self { hdr, body })
223 4 : }
224 :
225 244 : pub fn from_bytes(metadata_bytes: &[u8]) -> anyhow::Result<Self> {
226 244 : ensure!(
227 244 : metadata_bytes.len() == METADATA_MAX_SIZE,
228 0 : "metadata bytes size is wrong"
229 : );
230 244 : let hdr = TimelineMetadataHeader::des(&metadata_bytes[0..METADATA_HDR_SIZE])?;
231 :
232 244 : let metadata_size = hdr.size as usize;
233 244 : ensure!(
234 244 : metadata_size <= METADATA_MAX_SIZE,
235 0 : "corrupted metadata file"
236 : );
237 244 : let calculated_checksum = crc32c::crc32c(&metadata_bytes[METADATA_HDR_SIZE..metadata_size]);
238 244 : ensure!(
239 244 : hdr.checksum == calculated_checksum,
240 0 : "metadata checksum mismatch"
241 : );
242 :
243 244 : if hdr.format_version != METADATA_FORMAT_VERSION {
244 : // If metadata has the old format,
245 : // upgrade it and return the result
246 4 : TimelineMetadata::upgrade_timeline_metadata(metadata_bytes)
247 : } else {
248 240 : let body =
249 240 : TimelineMetadataBodyV2::des(&metadata_bytes[METADATA_HDR_SIZE..metadata_size])?;
250 240 : ensure!(
251 240 : body.disk_consistent_lsn.is_aligned(),
252 0 : "disk_consistent_lsn is not aligned"
253 : );
254 240 : Ok(TimelineMetadata { hdr, body })
255 : }
256 244 : }
257 :
258 180 : pub fn to_bytes(&self) -> Result<Vec<u8>, SerializeError> {
259 180 : let body_bytes = self.body.ser()?;
260 180 : let metadata_size = METADATA_HDR_SIZE + body_bytes.len();
261 180 : let hdr = TimelineMetadataHeader {
262 180 : size: metadata_size as u16,
263 180 : format_version: METADATA_FORMAT_VERSION,
264 180 : checksum: crc32c::crc32c(&body_bytes),
265 180 : };
266 180 : let hdr_bytes = hdr.ser()?;
267 180 : let mut metadata_bytes = vec![0u8; METADATA_MAX_SIZE];
268 180 : metadata_bytes[0..METADATA_HDR_SIZE].copy_from_slice(&hdr_bytes);
269 180 : metadata_bytes[METADATA_HDR_SIZE..metadata_size].copy_from_slice(&body_bytes);
270 180 : Ok(metadata_bytes)
271 180 : }
272 :
273 : /// [`Lsn`] that corresponds to the corresponding timeline directory
274 : /// contents, stored locally in the pageserver workdir.
275 43644 : pub fn disk_consistent_lsn(&self) -> Lsn {
276 43644 : self.body.disk_consistent_lsn
277 43644 : }
278 :
279 896 : pub fn prev_record_lsn(&self) -> Option<Lsn> {
280 896 : self.body.prev_record_lsn
281 896 : }
282 :
283 6366 : pub fn ancestor_timeline(&self) -> Option<TimelineId> {
284 6366 : self.body.ancestor_timeline
285 6366 : }
286 :
287 1360 : pub fn ancestor_lsn(&self) -> Lsn {
288 1360 : self.body.ancestor_lsn
289 1360 : }
290 :
291 : /// When reparenting, the `ancestor_lsn` does not change.
292 : ///
293 : /// Returns true if anything was changed.
294 0 : pub fn reparent(&mut self, timeline: &TimelineId) {
295 0 : assert!(self.body.ancestor_timeline.is_some());
296 : // no assertion for redoing this: it's fine, we may have to repeat this multiple times over
297 0 : self.body.ancestor_timeline = Some(*timeline);
298 0 : }
299 :
300 : /// Returns true if anything was changed
301 0 : pub fn detach_from_ancestor(&mut self, branchpoint: &(TimelineId, Lsn)) {
302 0 : if let Some(ancestor) = self.body.ancestor_timeline {
303 0 : assert_eq!(ancestor, branchpoint.0);
304 0 : }
305 0 : if self.body.ancestor_lsn != Lsn(0) {
306 0 : assert_eq!(self.body.ancestor_lsn, branchpoint.1);
307 0 : }
308 0 : self.body.ancestor_timeline = None;
309 0 : self.body.ancestor_lsn = Lsn(0);
310 0 : }
311 :
312 896 : pub fn latest_gc_cutoff_lsn(&self) -> Lsn {
313 896 : self.body.latest_gc_cutoff_lsn
314 896 : }
315 :
316 896 : pub fn initdb_lsn(&self) -> Lsn {
317 896 : self.body.initdb_lsn
318 896 : }
319 :
320 904 : pub fn pg_version(&self) -> u32 {
321 904 : self.body.pg_version
322 904 : }
323 :
324 : // Checksums make it awkward to build a valid instance by hand. This helper
325 : // provides a TimelineMetadata with a valid checksum in its header.
326 168 : pub fn example() -> Self {
327 168 : let instance = Self::new(
328 168 : "0/16960E8".parse::<Lsn>().unwrap(),
329 168 : None,
330 168 : None,
331 168 : Lsn::from_hex("00000000").unwrap(),
332 168 : Lsn::from_hex("00000000").unwrap(),
333 168 : Lsn::from_hex("00000000").unwrap(),
334 168 : 0,
335 168 : );
336 168 : let bytes = instance.to_bytes().unwrap();
337 168 : Self::from_bytes(&bytes).unwrap()
338 168 : }
339 :
340 2448 : pub(crate) fn apply(&mut self, update: &MetadataUpdate) {
341 2448 : self.body.disk_consistent_lsn = update.disk_consistent_lsn;
342 2448 : self.body.prev_record_lsn = update.prev_record_lsn;
343 2448 : self.body.latest_gc_cutoff_lsn = update.latest_gc_cutoff_lsn;
344 2448 : }
345 : }
346 :
347 : pub(crate) mod modern_serde {
348 : use super::{TimelineMetadata, TimelineMetadataBodyV2, TimelineMetadataHeader};
349 : use serde::{Deserialize, Serialize};
350 :
351 104 : pub(crate) fn deserialize<'de, D>(deserializer: D) -> Result<TimelineMetadata, D::Error>
352 104 : where
353 104 : D: serde::de::Deserializer<'de>,
354 104 : {
355 : // for legacy reasons versions 1-5 had TimelineMetadata serialized as a Vec<u8> field with
356 : // BeSer.
357 : struct Visitor;
358 :
359 : impl<'d> serde::de::Visitor<'d> for Visitor {
360 : type Value = TimelineMetadata;
361 :
362 0 : fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
363 0 : f.write_str("BeSer bytes or json structure")
364 0 : }
365 :
366 32 : fn visit_seq<A>(self, seq: A) -> Result<Self::Value, A::Error>
367 32 : where
368 32 : A: serde::de::SeqAccess<'d>,
369 32 : {
370 : use serde::de::Error;
371 32 : let de = serde::de::value::SeqAccessDeserializer::new(seq);
372 32 : Vec::<u8>::deserialize(de)
373 32 : .map(|v| TimelineMetadata::from_bytes(&v).map_err(A::Error::custom))?
374 32 : }
375 :
376 72 : fn visit_map<A>(self, map: A) -> Result<Self::Value, A::Error>
377 72 : where
378 72 : A: serde::de::MapAccess<'d>,
379 72 : {
380 : use serde::de::Error;
381 :
382 72 : let de = serde::de::value::MapAccessDeserializer::new(map);
383 72 : let body = TimelineMetadataBodyV2::deserialize(de)?;
384 72 : let hdr = TimelineMetadataHeader::try_from(&body).map_err(A::Error::custom)?;
385 :
386 72 : Ok(TimelineMetadata { hdr, body })
387 72 : }
388 : }
389 :
390 104 : deserializer.deserialize_any(Visitor)
391 104 : }
392 :
393 6094 : pub(crate) fn serialize<S>(
394 6094 : metadata: &TimelineMetadata,
395 6094 : serializer: S,
396 6094 : ) -> Result<S::Ok, S::Error>
397 6094 : where
398 6094 : S: serde::Serializer,
399 6094 : {
400 6094 : // header is not needed, upon reading we've upgraded all v1 to v2
401 6094 : metadata.body.serialize(serializer)
402 6094 : }
403 :
404 : #[test]
405 4 : fn deserializes_bytes_as_well_as_equivalent_body_v2() {
406 4 : #[derive(serde::Deserialize, serde::Serialize)]
407 : struct Wrapper(
408 : #[serde(deserialize_with = "deserialize", serialize_with = "serialize")]
409 : TimelineMetadata,
410 : );
411 :
412 4 : let too_many_bytes = "[216,111,252,208,0,54,0,4,0,0,0,0,1,73,253,144,1,0,0,0,0,1,73,253,24,0,0,0,0,0,0,0,0,0,0,0,0,0,1,73,253,24,0,0,0,0,1,73,253,24,0,0,0,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]";
413 4 :
414 4 : let wrapper_from_bytes = serde_json::from_str::<Wrapper>(too_many_bytes).unwrap();
415 4 :
416 4 : let serialized = serde_json::to_value(&wrapper_from_bytes).unwrap();
417 4 :
418 4 : assert_eq!(
419 4 : serialized,
420 4 : serde_json::json! {{
421 4 : "disk_consistent_lsn": "0/149FD90",
422 4 : "prev_record_lsn": "0/149FD18",
423 4 : "ancestor_timeline": null,
424 4 : "ancestor_lsn": "0/0",
425 4 : "latest_gc_cutoff_lsn": "0/149FD18",
426 4 : "initdb_lsn": "0/149FD18",
427 4 : "pg_version": 15
428 4 : }}
429 4 : );
430 :
431 4 : let wrapper_from_json = serde_json::value::from_value::<Wrapper>(serialized).unwrap();
432 4 :
433 4 : assert_eq!(wrapper_from_bytes.0, wrapper_from_json.0);
434 4 : }
435 : }
436 :
437 : /// Parts of the metadata which are regularly modified.
438 : pub(crate) struct MetadataUpdate {
439 : disk_consistent_lsn: Lsn,
440 : prev_record_lsn: Option<Lsn>,
441 : latest_gc_cutoff_lsn: Lsn,
442 : }
443 :
444 : impl MetadataUpdate {
445 2448 : pub(crate) fn new(
446 2448 : disk_consistent_lsn: Lsn,
447 2448 : prev_record_lsn: Option<Lsn>,
448 2448 : latest_gc_cutoff_lsn: Lsn,
449 2448 : ) -> Self {
450 2448 : Self {
451 2448 : disk_consistent_lsn,
452 2448 : prev_record_lsn,
453 2448 : latest_gc_cutoff_lsn,
454 2448 : }
455 2448 : }
456 : }
457 :
458 : #[cfg(test)]
459 : mod tests {
460 : use super::*;
461 : use crate::tenant::harness::TIMELINE_ID;
462 :
463 : #[test]
464 4 : fn metadata_serializes_correctly() {
465 4 : let original_metadata = TimelineMetadata::new(
466 4 : Lsn(0x200),
467 4 : Some(Lsn(0x100)),
468 4 : Some(TIMELINE_ID),
469 4 : Lsn(0),
470 4 : Lsn(0),
471 4 : Lsn(0),
472 4 : // Any version will do here, so use the default
473 4 : crate::DEFAULT_PG_VERSION,
474 4 : );
475 4 :
476 4 : let metadata_bytes = original_metadata
477 4 : .to_bytes()
478 4 : .expect("Should serialize correct metadata to bytes");
479 4 :
480 4 : let deserialized_metadata = TimelineMetadata::from_bytes(&metadata_bytes)
481 4 : .expect("Should deserialize its own bytes");
482 4 :
483 4 : assert_eq!(
484 : deserialized_metadata.body, original_metadata.body,
485 0 : "Metadata that was serialized to bytes and deserialized back should not change"
486 : );
487 4 : }
488 :
489 : // Generate old version metadata and read it with current code.
490 : // Ensure that it is upgraded correctly
491 : #[test]
492 4 : fn test_metadata_upgrade() {
493 : #[derive(Debug, Clone, PartialEq, Eq)]
494 : struct TimelineMetadataV1 {
495 : hdr: TimelineMetadataHeader,
496 : body: TimelineMetadataBodyV1,
497 : }
498 :
499 4 : let metadata_v1 = TimelineMetadataV1 {
500 4 : hdr: TimelineMetadataHeader {
501 4 : checksum: 0,
502 4 : size: 0,
503 4 : format_version: METADATA_OLD_FORMAT_VERSION,
504 4 : },
505 4 : body: TimelineMetadataBodyV1 {
506 4 : disk_consistent_lsn: Lsn(0x200),
507 4 : prev_record_lsn: Some(Lsn(0x100)),
508 4 : ancestor_timeline: Some(TIMELINE_ID),
509 4 : ancestor_lsn: Lsn(0),
510 4 : latest_gc_cutoff_lsn: Lsn(0),
511 4 : initdb_lsn: Lsn(0),
512 4 : },
513 4 : };
514 :
515 : impl TimelineMetadataV1 {
516 4 : pub fn to_bytes(&self) -> anyhow::Result<Vec<u8>> {
517 4 : let body_bytes = self.body.ser()?;
518 4 : let metadata_size = METADATA_HDR_SIZE + body_bytes.len();
519 4 : let hdr = TimelineMetadataHeader {
520 4 : size: metadata_size as u16,
521 4 : format_version: METADATA_OLD_FORMAT_VERSION,
522 4 : checksum: crc32c::crc32c(&body_bytes),
523 4 : };
524 4 : let hdr_bytes = hdr.ser()?;
525 4 : let mut metadata_bytes = vec![0u8; METADATA_MAX_SIZE];
526 4 : metadata_bytes[0..METADATA_HDR_SIZE].copy_from_slice(&hdr_bytes);
527 4 : metadata_bytes[METADATA_HDR_SIZE..metadata_size].copy_from_slice(&body_bytes);
528 4 : Ok(metadata_bytes)
529 4 : }
530 : }
531 :
532 4 : let metadata_bytes = metadata_v1
533 4 : .to_bytes()
534 4 : .expect("Should serialize correct metadata to bytes");
535 4 :
536 4 : // This should deserialize to the latest version format
537 4 : let deserialized_metadata = TimelineMetadata::from_bytes(&metadata_bytes)
538 4 : .expect("Should deserialize its own bytes");
539 4 :
540 4 : let expected_metadata = TimelineMetadata::new(
541 4 : Lsn(0x200),
542 4 : Some(Lsn(0x100)),
543 4 : Some(TIMELINE_ID),
544 4 : Lsn(0),
545 4 : Lsn(0),
546 4 : Lsn(0),
547 4 : 14, // All timelines created before this version had pg_version 14
548 4 : );
549 4 :
550 4 : assert_eq!(
551 : deserialized_metadata.body, expected_metadata.body,
552 0 : "Metadata of the old version {} should be upgraded to the latest version {}",
553 : METADATA_OLD_FORMAT_VERSION, METADATA_FORMAT_VERSION
554 : );
555 4 : }
556 :
557 : #[test]
558 4 : fn test_metadata_bincode_serde_ensure_roundtrip() {
559 4 : let original_metadata = TimelineMetadata::new(
560 4 : Lsn(0x200),
561 4 : Some(Lsn(0x100)),
562 4 : Some(TIMELINE_ID),
563 4 : Lsn(0),
564 4 : Lsn(0),
565 4 : Lsn(0),
566 4 : // Any version will do here, so use the default
567 4 : crate::DEFAULT_PG_VERSION,
568 4 : );
569 4 : let expected_bytes = vec![
570 4 : /* TimelineMetadataHeader */
571 4 : 74, 104, 158, 105, 0, 70, 0, 4, // checksum, size, format_version (4 + 2 + 2)
572 4 : /* TimelineMetadataBodyV2 */
573 4 : 0, 0, 0, 0, 0, 0, 2, 0, // disk_consistent_lsn (8 bytes)
574 4 : 1, 0, 0, 0, 0, 0, 0, 1, 0, // prev_record_lsn (9 bytes)
575 4 : 1, 17, 34, 51, 68, 85, 102, 119, 136, 17, 34, 51, 68, 85, 102, 119,
576 4 : 136, // ancestor_timeline (17 bytes)
577 4 : 0, 0, 0, 0, 0, 0, 0, 0, // ancestor_lsn (8 bytes)
578 4 : 0, 0, 0, 0, 0, 0, 0, 0, // latest_gc_cutoff_lsn (8 bytes)
579 4 : 0, 0, 0, 0, 0, 0, 0, 0, // initdb_lsn (8 bytes)
580 4 : 0, 0, 0, 16, // pg_version (4 bytes)
581 4 : /* padding bytes */
582 4 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
583 4 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
584 4 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
585 4 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
586 4 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
587 4 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
588 4 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
589 4 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
590 4 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
591 4 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
592 4 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
593 4 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
594 4 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
595 4 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
596 4 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
597 4 : 0, 0, 0, 0, 0, 0, 0,
598 4 : ];
599 4 : let metadata_ser_bytes = original_metadata.to_bytes().unwrap();
600 4 : assert_eq!(metadata_ser_bytes, expected_bytes);
601 :
602 4 : let expected_metadata = {
603 4 : let mut temp_metadata = original_metadata;
604 4 : let body_bytes = temp_metadata
605 4 : .body
606 4 : .ser()
607 4 : .expect("Cannot serialize the metadata body");
608 4 : let metadata_size = METADATA_HDR_SIZE + body_bytes.len();
609 4 : let hdr = TimelineMetadataHeader {
610 4 : size: metadata_size as u16,
611 4 : format_version: METADATA_FORMAT_VERSION,
612 4 : checksum: crc32c::crc32c(&body_bytes),
613 4 : };
614 4 : temp_metadata.hdr = hdr;
615 4 : temp_metadata
616 4 : };
617 4 : let des_metadata = TimelineMetadata::from_bytes(&metadata_ser_bytes).unwrap();
618 4 : assert_eq!(des_metadata, expected_metadata);
619 4 : }
620 : }
|
