Line data Source code
1 : //! Describes the legacy now hopefully no longer modified per-timeline metadata.
2 : //!
3 : //! It is stored in `index_part.json` managed by [`remote_timeline_client`]. For many tenants and
4 : //! their timelines, this struct and its original serialization format is still needed because
5 : //! they were written a long time ago.
6 : //!
7 : //! Instead of changing and adding versioning to this, just change [`IndexPart`] with soft json
8 : //! versioning.
9 : //!
10 : //! To clean up this module we need to migrate all index_part.json files to a later version.
11 : //! While doing this, we need to be mindful about s3 based recovery as well, so it might take
12 : //! however long we keep the old versions to be able to delete the old code. After that, we can
13 : //! remove everything else than [`TimelineMetadataBodyV2`], rename it as `TimelineMetadata` and
14 : //! move it to `index.rs`. Before doing all of this, we need to keep the structures for backwards
15 : //! compatibility.
16 : //!
17 : //! [`remote_timeline_client`]: super::remote_timeline_client
18 : //! [`IndexPart`]: super::remote_timeline_client::index::IndexPart
19 :
20 : use anyhow::ensure;
21 : use serde::{Deserialize, Serialize};
22 : use utils::bin_ser::{BeSer, SerializeError};
23 : use utils::id::TimelineId;
24 : use utils::lsn::Lsn;
25 :
26 : /// Use special format number to enable backward compatibility.
27 : const METADATA_FORMAT_VERSION: u16 = 4;
28 :
29 : /// Previous supported format versions.
30 : ///
31 : /// In practice, none of these should remain, all are [`METADATA_FORMAT_VERSION`], but confirming
32 : /// that requires a scrubber run which is yet to be done.
33 : const METADATA_OLD_FORMAT_VERSION: u16 = 3;
34 :
35 : /// When the file existed on disk we assumed that a write of up to METADATA_MAX_SIZE bytes is atomic.
36 : ///
37 : /// This is the same assumption that PostgreSQL makes with the control file,
38 : ///
39 : /// see PG_CONTROL_MAX_SAFE_SIZE
40 : const METADATA_MAX_SIZE: usize = 512;
41 :
42 : /// Legacy metadata stored as a component of `index_part.json` per timeline.
43 : ///
44 : /// Do not make new changes to this type or the module. In production, we have two different kinds
45 : /// of serializations of this type: bincode and json. Bincode version reflects what used to be
46 : /// stored on disk in earlier versions and does internal crc32 checksumming.
47 : ///
48 : /// This type should not implement `serde::Serialize` or `serde::Deserialize` because there would
49 : /// be a confusion whether you want the old version ([`TimelineMetadata::from_bytes`]) or the modern
50 : /// as-exists in `index_part.json` ([`self::modern_serde`]).
51 : ///
52 : /// ```compile_fail
53 : /// #[derive(serde::Serialize)]
54 : /// struct DoNotDoThis(pageserver::tenant::metadata::TimelineMetadata);
55 : /// ```
56 : ///
57 : /// ```compile_fail
58 : /// #[derive(serde::Deserialize)]
59 : /// struct NeitherDoThis(pageserver::tenant::metadata::TimelineMetadata);
60 : /// ```
61 : #[derive(Debug, Clone, PartialEq, Eq)]
62 : pub struct TimelineMetadata {
63 : hdr: TimelineMetadataHeader,
64 : body: TimelineMetadataBodyV2,
65 : }
66 :
67 0 : #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
68 : struct TimelineMetadataHeader {
69 : checksum: u32, // CRC of serialized metadata body
70 : size: u16, // size of serialized metadata
71 : format_version: u16, // metadata format version (used for compatibility checks)
72 : }
73 :
74 : impl TryFrom<&TimelineMetadataBodyV2> for TimelineMetadataHeader {
75 : type Error = Crc32CalculationFailed;
76 :
77 312 : fn try_from(value: &TimelineMetadataBodyV2) -> Result<Self, Self::Error> {
78 : #[derive(Default)]
79 : struct Crc32Sink {
80 : crc: u32,
81 : count: usize,
82 : }
83 :
84 : impl std::io::Write for Crc32Sink {
85 5304 : fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
86 5304 : self.crc = crc32c::crc32c_append(self.crc, buf);
87 5304 : self.count += buf.len();
88 5304 : Ok(buf.len())
89 5304 : }
90 :
91 0 : fn flush(&mut self) -> std::io::Result<()> {
92 0 : Ok(())
93 0 : }
94 : }
95 :
96 : // jump through hoops to calculate the crc32 so that TimelineMetadata::ne works
97 : // across serialization versions
98 312 : let mut sink = Crc32Sink::default();
99 312 : <TimelineMetadataBodyV2 as utils::bin_ser::BeSer>::ser_into(value, &mut sink)
100 312 : .map_err(Crc32CalculationFailed)?;
101 :
102 312 : let size = METADATA_HDR_SIZE + sink.count;
103 312 :
104 312 : Ok(TimelineMetadataHeader {
105 312 : checksum: sink.crc,
106 312 : size: size as u16,
107 312 : format_version: METADATA_FORMAT_VERSION,
108 312 : })
109 312 : }
110 : }
111 :
112 : #[derive(thiserror::Error, Debug)]
113 : #[error("re-serializing for crc32 failed")]
114 : struct Crc32CalculationFailed(#[source] utils::bin_ser::SerializeError);
115 :
116 : const METADATA_HDR_SIZE: usize = size_of::<TimelineMetadataHeader>();
117 :
118 1596 : #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
119 : struct TimelineMetadataBodyV2 {
120 : disk_consistent_lsn: Lsn,
121 : // This is only set if we know it. We track it in memory when the page
122 : // server is running, but we only track the value corresponding to
123 : // 'last_record_lsn', not 'disk_consistent_lsn' which can lag behind by a
124 : // lot. We only store it in the metadata file when we flush *all* the
125 : // in-memory data so that 'last_record_lsn' is the same as
126 : // 'disk_consistent_lsn'. That's OK, because after page server restart, as
127 : // soon as we reprocess at least one record, we will have a valid
128 : // 'prev_record_lsn' value in memory again. This is only really needed when
129 : // doing a clean shutdown, so that there is no more WAL beyond
130 : // 'disk_consistent_lsn'
131 : prev_record_lsn: Option<Lsn>,
132 : ancestor_timeline: Option<TimelineId>,
133 : ancestor_lsn: Lsn,
134 :
135 : // The LSN at which GC was last executed. Synonym of [`Timeline::applied_gc_cutoff_lsn`].
136 : latest_gc_cutoff_lsn: Lsn,
137 :
138 : initdb_lsn: Lsn,
139 : pg_version: u32,
140 : }
141 :
142 0 : #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
143 : struct TimelineMetadataBodyV1 {
144 : disk_consistent_lsn: Lsn,
145 : // This is only set if we know it. We track it in memory when the page
146 : // server is running, but we only track the value corresponding to
147 : // 'last_record_lsn', not 'disk_consistent_lsn' which can lag behind by a
148 : // lot. We only store it in the metadata file when we flush *all* the
149 : // in-memory data so that 'last_record_lsn' is the same as
150 : // 'disk_consistent_lsn'. That's OK, because after page server restart, as
151 : // soon as we reprocess at least one record, we will have a valid
152 : // 'prev_record_lsn' value in memory again. This is only really needed when
153 : // doing a clean shutdown, so that there is no more WAL beyond
154 : // 'disk_consistent_lsn'
155 : prev_record_lsn: Option<Lsn>,
156 : ancestor_timeline: Option<TimelineId>,
157 : ancestor_lsn: Lsn,
158 : latest_gc_cutoff_lsn: Lsn,
159 : initdb_lsn: Lsn,
160 : }
161 :
162 : impl TimelineMetadata {
163 3384 : pub fn new(
164 3384 : disk_consistent_lsn: Lsn,
165 3384 : prev_record_lsn: Option<Lsn>,
166 3384 : ancestor_timeline: Option<TimelineId>,
167 3384 : ancestor_lsn: Lsn,
168 3384 : latest_gc_cutoff_lsn: Lsn,
169 3384 : initdb_lsn: Lsn,
170 3384 : pg_version: u32,
171 3384 : ) -> Self {
172 3384 : Self {
173 3384 : hdr: TimelineMetadataHeader {
174 3384 : checksum: 0,
175 3384 : size: 0,
176 3384 : format_version: METADATA_FORMAT_VERSION,
177 3384 : },
178 3384 : body: TimelineMetadataBodyV2 {
179 3384 : disk_consistent_lsn,
180 3384 : prev_record_lsn,
181 3384 : ancestor_timeline,
182 3384 : ancestor_lsn,
183 3384 : latest_gc_cutoff_lsn,
184 3384 : initdb_lsn,
185 3384 : pg_version,
186 3384 : },
187 3384 : }
188 3384 : }
189 :
190 : #[cfg(test)]
191 84 : pub(crate) fn with_recalculated_checksum(mut self) -> anyhow::Result<Self> {
192 84 : self.hdr = TimelineMetadataHeader::try_from(&self.body)?;
193 84 : Ok(self)
194 84 : }
195 :
196 12 : fn upgrade_timeline_metadata(metadata_bytes: &[u8]) -> anyhow::Result<Self> {
197 12 : let mut hdr = TimelineMetadataHeader::des(&metadata_bytes[0..METADATA_HDR_SIZE])?;
198 :
199 : // backward compatible only up to this version
200 12 : ensure!(
201 12 : hdr.format_version == METADATA_OLD_FORMAT_VERSION,
202 0 : "unsupported metadata format version {}",
203 : hdr.format_version
204 : );
205 :
206 12 : let metadata_size = hdr.size as usize;
207 :
208 12 : let body: TimelineMetadataBodyV1 =
209 12 : TimelineMetadataBodyV1::des(&metadata_bytes[METADATA_HDR_SIZE..metadata_size])?;
210 :
211 12 : let body = TimelineMetadataBodyV2 {
212 12 : disk_consistent_lsn: body.disk_consistent_lsn,
213 12 : prev_record_lsn: body.prev_record_lsn,
214 12 : ancestor_timeline: body.ancestor_timeline,
215 12 : ancestor_lsn: body.ancestor_lsn,
216 12 : latest_gc_cutoff_lsn: body.latest_gc_cutoff_lsn,
217 12 : initdb_lsn: body.initdb_lsn,
218 12 : pg_version: 14, // All timelines created before this version had pg_version 14
219 12 : };
220 12 :
221 12 : hdr.format_version = METADATA_FORMAT_VERSION;
222 12 :
223 12 : Ok(Self { hdr, body })
224 12 : }
225 :
226 732 : pub fn from_bytes(metadata_bytes: &[u8]) -> anyhow::Result<Self> {
227 732 : ensure!(
228 732 : metadata_bytes.len() == METADATA_MAX_SIZE,
229 0 : "metadata bytes size is wrong"
230 : );
231 732 : let hdr = TimelineMetadataHeader::des(&metadata_bytes[0..METADATA_HDR_SIZE])?;
232 :
233 732 : let metadata_size = hdr.size as usize;
234 732 : ensure!(
235 732 : metadata_size <= METADATA_MAX_SIZE,
236 0 : "corrupted metadata file"
237 : );
238 732 : let calculated_checksum = crc32c::crc32c(&metadata_bytes[METADATA_HDR_SIZE..metadata_size]);
239 732 : ensure!(
240 732 : hdr.checksum == calculated_checksum,
241 0 : "metadata checksum mismatch"
242 : );
243 :
244 732 : if hdr.format_version != METADATA_FORMAT_VERSION {
245 : // If metadata has the old format,
246 : // upgrade it and return the result
247 12 : TimelineMetadata::upgrade_timeline_metadata(metadata_bytes)
248 : } else {
249 720 : let body =
250 720 : TimelineMetadataBodyV2::des(&metadata_bytes[METADATA_HDR_SIZE..metadata_size])?;
251 720 : ensure!(
252 720 : body.disk_consistent_lsn.is_aligned(),
253 0 : "disk_consistent_lsn is not aligned"
254 : );
255 720 : Ok(TimelineMetadata { hdr, body })
256 : }
257 732 : }
258 :
259 540 : pub fn to_bytes(&self) -> Result<Vec<u8>, SerializeError> {
260 540 : let body_bytes = self.body.ser()?;
261 540 : let metadata_size = METADATA_HDR_SIZE + body_bytes.len();
262 540 : let hdr = TimelineMetadataHeader {
263 540 : size: metadata_size as u16,
264 540 : format_version: METADATA_FORMAT_VERSION,
265 540 : checksum: crc32c::crc32c(&body_bytes),
266 540 : };
267 540 : let hdr_bytes = hdr.ser()?;
268 540 : let mut metadata_bytes = vec![0u8; METADATA_MAX_SIZE];
269 540 : metadata_bytes[0..METADATA_HDR_SIZE].copy_from_slice(&hdr_bytes);
270 540 : metadata_bytes[METADATA_HDR_SIZE..metadata_size].copy_from_slice(&body_bytes);
271 540 : Ok(metadata_bytes)
272 540 : }
273 :
274 : /// [`Lsn`] that corresponds to the corresponding timeline directory
275 : /// contents, stored locally in the pageserver workdir.
276 129399 : pub fn disk_consistent_lsn(&self) -> Lsn {
277 129399 : self.body.disk_consistent_lsn
278 129399 : }
279 :
280 2784 : pub fn prev_record_lsn(&self) -> Option<Lsn> {
281 2784 : self.body.prev_record_lsn
282 2784 : }
283 :
284 19270 : pub fn ancestor_timeline(&self) -> Option<TimelineId> {
285 19270 : self.body.ancestor_timeline
286 19270 : }
287 :
288 4212 : pub fn ancestor_lsn(&self) -> Lsn {
289 4212 : self.body.ancestor_lsn
290 4212 : }
291 :
292 : /// When reparenting, the `ancestor_lsn` does not change.
293 : ///
294 : /// Returns true if anything was changed.
295 0 : pub fn reparent(&mut self, timeline: &TimelineId) {
296 0 : assert!(self.body.ancestor_timeline.is_some());
297 : // no assertion for redoing this: it's fine, we may have to repeat this multiple times over
298 0 : self.body.ancestor_timeline = Some(*timeline);
299 0 : }
300 :
301 : /// Returns true if anything was changed
302 0 : pub fn detach_from_ancestor(&mut self, branchpoint: &(TimelineId, Lsn)) {
303 0 : // Detaching from ancestor now doesn't always detach directly to the direct ancestor, but we
304 0 : // ensure the LSN is the same. So we don't check the timeline ID.
305 0 : if self.body.ancestor_lsn != Lsn(0) {
306 0 : assert_eq!(self.body.ancestor_lsn, branchpoint.1);
307 0 : }
308 0 : self.body.ancestor_timeline = None;
309 0 : self.body.ancestor_lsn = Lsn(0);
310 0 : }
311 :
312 2784 : pub fn latest_gc_cutoff_lsn(&self) -> Lsn {
313 2784 : self.body.latest_gc_cutoff_lsn
314 2784 : }
315 :
316 2784 : pub fn initdb_lsn(&self) -> Lsn {
317 2784 : self.body.initdb_lsn
318 2784 : }
319 :
320 2808 : pub fn pg_version(&self) -> u32 {
321 2808 : self.body.pg_version
322 2808 : }
323 :
324 : // Checksums make it awkward to build a valid instance by hand. This helper
325 : // provides a TimelineMetadata with a valid checksum in its header.
326 504 : pub fn example() -> Self {
327 504 : let instance = Self::new(
328 504 : "0/16960E8".parse::<Lsn>().unwrap(),
329 504 : None,
330 504 : None,
331 504 : Lsn::from_hex("00000000").unwrap(),
332 504 : Lsn::from_hex("00000000").unwrap(),
333 504 : Lsn::from_hex("00000000").unwrap(),
334 504 : 0,
335 504 : );
336 504 : let bytes = instance.to_bytes().unwrap();
337 504 : Self::from_bytes(&bytes).unwrap()
338 504 : }
339 :
340 7404 : pub(crate) fn apply(&mut self, update: &MetadataUpdate) {
341 7404 : self.body.disk_consistent_lsn = update.disk_consistent_lsn;
342 7404 : self.body.prev_record_lsn = update.prev_record_lsn;
343 7404 : self.body.latest_gc_cutoff_lsn = update.latest_gc_cutoff_lsn;
344 7404 : }
345 : }
346 :
347 : pub(crate) mod modern_serde {
348 : use serde::{Deserialize, Serialize};
349 :
350 : use super::{TimelineMetadata, TimelineMetadataBodyV2, TimelineMetadataHeader};
351 :
352 324 : pub(crate) fn deserialize<'de, D>(deserializer: D) -> Result<TimelineMetadata, D::Error>
353 324 : where
354 324 : D: serde::de::Deserializer<'de>,
355 324 : {
356 : // for legacy reasons versions 1-5 had TimelineMetadata serialized as a Vec<u8> field with
357 : // BeSer.
358 : struct Visitor;
359 :
360 : impl<'d> serde::de::Visitor<'d> for Visitor {
361 : type Value = TimelineMetadata;
362 :
363 0 : fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
364 0 : f.write_str("BeSer bytes or json structure")
365 0 : }
366 :
367 96 : fn visit_seq<A>(self, seq: A) -> Result<Self::Value, A::Error>
368 96 : where
369 96 : A: serde::de::SeqAccess<'d>,
370 96 : {
371 : use serde::de::Error;
372 96 : let de = serde::de::value::SeqAccessDeserializer::new(seq);
373 96 : Vec::<u8>::deserialize(de)
374 96 : .map(|v| TimelineMetadata::from_bytes(&v).map_err(A::Error::custom))?
375 96 : }
376 :
377 228 : fn visit_map<A>(self, map: A) -> Result<Self::Value, A::Error>
378 228 : where
379 228 : A: serde::de::MapAccess<'d>,
380 228 : {
381 : use serde::de::Error;
382 :
383 228 : let de = serde::de::value::MapAccessDeserializer::new(map);
384 228 : let body = TimelineMetadataBodyV2::deserialize(de)?;
385 228 : let hdr = TimelineMetadataHeader::try_from(&body).map_err(A::Error::custom)?;
386 :
387 228 : Ok(TimelineMetadata { hdr, body })
388 228 : }
389 : }
390 :
391 324 : deserializer.deserialize_any(Visitor)
392 324 : }
393 :
394 18406 : pub(crate) fn serialize<S>(
395 18406 : metadata: &TimelineMetadata,
396 18406 : serializer: S,
397 18406 : ) -> Result<S::Ok, S::Error>
398 18406 : where
399 18406 : S: serde::Serializer,
400 18406 : {
401 18406 : // header is not needed, upon reading we've upgraded all v1 to v2
402 18406 : metadata.body.serialize(serializer)
403 18406 : }
404 :
405 : #[test]
406 12 : fn deserializes_bytes_as_well_as_equivalent_body_v2() {
407 12 : #[derive(serde::Deserialize, serde::Serialize)]
408 : struct Wrapper(
409 : #[serde(deserialize_with = "deserialize", serialize_with = "serialize")]
410 : TimelineMetadata,
411 : );
412 :
413 12 : let too_many_bytes = "[216,111,252,208,0,54,0,4,0,0,0,0,1,73,253,144,1,0,0,0,0,1,73,253,24,0,0,0,0,0,0,0,0,0,0,0,0,0,1,73,253,24,0,0,0,0,1,73,253,24,0,0,0,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]";
414 12 :
415 12 : let wrapper_from_bytes = serde_json::from_str::<Wrapper>(too_many_bytes).unwrap();
416 12 :
417 12 : let serialized = serde_json::to_value(&wrapper_from_bytes).unwrap();
418 12 :
419 12 : assert_eq!(
420 12 : serialized,
421 12 : serde_json::json! {{
422 12 : "disk_consistent_lsn": "0/149FD90",
423 12 : "prev_record_lsn": "0/149FD18",
424 12 : "ancestor_timeline": null,
425 12 : "ancestor_lsn": "0/0",
426 12 : "latest_gc_cutoff_lsn": "0/149FD18",
427 12 : "initdb_lsn": "0/149FD18",
428 12 : "pg_version": 15
429 12 : }}
430 12 : );
431 :
432 12 : let wrapper_from_json = serde_json::value::from_value::<Wrapper>(serialized).unwrap();
433 12 :
434 12 : assert_eq!(wrapper_from_bytes.0, wrapper_from_json.0);
435 12 : }
436 : }
437 :
438 : /// Parts of the metadata which are regularly modified.
439 : pub(crate) struct MetadataUpdate {
440 : disk_consistent_lsn: Lsn,
441 : prev_record_lsn: Option<Lsn>,
442 : latest_gc_cutoff_lsn: Lsn,
443 : }
444 :
445 : impl MetadataUpdate {
446 7404 : pub(crate) fn new(
447 7404 : disk_consistent_lsn: Lsn,
448 7404 : prev_record_lsn: Option<Lsn>,
449 7404 : latest_gc_cutoff_lsn: Lsn,
450 7404 : ) -> Self {
451 7404 : Self {
452 7404 : disk_consistent_lsn,
453 7404 : prev_record_lsn,
454 7404 : latest_gc_cutoff_lsn,
455 7404 : }
456 7404 : }
457 : }
458 :
459 : #[cfg(test)]
460 : mod tests {
461 : use super::*;
462 : use crate::tenant::harness::TIMELINE_ID;
463 :
464 : #[test]
465 12 : fn metadata_serializes_correctly() {
466 12 : let original_metadata = TimelineMetadata::new(
467 12 : Lsn(0x200),
468 12 : Some(Lsn(0x100)),
469 12 : Some(TIMELINE_ID),
470 12 : Lsn(0),
471 12 : Lsn(0),
472 12 : Lsn(0),
473 12 : // Any version will do here, so use the default
474 12 : crate::DEFAULT_PG_VERSION,
475 12 : );
476 12 :
477 12 : let metadata_bytes = original_metadata
478 12 : .to_bytes()
479 12 : .expect("Should serialize correct metadata to bytes");
480 12 :
481 12 : let deserialized_metadata = TimelineMetadata::from_bytes(&metadata_bytes)
482 12 : .expect("Should deserialize its own bytes");
483 12 :
484 12 : assert_eq!(
485 : deserialized_metadata.body, original_metadata.body,
486 0 : "Metadata that was serialized to bytes and deserialized back should not change"
487 : );
488 12 : }
489 :
490 : // Generate old version metadata and read it with current code.
491 : // Ensure that it is upgraded correctly
492 : #[test]
493 12 : fn test_metadata_upgrade() {
494 : #[derive(Debug, Clone, PartialEq, Eq)]
495 : struct TimelineMetadataV1 {
496 : hdr: TimelineMetadataHeader,
497 : body: TimelineMetadataBodyV1,
498 : }
499 :
500 12 : let metadata_v1 = TimelineMetadataV1 {
501 12 : hdr: TimelineMetadataHeader {
502 12 : checksum: 0,
503 12 : size: 0,
504 12 : format_version: METADATA_OLD_FORMAT_VERSION,
505 12 : },
506 12 : body: TimelineMetadataBodyV1 {
507 12 : disk_consistent_lsn: Lsn(0x200),
508 12 : prev_record_lsn: Some(Lsn(0x100)),
509 12 : ancestor_timeline: Some(TIMELINE_ID),
510 12 : ancestor_lsn: Lsn(0),
511 12 : latest_gc_cutoff_lsn: Lsn(0),
512 12 : initdb_lsn: Lsn(0),
513 12 : },
514 12 : };
515 :
516 : impl TimelineMetadataV1 {
517 12 : pub fn to_bytes(&self) -> anyhow::Result<Vec<u8>> {
518 12 : let body_bytes = self.body.ser()?;
519 12 : let metadata_size = METADATA_HDR_SIZE + body_bytes.len();
520 12 : let hdr = TimelineMetadataHeader {
521 12 : size: metadata_size as u16,
522 12 : format_version: METADATA_OLD_FORMAT_VERSION,
523 12 : checksum: crc32c::crc32c(&body_bytes),
524 12 : };
525 12 : let hdr_bytes = hdr.ser()?;
526 12 : let mut metadata_bytes = vec![0u8; METADATA_MAX_SIZE];
527 12 : metadata_bytes[0..METADATA_HDR_SIZE].copy_from_slice(&hdr_bytes);
528 12 : metadata_bytes[METADATA_HDR_SIZE..metadata_size].copy_from_slice(&body_bytes);
529 12 : Ok(metadata_bytes)
530 12 : }
531 : }
532 :
533 12 : let metadata_bytes = metadata_v1
534 12 : .to_bytes()
535 12 : .expect("Should serialize correct metadata to bytes");
536 12 :
537 12 : // This should deserialize to the latest version format
538 12 : let deserialized_metadata = TimelineMetadata::from_bytes(&metadata_bytes)
539 12 : .expect("Should deserialize its own bytes");
540 12 :
541 12 : let expected_metadata = TimelineMetadata::new(
542 12 : Lsn(0x200),
543 12 : Some(Lsn(0x100)),
544 12 : Some(TIMELINE_ID),
545 12 : Lsn(0),
546 12 : Lsn(0),
547 12 : Lsn(0),
548 12 : 14, // All timelines created before this version had pg_version 14
549 12 : );
550 12 :
551 12 : assert_eq!(
552 : deserialized_metadata.body, expected_metadata.body,
553 0 : "Metadata of the old version {} should be upgraded to the latest version {}",
554 : METADATA_OLD_FORMAT_VERSION, METADATA_FORMAT_VERSION
555 : );
556 12 : }
557 :
558 : #[test]
559 12 : fn test_metadata_bincode_serde_ensure_roundtrip() {
560 12 : let original_metadata = TimelineMetadata::new(
561 12 : Lsn(0x200),
562 12 : Some(Lsn(0x100)),
563 12 : Some(TIMELINE_ID),
564 12 : Lsn(0),
565 12 : Lsn(0),
566 12 : Lsn(0),
567 12 : // Updating this version to 17 will cause the test to fail at the
568 12 : // next assert_eq!().
569 12 : 16,
570 12 : );
571 12 : let expected_bytes = vec![
572 12 : /* TimelineMetadataHeader */
573 12 : 74, 104, 158, 105, 0, 70, 0, 4, // checksum, size, format_version (4 + 2 + 2)
574 12 : /* TimelineMetadataBodyV2 */
575 12 : 0, 0, 0, 0, 0, 0, 2, 0, // disk_consistent_lsn (8 bytes)
576 12 : 1, 0, 0, 0, 0, 0, 0, 1, 0, // prev_record_lsn (9 bytes)
577 12 : 1, 17, 34, 51, 68, 85, 102, 119, 136, 17, 34, 51, 68, 85, 102, 119,
578 12 : 136, // ancestor_timeline (17 bytes)
579 12 : 0, 0, 0, 0, 0, 0, 0, 0, // ancestor_lsn (8 bytes)
580 12 : 0, 0, 0, 0, 0, 0, 0, 0, // latest_gc_cutoff_lsn (8 bytes)
581 12 : 0, 0, 0, 0, 0, 0, 0, 0, // initdb_lsn (8 bytes)
582 12 : 0, 0, 0, 16, // pg_version (4 bytes)
583 12 : /* padding bytes */
584 12 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
585 12 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
586 12 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
587 12 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
588 12 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
589 12 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
590 12 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
591 12 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
592 12 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
593 12 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
594 12 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
595 12 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
596 12 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
597 12 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
598 12 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
599 12 : 0, 0, 0, 0, 0, 0, 0,
600 12 : ];
601 12 : let metadata_ser_bytes = original_metadata.to_bytes().unwrap();
602 12 : assert_eq!(metadata_ser_bytes, expected_bytes);
603 :
604 12 : let expected_metadata = {
605 12 : let mut temp_metadata = original_metadata;
606 12 : let body_bytes = temp_metadata
607 12 : .body
608 12 : .ser()
609 12 : .expect("Cannot serialize the metadata body");
610 12 : let metadata_size = METADATA_HDR_SIZE + body_bytes.len();
611 12 : let hdr = TimelineMetadataHeader {
612 12 : size: metadata_size as u16,
613 12 : format_version: METADATA_FORMAT_VERSION,
614 12 : checksum: crc32c::crc32c(&body_bytes),
615 12 : };
616 12 : temp_metadata.hdr = hdr;
617 12 : temp_metadata
618 12 : };
619 12 : let des_metadata = TimelineMetadata::from_bytes(&metadata_ser_bytes).unwrap();
620 12 : assert_eq!(des_metadata, expected_metadata);
621 12 : }
622 : }
|
