Line data Source code
1 : //! Describes the legacy now hopefully no longer modified per-timeline metadata.
2 : //!
3 : //! It is stored in `index_part.json` managed by [`remote_timeline_client`]. For many tenants and
4 : //! their timelines, this struct and its original serialization format is still needed because
5 : //! they were written a long time ago.
6 : //!
7 : //! Instead of changing and adding versioning to this, just change [`IndexPart`] with soft json
8 : //! versioning.
9 : //!
10 : //! To clean up this module we need to migrate all index_part.json files to a later version.
11 : //! While doing this, we need to be mindful about s3 based recovery as well, so it might take
12 : //! however long we keep the old versions to be able to delete the old code. After that, we can
13 : //! remove everything else than [`TimelineMetadataBodyV2`], rename it as `TimelineMetadata` and
14 : //! move it to `index.rs`. Before doing all of this, we need to keep the structures for backwards
15 : //! compatibility.
16 : //!
17 : //! [`remote_timeline_client`]: super::remote_timeline_client
18 : //! [`IndexPart`]: super::remote_timeline_client::index::IndexPart
19 :
20 : use anyhow::ensure;
21 : use postgres_ffi::PgMajorVersion;
22 : use serde::{Deserialize, Serialize};
23 : use utils::bin_ser::{BeSer, SerializeError};
24 : use utils::id::TimelineId;
25 : use utils::lsn::Lsn;
26 :
27 : /// Use special format number to enable backward compatibility.
28 : const METADATA_FORMAT_VERSION: u16 = 4;
29 :
30 : /// Previous supported format versions.
31 : ///
32 : /// In practice, none of these should remain, all are [`METADATA_FORMAT_VERSION`], but confirming
33 : /// that requires a scrubber run which is yet to be done.
34 : const METADATA_OLD_FORMAT_VERSION: u16 = 3;
35 :
36 : /// When the file existed on disk we assumed that a write of up to METADATA_MAX_SIZE bytes is atomic.
37 : ///
38 : /// This is the same assumption that PostgreSQL makes with the control file,
39 : ///
40 : /// see PG_CONTROL_MAX_SAFE_SIZE
41 : const METADATA_MAX_SIZE: usize = 512;
42 :
43 : /// Legacy metadata stored as a component of `index_part.json` per timeline.
44 : ///
45 : /// Do not make new changes to this type or the module. In production, we have two different kinds
46 : /// of serializations of this type: bincode and json. Bincode version reflects what used to be
47 : /// stored on disk in earlier versions and does internal crc32 checksumming.
48 : ///
49 : /// This type should not implement `serde::Serialize` or `serde::Deserialize` because there would
50 : /// be a confusion whether you want the old version ([`TimelineMetadata::from_bytes`]) or the modern
51 : /// as-exists in `index_part.json` ([`self::modern_serde`]).
52 : ///
53 : /// ```compile_fail
54 : /// #[derive(serde::Serialize)]
55 : /// struct DoNotDoThis(pageserver::tenant::metadata::TimelineMetadata);
56 : /// ```
57 : ///
58 : /// ```compile_fail
59 : /// #[derive(serde::Deserialize)]
60 : /// struct NeitherDoThis(pageserver::tenant::metadata::TimelineMetadata);
61 : /// ```
62 : #[derive(Debug, Clone, PartialEq, Eq)]
63 : pub struct TimelineMetadata {
64 : hdr: TimelineMetadataHeader,
65 : body: TimelineMetadataBodyV2,
66 : }
67 :
68 0 : #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
69 : struct TimelineMetadataHeader {
70 : checksum: u32, // CRC of serialized metadata body
71 : size: u16, // size of serialized metadata
72 : format_version: u16, // metadata format version (used for compatibility checks)
73 : }
74 :
75 : impl TryFrom<&TimelineMetadataBodyV2> for TimelineMetadataHeader {
76 : type Error = Crc32CalculationFailed;
77 :
78 26 : fn try_from(value: &TimelineMetadataBodyV2) -> Result<Self, Self::Error> {
79 : #[derive(Default)]
80 : struct Crc32Sink {
81 : crc: u32,
82 : count: usize,
83 : }
84 :
85 : impl std::io::Write for Crc32Sink {
86 442 : fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
87 442 : self.crc = crc32c::crc32c_append(self.crc, buf);
88 442 : self.count += buf.len();
89 442 : Ok(buf.len())
90 442 : }
91 :
92 0 : fn flush(&mut self) -> std::io::Result<()> {
93 0 : Ok(())
94 0 : }
95 : }
96 :
97 : // jump through hoops to calculate the crc32 so that TimelineMetadata::ne works
98 : // across serialization versions
99 26 : let mut sink = Crc32Sink::default();
100 26 : <TimelineMetadataBodyV2 as utils::bin_ser::BeSer>::ser_into(value, &mut sink)
101 26 : .map_err(Crc32CalculationFailed)?;
102 :
103 26 : let size = METADATA_HDR_SIZE + sink.count;
104 :
105 26 : Ok(TimelineMetadataHeader {
106 26 : checksum: sink.crc,
107 26 : size: size as u16,
108 26 : format_version: METADATA_FORMAT_VERSION,
109 26 : })
110 26 : }
111 : }
112 :
113 : #[derive(thiserror::Error, Debug)]
114 : #[error("re-serializing for crc32 failed")]
115 : struct Crc32CalculationFailed(#[source] utils::bin_ser::SerializeError);
116 :
117 : const METADATA_HDR_SIZE: usize = size_of::<TimelineMetadataHeader>();
118 :
119 0 : #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
120 : struct TimelineMetadataBodyV2 {
121 : disk_consistent_lsn: Lsn,
122 : // This is only set if we know it. We track it in memory when the page
123 : // server is running, but we only track the value corresponding to
124 : // 'last_record_lsn', not 'disk_consistent_lsn' which can lag behind by a
125 : // lot. We only store it in the metadata file when we flush *all* the
126 : // in-memory data so that 'last_record_lsn' is the same as
127 : // 'disk_consistent_lsn'. That's OK, because after page server restart, as
128 : // soon as we reprocess at least one record, we will have a valid
129 : // 'prev_record_lsn' value in memory again. This is only really needed when
130 : // doing a clean shutdown, so that there is no more WAL beyond
131 : // 'disk_consistent_lsn'
132 : prev_record_lsn: Option<Lsn>,
133 : ancestor_timeline: Option<TimelineId>,
134 : ancestor_lsn: Lsn,
135 :
136 : // The LSN at which GC was last executed. Synonym of [`Timeline::applied_gc_cutoff_lsn`].
137 : latest_gc_cutoff_lsn: Lsn,
138 :
139 : initdb_lsn: Lsn,
140 : pg_version: PgMajorVersion,
141 : }
142 :
143 0 : #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
144 : struct TimelineMetadataBodyV1 {
145 : disk_consistent_lsn: Lsn,
146 : // This is only set if we know it. We track it in memory when the page
147 : // server is running, but we only track the value corresponding to
148 : // 'last_record_lsn', not 'disk_consistent_lsn' which can lag behind by a
149 : // lot. We only store it in the metadata file when we flush *all* the
150 : // in-memory data so that 'last_record_lsn' is the same as
151 : // 'disk_consistent_lsn'. That's OK, because after page server restart, as
152 : // soon as we reprocess at least one record, we will have a valid
153 : // 'prev_record_lsn' value in memory again. This is only really needed when
154 : // doing a clean shutdown, so that there is no more WAL beyond
155 : // 'disk_consistent_lsn'
156 : prev_record_lsn: Option<Lsn>,
157 : ancestor_timeline: Option<TimelineId>,
158 : ancestor_lsn: Lsn,
159 : latest_gc_cutoff_lsn: Lsn,
160 : initdb_lsn: Lsn,
161 : }
162 :
163 : impl TimelineMetadata {
164 285 : pub fn new(
165 285 : disk_consistent_lsn: Lsn,
166 285 : prev_record_lsn: Option<Lsn>,
167 285 : ancestor_timeline: Option<TimelineId>,
168 285 : ancestor_lsn: Lsn,
169 285 : latest_gc_cutoff_lsn: Lsn,
170 285 : initdb_lsn: Lsn,
171 285 : pg_version: PgMajorVersion,
172 285 : ) -> Self {
173 285 : Self {
174 285 : hdr: TimelineMetadataHeader {
175 285 : checksum: 0,
176 285 : size: 0,
177 285 : format_version: METADATA_FORMAT_VERSION,
178 285 : },
179 285 : body: TimelineMetadataBodyV2 {
180 285 : disk_consistent_lsn,
181 285 : prev_record_lsn,
182 285 : ancestor_timeline,
183 285 : ancestor_lsn,
184 285 : latest_gc_cutoff_lsn,
185 285 : initdb_lsn,
186 285 : pg_version,
187 285 : },
188 285 : }
189 285 : }
190 :
191 : #[cfg(test)]
192 7 : pub(crate) fn with_recalculated_checksum(mut self) -> anyhow::Result<Self> {
193 7 : self.hdr = TimelineMetadataHeader::try_from(&self.body)?;
194 7 : Ok(self)
195 7 : }
196 :
197 1 : fn upgrade_timeline_metadata(metadata_bytes: &[u8]) -> anyhow::Result<Self> {
198 1 : let mut hdr = TimelineMetadataHeader::des(&metadata_bytes[0..METADATA_HDR_SIZE])?;
199 :
200 : // backward compatible only up to this version
201 1 : ensure!(
202 1 : hdr.format_version == METADATA_OLD_FORMAT_VERSION,
203 0 : "unsupported metadata format version {}",
204 : hdr.format_version
205 : );
206 :
207 1 : let metadata_size = hdr.size as usize;
208 :
209 1 : let body: TimelineMetadataBodyV1 =
210 1 : TimelineMetadataBodyV1::des(&metadata_bytes[METADATA_HDR_SIZE..metadata_size])?;
211 :
212 1 : let body = TimelineMetadataBodyV2 {
213 1 : disk_consistent_lsn: body.disk_consistent_lsn,
214 1 : prev_record_lsn: body.prev_record_lsn,
215 1 : ancestor_timeline: body.ancestor_timeline,
216 1 : ancestor_lsn: body.ancestor_lsn,
217 1 : latest_gc_cutoff_lsn: body.latest_gc_cutoff_lsn,
218 1 : initdb_lsn: body.initdb_lsn,
219 1 : pg_version: PgMajorVersion::PG14, // All timelines created before this version had pg_version 14
220 1 : };
221 :
222 1 : hdr.format_version = METADATA_FORMAT_VERSION;
223 :
224 1 : Ok(Self { hdr, body })
225 1 : }
226 :
227 61 : pub fn from_bytes(metadata_bytes: &[u8]) -> anyhow::Result<Self> {
228 61 : ensure!(
229 61 : metadata_bytes.len() == METADATA_MAX_SIZE,
230 0 : "metadata bytes size is wrong"
231 : );
232 61 : let hdr = TimelineMetadataHeader::des(&metadata_bytes[0..METADATA_HDR_SIZE])?;
233 :
234 61 : let metadata_size = hdr.size as usize;
235 61 : ensure!(
236 61 : metadata_size <= METADATA_MAX_SIZE,
237 0 : "corrupted metadata file"
238 : );
239 61 : let calculated_checksum = crc32c::crc32c(&metadata_bytes[METADATA_HDR_SIZE..metadata_size]);
240 61 : ensure!(
241 61 : hdr.checksum == calculated_checksum,
242 0 : "metadata checksum mismatch"
243 : );
244 :
245 61 : if hdr.format_version != METADATA_FORMAT_VERSION {
246 : // If metadata has the old format,
247 : // upgrade it and return the result
248 1 : TimelineMetadata::upgrade_timeline_metadata(metadata_bytes)
249 : } else {
250 60 : let body =
251 60 : TimelineMetadataBodyV2::des(&metadata_bytes[METADATA_HDR_SIZE..metadata_size])?;
252 60 : ensure!(
253 60 : body.disk_consistent_lsn.is_aligned(),
254 0 : "disk_consistent_lsn is not aligned"
255 : );
256 60 : Ok(TimelineMetadata { hdr, body })
257 : }
258 61 : }
259 :
260 45 : pub fn to_bytes(&self) -> Result<Vec<u8>, SerializeError> {
261 45 : let body_bytes = self.body.ser()?;
262 45 : let metadata_size = METADATA_HDR_SIZE + body_bytes.len();
263 45 : let hdr = TimelineMetadataHeader {
264 45 : size: metadata_size as u16,
265 45 : format_version: METADATA_FORMAT_VERSION,
266 45 : checksum: crc32c::crc32c(&body_bytes),
267 45 : };
268 45 : let hdr_bytes = hdr.ser()?;
269 45 : let mut metadata_bytes = vec![0u8; METADATA_MAX_SIZE];
270 45 : metadata_bytes[0..METADATA_HDR_SIZE].copy_from_slice(&hdr_bytes);
271 45 : metadata_bytes[METADATA_HDR_SIZE..metadata_size].copy_from_slice(&body_bytes);
272 45 : Ok(metadata_bytes)
273 45 : }
274 :
275 : /// [`Lsn`] that corresponds to the corresponding timeline directory
276 : /// contents, stored locally in the pageserver workdir.
277 11070 : pub fn disk_consistent_lsn(&self) -> Lsn {
278 11070 : self.body.disk_consistent_lsn
279 11070 : }
280 :
281 235 : pub fn prev_record_lsn(&self) -> Option<Lsn> {
282 235 : self.body.prev_record_lsn
283 235 : }
284 :
285 1642 : pub fn ancestor_timeline(&self) -> Option<TimelineId> {
286 1642 : self.body.ancestor_timeline
287 1642 : }
288 :
289 354 : pub fn ancestor_lsn(&self) -> Lsn {
290 354 : self.body.ancestor_lsn
291 354 : }
292 :
293 : /// When reparenting, the `ancestor_lsn` does not change.
294 : ///
295 : /// Returns true if anything was changed.
296 0 : pub fn reparent(&mut self, timeline: &TimelineId) {
297 0 : assert!(self.body.ancestor_timeline.is_some());
298 : // no assertion for redoing this: it's fine, we may have to repeat this multiple times over
299 0 : self.body.ancestor_timeline = Some(*timeline);
300 0 : }
301 :
302 : /// Returns true if anything was changed
303 0 : pub fn detach_from_ancestor(&mut self, branchpoint: &(TimelineId, Lsn)) {
304 : // Detaching from ancestor now doesn't always detach directly to the direct ancestor, but we
305 : // ensure the LSN is the same. So we don't check the timeline ID.
306 0 : if self.body.ancestor_lsn != Lsn(0) {
307 0 : assert_eq!(self.body.ancestor_lsn, branchpoint.1);
308 0 : }
309 0 : self.body.ancestor_timeline = None;
310 0 : self.body.ancestor_lsn = Lsn(0);
311 0 : }
312 :
313 235 : pub fn latest_gc_cutoff_lsn(&self) -> Lsn {
314 235 : self.body.latest_gc_cutoff_lsn
315 235 : }
316 :
317 235 : pub fn initdb_lsn(&self) -> Lsn {
318 235 : self.body.initdb_lsn
319 235 : }
320 :
321 237 : pub fn pg_version(&self) -> PgMajorVersion {
322 237 : self.body.pg_version
323 237 : }
324 :
325 : // Checksums make it awkward to build a valid instance by hand. This helper
326 : // provides a TimelineMetadata with a valid checksum in its header.
327 42 : pub fn example() -> Self {
328 42 : let instance = Self::new(
329 42 : "0/16960E8".parse::<Lsn>().unwrap(),
330 42 : None,
331 42 : None,
332 42 : Lsn::from_hex("00000000").unwrap(),
333 42 : Lsn::from_hex("00000000").unwrap(),
334 42 : Lsn::from_hex("00000000").unwrap(),
335 42 : PgMajorVersion::PG14,
336 : );
337 42 : let bytes = instance.to_bytes().unwrap();
338 42 : Self::from_bytes(&bytes).unwrap()
339 42 : }
340 :
341 623 : pub(crate) fn apply(&mut self, update: &MetadataUpdate) {
342 623 : self.body.disk_consistent_lsn = update.disk_consistent_lsn;
343 623 : self.body.prev_record_lsn = update.prev_record_lsn;
344 623 : self.body.latest_gc_cutoff_lsn = update.latest_gc_cutoff_lsn;
345 623 : }
346 : }
347 :
348 : pub(crate) mod modern_serde {
349 : use serde::{Deserialize, Serialize};
350 :
351 : use super::{TimelineMetadata, TimelineMetadataBodyV2, TimelineMetadataHeader};
352 :
353 27 : pub(crate) fn deserialize<'de, D>(deserializer: D) -> Result<TimelineMetadata, D::Error>
354 27 : where
355 27 : D: serde::de::Deserializer<'de>,
356 : {
357 : // for legacy reasons versions 1-5 had TimelineMetadata serialized as a Vec<u8> field with
358 : // BeSer.
359 : struct Visitor;
360 :
361 : impl<'d> serde::de::Visitor<'d> for Visitor {
362 : type Value = TimelineMetadata;
363 :
364 0 : fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
365 0 : f.write_str("BeSer bytes or json structure")
366 0 : }
367 :
368 8 : fn visit_seq<A>(self, seq: A) -> Result<Self::Value, A::Error>
369 8 : where
370 8 : A: serde::de::SeqAccess<'d>,
371 : {
372 : use serde::de::Error;
373 8 : let de = serde::de::value::SeqAccessDeserializer::new(seq);
374 8 : Vec::<u8>::deserialize(de)
375 8 : .map(|v| TimelineMetadata::from_bytes(&v).map_err(A::Error::custom))?
376 8 : }
377 :
378 19 : fn visit_map<A>(self, map: A) -> Result<Self::Value, A::Error>
379 19 : where
380 19 : A: serde::de::MapAccess<'d>,
381 : {
382 : use serde::de::Error;
383 :
384 19 : let de = serde::de::value::MapAccessDeserializer::new(map);
385 19 : let body = TimelineMetadataBodyV2::deserialize(de)?;
386 19 : let hdr = TimelineMetadataHeader::try_from(&body).map_err(A::Error::custom)?;
387 :
388 19 : Ok(TimelineMetadata { hdr, body })
389 19 : }
390 : }
391 :
392 27 : deserializer.deserialize_any(Visitor)
393 27 : }
394 :
395 1578 : pub(crate) fn serialize<S>(
396 1578 : metadata: &TimelineMetadata,
397 1578 : serializer: S,
398 1578 : ) -> Result<S::Ok, S::Error>
399 1578 : where
400 1578 : S: serde::Serializer,
401 : {
402 : // header is not needed, upon reading we've upgraded all v1 to v2
403 1578 : metadata.body.serialize(serializer)
404 1578 : }
405 :
406 : #[test]
407 1 : fn deserializes_bytes_as_well_as_equivalent_body_v2() {
408 : #[derive(serde::Deserialize, serde::Serialize)]
409 : struct Wrapper(
410 : #[serde(deserialize_with = "deserialize", serialize_with = "serialize")]
411 : TimelineMetadata,
412 : );
413 :
414 1 : let too_many_bytes = "[216,111,252,208,0,54,0,4,0,0,0,0,1,73,253,144,1,0,0,0,0,1,73,253,24,0,0,0,0,0,0,0,0,0,0,0,0,0,1,73,253,24,0,0,0,0,1,73,253,24,0,0,0,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]";
415 :
416 1 : let wrapper_from_bytes = serde_json::from_str::<Wrapper>(too_many_bytes).unwrap();
417 :
418 1 : let serialized = serde_json::to_value(&wrapper_from_bytes).unwrap();
419 :
420 1 : assert_eq!(
421 : serialized,
422 1 : serde_json::json! {{
423 1 : "disk_consistent_lsn": "0/149FD90",
424 1 : "prev_record_lsn": "0/149FD18",
425 1 : "ancestor_timeline": null,
426 1 : "ancestor_lsn": "0/0",
427 1 : "latest_gc_cutoff_lsn": "0/149FD18",
428 1 : "initdb_lsn": "0/149FD18",
429 1 : "pg_version": 15
430 : }}
431 : );
432 :
433 1 : let wrapper_from_json = serde_json::value::from_value::<Wrapper>(serialized).unwrap();
434 :
435 1 : assert_eq!(wrapper_from_bytes.0, wrapper_from_json.0);
436 1 : }
437 : }
438 :
439 : /// Parts of the metadata which are regularly modified.
440 : pub(crate) struct MetadataUpdate {
441 : disk_consistent_lsn: Lsn,
442 : prev_record_lsn: Option<Lsn>,
443 : latest_gc_cutoff_lsn: Lsn,
444 : }
445 :
446 : impl MetadataUpdate {
447 623 : pub(crate) fn new(
448 623 : disk_consistent_lsn: Lsn,
449 623 : prev_record_lsn: Option<Lsn>,
450 623 : latest_gc_cutoff_lsn: Lsn,
451 623 : ) -> Self {
452 623 : Self {
453 623 : disk_consistent_lsn,
454 623 : prev_record_lsn,
455 623 : latest_gc_cutoff_lsn,
456 623 : }
457 623 : }
458 : }
459 :
460 : #[cfg(test)]
461 : mod tests {
462 : use super::*;
463 : use crate::tenant::harness::TIMELINE_ID;
464 :
465 : #[test]
466 1 : fn metadata_serializes_correctly() {
467 1 : let original_metadata = TimelineMetadata::new(
468 1 : Lsn(0x200),
469 1 : Some(Lsn(0x100)),
470 1 : Some(TIMELINE_ID),
471 1 : Lsn(0),
472 1 : Lsn(0),
473 1 : Lsn(0),
474 : // Any version will do here, so use the default
475 : crate::DEFAULT_PG_VERSION,
476 : );
477 :
478 1 : let metadata_bytes = original_metadata
479 1 : .to_bytes()
480 1 : .expect("Should serialize correct metadata to bytes");
481 :
482 1 : let deserialized_metadata = TimelineMetadata::from_bytes(&metadata_bytes)
483 1 : .expect("Should deserialize its own bytes");
484 :
485 1 : assert_eq!(
486 : deserialized_metadata.body, original_metadata.body,
487 0 : "Metadata that was serialized to bytes and deserialized back should not change"
488 : );
489 1 : }
490 :
491 : // Generate old version metadata and read it with current code.
492 : // Ensure that it is upgraded correctly
493 : #[test]
494 1 : fn test_metadata_upgrade() {
495 : #[derive(Debug, Clone, PartialEq, Eq)]
496 : struct TimelineMetadataV1 {
497 : hdr: TimelineMetadataHeader,
498 : body: TimelineMetadataBodyV1,
499 : }
500 :
501 1 : let metadata_v1 = TimelineMetadataV1 {
502 1 : hdr: TimelineMetadataHeader {
503 1 : checksum: 0,
504 1 : size: 0,
505 1 : format_version: METADATA_OLD_FORMAT_VERSION,
506 1 : },
507 1 : body: TimelineMetadataBodyV1 {
508 1 : disk_consistent_lsn: Lsn(0x200),
509 1 : prev_record_lsn: Some(Lsn(0x100)),
510 1 : ancestor_timeline: Some(TIMELINE_ID),
511 1 : ancestor_lsn: Lsn(0),
512 1 : latest_gc_cutoff_lsn: Lsn(0),
513 1 : initdb_lsn: Lsn(0),
514 1 : },
515 1 : };
516 :
517 : impl TimelineMetadataV1 {
518 1 : pub fn to_bytes(&self) -> anyhow::Result<Vec<u8>> {
519 1 : let body_bytes = self.body.ser()?;
520 1 : let metadata_size = METADATA_HDR_SIZE + body_bytes.len();
521 1 : let hdr = TimelineMetadataHeader {
522 1 : size: metadata_size as u16,
523 1 : format_version: METADATA_OLD_FORMAT_VERSION,
524 1 : checksum: crc32c::crc32c(&body_bytes),
525 1 : };
526 1 : let hdr_bytes = hdr.ser()?;
527 1 : let mut metadata_bytes = vec![0u8; METADATA_MAX_SIZE];
528 1 : metadata_bytes[0..METADATA_HDR_SIZE].copy_from_slice(&hdr_bytes);
529 1 : metadata_bytes[METADATA_HDR_SIZE..metadata_size].copy_from_slice(&body_bytes);
530 1 : Ok(metadata_bytes)
531 1 : }
532 : }
533 :
534 1 : let metadata_bytes = metadata_v1
535 1 : .to_bytes()
536 1 : .expect("Should serialize correct metadata to bytes");
537 :
538 : // This should deserialize to the latest version format
539 1 : let deserialized_metadata = TimelineMetadata::from_bytes(&metadata_bytes)
540 1 : .expect("Should deserialize its own bytes");
541 :
542 1 : let expected_metadata = TimelineMetadata::new(
543 1 : Lsn(0x200),
544 1 : Some(Lsn(0x100)),
545 1 : Some(TIMELINE_ID),
546 1 : Lsn(0),
547 1 : Lsn(0),
548 1 : Lsn(0),
549 1 : PgMajorVersion::PG14, // All timelines created before this version had pg_version 14
550 : );
551 :
552 1 : assert_eq!(
553 : deserialized_metadata.body, expected_metadata.body,
554 0 : "Metadata of the old version {METADATA_OLD_FORMAT_VERSION} should be upgraded to the latest version {METADATA_FORMAT_VERSION}"
555 : );
556 1 : }
557 :
558 : #[test]
559 1 : fn test_metadata_bincode_serde_ensure_roundtrip() {
560 1 : let original_metadata = TimelineMetadata::new(
561 1 : Lsn(0x200),
562 1 : Some(Lsn(0x100)),
563 1 : Some(TIMELINE_ID),
564 1 : Lsn(0),
565 1 : Lsn(0),
566 1 : Lsn(0),
567 : // Updating this version to 17 will cause the test to fail at the
568 : // next assert_eq!().
569 1 : PgMajorVersion::PG16,
570 : );
571 1 : let expected_bytes = vec![
572 : /* TimelineMetadataHeader */
573 : 74, 104, 158, 105, 0, 70, 0, 4, // checksum, size, format_version (4 + 2 + 2)
574 : /* TimelineMetadataBodyV2 */
575 : 0, 0, 0, 0, 0, 0, 2, 0, // disk_consistent_lsn (8 bytes)
576 : 1, 0, 0, 0, 0, 0, 0, 1, 0, // prev_record_lsn (9 bytes)
577 : 1, 17, 34, 51, 68, 85, 102, 119, 136, 17, 34, 51, 68, 85, 102, 119,
578 : 136, // ancestor_timeline (17 bytes)
579 : 0, 0, 0, 0, 0, 0, 0, 0, // ancestor_lsn (8 bytes)
580 : 0, 0, 0, 0, 0, 0, 0, 0, // latest_gc_cutoff_lsn (8 bytes)
581 : 0, 0, 0, 0, 0, 0, 0, 0, // initdb_lsn (8 bytes)
582 : 0, 0, 0, 16, // pg_version (4 bytes)
583 : /* padding bytes */
584 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
585 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
586 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
587 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
588 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
589 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
590 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
591 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
592 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
593 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
594 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
595 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
596 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
597 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
598 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
599 : 0, 0, 0, 0, 0, 0, 0,
600 : ];
601 1 : let metadata_ser_bytes = original_metadata.to_bytes().unwrap();
602 1 : assert_eq!(metadata_ser_bytes, expected_bytes);
603 :
604 1 : let expected_metadata = {
605 1 : let mut temp_metadata = original_metadata;
606 1 : let body_bytes = temp_metadata
607 1 : .body
608 1 : .ser()
609 1 : .expect("Cannot serialize the metadata body");
610 1 : let metadata_size = METADATA_HDR_SIZE + body_bytes.len();
611 1 : let hdr = TimelineMetadataHeader {
612 1 : size: metadata_size as u16,
613 1 : format_version: METADATA_FORMAT_VERSION,
614 1 : checksum: crc32c::crc32c(&body_bytes),
615 1 : };
616 1 : temp_metadata.hdr = hdr;
617 1 : temp_metadata
618 : };
619 1 : let des_metadata = TimelineMetadata::from_bytes(&metadata_ser_bytes).unwrap();
620 1 : assert_eq!(des_metadata, expected_metadata);
621 1 : }
622 : }
|
