Line data Source code
1 : //! Describes the legacy now hopefully no longer modified per-timeline metadata stored in
2 : //! `index_part.json` managed by [`remote_timeline_client`]. For many tenants and their timelines,
3 : //! this struct and it's original serialization format is still needed because they were written a
4 : //! long time ago.
5 : //!
6 : //! Instead of changing and adding versioning to this, just change [`IndexPart`] with soft json
7 : //! versioning.
8 : //!
9 : //! To clean up this module we need to migrate all index_part.json files to a later version.
10 : //! While doing this, we need to be mindful about s3 based recovery as well, so it might take
11 : //! however long we keep the old versions to be able to delete the old code. After that, we can
12 : //! remove everything else than [`TimelineMetadataBodyV2`], rename it as `TimelineMetadata` and
13 : //! move it to `index.rs`. Before doing all of this, we need to keep the structures for backwards
14 : //! compatibility.
15 : //!
16 : //! [`remote_timeline_client`]: super::remote_timeline_client
17 : //! [`IndexPart`]: super::remote_timeline_client::index::IndexPart
18 :
19 : use anyhow::ensure;
20 : use serde::{Deserialize, Serialize};
21 : use utils::bin_ser::SerializeError;
22 : use utils::{bin_ser::BeSer, id::TimelineId, lsn::Lsn};
23 :
24 : /// Use special format number to enable backward compatibility.
25 : const METADATA_FORMAT_VERSION: u16 = 4;
26 :
27 : /// Previous supported format versions.
28 : ///
29 : /// In practice, none of these should remain, all are [`METADATA_FORMAT_VERSION`], but confirming
30 : /// that requires a scrubber run which is yet to be done.
31 : const METADATA_OLD_FORMAT_VERSION: u16 = 3;
32 :
33 : /// When the file existed on disk we assumed that a write of up to METADATA_MAX_SIZE bytes is atomic.
34 : ///
35 : /// This is the same assumption that PostgreSQL makes with the control file,
36 : ///
37 : /// see PG_CONTROL_MAX_SAFE_SIZE
38 : const METADATA_MAX_SIZE: usize = 512;
39 :
40 : /// Legacy metadata stored as a component of `index_part.json` per timeline.
41 : ///
42 : /// Do not make new changes to this type or the module. In production, we have two different kinds
43 : /// of serializations of this type: bincode and json. Bincode version reflects what used to be
44 : /// stored on disk in earlier versions and does internal crc32 checksumming.
45 : ///
46 : /// This type should not implement `serde::Serialize` or `serde::Deserialize` because there would
47 : /// be a confusion whether you want the old version ([`TimelineMetadata::from_bytes`]) or the modern
48 : /// as-exists in `index_part.json` ([`self::modern_serde`]).
49 : ///
50 : /// ```compile_fail
51 : /// #[derive(serde::Serialize)]
52 : /// struct DoNotDoThis(pageserver::tenant::metadata::TimelineMetadata);
53 : /// ```
54 : ///
55 : /// ```compile_fail
56 : /// #[derive(serde::Deserialize)]
57 : /// struct NeitherDoThis(pageserver::tenant::metadata::TimelineMetadata);
58 : /// ```
59 : #[derive(Debug, Clone, PartialEq, Eq)]
60 : pub struct TimelineMetadata {
61 : hdr: TimelineMetadataHeader,
62 : body: TimelineMetadataBodyV2,
63 : }
64 :
65 52 : #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
66 : struct TimelineMetadataHeader {
67 : checksum: u32, // CRC of serialized metadata body
68 : size: u16, // size of serialized metadata
69 : format_version: u16, // metadata format version (used for compatibility checks)
70 : }
71 :
72 : impl TryFrom<&TimelineMetadataBodyV2> for TimelineMetadataHeader {
73 : type Error = Crc32CalculationFailed;
74 :
75 26 : fn try_from(value: &TimelineMetadataBodyV2) -> Result<Self, Self::Error> {
76 26 : #[derive(Default)]
77 26 : struct Crc32Sink {
78 26 : crc: u32,
79 26 : count: usize,
80 26 : }
81 26 :
82 26 : impl std::io::Write for Crc32Sink {
83 292 : fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
84 292 : self.crc = crc32c::crc32c_append(self.crc, buf);
85 292 : self.count += buf.len();
86 292 : Ok(buf.len())
87 292 : }
88 26 :
89 26 : fn flush(&mut self) -> std::io::Result<()> {
90 0 : Ok(())
91 0 : }
92 26 : }
93 26 :
94 26 : // jump through hoops to calculate the crc32 so that TimelineMetadata::ne works
95 26 : // across serialization versions
96 26 : let mut sink = Crc32Sink::default();
97 26 : <TimelineMetadataBodyV2 as utils::bin_ser::BeSer>::ser_into(value, &mut sink)
98 26 : .map_err(Crc32CalculationFailed)?;
99 :
100 26 : let size = METADATA_HDR_SIZE + sink.count;
101 26 :
102 26 : Ok(TimelineMetadataHeader {
103 26 : checksum: sink.crc,
104 26 : size: size as u16,
105 26 : format_version: METADATA_FORMAT_VERSION,
106 26 : })
107 26 : }
108 : }
109 :
110 0 : #[derive(thiserror::Error, Debug)]
111 : #[error("re-serializing for crc32 failed")]
112 : struct Crc32CalculationFailed(#[source] utils::bin_ser::SerializeError);
113 :
114 : const METADATA_HDR_SIZE: usize = std::mem::size_of::<TimelineMetadataHeader>();
115 :
116 240 : #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
117 : struct TimelineMetadataBodyV2 {
118 : disk_consistent_lsn: Lsn,
119 : // This is only set if we know it. We track it in memory when the page
120 : // server is running, but we only track the value corresponding to
121 : // 'last_record_lsn', not 'disk_consistent_lsn' which can lag behind by a
122 : // lot. We only store it in the metadata file when we flush *all* the
123 : // in-memory data so that 'last_record_lsn' is the same as
124 : // 'disk_consistent_lsn'. That's OK, because after page server restart, as
125 : // soon as we reprocess at least one record, we will have a valid
126 : // 'prev_record_lsn' value in memory again. This is only really needed when
127 : // doing a clean shutdown, so that there is no more WAL beyond
128 : // 'disk_consistent_lsn'
129 : prev_record_lsn: Option<Lsn>,
130 : ancestor_timeline: Option<TimelineId>,
131 : ancestor_lsn: Lsn,
132 : latest_gc_cutoff_lsn: Lsn,
133 : initdb_lsn: Lsn,
134 : pg_version: u32,
135 : }
136 :
137 2 : #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
138 : struct TimelineMetadataBodyV1 {
139 : disk_consistent_lsn: Lsn,
140 : // This is only set if we know it. We track it in memory when the page
141 : // server is running, but we only track the value corresponding to
142 : // 'last_record_lsn', not 'disk_consistent_lsn' which can lag behind by a
143 : // lot. We only store it in the metadata file when we flush *all* the
144 : // in-memory data so that 'last_record_lsn' is the same as
145 : // 'disk_consistent_lsn'. That's OK, because after page server restart, as
146 : // soon as we reprocess at least one record, we will have a valid
147 : // 'prev_record_lsn' value in memory again. This is only really needed when
148 : // doing a clean shutdown, so that there is no more WAL beyond
149 : // 'disk_consistent_lsn'
150 : prev_record_lsn: Option<Lsn>,
151 : ancestor_timeline: Option<TimelineId>,
152 : ancestor_lsn: Lsn,
153 : latest_gc_cutoff_lsn: Lsn,
154 : initdb_lsn: Lsn,
155 : }
156 :
157 : impl TimelineMetadata {
158 396 : pub fn new(
159 396 : disk_consistent_lsn: Lsn,
160 396 : prev_record_lsn: Option<Lsn>,
161 396 : ancestor_timeline: Option<TimelineId>,
162 396 : ancestor_lsn: Lsn,
163 396 : latest_gc_cutoff_lsn: Lsn,
164 396 : initdb_lsn: Lsn,
165 396 : pg_version: u32,
166 396 : ) -> Self {
167 396 : Self {
168 396 : hdr: TimelineMetadataHeader {
169 396 : checksum: 0,
170 396 : size: 0,
171 396 : format_version: METADATA_FORMAT_VERSION,
172 396 : },
173 396 : body: TimelineMetadataBodyV2 {
174 396 : disk_consistent_lsn,
175 396 : prev_record_lsn,
176 396 : ancestor_timeline,
177 396 : ancestor_lsn,
178 396 : latest_gc_cutoff_lsn,
179 396 : initdb_lsn,
180 396 : pg_version,
181 396 : },
182 396 : }
183 396 : }
184 :
185 : #[cfg(test)]
186 2 : pub(crate) fn with_recalculated_checksum(mut self) -> anyhow::Result<Self> {
187 2 : self.hdr = TimelineMetadataHeader::try_from(&self.body)?;
188 2 : Ok(self)
189 2 : }
190 :
191 2 : fn upgrade_timeline_metadata(metadata_bytes: &[u8]) -> anyhow::Result<Self> {
192 2 : let mut hdr = TimelineMetadataHeader::des(&metadata_bytes[0..METADATA_HDR_SIZE])?;
193 :
194 : // backward compatible only up to this version
195 2 : ensure!(
196 2 : hdr.format_version == METADATA_OLD_FORMAT_VERSION,
197 0 : "unsupported metadata format version {}",
198 : hdr.format_version
199 : );
200 :
201 2 : let metadata_size = hdr.size as usize;
202 :
203 2 : let body: TimelineMetadataBodyV1 =
204 2 : TimelineMetadataBodyV1::des(&metadata_bytes[METADATA_HDR_SIZE..metadata_size])?;
205 :
206 2 : let body = TimelineMetadataBodyV2 {
207 2 : disk_consistent_lsn: body.disk_consistent_lsn,
208 2 : prev_record_lsn: body.prev_record_lsn,
209 2 : ancestor_timeline: body.ancestor_timeline,
210 2 : ancestor_lsn: body.ancestor_lsn,
211 2 : latest_gc_cutoff_lsn: body.latest_gc_cutoff_lsn,
212 2 : initdb_lsn: body.initdb_lsn,
213 2 : pg_version: 14, // All timelines created before this version had pg_version 14
214 2 : };
215 2 :
216 2 : hdr.format_version = METADATA_FORMAT_VERSION;
217 2 :
218 2 : Ok(Self { hdr, body })
219 2 : }
220 :
221 50 : pub fn from_bytes(metadata_bytes: &[u8]) -> anyhow::Result<Self> {
222 50 : ensure!(
223 50 : metadata_bytes.len() == METADATA_MAX_SIZE,
224 0 : "metadata bytes size is wrong"
225 : );
226 50 : let hdr = TimelineMetadataHeader::des(&metadata_bytes[0..METADATA_HDR_SIZE])?;
227 :
228 50 : let metadata_size = hdr.size as usize;
229 50 : ensure!(
230 50 : metadata_size <= METADATA_MAX_SIZE,
231 0 : "corrupted metadata file"
232 : );
233 50 : let calculated_checksum = crc32c::crc32c(&metadata_bytes[METADATA_HDR_SIZE..metadata_size]);
234 50 : ensure!(
235 50 : hdr.checksum == calculated_checksum,
236 0 : "metadata checksum mismatch"
237 : );
238 :
239 50 : if hdr.format_version != METADATA_FORMAT_VERSION {
240 : // If metadata has the old format,
241 : // upgrade it and return the result
242 2 : TimelineMetadata::upgrade_timeline_metadata(metadata_bytes)
243 : } else {
244 48 : let body =
245 48 : TimelineMetadataBodyV2::des(&metadata_bytes[METADATA_HDR_SIZE..metadata_size])?;
246 48 : ensure!(
247 48 : body.disk_consistent_lsn.is_aligned(),
248 0 : "disk_consistent_lsn is not aligned"
249 : );
250 48 : Ok(TimelineMetadata { hdr, body })
251 : }
252 50 : }
253 :
254 18 : pub fn to_bytes(&self) -> Result<Vec<u8>, SerializeError> {
255 18 : let body_bytes = self.body.ser()?;
256 18 : let metadata_size = METADATA_HDR_SIZE + body_bytes.len();
257 18 : let hdr = TimelineMetadataHeader {
258 18 : size: metadata_size as u16,
259 18 : format_version: METADATA_FORMAT_VERSION,
260 18 : checksum: crc32c::crc32c(&body_bytes),
261 18 : };
262 18 : let hdr_bytes = hdr.ser()?;
263 18 : let mut metadata_bytes = vec![0u8; METADATA_MAX_SIZE];
264 18 : metadata_bytes[0..METADATA_HDR_SIZE].copy_from_slice(&hdr_bytes);
265 18 : metadata_bytes[METADATA_HDR_SIZE..metadata_size].copy_from_slice(&body_bytes);
266 18 : Ok(metadata_bytes)
267 18 : }
268 :
269 : /// [`Lsn`] that corresponds to the corresponding timeline directory
270 : /// contents, stored locally in the pageserver workdir.
271 20866 : pub fn disk_consistent_lsn(&self) -> Lsn {
272 20866 : self.body.disk_consistent_lsn
273 20866 : }
274 :
275 380 : pub fn prev_record_lsn(&self) -> Option<Lsn> {
276 380 : self.body.prev_record_lsn
277 380 : }
278 :
279 392 : pub fn ancestor_timeline(&self) -> Option<TimelineId> {
280 392 : self.body.ancestor_timeline
281 392 : }
282 :
283 380 : pub fn ancestor_lsn(&self) -> Lsn {
284 380 : self.body.ancestor_lsn
285 380 : }
286 :
287 : /// When reparenting, the `ancestor_lsn` does not change.
288 0 : pub fn reparent(&mut self, timeline: &TimelineId) {
289 0 : assert!(self.body.ancestor_timeline.is_some());
290 : // no assertion for redoing this: it's fine, we may have to repeat this multiple times over
291 0 : self.body.ancestor_timeline = Some(*timeline);
292 0 : }
293 :
294 0 : pub fn detach_from_ancestor(&mut self, branchpoint: &(TimelineId, Lsn)) {
295 0 : if let Some(ancestor) = self.body.ancestor_timeline {
296 0 : assert_eq!(ancestor, branchpoint.0);
297 0 : }
298 0 : if self.body.ancestor_lsn != Lsn(0) {
299 0 : assert_eq!(self.body.ancestor_lsn, branchpoint.1);
300 0 : }
301 0 : self.body.ancestor_timeline = None;
302 0 : self.body.ancestor_lsn = Lsn(0);
303 0 : }
304 :
305 380 : pub fn latest_gc_cutoff_lsn(&self) -> Lsn {
306 380 : self.body.latest_gc_cutoff_lsn
307 380 : }
308 :
309 380 : pub fn initdb_lsn(&self) -> Lsn {
310 380 : self.body.initdb_lsn
311 380 : }
312 :
313 380 : pub fn pg_version(&self) -> u32 {
314 380 : self.body.pg_version
315 380 : }
316 :
317 : // Checksums make it awkward to build a valid instance by hand. This helper
318 : // provides a TimelineMetadata with a valid checksum in its header.
319 : #[cfg(test)]
320 12 : pub fn example() -> Self {
321 12 : let instance = Self::new(
322 12 : "0/16960E8".parse::<Lsn>().unwrap(),
323 12 : None,
324 12 : None,
325 12 : Lsn::from_hex("00000000").unwrap(),
326 12 : Lsn::from_hex("00000000").unwrap(),
327 12 : Lsn::from_hex("00000000").unwrap(),
328 12 : 0,
329 12 : );
330 12 : let bytes = instance.to_bytes().unwrap();
331 12 : Self::from_bytes(&bytes).unwrap()
332 12 : }
333 :
334 1114 : pub(crate) fn apply(&mut self, update: &MetadataUpdate) {
335 1114 : self.body.disk_consistent_lsn = update.disk_consistent_lsn;
336 1114 : self.body.prev_record_lsn = update.prev_record_lsn;
337 1114 : self.body.latest_gc_cutoff_lsn = update.latest_gc_cutoff_lsn;
338 1114 : }
339 : }
340 :
341 : pub(crate) mod modern_serde {
342 : use super::{TimelineMetadata, TimelineMetadataBodyV2, TimelineMetadataHeader};
343 : use serde::{Deserialize, Serialize};
344 :
345 40 : pub(crate) fn deserialize<'de, D>(deserializer: D) -> Result<TimelineMetadata, D::Error>
346 40 : where
347 40 : D: serde::de::Deserializer<'de>,
348 40 : {
349 40 : // for legacy reasons versions 1-5 had TimelineMetadata serialized as a Vec<u8> field with
350 40 : // BeSer.
351 40 : struct Visitor;
352 40 :
353 40 : impl<'d> serde::de::Visitor<'d> for Visitor {
354 40 : type Value = TimelineMetadata;
355 40 :
356 40 : fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
357 0 : f.write_str("BeSer bytes or json structure")
358 0 : }
359 40 :
360 40 : fn visit_seq<A>(self, seq: A) -> Result<Self::Value, A::Error>
361 16 : where
362 16 : A: serde::de::SeqAccess<'d>,
363 16 : {
364 16 : use serde::de::Error;
365 16 : let de = serde::de::value::SeqAccessDeserializer::new(seq);
366 16 : Vec::<u8>::deserialize(de)
367 16 : .map(|v| TimelineMetadata::from_bytes(&v).map_err(A::Error::custom))?
368 40 : }
369 40 :
370 40 : fn visit_map<A>(self, map: A) -> Result<Self::Value, A::Error>
371 24 : where
372 24 : A: serde::de::MapAccess<'d>,
373 24 : {
374 24 : use serde::de::Error;
375 24 :
376 24 : let de = serde::de::value::MapAccessDeserializer::new(map);
377 40 : let body = TimelineMetadataBodyV2::deserialize(de)?;
378 40 : let hdr = TimelineMetadataHeader::try_from(&body).map_err(A::Error::custom)?;
379 40 :
380 40 : Ok(TimelineMetadata { hdr, body })
381 40 : }
382 40 : }
383 40 :
384 40 : deserializer.deserialize_any(Visitor)
385 40 : }
386 :
387 2779 : pub(crate) fn serialize<S>(
388 2779 : metadata: &TimelineMetadata,
389 2779 : serializer: S,
390 2779 : ) -> Result<S::Ok, S::Error>
391 2779 : where
392 2779 : S: serde::Serializer,
393 2779 : {
394 2779 : // header is not needed, upon reading we've upgraded all v1 to v2
395 2779 : metadata.body.serialize(serializer)
396 2779 : }
397 :
398 : #[test]
399 2 : fn deserializes_bytes_as_well_as_equivalent_body_v2() {
400 4 : #[derive(serde::Deserialize, serde::Serialize)]
401 2 : struct Wrapper(
402 2 : #[serde(deserialize_with = "deserialize", serialize_with = "serialize")]
403 2 : TimelineMetadata,
404 2 : );
405 2 :
406 2 : let too_many_bytes = "[216,111,252,208,0,54,0,4,0,0,0,0,1,73,253,144,1,0,0,0,0,1,73,253,24,0,0,0,0,0,0,0,0,0,0,0,0,0,1,73,253,24,0,0,0,0,1,73,253,24,0,0,0,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]";
407 2 :
408 2 : let wrapper_from_bytes = serde_json::from_str::<Wrapper>(too_many_bytes).unwrap();
409 2 :
410 2 : let serialized = serde_json::to_value(&wrapper_from_bytes).unwrap();
411 2 :
412 2 : assert_eq!(
413 2 : serialized,
414 2 : serde_json::json! {{
415 2 : "disk_consistent_lsn": "0/149FD90",
416 2 : "prev_record_lsn": "0/149FD18",
417 2 : "ancestor_timeline": null,
418 2 : "ancestor_lsn": "0/0",
419 2 : "latest_gc_cutoff_lsn": "0/149FD18",
420 2 : "initdb_lsn": "0/149FD18",
421 2 : "pg_version": 15
422 2 : }}
423 2 : );
424 :
425 2 : let wrapper_from_json = serde_json::value::from_value::<Wrapper>(serialized).unwrap();
426 2 :
427 2 : assert_eq!(wrapper_from_bytes.0, wrapper_from_json.0);
428 2 : }
429 : }
430 :
431 : /// Parts of the metadata which are regularly modified.
432 : pub(crate) struct MetadataUpdate {
433 : disk_consistent_lsn: Lsn,
434 : prev_record_lsn: Option<Lsn>,
435 : latest_gc_cutoff_lsn: Lsn,
436 : }
437 :
438 : impl MetadataUpdate {
439 1114 : pub(crate) fn new(
440 1114 : disk_consistent_lsn: Lsn,
441 1114 : prev_record_lsn: Option<Lsn>,
442 1114 : latest_gc_cutoff_lsn: Lsn,
443 1114 : ) -> Self {
444 1114 : Self {
445 1114 : disk_consistent_lsn,
446 1114 : prev_record_lsn,
447 1114 : latest_gc_cutoff_lsn,
448 1114 : }
449 1114 : }
450 : }
451 :
452 : #[cfg(test)]
453 : mod tests {
454 : use super::*;
455 : use crate::tenant::harness::TIMELINE_ID;
456 :
457 : #[test]
458 2 : fn metadata_serializes_correctly() {
459 2 : let original_metadata = TimelineMetadata::new(
460 2 : Lsn(0x200),
461 2 : Some(Lsn(0x100)),
462 2 : Some(TIMELINE_ID),
463 2 : Lsn(0),
464 2 : Lsn(0),
465 2 : Lsn(0),
466 2 : // Any version will do here, so use the default
467 2 : crate::DEFAULT_PG_VERSION,
468 2 : );
469 2 :
470 2 : let metadata_bytes = original_metadata
471 2 : .to_bytes()
472 2 : .expect("Should serialize correct metadata to bytes");
473 2 :
474 2 : let deserialized_metadata = TimelineMetadata::from_bytes(&metadata_bytes)
475 2 : .expect("Should deserialize its own bytes");
476 2 :
477 2 : assert_eq!(
478 : deserialized_metadata.body, original_metadata.body,
479 0 : "Metadata that was serialized to bytes and deserialized back should not change"
480 : );
481 2 : }
482 :
483 : // Generate old version metadata and read it with current code.
484 : // Ensure that it is upgraded correctly
485 : #[test]
486 2 : fn test_metadata_upgrade() {
487 2 : #[derive(Debug, Clone, PartialEq, Eq)]
488 2 : struct TimelineMetadataV1 {
489 2 : hdr: TimelineMetadataHeader,
490 2 : body: TimelineMetadataBodyV1,
491 2 : }
492 2 :
493 2 : let metadata_v1 = TimelineMetadataV1 {
494 2 : hdr: TimelineMetadataHeader {
495 2 : checksum: 0,
496 2 : size: 0,
497 2 : format_version: METADATA_OLD_FORMAT_VERSION,
498 2 : },
499 2 : body: TimelineMetadataBodyV1 {
500 2 : disk_consistent_lsn: Lsn(0x200),
501 2 : prev_record_lsn: Some(Lsn(0x100)),
502 2 : ancestor_timeline: Some(TIMELINE_ID),
503 2 : ancestor_lsn: Lsn(0),
504 2 : latest_gc_cutoff_lsn: Lsn(0),
505 2 : initdb_lsn: Lsn(0),
506 2 : },
507 2 : };
508 2 :
509 2 : impl TimelineMetadataV1 {
510 2 : pub fn to_bytes(&self) -> anyhow::Result<Vec<u8>> {
511 2 : let body_bytes = self.body.ser()?;
512 2 : let metadata_size = METADATA_HDR_SIZE + body_bytes.len();
513 2 : let hdr = TimelineMetadataHeader {
514 2 : size: metadata_size as u16,
515 2 : format_version: METADATA_OLD_FORMAT_VERSION,
516 2 : checksum: crc32c::crc32c(&body_bytes),
517 2 : };
518 2 : let hdr_bytes = hdr.ser()?;
519 2 : let mut metadata_bytes = vec![0u8; METADATA_MAX_SIZE];
520 2 : metadata_bytes[0..METADATA_HDR_SIZE].copy_from_slice(&hdr_bytes);
521 2 : metadata_bytes[METADATA_HDR_SIZE..metadata_size].copy_from_slice(&body_bytes);
522 2 : Ok(metadata_bytes)
523 2 : }
524 2 : }
525 2 :
526 2 : let metadata_bytes = metadata_v1
527 2 : .to_bytes()
528 2 : .expect("Should serialize correct metadata to bytes");
529 2 :
530 2 : // This should deserialize to the latest version format
531 2 : let deserialized_metadata = TimelineMetadata::from_bytes(&metadata_bytes)
532 2 : .expect("Should deserialize its own bytes");
533 2 :
534 2 : let expected_metadata = TimelineMetadata::new(
535 2 : Lsn(0x200),
536 2 : Some(Lsn(0x100)),
537 2 : Some(TIMELINE_ID),
538 2 : Lsn(0),
539 2 : Lsn(0),
540 2 : Lsn(0),
541 2 : 14, // All timelines created before this version had pg_version 14
542 2 : );
543 2 :
544 2 : assert_eq!(
545 : deserialized_metadata.body, expected_metadata.body,
546 0 : "Metadata of the old version {} should be upgraded to the latest version {}",
547 : METADATA_OLD_FORMAT_VERSION, METADATA_FORMAT_VERSION
548 : );
549 2 : }
550 :
551 : #[test]
552 2 : fn test_metadata_bincode_serde_ensure_roundtrip() {
553 2 : let original_metadata = TimelineMetadata::new(
554 2 : Lsn(0x200),
555 2 : Some(Lsn(0x100)),
556 2 : Some(TIMELINE_ID),
557 2 : Lsn(0),
558 2 : Lsn(0),
559 2 : Lsn(0),
560 2 : // Any version will do here, so use the default
561 2 : crate::DEFAULT_PG_VERSION,
562 2 : );
563 2 : let expected_bytes = vec![
564 2 : /* TimelineMetadataHeader */
565 2 : 4, 37, 101, 34, 0, 70, 0, 4, // checksum, size, format_version (4 + 2 + 2)
566 2 : /* TimelineMetadataBodyV2 */
567 2 : 0, 0, 0, 0, 0, 0, 2, 0, // disk_consistent_lsn (8 bytes)
568 2 : 1, 0, 0, 0, 0, 0, 0, 1, 0, // prev_record_lsn (9 bytes)
569 2 : 1, 17, 34, 51, 68, 85, 102, 119, 136, 17, 34, 51, 68, 85, 102, 119,
570 2 : 136, // ancestor_timeline (17 bytes)
571 2 : 0, 0, 0, 0, 0, 0, 0, 0, // ancestor_lsn (8 bytes)
572 2 : 0, 0, 0, 0, 0, 0, 0, 0, // latest_gc_cutoff_lsn (8 bytes)
573 2 : 0, 0, 0, 0, 0, 0, 0, 0, // initdb_lsn (8 bytes)
574 2 : 0, 0, 0, 15, // pg_version (4 bytes)
575 2 : /* padding bytes */
576 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
577 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
578 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
579 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
580 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
581 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
582 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
583 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
584 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
585 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
586 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
587 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
588 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
589 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
590 2 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
591 2 : 0, 0, 0, 0, 0, 0, 0,
592 2 : ];
593 2 : let metadata_ser_bytes = original_metadata.to_bytes().unwrap();
594 2 : assert_eq!(metadata_ser_bytes, expected_bytes);
595 :
596 2 : let expected_metadata = {
597 2 : let mut temp_metadata = original_metadata;
598 2 : let body_bytes = temp_metadata
599 2 : .body
600 2 : .ser()
601 2 : .expect("Cannot serialize the metadata body");
602 2 : let metadata_size = METADATA_HDR_SIZE + body_bytes.len();
603 2 : let hdr = TimelineMetadataHeader {
604 2 : size: metadata_size as u16,
605 2 : format_version: METADATA_FORMAT_VERSION,
606 2 : checksum: crc32c::crc32c(&body_bytes),
607 2 : };
608 2 : temp_metadata.hdr = hdr;
609 2 : temp_metadata
610 2 : };
611 2 : let des_metadata = TimelineMetadata::from_bytes(&metadata_ser_bytes).unwrap();
612 2 : assert_eq!(des_metadata, expected_metadata);
613 2 : }
614 : }
|
