LCOV - code coverage report
Current view: top level - pageserver/src/tenant - metadata.rs (source / functions) Coverage Total Hit
Test: 07bee600374ccd486c69370d0972d9035964fe68.info Lines: 92.1 % 369 340
Test Date: 2025-02-20 13:11:02 Functions: 58.2 % 67 39

            Line data    Source code
       1              : //! Describes the legacy now hopefully no longer modified per-timeline metadata.
       2              : //!
       3              : //! It is stored in `index_part.json` managed by [`remote_timeline_client`]. For many tenants and
       4              : //! their timelines, this struct and its original serialization format is still needed because
       5              : //! they were written a long time ago.
       6              : //!
       7              : //! Instead of changing and adding versioning to this, just change [`IndexPart`] with soft json
       8              : //! versioning.
       9              : //!
      10              : //! To clean up this module we need to migrate all index_part.json files to a later version.
      11              : //! While doing this, we need to be mindful about s3 based recovery as well, so it might take
      12              : //! however long we keep the old versions to be able to delete the old code. After that, we can
      13              : //! remove everything else than [`TimelineMetadataBodyV2`], rename it as `TimelineMetadata` and
      14              : //! move it to `index.rs`. Before doing all of this, we need to keep the structures for backwards
      15              : //! compatibility.
      16              : //!
      17              : //! [`remote_timeline_client`]: super::remote_timeline_client
      18              : //! [`IndexPart`]: super::remote_timeline_client::index::IndexPart
      19              : 
      20              : use anyhow::ensure;
      21              : use serde::{Deserialize, Serialize};
      22              : use utils::bin_ser::SerializeError;
      23              : use utils::{bin_ser::BeSer, id::TimelineId, lsn::Lsn};
      24              : 
      25              : /// Use special format number to enable backward compatibility.
      26              : const METADATA_FORMAT_VERSION: u16 = 4;
      27              : 
      28              : /// Previous supported format versions.
      29              : ///
      30              : /// In practice, none of these should remain, all are [`METADATA_FORMAT_VERSION`], but confirming
      31              : /// that requires a scrubber run which is yet to be done.
      32              : const METADATA_OLD_FORMAT_VERSION: u16 = 3;
      33              : 
      34              : /// When the file existed on disk we assumed that a write of up to METADATA_MAX_SIZE bytes is atomic.
      35              : ///
      36              : /// This is the same assumption that PostgreSQL makes with the control file,
      37              : ///
      38              : /// see PG_CONTROL_MAX_SAFE_SIZE
      39              : const METADATA_MAX_SIZE: usize = 512;
      40              : 
      41              : /// Legacy metadata stored as a component of `index_part.json` per timeline.
      42              : ///
      43              : /// Do not make new changes to this type or the module. In production, we have two different kinds
      44              : /// of serializations of this type: bincode and json. Bincode version reflects what used to be
      45              : /// stored on disk in earlier versions and does internal crc32 checksumming.
      46              : ///
      47              : /// This type should not implement `serde::Serialize` or `serde::Deserialize` because there would
      48              : /// be a confusion whether you want the old version ([`TimelineMetadata::from_bytes`]) or the modern
      49              : /// as-exists in `index_part.json` ([`self::modern_serde`]).
      50              : ///
      51              : /// ```compile_fail
      52              : /// #[derive(serde::Serialize)]
      53              : /// struct DoNotDoThis(pageserver::tenant::metadata::TimelineMetadata);
      54              : /// ```
      55              : ///
      56              : /// ```compile_fail
      57              : /// #[derive(serde::Deserialize)]
      58              : /// struct NeitherDoThis(pageserver::tenant::metadata::TimelineMetadata);
      59              : /// ```
      60              : #[derive(Debug, Clone, PartialEq, Eq)]
      61              : pub struct TimelineMetadata {
      62              :     hdr: TimelineMetadataHeader,
      63              :     body: TimelineMetadataBodyV2,
      64              : }
      65              : 
      66            0 : #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
      67              : struct TimelineMetadataHeader {
      68              :     checksum: u32,       // CRC of serialized metadata body
      69              :     size: u16,           // size of serialized metadata
      70              :     format_version: u16, // metadata format version (used for compatibility checks)
      71              : }
      72              : 
      73              : impl TryFrom<&TimelineMetadataBodyV2> for TimelineMetadataHeader {
      74              :     type Error = Crc32CalculationFailed;
      75              : 
      76           96 :     fn try_from(value: &TimelineMetadataBodyV2) -> Result<Self, Self::Error> {
      77              :         #[derive(Default)]
      78              :         struct Crc32Sink {
      79              :             crc: u32,
      80              :             count: usize,
      81              :         }
      82              : 
      83              :         impl std::io::Write for Crc32Sink {
      84         1576 :             fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
      85         1576 :                 self.crc = crc32c::crc32c_append(self.crc, buf);
      86         1576 :                 self.count += buf.len();
      87         1576 :                 Ok(buf.len())
      88         1576 :             }
      89              : 
      90            0 :             fn flush(&mut self) -> std::io::Result<()> {
      91            0 :                 Ok(())
      92            0 :             }
      93              :         }
      94              : 
      95              :         // jump through hoops to calculate the crc32 so that TimelineMetadata::ne works
      96              :         // across serialization versions
      97           96 :         let mut sink = Crc32Sink::default();
      98           96 :         <TimelineMetadataBodyV2 as utils::bin_ser::BeSer>::ser_into(value, &mut sink)
      99           96 :             .map_err(Crc32CalculationFailed)?;
     100              : 
     101           96 :         let size = METADATA_HDR_SIZE + sink.count;
     102           96 : 
     103           96 :         Ok(TimelineMetadataHeader {
     104           96 :             checksum: sink.crc,
     105           96 :             size: size as u16,
     106           96 :             format_version: METADATA_FORMAT_VERSION,
     107           96 :         })
     108           96 :     }
     109              : }
     110              : 
     111              : #[derive(thiserror::Error, Debug)]
     112              : #[error("re-serializing for crc32 failed")]
     113              : struct Crc32CalculationFailed(#[source] utils::bin_ser::SerializeError);
     114              : 
     115              : const METADATA_HDR_SIZE: usize = size_of::<TimelineMetadataHeader>();
     116              : 
     117          504 : #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
     118              : struct TimelineMetadataBodyV2 {
     119              :     disk_consistent_lsn: Lsn,
     120              :     // This is only set if we know it. We track it in memory when the page
     121              :     // server is running, but we only track the value corresponding to
     122              :     // 'last_record_lsn', not 'disk_consistent_lsn' which can lag behind by a
     123              :     // lot. We only store it in the metadata file when we flush *all* the
     124              :     // in-memory data so that 'last_record_lsn' is the same as
     125              :     // 'disk_consistent_lsn'.  That's OK, because after page server restart, as
     126              :     // soon as we reprocess at least one record, we will have a valid
     127              :     // 'prev_record_lsn' value in memory again. This is only really needed when
     128              :     // doing a clean shutdown, so that there is no more WAL beyond
     129              :     // 'disk_consistent_lsn'
     130              :     prev_record_lsn: Option<Lsn>,
     131              :     ancestor_timeline: Option<TimelineId>,
     132              :     ancestor_lsn: Lsn,
     133              : 
     134              :     // The LSN at which GC was last executed.  Synonym of [`Timeline::applied_gc_cutoff_lsn`].
     135              :     latest_gc_cutoff_lsn: Lsn,
     136              : 
     137              :     initdb_lsn: Lsn,
     138              :     pg_version: u32,
     139              : }
     140              : 
     141            0 : #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
     142              : struct TimelineMetadataBodyV1 {
     143              :     disk_consistent_lsn: Lsn,
     144              :     // This is only set if we know it. We track it in memory when the page
     145              :     // server is running, but we only track the value corresponding to
     146              :     // 'last_record_lsn', not 'disk_consistent_lsn' which can lag behind by a
     147              :     // lot. We only store it in the metadata file when we flush *all* the
     148              :     // in-memory data so that 'last_record_lsn' is the same as
     149              :     // 'disk_consistent_lsn'.  That's OK, because after page server restart, as
     150              :     // soon as we reprocess at least one record, we will have a valid
     151              :     // 'prev_record_lsn' value in memory again. This is only really needed when
     152              :     // doing a clean shutdown, so that there is no more WAL beyond
     153              :     // 'disk_consistent_lsn'
     154              :     prev_record_lsn: Option<Lsn>,
     155              :     ancestor_timeline: Option<TimelineId>,
     156              :     ancestor_lsn: Lsn,
     157              :     latest_gc_cutoff_lsn: Lsn,
     158              :     initdb_lsn: Lsn,
     159              : }
     160              : 
     161              : impl TimelineMetadata {
     162         1092 :     pub fn new(
     163         1092 :         disk_consistent_lsn: Lsn,
     164         1092 :         prev_record_lsn: Option<Lsn>,
     165         1092 :         ancestor_timeline: Option<TimelineId>,
     166         1092 :         ancestor_lsn: Lsn,
     167         1092 :         latest_gc_cutoff_lsn: Lsn,
     168         1092 :         initdb_lsn: Lsn,
     169         1092 :         pg_version: u32,
     170         1092 :     ) -> Self {
     171         1092 :         Self {
     172         1092 :             hdr: TimelineMetadataHeader {
     173         1092 :                 checksum: 0,
     174         1092 :                 size: 0,
     175         1092 :                 format_version: METADATA_FORMAT_VERSION,
     176         1092 :             },
     177         1092 :             body: TimelineMetadataBodyV2 {
     178         1092 :                 disk_consistent_lsn,
     179         1092 :                 prev_record_lsn,
     180         1092 :                 ancestor_timeline,
     181         1092 :                 ancestor_lsn,
     182         1092 :                 latest_gc_cutoff_lsn,
     183         1092 :                 initdb_lsn,
     184         1092 :                 pg_version,
     185         1092 :             },
     186         1092 :         }
     187         1092 :     }
     188              : 
     189              :     #[cfg(test)]
     190           24 :     pub(crate) fn with_recalculated_checksum(mut self) -> anyhow::Result<Self> {
     191           24 :         self.hdr = TimelineMetadataHeader::try_from(&self.body)?;
     192           24 :         Ok(self)
     193           24 :     }
     194              : 
     195            4 :     fn upgrade_timeline_metadata(metadata_bytes: &[u8]) -> anyhow::Result<Self> {
     196            4 :         let mut hdr = TimelineMetadataHeader::des(&metadata_bytes[0..METADATA_HDR_SIZE])?;
     197              : 
     198              :         // backward compatible only up to this version
     199            4 :         ensure!(
     200            4 :             hdr.format_version == METADATA_OLD_FORMAT_VERSION,
     201            0 :             "unsupported metadata format version {}",
     202              :             hdr.format_version
     203              :         );
     204              : 
     205            4 :         let metadata_size = hdr.size as usize;
     206              : 
     207            4 :         let body: TimelineMetadataBodyV1 =
     208            4 :             TimelineMetadataBodyV1::des(&metadata_bytes[METADATA_HDR_SIZE..metadata_size])?;
     209              : 
     210            4 :         let body = TimelineMetadataBodyV2 {
     211            4 :             disk_consistent_lsn: body.disk_consistent_lsn,
     212            4 :             prev_record_lsn: body.prev_record_lsn,
     213            4 :             ancestor_timeline: body.ancestor_timeline,
     214            4 :             ancestor_lsn: body.ancestor_lsn,
     215            4 :             latest_gc_cutoff_lsn: body.latest_gc_cutoff_lsn,
     216            4 :             initdb_lsn: body.initdb_lsn,
     217            4 :             pg_version: 14, // All timelines created before this version had pg_version 14
     218            4 :         };
     219            4 : 
     220            4 :         hdr.format_version = METADATA_FORMAT_VERSION;
     221            4 : 
     222            4 :         Ok(Self { hdr, body })
     223            4 :     }
     224              : 
     225          244 :     pub fn from_bytes(metadata_bytes: &[u8]) -> anyhow::Result<Self> {
     226          244 :         ensure!(
     227          244 :             metadata_bytes.len() == METADATA_MAX_SIZE,
     228            0 :             "metadata bytes size is wrong"
     229              :         );
     230          244 :         let hdr = TimelineMetadataHeader::des(&metadata_bytes[0..METADATA_HDR_SIZE])?;
     231              : 
     232          244 :         let metadata_size = hdr.size as usize;
     233          244 :         ensure!(
     234          244 :             metadata_size <= METADATA_MAX_SIZE,
     235            0 :             "corrupted metadata file"
     236              :         );
     237          244 :         let calculated_checksum = crc32c::crc32c(&metadata_bytes[METADATA_HDR_SIZE..metadata_size]);
     238          244 :         ensure!(
     239          244 :             hdr.checksum == calculated_checksum,
     240            0 :             "metadata checksum mismatch"
     241              :         );
     242              : 
     243          244 :         if hdr.format_version != METADATA_FORMAT_VERSION {
     244              :             // If metadata has the old format,
     245              :             // upgrade it and return the result
     246            4 :             TimelineMetadata::upgrade_timeline_metadata(metadata_bytes)
     247              :         } else {
     248          240 :             let body =
     249          240 :                 TimelineMetadataBodyV2::des(&metadata_bytes[METADATA_HDR_SIZE..metadata_size])?;
     250          240 :             ensure!(
     251          240 :                 body.disk_consistent_lsn.is_aligned(),
     252            0 :                 "disk_consistent_lsn is not aligned"
     253              :             );
     254          240 :             Ok(TimelineMetadata { hdr, body })
     255              :         }
     256          244 :     }
     257              : 
     258          180 :     pub fn to_bytes(&self) -> Result<Vec<u8>, SerializeError> {
     259          180 :         let body_bytes = self.body.ser()?;
     260          180 :         let metadata_size = METADATA_HDR_SIZE + body_bytes.len();
     261          180 :         let hdr = TimelineMetadataHeader {
     262          180 :             size: metadata_size as u16,
     263          180 :             format_version: METADATA_FORMAT_VERSION,
     264          180 :             checksum: crc32c::crc32c(&body_bytes),
     265          180 :         };
     266          180 :         let hdr_bytes = hdr.ser()?;
     267          180 :         let mut metadata_bytes = vec![0u8; METADATA_MAX_SIZE];
     268          180 :         metadata_bytes[0..METADATA_HDR_SIZE].copy_from_slice(&hdr_bytes);
     269          180 :         metadata_bytes[METADATA_HDR_SIZE..metadata_size].copy_from_slice(&body_bytes);
     270          180 :         Ok(metadata_bytes)
     271          180 :     }
     272              : 
     273              :     /// [`Lsn`] that corresponds to the corresponding timeline directory
     274              :     /// contents, stored locally in the pageserver workdir.
     275        43644 :     pub fn disk_consistent_lsn(&self) -> Lsn {
     276        43644 :         self.body.disk_consistent_lsn
     277        43644 :     }
     278              : 
     279          896 :     pub fn prev_record_lsn(&self) -> Option<Lsn> {
     280          896 :         self.body.prev_record_lsn
     281          896 :     }
     282              : 
     283         6366 :     pub fn ancestor_timeline(&self) -> Option<TimelineId> {
     284         6366 :         self.body.ancestor_timeline
     285         6366 :     }
     286              : 
     287         1360 :     pub fn ancestor_lsn(&self) -> Lsn {
     288         1360 :         self.body.ancestor_lsn
     289         1360 :     }
     290              : 
     291              :     /// When reparenting, the `ancestor_lsn` does not change.
     292              :     ///
     293              :     /// Returns true if anything was changed.
     294            0 :     pub fn reparent(&mut self, timeline: &TimelineId) {
     295            0 :         assert!(self.body.ancestor_timeline.is_some());
     296              :         // no assertion for redoing this: it's fine, we may have to repeat this multiple times over
     297            0 :         self.body.ancestor_timeline = Some(*timeline);
     298            0 :     }
     299              : 
     300              :     /// Returns true if anything was changed
     301            0 :     pub fn detach_from_ancestor(&mut self, branchpoint: &(TimelineId, Lsn)) {
     302            0 :         if let Some(ancestor) = self.body.ancestor_timeline {
     303            0 :             assert_eq!(ancestor, branchpoint.0);
     304            0 :         }
     305            0 :         if self.body.ancestor_lsn != Lsn(0) {
     306            0 :             assert_eq!(self.body.ancestor_lsn, branchpoint.1);
     307            0 :         }
     308            0 :         self.body.ancestor_timeline = None;
     309            0 :         self.body.ancestor_lsn = Lsn(0);
     310            0 :     }
     311              : 
     312          896 :     pub fn latest_gc_cutoff_lsn(&self) -> Lsn {
     313          896 :         self.body.latest_gc_cutoff_lsn
     314          896 :     }
     315              : 
     316          896 :     pub fn initdb_lsn(&self) -> Lsn {
     317          896 :         self.body.initdb_lsn
     318          896 :     }
     319              : 
     320          904 :     pub fn pg_version(&self) -> u32 {
     321          904 :         self.body.pg_version
     322          904 :     }
     323              : 
     324              :     // Checksums make it awkward to build a valid instance by hand.  This helper
     325              :     // provides a TimelineMetadata with a valid checksum in its header.
     326          168 :     pub fn example() -> Self {
     327          168 :         let instance = Self::new(
     328          168 :             "0/16960E8".parse::<Lsn>().unwrap(),
     329          168 :             None,
     330          168 :             None,
     331          168 :             Lsn::from_hex("00000000").unwrap(),
     332          168 :             Lsn::from_hex("00000000").unwrap(),
     333          168 :             Lsn::from_hex("00000000").unwrap(),
     334          168 :             0,
     335          168 :         );
     336          168 :         let bytes = instance.to_bytes().unwrap();
     337          168 :         Self::from_bytes(&bytes).unwrap()
     338          168 :     }
     339              : 
     340         2448 :     pub(crate) fn apply(&mut self, update: &MetadataUpdate) {
     341         2448 :         self.body.disk_consistent_lsn = update.disk_consistent_lsn;
     342         2448 :         self.body.prev_record_lsn = update.prev_record_lsn;
     343         2448 :         self.body.latest_gc_cutoff_lsn = update.latest_gc_cutoff_lsn;
     344         2448 :     }
     345              : }
     346              : 
     347              : pub(crate) mod modern_serde {
     348              :     use super::{TimelineMetadata, TimelineMetadataBodyV2, TimelineMetadataHeader};
     349              :     use serde::{Deserialize, Serialize};
     350              : 
     351          104 :     pub(crate) fn deserialize<'de, D>(deserializer: D) -> Result<TimelineMetadata, D::Error>
     352          104 :     where
     353          104 :         D: serde::de::Deserializer<'de>,
     354          104 :     {
     355              :         // for legacy reasons versions 1-5 had TimelineMetadata serialized as a Vec<u8> field with
     356              :         // BeSer.
     357              :         struct Visitor;
     358              : 
     359              :         impl<'d> serde::de::Visitor<'d> for Visitor {
     360              :             type Value = TimelineMetadata;
     361              : 
     362            0 :             fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
     363            0 :                 f.write_str("BeSer bytes or json structure")
     364            0 :             }
     365              : 
     366           32 :             fn visit_seq<A>(self, seq: A) -> Result<Self::Value, A::Error>
     367           32 :             where
     368           32 :                 A: serde::de::SeqAccess<'d>,
     369           32 :             {
     370              :                 use serde::de::Error;
     371           32 :                 let de = serde::de::value::SeqAccessDeserializer::new(seq);
     372           32 :                 Vec::<u8>::deserialize(de)
     373           32 :                     .map(|v| TimelineMetadata::from_bytes(&v).map_err(A::Error::custom))?
     374           32 :             }
     375              : 
     376           72 :             fn visit_map<A>(self, map: A) -> Result<Self::Value, A::Error>
     377           72 :             where
     378           72 :                 A: serde::de::MapAccess<'d>,
     379           72 :             {
     380              :                 use serde::de::Error;
     381              : 
     382           72 :                 let de = serde::de::value::MapAccessDeserializer::new(map);
     383           72 :                 let body = TimelineMetadataBodyV2::deserialize(de)?;
     384           72 :                 let hdr = TimelineMetadataHeader::try_from(&body).map_err(A::Error::custom)?;
     385              : 
     386           72 :                 Ok(TimelineMetadata { hdr, body })
     387           72 :             }
     388              :         }
     389              : 
     390          104 :         deserializer.deserialize_any(Visitor)
     391          104 :     }
     392              : 
     393         6094 :     pub(crate) fn serialize<S>(
     394         6094 :         metadata: &TimelineMetadata,
     395         6094 :         serializer: S,
     396         6094 :     ) -> Result<S::Ok, S::Error>
     397         6094 :     where
     398         6094 :         S: serde::Serializer,
     399         6094 :     {
     400         6094 :         // header is not needed, upon reading we've upgraded all v1 to v2
     401         6094 :         metadata.body.serialize(serializer)
     402         6094 :     }
     403              : 
     404              :     #[test]
     405            4 :     fn deserializes_bytes_as_well_as_equivalent_body_v2() {
     406            4 :         #[derive(serde::Deserialize, serde::Serialize)]
     407              :         struct Wrapper(
     408              :             #[serde(deserialize_with = "deserialize", serialize_with = "serialize")]
     409              :             TimelineMetadata,
     410              :         );
     411              : 
     412            4 :         let too_many_bytes = "[216,111,252,208,0,54,0,4,0,0,0,0,1,73,253,144,1,0,0,0,0,1,73,253,24,0,0,0,0,0,0,0,0,0,0,0,0,0,1,73,253,24,0,0,0,0,1,73,253,24,0,0,0,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]";
     413            4 : 
     414            4 :         let wrapper_from_bytes = serde_json::from_str::<Wrapper>(too_many_bytes).unwrap();
     415            4 : 
     416            4 :         let serialized = serde_json::to_value(&wrapper_from_bytes).unwrap();
     417            4 : 
     418            4 :         assert_eq!(
     419            4 :             serialized,
     420            4 :             serde_json::json! {{
     421            4 :                 "disk_consistent_lsn": "0/149FD90",
     422            4 :                 "prev_record_lsn": "0/149FD18",
     423            4 :                 "ancestor_timeline": null,
     424            4 :                 "ancestor_lsn": "0/0",
     425            4 :                 "latest_gc_cutoff_lsn": "0/149FD18",
     426            4 :                 "initdb_lsn": "0/149FD18",
     427            4 :                 "pg_version": 15
     428            4 :             }}
     429            4 :         );
     430              : 
     431            4 :         let wrapper_from_json = serde_json::value::from_value::<Wrapper>(serialized).unwrap();
     432            4 : 
     433            4 :         assert_eq!(wrapper_from_bytes.0, wrapper_from_json.0);
     434            4 :     }
     435              : }
     436              : 
     437              : /// Parts of the metadata which are regularly modified.
     438              : pub(crate) struct MetadataUpdate {
     439              :     disk_consistent_lsn: Lsn,
     440              :     prev_record_lsn: Option<Lsn>,
     441              :     latest_gc_cutoff_lsn: Lsn,
     442              : }
     443              : 
     444              : impl MetadataUpdate {
     445         2448 :     pub(crate) fn new(
     446         2448 :         disk_consistent_lsn: Lsn,
     447         2448 :         prev_record_lsn: Option<Lsn>,
     448         2448 :         latest_gc_cutoff_lsn: Lsn,
     449         2448 :     ) -> Self {
     450         2448 :         Self {
     451         2448 :             disk_consistent_lsn,
     452         2448 :             prev_record_lsn,
     453         2448 :             latest_gc_cutoff_lsn,
     454         2448 :         }
     455         2448 :     }
     456              : }
     457              : 
     458              : #[cfg(test)]
     459              : mod tests {
     460              :     use super::*;
     461              :     use crate::tenant::harness::TIMELINE_ID;
     462              : 
     463              :     #[test]
     464            4 :     fn metadata_serializes_correctly() {
     465            4 :         let original_metadata = TimelineMetadata::new(
     466            4 :             Lsn(0x200),
     467            4 :             Some(Lsn(0x100)),
     468            4 :             Some(TIMELINE_ID),
     469            4 :             Lsn(0),
     470            4 :             Lsn(0),
     471            4 :             Lsn(0),
     472            4 :             // Any version will do here, so use the default
     473            4 :             crate::DEFAULT_PG_VERSION,
     474            4 :         );
     475            4 : 
     476            4 :         let metadata_bytes = original_metadata
     477            4 :             .to_bytes()
     478            4 :             .expect("Should serialize correct metadata to bytes");
     479            4 : 
     480            4 :         let deserialized_metadata = TimelineMetadata::from_bytes(&metadata_bytes)
     481            4 :             .expect("Should deserialize its own bytes");
     482            4 : 
     483            4 :         assert_eq!(
     484              :             deserialized_metadata.body, original_metadata.body,
     485            0 :             "Metadata that was serialized to bytes and deserialized back should not change"
     486              :         );
     487            4 :     }
     488              : 
     489              :     // Generate old version metadata and read it with current code.
     490              :     // Ensure that it is upgraded correctly
     491              :     #[test]
     492            4 :     fn test_metadata_upgrade() {
     493              :         #[derive(Debug, Clone, PartialEq, Eq)]
     494              :         struct TimelineMetadataV1 {
     495              :             hdr: TimelineMetadataHeader,
     496              :             body: TimelineMetadataBodyV1,
     497              :         }
     498              : 
     499            4 :         let metadata_v1 = TimelineMetadataV1 {
     500            4 :             hdr: TimelineMetadataHeader {
     501            4 :                 checksum: 0,
     502            4 :                 size: 0,
     503            4 :                 format_version: METADATA_OLD_FORMAT_VERSION,
     504            4 :             },
     505            4 :             body: TimelineMetadataBodyV1 {
     506            4 :                 disk_consistent_lsn: Lsn(0x200),
     507            4 :                 prev_record_lsn: Some(Lsn(0x100)),
     508            4 :                 ancestor_timeline: Some(TIMELINE_ID),
     509            4 :                 ancestor_lsn: Lsn(0),
     510            4 :                 latest_gc_cutoff_lsn: Lsn(0),
     511            4 :                 initdb_lsn: Lsn(0),
     512            4 :             },
     513            4 :         };
     514              : 
     515              :         impl TimelineMetadataV1 {
     516            4 :             pub fn to_bytes(&self) -> anyhow::Result<Vec<u8>> {
     517            4 :                 let body_bytes = self.body.ser()?;
     518            4 :                 let metadata_size = METADATA_HDR_SIZE + body_bytes.len();
     519            4 :                 let hdr = TimelineMetadataHeader {
     520            4 :                     size: metadata_size as u16,
     521            4 :                     format_version: METADATA_OLD_FORMAT_VERSION,
     522            4 :                     checksum: crc32c::crc32c(&body_bytes),
     523            4 :                 };
     524            4 :                 let hdr_bytes = hdr.ser()?;
     525            4 :                 let mut metadata_bytes = vec![0u8; METADATA_MAX_SIZE];
     526            4 :                 metadata_bytes[0..METADATA_HDR_SIZE].copy_from_slice(&hdr_bytes);
     527            4 :                 metadata_bytes[METADATA_HDR_SIZE..metadata_size].copy_from_slice(&body_bytes);
     528            4 :                 Ok(metadata_bytes)
     529            4 :             }
     530              :         }
     531              : 
     532            4 :         let metadata_bytes = metadata_v1
     533            4 :             .to_bytes()
     534            4 :             .expect("Should serialize correct metadata to bytes");
     535            4 : 
     536            4 :         // This should deserialize to the latest version format
     537            4 :         let deserialized_metadata = TimelineMetadata::from_bytes(&metadata_bytes)
     538            4 :             .expect("Should deserialize its own bytes");
     539            4 : 
     540            4 :         let expected_metadata = TimelineMetadata::new(
     541            4 :             Lsn(0x200),
     542            4 :             Some(Lsn(0x100)),
     543            4 :             Some(TIMELINE_ID),
     544            4 :             Lsn(0),
     545            4 :             Lsn(0),
     546            4 :             Lsn(0),
     547            4 :             14, // All timelines created before this version had pg_version 14
     548            4 :         );
     549            4 : 
     550            4 :         assert_eq!(
     551              :             deserialized_metadata.body, expected_metadata.body,
     552            0 :             "Metadata of the old version {} should be upgraded to the latest version {}",
     553              :             METADATA_OLD_FORMAT_VERSION, METADATA_FORMAT_VERSION
     554              :         );
     555            4 :     }
     556              : 
     557              :     #[test]
     558            4 :     fn test_metadata_bincode_serde_ensure_roundtrip() {
     559            4 :         let original_metadata = TimelineMetadata::new(
     560            4 :             Lsn(0x200),
     561            4 :             Some(Lsn(0x100)),
     562            4 :             Some(TIMELINE_ID),
     563            4 :             Lsn(0),
     564            4 :             Lsn(0),
     565            4 :             Lsn(0),
     566            4 :             // Any version will do here, so use the default
     567            4 :             crate::DEFAULT_PG_VERSION,
     568            4 :         );
     569            4 :         let expected_bytes = vec![
     570            4 :             /* TimelineMetadataHeader */
     571            4 :             74, 104, 158, 105, 0, 70, 0, 4, // checksum, size, format_version (4 + 2 + 2)
     572            4 :             /* TimelineMetadataBodyV2 */
     573            4 :             0, 0, 0, 0, 0, 0, 2, 0, // disk_consistent_lsn (8 bytes)
     574            4 :             1, 0, 0, 0, 0, 0, 0, 1, 0, // prev_record_lsn (9 bytes)
     575            4 :             1, 17, 34, 51, 68, 85, 102, 119, 136, 17, 34, 51, 68, 85, 102, 119,
     576            4 :             136, // ancestor_timeline (17 bytes)
     577            4 :             0, 0, 0, 0, 0, 0, 0, 0, // ancestor_lsn (8 bytes)
     578            4 :             0, 0, 0, 0, 0, 0, 0, 0, // latest_gc_cutoff_lsn (8 bytes)
     579            4 :             0, 0, 0, 0, 0, 0, 0, 0, // initdb_lsn (8 bytes)
     580            4 :             0, 0, 0, 16, // pg_version (4 bytes)
     581            4 :             /* padding bytes */
     582            4 :             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
     583            4 :             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
     584            4 :             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
     585            4 :             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
     586            4 :             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
     587            4 :             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
     588            4 :             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
     589            4 :             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
     590            4 :             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
     591            4 :             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
     592            4 :             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
     593            4 :             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
     594            4 :             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
     595            4 :             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
     596            4 :             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
     597            4 :             0, 0, 0, 0, 0, 0, 0,
     598            4 :         ];
     599            4 :         let metadata_ser_bytes = original_metadata.to_bytes().unwrap();
     600            4 :         assert_eq!(metadata_ser_bytes, expected_bytes);
     601              : 
     602            4 :         let expected_metadata = {
     603            4 :             let mut temp_metadata = original_metadata;
     604            4 :             let body_bytes = temp_metadata
     605            4 :                 .body
     606            4 :                 .ser()
     607            4 :                 .expect("Cannot serialize the metadata body");
     608            4 :             let metadata_size = METADATA_HDR_SIZE + body_bytes.len();
     609            4 :             let hdr = TimelineMetadataHeader {
     610            4 :                 size: metadata_size as u16,
     611            4 :                 format_version: METADATA_FORMAT_VERSION,
     612            4 :                 checksum: crc32c::crc32c(&body_bytes),
     613            4 :             };
     614            4 :             temp_metadata.hdr = hdr;
     615            4 :             temp_metadata
     616            4 :         };
     617            4 :         let des_metadata = TimelineMetadata::from_bytes(&metadata_ser_bytes).unwrap();
     618            4 :         assert_eq!(des_metadata, expected_metadata);
     619            4 :     }
     620              : }
        

Generated by: LCOV version 2.1-beta