Line data Source code
1 : use chrono::NaiveDateTime;
2 : use pageserver_api::shard::ShardStripeSize;
3 : use serde::{Deserialize, Serialize};
4 : use utils::id::TimelineId;
5 : use utils::lsn::Lsn;
6 :
7 : /// Tenant shard manifest, stored in remote storage. Contains offloaded timelines and other tenant
8 : /// shard-wide information that must be persisted in remote storage.
9 : ///
10 : /// The manifest is always updated on tenant attach, and as needed.
11 68 : #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
12 : pub struct TenantManifest {
13 : /// The manifest version. Incremented on manifest format changes, even non-breaking ones.
14 : /// Manifests must generally always be backwards and forwards compatible for one release, to
15 : /// allow release rollbacks.
16 : pub version: usize,
17 :
18 : /// This tenant's stripe size. This is only advisory, and used to recover tenant data from
19 : /// remote storage. The autoritative source is the storage controller. If None, assume the
20 : /// original default value of 32768 blocks (256 MB).
21 : #[serde(skip_serializing_if = "Option::is_none")]
22 : pub stripe_size: Option<ShardStripeSize>,
23 :
24 : /// The list of offloaded timelines together with enough information
25 : /// to not have to actually load them.
26 : ///
27 : /// Note: the timelines mentioned in this list might be deleted, i.e.
28 : /// we don't hold an invariant that the references aren't dangling.
29 : /// Existence of index-part.json is the actual indicator of timeline existence.
30 : #[serde(default)]
31 : pub offloaded_timelines: Vec<OffloadedTimelineManifest>,
32 : }
33 :
34 : /// The remote level representation of an offloaded timeline.
35 : ///
36 : /// Very similar to [`pageserver_api::models::OffloadedTimelineInfo`],
37 : /// but the two datastructures serve different needs, this is for a persistent disk format
38 : /// that must be backwards compatible, while the other is only for informative purposes.
39 48 : #[derive(Clone, Debug, Serialize, Deserialize, Copy, PartialEq, Eq)]
40 : pub struct OffloadedTimelineManifest {
41 : pub timeline_id: TimelineId,
42 : /// Whether the timeline has a parent it has been branched off from or not
43 : pub ancestor_timeline_id: Option<TimelineId>,
44 : /// Whether to retain the branch lsn at the ancestor or not
45 : pub ancestor_retain_lsn: Option<Lsn>,
46 : /// The time point when the timeline was archived
47 : pub archived_at: NaiveDateTime,
48 : }
49 :
50 : /// The newest manifest version. This should be incremented on changes, even non-breaking ones. We
51 : /// do not use deny_unknown_fields, so new fields are not breaking.
52 : ///
53 : /// 1: initial version
54 : /// 2: +stripe_size
55 : ///
56 : /// When adding new versions, also add a parse_vX test case below.
57 : pub const LATEST_TENANT_MANIFEST_VERSION: usize = 2;
58 :
59 : impl TenantManifest {
60 : /// Returns true if the manifests are equal, ignoring the version number. This avoids
61 : /// re-uploading all manifests just because the version number is bumped.
62 16 : pub fn eq_ignoring_version(&self, other: &Self) -> bool {
63 16 : // Fast path: if the version is equal, just compare directly.
64 16 : if self.version == other.version {
65 16 : return self == other;
66 0 : }
67 0 :
68 0 : // We could alternatively just clone and modify the version here.
69 0 : let Self {
70 0 : version: _, // ignore version
71 0 : stripe_size,
72 0 : offloaded_timelines,
73 0 : } = self;
74 0 :
75 0 : stripe_size == &other.stripe_size && offloaded_timelines == &other.offloaded_timelines
76 16 : }
77 :
78 : /// Decodes a manifest from JSON.
79 28 : pub fn from_json_bytes(bytes: &[u8]) -> Result<Self, serde_json::Error> {
80 28 : serde_json::from_slice(bytes)
81 28 : }
82 :
83 : /// Encodes a manifest as JSON.
84 452 : pub fn to_json_bytes(&self) -> serde_json::Result<Vec<u8>> {
85 452 : serde_json::to_vec(self)
86 452 : }
87 : }
88 :
89 : #[cfg(test)]
90 : mod tests {
91 : use std::str::FromStr;
92 :
93 : use utils::id::TimelineId;
94 :
95 : use super::*;
96 :
97 : /// Empty manifests should be parsed. Version is required.
98 : #[test]
99 4 : fn parse_empty() -> anyhow::Result<()> {
100 4 : let json = r#"{
101 4 : "version": 0
102 4 : }"#;
103 4 : let expected = TenantManifest {
104 4 : version: 0,
105 4 : stripe_size: None,
106 4 : offloaded_timelines: Vec::new(),
107 4 : };
108 4 : assert_eq!(expected, TenantManifest::from_json_bytes(json.as_bytes())?);
109 4 : Ok(())
110 4 : }
111 :
112 : /// Unknown fields should be ignored, for forwards compatibility.
113 : #[test]
114 4 : fn parse_unknown_fields() -> anyhow::Result<()> {
115 4 : let json = r#"{
116 4 : "version": 1,
117 4 : "foo": "bar"
118 4 : }"#;
119 4 : let expected = TenantManifest {
120 4 : version: 1,
121 4 : stripe_size: None,
122 4 : offloaded_timelines: Vec::new(),
123 4 : };
124 4 : assert_eq!(expected, TenantManifest::from_json_bytes(json.as_bytes())?);
125 4 : Ok(())
126 4 : }
127 :
128 : /// v1 manifests should be parsed, for backwards compatibility.
129 : #[test]
130 4 : fn parse_v1() -> anyhow::Result<()> {
131 4 : let json = r#"{
132 4 : "version": 1,
133 4 : "offloaded_timelines": [
134 4 : {
135 4 : "timeline_id": "5c4df612fd159e63c1b7853fe94d97da",
136 4 : "archived_at": "2025-03-07T11:07:11.373105434"
137 4 : },
138 4 : {
139 4 : "timeline_id": "f3def5823ad7080d2ea538d8e12163fa",
140 4 : "ancestor_timeline_id": "5c4df612fd159e63c1b7853fe94d97da",
141 4 : "ancestor_retain_lsn": "0/1F79038",
142 4 : "archived_at": "2025-03-05T11:10:22.257901390"
143 4 : }
144 4 : ]
145 4 : }"#;
146 4 : let expected = TenantManifest {
147 : version: 1,
148 4 : stripe_size: None,
149 4 : offloaded_timelines: vec![
150 4 : OffloadedTimelineManifest {
151 4 : timeline_id: TimelineId::from_str("5c4df612fd159e63c1b7853fe94d97da")?,
152 4 : ancestor_timeline_id: None,
153 4 : ancestor_retain_lsn: None,
154 4 : archived_at: NaiveDateTime::from_str("2025-03-07T11:07:11.373105434")?,
155 : },
156 : OffloadedTimelineManifest {
157 4 : timeline_id: TimelineId::from_str("f3def5823ad7080d2ea538d8e12163fa")?,
158 4 : ancestor_timeline_id: Some(TimelineId::from_str(
159 4 : "5c4df612fd159e63c1b7853fe94d97da",
160 4 : )?),
161 4 : ancestor_retain_lsn: Some(Lsn::from_str("0/1F79038")?),
162 4 : archived_at: NaiveDateTime::from_str("2025-03-05T11:10:22.257901390")?,
163 : },
164 : ],
165 : };
166 4 : assert_eq!(expected, TenantManifest::from_json_bytes(json.as_bytes())?);
167 4 : Ok(())
168 4 : }
169 :
170 : /// v2 manifests should be parsed, for backwards compatibility.
171 : #[test]
172 4 : fn parse_v2() -> anyhow::Result<()> {
173 4 : let json = r#"{
174 4 : "version": 2,
175 4 : "stripe_size": 32768,
176 4 : "offloaded_timelines": [
177 4 : {
178 4 : "timeline_id": "5c4df612fd159e63c1b7853fe94d97da",
179 4 : "archived_at": "2025-03-07T11:07:11.373105434"
180 4 : },
181 4 : {
182 4 : "timeline_id": "f3def5823ad7080d2ea538d8e12163fa",
183 4 : "ancestor_timeline_id": "5c4df612fd159e63c1b7853fe94d97da",
184 4 : "ancestor_retain_lsn": "0/1F79038",
185 4 : "archived_at": "2025-03-05T11:10:22.257901390"
186 4 : }
187 4 : ]
188 4 : }"#;
189 4 : let expected = TenantManifest {
190 : version: 2,
191 4 : stripe_size: Some(ShardStripeSize(32768)),
192 4 : offloaded_timelines: vec![
193 4 : OffloadedTimelineManifest {
194 4 : timeline_id: TimelineId::from_str("5c4df612fd159e63c1b7853fe94d97da")?,
195 4 : ancestor_timeline_id: None,
196 4 : ancestor_retain_lsn: None,
197 4 : archived_at: NaiveDateTime::from_str("2025-03-07T11:07:11.373105434")?,
198 : },
199 : OffloadedTimelineManifest {
200 4 : timeline_id: TimelineId::from_str("f3def5823ad7080d2ea538d8e12163fa")?,
201 4 : ancestor_timeline_id: Some(TimelineId::from_str(
202 4 : "5c4df612fd159e63c1b7853fe94d97da",
203 4 : )?),
204 4 : ancestor_retain_lsn: Some(Lsn::from_str("0/1F79038")?),
205 4 : archived_at: NaiveDateTime::from_str("2025-03-05T11:10:22.257901390")?,
206 : },
207 : ],
208 : };
209 4 : assert_eq!(expected, TenantManifest::from_json_bytes(json.as_bytes())?);
210 4 : Ok(())
211 4 : }
212 : }
|
