TLA Line data Source code
1 : //! `ComputeSpec` represents the contents of the spec.json file.
2 : //!
3 : //! The spec.json file is used to pass information to 'compute_ctl'. It contains
4 : //! all the information needed to start up the right version of PostgreSQL,
5 : //! and connect it to the storage nodes.
6 : use std::collections::HashMap;
7 :
8 : use serde::{Deserialize, Serialize};
9 : use utils::id::{TenantId, TimelineId};
10 : use utils::lsn::Lsn;
11 :
12 : use regex::Regex;
13 : use remote_storage::RemotePath;
14 :
15 : /// String type alias representing Postgres identifier and
16 : /// intended to be used for DB / role names.
17 : pub type PgIdent = String;
18 :
19 : /// Cluster spec or configuration represented as an optional number of
20 : /// delta operations + final cluster state description.
21 CBC 28435 : #[derive(Clone, Debug, Default, Deserialize, Serialize)]
22 : pub struct ComputeSpec {
23 : pub format_version: f32,
24 :
25 : // The control plane also includes a 'timestamp' field in the JSON document,
26 : // but we don't use it for anything. Serde will ignore missing fields when
27 : // deserializing it.
28 : pub operation_uuid: Option<String>,
29 :
30 : /// Compute features to enable. These feature flags are provided, when we
31 : /// know all the details about client's compute, so they cannot be used
32 : /// to change `Empty` compute behavior.
33 : #[serde(default)]
34 : pub features: Vec<ComputeFeature>,
35 :
36 : /// Expected cluster state at the end of transition process.
37 : pub cluster: Cluster,
38 : pub delta_operations: Option<Vec<DeltaOp>>,
39 :
40 : /// An optinal hint that can be passed to speed up startup time if we know
41 : /// that no pg catalog mutations (like role creation, database creation,
42 : /// extension creation) need to be done on the actual database to start.
43 : #[serde(default)] // Default false
44 : pub skip_pg_catalog_updates: bool,
45 :
46 : // Information needed to connect to the storage layer.
47 : //
48 : // `tenant_id`, `timeline_id` and `pageserver_connstring` are always needed.
49 : //
50 : // Depending on `mode`, this can be a primary read-write node, a read-only
51 : // replica, or a read-only node pinned at an older LSN.
52 : // `safekeeper_connstrings` must be set for a primary.
53 : //
54 : // For backwards compatibility, the control plane may leave out all of
55 : // these, and instead set the "neon.tenant_id", "neon.timeline_id",
56 : // etc. GUCs in cluster.settings. TODO: Once the control plane has been
57 : // updated to fill these fields, we can make these non optional.
58 : pub tenant_id: Option<TenantId>,
59 :
60 : pub timeline_id: Option<TimelineId>,
61 :
62 : pub pageserver_connstring: Option<String>,
63 :
64 : #[serde(default)]
65 : pub safekeeper_connstrings: Vec<String>,
66 :
67 : #[serde(default)]
68 : pub mode: ComputeMode,
69 :
70 : /// If set, 'storage_auth_token' is used as the password to authenticate to
71 : /// the pageserver and safekeepers.
72 : pub storage_auth_token: Option<String>,
73 :
74 : // information about available remote extensions
75 : pub remote_extensions: Option<RemoteExtSpec>,
76 :
77 : pub pgbouncer_settings: Option<HashMap<String, String>>,
78 : }
79 :
80 : /// Feature flag to signal `compute_ctl` to enable certain experimental functionality.
81 4 : #[derive(Serialize, Clone, Copy, Debug, Deserialize, PartialEq, Eq)]
82 : #[serde(rename_all = "snake_case")]
83 : pub enum ComputeFeature {
84 : // XXX: Add more feature flags here.
85 :
86 : // This is a special feature flag that is used to represent unknown feature flags.
87 : // Basically all unknown to enum flags are represented as this one. See unit test
88 : // `parse_unknown_features()` for more details.
89 : #[serde(other)]
90 : UnknownFeature,
91 : }
92 :
93 59 : #[derive(Clone, Debug, Default, Deserialize, Serialize)]
94 : pub struct RemoteExtSpec {
95 : pub public_extensions: Option<Vec<String>>,
96 : pub custom_extensions: Option<Vec<String>>,
97 : pub library_index: HashMap<String, String>,
98 : pub extension_data: HashMap<String, ExtensionData>,
99 : }
100 :
101 60 : #[derive(Clone, Debug, Serialize, Deserialize)]
102 : pub struct ExtensionData {
103 : pub control_data: HashMap<String, String>,
104 : pub archive_path: String,
105 : }
106 :
107 : impl RemoteExtSpec {
108 2 : pub fn get_ext(
109 2 : &self,
110 2 : ext_name: &str,
111 2 : is_library: bool,
112 2 : build_tag: &str,
113 2 : pg_major_version: &str,
114 2 : ) -> anyhow::Result<(String, RemotePath)> {
115 2 : let mut real_ext_name = ext_name;
116 2 : if is_library {
117 : // sometimes library names might have a suffix like
118 : // library.so or library.so.3. We strip this off
119 : // because library_index is based on the name without the file extension
120 UBC 0 : let strip_lib_suffix = Regex::new(r"\.so.*").unwrap();
121 0 : let lib_raw_name = strip_lib_suffix.replace(real_ext_name, "").to_string();
122 0 :
123 0 : real_ext_name = self
124 0 : .library_index
125 0 : .get(&lib_raw_name)
126 0 : .ok_or(anyhow::anyhow!("library {} is not found", lib_raw_name))?;
127 CBC 2 : }
128 :
129 : // Check if extension is present in public or custom.
130 : // If not, then it is not allowed to be used by this compute.
131 2 : if let Some(public_extensions) = &self.public_extensions {
132 UBC 0 : if !public_extensions.contains(&real_ext_name.to_string()) {
133 0 : if let Some(custom_extensions) = &self.custom_extensions {
134 0 : if !custom_extensions.contains(&real_ext_name.to_string()) {
135 0 : return Err(anyhow::anyhow!("extension {} is not found", real_ext_name));
136 0 : }
137 0 : }
138 0 : }
139 CBC 2 : }
140 :
141 2 : match self.extension_data.get(real_ext_name) {
142 2 : Some(_ext_data) => {
143 2 : // Construct the path to the extension archive
144 2 : // BUILD_TAG/PG_MAJOR_VERSION/extensions/EXTENSION_NAME.tar.zst
145 2 : //
146 2 : // Keep it in sync with path generation in
147 2 : // https://github.com/neondatabase/build-custom-extensions/tree/main
148 2 : let archive_path_str =
149 2 : format!("{build_tag}/{pg_major_version}/extensions/{real_ext_name}.tar.zst");
150 2 : Ok((
151 2 : real_ext_name.to_string(),
152 2 : RemotePath::from_string(&archive_path_str)?,
153 : ))
154 : }
155 UBC 0 : None => Err(anyhow::anyhow!(
156 0 : "real_ext_name {} is not found",
157 0 : real_ext_name
158 0 : )),
159 : }
160 CBC 2 : }
161 : }
162 :
163 9698 : #[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
164 : pub enum ComputeMode {
165 : /// A read-write node
166 : #[default]
167 : Primary,
168 : /// A read-only node, pinned at a particular LSN
169 : Static(Lsn),
170 : /// A read-only node that follows the tip of the branch in hot standby mode
171 : ///
172 : /// Future versions may want to distinguish between replicas with hot standby
173 : /// feedback and other kinds of replication configurations.
174 : Replica,
175 : }
176 :
177 14735 : #[derive(Clone, Debug, Default, Deserialize, Serialize)]
178 : pub struct Cluster {
179 : pub cluster_id: Option<String>,
180 : pub name: Option<String>,
181 : pub state: Option<String>,
182 : pub roles: Vec<Role>,
183 : pub databases: Vec<Database>,
184 :
185 : /// Desired contents of 'postgresql.conf' file. (The 'compute_ctl'
186 : /// tool may add additional settings to the final file.)
187 : pub postgresql_conf: Option<String>,
188 :
189 : /// Additional settings that will be appended to the 'postgresql.conf' file.
190 : pub settings: GenericOptions,
191 : }
192 :
193 : /// Single cluster state changing operation that could not be represented as
194 : /// a static `Cluster` structure. For example:
195 : /// - DROP DATABASE
196 : /// - DROP ROLE
197 : /// - ALTER ROLE name RENAME TO new_name
198 : /// - ALTER DATABASE name RENAME TO new_name
199 120 : #[derive(Clone, Debug, Deserialize, Serialize)]
200 : pub struct DeltaOp {
201 : pub action: String,
202 : pub name: PgIdent,
203 : pub new_name: Option<PgIdent>,
204 : }
205 :
206 : /// Rust representation of Postgres role info with only those fields
207 : /// that matter for us.
208 180 : #[derive(Clone, Debug, Deserialize, Serialize)]
209 : pub struct Role {
210 : pub name: PgIdent,
211 : pub encrypted_password: Option<String>,
212 : pub options: GenericOptions,
213 : }
214 :
215 : /// Rust representation of Postgres database info with only those fields
216 : /// that matter for us.
217 2653 : #[derive(Clone, Debug, Deserialize, Serialize)]
218 : pub struct Database {
219 : pub name: PgIdent,
220 : pub owner: PgIdent,
221 : pub options: GenericOptions,
222 : // These are derived flags, not present in the spec file.
223 : // They are never set by the control plane.
224 : #[serde(skip_deserializing, default)]
225 : pub restrict_conn: bool,
226 : #[serde(skip_deserializing, default)]
227 : pub invalid: bool,
228 : }
229 :
230 : /// Common type representing both SQL statement params with or without value,
231 : /// like `LOGIN` or `OWNER username` in the `CREATE/ALTER ROLE`, and config
232 : /// options like `wal_level = logical`.
233 910 : #[derive(Clone, Debug, Deserialize, Serialize)]
234 : pub struct GenericOption {
235 : pub name: String,
236 : pub value: Option<String>,
237 : pub vartype: String,
238 : }
239 :
240 : /// Optional collection of `GenericOption`'s. Type alias allows us to
241 : /// declare a `trait` on it.
242 : pub type GenericOptions = Option<Vec<GenericOption>>;
243 :
244 : #[cfg(test)]
245 : mod tests {
246 : use super::*;
247 : use std::fs::File;
248 :
249 1 : #[test]
250 1 : fn parse_spec_file() {
251 1 : let file = File::open("tests/cluster_spec.json").unwrap();
252 1 : let spec: ComputeSpec = serde_json::from_reader(file).unwrap();
253 :
254 : // Features list defaults to empty vector.
255 1 : assert!(spec.features.is_empty());
256 1 : }
257 :
258 1 : #[test]
259 1 : fn parse_unknown_fields() {
260 1 : // Forward compatibility test
261 1 : let file = File::open("tests/cluster_spec.json").unwrap();
262 1 : let mut json: serde_json::Value = serde_json::from_reader(file).unwrap();
263 1 : let ob = json.as_object_mut().unwrap();
264 1 : ob.insert("unknown_field_123123123".into(), "hello".into());
265 1 : let _spec: ComputeSpec = serde_json::from_value(json).unwrap();
266 1 : }
267 :
268 1 : #[test]
269 1 : fn parse_unknown_features() {
270 1 : // Test that unknown feature flags do not cause any errors.
271 1 : let file = File::open("tests/cluster_spec.json").unwrap();
272 1 : let mut json: serde_json::Value = serde_json::from_reader(file).unwrap();
273 1 : let ob = json.as_object_mut().unwrap();
274 1 :
275 1 : // Add unknown feature flags.
276 1 : let features = vec!["foo_bar_feature", "baz_feature"];
277 1 : ob.insert("features".into(), features.into());
278 1 :
279 1 : let spec: ComputeSpec = serde_json::from_value(json).unwrap();
280 :
281 1 : assert!(spec.features.len() == 2);
282 1 : assert!(spec.features.contains(&ComputeFeature::UnknownFeature));
283 1 : assert_eq!(spec.features, vec![ComputeFeature::UnknownFeature; 2]);
284 1 : }
285 : }
|
