TLA Line data Source code
1 : //! `ComputeSpec` represents the contents of the spec.json file.
2 : //!
3 : //! The spec.json file is used to pass information to 'compute_ctl'. It contains
4 : //! all the information needed to start up the right version of PostgreSQL,
5 : //! and connect it to the storage nodes.
6 : use std::collections::HashMap;
7 :
8 : use serde::{Deserialize, Serialize};
9 : use serde_with::{serde_as, DisplayFromStr};
10 : use utils::id::{TenantId, TimelineId};
11 : use utils::lsn::Lsn;
12 :
13 : use regex::Regex;
14 : use remote_storage::RemotePath;
15 :
16 : /// String type alias representing Postgres identifier and
17 : /// intended to be used for DB / role names.
18 : pub type PgIdent = String;
19 :
20 : /// Cluster spec or configuration represented as an optional number of
21 : /// delta operations + final cluster state description.
22 : #[serde_as]
23 CBC 17997 : #[derive(Clone, Debug, Default, Deserialize, Serialize)]
24 : pub struct ComputeSpec {
25 : pub format_version: f32,
26 :
27 : // The control plane also includes a 'timestamp' field in the JSON document,
28 : // but we don't use it for anything. Serde will ignore missing fields when
29 : // deserializing it.
30 : pub operation_uuid: Option<String>,
31 : /// Expected cluster state at the end of transition process.
32 : pub cluster: Cluster,
33 : pub delta_operations: Option<Vec<DeltaOp>>,
34 :
35 : /// An optinal hint that can be passed to speed up startup time if we know
36 : /// that no pg catalog mutations (like role creation, database creation,
37 : /// extension creation) need to be done on the actual database to start.
38 : #[serde(default)] // Default false
39 : pub skip_pg_catalog_updates: bool,
40 :
41 : // Information needed to connect to the storage layer.
42 : //
43 : // `tenant_id`, `timeline_id` and `pageserver_connstring` are always needed.
44 : //
45 : // Depending on `mode`, this can be a primary read-write node, a read-only
46 : // replica, or a read-only node pinned at an older LSN.
47 : // `safekeeper_connstrings` must be set for a primary.
48 : //
49 : // For backwards compatibility, the control plane may leave out all of
50 : // these, and instead set the "neon.tenant_id", "neon.timeline_id",
51 : // etc. GUCs in cluster.settings. TODO: Once the control plane has been
52 : // updated to fill these fields, we can make these non optional.
53 : #[serde_as(as = "Option<DisplayFromStr>")]
54 : pub tenant_id: Option<TenantId>,
55 : #[serde_as(as = "Option<DisplayFromStr>")]
56 : pub timeline_id: Option<TimelineId>,
57 : #[serde_as(as = "Option<DisplayFromStr>")]
58 : pub pageserver_connstring: Option<String>,
59 : #[serde(default)]
60 : pub safekeeper_connstrings: Vec<String>,
61 :
62 : #[serde(default)]
63 : pub mode: ComputeMode,
64 :
65 : /// If set, 'storage_auth_token' is used as the password to authenticate to
66 : /// the pageserver and safekeepers.
67 : pub storage_auth_token: Option<String>,
68 :
69 : // information about available remote extensions
70 : pub remote_extensions: Option<RemoteExtSpec>,
71 : }
72 :
73 36 : #[derive(Clone, Debug, Default, Deserialize, Serialize)]
74 : pub struct RemoteExtSpec {
75 : pub public_extensions: Option<Vec<String>>,
76 : pub custom_extensions: Option<Vec<String>>,
77 : pub library_index: HashMap<String, String>,
78 : pub extension_data: HashMap<String, ExtensionData>,
79 : }
80 :
81 40 : #[derive(Clone, Debug, Serialize, Deserialize)]
82 : pub struct ExtensionData {
83 : pub control_data: HashMap<String, String>,
84 : pub archive_path: String,
85 : }
86 :
87 : impl RemoteExtSpec {
88 UBC 0 : pub fn get_ext(
89 0 : &self,
90 0 : ext_name: &str,
91 0 : is_library: bool,
92 0 : build_tag: &str,
93 0 : pg_major_version: &str,
94 0 : ) -> anyhow::Result<(String, RemotePath)> {
95 0 : let mut real_ext_name = ext_name;
96 0 : if is_library {
97 : // sometimes library names might have a suffix like
98 : // library.so or library.so.3. We strip this off
99 : // because library_index is based on the name without the file extension
100 0 : let strip_lib_suffix = Regex::new(r"\.so.*").unwrap();
101 0 : let lib_raw_name = strip_lib_suffix.replace(real_ext_name, "").to_string();
102 0 :
103 0 : real_ext_name = self
104 0 : .library_index
105 0 : .get(&lib_raw_name)
106 0 : .ok_or(anyhow::anyhow!("library {} is not found", lib_raw_name))?;
107 0 : }
108 :
109 : // Check if extension is present in public or custom.
110 : // If not, then it is not allowed to be used by this compute.
111 0 : if let Some(public_extensions) = &self.public_extensions {
112 0 : if !public_extensions.contains(&real_ext_name.to_string()) {
113 0 : if let Some(custom_extensions) = &self.custom_extensions {
114 0 : if !custom_extensions.contains(&real_ext_name.to_string()) {
115 0 : return Err(anyhow::anyhow!("extension {} is not found", real_ext_name));
116 0 : }
117 0 : }
118 0 : }
119 0 : }
120 :
121 0 : match self.extension_data.get(real_ext_name) {
122 0 : Some(_ext_data) => {
123 0 : // Construct the path to the extension archive
124 0 : // BUILD_TAG/PG_MAJOR_VERSION/extensions/EXTENSION_NAME.tar.zst
125 0 : //
126 0 : // Keep it in sync with path generation in
127 0 : // https://github.com/neondatabase/build-custom-extensions/tree/main
128 0 : let archive_path_str =
129 0 : format!("{build_tag}/{pg_major_version}/extensions/{real_ext_name}.tar.zst");
130 0 : Ok((
131 0 : real_ext_name.to_string(),
132 0 : RemotePath::from_string(&archive_path_str)?,
133 : ))
134 : }
135 0 : None => Err(anyhow::anyhow!(
136 0 : "real_ext_name {} is not found",
137 0 : real_ext_name
138 0 : )),
139 : }
140 0 : }
141 : }
142 :
143 : #[serde_as]
144 CBC 10494 : #[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
145 : pub enum ComputeMode {
146 : /// A read-write node
147 : #[default]
148 : Primary,
149 : /// A read-only node, pinned at a particular LSN
150 : Static(#[serde_as(as = "DisplayFromStr")] Lsn),
151 : /// A read-only node that follows the tip of the branch in hot standby mode
152 : ///
153 : /// Future versions may want to distinguish between replicas with hot standby
154 : /// feedback and other kinds of replication configurations.
155 : Replica,
156 : }
157 :
158 9667 : #[derive(Clone, Debug, Default, Deserialize, Serialize)]
159 : pub struct Cluster {
160 : pub cluster_id: Option<String>,
161 : pub name: Option<String>,
162 : pub state: Option<String>,
163 : pub roles: Vec<Role>,
164 : pub databases: Vec<Database>,
165 :
166 : /// Desired contents of 'postgresql.conf' file. (The 'compute_ctl'
167 : /// tool may add additional settings to the final file.)
168 : pub postgresql_conf: Option<String>,
169 :
170 : /// Additional settings that will be appended to the 'postgresql.conf' file.
171 : pub settings: GenericOptions,
172 : }
173 :
174 : /// Single cluster state changing operation that could not be represented as
175 : /// a static `Cluster` structure. For example:
176 : /// - DROP DATABASE
177 : /// - DROP ROLE
178 : /// - ALTER ROLE name RENAME TO new_name
179 : /// - ALTER DATABASE name RENAME TO new_name
180 96 : #[derive(Clone, Debug, Deserialize, Serialize)]
181 : pub struct DeltaOp {
182 : pub action: String,
183 : pub name: PgIdent,
184 : pub new_name: Option<PgIdent>,
185 : }
186 :
187 : /// Rust representation of Postgres role info with only those fields
188 : /// that matter for us.
189 144 : #[derive(Clone, Debug, Deserialize, Serialize)]
190 : pub struct Role {
191 : pub name: PgIdent,
192 : pub encrypted_password: Option<String>,
193 : pub options: GenericOptions,
194 : }
195 :
196 : /// Rust representation of Postgres database info with only those fields
197 : /// that matter for us.
198 68 : #[derive(Clone, Debug, Deserialize, Serialize)]
199 : pub struct Database {
200 : pub name: PgIdent,
201 : pub owner: PgIdent,
202 : pub options: GenericOptions,
203 : // These are derived flags, not present in the spec file.
204 : // They are never set by the control plane.
205 : #[serde(skip_deserializing, default)]
206 : pub restrict_conn: bool,
207 : #[serde(skip_deserializing, default)]
208 : pub invalid: bool,
209 : }
210 :
211 : /// Common type representing both SQL statement params with or without value,
212 : /// like `LOGIN` or `OWNER username` in the `CREATE/ALTER ROLE`, and config
213 : /// options like `wal_level = logical`.
214 728 : #[derive(Clone, Debug, Deserialize, Serialize)]
215 : pub struct GenericOption {
216 : pub name: String,
217 : pub value: Option<String>,
218 : pub vartype: String,
219 : }
220 :
221 : /// Optional collection of `GenericOption`'s. Type alias allows us to
222 : /// declare a `trait` on it.
223 : pub type GenericOptions = Option<Vec<GenericOption>>;
224 :
225 : #[cfg(test)]
226 : mod tests {
227 : use super::*;
228 : use std::fs::File;
229 :
230 1 : #[test]
231 1 : fn parse_spec_file() {
232 1 : let file = File::open("tests/cluster_spec.json").unwrap();
233 1 : let _spec: ComputeSpec = serde_json::from_reader(file).unwrap();
234 1 : }
235 :
236 1 : #[test]
237 1 : fn parse_unknown_fields() {
238 1 : // Forward compatibility test
239 1 : let file = File::open("tests/cluster_spec.json").unwrap();
240 1 : let mut json: serde_json::Value = serde_json::from_reader(file).unwrap();
241 1 : let ob = json.as_object_mut().unwrap();
242 1 : ob.insert("unknown_field_123123123".into(), "hello".into());
243 1 : let _spec: ComputeSpec = serde_json::from_value(json).unwrap();
244 1 : }
245 : }
|
