Line data Source code
1 : //! The ComputeSpec contains all the information needed to start up
2 : //! the right version of PostgreSQL, and connect it to the storage nodes.
3 : //! It can be passed as part of the `config.json`, or the control plane can
4 : //! provide it by calling the compute_ctl's `/compute_ctl` endpoint, or
5 : //! compute_ctl can fetch it by calling the control plane's API.
6 : use std::collections::HashMap;
7 :
8 : use indexmap::IndexMap;
9 : use regex::Regex;
10 : use remote_storage::RemotePath;
11 : use serde::{Deserialize, Serialize};
12 : use utils::id::{TenantId, TimelineId};
13 : use utils::lsn::Lsn;
14 :
15 : use crate::responses::TlsConfig;
16 :
17 : /// String type alias representing Postgres identifier and
18 : /// intended to be used for DB / role names.
19 : pub type PgIdent = String;
20 :
21 : /// String type alias representing Postgres extension version
22 : pub type ExtVersion = String;
23 :
24 6 : fn default_reconfigure_concurrency() -> usize {
25 6 : 1
26 6 : }
27 :
28 : /// Cluster spec or configuration represented as an optional number of
29 : /// delta operations + final cluster state description.
30 45 : #[derive(Clone, Debug, Default, Deserialize, Serialize)]
31 : pub struct ComputeSpec {
32 : pub format_version: f32,
33 :
34 : // The control plane also includes a 'timestamp' field in the JSON document,
35 : // but we don't use it for anything. Serde will ignore missing fields when
36 : // deserializing it.
37 : pub operation_uuid: Option<String>,
38 :
39 : /// Compute features to enable. These feature flags are provided, when we
40 : /// know all the details about client's compute, so they cannot be used
41 : /// to change `Empty` compute behavior.
42 : #[serde(default)]
43 : pub features: Vec<ComputeFeature>,
44 :
45 : /// If compute_ctl was passed `--resize-swap-on-bind`, a value of `Some(_)` instructs
46 : /// compute_ctl to `/neonvm/bin/resize-swap` with the given size, when the spec is first
47 : /// received.
48 : ///
49 : /// Both this field and `--resize-swap-on-bind` are required, so that the control plane's
50 : /// spec generation doesn't need to be aware of the actual compute it's running on, while
51 : /// guaranteeing gradual rollout of swap. Otherwise, without `--resize-swap-on-bind`, we could
52 : /// end up trying to resize swap in VMs without it -- or end up *not* resizing swap, thus
53 : /// giving every VM much more swap than it should have (32GiB).
54 : ///
55 : /// Eventually we may remove `--resize-swap-on-bind` and exclusively use `swap_size_bytes` for
56 : /// enabling the swap resizing behavior once rollout is complete.
57 : ///
58 : /// See neondatabase/cloud#12047 for more.
59 : #[serde(default)]
60 : pub swap_size_bytes: Option<u64>,
61 :
62 : /// If compute_ctl was passed `--set-disk-quota-for-fs`, a value of `Some(_)` instructs
63 : /// compute_ctl to run `/neonvm/bin/set-disk-quota` with the given size and fs, when the
64 : /// spec is first received.
65 : ///
66 : /// Both this field and `--set-disk-quota-for-fs` are required, so that the control plane's
67 : /// spec generation doesn't need to be aware of the actual compute it's running on, while
68 : /// guaranteeing gradual rollout of disk quota.
69 : #[serde(default)]
70 : pub disk_quota_bytes: Option<u64>,
71 :
72 : /// Disables the vm-monitor behavior that resizes LFC on upscale/downscale, instead relying on
73 : /// the initial size of LFC.
74 : ///
75 : /// This is intended for use when the LFC size is being overridden from the default but
76 : /// autoscaling is still enabled, and we don't want the vm-monitor to interfere with the custom
77 : /// LFC sizing.
78 : #[serde(default)]
79 : pub disable_lfc_resizing: Option<bool>,
80 :
81 : /// Expected cluster state at the end of transition process.
82 : pub cluster: Cluster,
83 : pub delta_operations: Option<Vec<DeltaOp>>,
84 :
85 : /// An optional hint that can be passed to speed up startup time if we know
86 : /// that no pg catalog mutations (like role creation, database creation,
87 : /// extension creation) need to be done on the actual database to start.
88 : #[serde(default)] // Default false
89 : pub skip_pg_catalog_updates: bool,
90 :
91 : // Information needed to connect to the storage layer.
92 : //
93 : // `tenant_id`, `timeline_id` and `pageserver_connstring` are always needed.
94 : //
95 : // Depending on `mode`, this can be a primary read-write node, a read-only
96 : // replica, or a read-only node pinned at an older LSN.
97 : // `safekeeper_connstrings` must be set for a primary.
98 : //
99 : // For backwards compatibility, the control plane may leave out all of
100 : // these, and instead set the "neon.tenant_id", "neon.timeline_id",
101 : // etc. GUCs in cluster.settings. TODO: Once the control plane has been
102 : // updated to fill these fields, we can make these non optional.
103 : pub tenant_id: Option<TenantId>,
104 : pub timeline_id: Option<TimelineId>,
105 : pub pageserver_connstring: Option<String>,
106 :
107 : // More neon ids that we expose to the compute_ctl
108 : // and to postgres as neon extension GUCs.
109 : pub project_id: Option<String>,
110 : pub branch_id: Option<String>,
111 : pub endpoint_id: Option<String>,
112 :
113 : /// Safekeeper membership config generation. It is put in
114 : /// neon.safekeepers GUC and serves two purposes:
115 : /// 1) Non zero value forces walproposer to use membership configurations.
116 : /// 2) If walproposer wants to update list of safekeepers to connect to
117 : /// taking them from some safekeeper mconf, it should check what value
118 : /// is newer by comparing the generation.
119 : ///
120 : /// Note: it could be SafekeeperGeneration, but this needs linking
121 : /// compute_ctl with postgres_ffi.
122 : #[serde(default)]
123 : pub safekeepers_generation: Option<u32>,
124 : #[serde(default)]
125 : pub safekeeper_connstrings: Vec<String>,
126 :
127 : #[serde(default)]
128 : pub mode: ComputeMode,
129 :
130 : /// If set, 'storage_auth_token' is used as the password to authenticate to
131 : /// the pageserver and safekeepers.
132 : pub storage_auth_token: Option<String>,
133 :
134 : // information about available remote extensions
135 : pub remote_extensions: Option<RemoteExtSpec>,
136 :
137 : pub pgbouncer_settings: Option<IndexMap<String, String>>,
138 :
139 : // Stripe size for pageserver sharding, in pages
140 : #[serde(default)]
141 : pub shard_stripe_size: Option<usize>,
142 :
143 : /// Local Proxy configuration used for JWT authentication
144 : #[serde(default)]
145 : pub local_proxy_config: Option<LocalProxySpec>,
146 :
147 : /// Number of concurrent connections during the parallel RunInEachDatabase
148 : /// phase of the apply config process.
149 : ///
150 : /// We need a higher concurrency during reconfiguration in case of many DBs,
151 : /// but instance is already running and used by client. We can easily get out of
152 : /// `max_connections` limit, and the current code won't handle that.
153 : ///
154 : /// Default is 1, but also allow control plane to override this value for specific
155 : /// projects. It's also recommended to bump `superuser_reserved_connections` +=
156 : /// `reconfigure_concurrency` for such projects to ensure that we always have
157 : /// enough spare connections for reconfiguration process to succeed.
158 : #[serde(default = "default_reconfigure_concurrency")]
159 : pub reconfigure_concurrency: usize,
160 :
161 : /// If set to true, the compute_ctl will drop all subscriptions before starting the
162 : /// compute. This is needed when we start an endpoint on a branch, so that child
163 : /// would not compete with parent branch subscriptions
164 : /// over the same replication content from publisher.
165 : #[serde(default)] // Default false
166 : pub drop_subscriptions_before_start: bool,
167 :
168 : /// Log level for compute audit logging
169 : #[serde(default)]
170 : pub audit_log_level: ComputeAudit,
171 :
172 : /// Hostname and the port of the otel collector. Leave empty to disable Postgres logs forwarding.
173 : /// Example: config-shy-breeze-123-collector-monitoring.neon-telemetry.svc.cluster.local:10514
174 : pub logs_export_host: Option<String>,
175 :
176 : /// Address of endpoint storage service
177 : pub endpoint_storage_addr: Option<String>,
178 : /// JWT for authorizing requests to endpoint storage service
179 : pub endpoint_storage_token: Option<String>,
180 :
181 : /// If true, download LFC state from endpoint_storage and pass it to Postgres on startup
182 : #[serde(default)]
183 : pub prewarm_lfc_on_startup: bool,
184 : }
185 :
186 : /// Feature flag to signal `compute_ctl` to enable certain experimental functionality.
187 3 : #[derive(Serialize, Clone, Copy, Debug, Deserialize, PartialEq, Eq)]
188 : #[serde(rename_all = "snake_case")]
189 : pub enum ComputeFeature {
190 : // XXX: Add more feature flags here.
191 : /// Enable the experimental activity monitor logic, which uses `pg_stat_database` to
192 : /// track short-lived connections as user activity.
193 : ActivityMonitorExperimental,
194 :
195 : /// This is a special feature flag that is used to represent unknown feature flags.
196 : /// Basically all unknown to enum flags are represented as this one. See unit test
197 : /// `parse_unknown_features()` for more details.
198 : #[serde(other)]
199 : UnknownFeature,
200 : }
201 :
202 44 : #[derive(Clone, Debug, Default, Deserialize, Serialize)]
203 : pub struct RemoteExtSpec {
204 : pub public_extensions: Option<Vec<String>>,
205 : pub custom_extensions: Option<Vec<String>>,
206 : pub library_index: HashMap<String, String>,
207 : pub extension_data: HashMap<String, ExtensionData>,
208 : }
209 :
210 18 : #[derive(Clone, Debug, Serialize, Deserialize)]
211 : pub struct ExtensionData {
212 : pub control_data: HashMap<String, String>,
213 : pub archive_path: String,
214 : }
215 :
216 : impl RemoteExtSpec {
217 6 : pub fn get_ext(
218 6 : &self,
219 6 : ext_name: &str,
220 6 : is_library: bool,
221 6 : build_tag: &str,
222 6 : pg_major_version: &str,
223 6 : ) -> anyhow::Result<(String, RemotePath)> {
224 6 : let mut real_ext_name = ext_name;
225 6 : if is_library {
226 : // sometimes library names might have a suffix like
227 : // library.so or library.so.3. We strip this off
228 : // because library_index is based on the name without the file extension
229 1 : let strip_lib_suffix = Regex::new(r"\.so.*").unwrap();
230 1 : let lib_raw_name = strip_lib_suffix.replace(real_ext_name, "").to_string();
231 1 :
232 1 : real_ext_name = self
233 1 : .library_index
234 1 : .get(&lib_raw_name)
235 1 : .ok_or(anyhow::anyhow!("library {} is not found", lib_raw_name))?;
236 5 : }
237 :
238 : // Check if extension is present in public or custom.
239 : // If not, then it is not allowed to be used by this compute.
240 6 : if !self
241 6 : .public_extensions
242 6 : .as_ref()
243 6 : .is_some_and(|exts| exts.iter().any(|e| e == real_ext_name))
244 4 : && !self
245 4 : .custom_extensions
246 4 : .as_ref()
247 4 : .is_some_and(|exts| exts.iter().any(|e| e == real_ext_name))
248 : {
249 3 : return Err(anyhow::anyhow!("extension {} is not found", real_ext_name));
250 3 : }
251 3 :
252 3 : match self.extension_data.get(real_ext_name) {
253 3 : Some(_ext_data) => {
254 : // We have decided to use the Go naming convention due to Kubernetes.
255 :
256 3 : let arch = match std::env::consts::ARCH {
257 3 : "x86_64" => "amd64",
258 0 : "aarch64" => "arm64",
259 0 : arch => arch,
260 : };
261 :
262 : // Construct the path to the extension archive
263 : // BUILD_TAG/PG_MAJOR_VERSION/extensions/EXTENSION_NAME.tar.zst
264 : //
265 : // Keep it in sync with path generation in
266 : // https://github.com/neondatabase/build-custom-extensions/tree/main
267 3 : let archive_path_str = format!(
268 3 : "{build_tag}/{arch}/{pg_major_version}/extensions/{real_ext_name}.tar.zst"
269 3 : );
270 3 : Ok((
271 3 : real_ext_name.to_string(),
272 3 : RemotePath::from_string(&archive_path_str)?,
273 : ))
274 : }
275 0 : None => Err(anyhow::anyhow!(
276 0 : "real_ext_name {} is not found",
277 0 : real_ext_name
278 0 : )),
279 : }
280 6 : }
281 : }
282 :
283 0 : #[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
284 : pub enum ComputeMode {
285 : /// A read-write node
286 : #[default]
287 : Primary,
288 : /// A read-only node, pinned at a particular LSN
289 : Static(Lsn),
290 : /// A read-only node that follows the tip of the branch in hot standby mode
291 : ///
292 : /// Future versions may want to distinguish between replicas with hot standby
293 : /// feedback and other kinds of replication configurations.
294 : Replica,
295 : }
296 :
297 : impl ComputeMode {
298 : /// Convert the compute mode to a string that can be used to identify the type of compute,
299 : /// which means that if it's a static compute, the LSN will not be included.
300 0 : pub fn to_type_str(&self) -> &'static str {
301 0 : match self {
302 0 : ComputeMode::Primary => "primary",
303 0 : ComputeMode::Static(_) => "static",
304 0 : ComputeMode::Replica => "replica",
305 : }
306 0 : }
307 : }
308 :
309 : /// Log level for audit logging
310 0 : #[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
311 : pub enum ComputeAudit {
312 : #[default]
313 : Disabled,
314 : // Deprecated, use Base instead
315 : Log,
316 : // (pgaudit.log = 'ddl', pgaudit.log_parameter='off')
317 : // logged to the standard postgresql log stream
318 : Base,
319 : // Deprecated, use Full or Extended instead
320 : Hipaa,
321 : // (pgaudit.log = 'all, -misc', pgaudit.log_parameter='off')
322 : // logged to separate files collected by rsyslog
323 : // into dedicated log storage with strict access
324 : Extended,
325 : // (pgaudit.log='all', pgaudit.log_parameter='on'),
326 : // logged to separate files collected by rsyslog
327 : // into dedicated log storage with strict access.
328 : Full,
329 : }
330 :
331 36 : #[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
332 : pub struct Cluster {
333 : pub cluster_id: Option<String>,
334 : pub name: Option<String>,
335 : pub state: Option<String>,
336 : pub roles: Vec<Role>,
337 : pub databases: Vec<Database>,
338 :
339 : /// Desired contents of 'postgresql.conf' file. (The 'compute_ctl'
340 : /// tool may add additional settings to the final file.)
341 : pub postgresql_conf: Option<String>,
342 :
343 : /// Additional settings that will be appended to the 'postgresql.conf' file.
344 : pub settings: GenericOptions,
345 : }
346 :
347 : /// Single cluster state changing operation that could not be represented as
348 : /// a static `Cluster` structure. For example:
349 : /// - DROP DATABASE
350 : /// - DROP ROLE
351 : /// - ALTER ROLE name RENAME TO new_name
352 : /// - ALTER DATABASE name RENAME TO new_name
353 60 : #[derive(Clone, Debug, Deserialize, Serialize)]
354 : pub struct DeltaOp {
355 : pub action: String,
356 : pub name: PgIdent,
357 : pub new_name: Option<PgIdent>,
358 : }
359 :
360 : /// Rust representation of Postgres role info with only those fields
361 : /// that matter for us.
362 90 : #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
363 : pub struct Role {
364 : pub name: PgIdent,
365 : pub encrypted_password: Option<String>,
366 : pub options: GenericOptions,
367 : }
368 :
369 : /// Rust representation of Postgres database info with only those fields
370 : /// that matter for us.
371 42 : #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
372 : pub struct Database {
373 : pub name: PgIdent,
374 : pub owner: PgIdent,
375 : pub options: GenericOptions,
376 : // These are derived flags, not present in the spec file.
377 : // They are never set by the control plane.
378 : #[serde(skip_deserializing, default)]
379 : pub restrict_conn: bool,
380 : #[serde(skip_deserializing, default)]
381 : pub invalid: bool,
382 : }
383 :
384 : /// Common type representing both SQL statement params with or without value,
385 : /// like `LOGIN` or `OWNER username` in the `CREATE/ALTER ROLE`, and config
386 : /// options like `wal_level = logical`.
387 486 : #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
388 : pub struct GenericOption {
389 : pub name: String,
390 : pub value: Option<String>,
391 : pub vartype: String,
392 : }
393 :
394 : /// Optional collection of `GenericOption`'s. Type alias allows us to
395 : /// declare a `trait` on it.
396 : pub type GenericOptions = Option<Vec<GenericOption>>;
397 :
398 : /// Configured the local_proxy application with the relevant JWKS and roles it should
399 : /// use for authorizing connect requests using JWT.
400 0 : #[derive(Clone, Debug, Deserialize, Serialize)]
401 : pub struct LocalProxySpec {
402 : #[serde(default)]
403 : #[serde(skip_serializing_if = "Option::is_none")]
404 : pub jwks: Option<Vec<JwksSettings>>,
405 : #[serde(default)]
406 : #[serde(skip_serializing_if = "Option::is_none")]
407 : pub tls: Option<TlsConfig>,
408 : }
409 :
410 0 : #[derive(Clone, Debug, Deserialize, Serialize)]
411 : pub struct JwksSettings {
412 : pub id: String,
413 : pub role_names: Vec<String>,
414 : pub jwks_url: String,
415 : pub provider_name: String,
416 : pub jwt_audience: Option<String>,
417 : }
418 :
419 : #[cfg(test)]
420 : mod tests {
421 : use std::fs::File;
422 :
423 : use super::*;
424 :
425 : #[test]
426 1 : fn allow_installing_remote_extensions() {
427 1 : let rspec: RemoteExtSpec = serde_json::from_value(serde_json::json!({
428 1 : "public_extensions": null,
429 1 : "custom_extensions": null,
430 1 : "library_index": {},
431 1 : "extension_data": {},
432 1 : }))
433 1 : .unwrap();
434 1 :
435 1 : rspec
436 1 : .get_ext("ext", false, "latest", "v17")
437 1 : .expect_err("Extension should not be found");
438 1 :
439 1 : let rspec: RemoteExtSpec = serde_json::from_value(serde_json::json!({
440 1 : "public_extensions": [],
441 1 : "custom_extensions": null,
442 1 : "library_index": {},
443 1 : "extension_data": {},
444 1 : }))
445 1 : .unwrap();
446 1 :
447 1 : rspec
448 1 : .get_ext("ext", false, "latest", "v17")
449 1 : .expect_err("Extension should not be found");
450 1 :
451 1 : let rspec: RemoteExtSpec = serde_json::from_value(serde_json::json!({
452 1 : "public_extensions": [],
453 1 : "custom_extensions": [],
454 1 : "library_index": {
455 1 : "ext": "ext"
456 1 : },
457 1 : "extension_data": {
458 1 : "ext": {
459 1 : "control_data": {
460 1 : "ext.control": ""
461 1 : },
462 1 : "archive_path": ""
463 1 : }
464 1 : },
465 1 : }))
466 1 : .unwrap();
467 1 :
468 1 : rspec
469 1 : .get_ext("ext", false, "latest", "v17")
470 1 : .expect_err("Extension should not be found");
471 1 :
472 1 : let rspec: RemoteExtSpec = serde_json::from_value(serde_json::json!({
473 1 : "public_extensions": [],
474 1 : "custom_extensions": ["ext"],
475 1 : "library_index": {
476 1 : "ext": "ext"
477 1 : },
478 1 : "extension_data": {
479 1 : "ext": {
480 1 : "control_data": {
481 1 : "ext.control": ""
482 1 : },
483 1 : "archive_path": ""
484 1 : }
485 1 : },
486 1 : }))
487 1 : .unwrap();
488 1 :
489 1 : rspec
490 1 : .get_ext("ext", false, "latest", "v17")
491 1 : .expect("Extension should be found");
492 1 :
493 1 : let rspec: RemoteExtSpec = serde_json::from_value(serde_json::json!({
494 1 : "public_extensions": ["ext"],
495 1 : "custom_extensions": [],
496 1 : "library_index": {
497 1 : "extlib": "ext",
498 1 : },
499 1 : "extension_data": {
500 1 : "ext": {
501 1 : "control_data": {
502 1 : "ext.control": ""
503 1 : },
504 1 : "archive_path": ""
505 1 : }
506 1 : },
507 1 : }))
508 1 : .unwrap();
509 1 :
510 1 : rspec
511 1 : .get_ext("ext", false, "latest", "v17")
512 1 : .expect("Extension should be found");
513 1 :
514 1 : // test library index for the case when library name
515 1 : // doesn't match the extension name
516 1 : rspec
517 1 : .get_ext("extlib", true, "latest", "v17")
518 1 : .expect("Library should be found");
519 1 : }
520 :
521 : #[test]
522 1 : fn parse_spec_file() {
523 1 : let file = File::open("tests/cluster_spec.json").unwrap();
524 1 : let spec: ComputeSpec = serde_json::from_reader(file).unwrap();
525 1 :
526 1 : // Features list defaults to empty vector.
527 1 : assert!(spec.features.is_empty());
528 :
529 : // Reconfigure concurrency defaults to 1.
530 1 : assert_eq!(spec.reconfigure_concurrency, 1);
531 1 : }
532 :
533 : #[test]
534 1 : fn parse_unknown_fields() {
535 1 : // Forward compatibility test
536 1 : let file = File::open("tests/cluster_spec.json").unwrap();
537 1 : let mut json: serde_json::Value = serde_json::from_reader(file).unwrap();
538 1 : let ob = json.as_object_mut().unwrap();
539 1 : ob.insert("unknown_field_123123123".into(), "hello".into());
540 1 : let _spec: ComputeSpec = serde_json::from_value(json).unwrap();
541 1 : }
542 :
543 : #[test]
544 1 : fn parse_unknown_features() {
545 1 : // Test that unknown feature flags do not cause any errors.
546 1 : let file = File::open("tests/cluster_spec.json").unwrap();
547 1 : let mut json: serde_json::Value = serde_json::from_reader(file).unwrap();
548 1 : let ob = json.as_object_mut().unwrap();
549 1 :
550 1 : // Add unknown feature flags.
551 1 : let features = vec!["foo_bar_feature", "baz_feature"];
552 1 : ob.insert("features".into(), features.into());
553 1 :
554 1 : let spec: ComputeSpec = serde_json::from_value(json).unwrap();
555 1 :
556 1 : assert!(spec.features.len() == 2);
557 1 : assert!(spec.features.contains(&ComputeFeature::UnknownFeature));
558 1 : assert_eq!(spec.features, vec![ComputeFeature::UnknownFeature; 2]);
559 1 : }
560 :
561 : #[test]
562 1 : fn parse_known_features() {
563 1 : // Test that we can properly parse known feature flags.
564 1 : let file = File::open("tests/cluster_spec.json").unwrap();
565 1 : let mut json: serde_json::Value = serde_json::from_reader(file).unwrap();
566 1 : let ob = json.as_object_mut().unwrap();
567 1 :
568 1 : // Add known feature flags.
569 1 : let features = vec!["activity_monitor_experimental"];
570 1 : ob.insert("features".into(), features.into());
571 1 :
572 1 : let spec: ComputeSpec = serde_json::from_value(json).unwrap();
573 1 :
574 1 : assert_eq!(
575 1 : spec.features,
576 1 : vec![ComputeFeature::ActivityMonitorExperimental]
577 1 : );
578 1 : }
579 : }
|
