Line data Source code
1 : //! This module is responsible for locating and loading paths in a local setup.
2 : //!
3 : //! Now it also provides init method which acts like a stub for proper installation
4 : //! script which will use local paths.
5 :
6 : use anyhow::{bail, ensure, Context};
7 :
8 : use clap::ValueEnum;
9 : use postgres_backend::AuthType;
10 : use reqwest::Url;
11 : use serde::{Deserialize, Serialize};
12 : use std::collections::HashMap;
13 : use std::env;
14 : use std::fs;
15 : use std::net::IpAddr;
16 : use std::net::Ipv4Addr;
17 : use std::net::SocketAddr;
18 : use std::path::{Path, PathBuf};
19 : use std::process::{Command, Stdio};
20 : use utils::{
21 : auth::{encode_from_key_file, Claims},
22 : id::{NodeId, TenantId, TenantTimelineId, TimelineId},
23 : };
24 :
25 : use crate::safekeeper::SafekeeperNode;
26 :
27 : pub const DEFAULT_PG_VERSION: u32 = 15;
28 :
29 : //
30 : // This data structures represents neon_local CLI config
31 : //
32 : // It is deserialized from the .neon/config file, or the config file passed
33 : // to 'neon_local init --config=<path>' option. See control_plane/simple.conf for
34 : // an example.
35 : //
36 30 : #[derive(Serialize, Deserialize, PartialEq, Eq, Clone, Debug)]
37 : pub struct LocalEnv {
38 : // Base directory for all the nodes (the pageserver, safekeepers and
39 : // compute endpoints).
40 : //
41 : // This is not stored in the config file. Rather, this is the path where the
42 : // config file itself is. It is read from the NEON_REPO_DIR env variable or
43 : // '.neon' if not given.
44 : #[serde(skip)]
45 : pub base_data_dir: PathBuf,
46 :
47 : // Path to postgres distribution. It's expected that "bin", "include",
48 : // "lib", "share" from postgres distribution are there. If at some point
49 : // in time we will be able to run against vanilla postgres we may split that
50 : // to four separate paths and match OS-specific installation layout.
51 : #[serde(default)]
52 : pub pg_distrib_dir: PathBuf,
53 :
54 : // Path to pageserver binary.
55 : #[serde(default)]
56 : pub neon_distrib_dir: PathBuf,
57 :
58 : // Default tenant ID to use with the 'neon_local' command line utility, when
59 : // --tenant_id is not explicitly specified.
60 : #[serde(default)]
61 : pub default_tenant_id: Option<TenantId>,
62 :
63 : // used to issue tokens during e.g pg start
64 : #[serde(default)]
65 : pub private_key_path: PathBuf,
66 :
67 : pub broker: NeonBroker,
68 :
69 : /// This Vec must always contain at least one pageserver
70 : pub pageservers: Vec<PageServerConf>,
71 :
72 : #[serde(default)]
73 : pub safekeepers: Vec<SafekeeperConf>,
74 :
75 : // Control plane upcall API for pageserver: if None, we will not run attachment_service. If set, this will
76 : // be propagated into each pageserver's configuration.
77 : #[serde(default)]
78 : pub control_plane_api: Option<Url>,
79 :
80 : // Control plane upcall API for attachment service. If set, this will be propagated into the
81 : // attachment service's configuration.
82 : #[serde(default)]
83 : pub control_plane_compute_hook_api: Option<Url>,
84 :
85 : /// Keep human-readable aliases in memory (and persist them to config), to hide ZId hex strings from the user.
86 : #[serde(default)]
87 : // A `HashMap<String, HashMap<TenantId, TimelineId>>` would be more appropriate here,
88 : // but deserialization into a generic toml object as `toml::Value::try_from` fails with an error.
89 : // https://toml.io/en/v1.0.0 does not contain a concept of "a table inside another table".
90 : branch_name_mappings: HashMap<String, Vec<(TenantId, TimelineId)>>,
91 : }
92 :
93 : /// Broker config for cluster internal communication.
94 14 : #[derive(Serialize, Deserialize, PartialEq, Eq, Clone, Debug)]
95 : #[serde(default)]
96 : pub struct NeonBroker {
97 : /// Broker listen address for storage nodes coordination, e.g. '127.0.0.1:50051'.
98 : pub listen_addr: SocketAddr,
99 : }
100 :
101 : // Dummy Default impl to satisfy Deserialize derive.
102 : impl Default for NeonBroker {
103 2 : fn default() -> Self {
104 2 : NeonBroker {
105 2 : listen_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 0),
106 2 : }
107 2 : }
108 : }
109 :
110 : impl NeonBroker {
111 0 : pub fn client_url(&self) -> Url {
112 0 : Url::parse(&format!("http://{}", self.listen_addr)).expect("failed to construct url")
113 0 : }
114 : }
115 :
116 44 : #[derive(Serialize, Deserialize, PartialEq, Eq, Clone, Debug)]
117 : #[serde(default)]
118 : pub struct PageServerConf {
119 : // node id
120 : pub id: NodeId,
121 :
122 : // Pageserver connection settings
123 : pub listen_pg_addr: String,
124 : pub listen_http_addr: String,
125 :
126 : // auth type used for the PG and HTTP ports
127 : pub pg_auth_type: AuthType,
128 : pub http_auth_type: AuthType,
129 : }
130 :
131 : impl Default for PageServerConf {
132 4 : fn default() -> Self {
133 4 : Self {
134 4 : id: NodeId(0),
135 4 : listen_pg_addr: String::new(),
136 4 : listen_http_addr: String::new(),
137 4 : pg_auth_type: AuthType::Trust,
138 4 : http_auth_type: AuthType::Trust,
139 4 : }
140 4 : }
141 : }
142 :
143 28 : #[derive(Serialize, Deserialize, PartialEq, Eq, Clone, Debug)]
144 : #[serde(default)]
145 : pub struct SafekeeperConf {
146 : pub id: NodeId,
147 : pub pg_port: u16,
148 : pub pg_tenant_only_port: Option<u16>,
149 : pub http_port: u16,
150 : pub sync: bool,
151 : pub remote_storage: Option<String>,
152 : pub backup_threads: Option<u32>,
153 : pub auth_enabled: bool,
154 : }
155 :
156 : impl Default for SafekeeperConf {
157 4 : fn default() -> Self {
158 4 : Self {
159 4 : id: NodeId(0),
160 4 : pg_port: 0,
161 4 : pg_tenant_only_port: None,
162 4 : http_port: 0,
163 4 : sync: true,
164 4 : remote_storage: None,
165 4 : backup_threads: None,
166 4 : auth_enabled: false,
167 4 : }
168 4 : }
169 : }
170 :
171 2 : #[derive(Clone, Copy)]
172 : pub enum InitForceMode {
173 : MustNotExist,
174 : EmptyDirOk,
175 : RemoveAllContents,
176 : }
177 :
178 : impl ValueEnum for InitForceMode {
179 4 : fn value_variants<'a>() -> &'a [Self] {
180 4 : &[
181 4 : Self::MustNotExist,
182 4 : Self::EmptyDirOk,
183 4 : Self::RemoveAllContents,
184 4 : ]
185 4 : }
186 :
187 10 : fn to_possible_value(&self) -> Option<clap::builder::PossibleValue> {
188 10 : Some(clap::builder::PossibleValue::new(match self {
189 6 : InitForceMode::MustNotExist => "must-not-exist",
190 2 : InitForceMode::EmptyDirOk => "empty-dir-ok",
191 2 : InitForceMode::RemoveAllContents => "remove-all-contents",
192 : }))
193 10 : }
194 : }
195 :
196 : impl SafekeeperConf {
197 : /// Compute is served by port on which only tenant scoped tokens allowed, if
198 : /// it is configured.
199 0 : pub fn get_compute_port(&self) -> u16 {
200 0 : self.pg_tenant_only_port.unwrap_or(self.pg_port)
201 0 : }
202 : }
203 :
204 : impl LocalEnv {
205 0 : pub fn pg_distrib_dir_raw(&self) -> PathBuf {
206 0 : self.pg_distrib_dir.clone()
207 0 : }
208 :
209 0 : pub fn pg_distrib_dir(&self, pg_version: u32) -> anyhow::Result<PathBuf> {
210 0 : let path = self.pg_distrib_dir.clone();
211 0 :
212 0 : #[allow(clippy::manual_range_patterns)]
213 0 : match pg_version {
214 0 : 14 | 15 | 16 => Ok(path.join(format!("v{pg_version}"))),
215 0 : _ => bail!("Unsupported postgres version: {}", pg_version),
216 : }
217 0 : }
218 :
219 0 : pub fn pg_bin_dir(&self, pg_version: u32) -> anyhow::Result<PathBuf> {
220 0 : Ok(self.pg_distrib_dir(pg_version)?.join("bin"))
221 0 : }
222 0 : pub fn pg_lib_dir(&self, pg_version: u32) -> anyhow::Result<PathBuf> {
223 0 : Ok(self.pg_distrib_dir(pg_version)?.join("lib"))
224 0 : }
225 :
226 0 : pub fn pageserver_bin(&self) -> PathBuf {
227 0 : self.neon_distrib_dir.join("pageserver")
228 0 : }
229 :
230 0 : pub fn attachment_service_bin(&self) -> PathBuf {
231 0 : // Irrespective of configuration, attachment service binary is always
232 0 : // run from the same location as neon_local. This means that for compatibility
233 0 : // tests that run old pageserver/safekeeper, they still run latest attachment service.
234 0 : let neon_local_bin_dir = env::current_exe().unwrap().parent().unwrap().to_owned();
235 0 : neon_local_bin_dir.join("attachment_service")
236 0 : }
237 :
238 0 : pub fn safekeeper_bin(&self) -> PathBuf {
239 0 : self.neon_distrib_dir.join("safekeeper")
240 0 : }
241 :
242 0 : pub fn storage_broker_bin(&self) -> PathBuf {
243 0 : self.neon_distrib_dir.join("storage_broker")
244 0 : }
245 :
246 0 : pub fn endpoints_path(&self) -> PathBuf {
247 0 : self.base_data_dir.join("endpoints")
248 0 : }
249 :
250 0 : pub fn pageserver_data_dir(&self, pageserver_id: NodeId) -> PathBuf {
251 0 : self.base_data_dir
252 0 : .join(format!("pageserver_{pageserver_id}"))
253 0 : }
254 :
255 0 : pub fn safekeeper_data_dir(&self, data_dir_name: &str) -> PathBuf {
256 0 : self.base_data_dir.join("safekeepers").join(data_dir_name)
257 0 : }
258 :
259 0 : pub fn get_pageserver_conf(&self, id: NodeId) -> anyhow::Result<&PageServerConf> {
260 0 : if let Some(conf) = self.pageservers.iter().find(|node| node.id == id) {
261 0 : Ok(conf)
262 : } else {
263 0 : let have_ids = self
264 0 : .pageservers
265 0 : .iter()
266 0 : .map(|node| format!("{}:{}", node.id, node.listen_http_addr))
267 0 : .collect::<Vec<_>>();
268 0 : let joined = have_ids.join(",");
269 0 : bail!("could not find pageserver {id}, have ids {joined}")
270 : }
271 0 : }
272 :
273 0 : pub fn register_branch_mapping(
274 0 : &mut self,
275 0 : branch_name: String,
276 0 : tenant_id: TenantId,
277 0 : timeline_id: TimelineId,
278 0 : ) -> anyhow::Result<()> {
279 0 : let existing_values = self
280 0 : .branch_name_mappings
281 0 : .entry(branch_name.clone())
282 0 : .or_default();
283 0 :
284 0 : let existing_ids = existing_values
285 0 : .iter()
286 0 : .find(|(existing_tenant_id, _)| existing_tenant_id == &tenant_id);
287 :
288 0 : if let Some((_, old_timeline_id)) = existing_ids {
289 0 : if old_timeline_id == &timeline_id {
290 0 : Ok(())
291 : } else {
292 0 : bail!("branch '{branch_name}' is already mapped to timeline {old_timeline_id}, cannot map to another timeline {timeline_id}");
293 : }
294 : } else {
295 0 : existing_values.push((tenant_id, timeline_id));
296 0 : Ok(())
297 : }
298 0 : }
299 :
300 0 : pub fn get_branch_timeline_id(
301 0 : &self,
302 0 : branch_name: &str,
303 0 : tenant_id: TenantId,
304 0 : ) -> Option<TimelineId> {
305 0 : self.branch_name_mappings
306 0 : .get(branch_name)?
307 0 : .iter()
308 0 : .find(|(mapped_tenant_id, _)| mapped_tenant_id == &tenant_id)
309 0 : .map(|&(_, timeline_id)| timeline_id)
310 0 : .map(TimelineId::from)
311 0 : }
312 :
313 0 : pub fn timeline_name_mappings(&self) -> HashMap<TenantTimelineId, String> {
314 0 : self.branch_name_mappings
315 0 : .iter()
316 0 : .flat_map(|(name, tenant_timelines)| {
317 0 : tenant_timelines.iter().map(|&(tenant_id, timeline_id)| {
318 0 : (TenantTimelineId::new(tenant_id, timeline_id), name.clone())
319 0 : })
320 0 : })
321 0 : .collect()
322 0 : }
323 :
324 : /// Create a LocalEnv from a config file.
325 : ///
326 : /// Unlike 'load_config', this function fills in any defaults that are missing
327 : /// from the config file.
328 4 : pub fn parse_config(toml: &str) -> anyhow::Result<Self> {
329 4 : let mut env: LocalEnv = toml::from_str(toml)?;
330 :
331 : // Find postgres binaries.
332 : // Follow POSTGRES_DISTRIB_DIR if set, otherwise look in "pg_install".
333 : // Note that later in the code we assume, that distrib dirs follow the same pattern
334 : // for all postgres versions.
335 2 : if env.pg_distrib_dir == Path::new("") {
336 2 : if let Some(postgres_bin) = env::var_os("POSTGRES_DISTRIB_DIR") {
337 0 : env.pg_distrib_dir = postgres_bin.into();
338 0 : } else {
339 2 : let cwd = env::current_dir()?;
340 2 : env.pg_distrib_dir = cwd.join("pg_install")
341 : }
342 0 : }
343 :
344 : // Find neon binaries.
345 2 : if env.neon_distrib_dir == Path::new("") {
346 2 : env.neon_distrib_dir = env::current_exe()?.parent().unwrap().to_owned();
347 0 : }
348 :
349 2 : if env.pageservers.is_empty() {
350 0 : anyhow::bail!("Configuration must contain at least one pageserver");
351 2 : }
352 2 :
353 2 : env.base_data_dir = base_path();
354 2 :
355 2 : Ok(env)
356 4 : }
357 :
358 : /// Locate and load config
359 0 : pub fn load_config() -> anyhow::Result<Self> {
360 0 : let repopath = base_path();
361 0 :
362 0 : if !repopath.exists() {
363 0 : bail!(
364 0 : "Neon config is not found in {}. You need to run 'neon_local init' first",
365 0 : repopath.to_str().unwrap()
366 0 : );
367 0 : }
368 :
369 : // TODO: check that it looks like a neon repository
370 :
371 : // load and parse file
372 0 : let config = fs::read_to_string(repopath.join("config"))?;
373 0 : let mut env: LocalEnv = toml::from_str(config.as_str())?;
374 :
375 0 : env.base_data_dir = repopath;
376 0 :
377 0 : Ok(env)
378 0 : }
379 :
380 0 : pub fn persist_config(&self, base_path: &Path) -> anyhow::Result<()> {
381 0 : // Currently, the user first passes a config file with 'neon_local init --config=<path>'
382 0 : // We read that in, in `create_config`, and fill any missing defaults. Then it's saved
383 0 : // to .neon/config. TODO: We lose any formatting and comments along the way, which is
384 0 : // a bit sad.
385 0 : let mut conf_content = r#"# This file describes a local deployment of the page server
386 0 : # and safekeeeper node. It is read by the 'neon_local' command-line
387 0 : # utility.
388 0 : "#
389 0 : .to_string();
390 0 :
391 0 : // Convert the LocalEnv to a toml file.
392 0 : //
393 0 : // This could be as simple as this:
394 0 : //
395 0 : // conf_content += &toml::to_string_pretty(env)?;
396 0 : //
397 0 : // But it results in a "values must be emitted before tables". I'm not sure
398 0 : // why, AFAICS the table, i.e. 'safekeepers: Vec<SafekeeperConf>' is last.
399 0 : // Maybe rust reorders the fields to squeeze avoid padding or something?
400 0 : // In any case, converting to toml::Value first, and serializing that, works.
401 0 : // See https://github.com/alexcrichton/toml-rs/issues/142
402 0 : conf_content += &toml::to_string_pretty(&toml::Value::try_from(self)?)?;
403 :
404 0 : let target_config_path = base_path.join("config");
405 0 : fs::write(&target_config_path, conf_content).with_context(|| {
406 0 : format!(
407 0 : "Failed to write config file into path '{}'",
408 0 : target_config_path.display()
409 0 : )
410 0 : })
411 0 : }
412 :
413 : // this function is used only for testing purposes in CLI e g generate tokens during init
414 0 : pub fn generate_auth_token(&self, claims: &Claims) -> anyhow::Result<String> {
415 0 : let private_key_path = self.get_private_key_path();
416 0 : let key_data = fs::read(private_key_path)?;
417 0 : encode_from_key_file(claims, &key_data)
418 0 : }
419 :
420 0 : pub fn get_private_key_path(&self) -> PathBuf {
421 0 : if self.private_key_path.is_absolute() {
422 0 : self.private_key_path.to_path_buf()
423 : } else {
424 0 : self.base_data_dir.join(&self.private_key_path)
425 : }
426 0 : }
427 :
428 : //
429 : // Initialize a new Neon repository
430 : //
431 0 : pub fn init(&mut self, pg_version: u32, force: &InitForceMode) -> anyhow::Result<()> {
432 0 : // check if config already exists
433 0 : let base_path = &self.base_data_dir;
434 0 : ensure!(
435 0 : base_path != Path::new(""),
436 0 : "repository base path is missing"
437 : );
438 :
439 0 : if base_path.exists() {
440 0 : match force {
441 : InitForceMode::MustNotExist => {
442 0 : bail!(
443 0 : "directory '{}' already exists. Perhaps already initialized?",
444 0 : base_path.display()
445 0 : );
446 : }
447 : InitForceMode::EmptyDirOk => {
448 0 : if let Some(res) = std::fs::read_dir(base_path)?.next() {
449 0 : res.context("check if directory is empty")?;
450 0 : anyhow::bail!("directory not empty: {base_path:?}");
451 0 : }
452 : }
453 : InitForceMode::RemoveAllContents => {
454 0 : println!("removing all contents of '{}'", base_path.display());
455 : // instead of directly calling `remove_dir_all`, we keep the original dir but removing
456 : // all contents inside. This helps if the developer symbol links another directory (i.e.,
457 : // S3 local SSD) to the `.neon` base directory.
458 0 : for entry in std::fs::read_dir(base_path)? {
459 0 : let entry = entry?;
460 0 : let path = entry.path();
461 0 : if path.is_dir() {
462 0 : fs::remove_dir_all(&path)?;
463 : } else {
464 0 : fs::remove_file(&path)?;
465 : }
466 : }
467 : }
468 : }
469 0 : }
470 :
471 0 : if !self.pg_bin_dir(pg_version)?.join("postgres").exists() {
472 0 : bail!(
473 0 : "Can't find postgres binary at {}",
474 0 : self.pg_bin_dir(pg_version)?.display()
475 : );
476 0 : }
477 0 : for binary in ["pageserver", "safekeeper"] {
478 0 : if !self.neon_distrib_dir.join(binary).exists() {
479 0 : bail!(
480 0 : "Can't find binary '{binary}' in neon distrib dir '{}'",
481 0 : self.neon_distrib_dir.display()
482 0 : );
483 0 : }
484 : }
485 :
486 0 : if !base_path.exists() {
487 0 : fs::create_dir(base_path)?;
488 0 : }
489 :
490 : // Generate keypair for JWT.
491 : //
492 : // The keypair is only needed if authentication is enabled in any of the
493 : // components. For convenience, we generate the keypair even if authentication
494 : // is not enabled, so that you can easily enable it after the initialization
495 : // step. However, if the key generation fails, we treat it as non-fatal if
496 : // authentication was not enabled.
497 0 : if self.private_key_path == PathBuf::new() {
498 0 : match generate_auth_keys(
499 0 : base_path.join("auth_private_key.pem").as_path(),
500 0 : base_path.join("auth_public_key.pem").as_path(),
501 0 : ) {
502 0 : Ok(()) => {
503 0 : self.private_key_path = PathBuf::from("auth_private_key.pem");
504 0 : }
505 0 : Err(e) => {
506 0 : if !self.auth_keys_needed() {
507 0 : eprintln!("Could not generate keypair for JWT authentication: {e}");
508 0 : eprintln!("Continuing anyway because authentication was not enabled");
509 0 : self.private_key_path = PathBuf::from("auth_private_key.pem");
510 0 : } else {
511 0 : return Err(e);
512 : }
513 : }
514 : }
515 0 : }
516 :
517 0 : fs::create_dir_all(self.endpoints_path())?;
518 :
519 0 : for safekeeper in &self.safekeepers {
520 0 : fs::create_dir_all(SafekeeperNode::datadir_path_by_id(self, safekeeper.id))?;
521 : }
522 :
523 0 : self.persist_config(base_path)
524 0 : }
525 :
526 0 : fn auth_keys_needed(&self) -> bool {
527 0 : self.pageservers.iter().any(|ps| {
528 0 : ps.pg_auth_type == AuthType::NeonJWT || ps.http_auth_type == AuthType::NeonJWT
529 0 : }) || self.safekeepers.iter().any(|sk| sk.auth_enabled)
530 0 : }
531 : }
532 :
533 2 : fn base_path() -> PathBuf {
534 2 : match std::env::var_os("NEON_REPO_DIR") {
535 0 : Some(val) => PathBuf::from(val),
536 2 : None => PathBuf::from(".neon"),
537 : }
538 2 : }
539 :
540 : /// Generate a public/private key pair for JWT authentication
541 0 : fn generate_auth_keys(private_key_path: &Path, public_key_path: &Path) -> anyhow::Result<()> {
542 : // Generate the key pair
543 : //
544 : // openssl genpkey -algorithm ed25519 -out auth_private_key.pem
545 0 : let keygen_output = Command::new("openssl")
546 0 : .arg("genpkey")
547 0 : .args(["-algorithm", "ed25519"])
548 0 : .args(["-out", private_key_path.to_str().unwrap()])
549 0 : .stdout(Stdio::null())
550 0 : .output()
551 0 : .context("failed to generate auth private key")?;
552 0 : if !keygen_output.status.success() {
553 0 : bail!(
554 0 : "openssl failed: '{}'",
555 0 : String::from_utf8_lossy(&keygen_output.stderr)
556 0 : );
557 0 : }
558 : // Extract the public key from the private key file
559 : //
560 : // openssl pkey -in auth_private_key.pem -pubout -out auth_public_key.pem
561 0 : let keygen_output = Command::new("openssl")
562 0 : .arg("pkey")
563 0 : .args(["-in", private_key_path.to_str().unwrap()])
564 0 : .arg("-pubout")
565 0 : .args(["-out", public_key_path.to_str().unwrap()])
566 0 : .output()
567 0 : .context("failed to extract public key from private key")?;
568 0 : if !keygen_output.status.success() {
569 0 : bail!(
570 0 : "openssl failed: '{}'",
571 0 : String::from_utf8_lossy(&keygen_output.stderr)
572 0 : );
573 0 : }
574 0 : Ok(())
575 0 : }
576 :
577 : #[cfg(test)]
578 : mod tests {
579 : use super::*;
580 :
581 2 : #[test]
582 2 : fn simple_conf_parsing() {
583 2 : let simple_conf_toml = include_str!("../simple.conf");
584 2 : let simple_conf_parse_result = LocalEnv::parse_config(simple_conf_toml);
585 2 : assert!(
586 2 : simple_conf_parse_result.is_ok(),
587 0 : "failed to parse simple config {simple_conf_toml}, reason: {simple_conf_parse_result:?}"
588 : );
589 :
590 2 : let string_to_replace = "listen_addr = '127.0.0.1:50051'";
591 2 : let spoiled_url_str = "listen_addr = '!@$XOXO%^&'";
592 2 : let spoiled_url_toml = simple_conf_toml.replace(string_to_replace, spoiled_url_str);
593 2 : assert!(
594 2 : spoiled_url_toml.contains(spoiled_url_str),
595 0 : "Failed to replace string {string_to_replace} in the toml file {simple_conf_toml}"
596 : );
597 2 : let spoiled_url_parse_result = LocalEnv::parse_config(&spoiled_url_toml);
598 2 : assert!(
599 2 : spoiled_url_parse_result.is_err(),
600 0 : "expected toml with invalid Url {spoiled_url_toml} to fail the parsing, but got {spoiled_url_parse_result:?}"
601 : );
602 2 : }
603 : }
|
