Line data Source code
1 : use std::num::NonZeroU32;
2 : use std::path::PathBuf;
3 : use std::sync::Arc;
4 : use std::time::Duration;
5 :
6 : use anyhow::{Context, anyhow};
7 : use camino::Utf8PathBuf;
8 : use clap::Parser;
9 : use futures::future::OptionFuture;
10 : use http_utils::tls_certs::ReloadingCertificateResolver;
11 : use hyper0::Uri;
12 : use metrics::BuildInfo;
13 : use metrics::launch_timestamp::LaunchTimestamp;
14 : use reqwest::Certificate;
15 : use storage_controller::http::make_router;
16 : use storage_controller::metrics::preinitialize_metrics;
17 : use storage_controller::persistence::Persistence;
18 : use storage_controller::service::chaos_injector::ChaosInjector;
19 : use storage_controller::service::{
20 : Config, HEARTBEAT_INTERVAL_DEFAULT, LONG_RECONCILE_THRESHOLD_DEFAULT,
21 : MAX_OFFLINE_INTERVAL_DEFAULT, MAX_WARMING_UP_INTERVAL_DEFAULT,
22 : PRIORITY_RECONCILER_CONCURRENCY_DEFAULT, RECONCILER_CONCURRENCY_DEFAULT, Service,
23 : };
24 : use tokio::signal::unix::SignalKind;
25 : use tokio_util::sync::CancellationToken;
26 : use tracing::Instrument;
27 : use utils::auth::{JwtAuth, SwappableJwtAuth};
28 : use utils::logging::{self, LogFormat};
29 : use utils::sentry_init::init_sentry;
30 : use utils::{project_build_tag, project_git_version, tcp_listener};
31 :
32 : project_git_version!(GIT_VERSION);
33 : project_build_tag!(BUILD_TAG);
34 :
35 : #[global_allocator]
36 : static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
37 :
38 : /// Configure jemalloc to profile heap allocations by sampling stack traces every 2 MB (1 << 21).
39 : /// This adds roughly 3% overhead for allocations on average, which is acceptable considering
40 : /// performance-sensitive code will avoid allocations as far as possible anyway.
41 : #[allow(non_upper_case_globals)]
42 : #[unsafe(export_name = "malloc_conf")]
43 : pub static malloc_conf: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:21\0";
44 :
45 : const DEFAULT_SSL_KEY_FILE: &str = "server.key";
46 : const DEFAULT_SSL_CERT_FILE: &str = "server.crt";
47 : const DEFAULT_SSL_CERT_RELOAD_PERIOD: &str = "60s";
48 :
49 : #[derive(Parser)]
50 : #[command(author, version, about, long_about = None)]
51 : #[command(arg_required_else_help(true))]
52 : #[clap(group(
53 : clap::ArgGroup::new("listen-addresses")
54 : .required(true)
55 : .multiple(true)
56 : .args(&["listen", "listen_https"]),
57 : ))]
58 : struct Cli {
59 : /// Host and port to listen HTTP on, like `127.0.0.1:1234`.
60 : /// At least one of ["listen", "listen_https"] should be specified.
61 : // TODO: Make this option dev-only when https is out everywhere.
62 : #[arg(short, long)]
63 : listen: Option<std::net::SocketAddr>,
64 : /// Host and port to listen HTTPS on, like `127.0.0.1:1234`.
65 : /// At least one of ["listen", "listen_https"] should be specified.
66 : #[arg(long)]
67 : listen_https: Option<std::net::SocketAddr>,
68 :
69 : /// Public key for JWT authentication of clients
70 : #[arg(long)]
71 : public_key: Option<String>,
72 :
73 : /// Token for authenticating this service with the pageservers it controls
74 : #[arg(long)]
75 : jwt_token: Option<String>,
76 :
77 : /// Token for authenticating this service with the safekeepers it controls
78 : #[arg(long)]
79 : safekeeper_jwt_token: Option<String>,
80 :
81 : /// Token for authenticating this service with the control plane, when calling
82 : /// the compute notification endpoint
83 : #[arg(long)]
84 : control_plane_jwt_token: Option<String>,
85 :
86 : #[arg(long)]
87 : peer_jwt_token: Option<String>,
88 :
89 : /// URL to control plane compute notification endpoint
90 : #[arg(long)]
91 : compute_hook_url: Option<String>,
92 :
93 : /// URL to control plane storage API prefix
94 : #[arg(long)]
95 : control_plane_url: Option<String>,
96 :
97 : /// URL to connect to postgres, like postgresql://localhost:1234/storage_controller
98 : #[arg(long)]
99 : database_url: Option<String>,
100 :
101 : /// Flag to enable dev mode, which permits running without auth
102 : #[arg(long, default_value = "false")]
103 0 : dev: bool,
104 :
105 : /// Grace period before marking unresponsive pageserver offline
106 : #[arg(long)]
107 : max_offline_interval: Option<humantime::Duration>,
108 :
109 : /// More tolerant grace period before marking unresponsive pagserver offline used
110 : /// around pageserver restarts
111 : #[arg(long)]
112 : max_warming_up_interval: Option<humantime::Duration>,
113 :
114 : /// Size threshold for automatically splitting shards (disabled by default)
115 : #[arg(long)]
116 : split_threshold: Option<u64>,
117 :
118 : /// Maximum number of shards during autosplits. 0 disables autosplits.
119 : // TODO: defaults to 8 for backwards compatibility, should default to 255.
120 : #[arg(long, default_value = "8")]
121 0 : max_split_shards: u8,
122 :
123 : /// Size threshold for initial shard splits of unsharded tenants. 0 disables initial splits.
124 : // TODO: defaults to 64 GB for backwards compatibility. Should default to None.
125 : #[arg(long, default_value = "68719476736")]
126 0 : initial_split_threshold: u64,
127 :
128 : /// Number of target shards for initial splits. 0 or 1 disables initial splits.
129 : // TODO: defaults to 8 for backwards compatibility. Should default to 2.
130 : #[arg(long, default_value = "8")]
131 0 : initial_split_shards: u8,
132 :
133 : /// Maximum number of normal-priority reconcilers that may run in parallel
134 : #[arg(long)]
135 : reconciler_concurrency: Option<usize>,
136 :
137 : /// Maximum number of high-priority reconcilers that may run in parallel
138 : #[arg(long)]
139 : priority_reconciler_concurrency: Option<usize>,
140 :
141 : /// Tenant API rate limit, as requests per second per tenant.
142 : #[arg(long, default_value = "10")]
143 0 : tenant_rate_limit: NonZeroU32,
144 :
145 : /// How long to wait for the initial database connection to be available.
146 : #[arg(long, default_value = "5s")]
147 0 : db_connect_timeout: humantime::Duration,
148 :
149 : #[arg(long, default_value = "false")]
150 0 : start_as_candidate: bool,
151 :
152 : // TODO: make this mandatory once the helm chart gets updated
153 : #[arg(long)]
154 : address_for_peers: Option<Uri>,
155 :
156 : /// `neon_local` sets this to the path of the neon_local repo dir.
157 : /// Only relevant for testing.
158 : // TODO: make `cfg(feature = "testing")`
159 : #[arg(long)]
160 : neon_local_repo_dir: Option<PathBuf>,
161 :
162 : /// Chaos testing: exercise tenant migrations
163 : #[arg(long)]
164 : chaos_interval: Option<humantime::Duration>,
165 :
166 : /// Chaos testing: exercise an immediate exit
167 : #[arg(long)]
168 : chaos_exit_crontab: Option<cron::Schedule>,
169 :
170 : /// Maximum acceptable lag for the secondary location while draining
171 : /// a pageserver
172 : #[arg(long)]
173 : max_secondary_lag_bytes: Option<u64>,
174 :
175 : /// Period with which to send heartbeats to registered nodes
176 : #[arg(long)]
177 : heartbeat_interval: Option<humantime::Duration>,
178 :
179 : #[arg(long)]
180 : long_reconcile_threshold: Option<humantime::Duration>,
181 :
182 : /// Flag to use https for requests to pageserver API.
183 : #[arg(long, default_value = "false")]
184 0 : use_https_pageserver_api: bool,
185 :
186 : // Whether to put timelines onto safekeepers
187 : #[arg(long, default_value = "false")]
188 0 : timelines_onto_safekeepers: bool,
189 :
190 : /// Flag to use https for requests to safekeeper API.
191 : #[arg(long, default_value = "false")]
192 0 : use_https_safekeeper_api: bool,
193 :
194 : /// Path to a file with certificate's private key for https API.
195 : #[arg(long, default_value = DEFAULT_SSL_KEY_FILE)]
196 0 : ssl_key_file: Utf8PathBuf,
197 : /// Path to a file with a X509 certificate for https API.
198 : #[arg(long, default_value = DEFAULT_SSL_CERT_FILE)]
199 0 : ssl_cert_file: Utf8PathBuf,
200 : /// Period to reload certificate and private key from files.
201 : #[arg(long, default_value = DEFAULT_SSL_CERT_RELOAD_PERIOD)]
202 0 : ssl_cert_reload_period: humantime::Duration,
203 : /// Trusted root CA certificates to use in https APIs.
204 : #[arg(long)]
205 : ssl_ca_file: Option<PathBuf>,
206 :
207 : /// Neon local specific flag. When set, ignore [`Cli::control_plane_url`] and deliver
208 : /// the compute notification directly (instead of via control plane).
209 : #[arg(long, default_value = "false")]
210 0 : use_local_compute_notifications: bool,
211 : }
212 :
213 : enum StrictMode {
214 : /// In strict mode, we will require that all secrets are loaded, i.e. security features
215 : /// may not be implicitly turned off by omitting secrets in the environment.
216 : Strict,
217 : /// In dev mode, secrets are optional, and omitting a particular secret will implicitly
218 : /// disable the auth related to it (e.g. no pageserver jwt key -> send unauthenticated
219 : /// requests, no public key -> don't authenticate incoming requests).
220 : Dev,
221 : }
222 :
223 : impl Default for StrictMode {
224 0 : fn default() -> Self {
225 0 : Self::Strict
226 0 : }
227 : }
228 :
229 : /// Secrets may either be provided on the command line (for testing), or loaded from AWS SecretManager: this
230 : /// type encapsulates the logic to decide which and do the loading.
231 : struct Secrets {
232 : database_url: String,
233 : public_key: Option<JwtAuth>,
234 : pageserver_jwt_token: Option<String>,
235 : safekeeper_jwt_token: Option<String>,
236 : control_plane_jwt_token: Option<String>,
237 : peer_jwt_token: Option<String>,
238 : }
239 :
240 : impl Secrets {
241 : const DATABASE_URL_ENV: &'static str = "DATABASE_URL";
242 : const PAGESERVER_JWT_TOKEN_ENV: &'static str = "PAGESERVER_JWT_TOKEN";
243 : const SAFEKEEPER_JWT_TOKEN_ENV: &'static str = "SAFEKEEPER_JWT_TOKEN";
244 : const CONTROL_PLANE_JWT_TOKEN_ENV: &'static str = "CONTROL_PLANE_JWT_TOKEN";
245 : const PEER_JWT_TOKEN_ENV: &'static str = "PEER_JWT_TOKEN";
246 : const PUBLIC_KEY_ENV: &'static str = "PUBLIC_KEY";
247 :
248 : /// Load secrets from, in order of preference:
249 : /// - CLI args if database URL is provided on the CLI
250 : /// - Environment variables if DATABASE_URL is set.
251 0 : async fn load(args: &Cli) -> anyhow::Result<Self> {
252 0 : let Some(database_url) = Self::load_secret(&args.database_url, Self::DATABASE_URL_ENV)
253 : else {
254 0 : anyhow::bail!(
255 0 : "Database URL is not set (set `--database-url`, or `DATABASE_URL` environment)"
256 0 : )
257 : };
258 :
259 0 : let public_key = match Self::load_secret(&args.public_key, Self::PUBLIC_KEY_ENV) {
260 0 : Some(v) => Some(JwtAuth::from_key(v).context("Loading public key")?),
261 0 : None => None,
262 : };
263 :
264 0 : let this = Self {
265 0 : database_url,
266 0 : public_key,
267 0 : pageserver_jwt_token: Self::load_secret(
268 0 : &args.jwt_token,
269 0 : Self::PAGESERVER_JWT_TOKEN_ENV,
270 0 : ),
271 0 : safekeeper_jwt_token: Self::load_secret(
272 0 : &args.safekeeper_jwt_token,
273 0 : Self::SAFEKEEPER_JWT_TOKEN_ENV,
274 0 : ),
275 0 : control_plane_jwt_token: Self::load_secret(
276 0 : &args.control_plane_jwt_token,
277 0 : Self::CONTROL_PLANE_JWT_TOKEN_ENV,
278 0 : ),
279 0 : peer_jwt_token: Self::load_secret(&args.peer_jwt_token, Self::PEER_JWT_TOKEN_ENV),
280 0 : };
281 0 :
282 0 : Ok(this)
283 0 : }
284 :
285 0 : fn load_secret(cli: &Option<String>, env_name: &str) -> Option<String> {
286 0 : if let Some(v) = cli {
287 0 : Some(v.clone())
288 0 : } else if let Ok(v) = std::env::var(env_name) {
289 0 : Some(v)
290 : } else {
291 0 : None
292 : }
293 0 : }
294 : }
295 :
296 0 : fn main() -> anyhow::Result<()> {
297 0 : logging::init(
298 0 : LogFormat::Plain,
299 0 : logging::TracingErrorLayerEnablement::Disabled,
300 0 : logging::Output::Stdout,
301 0 : )?;
302 :
303 : // log using tracing so we don't get confused output by default hook writing to stderr
304 0 : utils::logging::replace_panic_hook_with_tracing_panic_hook().forget();
305 0 :
306 0 : let _sentry_guard = init_sentry(Some(GIT_VERSION.into()), &[]);
307 0 :
308 0 : let hook = std::panic::take_hook();
309 0 : std::panic::set_hook(Box::new(move |info| {
310 0 : // let sentry send a message (and flush)
311 0 : // and trace the error
312 0 : hook(info);
313 0 :
314 0 : std::process::exit(1);
315 0 : }));
316 0 :
317 0 : tokio::runtime::Builder::new_current_thread()
318 0 : // We use spawn_blocking for database operations, so require approximately
319 0 : // as many blocking threads as we will open database connections.
320 0 : .max_blocking_threads(Persistence::MAX_CONNECTIONS as usize)
321 0 : .enable_all()
322 0 : .build()
323 0 : .unwrap()
324 0 : .block_on(async_main())
325 0 : }
326 :
327 0 : async fn async_main() -> anyhow::Result<()> {
328 0 : let launch_ts = Box::leak(Box::new(LaunchTimestamp::generate()));
329 0 :
330 0 : preinitialize_metrics();
331 0 :
332 0 : let args = Cli::parse();
333 0 : tracing::info!(
334 0 : "version: {}, launch_timestamp: {}, build_tag {}",
335 0 : GIT_VERSION,
336 0 : launch_ts.to_string(),
337 : BUILD_TAG,
338 : );
339 :
340 0 : let build_info = BuildInfo {
341 0 : revision: GIT_VERSION,
342 0 : build_tag: BUILD_TAG,
343 0 : };
344 :
345 0 : let strict_mode = if args.dev {
346 0 : StrictMode::Dev
347 : } else {
348 0 : StrictMode::Strict
349 : };
350 :
351 0 : let secrets = Secrets::load(&args).await?;
352 :
353 : // Validate required secrets and arguments are provided in strict mode
354 0 : match strict_mode {
355 : StrictMode::Strict
356 0 : if (secrets.public_key.is_none()
357 0 : || secrets.pageserver_jwt_token.is_none()
358 0 : || secrets.control_plane_jwt_token.is_none()
359 0 : || secrets.safekeeper_jwt_token.is_none()) =>
360 0 : {
361 0 : // Production systems should always have secrets configured: if public_key was not set
362 0 : // then we would implicitly disable auth.
363 0 : anyhow::bail!(
364 0 : "Insecure config! One or more secrets is not set. This is only permitted in `--dev` mode"
365 0 : );
366 : }
367 : StrictMode::Strict
368 0 : if args.compute_hook_url.is_none() && args.control_plane_url.is_none() =>
369 0 : {
370 0 : // Production systems should always have a control plane URL set, to prevent falling
371 0 : // back to trying to use neon_local.
372 0 : anyhow::bail!(
373 0 : "neither `--compute-hook-url` nor `--control-plane-url` are set: this is only permitted in `--dev` mode"
374 0 : );
375 : }
376 0 : StrictMode::Strict if args.use_local_compute_notifications => {
377 0 : anyhow::bail!("`--use-local-compute-notifications` is only permitted in `--dev` mode");
378 : }
379 : StrictMode::Strict => {
380 0 : tracing::info!("Starting in strict mode: configuration is OK.")
381 : }
382 : StrictMode::Dev => {
383 0 : tracing::warn!("Starting in dev mode: this may be an insecure configuration.")
384 : }
385 : }
386 :
387 0 : let ssl_ca_certs = match args.ssl_ca_file.as_ref() {
388 0 : Some(ssl_ca_file) => {
389 0 : tracing::info!("Using ssl root CA file: {ssl_ca_file:?}");
390 0 : let buf = tokio::fs::read(ssl_ca_file).await?;
391 0 : Certificate::from_pem_bundle(&buf)?
392 : }
393 0 : None => Vec::new(),
394 : };
395 :
396 0 : let config = Config {
397 0 : pageserver_jwt_token: secrets.pageserver_jwt_token,
398 0 : safekeeper_jwt_token: secrets.safekeeper_jwt_token,
399 0 : control_plane_jwt_token: secrets.control_plane_jwt_token,
400 0 : peer_jwt_token: secrets.peer_jwt_token,
401 0 : compute_hook_url: args.compute_hook_url,
402 0 : control_plane_url: args.control_plane_url,
403 0 : max_offline_interval: args
404 0 : .max_offline_interval
405 0 : .map(humantime::Duration::into)
406 0 : .unwrap_or(MAX_OFFLINE_INTERVAL_DEFAULT),
407 0 : max_warming_up_interval: args
408 0 : .max_warming_up_interval
409 0 : .map(humantime::Duration::into)
410 0 : .unwrap_or(MAX_WARMING_UP_INTERVAL_DEFAULT),
411 0 : reconciler_concurrency: args
412 0 : .reconciler_concurrency
413 0 : .unwrap_or(RECONCILER_CONCURRENCY_DEFAULT),
414 0 : priority_reconciler_concurrency: args
415 0 : .priority_reconciler_concurrency
416 0 : .unwrap_or(PRIORITY_RECONCILER_CONCURRENCY_DEFAULT),
417 0 : tenant_rate_limit: args.tenant_rate_limit,
418 0 : split_threshold: args.split_threshold,
419 0 : max_split_shards: args.max_split_shards,
420 0 : initial_split_threshold: Some(args.initial_split_threshold),
421 0 : initial_split_shards: args.initial_split_shards,
422 0 : neon_local_repo_dir: args.neon_local_repo_dir,
423 0 : max_secondary_lag_bytes: args.max_secondary_lag_bytes,
424 0 : heartbeat_interval: args
425 0 : .heartbeat_interval
426 0 : .map(humantime::Duration::into)
427 0 : .unwrap_or(HEARTBEAT_INTERVAL_DEFAULT),
428 0 : long_reconcile_threshold: args
429 0 : .long_reconcile_threshold
430 0 : .map(humantime::Duration::into)
431 0 : .unwrap_or(LONG_RECONCILE_THRESHOLD_DEFAULT),
432 0 : address_for_peers: args.address_for_peers,
433 0 : start_as_candidate: args.start_as_candidate,
434 0 : use_https_pageserver_api: args.use_https_pageserver_api,
435 0 : use_https_safekeeper_api: args.use_https_safekeeper_api,
436 0 : ssl_ca_certs,
437 0 : timelines_onto_safekeepers: args.timelines_onto_safekeepers,
438 0 : use_local_compute_notifications: args.use_local_compute_notifications,
439 0 : };
440 0 :
441 0 : // Validate that we can connect to the database
442 0 : Persistence::await_connection(&secrets.database_url, args.db_connect_timeout.into()).await?;
443 :
444 0 : let persistence = Arc::new(Persistence::new(secrets.database_url).await);
445 :
446 0 : let service = Service::spawn(config, persistence.clone()).await?;
447 :
448 0 : let auth = secrets
449 0 : .public_key
450 0 : .map(|jwt_auth| Arc::new(SwappableJwtAuth::new(jwt_auth)));
451 0 : let router = make_router(service.clone(), auth, build_info)
452 0 : .build()
453 0 : .map_err(|err| anyhow!(err))?;
454 0 : let http_service =
455 0 : Arc::new(http_utils::RequestServiceBuilder::new(router).map_err(|err| anyhow!(err))?);
456 :
457 0 : let api_shutdown = CancellationToken::new();
458 :
459 : // Start HTTP server
460 0 : let http_server_task: OptionFuture<_> = match args.listen {
461 0 : Some(http_addr) => {
462 0 : let http_listener = tcp_listener::bind(http_addr)?;
463 0 : let http_server =
464 0 : http_utils::server::Server::new(Arc::clone(&http_service), http_listener, None)?;
465 :
466 0 : tracing::info!("Serving HTTP on {}", http_addr);
467 0 : Some(tokio::task::spawn(http_server.serve(api_shutdown.clone())))
468 : }
469 0 : None => None,
470 : }
471 0 : .into();
472 :
473 : // Start HTTPS server
474 0 : let https_server_task: OptionFuture<_> = match args.listen_https {
475 0 : Some(https_addr) => {
476 0 : let https_listener = tcp_listener::bind(https_addr)?;
477 :
478 0 : let resolver = ReloadingCertificateResolver::new(
479 0 : &args.ssl_key_file,
480 0 : &args.ssl_cert_file,
481 0 : *args.ssl_cert_reload_period,
482 0 : )
483 0 : .await?;
484 :
485 0 : let server_config = rustls::ServerConfig::builder()
486 0 : .with_no_client_auth()
487 0 : .with_cert_resolver(resolver);
488 0 :
489 0 : let tls_acceptor = tokio_rustls::TlsAcceptor::from(Arc::new(server_config));
490 0 : let https_server =
491 0 : http_utils::server::Server::new(http_service, https_listener, Some(tls_acceptor))?;
492 :
493 0 : tracing::info!("Serving HTTPS on {}", https_addr);
494 0 : Some(tokio::task::spawn(https_server.serve(api_shutdown.clone())))
495 : }
496 0 : None => None,
497 : }
498 0 : .into();
499 0 :
500 0 : let chaos_task = args.chaos_interval.map(|interval| {
501 0 : let service = service.clone();
502 0 : let cancel = CancellationToken::new();
503 0 : let cancel_bg = cancel.clone();
504 0 : let chaos_exit_crontab = args.chaos_exit_crontab;
505 0 : (
506 0 : tokio::task::spawn(
507 0 : async move {
508 0 : let mut chaos_injector =
509 0 : ChaosInjector::new(service, interval.into(), chaos_exit_crontab);
510 0 : chaos_injector.run(cancel_bg).await
511 0 : }
512 0 : .instrument(tracing::info_span!("chaos_injector")),
513 : ),
514 0 : cancel,
515 0 : )
516 0 : });
517 :
518 : // Wait until we receive a signal
519 0 : let mut sigint = tokio::signal::unix::signal(SignalKind::interrupt())?;
520 0 : let mut sigquit = tokio::signal::unix::signal(SignalKind::quit())?;
521 0 : let mut sigterm = tokio::signal::unix::signal(SignalKind::terminate())?;
522 0 : tokio::pin!(http_server_task, https_server_task);
523 0 : tokio::select! {
524 0 : _ = sigint.recv() => {},
525 0 : _ = sigterm.recv() => {},
526 0 : _ = sigquit.recv() => {},
527 0 : Some(err) = &mut http_server_task => {
528 0 : panic!("HTTP server task failed: {err:#?}");
529 : }
530 0 : Some(err) = &mut https_server_task => {
531 0 : panic!("HTTPS server task failed: {err:#?}");
532 : }
533 : }
534 0 : tracing::info!("Terminating on signal");
535 :
536 : // Stop HTTP and HTTPS servers first, so that we don't have to service requests
537 : // while shutting down Service.
538 0 : api_shutdown.cancel();
539 0 :
540 0 : // If the deadline is exceeded, we will fall through and shut down the service anyway,
541 0 : // any request handlers in flight will experience cancellation & their clients will
542 0 : // see a torn connection.
543 0 : let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
544 0 :
545 0 : match tokio::time::timeout_at(deadline, http_server_task).await {
546 0 : Ok(Some(Ok(_))) => tracing::info!("Joined HTTP server task"),
547 0 : Ok(Some(Err(e))) => tracing::error!("Error joining HTTP server task: {e}"),
548 0 : Ok(None) => {} // HTTP is disabled.
549 0 : Err(_) => tracing::warn!("Timed out joining HTTP server task"),
550 : }
551 :
552 0 : match tokio::time::timeout_at(deadline, https_server_task).await {
553 0 : Ok(Some(Ok(_))) => tracing::info!("Joined HTTPS server task"),
554 0 : Ok(Some(Err(e))) => tracing::error!("Error joining HTTPS server task: {e}"),
555 0 : Ok(None) => {} // HTTPS is disabled.
556 0 : Err(_) => tracing::warn!("Timed out joining HTTPS server task"),
557 : }
558 :
559 : // If we were injecting chaos, stop that so that we're not calling into Service while it shuts down
560 0 : if let Some((chaos_jh, chaos_cancel)) = chaos_task {
561 0 : chaos_cancel.cancel();
562 0 : chaos_jh.await.ok();
563 0 : }
564 :
565 0 : service.shutdown().await;
566 0 : tracing::info!("Service shutdown complete");
567 :
568 0 : std::process::exit(0);
569 0 : }
|
