TLA Line data Source code
1 : use std::sync::Arc;
2 :
3 : use anyhow::Context;
4 : use camino::{Utf8Path, Utf8PathBuf};
5 : use pageserver_api::models::TenantState;
6 : use remote_storage::{DownloadError, GenericRemoteStorage, RemotePath};
7 : use tokio::sync::OwnedMutexGuard;
8 : use tokio_util::sync::CancellationToken;
9 : use tracing::{error, info, instrument, warn, Instrument, Span};
10 :
11 : use utils::{
12 : backoff, completion, crashsafe, fs_ext,
13 : id::{TenantId, TimelineId},
14 : };
15 :
16 : use crate::{
17 : config::PageServerConf,
18 : context::RequestContext,
19 : task_mgr::{self, TaskKind},
20 : InitializationOrder,
21 : };
22 :
23 : use super::{
24 : mgr::{GetTenantError, TenantsMap},
25 : remote_timeline_client::{FAILED_REMOTE_OP_RETRIES, FAILED_UPLOAD_WARN_THRESHOLD},
26 : span,
27 : timeline::delete::DeleteTimelineFlow,
28 : tree_sort_timelines, DeleteTimelineError, Tenant,
29 : };
30 :
31 : const SHOULD_RESUME_DELETION_FETCH_MARK_ATTEMPTS: u32 = 3;
32 :
33 CBC 214 : #[derive(Debug, thiserror::Error)]
34 : pub(crate) enum DeleteTenantError {
35 : #[error("GetTenant {0}")]
36 : Get(#[from] GetTenantError),
37 :
38 : #[error("Invalid state {0}. Expected Active or Broken")]
39 : InvalidState(TenantState),
40 :
41 : #[error("Tenant deletion is already in progress")]
42 : AlreadyInProgress,
43 :
44 : #[error("Timeline {0}")]
45 : Timeline(#[from] DeleteTimelineError),
46 :
47 : #[error(transparent)]
48 : Other(#[from] anyhow::Error),
49 : }
50 :
51 : type DeletionGuard = tokio::sync::OwnedMutexGuard<DeleteTenantFlow>;
52 :
53 840 : fn remote_tenant_delete_mark_path(
54 840 : conf: &PageServerConf,
55 840 : tenant_id: &TenantId,
56 840 : ) -> anyhow::Result<RemotePath> {
57 840 : let tenant_remote_path = conf
58 840 : .tenant_path(tenant_id)
59 840 : .strip_prefix(&conf.workdir)
60 840 : .context("Failed to strip workdir prefix")
61 840 : .and_then(RemotePath::new)
62 840 : .context("tenant path")?;
63 840 : Ok(tenant_remote_path.join(Utf8Path::new("deleted")))
64 840 : }
65 :
66 80 : async fn create_remote_delete_mark(
67 80 : conf: &PageServerConf,
68 80 : remote_storage: &GenericRemoteStorage,
69 80 : tenant_id: &TenantId,
70 80 : ) -> Result<(), DeleteTenantError> {
71 80 : let remote_mark_path = remote_tenant_delete_mark_path(conf, tenant_id)?;
72 :
73 80 : let data: &[u8] = &[];
74 80 : backoff::retry(
75 120 : || async {
76 120 : remote_storage
77 120 : .upload(data, 0, &remote_mark_path, None)
78 160 : .await
79 120 : },
80 80 : |_e| false,
81 80 : FAILED_UPLOAD_WARN_THRESHOLD,
82 80 : FAILED_REMOTE_OP_RETRIES,
83 80 : "mark_upload",
84 80 : // TODO: use a cancellation token (https://github.com/neondatabase/neon/issues/5066)
85 80 : backoff::Cancel::new(CancellationToken::new(), || unreachable!()),
86 80 : )
87 160 : .await
88 80 : .context("mark_upload")?;
89 :
90 80 : Ok(())
91 80 : }
92 :
93 76 : async fn create_local_delete_mark(
94 76 : conf: &PageServerConf,
95 76 : tenant_id: &TenantId,
96 76 : ) -> Result<(), DeleteTenantError> {
97 76 : let marker_path = conf.tenant_deleted_mark_file_path(tenant_id);
98 76 :
99 76 : // Note: we're ok to replace existing file.
100 76 : let _ = std::fs::OpenOptions::new()
101 76 : .write(true)
102 76 : .create(true)
103 76 : .open(&marker_path)
104 76 : .with_context(|| format!("could not create delete marker file {marker_path:?}"))?;
105 :
106 76 : crashsafe::fsync_file_and_parent(&marker_path).context("sync_mark")?;
107 :
108 76 : Ok(())
109 76 : }
110 :
111 95 : async fn schedule_ordered_timeline_deletions(
112 95 : tenant: &Arc<Tenant>,
113 95 : ) -> Result<Vec<(Arc<tokio::sync::Mutex<DeleteTimelineFlow>>, TimelineId)>, DeleteTenantError> {
114 95 : // Tenant is stopping at this point. We know it will be deleted.
115 95 : // No new timelines should be created.
116 95 : // Tree sort timelines to delete from leafs to the root.
117 95 : // NOTE: by calling clone we release the mutex which creates a possibility for a race: pending deletion
118 95 : // can complete and remove timeline from the map in between our call to clone
119 95 : // and `DeleteTimelineFlow::run`, so `run` wont find timeline in `timelines` map.
120 95 : // timelines.lock is currently synchronous so we cant hold it across await point.
121 95 : // So just ignore NotFound error if we get it from `run`.
122 95 : // Beware: in case it becomes async and we try to hold it here, `run` also locks it, which can create a deadlock.
123 95 : let timelines = tenant.timelines.lock().unwrap().clone();
124 95 : let sorted =
125 161 : tree_sort_timelines(timelines, |t| t.get_ancestor_timeline_id()).context("tree sort")?;
126 :
127 95 : let mut already_running_deletions = vec![];
128 :
129 139 : for (timeline_id, _) in sorted.into_iter().rev() {
130 4590 : if let Err(e) = DeleteTimelineFlow::run(tenant, timeline_id, true).await {
131 26 : match e {
132 : DeleteTimelineError::NotFound => {
133 : // Timeline deletion finished after call to clone above but before call
134 : // to `DeleteTimelineFlow::run` and removed timeline from the map.
135 3 : continue;
136 : }
137 4 : DeleteTimelineError::AlreadyInProgress(guard) => {
138 4 : already_running_deletions.push((guard, timeline_id));
139 4 : continue;
140 : }
141 19 : e => return Err(DeleteTenantError::Timeline(e)),
142 : }
143 113 : }
144 : }
145 :
146 76 : Ok(already_running_deletions)
147 95 : }
148 :
149 66 : async fn ensure_timelines_dir_empty(timelines_path: &Utf8Path) -> Result<(), DeleteTenantError> {
150 : // Assert timelines dir is empty.
151 131 : if !fs_ext::is_directory_empty(timelines_path).await? {
152 : // Display first 10 items in directory
153 UBC 0 : let list = &fs_ext::list_dir(timelines_path).await.context("list_dir")?[..10];
154 0 : return Err(DeleteTenantError::Other(anyhow::anyhow!(
155 0 : "Timelines directory is not empty after all timelines deletion: {list:?}"
156 0 : )));
157 CBC 66 : }
158 66 :
159 66 : Ok(())
160 66 : }
161 :
162 72 : async fn remove_tenant_remote_delete_mark(
163 72 : conf: &PageServerConf,
164 72 : remote_storage: Option<&GenericRemoteStorage>,
165 72 : tenant_id: &TenantId,
166 72 : ) -> Result<(), DeleteTenantError> {
167 72 : if let Some(remote_storage) = remote_storage {
168 72 : let path = remote_tenant_delete_mark_path(conf, tenant_id)?;
169 72 : backoff::retry(
170 321 : || async { remote_storage.delete(&path).await },
171 72 : |_e| false,
172 72 : FAILED_UPLOAD_WARN_THRESHOLD,
173 72 : FAILED_REMOTE_OP_RETRIES,
174 72 : "remove_tenant_remote_delete_mark",
175 72 : // TODO: use a cancellation token (https://github.com/neondatabase/neon/issues/5066)
176 72 : backoff::Cancel::new(CancellationToken::new(), || unreachable!()),
177 72 : )
178 321 : .await
179 72 : .context("remove_tenant_remote_delete_mark")?;
180 UBC 0 : }
181 CBC 72 : Ok(())
182 72 : }
183 :
184 : // Cleanup fs traces: tenant config, timelines dir local delete mark, tenant dir
185 68 : async fn cleanup_remaining_fs_traces(
186 68 : conf: &PageServerConf,
187 68 : tenant_id: &TenantId,
188 68 : ) -> Result<(), DeleteTenantError> {
189 316 : let rm = |p: Utf8PathBuf, is_dir: bool| async move {
190 316 : if is_dir {
191 120 : tokio::fs::remove_dir(&p).await
192 68 : } else {
193 196 : tokio::fs::remove_file(&p).await
194 68 : }
195 316 : .or_else(fs_ext::ignore_not_found)
196 316 : .with_context(|| format!("failed to delete {p}"))
197 316 : };
198 68 :
199 68 : rm(conf.tenant_config_path(tenant_id), false).await?;
200 68 : rm(conf.tenant_location_config_path(tenant_id), false).await?;
201 :
202 68 : fail::fail_point!("tenant-delete-before-remove-timelines-dir", |_| {
203 4 : Err(anyhow::anyhow!(
204 4 : "failpoint: tenant-delete-before-remove-timelines-dir"
205 4 : ))?
206 68 : });
207 :
208 64 : rm(conf.timelines_path(tenant_id), true).await?;
209 :
210 64 : fail::fail_point!("tenant-delete-before-remove-deleted-mark", |_| {
211 4 : Err(anyhow::anyhow!(
212 4 : "failpoint: tenant-delete-before-remove-deleted-mark"
213 4 : ))?
214 64 : });
215 :
216 : // Make sure previous deletions are ordered before mark removal.
217 : // Otherwise there is no guarantee that they reach the disk before mark deletion.
218 : // So its possible for mark to reach disk first and for other deletions
219 : // to be reordered later and thus missed if a crash occurs.
220 : // Note that we dont need to sync after mark file is removed
221 : // because we can tolerate the case when mark file reappears on startup.
222 60 : let tenant_path = &conf.tenant_path(tenant_id);
223 60 : if tenant_path.exists() {
224 60 : crashsafe::fsync_async(&conf.tenant_path(tenant_id))
225 119 : .await
226 60 : .context("fsync_pre_mark_remove")?;
227 UBC 0 : }
228 :
229 CBC 60 : rm(conf.tenant_deleted_mark_file_path(tenant_id), false).await?;
230 :
231 60 : fail::fail_point!("tenant-delete-before-remove-tenant-dir", |_| {
232 4 : Err(anyhow::anyhow!(
233 4 : "failpoint: tenant-delete-before-remove-tenant-dir"
234 4 : ))?
235 60 : });
236 :
237 56 : rm(conf.tenant_path(tenant_id), true).await?;
238 :
239 56 : Ok(())
240 68 : }
241 :
242 688 : pub(crate) async fn remote_delete_mark_exists(
243 688 : conf: &PageServerConf,
244 688 : tenant_id: &TenantId,
245 688 : remote_storage: &GenericRemoteStorage,
246 688 : ) -> anyhow::Result<bool> {
247 : // If remote storage is there we rely on it
248 688 : let remote_mark_path = remote_tenant_delete_mark_path(conf, tenant_id).context("path")?;
249 :
250 688 : let result = backoff::retry(
251 1716 : || async { remote_storage.download(&remote_mark_path).await },
252 747 : |e| matches!(e, DownloadError::NotFound),
253 688 : SHOULD_RESUME_DELETION_FETCH_MARK_ATTEMPTS,
254 688 : SHOULD_RESUME_DELETION_FETCH_MARK_ATTEMPTS,
255 688 : "fetch_tenant_deletion_mark",
256 688 : // TODO: use a cancellation token (https://github.com/neondatabase/neon/issues/5066)
257 688 : backoff::Cancel::new(CancellationToken::new(), || unreachable!()),
258 688 : )
259 1716 : .await;
260 :
261 683 : match result {
262 5 : Ok(_) => Ok(true),
263 683 : Err(DownloadError::NotFound) => Ok(false),
264 UBC 0 : Err(e) => Err(anyhow::anyhow!(e)).context("remote_delete_mark_exists")?,
265 : }
266 CBC 688 : }
267 :
268 : /// Orchestrates tenant shut down of all tasks, removes its in-memory structures,
269 : /// and deletes its data from both disk and s3.
270 : /// The sequence of steps:
271 : /// 1. Upload remote deletion mark.
272 : /// 2. Create local mark file.
273 : /// 3. Shutdown tasks
274 : /// 4. Run ordered timeline deletions
275 : /// 5. Wait for timeline deletion operations that were scheduled before tenant deletion was requested
276 : /// 6. Remove remote mark
277 : /// 7. Cleanup remaining fs traces, tenant dir, config, timelines dir, local delete mark
278 : /// It is resumable from any step in case a crash/restart occurs.
279 : /// There are three entrypoints to the process:
280 : /// 1. [`DeleteTenantFlow::run`] this is the main one called by a management api handler.
281 : /// 2. [`DeleteTenantFlow::resume_from_load`] is called during restarts when local or remote deletion marks are still there.
282 : /// 3. [`DeleteTenantFlow::resume_from_attach`] is called when deletion is resumed tenant is found to be deleted during attach process.
283 : /// Note the only other place that messes around timeline delete mark is the `Tenant::spawn_load` function.
284 748 : #[derive(Default)]
285 : pub enum DeleteTenantFlow {
286 : #[default]
287 : NotStarted,
288 : InProgress,
289 : Finished,
290 : }
291 :
292 : impl DeleteTenantFlow {
293 : // These steps are run in the context of management api request handler.
294 : // Long running steps are continued to run in the background.
295 : // NB: If this fails half-way through, and is retried, the retry will go through
296 : // all the same steps again. Make sure the code here is idempotent, and don't
297 : // error out if some of the shutdown tasks have already been completed!
298 : // NOTE: static needed for background part.
299 : // We assume that calling code sets up the span with tenant_id.
300 273 : #[instrument(skip_all)]
301 : pub(crate) async fn run(
302 : conf: &'static PageServerConf,
303 : remote_storage: Option<GenericRemoteStorage>,
304 : tenants: &'static tokio::sync::RwLock<TenantsMap>,
305 : tenant_id: TenantId,
306 : ) -> Result<(), DeleteTenantError> {
307 : span::debug_assert_current_span_has_tenant_id();
308 :
309 : let (tenant, mut guard) = Self::prepare(tenants, tenant_id).await?;
310 :
311 : if let Err(e) = Self::run_inner(&mut guard, conf, remote_storage.as_ref(), &tenant).await {
312 : tenant.set_broken(format!("{e:#}")).await;
313 : return Err(e);
314 : }
315 :
316 : Self::schedule_background(guard, conf, remote_storage, tenants, tenant);
317 :
318 : Ok(())
319 : }
320 :
321 : // Helper function needed to be able to match once on returned error and transition tenant into broken state.
322 : // This is needed because tenant.shutwodn is not idempotent. If tenant state is set to stopping another call to tenant.shutdown
323 : // will result in an error, but here we need to be able to retry shutdown when tenant deletion is retried.
324 : // So the solution is to set tenant state to broken.
325 84 : async fn run_inner(
326 84 : guard: &mut OwnedMutexGuard<Self>,
327 84 : conf: &'static PageServerConf,
328 84 : remote_storage: Option<&GenericRemoteStorage>,
329 84 : tenant: &Tenant,
330 84 : ) -> Result<(), DeleteTenantError> {
331 84 : guard.mark_in_progress()?;
332 :
333 84 : fail::fail_point!("tenant-delete-before-create-remote-mark", |_| {
334 4 : Err(anyhow::anyhow!(
335 4 : "failpoint: tenant-delete-before-create-remote-mark"
336 4 : ))?
337 84 : });
338 :
339 : // IDEA: implement detach as delete without remote storage. Then they would use the same lock (deletion_progress) so wont contend.
340 : // Though sounds scary, different mark name?
341 : // Detach currently uses remove_dir_all so in case of a crash we can end up in a weird state.
342 80 : if let Some(remote_storage) = &remote_storage {
343 80 : create_remote_delete_mark(conf, remote_storage, &tenant.tenant_id)
344 160 : .await
345 80 : .context("remote_mark")?
346 UBC 0 : }
347 :
348 CBC 80 : fail::fail_point!("tenant-delete-before-create-local-mark", |_| {
349 4 : Err(anyhow::anyhow!(
350 4 : "failpoint: tenant-delete-before-create-local-mark"
351 4 : ))?
352 80 : });
353 :
354 76 : create_local_delete_mark(conf, &tenant.tenant_id)
355 UBC 0 : .await
356 CBC 76 : .context("local delete mark")?;
357 :
358 76 : fail::fail_point!("tenant-delete-before-background", |_| {
359 4 : Err(anyhow::anyhow!(
360 4 : "failpoint: tenant-delete-before-background"
361 4 : ))?
362 76 : });
363 :
364 72 : Ok(())
365 84 : }
366 :
367 84 : fn mark_in_progress(&mut self) -> anyhow::Result<()> {
368 84 : match self {
369 UBC 0 : Self::Finished => anyhow::bail!("Bug. Is in finished state"),
370 CBC 24 : Self::InProgress { .. } => { /* We're in a retry */ }
371 60 : Self::NotStarted => { /* Fresh start */ }
372 : }
373 :
374 84 : *self = Self::InProgress;
375 84 :
376 84 : Ok(())
377 84 : }
378 :
379 706 : pub(crate) async fn should_resume_deletion(
380 706 : conf: &'static PageServerConf,
381 706 : remote_storage: Option<&GenericRemoteStorage>,
382 706 : tenant: &Tenant,
383 706 : ) -> Result<Option<DeletionGuard>, DeleteTenantError> {
384 706 : let acquire = |t: &Tenant| {
385 23 : Some(
386 23 : Arc::clone(&t.delete_progress)
387 23 : .try_lock_owned()
388 23 : .expect("we're the only owner during init"),
389 23 : )
390 706 : };
391 706 :
392 706 : let tenant_id = tenant.tenant_id;
393 706 : // Check local mark first, if its there there is no need to go to s3 to check whether remote one exists.
394 706 : if conf.tenant_deleted_mark_file_path(&tenant_id).exists() {
395 18 : return Ok(acquire(tenant));
396 688 : }
397 :
398 688 : let remote_storage = match remote_storage {
399 688 : Some(remote_storage) => remote_storage,
400 UBC 0 : None => return Ok(None),
401 : };
402 :
403 CBC 1716 : if remote_delete_mark_exists(conf, &tenant_id, remote_storage).await? {
404 5 : Ok(acquire(tenant))
405 : } else {
406 683 : Ok(None)
407 : }
408 706 : }
409 :
410 20 : pub(crate) async fn resume_from_load(
411 20 : guard: DeletionGuard,
412 20 : tenant: &Arc<Tenant>,
413 20 : init_order: Option<&InitializationOrder>,
414 20 : tenants: &'static tokio::sync::RwLock<TenantsMap>,
415 20 : ctx: &RequestContext,
416 20 : ) -> Result<(), DeleteTenantError> {
417 20 : let (_, progress) = completion::channel();
418 20 :
419 20 : tenant
420 20 : .set_stopping(progress, true, false)
421 UBC 0 : .await
422 CBC 20 : .expect("cant be stopping or broken");
423 20 :
424 20 : // Do not consume valuable resources during the load phase, continue deletion once init phase is complete.
425 20 : let background_jobs_can_start = init_order.as_ref().map(|x| &x.background_jobs_can_start);
426 20 : if let Some(background) = background_jobs_can_start {
427 20 : info!("waiting for backgound jobs barrier");
428 20 : background.clone().wait().await;
429 20 : info!("ready for backgound jobs barrier");
430 UBC 0 : }
431 :
432 : // Tenant may not be loadable if we fail late in cleanup_remaining_fs_traces (e g remove timelines dir)
433 CBC 20 : let timelines_path = tenant.conf.timelines_path(&tenant.tenant_id);
434 20 : if timelines_path.exists() {
435 70 : tenant.load(init_order, None, ctx).await.context("load")?;
436 2 : }
437 :
438 20 : Self::background(
439 20 : guard,
440 20 : tenant.conf,
441 20 : tenant.remote_storage.clone(),
442 20 : tenants,
443 20 : tenant,
444 20 : )
445 1046 : .await
446 20 : }
447 :
448 3 : pub(crate) async fn resume_from_attach(
449 3 : guard: DeletionGuard,
450 3 : tenant: &Arc<Tenant>,
451 3 : tenants: &'static tokio::sync::RwLock<TenantsMap>,
452 3 : ctx: &RequestContext,
453 3 : ) -> Result<(), DeleteTenantError> {
454 3 : let (_, progress) = completion::channel();
455 3 :
456 3 : tenant
457 3 : .set_stopping(progress, false, true)
458 UBC 0 : .await
459 CBC 3 : .expect("cant be stopping or broken");
460 3 :
461 3 : tenant
462 3 : .attach(ctx, super::AttachMarkerMode::Expect)
463 34 : .await
464 3 : .context("attach")?;
465 :
466 3 : Self::background(
467 3 : guard,
468 3 : tenant.conf,
469 3 : tenant.remote_storage.clone(),
470 3 : tenants,
471 3 : tenant,
472 3 : )
473 155 : .await
474 3 : }
475 :
476 91 : async fn prepare(
477 91 : tenants: &tokio::sync::RwLock<TenantsMap>,
478 91 : tenant_id: TenantId,
479 91 : ) -> Result<(Arc<Tenant>, tokio::sync::OwnedMutexGuard<Self>), DeleteTenantError> {
480 91 : let m = tenants.read().await;
481 :
482 91 : let tenant = m
483 91 : .get(&tenant_id)
484 91 : .ok_or(GetTenantError::NotFound(tenant_id))?;
485 :
486 : // FIXME: unsure about active only. Our init jobs may not be cancellable properly,
487 : // so at least for now allow deletions only for active tenants. TODO recheck
488 : // Broken and Stopping is needed for retries.
489 : if !matches!(
490 88 : tenant.current_state(),
491 : TenantState::Active | TenantState::Broken { .. }
492 : ) {
493 UBC 0 : return Err(DeleteTenantError::InvalidState(tenant.current_state()));
494 CBC 88 : }
495 :
496 88 : let guard = Arc::clone(&tenant.delete_progress)
497 88 : .try_lock_owned()
498 88 : .map_err(|_| DeleteTenantError::AlreadyInProgress)?;
499 :
500 88 : fail::fail_point!("tenant-delete-before-shutdown", |_| {
501 4 : Err(anyhow::anyhow!("failpoint: tenant-delete-before-shutdown"))?
502 88 : });
503 :
504 : // make pageserver shutdown not to wait for our completion
505 84 : let (_, progress) = completion::channel();
506 84 :
507 84 : // It would be good to only set stopping here and continue shutdown in the background, but shutdown is not idempotent.
508 84 : // i e it is an error to do:
509 84 : // tenant.set_stopping
510 84 : // tenant.shutdown
511 84 : // Its also bad that we're holding tenants.read here.
512 84 : // TODO relax set_stopping to be idempotent?
513 387 : if tenant.shutdown(progress, false).await.is_err() {
514 UBC 0 : return Err(DeleteTenantError::Other(anyhow::anyhow!(
515 0 : "tenant shutdown is already in progress"
516 0 : )));
517 CBC 84 : }
518 84 :
519 84 : Ok((Arc::clone(tenant), guard))
520 91 : }
521 :
522 72 : fn schedule_background(
523 72 : guard: OwnedMutexGuard<Self>,
524 72 : conf: &'static PageServerConf,
525 72 : remote_storage: Option<GenericRemoteStorage>,
526 72 : tenants: &'static tokio::sync::RwLock<TenantsMap>,
527 72 : tenant: Arc<Tenant>,
528 72 : ) {
529 72 : let tenant_id = tenant.tenant_id;
530 72 :
531 72 : task_mgr::spawn(
532 72 : task_mgr::BACKGROUND_RUNTIME.handle(),
533 72 : TaskKind::TimelineDeletionWorker,
534 72 : Some(tenant_id),
535 72 : None,
536 72 : "tenant_delete",
537 : false,
538 72 : async move {
539 39 : if let Err(err) =
540 4274 : Self::background(guard, conf, remote_storage, tenants, &tenant).await
541 : {
542 39 : error!("Error: {err:#}");
543 39 : tenant.set_broken(format!("{err:#}")).await;
544 33 : };
545 72 : Ok(())
546 72 : }
547 72 : .instrument({
548 72 : let span = tracing::info_span!(parent: None, "delete_tenant", tenant_id=%tenant_id);
549 72 : span.follows_from(Span::current());
550 72 : span
551 72 : }),
552 72 : );
553 72 : }
554 :
555 95 : async fn background(
556 95 : mut guard: OwnedMutexGuard<Self>,
557 95 : conf: &PageServerConf,
558 95 : remote_storage: Option<GenericRemoteStorage>,
559 95 : tenants: &'static tokio::sync::RwLock<TenantsMap>,
560 95 : tenant: &Arc<Tenant>,
561 95 : ) -> Result<(), DeleteTenantError> {
562 : // Tree sort timelines, schedule delete for them. Mention retries from the console side.
563 : // Note that if deletion fails we dont mark timelines as broken,
564 : // the whole tenant will become broken as by `Self::schedule_background` logic
565 95 : let already_running_timeline_deletions = schedule_ordered_timeline_deletions(tenant)
566 4590 : .await
567 95 : .context("schedule_ordered_timeline_deletions")?;
568 :
569 76 : fail::fail_point!("tenant-delete-before-polling-ongoing-deletions", |_| {
570 4 : Err(anyhow::anyhow!(
571 4 : "failpoint: tenant-delete-before-polling-ongoing-deletions"
572 4 : ))?
573 76 : });
574 :
575 : // Wait for deletions that were already running at the moment when tenant deletion was requested.
576 : // When we can lock deletion guard it means that corresponding timeline deletion finished.
577 76 : for (guard, timeline_id) in already_running_timeline_deletions {
578 4 : let flow = guard.lock().await;
579 4 : if !flow.is_finished() {
580 UBC 0 : return Err(DeleteTenantError::Other(anyhow::anyhow!(
581 0 : "already running timeline deletion failed: {timeline_id}"
582 0 : )));
583 CBC 4 : }
584 : }
585 :
586 72 : let timelines_path = conf.timelines_path(&tenant.tenant_id);
587 72 : // May not exist if we fail in cleanup_remaining_fs_traces after removing it
588 72 : if timelines_path.exists() {
589 : // sanity check to guard against layout changes
590 66 : ensure_timelines_dir_empty(&timelines_path)
591 131 : .await
592 66 : .context("timelines dir not empty")?;
593 6 : }
594 :
595 321 : remove_tenant_remote_delete_mark(conf, remote_storage.as_ref(), &tenant.tenant_id).await?;
596 :
597 72 : fail::fail_point!("tenant-delete-before-cleanup-remaining-fs-traces", |_| {
598 4 : Err(anyhow::anyhow!(
599 4 : "failpoint: tenant-delete-before-cleanup-remaining-fs-traces"
600 4 : ))?
601 72 : });
602 :
603 68 : cleanup_remaining_fs_traces(conf, &tenant.tenant_id)
604 433 : .await
605 68 : .context("cleanup_remaining_fs_traces")?;
606 :
607 56 : let mut locked = tenants.write().await;
608 56 : if locked.remove(&tenant.tenant_id).is_none() {
609 UBC 0 : warn!("Tenant got removed from tenants map during deletion");
610 CBC 56 : };
611 :
612 56 : *guard = Self::Finished;
613 56 :
614 56 : Ok(())
615 95 : }
616 : }
|
