Line data Source code
1 : use std::{
2 : path::{Path, PathBuf},
3 : sync::Arc,
4 : };
5 :
6 : use anyhow::Context;
7 : use pageserver_api::models::TenantState;
8 : use remote_storage::{DownloadError, GenericRemoteStorage, RemotePath};
9 : use tokio::sync::OwnedMutexGuard;
10 : use tokio_util::sync::CancellationToken;
11 : use tracing::{error, info, instrument, warn, Instrument, Span};
12 :
13 : use utils::{
14 : backoff, completion, crashsafe, fs_ext,
15 : id::{TenantId, TimelineId},
16 : };
17 :
18 : use crate::{
19 : config::PageServerConf,
20 : context::RequestContext,
21 : task_mgr::{self, TaskKind},
22 : InitializationOrder,
23 : };
24 :
25 : use super::{
26 : mgr::{GetTenantError, TenantsMap},
27 : remote_timeline_client::{FAILED_REMOTE_OP_RETRIES, FAILED_UPLOAD_WARN_THRESHOLD},
28 : span,
29 : timeline::delete::DeleteTimelineFlow,
30 : tree_sort_timelines, DeleteTimelineError, Tenant,
31 : };
32 :
33 : const SHOULD_RESUME_DELETION_FETCH_MARK_ATTEMPTS: u32 = 3;
34 :
35 300 : #[derive(Debug, thiserror::Error)]
36 : pub enum DeleteTenantError {
37 : #[error("GetTenant {0}")]
38 : Get(#[from] GetTenantError),
39 :
40 : #[error("Invalid state {0}. Expected Active or Broken")]
41 : InvalidState(TenantState),
42 :
43 : #[error("Tenant deletion is already in progress")]
44 : AlreadyInProgress,
45 :
46 : #[error("Timeline {0}")]
47 : Timeline(#[from] DeleteTimelineError),
48 :
49 : #[error(transparent)]
50 : Other(#[from] anyhow::Error),
51 : }
52 :
53 : type DeletionGuard = tokio::sync::OwnedMutexGuard<DeleteTenantFlow>;
54 :
55 537 : fn remote_tenant_delete_mark_path(
56 537 : conf: &PageServerConf,
57 537 : tenant_id: &TenantId,
58 537 : ) -> anyhow::Result<RemotePath> {
59 537 : let tenant_remote_path = conf
60 537 : .tenant_path(tenant_id)
61 537 : .strip_prefix(&conf.workdir)
62 537 : .context("Failed to strip workdir prefix")
63 537 : .and_then(RemotePath::new)
64 537 : .context("tenant path")?;
65 537 : Ok(tenant_remote_path.join(Path::new("deleted")))
66 537 : }
67 :
68 80 : async fn create_remote_delete_mark(
69 80 : conf: &PageServerConf,
70 80 : remote_storage: &GenericRemoteStorage,
71 80 : tenant_id: &TenantId,
72 80 : ) -> Result<(), DeleteTenantError> {
73 80 : let remote_mark_path = remote_tenant_delete_mark_path(conf, tenant_id)?;
74 :
75 80 : let data: &[u8] = &[];
76 80 : backoff::retry(
77 120 : || async {
78 120 : remote_storage
79 120 : .upload(data, 0, &remote_mark_path, None)
80 190 : .await
81 120 : },
82 80 : |_e| false,
83 80 : FAILED_UPLOAD_WARN_THRESHOLD,
84 80 : FAILED_REMOTE_OP_RETRIES,
85 80 : "mark_upload",
86 80 : // TODO: use a cancellation token (https://github.com/neondatabase/neon/issues/5066)
87 80 : backoff::Cancel::new(CancellationToken::new(), || unreachable!()),
88 80 : )
89 190 : .await
90 80 : .context("mark_upload")?;
91 :
92 80 : Ok(())
93 80 : }
94 :
95 110 : async fn create_local_delete_mark(
96 110 : conf: &PageServerConf,
97 110 : tenant_id: &TenantId,
98 110 : ) -> Result<(), DeleteTenantError> {
99 110 : let marker_path = conf.tenant_deleted_mark_file_path(tenant_id);
100 110 :
101 110 : // Note: we're ok to replace existing file.
102 110 : let _ = std::fs::OpenOptions::new()
103 110 : .write(true)
104 110 : .create(true)
105 110 : .open(&marker_path)
106 110 : .with_context(|| format!("could not create delete marker file {marker_path:?}"))?;
107 :
108 110 : crashsafe::fsync_file_and_parent(&marker_path).context("sync_mark")?;
109 :
110 110 : Ok(())
111 110 : }
112 :
113 135 : async fn schedule_ordered_timeline_deletions(
114 135 : tenant: &Arc<Tenant>,
115 135 : ) -> Result<Vec<(Arc<tokio::sync::Mutex<DeleteTimelineFlow>>, TimelineId)>, DeleteTenantError> {
116 135 : // Tenant is stopping at this point. We know it will be deleted.
117 135 : // No new timelines should be created.
118 135 : // Tree sort timelines to delete from leafs to the root.
119 135 : // NOTE: by calling clone we release the mutex which creates a possibility for a race: pending deletion
120 135 : // can complete and remove timeline from the map in between our call to clone
121 135 : // and `DeleteTimelineFlow::run`, so `run` wont find timeline in `timelines` map.
122 135 : // timelines.lock is currently synchronous so we cant hold it across await point.
123 135 : // So just ignore NotFound error if we get it from `run`.
124 135 : // Beware: in case it becomes async and we try to hold it here, `run` also locks it, which can create a deadlock.
125 135 : let timelines = tenant.timelines.lock().unwrap().clone();
126 135 : let sorted =
127 223 : tree_sort_timelines(timelines, |t| t.get_ancestor_timeline_id()).context("tree sort")?;
128 :
129 135 : let mut already_running_deletions = vec![];
130 :
131 195 : for (timeline_id, _) in sorted.into_iter().rev() {
132 6342 : if let Err(e) = DeleteTimelineFlow::run(tenant, timeline_id, true).await {
133 33 : match e {
134 : DeleteTimelineError::NotFound => {
135 : // Timeline deletion finished after call to clone above but before call
136 : // to `DeleteTimelineFlow::run` and removed timeline from the map.
137 4 : continue;
138 : }
139 4 : DeleteTimelineError::AlreadyInProgress(guard) => {
140 4 : already_running_deletions.push((guard, timeline_id));
141 4 : continue;
142 : }
143 25 : e => return Err(DeleteTenantError::Timeline(e)),
144 : }
145 162 : }
146 : }
147 :
148 110 : Ok(already_running_deletions)
149 135 : }
150 :
151 95 : async fn ensure_timelines_dir_empty(timelines_path: &Path) -> Result<(), DeleteTenantError> {
152 : // Assert timelines dir is empty.
153 188 : if !fs_ext::is_directory_empty(timelines_path).await? {
154 : // Display first 10 items in directory
155 0 : let list = &fs_ext::list_dir(timelines_path).await.context("list_dir")?[..10];
156 0 : return Err(DeleteTenantError::Other(anyhow::anyhow!(
157 0 : "Timelines directory is not empty after all timelines deletion: {list:?}"
158 0 : )));
159 95 : }
160 95 :
161 95 : Ok(())
162 95 : }
163 :
164 104 : async fn remove_tenant_remote_delete_mark(
165 104 : conf: &PageServerConf,
166 104 : remote_storage: Option<&GenericRemoteStorage>,
167 104 : tenant_id: &TenantId,
168 104 : ) -> Result<(), DeleteTenantError> {
169 104 : if let Some(remote_storage) = remote_storage {
170 72 : let path = remote_tenant_delete_mark_path(conf, tenant_id)?;
171 72 : backoff::retry(
172 428 : || async { remote_storage.delete(&path).await },
173 72 : |_e| false,
174 72 : FAILED_UPLOAD_WARN_THRESHOLD,
175 72 : FAILED_REMOTE_OP_RETRIES,
176 72 : "remove_tenant_remote_delete_mark",
177 72 : // TODO: use a cancellation token (https://github.com/neondatabase/neon/issues/5066)
178 72 : backoff::Cancel::new(CancellationToken::new(), || unreachable!()),
179 72 : )
180 428 : .await
181 72 : .context("remove_tenant_remote_delete_mark")?;
182 32 : }
183 104 : Ok(())
184 104 : }
185 :
186 : // Cleanup fs traces: tenant config, timelines dir local delete mark, tenant dir
187 98 : async fn cleanup_remaining_fs_traces(
188 98 : conf: &PageServerConf,
189 98 : tenant_id: &TenantId,
190 98 : ) -> Result<(), DeleteTenantError> {
191 356 : let rm = |p: PathBuf, is_dir: bool| async move {
192 356 : if is_dir {
193 172 : tokio::fs::remove_dir(&p).await
194 98 : } else {
195 184 : tokio::fs::remove_file(&p).await
196 98 : }
197 356 : .or_else(fs_ext::ignore_not_found)
198 356 : .with_context(|| {
199 0 : let to_display = p.display();
200 0 : format!("failed to delete {to_display}")
201 356 : })
202 356 : };
203 98 :
204 98 : rm(conf.tenant_config_path(tenant_id), false).await?;
205 :
206 98 : fail::fail_point!("tenant-delete-before-remove-timelines-dir", |_| {
207 6 : Err(anyhow::anyhow!(
208 6 : "failpoint: tenant-delete-before-remove-timelines-dir"
209 6 : ))?
210 98 : });
211 :
212 92 : rm(conf.timelines_path(tenant_id), true).await?;
213 :
214 92 : fail::fail_point!("tenant-delete-before-remove-deleted-mark", |_| {
215 6 : Err(anyhow::anyhow!(
216 6 : "failpoint: tenant-delete-before-remove-deleted-mark"
217 6 : ))?
218 92 : });
219 :
220 : // Make sure previous deletions are ordered before mark removal.
221 : // Otherwise there is no guarantee that they reach the disk before mark deletion.
222 : // So its possible for mark to reach disk first and for other deletions
223 : // to be reordered later and thus missed if a crash occurs.
224 : // Note that we dont need to sync after mark file is removed
225 : // because we can tolerate the case when mark file reappears on startup.
226 86 : let tenant_path = &conf.tenant_path(tenant_id);
227 86 : if tenant_path.exists() {
228 86 : crashsafe::fsync_async(&conf.tenant_path(tenant_id))
229 171 : .await
230 86 : .context("fsync_pre_mark_remove")?;
231 0 : }
232 :
233 86 : rm(conf.tenant_deleted_mark_file_path(tenant_id), false).await?;
234 :
235 86 : fail::fail_point!("tenant-delete-before-remove-tenant-dir", |_| {
236 6 : Err(anyhow::anyhow!(
237 6 : "failpoint: tenant-delete-before-remove-tenant-dir"
238 6 : ))?
239 86 : });
240 :
241 80 : rm(conf.tenant_path(tenant_id), true).await?;
242 :
243 80 : Ok(())
244 98 : }
245 :
246 385 : pub(crate) async fn remote_delete_mark_exists(
247 385 : conf: &PageServerConf,
248 385 : tenant_id: &TenantId,
249 385 : remote_storage: &GenericRemoteStorage,
250 385 : ) -> anyhow::Result<bool> {
251 : // If remote storage is there we rely on it
252 385 : let remote_mark_path = remote_tenant_delete_mark_path(conf, tenant_id).context("path")?;
253 :
254 385 : let result = backoff::retry(
255 1619 : || async { remote_storage.download(&remote_mark_path).await },
256 435 : |e| matches!(e, DownloadError::NotFound),
257 385 : SHOULD_RESUME_DELETION_FETCH_MARK_ATTEMPTS,
258 385 : SHOULD_RESUME_DELETION_FETCH_MARK_ATTEMPTS,
259 385 : "fetch_tenant_deletion_mark",
260 385 : // TODO: use a cancellation token (https://github.com/neondatabase/neon/issues/5066)
261 385 : backoff::Cancel::new(CancellationToken::new(), || unreachable!()),
262 385 : )
263 1619 : .await;
264 :
265 380 : match result {
266 5 : Ok(_) => Ok(true),
267 380 : Err(DownloadError::NotFound) => Ok(false),
268 0 : Err(e) => Err(anyhow::anyhow!(e)).context("remote_delete_mark_exists")?,
269 : }
270 385 : }
271 :
272 : /// Orchestrates tenant shut down of all tasks, removes its in-memory structures,
273 : /// and deletes its data from both disk and s3.
274 : /// The sequence of steps:
275 : /// 1. Upload remote deletion mark.
276 : /// 2. Create local mark file.
277 : /// 3. Shutdown tasks
278 : /// 4. Run ordered timeline deletions
279 : /// 5. Wait for timeline deletion operations that were scheduled before tenant deletion was requested
280 : /// 6. Remove remote mark
281 : /// 7. Cleanup remaining fs traces, tenant dir, config, timelines dir, local delete mark
282 : /// It is resumable from any step in case a crash/restart occurs.
283 : /// There are three entrypoints to the process:
284 : /// 1. [`DeleteTenantFlow::run`] this is the main one called by a management api handler.
285 : /// 2. [`DeleteTenantFlow::resume_from_load`] is called during restarts when local or remote deletion marks are still there.
286 : /// 3. [`DeleteTenantFlow::resume_from_attach`] is called when deletion is resumed tenant is found to be deleted during attach process.
287 : /// Note the only other place that messes around timeline delete mark is the `Tenant::spawn_load` function.
288 779 : #[derive(Default)]
289 : pub enum DeleteTenantFlow {
290 : #[default]
291 : NotStarted,
292 : InProgress,
293 : Finished,
294 : }
295 :
296 : impl DeleteTenantFlow {
297 : // These steps are run in the context of management api request handler.
298 : // Long running steps are continued to run in the background.
299 : // NB: If this fails half-way through, and is retried, the retry will go through
300 : // all the same steps again. Make sure the code here is idempotent, and don't
301 : // error out if some of the shutdown tasks have already been completed!
302 : // NOTE: static needed for background part.
303 : // We assume that calling code sets up the span with tenant_id.
304 396 : #[instrument(skip_all)]
305 : pub(crate) async fn run(
306 : conf: &'static PageServerConf,
307 : remote_storage: Option<GenericRemoteStorage>,
308 : tenants: &'static tokio::sync::RwLock<TenantsMap>,
309 : tenant_id: TenantId,
310 : ) -> Result<(), DeleteTenantError> {
311 : span::debug_assert_current_span_has_tenant_id();
312 :
313 : let (tenant, mut guard) = Self::prepare(tenants, tenant_id).await?;
314 :
315 : if let Err(e) = Self::run_inner(&mut guard, conf, remote_storage.as_ref(), &tenant).await {
316 : tenant.set_broken(format!("{e:#}")).await;
317 : return Err(e);
318 : }
319 :
320 : Self::schedule_background(guard, conf, remote_storage, tenants, tenant);
321 :
322 : Ok(())
323 : }
324 :
325 : // Helper function needed to be able to match once on returned error and transition tenant into broken state.
326 : // This is needed because tenant.shutwodn is not idempotent. If tenant state is set to stopping another call to tenant.shutdown
327 : // will result in an error, but here we need to be able to retry shutdown when tenant deletion is retried.
328 : // So the solution is to set tenant state to broken.
329 122 : async fn run_inner(
330 122 : guard: &mut OwnedMutexGuard<Self>,
331 122 : conf: &'static PageServerConf,
332 122 : remote_storage: Option<&GenericRemoteStorage>,
333 122 : tenant: &Tenant,
334 122 : ) -> Result<(), DeleteTenantError> {
335 122 : guard.mark_in_progress()?;
336 :
337 122 : fail::fail_point!("tenant-delete-before-create-remote-mark", |_| {
338 6 : Err(anyhow::anyhow!(
339 6 : "failpoint: tenant-delete-before-create-remote-mark"
340 6 : ))?
341 122 : });
342 :
343 : // IDEA: implement detach as delete without remote storage. Then they would use the same lock (deletion_progress) so wont contend.
344 : // Though sounds scary, different mark name?
345 : // Detach currently uses remove_dir_all so in case of a crash we can end up in a weird state.
346 116 : if let Some(remote_storage) = &remote_storage {
347 80 : create_remote_delete_mark(conf, remote_storage, &tenant.tenant_id)
348 190 : .await
349 80 : .context("remote_mark")?
350 36 : }
351 :
352 116 : fail::fail_point!("tenant-delete-before-create-local-mark", |_| {
353 6 : Err(anyhow::anyhow!(
354 6 : "failpoint: tenant-delete-before-create-local-mark"
355 6 : ))?
356 116 : });
357 :
358 110 : create_local_delete_mark(conf, &tenant.tenant_id)
359 0 : .await
360 110 : .context("local delete mark")?;
361 :
362 110 : fail::fail_point!("tenant-delete-before-background", |_| {
363 6 : Err(anyhow::anyhow!(
364 6 : "failpoint: tenant-delete-before-background"
365 6 : ))?
366 110 : });
367 :
368 104 : Ok(())
369 122 : }
370 :
371 122 : fn mark_in_progress(&mut self) -> anyhow::Result<()> {
372 122 : match self {
373 0 : Self::Finished => anyhow::bail!("Bug. Is in finished state"),
374 35 : Self::InProgress { .. } => { /* We're in a retry */ }
375 87 : Self::NotStarted => { /* Fresh start */ }
376 : }
377 :
378 122 : *self = Self::InProgress;
379 122 :
380 122 : Ok(())
381 122 : }
382 :
383 739 : pub async fn should_resume_deletion(
384 739 : conf: &'static PageServerConf,
385 739 : remote_storage: Option<&GenericRemoteStorage>,
386 739 : tenant: &Tenant,
387 739 : ) -> Result<Option<DeletionGuard>, DeleteTenantError> {
388 739 : let acquire = |t: &Tenant| {
389 31 : Some(
390 31 : Arc::clone(&t.delete_progress)
391 31 : .try_lock_owned()
392 31 : .expect("we're the only owner during init"),
393 31 : )
394 739 : };
395 739 :
396 739 : let tenant_id = tenant.tenant_id;
397 739 : // Check local mark first, if its there there is no need to go to s3 to check whether remote one exists.
398 739 : if conf.tenant_deleted_mark_file_path(&tenant_id).exists() {
399 26 : return Ok(acquire(tenant));
400 713 : }
401 :
402 713 : let remote_storage = match remote_storage {
403 385 : Some(remote_storage) => remote_storage,
404 328 : None => return Ok(None),
405 : };
406 :
407 1619 : if remote_delete_mark_exists(conf, &tenant_id, remote_storage).await? {
408 5 : Ok(acquire(tenant))
409 : } else {
410 380 : Ok(None)
411 : }
412 739 : }
413 :
414 28 : pub(crate) async fn resume_from_load(
415 28 : guard: DeletionGuard,
416 28 : tenant: &Arc<Tenant>,
417 28 : init_order: Option<&InitializationOrder>,
418 28 : tenants: &'static tokio::sync::RwLock<TenantsMap>,
419 28 : ctx: &RequestContext,
420 28 : ) -> Result<(), DeleteTenantError> {
421 28 : let (_, progress) = completion::channel();
422 28 :
423 28 : tenant
424 28 : .set_stopping(progress, true, false)
425 0 : .await
426 28 : .expect("cant be stopping or broken");
427 28 :
428 28 : // Do not consume valuable resources during the load phase, continue deletion once init phase is complete.
429 28 : let background_jobs_can_start = init_order.as_ref().map(|x| &x.background_jobs_can_start);
430 28 : if let Some(background) = background_jobs_can_start {
431 28 : info!("waiting for backgound jobs barrier");
432 28 : background.clone().wait().await;
433 28 : info!("ready for backgound jobs barrier");
434 0 : }
435 :
436 : // Tenant may not be loadable if we fail late in cleanup_remaining_fs_traces (e g remove timelines dir)
437 28 : let timelines_path = tenant.conf.timelines_path(&tenant.tenant_id);
438 28 : if timelines_path.exists() {
439 159 : tenant.load(init_order, ctx).await.context("load")?;
440 3 : }
441 :
442 28 : Self::background(
443 28 : guard,
444 28 : tenant.conf,
445 28 : tenant.remote_storage.clone(),
446 28 : tenants,
447 28 : tenant,
448 28 : )
449 1281 : .await
450 28 : }
451 :
452 3 : pub(crate) async fn resume_from_attach(
453 3 : guard: DeletionGuard,
454 3 : tenant: &Arc<Tenant>,
455 3 : tenants: &'static tokio::sync::RwLock<TenantsMap>,
456 3 : ctx: &RequestContext,
457 3 : ) -> Result<(), DeleteTenantError> {
458 3 : let (_, progress) = completion::channel();
459 3 :
460 3 : tenant
461 3 : .set_stopping(progress, false, true)
462 0 : .await
463 3 : .expect("cant be stopping or broken");
464 3 :
465 34 : tenant.attach(ctx).await.context("attach")?;
466 :
467 3 : Self::background(
468 3 : guard,
469 3 : tenant.conf,
470 3 : tenant.remote_storage.clone(),
471 3 : tenants,
472 3 : tenant,
473 3 : )
474 165 : .await
475 3 : }
476 :
477 132 : async fn prepare(
478 132 : tenants: &tokio::sync::RwLock<TenantsMap>,
479 132 : tenant_id: TenantId,
480 132 : ) -> Result<(Arc<Tenant>, tokio::sync::OwnedMutexGuard<Self>), DeleteTenantError> {
481 132 : let m = tenants.read().await;
482 :
483 132 : let tenant = m
484 132 : .get(&tenant_id)
485 132 : .ok_or(GetTenantError::NotFound(tenant_id))?;
486 :
487 : // FIXME: unsure about active only. Our init jobs may not be cancellable properly,
488 : // so at least for now allow deletions only for active tenants. TODO recheck
489 : // Broken and Stopping is needed for retries.
490 : if !matches!(
491 128 : tenant.current_state(),
492 : TenantState::Active | TenantState::Broken { .. }
493 : ) {
494 0 : return Err(DeleteTenantError::InvalidState(tenant.current_state()));
495 128 : }
496 :
497 128 : let guard = Arc::clone(&tenant.delete_progress)
498 128 : .try_lock_owned()
499 128 : .map_err(|_| DeleteTenantError::AlreadyInProgress)?;
500 :
501 128 : fail::fail_point!("tenant-delete-before-shutdown", |_| {
502 6 : Err(anyhow::anyhow!("failpoint: tenant-delete-before-shutdown"))?
503 128 : });
504 :
505 : // make pageserver shutdown not to wait for our completion
506 122 : let (_, progress) = completion::channel();
507 122 :
508 122 : // It would be good to only set stopping here and continue shutdown in the background, but shutdown is not idempotent.
509 122 : // i e it is an error to do:
510 122 : // tenant.set_stopping
511 122 : // tenant.shutdown
512 122 : // Its also bad that we're holding tenants.read here.
513 122 : // TODO relax set_stopping to be idempotent?
514 485 : if tenant.shutdown(progress, false).await.is_err() {
515 0 : return Err(DeleteTenantError::Other(anyhow::anyhow!(
516 0 : "tenant shutdown is already in progress"
517 0 : )));
518 122 : }
519 122 :
520 122 : Ok((Arc::clone(tenant), guard))
521 132 : }
522 :
523 104 : fn schedule_background(
524 104 : guard: OwnedMutexGuard<Self>,
525 104 : conf: &'static PageServerConf,
526 104 : remote_storage: Option<GenericRemoteStorage>,
527 104 : tenants: &'static tokio::sync::RwLock<TenantsMap>,
528 104 : tenant: Arc<Tenant>,
529 104 : ) {
530 104 : let tenant_id = tenant.tenant_id;
531 104 :
532 104 : task_mgr::spawn(
533 104 : task_mgr::BACKGROUND_RUNTIME.handle(),
534 104 : TaskKind::TimelineDeletionWorker,
535 104 : Some(tenant_id),
536 104 : None,
537 104 : "tenant_delete",
538 : false,
539 104 : async move {
540 55 : if let Err(err) =
541 6037 : Self::background(guard, conf, remote_storage, tenants, &tenant).await
542 : {
543 55 : error!("Error: {err:#}");
544 55 : tenant.set_broken(format!("{err:#}")).await;
545 49 : };
546 104 : Ok(())
547 104 : }
548 104 : .instrument({
549 104 : let span = tracing::info_span!(parent: None, "delete_tenant", tenant_id=%tenant_id);
550 104 : span.follows_from(Span::current());
551 104 : span
552 104 : }),
553 104 : );
554 104 : }
555 :
556 135 : async fn background(
557 135 : mut guard: OwnedMutexGuard<Self>,
558 135 : conf: &PageServerConf,
559 135 : remote_storage: Option<GenericRemoteStorage>,
560 135 : tenants: &'static tokio::sync::RwLock<TenantsMap>,
561 135 : tenant: &Arc<Tenant>,
562 135 : ) -> Result<(), DeleteTenantError> {
563 : // Tree sort timelines, schedule delete for them. Mention retries from the console side.
564 : // Note that if deletion fails we dont mark timelines as broken,
565 : // the whole tenant will become broken as by `Self::schedule_background` logic
566 135 : let already_running_timeline_deletions = schedule_ordered_timeline_deletions(tenant)
567 6342 : .await
568 135 : .context("schedule_ordered_timeline_deletions")?;
569 :
570 110 : fail::fail_point!("tenant-delete-before-polling-ongoing-deletions", |_| {
571 6 : Err(anyhow::anyhow!(
572 6 : "failpoint: tenant-delete-before-polling-ongoing-deletions"
573 6 : ))?
574 110 : });
575 :
576 : // Wait for deletions that were already running at the moment when tenant deletion was requested.
577 : // When we can lock deletion guard it means that corresponding timeline deletion finished.
578 108 : for (guard, timeline_id) in already_running_timeline_deletions {
579 4 : let flow = guard.lock().await;
580 4 : if !flow.is_finished() {
581 0 : return Err(DeleteTenantError::Other(anyhow::anyhow!(
582 0 : "already running timeline deletion failed: {timeline_id}"
583 0 : )));
584 4 : }
585 : }
586 :
587 104 : let timelines_path = conf.timelines_path(&tenant.tenant_id);
588 104 : // May not exist if we fail in cleanup_remaining_fs_traces after removing it
589 104 : if timelines_path.exists() {
590 : // sanity check to guard against layout changes
591 95 : ensure_timelines_dir_empty(&timelines_path)
592 188 : .await
593 95 : .context("timelines dir not empty")?;
594 9 : }
595 :
596 428 : remove_tenant_remote_delete_mark(conf, remote_storage.as_ref(), &tenant.tenant_id).await?;
597 :
598 104 : fail::fail_point!("tenant-delete-before-cleanup-remaining-fs-traces", |_| {
599 6 : Err(anyhow::anyhow!(
600 6 : "failpoint: tenant-delete-before-cleanup-remaining-fs-traces"
601 6 : ))?
602 104 : });
603 :
604 98 : cleanup_remaining_fs_traces(conf, &tenant.tenant_id)
605 522 : .await
606 98 : .context("cleanup_remaining_fs_traces")?;
607 :
608 80 : let mut locked = tenants.write().await;
609 80 : if locked.remove(&tenant.tenant_id).is_none() {
610 0 : warn!("Tenant got removed from tenants map during deletion");
611 80 : };
612 :
613 80 : *guard = Self::Finished;
614 80 :
615 80 : Ok(())
616 135 : }
617 : }
|
