LCOV - code coverage report
Current view: top level - pageserver/src/tenant/timeline - delete.rs (source / functions) Coverage Total Hit
Test: 1b0a6a0c05cee5a7de360813c8034804e105ce1c.info Lines: 36.2 % 240 87
Test Date: 2025-03-12 00:01:28 Functions: 16.7 % 24 4

            Line data    Source code
       1              : use std::ops::{Deref, DerefMut};
       2              : use std::sync::Arc;
       3              : 
       4              : use anyhow::Context;
       5              : use pageserver_api::models::TimelineState;
       6              : use pageserver_api::shard::TenantShardId;
       7              : use remote_storage::DownloadError;
       8              : use tokio::sync::OwnedMutexGuard;
       9              : use tracing::{Instrument, error, info, info_span, instrument};
      10              : use utils::id::TimelineId;
      11              : use utils::{crashsafe, fs_ext, pausable_failpoint};
      12              : 
      13              : use crate::config::PageServerConf;
      14              : use crate::context::RequestContext;
      15              : use crate::task_mgr::{self, TaskKind};
      16              : use crate::tenant::metadata::TimelineMetadata;
      17              : use crate::tenant::remote_timeline_client::{
      18              :     PersistIndexPartWithDeletedFlagError, RemoteTimelineClient,
      19              : };
      20              : use crate::tenant::{
      21              :     CreateTimelineCause, DeleteTimelineError, MaybeDeletedIndexPart, Tenant, TenantManifestError,
      22              :     Timeline, TimelineOrOffloaded,
      23              : };
      24              : use crate::virtual_file::MaybeFatalIo;
      25              : 
      26              : /// Mark timeline as deleted in S3 so we won't pick it up next time
      27              : /// during attach or pageserver restart.
      28              : /// See comment in persist_index_part_with_deleted_flag.
      29            0 : async fn set_deleted_in_remote_index(
      30            0 :     remote_client: &Arc<RemoteTimelineClient>,
      31            0 : ) -> Result<(), DeleteTimelineError> {
      32            0 :     let res = remote_client.persist_index_part_with_deleted_flag().await;
      33            0 :     match res {
      34              :         // If we (now, or already) marked it successfully as deleted, we can proceed
      35            0 :         Ok(()) | Err(PersistIndexPartWithDeletedFlagError::AlreadyDeleted(_)) => (),
      36              :         // Bail out otherwise
      37              :         //
      38              :         // AlreadyInProgress shouldn't happen, because the 'delete_lock' prevents
      39              :         // two tasks from performing the deletion at the same time. The first task
      40              :         // that starts deletion should run it to completion.
      41            0 :         Err(e @ PersistIndexPartWithDeletedFlagError::AlreadyInProgress(_))
      42            0 :         | Err(e @ PersistIndexPartWithDeletedFlagError::Other(_)) => {
      43            0 :             return Err(DeleteTimelineError::Other(anyhow::anyhow!(e)));
      44              :         }
      45              :     }
      46            0 :     Ok(())
      47            0 : }
      48              : 
      49              : /// Grab the compaction and gc locks, and actually perform the deletion.
      50              : ///
      51              : /// The locks prevent GC or compaction from running at the same time. The background tasks do not
      52              : /// register themselves with the timeline it's operating on, so it might still be running even
      53              : /// though we called `shutdown_tasks`.
      54              : ///
      55              : /// Note that there are still other race conditions between
      56              : /// GC, compaction and timeline deletion. See
      57              : /// <https://github.com/neondatabase/neon/issues/2671>
      58              : ///
      59              : /// No timeout here, GC & Compaction should be responsive to the
      60              : /// `TimelineState::Stopping` change.
      61              : // pub(super): documentation link
      62            4 : pub(super) async fn delete_local_timeline_directory(
      63            4 :     conf: &PageServerConf,
      64            4 :     tenant_shard_id: TenantShardId,
      65            4 :     timeline: &Timeline,
      66            4 : ) {
      67            4 :     // Always ensure the lock order is compaction -> gc.
      68            4 :     let compaction_lock = timeline.compaction_lock.lock();
      69            4 :     let _compaction_lock = crate::timed(
      70            4 :         compaction_lock,
      71            4 :         "acquires compaction lock",
      72            4 :         std::time::Duration::from_secs(5),
      73            4 :     )
      74            4 :     .await;
      75              : 
      76            4 :     let gc_lock = timeline.gc_lock.lock();
      77            4 :     let _gc_lock = crate::timed(
      78            4 :         gc_lock,
      79            4 :         "acquires gc lock",
      80            4 :         std::time::Duration::from_secs(5),
      81            4 :     )
      82            4 :     .await;
      83              : 
      84              :     // NB: storage_sync upload tasks that reference these layers have been cancelled
      85              :     //     by the caller.
      86              : 
      87            4 :     let local_timeline_directory = conf.timeline_path(&tenant_shard_id, &timeline.timeline_id);
      88            4 : 
      89            4 :     // NB: This need not be atomic because the deleted flag in the IndexPart
      90            4 :     // will be observed during tenant/timeline load. The deletion will be resumed there.
      91            4 :     //
      92            4 :     // ErrorKind::NotFound can happen e.g. if we race with tenant detach, because,
      93            4 :     // no locks are shared.
      94            4 :     tokio::fs::remove_dir_all(local_timeline_directory)
      95            4 :         .await
      96            4 :         .or_else(fs_ext::ignore_not_found)
      97            4 :         .fatal_err("removing timeline directory");
      98            4 : 
      99            4 :     // Make sure previous deletions are ordered before mark removal.
     100            4 :     // Otherwise there is no guarantee that they reach the disk before mark deletion.
     101            4 :     // So its possible for mark to reach disk first and for other deletions
     102            4 :     // to be reordered later and thus missed if a crash occurs.
     103            4 :     // Note that we dont need to sync after mark file is removed
     104            4 :     // because we can tolerate the case when mark file reappears on startup.
     105            4 :     let timeline_path = conf.timelines_path(&tenant_shard_id);
     106            4 :     crashsafe::fsync_async(timeline_path)
     107            4 :         .await
     108            4 :         .fatal_err("fsync after removing timeline directory");
     109            4 : 
     110            4 :     info!("finished deleting layer files, releasing locks");
     111            4 : }
     112              : 
     113              : /// It is important that this gets called when DeletionGuard is being held.
     114              : /// For more context see comments in [`make_timeline_delete_guard`]
     115            0 : async fn remove_maybe_offloaded_timeline_from_tenant(
     116            0 :     tenant: &Tenant,
     117            0 :     timeline: &TimelineOrOffloaded,
     118            0 :     _: &DeletionGuard, // using it as a witness
     119            0 : ) -> anyhow::Result<()> {
     120            0 :     // Remove the timeline from the map.
     121            0 :     // This observes the locking order between timelines and timelines_offloaded
     122            0 :     let mut timelines = tenant.timelines.lock().unwrap();
     123            0 :     let mut timelines_offloaded = tenant.timelines_offloaded.lock().unwrap();
     124            0 :     let offloaded_children_exist = timelines_offloaded
     125            0 :         .iter()
     126            0 :         .any(|(_, entry)| entry.ancestor_timeline_id == Some(timeline.timeline_id()));
     127            0 :     let children_exist = timelines
     128            0 :         .iter()
     129            0 :         .any(|(_, entry)| entry.get_ancestor_timeline_id() == Some(timeline.timeline_id()));
     130            0 :     // XXX this can happen because of race conditions with branch creation.
     131            0 :     // We already deleted the remote layer files, so it's probably best to panic.
     132            0 :     if children_exist || offloaded_children_exist {
     133            0 :         panic!("Timeline grew children while we removed layer files");
     134            0 :     }
     135            0 : 
     136            0 :     match timeline {
     137            0 :         TimelineOrOffloaded::Timeline(timeline) => {
     138            0 :             timelines.remove(&timeline.timeline_id).expect(
     139            0 :                 "timeline that we were deleting was concurrently removed from 'timelines' map",
     140            0 :             );
     141            0 :             tenant
     142            0 :                 .scheduled_compaction_tasks
     143            0 :                 .lock()
     144            0 :                 .unwrap()
     145            0 :                 .remove(&timeline.timeline_id);
     146            0 :         }
     147            0 :         TimelineOrOffloaded::Offloaded(timeline) => {
     148            0 :             let offloaded_timeline = timelines_offloaded
     149            0 :                 .remove(&timeline.timeline_id)
     150            0 :                 .expect("timeline that we were deleting was concurrently removed from 'timelines_offloaded' map");
     151            0 :             offloaded_timeline.delete_from_ancestor_with_timelines(&timelines);
     152            0 :         }
     153              :     }
     154              : 
     155            0 :     drop(timelines_offloaded);
     156            0 :     drop(timelines);
     157            0 : 
     158            0 :     Ok(())
     159            0 : }
     160              : 
     161              : /// Orchestrates timeline shut down of all timeline tasks, removes its in-memory structures,
     162              : /// and deletes its data from both disk and s3.
     163              : /// The sequence of steps:
     164              : /// 1. Set deleted_at in remote index part.
     165              : /// 2. Create local mark file.
     166              : /// 3. Delete local files except metadata (it is simpler this way, to be able to reuse timeline initialization code that expects metadata)
     167              : /// 4. Delete remote layers
     168              : /// 5. Delete index part
     169              : /// 6. Delete meta, timeline directory
     170              : /// 7. Delete mark file
     171              : ///
     172              : /// It is resumable from any step in case a crash/restart occurs.
     173              : /// There are two entrypoints to the process:
     174              : /// 1. [`DeleteTimelineFlow::run`] this is the main one called by a management api handler.
     175              : /// 2. [`DeleteTimelineFlow::resume_deletion`] is called during restarts when local metadata is still present
     176              : ///    and we possibly neeed to continue deletion of remote files.
     177              : ///
     178              : /// Note the only other place that messes around timeline delete mark is the logic that scans directory with timelines during tenant load.
     179              : #[derive(Default)]
     180              : pub enum DeleteTimelineFlow {
     181              :     #[default]
     182              :     NotStarted,
     183              :     InProgress,
     184              :     Finished,
     185              : }
     186              : 
     187              : impl DeleteTimelineFlow {
     188              :     // These steps are run in the context of management api request handler.
     189              :     // Long running steps are continued to run in the background.
     190              :     // NB: If this fails half-way through, and is retried, the retry will go through
     191              :     // all the same steps again. Make sure the code here is idempotent, and don't
     192              :     // error out if some of the shutdown tasks have already been completed!
     193              :     #[instrument(skip_all)]
     194              :     pub async fn run(
     195              :         tenant: &Arc<Tenant>,
     196              :         timeline_id: TimelineId,
     197              :     ) -> Result<(), DeleteTimelineError> {
     198              :         super::debug_assert_current_span_has_tenant_and_timeline_id();
     199              : 
     200              :         let (timeline, mut guard) =
     201              :             make_timeline_delete_guard(tenant, timeline_id, TimelineDeleteGuardKind::Delete)?;
     202              : 
     203              :         guard.mark_in_progress()?;
     204              : 
     205              :         // Now that the Timeline is in Stopping state, request all the related tasks to shut down.
     206              :         if let TimelineOrOffloaded::Timeline(timeline) = &timeline {
     207              :             timeline.shutdown(super::ShutdownMode::Hard).await;
     208              :         }
     209              : 
     210              :         tenant.gc_block.before_delete(&timeline.timeline_id());
     211              : 
     212            0 :         fail::fail_point!("timeline-delete-before-index-deleted-at", |_| {
     213            0 :             Err(anyhow::anyhow!(
     214            0 :                 "failpoint: timeline-delete-before-index-deleted-at"
     215            0 :             ))?
     216            0 :         });
     217              : 
     218              :         let remote_client = match timeline.maybe_remote_client() {
     219              :             Some(remote_client) => remote_client,
     220              :             None => {
     221              :                 let remote_client = tenant
     222              :                     .build_timeline_client(timeline.timeline_id(), tenant.remote_storage.clone());
     223              :                 let result = match remote_client
     224              :                     .download_index_file(&tenant.cancel)
     225              :                     .instrument(info_span!("download_index_file"))
     226              :                     .await
     227              :                 {
     228              :                     Ok(r) => r,
     229              :                     Err(DownloadError::NotFound) => {
     230              :                         // Deletion is already complete
     231              :                         tracing::info!("Timeline already deleted in remote storage");
     232              :                         return Ok(());
     233              :                     }
     234              :                     Err(e) => {
     235              :                         return Err(DeleteTimelineError::Other(anyhow::anyhow!(
     236              :                             "error: {:?}",
     237              :                             e
     238              :                         )));
     239              :                     }
     240              :                 };
     241              :                 let index_part = match result {
     242              :                     MaybeDeletedIndexPart::Deleted(p) => {
     243              :                         tracing::info!("Timeline already set as deleted in remote index");
     244              :                         p
     245              :                     }
     246              :                     MaybeDeletedIndexPart::IndexPart(p) => p,
     247              :                 };
     248              :                 let remote_client = Arc::new(remote_client);
     249              : 
     250              :                 remote_client
     251              :                     .init_upload_queue(&index_part)
     252              :                     .map_err(DeleteTimelineError::Other)?;
     253              :                 remote_client.shutdown().await;
     254              :                 remote_client
     255              :             }
     256              :         };
     257              :         set_deleted_in_remote_index(&remote_client).await?;
     258              : 
     259            0 :         fail::fail_point!("timeline-delete-before-schedule", |_| {
     260            0 :             Err(anyhow::anyhow!(
     261            0 :                 "failpoint: timeline-delete-before-schedule"
     262            0 :             ))?
     263            0 :         });
     264              : 
     265              :         Self::schedule_background(
     266              :             guard,
     267              :             tenant.conf,
     268              :             Arc::clone(tenant),
     269              :             timeline,
     270              :             remote_client,
     271              :         );
     272              : 
     273              :         Ok(())
     274              :     }
     275              : 
     276            0 :     fn mark_in_progress(&mut self) -> anyhow::Result<()> {
     277            0 :         match self {
     278            0 :             Self::Finished => anyhow::bail!("Bug. Is in finished state"),
     279            0 :             Self::InProgress { .. } => { /* We're in a retry */ }
     280            0 :             Self::NotStarted => { /* Fresh start */ }
     281              :         }
     282              : 
     283            0 :         *self = Self::InProgress;
     284            0 : 
     285            0 :         Ok(())
     286            0 :     }
     287              : 
     288              :     /// Shortcut to create Timeline in stopping state and spawn deletion task.
     289              :     #[instrument(skip_all, fields(%timeline_id))]
     290              :     pub(crate) async fn resume_deletion(
     291              :         tenant: Arc<Tenant>,
     292              :         timeline_id: TimelineId,
     293              :         local_metadata: &TimelineMetadata,
     294              :         remote_client: RemoteTimelineClient,
     295              :         ctx: &RequestContext,
     296              :     ) -> anyhow::Result<()> {
     297              :         // Note: here we even skip populating layer map. Timeline is essentially uninitialized.
     298              :         // RemoteTimelineClient is the only functioning part.
     299              :         let (timeline, _timeline_ctx) = tenant
     300              :             .create_timeline_struct(
     301              :                 timeline_id,
     302              :                 local_metadata,
     303              :                 None, // Ancestor is not needed for deletion.
     304              :                 None, // Previous heatmap is not needed for deletion
     305              :                 tenant.get_timeline_resources_for(remote_client),
     306              :                 // Important. We dont pass ancestor above because it can be missing.
     307              :                 // Thus we need to skip the validation here.
     308              :                 CreateTimelineCause::Delete,
     309              :                 crate::tenant::CreateTimelineIdempotency::FailWithConflict, // doesn't matter what we put here
     310              :                 None, // doesn't matter what we put here
     311              :                 None, // doesn't matter what we put here
     312              :                 ctx,
     313              :             )
     314              :             .context("create_timeline_struct")?;
     315              : 
     316              :         let mut guard = DeletionGuard(
     317              :             Arc::clone(&timeline.delete_progress)
     318              :                 .try_lock_owned()
     319              :                 .expect("cannot happen because we're the only owner"),
     320              :         );
     321              : 
     322              :         // We meed to do this because when console retries delete request we shouldnt answer with 404
     323              :         // because 404 means successful deletion.
     324              :         {
     325              :             let mut locked = tenant.timelines.lock().unwrap();
     326              :             locked.insert(timeline_id, Arc::clone(&timeline));
     327              :         }
     328              : 
     329              :         guard.mark_in_progress()?;
     330              : 
     331              :         let remote_client = timeline.remote_client.clone();
     332              :         let timeline = TimelineOrOffloaded::Timeline(timeline);
     333              :         Self::schedule_background(guard, tenant.conf, tenant, timeline, remote_client);
     334              : 
     335              :         Ok(())
     336              :     }
     337              : 
     338            0 :     fn schedule_background(
     339            0 :         guard: DeletionGuard,
     340            0 :         conf: &'static PageServerConf,
     341            0 :         tenant: Arc<Tenant>,
     342            0 :         timeline: TimelineOrOffloaded,
     343            0 :         remote_client: Arc<RemoteTimelineClient>,
     344            0 :     ) {
     345            0 :         let tenant_shard_id = timeline.tenant_shard_id();
     346            0 :         let timeline_id = timeline.timeline_id();
     347              : 
     348              :         // Take a tenant gate guard, because timeline deletion needs access to the tenant to update its manifest.
     349            0 :         let Ok(tenant_guard) = tenant.gate.enter() else {
     350              :             // It is safe to simply skip here, because we only schedule background work once the timeline is durably marked for deletion.
     351            0 :             info!("Tenant is shutting down, timeline deletion will be resumed when it next starts");
     352            0 :             return;
     353              :         };
     354              : 
     355              :         task_mgr::spawn(
     356            0 :             task_mgr::BACKGROUND_RUNTIME.handle(),
     357            0 :             TaskKind::TimelineDeletionWorker,
     358            0 :             tenant_shard_id,
     359            0 :             Some(timeline_id),
     360            0 :             "timeline_delete",
     361            0 :             async move {
     362            0 :                 let _guard = tenant_guard;
     363              : 
     364            0 :                 if let Err(err) = Self::background(guard, conf, &tenant, &timeline, remote_client).await {
     365              :                     // Only log as an error if it's not a cancellation.
     366            0 :                     if matches!(err, DeleteTimelineError::Cancelled) {
     367            0 :                         info!("Shutdown during timeline deletion");
     368              :                     }else {
     369            0 :                         error!("Error: {err:#}");
     370              :                     }
     371            0 :                     if let TimelineOrOffloaded::Timeline(timeline) = timeline {
     372            0 :                         timeline.set_broken(format!("{err:#}"))
     373            0 :                     }
     374            0 :                 };
     375            0 :                 Ok(())
     376            0 :             }
     377            0 :             .instrument(tracing::info_span!(parent: None, "delete_timeline", tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug(),timeline_id=%timeline_id)),
     378              :         );
     379            0 :     }
     380              : 
     381            0 :     async fn background(
     382            0 :         mut guard: DeletionGuard,
     383            0 :         conf: &PageServerConf,
     384            0 :         tenant: &Tenant,
     385            0 :         timeline: &TimelineOrOffloaded,
     386            0 :         remote_client: Arc<RemoteTimelineClient>,
     387            0 :     ) -> Result<(), DeleteTimelineError> {
     388            0 :         fail::fail_point!("timeline-delete-before-rm", |_| {
     389            0 :             Err(anyhow::anyhow!("failpoint: timeline-delete-before-rm"))?
     390            0 :         });
     391              : 
     392              :         // Offloaded timelines have no local state
     393              :         // TODO: once we persist offloaded information, delete the timeline from there, too
     394            0 :         if let TimelineOrOffloaded::Timeline(timeline) = timeline {
     395            0 :             delete_local_timeline_directory(conf, tenant.tenant_shard_id, timeline).await;
     396            0 :         }
     397              : 
     398            0 :         fail::fail_point!("timeline-delete-after-rm", |_| {
     399            0 :             Err(anyhow::anyhow!("failpoint: timeline-delete-after-rm"))?
     400            0 :         });
     401              : 
     402            0 :         remote_client.delete_all().await?;
     403              : 
     404            0 :         pausable_failpoint!("in_progress_delete");
     405              : 
     406            0 :         remove_maybe_offloaded_timeline_from_tenant(tenant, timeline, &guard).await?;
     407              : 
     408              :         // This is susceptible to race conditions, i.e. we won't continue deletions if there is a crash
     409              :         // between the deletion of the index-part.json and reaching of this code.
     410              :         // So indeed, the tenant manifest might refer to an offloaded timeline which has already been deleted.
     411              :         // However, we handle this case in tenant loading code so the next time we attach, the issue is
     412              :         // resolved.
     413            0 :         tenant.store_tenant_manifest().await.map_err(|e| match e {
     414            0 :             TenantManifestError::Cancelled => DeleteTimelineError::Cancelled,
     415            0 :             _ => DeleteTimelineError::Other(e.into()),
     416            0 :         })?;
     417              : 
     418            0 :         *guard = Self::Finished;
     419            0 : 
     420            0 :         Ok(())
     421            0 :     }
     422              : 
     423            0 :     pub(crate) fn is_not_started(&self) -> bool {
     424            0 :         matches!(self, Self::NotStarted)
     425            0 :     }
     426              : }
     427              : 
     428              : #[derive(Copy, Clone, PartialEq, Eq)]
     429              : pub(super) enum TimelineDeleteGuardKind {
     430              :     Offload,
     431              :     Delete,
     432              : }
     433              : 
     434            4 : pub(super) fn make_timeline_delete_guard(
     435            4 :     tenant: &Tenant,
     436            4 :     timeline_id: TimelineId,
     437            4 :     guard_kind: TimelineDeleteGuardKind,
     438            4 : ) -> Result<(TimelineOrOffloaded, DeletionGuard), DeleteTimelineError> {
     439            4 :     // Note the interaction between this guard and deletion guard.
     440            4 :     // Here we attempt to lock deletion guard when we're holding a lock on timelines.
     441            4 :     // This is important because when you take into account `remove_timeline_from_tenant`
     442            4 :     // we remove timeline from memory when we still hold the deletion guard.
     443            4 :     // So here when timeline deletion is finished timeline wont be present in timelines map at all
     444            4 :     // which makes the following sequence impossible:
     445            4 :     // T1: get preempted right before the try_lock on `Timeline::delete_progress`
     446            4 :     // T2: do a full deletion, acquire and drop `Timeline::delete_progress`
     447            4 :     // T1: acquire deletion lock, do another `DeleteTimelineFlow::run`
     448            4 :     // For more context see this discussion: `https://github.com/neondatabase/neon/pull/4552#discussion_r1253437346`
     449            4 :     let timelines = tenant.timelines.lock().unwrap();
     450            4 :     let timelines_offloaded = tenant.timelines_offloaded.lock().unwrap();
     451              : 
     452            4 :     let timeline = match timelines.get(&timeline_id) {
     453            4 :         Some(t) => TimelineOrOffloaded::Timeline(Arc::clone(t)),
     454            0 :         None => match timelines_offloaded.get(&timeline_id) {
     455            0 :             Some(t) => TimelineOrOffloaded::Offloaded(Arc::clone(t)),
     456            0 :             None => return Err(DeleteTimelineError::NotFound),
     457              :         },
     458              :     };
     459              : 
     460              :     // Ensure that there are no child timelines, because we are about to remove files,
     461              :     // which will break child branches
     462            4 :     let mut children = Vec::new();
     463            4 :     if guard_kind == TimelineDeleteGuardKind::Delete {
     464            0 :         children.extend(timelines_offloaded.iter().filter_map(|(id, entry)| {
     465            0 :             (entry.ancestor_timeline_id == Some(timeline_id)).then_some(*id)
     466            0 :         }));
     467            4 :     }
     468            8 :     children.extend(timelines.iter().filter_map(|(id, entry)| {
     469            8 :         (entry.get_ancestor_timeline_id() == Some(timeline_id)).then_some(*id)
     470            8 :     }));
     471            4 : 
     472            4 :     if !children.is_empty() {
     473            0 :         return Err(DeleteTimelineError::HasChildren(children));
     474            4 :     }
     475            4 : 
     476            4 :     // Note that using try_lock here is important to avoid a deadlock.
     477            4 :     // Here we take lock on timelines and then the deletion guard.
     478            4 :     // At the end of the operation we're holding the guard and need to lock timelines map
     479            4 :     // to remove the timeline from it.
     480            4 :     // Always if you have two locks that are taken in different order this can result in a deadlock.
     481            4 : 
     482            4 :     let delete_progress = Arc::clone(timeline.delete_progress());
     483            4 :     let delete_lock_guard = match delete_progress.try_lock_owned() {
     484            4 :         Ok(guard) => DeletionGuard(guard),
     485              :         Err(_) => {
     486              :             // Unfortunately if lock fails arc is consumed.
     487            0 :             return Err(DeleteTimelineError::AlreadyInProgress(Arc::clone(
     488            0 :                 timeline.delete_progress(),
     489            0 :             )));
     490              :         }
     491              :     };
     492              : 
     493            4 :     if guard_kind == TimelineDeleteGuardKind::Delete {
     494            0 :         if let TimelineOrOffloaded::Timeline(timeline) = &timeline {
     495            0 :             timeline.set_state(TimelineState::Stopping);
     496            0 :         }
     497            4 :     }
     498              : 
     499            4 :     Ok((timeline, delete_lock_guard))
     500            4 : }
     501              : 
     502              : pub(super) struct DeletionGuard(OwnedMutexGuard<DeleteTimelineFlow>);
     503              : 
     504              : impl Deref for DeletionGuard {
     505              :     type Target = DeleteTimelineFlow;
     506              : 
     507            0 :     fn deref(&self) -> &Self::Target {
     508            0 :         &self.0
     509            0 :     }
     510              : }
     511              : 
     512              : impl DerefMut for DeletionGuard {
     513            0 :     fn deref_mut(&mut self) -> &mut Self::Target {
     514            0 :         &mut self.0
     515            0 :     }
     516              : }
        

Generated by: LCOV version 2.1-beta