Line data    Source code

       1              : use std::sync::Arc;
       2              : 
       3              : use super::delete::{delete_local_timeline_directory, DeleteTimelineFlow, DeletionGuard};
       4              : use super::Timeline;
       5              : use crate::span::debug_assert_current_span_has_tenant_and_timeline_id;
       6              : use crate::tenant::{remote_timeline_client, OffloadedTimeline, Tenant, TimelineOrOffloaded};
       7              : 
       8            0 : pub(crate) async fn offload_timeline(
       9            0 :     tenant: &Tenant,
      10            0 :     timeline: &Arc<Timeline>,
      11            0 : ) -> anyhow::Result<()> {
      12            0 :     debug_assert_current_span_has_tenant_and_timeline_id();
      13            0 :     tracing::info!("offloading archived timeline");
      14              : 
      15            0 :     let (timeline, guard) = DeleteTimelineFlow::prepare(tenant, timeline.timeline_id)?;
      16              : 
      17            0 :     let TimelineOrOffloaded::Timeline(timeline) = timeline else {
      18            0 :         tracing::error!("timeline already offloaded, but given timeline object");
      19            0 :         return Ok(());
      20              :     };
      21              : 
      22            0 :     let is_archived = timeline.is_archived();
      23            0 :     match is_archived {
      24            0 :         Some(true) => (),
      25              :         Some(false) => {
      26            0 :             tracing::warn!(?is_archived, "tried offloading a non-archived timeline");
      27            0 :             anyhow::bail!("timeline isn't archived");
      28              :         }
      29              :         None => {
      30            0 :             tracing::warn!(
      31              :                 ?is_archived,
      32            0 :                 "tried offloading a timeline where manifest is not yet available"
      33              :             );
      34            0 :             anyhow::bail!("timeline manifest hasn't been loaded yet");
      35              :         }
      36              :     }
      37              : 
      38              :     // Now that the Timeline is in Stopping state, request all the related tasks to shut down.
      39            0 :     timeline.shutdown(super::ShutdownMode::Hard).await;
      40              : 
      41              :     // TODO extend guard mechanism above with method
      42              :     // to make deletions possible while offloading is in progress
      43              : 
      44            0 :     let conf = &tenant.conf;
      45            0 :     delete_local_timeline_directory(conf, tenant.tenant_shard_id, &timeline).await?;
      46              : 
      47            0 :     remove_timeline_from_tenant(tenant, &timeline, &guard).await?;
      48              : 
      49            0 :     {
      50            0 :         let mut offloaded_timelines = tenant.timelines_offloaded.lock().unwrap();
      51            0 :         offloaded_timelines.insert(
      52            0 :             timeline.timeline_id,
      53            0 :             Arc::new(
      54            0 :                 OffloadedTimeline::from_timeline(&timeline)
      55            0 :                     .expect("we checked above that timeline was ready"),
      56            0 :             ),
      57            0 :         );
      58            0 :     }
      59            0 : 
      60            0 :     // Last step: mark timeline as offloaded in S3
      61            0 :     // TODO: maybe move this step above, right above deletion of the local timeline directory,
      62            0 :     // then there is no potential race condition where we partially offload a timeline, and
      63            0 :     // at the next restart attach it again.
      64            0 :     // For that to happen, we'd need to make the manifest reflect our *intended* state,
      65            0 :     // not our actual state of offloaded timelines.
      66            0 :     let manifest = tenant.tenant_manifest();
      67            0 :     // TODO: generation support
      68            0 :     let generation = remote_timeline_client::TENANT_MANIFEST_GENERATION;
      69            0 :     remote_timeline_client::upload_tenant_manifest(
      70            0 :         &tenant.remote_storage,
      71            0 :         &tenant.tenant_shard_id,
      72            0 :         generation,
      73            0 :         &manifest,
      74            0 :         &tenant.cancel,
      75            0 :     )
      76            0 :     .await?;
      77              : 
      78            0 :     Ok(())
      79            0 : }
      80              : 
      81              : /// It is important that this gets called when DeletionGuard is being held.
      82              : /// For more context see comments in [`DeleteTimelineFlow::prepare`]
      83            0 : async fn remove_timeline_from_tenant(
      84            0 :     tenant: &Tenant,
      85            0 :     timeline: &Timeline,
      86            0 :     _: &DeletionGuard, // using it as a witness
      87            0 : ) -> anyhow::Result<()> {
      88            0 :     // Remove the timeline from the map.
      89            0 :     let mut timelines = tenant.timelines.lock().unwrap();
      90            0 :     let children_exist = timelines
      91            0 :         .iter()
      92            0 :         .any(|(_, entry)| entry.get_ancestor_timeline_id() == Some(timeline.timeline_id));
      93            0 :     // XXX this can happen because `branch_timeline` doesn't check `TimelineState::Stopping`.
      94            0 :     // We already deleted the layer files, so it's probably best to panic.
      95            0 :     // (Ideally, above remove_dir_all is atomic so we don't see this timeline after a restart)
      96            0 :     if children_exist {
      97            0 :         panic!("Timeline grew children while we removed layer files");
      98            0 :     }
      99            0 : 
     100            0 :     timelines
     101            0 :         .remove(&timeline.timeline_id)
     102            0 :         .expect("timeline that we were deleting was concurrently removed from 'timelines' map");
     103            0 : 
     104            0 :     drop(timelines);
     105            0 : 
     106            0 :     Ok(())
     107            0 : }