Line data Source code
1 : use std::sync::Arc;
2 :
3 : use pageserver_api::models::{TenantState, TimelineState};
4 :
5 : use super::Timeline;
6 : use super::delete::{DeletionGuard, delete_local_timeline_directory};
7 : use crate::span::debug_assert_current_span_has_tenant_and_timeline_id;
8 : use crate::tenant::remote_timeline_client::ShutdownIfArchivedError;
9 : use crate::tenant::timeline::delete::{TimelineDeleteGuardKind, make_timeline_delete_guard};
10 : use crate::tenant::{
11 : DeleteTimelineError, OffloadedTimeline, TenantManifestError, TenantShard, TimelineOrOffloaded,
12 : };
13 :
14 : #[derive(thiserror::Error, Debug)]
15 : pub(crate) enum OffloadError {
16 : #[error("Cancelled")]
17 : Cancelled,
18 : #[error("Timeline is not archived")]
19 : NotArchived,
20 : #[error(transparent)]
21 : RemoteStorage(anyhow::Error),
22 : #[error("Unexpected offload error: {0}")]
23 : Other(anyhow::Error),
24 : }
25 :
26 : impl From<TenantManifestError> for OffloadError {
27 0 : fn from(e: TenantManifestError) -> Self {
28 0 : match e {
29 0 : TenantManifestError::Cancelled => Self::Cancelled,
30 0 : TenantManifestError::RemoteStorage(e) => Self::RemoteStorage(e),
31 : }
32 0 : }
33 : }
34 :
35 12 : pub(crate) async fn offload_timeline(
36 12 : tenant: &TenantShard,
37 12 : timeline: &Arc<Timeline>,
38 12 : ) -> Result<(), OffloadError> {
39 12 : debug_assert_current_span_has_tenant_and_timeline_id();
40 12 : tracing::info!("offloading archived timeline");
41 :
42 12 : let delete_guard_res = make_timeline_delete_guard(
43 12 : tenant,
44 12 : timeline.timeline_id,
45 12 : TimelineDeleteGuardKind::Offload,
46 12 : );
47 0 : if let Err(DeleteTimelineError::HasChildren(children)) = delete_guard_res {
48 0 : let is_archived = timeline.is_archived();
49 0 : if is_archived == Some(true) {
50 0 : tracing::error!("timeline is archived but has non-archived children: {children:?}");
51 0 : return Err(OffloadError::NotArchived);
52 0 : }
53 0 : tracing::info!(
54 : ?is_archived,
55 0 : "timeline is not archived and has unarchived children"
56 : );
57 0 : return Err(OffloadError::NotArchived);
58 12 : };
59 12 : let (timeline, guard) =
60 12 : delete_guard_res.map_err(|e| OffloadError::Other(anyhow::anyhow!(e)))?;
61 :
62 12 : let TimelineOrOffloaded::Timeline(timeline) = timeline else {
63 0 : tracing::error!("timeline already offloaded, but given timeline object");
64 0 : return Ok(());
65 : };
66 :
67 12 : match timeline.remote_client.shutdown_if_archived().await {
68 12 : Ok(()) => {}
69 0 : Err(ShutdownIfArchivedError::NotInitialized(_)) => {
70 0 : // Either the timeline is being deleted, the operation is being retried, or we are shutting down.
71 0 : // Don't return cancelled here to keep it idempotent.
72 0 : }
73 0 : Err(ShutdownIfArchivedError::NotArchived) => return Err(OffloadError::NotArchived),
74 : }
75 12 : timeline.set_state(TimelineState::Stopping);
76 12 :
77 12 : // Now that the Timeline is in Stopping state, request all the related tasks to shut down.
78 12 : timeline.shutdown(super::ShutdownMode::Reload).await;
79 :
80 : // TODO extend guard mechanism above with method
81 : // to make deletions possible while offloading is in progress
82 :
83 12 : let conf = &tenant.conf;
84 12 : delete_local_timeline_directory(conf, tenant.tenant_shard_id, &timeline).await;
85 :
86 12 : let remaining_refcount = remove_timeline_from_tenant(tenant, &timeline, &guard);
87 12 :
88 12 : {
89 12 : let mut offloaded_timelines = tenant.timelines_offloaded.lock().unwrap();
90 12 : if matches!(
91 12 : tenant.current_state(),
92 : TenantState::Stopping { .. } | TenantState::Broken { .. }
93 : ) {
94 : // Cancel the operation if the tenant is shutting down. Do this while the
95 : // timelines_offloaded lock is held to prevent a race with Tenant::shutdown
96 : // for defusing the lock
97 0 : return Err(OffloadError::Cancelled);
98 12 : }
99 12 : offloaded_timelines.insert(
100 12 : timeline.timeline_id,
101 12 : Arc::new(
102 12 : OffloadedTimeline::from_timeline(&timeline)
103 12 : .expect("we checked above that timeline was ready"),
104 12 : ),
105 12 : );
106 12 : }
107 12 :
108 12 : // Last step: mark timeline as offloaded in S3
109 12 : // TODO: maybe move this step above, right above deletion of the local timeline directory,
110 12 : // then there is no potential race condition where we partially offload a timeline, and
111 12 : // at the next restart attach it again.
112 12 : // For that to happen, we'd need to make the manifest reflect our *intended* state,
113 12 : // not our actual state of offloaded timelines.
114 12 : tenant.maybe_upload_tenant_manifest().await?;
115 :
116 12 : tracing::info!("Timeline offload complete (remaining arc refcount: {remaining_refcount})");
117 :
118 12 : Ok(())
119 12 : }
120 :
121 : /// It is important that this gets called when DeletionGuard is being held.
122 : /// For more context see comments in [`make_timeline_delete_guard`]
123 : ///
124 : /// Returns the strong count of the timeline `Arc`
125 12 : fn remove_timeline_from_tenant(
126 12 : tenant: &TenantShard,
127 12 : timeline: &Timeline,
128 12 : _: &DeletionGuard, // using it as a witness
129 12 : ) -> usize {
130 12 : // Remove the timeline from the map.
131 12 : let mut timelines = tenant.timelines.lock().unwrap();
132 12 : let children_exist = timelines
133 12 : .iter()
134 24 : .any(|(_, entry)| entry.get_ancestor_timeline_id() == Some(timeline.timeline_id));
135 12 : // XXX this can happen because `branch_timeline` doesn't check `TimelineState::Stopping`.
136 12 : // We already deleted the layer files, so it's probably best to panic.
137 12 : // (Ideally, above remove_dir_all is atomic so we don't see this timeline after a restart)
138 12 : if children_exist {
139 0 : panic!("Timeline grew children while we removed layer files");
140 12 : }
141 12 :
142 12 : let timeline = timelines
143 12 : .remove(&timeline.timeline_id)
144 12 : .expect("timeline that we were deleting was concurrently removed from 'timelines' map");
145 12 :
146 12 : // Clear the compaction queue for this timeline
147 12 : tenant
148 12 : .scheduled_compaction_tasks
149 12 : .lock()
150 12 : .unwrap()
151 12 : .remove(&timeline.timeline_id);
152 12 :
153 12 : Arc::strong_count(&timeline)
154 12 : }
|
