Line data Source code
1 : use anyhow::Context;
2 : use camino::{Utf8Path, Utf8PathBuf};
3 : use pageserver_api::models::{
4 : HistoricLayerInfo, LayerAccessKind, LayerResidenceEventReason, LayerResidenceStatus,
5 : };
6 : use pageserver_api::shard::ShardIndex;
7 : use std::ops::Range;
8 : use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
9 : use std::sync::{Arc, Weak};
10 : use std::time::SystemTime;
11 : use tracing::Instrument;
12 : use utils::lsn::Lsn;
13 : use utils::sync::heavier_once_cell;
14 :
15 : use crate::config::PageServerConf;
16 : use crate::context::RequestContext;
17 : use crate::repository::Key;
18 : use crate::span::debug_assert_current_span_has_tenant_and_timeline_id;
19 : use crate::tenant::{remote_timeline_client::LayerFileMetadata, Timeline};
20 :
21 : use super::delta_layer::{self, DeltaEntry};
22 : use super::image_layer;
23 : use super::{
24 : AsLayerDesc, LayerAccessStats, LayerAccessStatsReset, LayerFileName, PersistentLayerDesc,
25 : ValueReconstructResult, ValueReconstructState,
26 : };
27 :
28 : use utils::generation::Generation;
29 :
30 : /// A Layer contains all data in a "rectangle" consisting of a range of keys and
31 : /// range of LSNs.
32 : ///
33 : /// There are two kinds of layers, in-memory and on-disk layers. In-memory
34 : /// layers are used to ingest incoming WAL, and provide fast access to the
35 : /// recent page versions. On-disk layers are stored as files on disk, and are
36 : /// immutable. This type represents the on-disk kind while in-memory kind are represented by
37 : /// [`InMemoryLayer`].
38 : ///
39 : /// Furthermore, there are two kinds of on-disk layers: delta and image layers.
40 : /// A delta layer contains all modifications within a range of LSNs and keys.
41 : /// An image layer is a snapshot of all the data in a key-range, at a single
42 : /// LSN.
43 : ///
44 : /// This type models the on-disk layers, which can be evicted and on-demand downloaded.
45 : ///
46 : /// [`InMemoryLayer`]: super::inmemory_layer::InMemoryLayer
47 33789294 : #[derive(Clone)]
48 : pub(crate) struct Layer(Arc<LayerInner>);
49 :
50 : impl std::fmt::Display for Layer {
51 34503 : fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
52 34503 : if matches!(self.0.generation, Generation::Broken) {
53 0 : write!(f, "{}-broken", self.layer_desc().short_id())
54 : } else {
55 34503 : write!(
56 34503 : f,
57 34503 : "{}{}",
58 34503 : self.layer_desc().short_id(),
59 34503 : self.0.generation.get_suffix()
60 34503 : )
61 : }
62 34503 : }
63 : }
64 :
65 : impl std::fmt::Debug for Layer {
66 0 : fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
67 0 : write!(f, "{}", self)
68 0 : }
69 : }
70 :
71 : impl AsLayerDesc for Layer {
72 51190438 : fn layer_desc(&self) -> &PersistentLayerDesc {
73 51190438 : self.0.layer_desc()
74 51190438 : }
75 : }
76 :
77 : impl Layer {
78 : /// Creates a layer value for a file we know to not be resident.
79 40759 : pub(crate) fn for_evicted(
80 40759 : conf: &'static PageServerConf,
81 40759 : timeline: &Arc<Timeline>,
82 40759 : file_name: LayerFileName,
83 40759 : metadata: LayerFileMetadata,
84 40759 : ) -> Self {
85 40759 : let desc = PersistentLayerDesc::from_filename(
86 40759 : timeline.tenant_shard_id,
87 40759 : timeline.timeline_id,
88 40759 : file_name,
89 40759 : metadata.file_size(),
90 40759 : );
91 40759 :
92 40759 : let access_stats = LayerAccessStats::for_loading_layer(LayerResidenceStatus::Evicted);
93 40759 :
94 40759 : let owner = Layer(Arc::new(LayerInner::new(
95 40759 : conf,
96 40759 : timeline,
97 40759 : access_stats,
98 40759 : desc,
99 40759 : None,
100 40759 : metadata.generation,
101 40759 : metadata.shard,
102 40759 : )));
103 :
104 40759 : debug_assert!(owner.0.needs_download_blocking().unwrap().is_some());
105 :
106 40759 : owner
107 40759 : }
108 :
109 : /// Creates a Layer value for a file we know to be resident in timeline directory.
110 12436 : pub(crate) fn for_resident(
111 12436 : conf: &'static PageServerConf,
112 12436 : timeline: &Arc<Timeline>,
113 12436 : file_name: LayerFileName,
114 12436 : metadata: LayerFileMetadata,
115 12436 : ) -> ResidentLayer {
116 12436 : let desc = PersistentLayerDesc::from_filename(
117 12436 : timeline.tenant_shard_id,
118 12436 : timeline.timeline_id,
119 12436 : file_name,
120 12436 : metadata.file_size(),
121 12436 : );
122 12436 :
123 12436 : let access_stats = LayerAccessStats::for_loading_layer(LayerResidenceStatus::Resident);
124 12436 :
125 12436 : let mut resident = None;
126 12436 :
127 12436 : let owner = Layer(Arc::new_cyclic(|owner| {
128 12436 : let inner = Arc::new(DownloadedLayer {
129 12436 : owner: owner.clone(),
130 12436 : kind: tokio::sync::OnceCell::default(),
131 12436 : version: 0,
132 12436 : });
133 12436 : resident = Some(inner.clone());
134 12436 :
135 12436 : LayerInner::new(
136 12436 : conf,
137 12436 : timeline,
138 12436 : access_stats,
139 12436 : desc,
140 12436 : Some(inner),
141 12436 : metadata.generation,
142 12436 : metadata.shard,
143 12436 : )
144 12436 : }));
145 12436 :
146 12436 : let downloaded = resident.expect("just initialized");
147 :
148 12436 : debug_assert!(owner.0.needs_download_blocking().unwrap().is_none());
149 :
150 12436 : timeline
151 12436 : .metrics
152 12436 : .resident_physical_size_add(metadata.file_size());
153 12436 :
154 12436 : ResidentLayer { downloaded, owner }
155 12436 : }
156 :
157 : /// Creates a Layer value for freshly written out new layer file by renaming it from a
158 : /// temporary path.
159 22365 : pub(crate) fn finish_creating(
160 22365 : conf: &'static PageServerConf,
161 22365 : timeline: &Arc<Timeline>,
162 22365 : desc: PersistentLayerDesc,
163 22365 : temp_path: &Utf8Path,
164 22365 : ) -> anyhow::Result<ResidentLayer> {
165 22365 : let mut resident = None;
166 22365 :
167 22365 : let owner = Layer(Arc::new_cyclic(|owner| {
168 22365 : let inner = Arc::new(DownloadedLayer {
169 22365 : owner: owner.clone(),
170 22365 : kind: tokio::sync::OnceCell::default(),
171 22365 : version: 0,
172 22365 : });
173 22365 : resident = Some(inner.clone());
174 22365 : let access_stats = LayerAccessStats::empty_will_record_residence_event_later();
175 22365 : access_stats.record_residence_event(
176 22365 : LayerResidenceStatus::Resident,
177 22365 : LayerResidenceEventReason::LayerCreate,
178 22365 : );
179 22365 : LayerInner::new(
180 22365 : conf,
181 22365 : timeline,
182 22365 : access_stats,
183 22365 : desc,
184 22365 : Some(inner),
185 22365 : timeline.generation,
186 22365 : timeline.get_shard_index(),
187 22365 : )
188 22365 : }));
189 22365 :
190 22365 : let downloaded = resident.expect("just initialized");
191 22365 :
192 22365 : // if the rename works, the path is as expected
193 22365 : std::fs::rename(temp_path, owner.local_path())
194 22365 : .with_context(|| format!("rename temporary file as correct path for {owner}"))?;
195 :
196 22365 : Ok(ResidentLayer { downloaded, owner })
197 22365 : }
198 :
199 : /// Requests the layer to be evicted and waits for this to be done.
200 : ///
201 : /// If the file is not resident, an [`EvictionError::NotFound`] is returned.
202 : ///
203 : /// If for a bad luck or blocking of the executor, we miss the actual eviction and the layer is
204 : /// re-downloaded, [`EvictionError::Downloaded`] is returned.
205 : ///
206 : /// Technically cancellation safe, but cancelling might shift the viewpoint of what generation
207 : /// of download-evict cycle on retry.
208 2554 : pub(crate) async fn evict_and_wait(&self) -> Result<(), EvictionError> {
209 2554 : self.0.evict_and_wait().await
210 2554 : }
211 :
212 : /// Delete the layer file when the `self` gets dropped, also try to schedule a remote index upload
213 : /// then.
214 : ///
215 : /// On drop, this will cause a call to [`crate::tenant::remote_timeline_client::RemoteTimelineClient::schedule_deletion_of_unlinked`].
216 : /// This means that the unlinking by [gc] or [compaction] must have happened strictly before
217 : /// the value this is called on gets dropped.
218 : ///
219 : /// This is ensured by both of those methods accepting references to Layer.
220 : ///
221 : /// [gc]: [`RemoteTimelineClient::schedule_gc_update`]
222 : /// [compaction]: [`RemoteTimelineClient::schedule_compaction_update`]
223 5219 : pub(crate) fn delete_on_drop(&self) {
224 5219 : self.0.delete_on_drop();
225 5219 : }
226 :
227 : /// Return data needed to reconstruct given page at LSN.
228 : ///
229 : /// It is up to the caller to collect more data from the previous layer and
230 : /// perform WAL redo, if necessary.
231 : ///
232 : /// # Cancellation-Safety
233 : ///
234 : /// This method is cancellation-safe.
235 16818696 : pub(crate) async fn get_value_reconstruct_data(
236 16818696 : &self,
237 16818696 : key: Key,
238 16818696 : lsn_range: Range<Lsn>,
239 16818696 : reconstruct_data: &mut ValueReconstructState,
240 16818696 : ctx: &RequestContext,
241 16818696 : ) -> anyhow::Result<ValueReconstructResult> {
242 : use anyhow::ensure;
243 :
244 16818686 : let layer = self.0.get_or_maybe_download(true, Some(ctx)).await?;
245 16818683 : self.0
246 16818683 : .access_stats
247 16818683 : .record_access(LayerAccessKind::GetValueReconstructData, ctx);
248 16818683 :
249 16818683 : if self.layer_desc().is_delta {
250 16394949 : ensure!(lsn_range.start >= self.layer_desc().lsn_range.start);
251 16394949 : ensure!(self.layer_desc().key_range.contains(&key));
252 : } else {
253 423734 : ensure!(self.layer_desc().key_range.contains(&key));
254 423734 : ensure!(lsn_range.start >= self.layer_desc().image_layer_lsn());
255 423734 : ensure!(lsn_range.end >= self.layer_desc().image_layer_lsn());
256 : }
257 :
258 16818683 : layer
259 16818683 : .get_value_reconstruct_data(key, lsn_range, reconstruct_data, &self.0, ctx)
260 16818683 : .instrument(tracing::debug_span!("get_value_reconstruct_data", layer=%self))
261 972406 : .await
262 16818678 : .with_context(|| format!("get_value_reconstruct_data for layer {self}"))
263 16818679 : }
264 :
265 : /// Download the layer if evicted.
266 : ///
267 : /// Will not error when the layer is already downloaded.
268 12 : pub(crate) async fn download(&self) -> anyhow::Result<()> {
269 32 : self.0.get_or_maybe_download(true, None).await?;
270 7 : Ok(())
271 12 : }
272 :
273 : /// Assuming the layer is already downloaded, returns a guard which will prohibit eviction
274 : /// while the guard exists.
275 : ///
276 : /// Returns None if the layer is currently evicted.
277 4905 : pub(crate) async fn keep_resident(&self) -> anyhow::Result<Option<ResidentLayer>> {
278 4905 : let downloaded = match self.0.get_or_maybe_download(false, None).await {
279 4235 : Ok(d) => d,
280 : // technically there are a lot of possible errors, but in practice it should only be
281 : // DownloadRequired which is tripped up. could work to improve this situation
282 : // statically later.
283 670 : Err(DownloadError::DownloadRequired) => return Ok(None),
284 0 : Err(e) => return Err(e.into()),
285 : };
286 :
287 4235 : Ok(Some(ResidentLayer {
288 4235 : downloaded,
289 4235 : owner: self.clone(),
290 4235 : }))
291 4905 : }
292 :
293 : /// Downloads if necessary and creates a guard, which will keep this layer from being evicted.
294 4176 : pub(crate) async fn download_and_keep_resident(&self) -> anyhow::Result<ResidentLayer> {
295 4176 : let downloaded = self.0.get_or_maybe_download(true, None).await?;
296 :
297 4176 : Ok(ResidentLayer {
298 4176 : downloaded,
299 4176 : owner: self.clone(),
300 4176 : })
301 4176 : }
302 :
303 3151 : pub(crate) fn info(&self, reset: LayerAccessStatsReset) -> HistoricLayerInfo {
304 3151 : self.0.info(reset)
305 3151 : }
306 :
307 4233 : pub(crate) fn access_stats(&self) -> &LayerAccessStats {
308 4233 : &self.0.access_stats
309 4233 : }
310 :
311 23817 : pub(crate) fn local_path(&self) -> &Utf8Path {
312 23817 : &self.0.path
313 23817 : }
314 :
315 24914 : pub(crate) fn metadata(&self) -> LayerFileMetadata {
316 24914 : self.0.metadata()
317 24914 : }
318 :
319 : /// Traditional debug dumping facility
320 : #[allow(unused)]
321 4 : pub(crate) async fn dump(&self, verbose: bool, ctx: &RequestContext) -> anyhow::Result<()> {
322 4 : self.0.desc.dump();
323 4 :
324 4 : if verbose {
325 : // for now, unconditionally download everything, even if that might not be wanted.
326 4 : let l = self.0.get_or_maybe_download(true, Some(ctx)).await?;
327 8 : l.dump(&self.0, ctx).await?
328 0 : }
329 :
330 4 : Ok(())
331 4 : }
332 :
333 : /// Waits until this layer has been dropped (and if needed, local file deletion and remote
334 : /// deletion scheduling has completed).
335 : ///
336 : /// Does not start local deletion, use [`Self::delete_on_drop`] for that
337 : /// separatedly.
338 : #[cfg(feature = "testing")]
339 1086 : pub(crate) fn wait_drop(&self) -> impl std::future::Future<Output = ()> + 'static {
340 1086 : let mut rx = self.0.status.subscribe();
341 :
342 1086 : async move {
343 : loop {
344 1086 : if let Err(tokio::sync::broadcast::error::RecvError::Closed) = rx.recv().await {
345 1086 : break;
346 0 : }
347 : }
348 1086 : }
349 1086 : }
350 : }
351 :
352 : /// The download-ness ([`DownloadedLayer`]) can be either resident or wanted evicted.
353 : ///
354 : /// However when we want something evicted, we cannot evict it right away as there might be current
355 : /// reads happening on it. For example: it has been searched from [`LayerMap::search`] but not yet
356 : /// read with [`Layer::get_value_reconstruct_data`].
357 : ///
358 : /// [`LayerMap::search`]: crate::tenant::layer_map::LayerMap::search
359 0 : #[derive(Debug)]
360 : enum ResidentOrWantedEvicted {
361 : Resident(Arc<DownloadedLayer>),
362 : WantedEvicted(Weak<DownloadedLayer>, usize),
363 : }
364 :
365 : impl ResidentOrWantedEvicted {
366 16827115 : fn get_and_upgrade(&mut self) -> Option<(Arc<DownloadedLayer>, bool)> {
367 16827115 : match self {
368 16827115 : ResidentOrWantedEvicted::Resident(strong) => Some((strong.clone(), false)),
369 0 : ResidentOrWantedEvicted::WantedEvicted(weak, _) => match weak.upgrade() {
370 0 : Some(strong) => {
371 0 : LAYER_IMPL_METRICS.inc_raced_wanted_evicted_accesses();
372 0 :
373 0 : *self = ResidentOrWantedEvicted::Resident(strong.clone());
374 0 :
375 0 : Some((strong, true))
376 : }
377 0 : None => None,
378 : },
379 : }
380 16827115 : }
381 :
382 : /// When eviction is first requested, drop down to holding a [`Weak`].
383 : ///
384 : /// Returns `Some` if this was the first time eviction was requested. Care should be taken to
385 : /// drop the possibly last strong reference outside of the mutex of
386 : /// heavier_once_cell::OnceCell.
387 2553 : fn downgrade(&mut self) -> Option<Arc<DownloadedLayer>> {
388 2553 : match self {
389 2552 : ResidentOrWantedEvicted::Resident(strong) => {
390 2552 : let weak = Arc::downgrade(strong);
391 2552 : let mut temp = ResidentOrWantedEvicted::WantedEvicted(weak, strong.version);
392 2552 : std::mem::swap(self, &mut temp);
393 2552 : match temp {
394 2552 : ResidentOrWantedEvicted::Resident(strong) => Some(strong),
395 0 : ResidentOrWantedEvicted::WantedEvicted(..) => unreachable!("just swapped"),
396 : }
397 : }
398 1 : ResidentOrWantedEvicted::WantedEvicted(..) => None,
399 : }
400 2553 : }
401 : }
402 :
403 : struct LayerInner {
404 : /// Only needed to check ondemand_download_behavior_treat_error_as_warn and creation of
405 : /// [`Self::path`].
406 : conf: &'static PageServerConf,
407 :
408 : /// Full path to the file; unclear if this should exist anymore.
409 : path: Utf8PathBuf,
410 :
411 : desc: PersistentLayerDesc,
412 :
413 : /// Timeline access is needed for remote timeline client and metrics.
414 : timeline: Weak<Timeline>,
415 :
416 : /// Cached knowledge of [`Timeline::remote_client`] being `Some`.
417 : have_remote_client: bool,
418 :
419 : access_stats: LayerAccessStats,
420 :
421 : /// This custom OnceCell is backed by std mutex, but only held for short time periods.
422 : /// Initialization and deinitialization are done while holding a permit.
423 : inner: heavier_once_cell::OnceCell<ResidentOrWantedEvicted>,
424 :
425 : /// Do we want to delete locally and remotely this when `LayerInner` is dropped
426 : wanted_deleted: AtomicBool,
427 :
428 : /// Do we want to evict this layer as soon as possible? After being set to `true`, all accesses
429 : /// will try to downgrade [`ResidentOrWantedEvicted`], which will eventually trigger
430 : /// [`LayerInner::on_downloaded_layer_drop`].
431 : wanted_evicted: AtomicBool,
432 :
433 : /// Version is to make sure we will only evict a specific download of a file.
434 : ///
435 : /// Incremented for each download, stored in `DownloadedLayer::version` or
436 : /// `ResidentOrWantedEvicted::WantedEvicted`.
437 : version: AtomicUsize,
438 :
439 : /// Allow subscribing to when the layer actually gets evicted.
440 : status: tokio::sync::broadcast::Sender<Status>,
441 :
442 : /// Counter for exponential backoff with the download
443 : consecutive_failures: AtomicUsize,
444 :
445 : /// The generation of this Layer.
446 : ///
447 : /// For loaded layers (resident or evicted) this comes from [`LayerFileMetadata::generation`],
448 : /// for created layers from [`Timeline::generation`].
449 : generation: Generation,
450 :
451 : /// The shard of this Layer.
452 : ///
453 : /// For layers created in this process, this will always be the [`ShardIndex`] of the
454 : /// current `ShardIdentity`` (TODO: add link once it's introduced).
455 : ///
456 : /// For loaded layers, this may be some other value if the tenant has undergone
457 : /// a shard split since the layer was originally written.
458 : shard: ShardIndex,
459 :
460 : last_evicted_at: std::sync::Mutex<Option<std::time::Instant>>,
461 : }
462 :
463 : impl std::fmt::Display for LayerInner {
464 22474 : fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
465 22474 : write!(f, "{}", self.layer_desc().short_id())
466 22474 : }
467 : }
468 :
469 : impl AsLayerDesc for LayerInner {
470 51239007 : fn layer_desc(&self) -> &PersistentLayerDesc {
471 51239007 : &self.desc
472 51239007 : }
473 : }
474 :
475 2553 : #[derive(Debug, Clone, Copy)]
476 : enum Status {
477 : Evicted,
478 : Downloaded,
479 : }
480 :
481 : impl Drop for LayerInner {
482 44601 : fn drop(&mut self) {
483 44601 : if !*self.wanted_deleted.get_mut() {
484 : // should we try to evict if the last wish was for eviction?
485 : // feels like there's some hazard of overcrowding near shutdown near by, but we don't
486 : // run drops during shutdown (yet)
487 39382 : return;
488 5219 : }
489 :
490 5219 : let span = tracing::info_span!(parent: None, "layer_delete", tenant_id = %self.layer_desc().tenant_shard_id.tenant_id, shard_id=%self.layer_desc().tenant_shard_id.shard_slug(), timeline_id = %self.layer_desc().timeline_id);
491 :
492 5219 : let path = std::mem::take(&mut self.path);
493 5219 : let file_name = self.layer_desc().filename();
494 5219 : let file_size = self.layer_desc().file_size;
495 5219 : let timeline = self.timeline.clone();
496 5219 : let meta = self.metadata();
497 5219 : let status = self.status.clone();
498 5219 :
499 5219 : crate::task_mgr::BACKGROUND_RUNTIME.spawn_blocking(move || {
500 5219 : let _g = span.entered();
501 5219 :
502 5219 : // carry this until we are finished for [`Layer::wait_drop`] support
503 5219 : let _status = status;
504 :
505 5219 : let removed = match std::fs::remove_file(path) {
506 5217 : Ok(()) => true,
507 2 : Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
508 2 : // until we no longer do detaches by removing all local files before removing the
509 2 : // tenant from the global map, we will always get these errors even if we knew what
510 2 : // is the latest state.
511 2 : //
512 2 : // we currently do not track the latest state, so we'll also end up here on evicted
513 2 : // layers.
514 2 : false
515 : }
516 0 : Err(e) => {
517 0 : tracing::error!("failed to remove wanted deleted layer: {e}");
518 0 : LAYER_IMPL_METRICS.inc_delete_removes_failed();
519 0 : false
520 : }
521 : };
522 :
523 5219 : if let Some(timeline) = timeline.upgrade() {
524 5219 : if removed {
525 5217 : timeline.metrics.resident_physical_size_sub(file_size);
526 5217 : }
527 5219 : if let Some(remote_client) = timeline.remote_client.as_ref() {
528 5219 : let res = remote_client.schedule_deletion_of_unlinked(vec![(file_name, meta)]);
529 :
530 5219 : if let Err(e) = res {
531 : // test_timeline_deletion_with_files_stuck_in_upload_queue is good at
532 : // demonstrating this deadlock (without spawn_blocking): stop will drop
533 : // queued items, which will have ResidentLayer's, and those drops would try
534 : // to re-entrantly lock the RemoteTimelineClient inner state.
535 14 : if !timeline.is_active() {
536 14 : tracing::info!("scheduling deletion on drop failed: {e:#}");
537 : } else {
538 0 : tracing::warn!("scheduling deletion on drop failed: {e:#}");
539 : }
540 14 : LAYER_IMPL_METRICS.inc_deletes_failed(DeleteFailed::DeleteSchedulingFailed);
541 5205 : } else {
542 5205 : LAYER_IMPL_METRICS.inc_completed_deletes();
543 5205 : }
544 0 : }
545 0 : } else {
546 0 : // no need to nag that timeline is gone: under normal situation on
547 0 : // task_mgr::remove_tenant_from_memory the timeline is gone before we get dropped.
548 0 : LAYER_IMPL_METRICS.inc_deletes_failed(DeleteFailed::TimelineGone);
549 0 : }
550 5219 : });
551 44601 : }
552 : }
553 :
554 : impl LayerInner {
555 75560 : fn new(
556 75560 : conf: &'static PageServerConf,
557 75560 : timeline: &Arc<Timeline>,
558 75560 : access_stats: LayerAccessStats,
559 75560 : desc: PersistentLayerDesc,
560 75560 : downloaded: Option<Arc<DownloadedLayer>>,
561 75560 : generation: Generation,
562 75560 : shard: ShardIndex,
563 75560 : ) -> Self {
564 75560 : let path = conf
565 75560 : .timeline_path(&timeline.tenant_shard_id, &timeline.timeline_id)
566 75560 : .join(desc.filename().to_string());
567 :
568 75560 : let (inner, version) = if let Some(inner) = downloaded {
569 34801 : let version = inner.version;
570 34801 : let resident = ResidentOrWantedEvicted::Resident(inner);
571 34801 : (heavier_once_cell::OnceCell::new(resident), version)
572 : } else {
573 40759 : (heavier_once_cell::OnceCell::default(), 0)
574 : };
575 :
576 75560 : LayerInner {
577 75560 : conf,
578 75560 : path,
579 75560 : desc,
580 75560 : timeline: Arc::downgrade(timeline),
581 75560 : have_remote_client: timeline.remote_client.is_some(),
582 75560 : access_stats,
583 75560 : wanted_deleted: AtomicBool::new(false),
584 75560 : wanted_evicted: AtomicBool::new(false),
585 75560 : inner,
586 75560 : version: AtomicUsize::new(version),
587 75560 : status: tokio::sync::broadcast::channel(1).0,
588 75560 : consecutive_failures: AtomicUsize::new(0),
589 75560 : generation,
590 75560 : shard,
591 75560 : last_evicted_at: std::sync::Mutex::default(),
592 75560 : }
593 75560 : }
594 :
595 5219 : fn delete_on_drop(&self) {
596 5219 : let res =
597 5219 : self.wanted_deleted
598 5219 : .compare_exchange(false, true, Ordering::Release, Ordering::Relaxed);
599 5219 :
600 5219 : if res.is_ok() {
601 5219 : LAYER_IMPL_METRICS.inc_started_deletes();
602 5219 : }
603 5219 : }
604 :
605 : /// Cancellation safe, however dropping the future and calling this method again might result
606 : /// in a new attempt to evict OR join the previously started attempt.
607 2554 : pub(crate) async fn evict_and_wait(&self) -> Result<(), EvictionError> {
608 2554 : use tokio::sync::broadcast::error::RecvError;
609 2554 :
610 2554 : assert!(self.have_remote_client);
611 :
612 2554 : let mut rx = self.status.subscribe();
613 :
614 2553 : let strong = {
615 2554 : match self.inner.get() {
616 2553 : Some(mut either) => {
617 2553 : self.wanted_evicted.store(true, Ordering::Relaxed);
618 2553 : either.downgrade()
619 : }
620 1 : None => return Err(EvictionError::NotFound),
621 : }
622 : };
623 :
624 2553 : if strong.is_some() {
625 2552 : // drop the DownloadedLayer outside of the holding the guard
626 2552 : drop(strong);
627 2552 : LAYER_IMPL_METRICS.inc_started_evictions();
628 2552 : }
629 :
630 2553 : match rx.recv().await {
631 2553 : Ok(Status::Evicted) => Ok(()),
632 0 : Ok(Status::Downloaded) => Err(EvictionError::Downloaded),
633 : Err(RecvError::Closed) => {
634 0 : unreachable!("sender cannot be dropped while we are in &self method")
635 : }
636 : Err(RecvError::Lagged(_)) => {
637 : // this is quite unlikely, but we are blocking a lot in the async context, so
638 : // we might be missing this because we are stuck on a LIFO slot on a thread
639 : // which is busy blocking for a 1TB database create_image_layers.
640 : //
641 : // use however late (compared to the initial expressing of wanted) as the
642 : // "outcome" now
643 0 : LAYER_IMPL_METRICS.inc_broadcast_lagged();
644 0 : match self.inner.get() {
645 0 : Some(_) => Err(EvictionError::Downloaded),
646 0 : None => Ok(()),
647 : }
648 : }
649 : }
650 2554 : }
651 :
652 : /// Cancellation safe.
653 16827793 : async fn get_or_maybe_download(
654 16827793 : self: &Arc<Self>,
655 16827793 : allow_download: bool,
656 16827793 : ctx: Option<&RequestContext>,
657 16827793 : ) -> Result<Arc<DownloadedLayer>, DownloadError> {
658 16827783 : let mut init_permit = None;
659 :
660 : loop {
661 16827783 : let download = move |permit| {
662 10296 : async move {
663 10296 : // disable any scheduled but not yet running eviction deletions for this
664 10296 : let next_version = 1 + self.version.fetch_add(1, Ordering::Relaxed);
665 10296 :
666 10296 : // count cancellations, which currently remain largely unexpected
667 10296 : let init_cancelled =
668 10296 : scopeguard::guard((), |_| LAYER_IMPL_METRICS.inc_init_cancelled());
669 10296 :
670 10296 : // no need to make the evict_and_wait wait for the actual download to complete
671 10296 : drop(self.status.send(Status::Downloaded));
672 :
673 10296 : let timeline = self
674 10296 : .timeline
675 10296 : .upgrade()
676 10296 : .ok_or_else(|| DownloadError::TimelineShutdown)?;
677 :
678 : // FIXME: grab a gate
679 :
680 10296 : let can_ever_evict = timeline.remote_client.as_ref().is_some();
681 :
682 : // check if we really need to be downloaded; could have been already downloaded by a
683 : // cancelled previous attempt.
684 10296 : let needs_download = self
685 10296 : .needs_download()
686 10103 : .await
687 10296 : .map_err(DownloadError::PreStatFailed)?;
688 :
689 10296 : let permit = if let Some(reason) = needs_download {
690 10296 : if let NeedsDownload::NotFile(ft) = reason {
691 0 : return Err(DownloadError::NotFile(ft));
692 10296 : }
693 10296 :
694 10296 : // only reset this after we've decided we really need to download. otherwise it'd
695 10296 : // be impossible to mark cancelled downloads for eviction, like one could imagine
696 10296 : // we would like to do for prefetching which was not needed.
697 10296 : self.wanted_evicted.store(false, Ordering::Release);
698 10296 :
699 10296 : if !can_ever_evict {
700 0 : return Err(DownloadError::NoRemoteStorage);
701 10296 : }
702 :
703 10296 : if let Some(ctx) = ctx {
704 9611 : self.check_expected_download(ctx)?;
705 685 : }
706 :
707 10296 : if !allow_download {
708 : // this does look weird, but for LayerInner the "downloading" means also changing
709 : // internal once related state ...
710 670 : return Err(DownloadError::DownloadRequired);
711 9626 : }
712 9626 :
713 9626 : tracing::info!(%reason, "downloading on-demand");
714 :
715 20382 : self.spawn_download_and_wait(timeline, permit).await?
716 : } else {
717 : // the file is present locally, probably by a previous but cancelled call to
718 : // get_or_maybe_download. alternatively we might be running without remote storage.
719 0 : LAYER_IMPL_METRICS.inc_init_needed_no_download();
720 0 :
721 0 : permit
722 : };
723 :
724 9618 : let since_last_eviction =
725 9618 : self.last_evicted_at.lock().unwrap().map(|ts| ts.elapsed());
726 9618 : if let Some(since_last_eviction) = since_last_eviction {
727 121 : // FIXME: this will not always be recorded correctly until #6028 (the no
728 121 : // download needed branch above)
729 121 : LAYER_IMPL_METRICS.record_redownloaded_after(since_last_eviction);
730 9497 : }
731 :
732 9618 : let res = Arc::new(DownloadedLayer {
733 9618 : owner: Arc::downgrade(self),
734 9618 : kind: tokio::sync::OnceCell::default(),
735 9618 : version: next_version,
736 9618 : });
737 9618 :
738 9618 : self.access_stats.record_residence_event(
739 9618 : LayerResidenceStatus::Resident,
740 9618 : LayerResidenceEventReason::ResidenceChange,
741 9618 : );
742 9618 :
743 9618 : let waiters = self.inner.initializer_count();
744 9618 : if waiters > 0 {
745 316 : tracing::info!(
746 316 : waiters,
747 316 : "completing the on-demand download for other tasks"
748 316 : );
749 9302 : }
750 :
751 9618 : scopeguard::ScopeGuard::into_inner(init_cancelled);
752 9618 :
753 9618 : Ok((ResidentOrWantedEvicted::Resident(res), permit))
754 10294 : }
755 10296 : .instrument(tracing::info_span!("get_or_maybe_download", layer=%self))
756 10296 : };
757 :
758 16827783 : if let Some(init_permit) = init_permit.take() {
759 : // use the already held initialization permit because it is impossible to hit the
760 : // below paths anymore essentially limiting the max loop iterations to 2.
761 0 : let (value, init_permit) = download(init_permit).await?;
762 0 : let mut guard = self.inner.set(value, init_permit);
763 0 : let (strong, _upgraded) = guard
764 0 : .get_and_upgrade()
765 0 : .expect("init creates strong reference, we held the init permit");
766 0 : return Ok(strong);
767 16827783 : }
768 :
769 0 : let (weak, permit) = {
770 16827783 : let mut locked = self.inner.get_or_init(download).await?;
771 :
772 16827105 : if let Some((strong, upgraded)) = locked.get_and_upgrade() {
773 16827105 : if upgraded {
774 0 : // when upgraded back, the Arc<DownloadedLayer> is still available, but
775 0 : // previously a `evict_and_wait` was received.
776 0 : self.wanted_evicted.store(false, Ordering::Relaxed);
777 0 :
778 0 : // error out any `evict_and_wait`
779 0 : drop(self.status.send(Status::Downloaded));
780 0 : LAYER_IMPL_METRICS
781 0 : .inc_eviction_cancelled(EvictionCancelled::UpgradedBackOnAccess);
782 16827105 : }
783 :
784 16827105 : return Ok(strong);
785 : } else {
786 : // path to here: the evict_blocking is stuck on spawn_blocking queue.
787 : //
788 : // reset the contents, deactivating the eviction and causing a
789 : // EvictionCancelled::LostToDownload or EvictionCancelled::VersionCheckFailed.
790 0 : locked.take_and_deinit()
791 0 : }
792 0 : };
793 0 :
794 0 : // unlock first, then drop the weak, but because upgrade failed, we
795 0 : // know it cannot be a problem.
796 0 :
797 0 : assert!(
798 0 : matches!(weak, ResidentOrWantedEvicted::WantedEvicted(..)),
799 0 : "unexpected {weak:?}, ResidentOrWantedEvicted::get_and_upgrade has a bug"
800 : );
801 :
802 0 : init_permit = Some(permit);
803 0 :
804 0 : LAYER_IMPL_METRICS.inc_retried_get_or_maybe_download();
805 : }
806 16827781 : }
807 :
808 : /// Nag or fail per RequestContext policy
809 9611 : fn check_expected_download(&self, ctx: &RequestContext) -> Result<(), DownloadError> {
810 9611 : use crate::context::DownloadBehavior::*;
811 9611 : let b = ctx.download_behavior();
812 9611 : match b {
813 9611 : Download => Ok(()),
814 : Warn | Error => {
815 0 : tracing::info!(
816 0 : "unexpectedly on-demand downloading for task kind {:?}",
817 0 : ctx.task_kind()
818 0 : );
819 0 : crate::metrics::UNEXPECTED_ONDEMAND_DOWNLOADS.inc();
820 :
821 0 : let really_error =
822 0 : matches!(b, Error) && !self.conf.ondemand_download_behavior_treat_error_as_warn;
823 :
824 0 : if really_error {
825 : // this check is only probablistic, seems like flakyness footgun
826 0 : Err(DownloadError::ContextAndConfigReallyDeniesDownloads)
827 : } else {
828 0 : Ok(())
829 : }
830 : }
831 : }
832 9611 : }
833 :
834 : /// Actual download, at most one is executed at the time.
835 9626 : async fn spawn_download_and_wait(
836 9626 : self: &Arc<Self>,
837 9626 : timeline: Arc<Timeline>,
838 9626 : permit: heavier_once_cell::InitPermit,
839 9626 : ) -> Result<heavier_once_cell::InitPermit, DownloadError> {
840 9626 : debug_assert_current_span_has_tenant_and_timeline_id();
841 9626 :
842 9626 : let task_name = format!("download layer {}", self);
843 9626 :
844 9626 : let (tx, rx) = tokio::sync::oneshot::channel();
845 9626 :
846 9626 : // this is sadly needed because of task_mgr::shutdown_tasks, otherwise we cannot
847 9626 : // block tenant::mgr::remove_tenant_from_memory.
848 9626 :
849 9626 : let this: Arc<Self> = self.clone();
850 9626 :
851 9626 : crate::task_mgr::spawn(
852 9626 : &tokio::runtime::Handle::current(),
853 9626 : crate::task_mgr::TaskKind::RemoteDownloadTask,
854 9626 : Some(self.desc.tenant_shard_id),
855 9626 : Some(self.desc.timeline_id),
856 9626 : &task_name,
857 9626 : false,
858 9626 : async move {
859 9626 :
860 9626 : let client = timeline
861 9626 : .remote_client
862 9626 : .as_ref()
863 9626 : .expect("checked above with have_remote_client");
864 :
865 9626 : let result = client.download_layer_file(
866 9626 : &this.desc.filename(),
867 9626 : &this.metadata(),
868 9626 : &crate::task_mgr::shutdown_token()
869 9626 : )
870 453629 : .await;
871 :
872 9626 : let result = match result {
873 9619 : Ok(size) => {
874 9619 : timeline.metrics.resident_physical_size_add(size);
875 9619 : Ok(())
876 : }
877 7 : Err(e) => {
878 7 : let consecutive_failures =
879 7 : this.consecutive_failures.fetch_add(1, Ordering::Relaxed);
880 7 :
881 7 : let backoff = utils::backoff::exponential_backoff_duration_seconds(
882 7 : consecutive_failures.min(u32::MAX as usize) as u32,
883 7 : 1.5,
884 7 : 60.0,
885 7 : );
886 7 :
887 7 : let backoff = std::time::Duration::from_secs_f64(backoff);
888 7 :
889 13 : tokio::select! {
890 13 : _ = tokio::time::sleep(backoff) => {},
891 13 : _ = crate::task_mgr::shutdown_token().cancelled_owned() => {},
892 13 : _ = timeline.cancel.cancelled() => {},
893 13 : };
894 :
895 7 : Err(e)
896 : }
897 : };
898 :
899 9626 : if let Err(res) = tx.send((result, permit)) {
900 2 : match res {
901 1 : (Ok(()), _) => {
902 1 : // our caller is cancellation safe so this is fine; if someone
903 1 : // else requests the layer, they'll find it already downloaded.
904 1 : //
905 1 : // See counter [`LayerImplMetrics::inc_init_needed_no_download`]
906 1 : //
907 1 : // FIXME(#6028): however, could be that we should consider marking the
908 1 : // layer for eviction? alas, cannot: because only DownloadedLayer will
909 1 : // handle that.
910 1 : },
911 1 : (Err(e), _) => {
912 1 : // our caller is cancellation safe, but we might be racing with
913 1 : // another attempt to initialize. before we have cancellation
914 1 : // token support: these attempts should converge regardless of
915 1 : // their completion order.
916 1 : tracing::error!("layer file download failed, and additionally failed to communicate this to caller: {e:?}");
917 1 : LAYER_IMPL_METRICS.inc_download_failed_without_requester();
918 : }
919 : }
920 9624 : }
921 :
922 9626 : Ok(())
923 9626 : }
924 9626 : .in_current_span(),
925 9626 : );
926 10983 : match rx.await {
927 9618 : Ok((Ok(()), permit)) => {
928 9618 : if let Some(reason) = self
929 9618 : .needs_download()
930 9399 : .await
931 9618 : .map_err(DownloadError::PostStatFailed)?
932 : {
933 : // this is really a bug in needs_download or remote timeline client
934 0 : panic!("post-condition failed: needs_download returned {reason:?}");
935 9618 : }
936 9618 :
937 9618 : self.consecutive_failures.store(0, Ordering::Relaxed);
938 9618 : tracing::info!("on-demand download successful");
939 :
940 9618 : Ok(permit)
941 : }
942 6 : Ok((Err(e), _permit)) => {
943 6 : // sleep already happened in the spawned task, if it was not cancelled
944 6 : let consecutive_failures = self.consecutive_failures.load(Ordering::Relaxed);
945 6 :
946 6 : match e.downcast_ref::<remote_storage::DownloadError>() {
947 : // If the download failed due to its cancellation token,
948 : // propagate the cancellation error upstream.
949 : Some(remote_storage::DownloadError::Cancelled) => {
950 0 : Err(DownloadError::DownloadCancelled)
951 : }
952 : _ => {
953 6 : tracing::error!(consecutive_failures, "layer file download failed: {e:#}");
954 6 : Err(DownloadError::DownloadFailed)
955 : }
956 : }
957 : }
958 0 : Err(_gone) => Err(DownloadError::DownloadCancelled),
959 : }
960 9624 : }
961 :
962 19914 : async fn needs_download(&self) -> Result<Option<NeedsDownload>, std::io::Error> {
963 19914 : match tokio::fs::metadata(&self.path).await {
964 9618 : Ok(m) => Ok(self.is_file_present_and_good_size(&m).err()),
965 10296 : Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(Some(NeedsDownload::NotFound)),
966 0 : Err(e) => Err(e),
967 : }
968 19914 : }
969 :
970 53195 : fn needs_download_blocking(&self) -> Result<Option<NeedsDownload>, std::io::Error> {
971 53195 : match self.path.metadata() {
972 12436 : Ok(m) => Ok(self.is_file_present_and_good_size(&m).err()),
973 40759 : Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(Some(NeedsDownload::NotFound)),
974 0 : Err(e) => Err(e),
975 : }
976 53195 : }
977 :
978 22054 : fn is_file_present_and_good_size(&self, m: &std::fs::Metadata) -> Result<(), NeedsDownload> {
979 22054 : // in future, this should include sha2-256 validation of the file.
980 22054 : if !m.is_file() {
981 0 : Err(NeedsDownload::NotFile(m.file_type()))
982 22054 : } else if m.len() != self.desc.file_size {
983 0 : Err(NeedsDownload::WrongSize {
984 0 : actual: m.len(),
985 0 : expected: self.desc.file_size,
986 0 : })
987 : } else {
988 22054 : Ok(())
989 : }
990 22054 : }
991 :
992 3151 : fn info(&self, reset: LayerAccessStatsReset) -> HistoricLayerInfo {
993 3151 : let layer_file_name = self.desc.filename().file_name();
994 3151 :
995 3151 : // this is not accurate: we could have the file locally but there was a cancellation
996 3151 : // and now we are not in sync, or we are currently downloading it.
997 3151 : let remote = self.inner.get().is_none();
998 3151 :
999 3151 : let access_stats = self.access_stats.as_api_model(reset);
1000 3151 :
1001 3151 : if self.desc.is_delta {
1002 2560 : let lsn_range = &self.desc.lsn_range;
1003 2560 :
1004 2560 : HistoricLayerInfo::Delta {
1005 2560 : layer_file_name,
1006 2560 : layer_file_size: self.desc.file_size,
1007 2560 : lsn_start: lsn_range.start,
1008 2560 : lsn_end: lsn_range.end,
1009 2560 : remote,
1010 2560 : access_stats,
1011 2560 : }
1012 : } else {
1013 591 : let lsn = self.desc.image_layer_lsn();
1014 591 :
1015 591 : HistoricLayerInfo::Image {
1016 591 : layer_file_name,
1017 591 : layer_file_size: self.desc.file_size,
1018 591 : lsn_start: lsn,
1019 591 : remote,
1020 591 : access_stats,
1021 591 : }
1022 : }
1023 3151 : }
1024 :
1025 : /// `DownloadedLayer` is being dropped, so it calls this method.
1026 2552 : fn on_downloaded_layer_drop(self: Arc<LayerInner>, version: usize) {
1027 2552 : let delete = self.wanted_deleted.load(Ordering::Acquire);
1028 2552 : let evict = self.wanted_evicted.load(Ordering::Acquire);
1029 2552 : let can_evict = self.have_remote_client;
1030 2552 :
1031 2552 : if delete {
1032 0 : // do nothing now, only in LayerInner::drop -- this was originally implemented because
1033 0 : // we could had already scheduled the deletion at the time.
1034 0 : //
1035 0 : // FIXME: this is not true anymore, we can safely evict wanted deleted files.
1036 2552 : } else if can_evict && evict {
1037 2552 : let span = tracing::info_span!(parent: None, "layer_evict", tenant_id = %self.desc.tenant_shard_id.tenant_id, shard_id = %self.desc.tenant_shard_id.shard_slug(), timeline_id = %self.desc.timeline_id, layer=%self, %version);
1038 :
1039 : // downgrade for queueing, in case there's a tear down already ongoing we should not
1040 : // hold it alive.
1041 2552 : let this = Arc::downgrade(&self);
1042 2552 : drop(self);
1043 2552 :
1044 2552 : // NOTE: this scope *must* never call `self.inner.get` because evict_and_wait might
1045 2552 : // drop while the `self.inner` is being locked, leading to a deadlock.
1046 2552 :
1047 2552 : crate::task_mgr::BACKGROUND_RUNTIME.spawn_blocking(move || {
1048 2552 : let _g = span.entered();
1049 :
1050 : // if LayerInner is already dropped here, do nothing because the delete on drop
1051 : // has already ran while we were in queue
1052 2552 : let Some(this) = this.upgrade() else {
1053 0 : LAYER_IMPL_METRICS.inc_eviction_cancelled(EvictionCancelled::LayerGone);
1054 0 : return;
1055 : };
1056 2552 : match this.evict_blocking(version) {
1057 2552 : Ok(()) => LAYER_IMPL_METRICS.inc_completed_evictions(),
1058 0 : Err(reason) => LAYER_IMPL_METRICS.inc_eviction_cancelled(reason),
1059 : }
1060 2552 : });
1061 0 : }
1062 2552 : }
1063 :
1064 2552 : fn evict_blocking(&self, only_version: usize) -> Result<(), EvictionCancelled> {
1065 : // deleted or detached timeline, don't do anything.
1066 2552 : let Some(timeline) = self.timeline.upgrade() else {
1067 0 : return Err(EvictionCancelled::TimelineGone);
1068 : };
1069 :
1070 : // to avoid starting a new download while we evict, keep holding on to the
1071 : // permit.
1072 2552 : let _permit = {
1073 2552 : let maybe_downloaded = self.inner.get();
1074 :
1075 2552 : let (_weak, permit) = match maybe_downloaded {
1076 2552 : Some(mut guard) => {
1077 2552 : if let ResidentOrWantedEvicted::WantedEvicted(_weak, version) = &*guard {
1078 2552 : if *version == only_version {
1079 2552 : guard.take_and_deinit()
1080 : } else {
1081 : // this was not for us; maybe there's another eviction job
1082 : // TODO: does it make any sense to stall here? unique versions do not
1083 : // matter, we only want to make sure not to evict a resident, which we
1084 : // are not doing.
1085 0 : return Err(EvictionCancelled::VersionCheckFailed);
1086 : }
1087 : } else {
1088 0 : return Err(EvictionCancelled::AlreadyReinitialized);
1089 : }
1090 : }
1091 : None => {
1092 : // already deinitialized, perhaps get_or_maybe_download did this and is
1093 : // currently waiting to reinitialize it
1094 0 : return Err(EvictionCancelled::LostToDownload);
1095 : }
1096 : };
1097 :
1098 2552 : permit
1099 2552 : };
1100 2552 :
1101 2552 : // now accesses to inner.get_or_init wait on the semaphore or the `_permit`
1102 2552 :
1103 2552 : self.access_stats.record_residence_event(
1104 2552 : LayerResidenceStatus::Evicted,
1105 2552 : LayerResidenceEventReason::ResidenceChange,
1106 2552 : );
1107 :
1108 2552 : let res = match capture_mtime_and_remove(&self.path) {
1109 2552 : Ok(local_layer_mtime) => {
1110 2552 : let duration = SystemTime::now().duration_since(local_layer_mtime);
1111 2552 : match duration {
1112 2552 : Ok(elapsed) => {
1113 2552 : timeline
1114 2552 : .metrics
1115 2552 : .evictions_with_low_residence_duration
1116 2552 : .read()
1117 2552 : .unwrap()
1118 2552 : .observe(elapsed);
1119 2552 : tracing::info!(
1120 2552 : residence_millis = elapsed.as_millis(),
1121 2552 : "evicted layer after known residence period"
1122 2552 : );
1123 : }
1124 : Err(_) => {
1125 0 : tracing::info!("evicted layer after unknown residence period");
1126 : }
1127 : }
1128 2552 : timeline.metrics.evictions.inc();
1129 2552 : timeline
1130 2552 : .metrics
1131 2552 : .resident_physical_size_sub(self.desc.file_size);
1132 2552 :
1133 2552 : Ok(())
1134 : }
1135 0 : Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
1136 0 : tracing::error!(
1137 0 : layer_size = %self.desc.file_size,
1138 0 : "failed to evict layer from disk, it was already gone (metrics will be inaccurate)"
1139 0 : );
1140 0 : Err(EvictionCancelled::FileNotFound)
1141 : }
1142 0 : Err(e) => {
1143 0 : tracing::error!("failed to evict file from disk: {e:#}");
1144 0 : Err(EvictionCancelled::RemoveFailed)
1145 : }
1146 : };
1147 :
1148 : // we are still holding the permit, so no new spawn_download_and_wait can happen
1149 2552 : drop(self.status.send(Status::Evicted));
1150 2552 :
1151 2552 : *self.last_evicted_at.lock().unwrap() = Some(std::time::Instant::now());
1152 2552 :
1153 2552 : res
1154 2552 : }
1155 :
1156 39759 : fn metadata(&self) -> LayerFileMetadata {
1157 39759 : LayerFileMetadata::new(self.desc.file_size, self.generation, self.shard)
1158 39759 : }
1159 : }
1160 :
1161 2552 : fn capture_mtime_and_remove(path: &Utf8Path) -> Result<SystemTime, std::io::Error> {
1162 2552 : let m = path.metadata()?;
1163 2552 : let local_layer_mtime = m.modified()?;
1164 2552 : std::fs::remove_file(path)?;
1165 2552 : Ok(local_layer_mtime)
1166 2552 : }
1167 :
1168 0 : #[derive(Debug, thiserror::Error)]
1169 : pub(crate) enum EvictionError {
1170 : #[error("layer was already evicted")]
1171 : NotFound,
1172 :
1173 : /// Evictions must always lose to downloads in races, and this time it happened.
1174 : #[error("layer was downloaded instead")]
1175 : Downloaded,
1176 : }
1177 :
1178 : /// Error internal to the [`LayerInner::get_or_maybe_download`]
1179 13 : #[derive(Debug, thiserror::Error)]
1180 : enum DownloadError {
1181 : #[error("timeline has already shutdown")]
1182 : TimelineShutdown,
1183 : #[error("no remote storage configured")]
1184 : NoRemoteStorage,
1185 : #[error("context denies downloading")]
1186 : ContextAndConfigReallyDeniesDownloads,
1187 : #[error("downloading is really required but not allowed by this method")]
1188 : DownloadRequired,
1189 : #[error("layer path exists, but it is not a file: {0:?}")]
1190 : NotFile(std::fs::FileType),
1191 : /// Why no error here? Because it will be reported by page_service. We should had also done
1192 : /// retries already.
1193 : #[error("downloading evicted layer file failed")]
1194 : DownloadFailed,
1195 : #[error("downloading failed, possibly for shutdown")]
1196 : DownloadCancelled,
1197 : #[error("pre-condition: stat before download failed")]
1198 : PreStatFailed(#[source] std::io::Error),
1199 : #[error("post-condition: stat after download failed")]
1200 : PostStatFailed(#[source] std::io::Error),
1201 : }
1202 :
1203 0 : #[derive(Debug, PartialEq)]
1204 : pub(crate) enum NeedsDownload {
1205 : NotFound,
1206 : NotFile(std::fs::FileType),
1207 : WrongSize { actual: u64, expected: u64 },
1208 : }
1209 :
1210 : impl std::fmt::Display for NeedsDownload {
1211 9626 : fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1212 9626 : match self {
1213 9626 : NeedsDownload::NotFound => write!(f, "file was not found"),
1214 0 : NeedsDownload::NotFile(ft) => write!(f, "path is not a file; {ft:?}"),
1215 0 : NeedsDownload::WrongSize { actual, expected } => {
1216 0 : write!(f, "file size mismatch {actual} vs. {expected}")
1217 : }
1218 : }
1219 9626 : }
1220 : }
1221 :
1222 : /// Existence of `DownloadedLayer` means that we have the file locally, and can later evict it.
1223 : pub(crate) struct DownloadedLayer {
1224 : owner: Weak<LayerInner>,
1225 : // Use tokio OnceCell as we do not need to deinitialize this, it'll just get dropped with the
1226 : // DownloadedLayer
1227 : kind: tokio::sync::OnceCell<anyhow::Result<LayerKind>>,
1228 : version: usize,
1229 : }
1230 :
1231 : impl std::fmt::Debug for DownloadedLayer {
1232 0 : fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1233 0 : f.debug_struct("DownloadedLayer")
1234 0 : // owner omitted because it is always "Weak"
1235 0 : .field("kind", &self.kind)
1236 0 : .field("version", &self.version)
1237 0 : .finish()
1238 0 : }
1239 : }
1240 :
1241 : impl Drop for DownloadedLayer {
1242 24645 : fn drop(&mut self) {
1243 24645 : if let Some(owner) = self.owner.upgrade() {
1244 2552 : owner.on_downloaded_layer_drop(self.version);
1245 22093 : } else {
1246 22093 : // no need to do anything, we are shutting down
1247 22093 : }
1248 24645 : }
1249 : }
1250 :
1251 : impl DownloadedLayer {
1252 : /// Initializes the `DeltaLayerInner` or `ImageLayerInner` within [`LayerKind`], or fails to
1253 : /// initialize it permanently.
1254 : ///
1255 : /// `owner` parameter is a strong reference at the same `LayerInner` as the
1256 : /// `DownloadedLayer::owner` would be when upgraded. Given how this method ends up called,
1257 : /// we will always have the LayerInner on the callstack, so we can just use it.
1258 16822845 : async fn get<'a>(
1259 16822845 : &'a self,
1260 16822845 : owner: &Arc<LayerInner>,
1261 16822845 : ctx: &RequestContext,
1262 16822845 : ) -> anyhow::Result<&'a LayerKind> {
1263 16822835 : let init = || async {
1264 33391 : assert_eq!(
1265 33391 : Weak::as_ptr(&self.owner),
1266 33391 : Arc::as_ptr(owner),
1267 0 : "these are the same, just avoiding the upgrade"
1268 : );
1269 :
1270 33391 : let res = if owner.desc.is_delta {
1271 12147 : let summary = Some(delta_layer::Summary::expected(
1272 12147 : owner.desc.tenant_shard_id.tenant_id,
1273 12147 : owner.desc.timeline_id,
1274 12147 : owner.desc.key_range.clone(),
1275 12147 : owner.desc.lsn_range.clone(),
1276 12147 : ));
1277 12147 : delta_layer::DeltaLayerInner::load(&owner.path, summary, ctx)
1278 453 : .await
1279 12147 : .map(|res| res.map(LayerKind::Delta))
1280 : } else {
1281 21244 : let lsn = owner.desc.image_layer_lsn();
1282 21244 : let summary = Some(image_layer::Summary::expected(
1283 21244 : owner.desc.tenant_shard_id.tenant_id,
1284 21244 : owner.desc.timeline_id,
1285 21244 : owner.desc.key_range.clone(),
1286 21244 : lsn,
1287 21244 : ));
1288 21244 : image_layer::ImageLayerInner::load(&owner.path, lsn, summary, ctx)
1289 260 : .await
1290 21244 : .map(|res| res.map(LayerKind::Image))
1291 : };
1292 :
1293 33390 : match res {
1294 33390 : Ok(Ok(layer)) => Ok(Ok(layer)),
1295 0 : Ok(Err(transient)) => Err(transient),
1296 1 : Err(permanent) => {
1297 1 : LAYER_IMPL_METRICS.inc_permanent_loading_failures();
1298 1 : // TODO(#5815): we are not logging all errors, so temporarily log them **once**
1299 1 : // here as well
1300 1 : let permanent = permanent.context("load layer");
1301 1 : tracing::error!("layer loading failed permanently: {permanent:#}");
1302 1 : Ok(Err(permanent))
1303 : }
1304 : }
1305 66782 : };
1306 16822835 : self.kind
1307 16822835 : .get_or_try_init(init)
1308 : // return transient errors using `?`
1309 1674 : .await?
1310 16822835 : .as_ref()
1311 16822835 : .map_err(|e| {
1312 20 : // errors are not clonabled, cannot but stringify
1313 20 : // test_broken_timeline matches this string
1314 20 : anyhow::anyhow!("layer loading failed: {e:#}")
1315 16822835 : })
1316 16822835 : }
1317 :
1318 16818693 : async fn get_value_reconstruct_data(
1319 16818693 : &self,
1320 16818693 : key: Key,
1321 16818693 : lsn_range: Range<Lsn>,
1322 16818693 : reconstruct_data: &mut ValueReconstructState,
1323 16818693 : owner: &Arc<LayerInner>,
1324 16818693 : ctx: &RequestContext,
1325 16818693 : ) -> anyhow::Result<ValueReconstructResult> {
1326 16818683 : use LayerKind::*;
1327 16818683 :
1328 16818683 : match self.get(owner, ctx).await? {
1329 16394928 : Delta(d) => {
1330 16394928 : d.get_value_reconstruct_data(key, lsn_range, reconstruct_data, ctx)
1331 957813 : .await
1332 : }
1333 423734 : Image(i) => {
1334 423734 : i.get_value_reconstruct_data(key, reconstruct_data, ctx)
1335 13034 : .await
1336 : }
1337 : }
1338 16818678 : }
1339 :
1340 4 : async fn dump(&self, owner: &Arc<LayerInner>, ctx: &RequestContext) -> anyhow::Result<()> {
1341 4 : use LayerKind::*;
1342 4 : match self.get(owner, ctx).await? {
1343 4 : Delta(d) => d.dump(ctx).await?,
1344 0 : Image(i) => i.dump(ctx).await?,
1345 : }
1346 :
1347 4 : Ok(())
1348 4 : }
1349 : }
1350 :
1351 : /// Wrapper around an actual layer implementation.
1352 0 : #[derive(Debug)]
1353 : enum LayerKind {
1354 : Delta(delta_layer::DeltaLayerInner),
1355 : Image(image_layer::ImageLayerInner),
1356 : }
1357 :
1358 : /// Guard for forcing a layer be resident while it exists.
1359 26664 : #[derive(Clone)]
1360 : pub(crate) struct ResidentLayer {
1361 : owner: Layer,
1362 : downloaded: Arc<DownloadedLayer>,
1363 : }
1364 :
1365 : impl std::fmt::Display for ResidentLayer {
1366 34468 : fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1367 34468 : write!(f, "{}", self.owner)
1368 34468 : }
1369 : }
1370 :
1371 : impl std::fmt::Debug for ResidentLayer {
1372 0 : fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1373 0 : write!(f, "{}", self.owner)
1374 0 : }
1375 : }
1376 :
1377 : impl ResidentLayer {
1378 : /// Release the eviction guard, converting back into a plain [`Layer`].
1379 : ///
1380 : /// You can access the [`Layer`] also by using `as_ref`.
1381 20638 : pub(crate) fn drop_eviction_guard(self) -> Layer {
1382 20638 : self.into()
1383 20638 : }
1384 :
1385 : /// Loads all keys stored in the layer. Returns key, lsn and value size.
1386 8296 : #[tracing::instrument(skip_all, fields(layer=%self))]
1387 : pub(crate) async fn load_keys<'a>(
1388 : &'a self,
1389 : ctx: &RequestContext,
1390 : ) -> anyhow::Result<Vec<DeltaEntry<'a>>> {
1391 : use LayerKind::*;
1392 :
1393 : let owner = &self.owner.0;
1394 :
1395 : match self.downloaded.get(owner, ctx).await? {
1396 : Delta(ref d) => {
1397 : owner
1398 : .access_stats
1399 : .record_access(LayerAccessKind::KeyIter, ctx);
1400 :
1401 : // this is valid because the DownloadedLayer::kind is a OnceCell, not a
1402 : // Mutex<OnceCell>, so we cannot go and deinitialize the value with OnceCell::take
1403 : // while it's being held.
1404 : delta_layer::DeltaLayerInner::load_keys(d, ctx)
1405 : .await
1406 : .context("Layer index is corrupted")
1407 : }
1408 : Image(_) => anyhow::bail!("cannot load_keys on a image layer"),
1409 : }
1410 : }
1411 :
1412 46082 : pub(crate) fn local_path(&self) -> &Utf8Path {
1413 46082 : &self.owner.0.path
1414 46082 : }
1415 :
1416 22399 : pub(crate) fn metadata(&self) -> LayerFileMetadata {
1417 22399 : self.owner.metadata()
1418 22399 : }
1419 : }
1420 :
1421 : impl AsLayerDesc for ResidentLayer {
1422 77602 : fn layer_desc(&self) -> &PersistentLayerDesc {
1423 77602 : self.owner.layer_desc()
1424 77602 : }
1425 : }
1426 :
1427 : impl AsRef<Layer> for ResidentLayer {
1428 33127 : fn as_ref(&self) -> &Layer {
1429 33127 : &self.owner
1430 33127 : }
1431 : }
1432 :
1433 : /// Drop the eviction guard.
1434 : impl From<ResidentLayer> for Layer {
1435 20638 : fn from(value: ResidentLayer) -> Self {
1436 20638 : value.owner
1437 20638 : }
1438 : }
1439 :
1440 : use metrics::IntCounter;
1441 :
1442 : pub(crate) struct LayerImplMetrics {
1443 : started_evictions: IntCounter,
1444 : completed_evictions: IntCounter,
1445 : cancelled_evictions: enum_map::EnumMap<EvictionCancelled, IntCounter>,
1446 :
1447 : started_deletes: IntCounter,
1448 : completed_deletes: IntCounter,
1449 : failed_deletes: enum_map::EnumMap<DeleteFailed, IntCounter>,
1450 :
1451 : rare_counters: enum_map::EnumMap<RareEvent, IntCounter>,
1452 : inits_cancelled: metrics::core::GenericCounter<metrics::core::AtomicU64>,
1453 : redownload_after: metrics::Histogram,
1454 : }
1455 :
1456 : impl Default for LayerImplMetrics {
1457 630 : fn default() -> Self {
1458 630 : use enum_map::Enum;
1459 630 :
1460 630 : // reminder: these will be pageserver_layer_* with "_total" suffix
1461 630 :
1462 630 : let started_evictions = metrics::register_int_counter!(
1463 630 : "pageserver_layer_started_evictions",
1464 630 : "Evictions started in the Layer implementation"
1465 630 : )
1466 630 : .unwrap();
1467 630 : let completed_evictions = metrics::register_int_counter!(
1468 630 : "pageserver_layer_completed_evictions",
1469 630 : "Evictions completed in the Layer implementation"
1470 630 : )
1471 630 : .unwrap();
1472 630 :
1473 630 : let cancelled_evictions = metrics::register_int_counter_vec!(
1474 630 : "pageserver_layer_cancelled_evictions_count",
1475 630 : "Different reasons for evictions to have been cancelled or failed",
1476 630 : &["reason"]
1477 630 : )
1478 630 : .unwrap();
1479 630 :
1480 5040 : let cancelled_evictions = enum_map::EnumMap::from_array(std::array::from_fn(|i| {
1481 5040 : let reason = EvictionCancelled::from_usize(i);
1482 5040 : let s = reason.as_str();
1483 5040 : cancelled_evictions.with_label_values(&[s])
1484 5040 : }));
1485 630 :
1486 630 : let started_deletes = metrics::register_int_counter!(
1487 630 : "pageserver_layer_started_deletes",
1488 630 : "Deletions on drop pending in the Layer implementation"
1489 630 : )
1490 630 : .unwrap();
1491 630 : let completed_deletes = metrics::register_int_counter!(
1492 630 : "pageserver_layer_completed_deletes",
1493 630 : "Deletions on drop completed in the Layer implementation"
1494 630 : )
1495 630 : .unwrap();
1496 630 :
1497 630 : let failed_deletes = metrics::register_int_counter_vec!(
1498 630 : "pageserver_layer_failed_deletes_count",
1499 630 : "Different reasons for deletions on drop to have failed",
1500 630 : &["reason"]
1501 630 : )
1502 630 : .unwrap();
1503 630 :
1504 1260 : let failed_deletes = enum_map::EnumMap::from_array(std::array::from_fn(|i| {
1505 1260 : let reason = DeleteFailed::from_usize(i);
1506 1260 : let s = reason.as_str();
1507 1260 : failed_deletes.with_label_values(&[s])
1508 1260 : }));
1509 630 :
1510 630 : let rare_counters = metrics::register_int_counter_vec!(
1511 630 : "pageserver_layer_assumed_rare_count",
1512 630 : "Times unexpected or assumed rare event happened",
1513 630 : &["event"]
1514 630 : )
1515 630 : .unwrap();
1516 630 :
1517 4410 : let rare_counters = enum_map::EnumMap::from_array(std::array::from_fn(|i| {
1518 4410 : let event = RareEvent::from_usize(i);
1519 4410 : let s = event.as_str();
1520 4410 : rare_counters.with_label_values(&[s])
1521 4410 : }));
1522 630 :
1523 630 : let inits_cancelled = metrics::register_int_counter!(
1524 630 : "pageserver_layer_inits_cancelled_count",
1525 630 : "Times Layer initialization was cancelled",
1526 630 : )
1527 630 : .unwrap();
1528 630 :
1529 630 : let redownload_after = {
1530 630 : let minute = 60.0;
1531 630 : let hour = 60.0 * minute;
1532 630 : metrics::register_histogram!(
1533 630 : "pageserver_layer_redownloaded_after",
1534 630 : "Time between evicting and re-downloading.",
1535 630 : vec![
1536 630 : 10.0,
1537 630 : 30.0,
1538 630 : minute,
1539 630 : 5.0 * minute,
1540 630 : 15.0 * minute,
1541 630 : 30.0 * minute,
1542 630 : hour,
1543 630 : 12.0 * hour,
1544 630 : ]
1545 630 : )
1546 630 : .unwrap()
1547 630 : };
1548 630 :
1549 630 : Self {
1550 630 : started_evictions,
1551 630 : completed_evictions,
1552 630 : cancelled_evictions,
1553 630 :
1554 630 : started_deletes,
1555 630 : completed_deletes,
1556 630 : failed_deletes,
1557 630 :
1558 630 : rare_counters,
1559 630 : inits_cancelled,
1560 630 : redownload_after,
1561 630 : }
1562 630 : }
1563 : }
1564 :
1565 : impl LayerImplMetrics {
1566 2552 : fn inc_started_evictions(&self) {
1567 2552 : self.started_evictions.inc();
1568 2552 : }
1569 2552 : fn inc_completed_evictions(&self) {
1570 2552 : self.completed_evictions.inc();
1571 2552 : }
1572 0 : fn inc_eviction_cancelled(&self, reason: EvictionCancelled) {
1573 0 : self.cancelled_evictions[reason].inc()
1574 0 : }
1575 :
1576 5219 : fn inc_started_deletes(&self) {
1577 5219 : self.started_deletes.inc();
1578 5219 : }
1579 5205 : fn inc_completed_deletes(&self) {
1580 5205 : self.completed_deletes.inc();
1581 5205 : }
1582 14 : fn inc_deletes_failed(&self, reason: DeleteFailed) {
1583 14 : self.failed_deletes[reason].inc();
1584 14 : }
1585 :
1586 : /// Counted separatedly from failed layer deletes because we will complete the layer deletion
1587 : /// attempt regardless of failure to delete local file.
1588 0 : fn inc_delete_removes_failed(&self) {
1589 0 : self.rare_counters[RareEvent::RemoveOnDropFailed].inc();
1590 0 : }
1591 :
1592 : /// Expected rare because requires a race with `evict_blocking` and `get_or_maybe_download`.
1593 0 : fn inc_retried_get_or_maybe_download(&self) {
1594 0 : self.rare_counters[RareEvent::RetriedGetOrMaybeDownload].inc();
1595 0 : }
1596 :
1597 : /// Expected rare because cancellations are unexpected, and failures are unexpected
1598 1 : fn inc_download_failed_without_requester(&self) {
1599 1 : self.rare_counters[RareEvent::DownloadFailedWithoutRequester].inc();
1600 1 : }
1601 :
1602 : /// The Weak in ResidentOrWantedEvicted::WantedEvicted was successfully upgraded.
1603 : ///
1604 : /// If this counter is always zero, we should replace ResidentOrWantedEvicted type with an
1605 : /// Option.
1606 0 : fn inc_raced_wanted_evicted_accesses(&self) {
1607 0 : self.rare_counters[RareEvent::UpgradedWantedEvicted].inc();
1608 0 : }
1609 :
1610 : /// These are only expected for [`Self::inc_init_cancelled`] amount when
1611 : /// running with remote storage.
1612 0 : fn inc_init_needed_no_download(&self) {
1613 0 : self.rare_counters[RareEvent::InitWithoutDownload].inc();
1614 0 : }
1615 :
1616 : /// Expected rare because all layer files should be readable and good
1617 1 : fn inc_permanent_loading_failures(&self) {
1618 1 : self.rare_counters[RareEvent::PermanentLoadingFailure].inc();
1619 1 : }
1620 :
1621 0 : fn inc_broadcast_lagged(&self) {
1622 0 : self.rare_counters[RareEvent::EvictAndWaitLagged].inc();
1623 0 : }
1624 :
1625 678 : fn inc_init_cancelled(&self) {
1626 678 : self.inits_cancelled.inc()
1627 678 : }
1628 :
1629 121 : fn record_redownloaded_after(&self, duration: std::time::Duration) {
1630 121 : self.redownload_after.observe(duration.as_secs_f64())
1631 121 : }
1632 : }
1633 :
1634 5040 : #[derive(enum_map::Enum)]
1635 : enum EvictionCancelled {
1636 : LayerGone,
1637 : TimelineGone,
1638 : VersionCheckFailed,
1639 : FileNotFound,
1640 : RemoveFailed,
1641 : AlreadyReinitialized,
1642 : /// Not evicted because of a pending reinitialization
1643 : LostToDownload,
1644 : /// After eviction, there was a new layer access which cancelled the eviction.
1645 : UpgradedBackOnAccess,
1646 : }
1647 :
1648 : impl EvictionCancelled {
1649 5040 : fn as_str(&self) -> &'static str {
1650 5040 : match self {
1651 630 : EvictionCancelled::LayerGone => "layer_gone",
1652 630 : EvictionCancelled::TimelineGone => "timeline_gone",
1653 630 : EvictionCancelled::VersionCheckFailed => "version_check_fail",
1654 630 : EvictionCancelled::FileNotFound => "file_not_found",
1655 630 : EvictionCancelled::RemoveFailed => "remove_failed",
1656 630 : EvictionCancelled::AlreadyReinitialized => "already_reinitialized",
1657 630 : EvictionCancelled::LostToDownload => "lost_to_download",
1658 630 : EvictionCancelled::UpgradedBackOnAccess => "upgraded_back_on_access",
1659 : }
1660 5040 : }
1661 : }
1662 :
1663 1274 : #[derive(enum_map::Enum)]
1664 : enum DeleteFailed {
1665 : TimelineGone,
1666 : DeleteSchedulingFailed,
1667 : }
1668 :
1669 : impl DeleteFailed {
1670 1260 : fn as_str(&self) -> &'static str {
1671 1260 : match self {
1672 630 : DeleteFailed::TimelineGone => "timeline_gone",
1673 630 : DeleteFailed::DeleteSchedulingFailed => "delete_scheduling_failed",
1674 : }
1675 1260 : }
1676 : }
1677 :
1678 4412 : #[derive(enum_map::Enum)]
1679 : enum RareEvent {
1680 : RemoveOnDropFailed,
1681 : RetriedGetOrMaybeDownload,
1682 : DownloadFailedWithoutRequester,
1683 : UpgradedWantedEvicted,
1684 : InitWithoutDownload,
1685 : PermanentLoadingFailure,
1686 : EvictAndWaitLagged,
1687 : }
1688 :
1689 : impl RareEvent {
1690 4410 : fn as_str(&self) -> &'static str {
1691 4410 : use RareEvent::*;
1692 4410 :
1693 4410 : match self {
1694 630 : RemoveOnDropFailed => "remove_on_drop_failed",
1695 630 : RetriedGetOrMaybeDownload => "retried_gomd",
1696 630 : DownloadFailedWithoutRequester => "download_failed_without",
1697 630 : UpgradedWantedEvicted => "raced_wanted_evicted",
1698 630 : InitWithoutDownload => "init_needed_no_download",
1699 630 : PermanentLoadingFailure => "permanent_loading_failure",
1700 630 : EvictAndWaitLagged => "broadcast_lagged",
1701 : }
1702 4410 : }
1703 : }
1704 :
1705 : pub(crate) static LAYER_IMPL_METRICS: once_cell::sync::Lazy<LayerImplMetrics> =
1706 : once_cell::sync::Lazy::new(LayerImplMetrics::default);
|
