Line data Source code
1 : use std::ops::Range;
2 : use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
3 : use std::sync::{Arc, Weak};
4 : use std::time::{Duration, SystemTime};
5 :
6 : use crate::PERF_TRACE_TARGET;
7 : use crate::metrics::{ONDEMAND_DOWNLOAD_BYTES, ONDEMAND_DOWNLOAD_COUNT};
8 : use anyhow::Context;
9 : use camino::{Utf8Path, Utf8PathBuf};
10 : use pageserver_api::keyspace::KeySpace;
11 : use pageserver_api::models::HistoricLayerInfo;
12 : use pageserver_api::shard::{ShardIdentity, ShardIndex, TenantShardId};
13 : use tracing::{Instrument, info_span};
14 : use utils::generation::Generation;
15 : use utils::id::TimelineId;
16 : use utils::lsn::Lsn;
17 : use utils::sync::{gate, heavier_once_cell};
18 :
19 : use super::delta_layer::{self};
20 : use super::image_layer::{self};
21 : use super::{
22 : AsLayerDesc, ImageLayerWriter, LayerAccessStats, LayerAccessStatsReset, LayerName,
23 : LayerVisibilityHint, PerfInstrumentFutureExt, PersistentLayerDesc, ValuesReconstructState,
24 : };
25 : use crate::config::PageServerConf;
26 : use crate::context::{RequestContext, RequestContextBuilder};
27 : use crate::span::debug_assert_current_span_has_tenant_and_timeline_id;
28 : use crate::task_mgr::TaskKind;
29 : use crate::tenant::Timeline;
30 : use crate::tenant::remote_timeline_client::LayerFileMetadata;
31 : use crate::tenant::timeline::{CompactionError, GetVectoredError};
32 :
33 : #[cfg(test)]
34 : mod tests;
35 :
36 : #[cfg(test)]
37 : mod failpoints;
38 :
39 : pub const S3_UPLOAD_LIMIT: u64 = 4_500_000_000;
40 :
41 : /// A Layer contains all data in a "rectangle" consisting of a range of keys and
42 : /// range of LSNs.
43 : ///
44 : /// There are two kinds of layers, in-memory and on-disk layers. In-memory
45 : /// layers are used to ingest incoming WAL, and provide fast access to the
46 : /// recent page versions. On-disk layers are stored as files on disk, and are
47 : /// immutable. This type represents the on-disk kind while in-memory kind are represented by
48 : /// [`InMemoryLayer`].
49 : ///
50 : /// Furthermore, there are two kinds of on-disk layers: delta and image layers.
51 : /// A delta layer contains all modifications within a range of LSNs and keys.
52 : /// An image layer is a snapshot of all the data in a key-range, at a single
53 : /// LSN.
54 : ///
55 : /// This type models the on-disk layers, which can be evicted and on-demand downloaded. As a
56 : /// general goal, read accesses should always win eviction and eviction should not wait for
57 : /// download.
58 : ///
59 : /// ### State transitions
60 : ///
61 : /// The internal state of `Layer` is composed of most importantly the on-filesystem state and the
62 : /// [`ResidentOrWantedEvicted`] enum. On-filesystem state can be either present (fully downloaded,
63 : /// right size) or deleted.
64 : ///
65 : /// Reads will always win requests to evict until `wait_for_turn_and_evict` has acquired the
66 : /// `heavier_once_cell::InitPermit` and has started to `evict_blocking`. Before the
67 : /// `heavier_once_cell::InitPermit` has been acquired, any read request
68 : /// (`get_or_maybe_download`) can "re-initialize" using the existing downloaded file and thus
69 : /// cancelling the eviction.
70 : ///
71 : /// ```text
72 : /// +-----------------+ get_or_maybe_download +--------------------------------+
73 : /// | not initialized |--------------------------->| Resident(Arc<DownloadedLayer>) |
74 : /// | ENOENT | /->| |
75 : /// +-----------------+ | +--------------------------------+
76 : /// ^ | | ^
77 : /// | get_or_maybe_download | | | get_or_maybe_download, either:
78 : /// evict_blocking | /-------------------------/ | | - upgrade weak to strong
79 : /// | | | | - re-initialize without download
80 : /// | | evict_and_wait | |
81 : /// +-----------------+ v |
82 : /// | not initialized | on_downloaded_layer_drop +--------------------------------------+
83 : /// | file is present |<---------------------------| WantedEvicted(Weak<DownloadedLayer>) |
84 : /// +-----------------+ +--------------------------------------+
85 : /// ```
86 : ///
87 : /// ### Unsupported
88 : ///
89 : /// - Evicting by the operator deleting files from the filesystem
90 : ///
91 : /// [`InMemoryLayer`]: super::inmemory_layer::InMemoryLayer
92 : #[derive(Clone)]
93 : pub(crate) struct Layer(Arc<LayerInner>);
94 :
95 : impl std::fmt::Display for Layer {
96 1102 : fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
97 1102 : write!(
98 1102 : f,
99 1102 : "{}{}",
100 1102 : self.layer_desc().short_id(),
101 1102 : self.0.generation.get_suffix()
102 : )
103 1102 : }
104 : }
105 :
106 : impl std::fmt::Debug for Layer {
107 2 : fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
108 2 : write!(f, "{self}")
109 2 : }
110 : }
111 :
112 : impl AsLayerDesc for Layer {
113 1329037 : fn layer_desc(&self) -> &PersistentLayerDesc {
114 1329037 : self.0.layer_desc()
115 1329037 : }
116 : }
117 :
118 : impl PartialEq for Layer {
119 1 : fn eq(&self, other: &Self) -> bool {
120 1 : Arc::as_ptr(&self.0) == Arc::as_ptr(&other.0)
121 1 : }
122 : }
123 :
124 982 : pub(crate) fn local_layer_path(
125 982 : conf: &PageServerConf,
126 982 : tenant_shard_id: &TenantShardId,
127 982 : timeline_id: &TimelineId,
128 982 : layer_file_name: &LayerName,
129 982 : generation: &Generation,
130 982 : ) -> Utf8PathBuf {
131 982 : let timeline_path = conf.timeline_path(tenant_shard_id, timeline_id);
132 :
133 982 : if generation.is_none() {
134 : // Without a generation, we may only use legacy path style
135 0 : timeline_path.join(layer_file_name.to_string())
136 : } else {
137 982 : timeline_path.join(format!("{}-v1{}", layer_file_name, generation.get_suffix()))
138 : }
139 982 : }
140 :
141 : pub(crate) enum LastEviction {
142 : Never,
143 : At(std::time::Instant),
144 : Evicting,
145 : }
146 :
147 : impl LastEviction {
148 13 : pub(crate) fn happened_after(&self, timepoint: std::time::Instant) -> bool {
149 13 : match self {
150 0 : LastEviction::Never => false,
151 13 : LastEviction::At(evicted_at) => evicted_at > &timepoint,
152 0 : LastEviction::Evicting => true,
153 : }
154 13 : }
155 : }
156 :
157 : impl Layer {
158 : /// Creates a layer value for a file we know to not be resident.
159 0 : pub(crate) fn for_evicted(
160 0 : conf: &'static PageServerConf,
161 0 : timeline: &Arc<Timeline>,
162 0 : file_name: LayerName,
163 0 : metadata: LayerFileMetadata,
164 0 : ) -> Self {
165 0 : let local_path = local_layer_path(
166 0 : conf,
167 0 : &timeline.tenant_shard_id,
168 0 : &timeline.timeline_id,
169 0 : &file_name,
170 0 : &metadata.generation,
171 : );
172 :
173 0 : let desc = PersistentLayerDesc::from_filename(
174 0 : timeline.tenant_shard_id,
175 0 : timeline.timeline_id,
176 0 : file_name,
177 0 : metadata.file_size,
178 : );
179 :
180 0 : let owner = Layer(Arc::new(LayerInner::new(
181 0 : conf,
182 0 : timeline,
183 0 : local_path,
184 0 : desc,
185 0 : None,
186 0 : metadata.generation,
187 0 : metadata.shard,
188 0 : )));
189 :
190 0 : debug_assert!(owner.0.needs_download_blocking().unwrap().is_some());
191 :
192 0 : owner
193 0 : }
194 :
195 : /// Creates a Layer value for a file we know to be resident in timeline directory.
196 61 : pub(crate) fn for_resident(
197 61 : conf: &'static PageServerConf,
198 61 : timeline: &Arc<Timeline>,
199 61 : local_path: Utf8PathBuf,
200 61 : file_name: LayerName,
201 61 : metadata: LayerFileMetadata,
202 61 : ) -> ResidentLayer {
203 61 : let desc = PersistentLayerDesc::from_filename(
204 61 : timeline.tenant_shard_id,
205 61 : timeline.timeline_id,
206 61 : file_name,
207 61 : metadata.file_size,
208 : );
209 :
210 61 : let mut resident = None;
211 :
212 61 : let owner = Layer(Arc::new_cyclic(|owner| {
213 61 : let inner = Arc::new(DownloadedLayer {
214 61 : owner: owner.clone(),
215 61 : kind: tokio::sync::OnceCell::default(),
216 61 : version: 0,
217 61 : });
218 61 : resident = Some(inner.clone());
219 :
220 61 : LayerInner::new(
221 61 : conf,
222 61 : timeline,
223 61 : local_path,
224 61 : desc,
225 61 : Some(inner),
226 61 : metadata.generation,
227 61 : metadata.shard,
228 : )
229 61 : }));
230 :
231 61 : let downloaded = resident.expect("just initialized");
232 :
233 61 : debug_assert!(owner.0.needs_download_blocking().unwrap().is_none());
234 :
235 61 : timeline
236 61 : .metrics
237 61 : .resident_physical_size_add(metadata.file_size);
238 :
239 61 : ResidentLayer { downloaded, owner }
240 61 : }
241 :
242 : /// Creates a Layer value for freshly written out new layer file by renaming it from a
243 : /// temporary path.
244 929 : pub(crate) fn finish_creating(
245 929 : conf: &'static PageServerConf,
246 929 : timeline: &Arc<Timeline>,
247 929 : desc: PersistentLayerDesc,
248 929 : temp_path: &Utf8Path,
249 929 : ) -> anyhow::Result<ResidentLayer> {
250 929 : let mut resident = None;
251 :
252 929 : let owner = Layer(Arc::new_cyclic(|owner| {
253 929 : let inner = Arc::new(DownloadedLayer {
254 929 : owner: owner.clone(),
255 929 : kind: tokio::sync::OnceCell::default(),
256 929 : version: 0,
257 929 : });
258 929 : resident = Some(inner.clone());
259 :
260 929 : let local_path = local_layer_path(
261 929 : conf,
262 929 : &timeline.tenant_shard_id,
263 929 : &timeline.timeline_id,
264 929 : &desc.layer_name(),
265 929 : &timeline.generation,
266 : );
267 :
268 929 : LayerInner::new(
269 929 : conf,
270 929 : timeline,
271 929 : local_path,
272 929 : desc,
273 929 : Some(inner),
274 929 : timeline.generation,
275 929 : timeline.get_shard_index(),
276 : )
277 929 : }));
278 :
279 929 : let downloaded = resident.expect("just initialized");
280 :
281 : // We never want to overwrite an existing file, so we use `RENAME_NOREPLACE`.
282 : // TODO: this leaves the temp file in place if the rename fails, risking us running
283 : // out of space. Should we clean it up here or does the calling context deal with this?
284 929 : utils::fs_ext::rename_noreplace(temp_path.as_std_path(), owner.local_path().as_std_path())
285 929 : .with_context(|| format!("rename temporary file as correct path for {owner}"))?;
286 :
287 929 : Ok(ResidentLayer { downloaded, owner })
288 929 : }
289 :
290 : /// Requests the layer to be evicted and waits for this to be done.
291 : ///
292 : /// If the file is not resident, an [`EvictionError::NotFound`] is returned.
293 : ///
294 : /// If for a bad luck or blocking of the executor, we miss the actual eviction and the layer is
295 : /// re-downloaded, [`EvictionError::Downloaded`] is returned.
296 : ///
297 : /// Timeout is mandatory, because waiting for eviction is only needed for our tests; eviction
298 : /// will happen regardless the future returned by this method completing unless there is a
299 : /// read access before eviction gets to complete.
300 : ///
301 : /// Technically cancellation safe, but cancelling might shift the viewpoint of what generation
302 : /// of download-evict cycle on retry.
303 26 : pub(crate) async fn evict_and_wait(&self, timeout: Duration) -> Result<(), EvictionError> {
304 26 : self.0.evict_and_wait(timeout).await
305 24 : }
306 :
307 : /// Delete the layer file when the `self` gets dropped, also try to schedule a remote index upload
308 : /// then.
309 : ///
310 : /// On drop, this will cause a call to [`crate::tenant::remote_timeline_client::RemoteTimelineClient::schedule_deletion_of_unlinked`].
311 : /// This means that the unlinking by [gc] or [compaction] must have happened strictly before
312 : /// the value this is called on gets dropped.
313 : ///
314 : /// This is ensured by both of those methods accepting references to Layer.
315 : ///
316 : /// [gc]: [`RemoteTimelineClient::schedule_gc_update`]
317 : /// [compaction]: [`RemoteTimelineClient::schedule_compaction_update`]
318 263 : pub(crate) fn delete_on_drop(&self) {
319 263 : self.0.delete_on_drop();
320 263 : }
321 :
322 138851 : pub(crate) async fn get_values_reconstruct_data(
323 138851 : &self,
324 138851 : keyspace: KeySpace,
325 138851 : lsn_range: Range<Lsn>,
326 138851 : reconstruct_data: &mut ValuesReconstructState,
327 138851 : ctx: &RequestContext,
328 138851 : ) -> Result<(), GetVectoredError> {
329 138851 : let downloaded = {
330 138851 : let ctx = RequestContextBuilder::from(ctx)
331 138851 : .perf_span(|crnt_perf_span| {
332 0 : info_span!(
333 : target: PERF_TRACE_TARGET,
334 0 : parent: crnt_perf_span,
335 : "GET_LAYER",
336 : )
337 0 : })
338 138851 : .attached_child();
339 :
340 138851 : self.0
341 138851 : .get_or_maybe_download(true, &ctx)
342 138851 : .maybe_perf_instrument(&ctx, |crnt_perf_context| crnt_perf_context.clone())
343 138851 : .await
344 138851 : .map_err(|err| match err {
345 : DownloadError::TimelineShutdown | DownloadError::DownloadCancelled => {
346 0 : GetVectoredError::Cancelled
347 : }
348 0 : other => GetVectoredError::Other(anyhow::anyhow!(other)),
349 0 : })?
350 : };
351 :
352 138851 : let this = ResidentLayer {
353 138851 : downloaded: downloaded.clone(),
354 138851 : owner: self.clone(),
355 138851 : };
356 :
357 138851 : self.record_access(ctx);
358 :
359 138851 : let ctx = RequestContextBuilder::from(ctx)
360 138851 : .perf_span(|crnt_perf_span| {
361 0 : info_span!(
362 : target: PERF_TRACE_TARGET,
363 0 : parent: crnt_perf_span,
364 : "VISIT_LAYER",
365 : )
366 0 : })
367 138851 : .attached_child();
368 :
369 138851 : downloaded
370 138851 : .get_values_reconstruct_data(this, keyspace, lsn_range, reconstruct_data, &ctx)
371 138851 : .instrument(tracing::debug_span!("get_values_reconstruct_data", layer=%self))
372 138851 : .maybe_perf_instrument(&ctx, |crnt_perf_span| crnt_perf_span.clone())
373 138851 : .await
374 138851 : .map_err(|err| match err {
375 0 : GetVectoredError::Other(err) => GetVectoredError::Other(
376 0 : err.context(format!("get_values_reconstruct_data for layer {self}")),
377 0 : ),
378 0 : err => err,
379 0 : })
380 138851 : }
381 :
382 : /// Download the layer if evicted.
383 : ///
384 : /// Will not error when the layer is already downloaded.
385 0 : pub(crate) async fn download(&self, ctx: &RequestContext) -> Result<(), DownloadError> {
386 0 : self.0.get_or_maybe_download(true, ctx).await?;
387 0 : Ok(())
388 0 : }
389 :
390 158 : pub(crate) async fn needs_download(&self) -> Result<Option<NeedsDownload>, std::io::Error> {
391 158 : self.0.needs_download().await
392 158 : }
393 :
394 : /// Assuming the layer is already downloaded, returns a guard which will prohibit eviction
395 : /// while the guard exists.
396 : ///
397 : /// Returns None if the layer is currently evicted or becoming evicted.
398 10 : pub(crate) async fn keep_resident(&self) -> Option<ResidentLayer> {
399 10 : let downloaded = self.0.inner.get().and_then(|rowe| rowe.get())?;
400 :
401 7 : Some(ResidentLayer {
402 7 : downloaded,
403 7 : owner: self.clone(),
404 7 : })
405 10 : }
406 :
407 : /// Weak indicator of is the layer resident or not. Good enough for eviction, which can deal
408 : /// with `EvictionError::NotFound`.
409 : ///
410 : /// Returns `true` if this layer might be resident, or `false`, if it most likely evicted or
411 : /// will be unless a read happens soon.
412 89 : pub(crate) fn is_likely_resident(&self) -> bool {
413 89 : self.0
414 89 : .inner
415 89 : .get()
416 89 : .map(|rowe| rowe.is_likely_resident())
417 89 : .unwrap_or(false)
418 89 : }
419 :
420 : /// Downloads if necessary and creates a guard, which will keep this layer from being evicted.
421 290 : pub(crate) async fn download_and_keep_resident(
422 290 : &self,
423 290 : ctx: &RequestContext,
424 290 : ) -> Result<ResidentLayer, DownloadError> {
425 290 : let downloaded = self.0.get_or_maybe_download(true, ctx).await?;
426 :
427 290 : Ok(ResidentLayer {
428 290 : downloaded,
429 290 : owner: self.clone(),
430 290 : })
431 290 : }
432 :
433 0 : pub(crate) fn info(&self, reset: LayerAccessStatsReset) -> HistoricLayerInfo {
434 0 : self.0.info(reset)
435 0 : }
436 :
437 17 : pub(crate) fn latest_activity(&self) -> SystemTime {
438 17 : self.0.access_stats.latest_activity()
439 17 : }
440 :
441 46 : pub(crate) fn visibility(&self) -> LayerVisibilityHint {
442 46 : self.0.access_stats.visibility()
443 46 : }
444 :
445 930 : pub(crate) fn local_path(&self) -> &Utf8Path {
446 930 : &self.0.path
447 930 : }
448 :
449 1337 : pub(crate) fn metadata(&self) -> LayerFileMetadata {
450 1337 : self.0.metadata()
451 1337 : }
452 :
453 13 : pub(crate) fn last_evicted_at(&self) -> LastEviction {
454 13 : match self.0.last_evicted_at.try_lock() {
455 13 : Ok(lock) => match *lock {
456 0 : None => LastEviction::Never,
457 13 : Some(at) => LastEviction::At(at),
458 : },
459 0 : Err(std::sync::TryLockError::WouldBlock) => LastEviction::Evicting,
460 0 : Err(std::sync::TryLockError::Poisoned(p)) => panic!("Lock poisoned: {p}"),
461 : }
462 13 : }
463 :
464 0 : pub(crate) fn get_timeline_id(&self) -> Option<TimelineId> {
465 0 : self.0
466 0 : .timeline
467 0 : .upgrade()
468 0 : .map(|timeline| timeline.timeline_id)
469 0 : }
470 :
471 : /// Traditional debug dumping facility
472 : #[allow(unused)]
473 2 : pub(crate) async fn dump(&self, verbose: bool, ctx: &RequestContext) -> anyhow::Result<()> {
474 2 : self.0.desc.dump();
475 :
476 2 : if verbose {
477 : // for now, unconditionally download everything, even if that might not be wanted.
478 2 : let l = self.0.get_or_maybe_download(true, ctx).await?;
479 2 : l.dump(&self.0, ctx).await?
480 0 : }
481 :
482 2 : Ok(())
483 2 : }
484 :
485 : /// Waits until this layer has been dropped (and if needed, local file deletion and remote
486 : /// deletion scheduling has completed).
487 : ///
488 : /// Does not start local deletion, use [`Self::delete_on_drop`] for that
489 : /// separatedly.
490 : #[cfg(any(feature = "testing", test))]
491 1 : pub(crate) fn wait_drop(&self) -> impl std::future::Future<Output = ()> + 'static {
492 1 : let mut rx = self.0.status.as_ref().unwrap().subscribe();
493 :
494 1 : async move {
495 : loop {
496 1 : if rx.changed().await.is_err() {
497 1 : break;
498 0 : }
499 : }
500 1 : }
501 1 : }
502 :
503 138851 : fn record_access(&self, ctx: &RequestContext) {
504 138851 : if self.0.access_stats.record_access(ctx) {
505 : // Visibility was modified to Visible: maybe log about this
506 0 : match ctx.task_kind() {
507 : TaskKind::CalculateSyntheticSize
508 : | TaskKind::OndemandLogicalSizeCalculation
509 : | TaskKind::GarbageCollector
510 0 : | TaskKind::MgmtRequest => {
511 0 : // This situation is expected in code paths do binary searches of the LSN space to resolve
512 0 : // an LSN to a timestamp, which happens during GC, during GC cutoff calculations in synthetic size,
513 0 : // and on-demand for certain HTTP API requests. On-demand logical size calculation is also included
514 0 : // because it is run as a sub-task of synthetic size.
515 0 : }
516 : _ => {
517 : // In all other contexts, it is unusual to do I/O involving layers which are not visible at
518 : // some branch tip, so we log the fact that we are accessing something that the visibility
519 : // calculation thought should not be visible.
520 : //
521 : // This case is legal in brief time windows: for example an in-flight getpage request can hold on to a layer object
522 : // which was covered by a concurrent compaction.
523 0 : tracing::info!(
524 : layer=%self,
525 0 : "became visible as a result of access",
526 : );
527 : }
528 : }
529 :
530 : // Update the timeline's visible bytes count
531 0 : if let Some(tl) = self.0.timeline.upgrade() {
532 0 : tl.metrics
533 0 : .visible_physical_size_gauge
534 0 : .add(self.0.desc.file_size)
535 0 : }
536 138851 : }
537 138851 : }
538 :
539 190 : pub(crate) fn set_visibility(&self, visibility: LayerVisibilityHint) {
540 190 : let old_visibility = self.0.access_stats.set_visibility(visibility.clone());
541 : use LayerVisibilityHint::*;
542 190 : match (old_visibility, visibility) {
543 : (Visible, Covered) => {
544 : // Subtract this layer's contribution to the visible size metric
545 14 : if let Some(tl) = self.0.timeline.upgrade() {
546 14 : debug_assert!(
547 14 : tl.metrics.visible_physical_size_gauge.get() >= self.0.desc.file_size
548 : );
549 14 : tl.metrics
550 14 : .visible_physical_size_gauge
551 14 : .sub(self.0.desc.file_size)
552 0 : }
553 : }
554 : (Covered, Visible) => {
555 : // Add this layer's contribution to the visible size metric
556 0 : if let Some(tl) = self.0.timeline.upgrade() {
557 0 : tl.metrics
558 0 : .visible_physical_size_gauge
559 0 : .add(self.0.desc.file_size)
560 0 : }
561 : }
562 176 : (Covered, Covered) | (Visible, Visible) => {
563 176 : // no change
564 176 : }
565 : }
566 190 : }
567 : }
568 :
569 : /// The download-ness ([`DownloadedLayer`]) can be either resident or wanted evicted.
570 : ///
571 : /// However when we want something evicted, we cannot evict it right away as there might be current
572 : /// reads happening on it. For example: it has been searched from [`LayerMap::search`] but not yet
573 : /// read with [`Layer::get_values_reconstruct_data`].
574 : ///
575 : /// [`LayerMap::search`]: crate::tenant::layer_map::LayerMap::search
576 : #[derive(Debug)]
577 : enum ResidentOrWantedEvicted {
578 : Resident(Arc<DownloadedLayer>),
579 : WantedEvicted(Weak<DownloadedLayer>, usize),
580 : }
581 :
582 : impl ResidentOrWantedEvicted {
583 : /// Non-mutating access to the a DownloadedLayer, if possible.
584 : ///
585 : /// This is not used on the read path (anything that calls
586 : /// [`LayerInner::get_or_maybe_download`]) because it was decided that reads always win
587 : /// evictions, and part of that winning is using [`ResidentOrWantedEvicted::get_and_upgrade`].
588 7 : fn get(&self) -> Option<Arc<DownloadedLayer>> {
589 7 : match self {
590 7 : ResidentOrWantedEvicted::Resident(strong) => Some(strong.clone()),
591 0 : ResidentOrWantedEvicted::WantedEvicted(weak, _) => weak.upgrade(),
592 : }
593 7 : }
594 :
595 : /// Best-effort query for residency right now, not as strong guarantee as receiving a strong
596 : /// reference from `ResidentOrWantedEvicted::get`.
597 55 : fn is_likely_resident(&self) -> bool {
598 55 : match self {
599 52 : ResidentOrWantedEvicted::Resident(_) => true,
600 3 : ResidentOrWantedEvicted::WantedEvicted(weak, _) => weak.strong_count() > 0,
601 : }
602 55 : }
603 :
604 : /// Upgrades any weak to strong if possible.
605 : ///
606 : /// Returns a strong reference if possible, along with a boolean telling if an upgrade
607 : /// happened.
608 139141 : fn get_and_upgrade(&mut self) -> Option<(Arc<DownloadedLayer>, bool)> {
609 139141 : match self {
610 139137 : ResidentOrWantedEvicted::Resident(strong) => Some((strong.clone(), false)),
611 4 : ResidentOrWantedEvicted::WantedEvicted(weak, _) => match weak.upgrade() {
612 0 : Some(strong) => {
613 0 : LAYER_IMPL_METRICS.inc_raced_wanted_evicted_accesses();
614 :
615 0 : *self = ResidentOrWantedEvicted::Resident(strong.clone());
616 :
617 0 : Some((strong, true))
618 : }
619 4 : None => None,
620 : },
621 : }
622 139141 : }
623 :
624 : /// When eviction is first requested, drop down to holding a [`Weak`].
625 : ///
626 : /// Returns `Some` if this was the first time eviction was requested. Care should be taken to
627 : /// drop the possibly last strong reference outside of the mutex of
628 : /// [`heavier_once_cell::OnceCell`].
629 23 : fn downgrade(&mut self) -> Option<Arc<DownloadedLayer>> {
630 23 : match self {
631 21 : ResidentOrWantedEvicted::Resident(strong) => {
632 21 : let weak = Arc::downgrade(strong);
633 21 : let mut temp = ResidentOrWantedEvicted::WantedEvicted(weak, strong.version);
634 21 : std::mem::swap(self, &mut temp);
635 21 : match temp {
636 21 : ResidentOrWantedEvicted::Resident(strong) => Some(strong),
637 0 : ResidentOrWantedEvicted::WantedEvicted(..) => unreachable!("just swapped"),
638 : }
639 : }
640 2 : ResidentOrWantedEvicted::WantedEvicted(..) => None,
641 : }
642 23 : }
643 : }
644 :
645 : struct LayerInner {
646 : /// Only needed to check ondemand_download_behavior_treat_error_as_warn and creation of
647 : /// [`Self::path`].
648 : conf: &'static PageServerConf,
649 :
650 : /// Full path to the file; unclear if this should exist anymore.
651 : path: Utf8PathBuf,
652 :
653 : desc: PersistentLayerDesc,
654 :
655 : /// Timeline access is needed for remote timeline client and metrics.
656 : ///
657 : /// There should not be an access to timeline for any reason without entering the
658 : /// [`Timeline::gate`] at the same time.
659 : timeline: Weak<Timeline>,
660 :
661 : access_stats: LayerAccessStats,
662 :
663 : /// This custom OnceCell is backed by std mutex, but only held for short time periods.
664 : ///
665 : /// Filesystem changes (download, evict) are only done while holding a permit which the
666 : /// `heavier_once_cell` provides.
667 : ///
668 : /// A number of fields in `Layer` are meant to only be updated when holding the InitPermit, but
669 : /// possibly read while not holding it.
670 : inner: heavier_once_cell::OnceCell<ResidentOrWantedEvicted>,
671 :
672 : /// Do we want to delete locally and remotely this when `LayerInner` is dropped
673 : wanted_deleted: AtomicBool,
674 :
675 : /// Version is to make sure we will only evict a specific initialization of the downloaded file.
676 : ///
677 : /// Incremented for each initialization, stored in `DownloadedLayer::version` or
678 : /// `ResidentOrWantedEvicted::WantedEvicted`.
679 : version: AtomicUsize,
680 :
681 : /// Allow subscribing to when the layer actually gets evicted, a non-cancellable download
682 : /// starts, or completes.
683 : ///
684 : /// Updates must only be posted while holding the InitPermit or the heavier_once_cell::Guard.
685 : /// Holding the InitPermit is the only time we can do state transitions, but we also need to
686 : /// cancel a pending eviction on upgrading a [`ResidentOrWantedEvicted::WantedEvicted`] back to
687 : /// [`ResidentOrWantedEvicted::Resident`] on access.
688 : ///
689 : /// The sender is wrapped in an Option to facilitate moving it out on [`LayerInner::drop`].
690 : status: Option<tokio::sync::watch::Sender<Status>>,
691 :
692 : /// Counter for exponential backoff with the download.
693 : ///
694 : /// This is atomic only for the purposes of having additional data only accessed while holding
695 : /// the InitPermit.
696 : consecutive_failures: AtomicUsize,
697 :
698 : /// The generation of this Layer.
699 : ///
700 : /// For loaded layers (resident or evicted) this comes from [`LayerFileMetadata::generation`],
701 : /// for created layers from [`Timeline::generation`].
702 : generation: Generation,
703 :
704 : /// The shard of this Layer.
705 : ///
706 : /// For layers created in this process, this will always be the [`ShardIndex`] of the
707 : /// current `ShardIdentity`` (TODO: add link once it's introduced).
708 : ///
709 : /// For loaded layers, this may be some other value if the tenant has undergone
710 : /// a shard split since the layer was originally written.
711 : shard: ShardIndex,
712 :
713 : /// When the Layer was last evicted but has not been downloaded since.
714 : ///
715 : /// This is used for skipping evicted layers from the previous heatmap (see
716 : /// `[Timeline::generate_heatmap]`) and for updating metrics
717 : /// (see [`LayerImplMetrics::redownload_after`]).
718 : last_evicted_at: std::sync::Mutex<Option<std::time::Instant>>,
719 :
720 : #[cfg(test)]
721 : failpoints: std::sync::Mutex<Vec<failpoints::Failpoint>>,
722 : }
723 :
724 : impl std::fmt::Display for LayerInner {
725 27 : fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
726 27 : write!(f, "{}", self.layer_desc().short_id())
727 27 : }
728 : }
729 :
730 : impl AsLayerDesc for LayerInner {
731 1330369 : fn layer_desc(&self) -> &PersistentLayerDesc {
732 1330369 : &self.desc
733 1330369 : }
734 : }
735 :
736 : #[derive(Debug, Clone, Copy)]
737 : enum Status {
738 : Resident,
739 : Evicted,
740 : Downloading,
741 : }
742 :
743 : impl Drop for LayerInner {
744 360 : fn drop(&mut self) {
745 : // if there was a pending eviction, mark it cancelled here to balance metrics
746 360 : if let Some((ResidentOrWantedEvicted::WantedEvicted(..), _)) = self.inner.take_and_deinit()
747 1 : {
748 1 : // eviction has already been started
749 1 : LAYER_IMPL_METRICS.inc_eviction_cancelled(EvictionCancelled::LayerGone);
750 1 :
751 1 : // eviction request is intentionally not honored as no one is present to wait for it
752 1 : // and we could be delaying shutdown for nothing.
753 359 : }
754 :
755 360 : let timeline = self.timeline.upgrade();
756 :
757 360 : if let Some(timeline) = timeline.as_ref() {
758 : // Only need to decrement metrics if the timeline still exists: otherwise
759 : // it will have already de-registered these metrics via TimelineMetrics::shutdown
760 352 : timeline.metrics.dec_layer(&self.desc);
761 :
762 352 : if matches!(self.access_stats.visibility(), LayerVisibilityHint::Visible) {
763 352 : debug_assert!(
764 352 : timeline.metrics.visible_physical_size_gauge.get() >= self.desc.file_size
765 : );
766 352 : timeline
767 352 : .metrics
768 352 : .visible_physical_size_gauge
769 352 : .sub(self.desc.file_size);
770 0 : }
771 8 : }
772 :
773 360 : if !*self.wanted_deleted.get_mut() {
774 99 : return;
775 261 : }
776 :
777 261 : let span = tracing::info_span!(parent: None, "layer_delete", tenant_id = %self.layer_desc().tenant_shard_id.tenant_id, shard_id=%self.layer_desc().tenant_shard_id.shard_slug(), timeline_id = %self.layer_desc().timeline_id);
778 :
779 261 : let path = std::mem::take(&mut self.path);
780 261 : let file_name = self.layer_desc().layer_name();
781 261 : let file_size = self.layer_desc().file_size;
782 261 : let meta = self.metadata();
783 261 : let status = self.status.take();
784 :
785 261 : Self::spawn_blocking(move || {
786 261 : let _g = span.entered();
787 :
788 : // carry this until we are finished for [`Layer::wait_drop`] support
789 261 : let _status = status;
790 :
791 261 : let Some(timeline) = timeline else {
792 : // no need to nag that timeline is gone: under normal situation on
793 : // task_mgr::remove_tenant_from_memory the timeline is gone before we get dropped.
794 0 : LAYER_IMPL_METRICS.inc_deletes_failed(DeleteFailed::TimelineGone);
795 0 : return;
796 : };
797 :
798 261 : let Ok(_guard) = timeline.gate.enter() else {
799 0 : LAYER_IMPL_METRICS.inc_deletes_failed(DeleteFailed::TimelineGone);
800 0 : return;
801 : };
802 :
803 261 : let removed = match std::fs::remove_file(path) {
804 260 : Ok(()) => true,
805 1 : Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
806 : // until we no longer do detaches by removing all local files before removing the
807 : // tenant from the global map, we will always get these errors even if we knew what
808 : // is the latest state.
809 : //
810 : // we currently do not track the latest state, so we'll also end up here on evicted
811 : // layers.
812 1 : false
813 : }
814 0 : Err(e) => {
815 0 : tracing::error!("failed to remove wanted deleted layer: {e}");
816 0 : LAYER_IMPL_METRICS.inc_delete_removes_failed();
817 0 : false
818 : }
819 : };
820 :
821 261 : if removed {
822 260 : timeline.metrics.resident_physical_size_sub(file_size);
823 260 : }
824 261 : let res = timeline
825 261 : .remote_client
826 261 : .schedule_deletion_of_unlinked(vec![(file_name, meta)]);
827 :
828 261 : if let Err(e) = res {
829 : // test_timeline_deletion_with_files_stuck_in_upload_queue is good at
830 : // demonstrating this deadlock (without spawn_blocking): stop will drop
831 : // queued items, which will have ResidentLayer's, and those drops would try
832 : // to re-entrantly lock the RemoteTimelineClient inner state.
833 1 : if !timeline.is_active() {
834 1 : tracing::info!("scheduling deletion on drop failed: {e:#}");
835 : } else {
836 0 : tracing::warn!("scheduling deletion on drop failed: {e:#}");
837 : }
838 1 : LAYER_IMPL_METRICS.inc_deletes_failed(DeleteFailed::DeleteSchedulingFailed);
839 260 : } else {
840 260 : LAYER_IMPL_METRICS.inc_completed_deletes();
841 260 : }
842 261 : });
843 360 : }
844 : }
845 :
846 : impl LayerInner {
847 : #[allow(clippy::too_many_arguments)]
848 990 : fn new(
849 990 : conf: &'static PageServerConf,
850 990 : timeline: &Arc<Timeline>,
851 990 : local_path: Utf8PathBuf,
852 990 : desc: PersistentLayerDesc,
853 990 : downloaded: Option<Arc<DownloadedLayer>>,
854 990 : generation: Generation,
855 990 : shard: ShardIndex,
856 990 : ) -> Self {
857 990 : let (inner, version, init_status) = if let Some(inner) = downloaded {
858 990 : let version = inner.version;
859 990 : let resident = ResidentOrWantedEvicted::Resident(inner);
860 990 : (
861 990 : heavier_once_cell::OnceCell::new(resident),
862 990 : version,
863 990 : Status::Resident,
864 990 : )
865 : } else {
866 0 : (heavier_once_cell::OnceCell::default(), 0, Status::Evicted)
867 : };
868 :
869 : // This object acts as a RAII guard on these metrics: increment on construction
870 990 : timeline.metrics.inc_layer(&desc);
871 :
872 : // New layers are visible by default. This metric is later updated on drop or in set_visibility
873 990 : timeline
874 990 : .metrics
875 990 : .visible_physical_size_gauge
876 990 : .add(desc.file_size);
877 :
878 990 : LayerInner {
879 990 : conf,
880 990 : path: local_path,
881 990 : desc,
882 990 : timeline: Arc::downgrade(timeline),
883 990 : access_stats: Default::default(),
884 990 : wanted_deleted: AtomicBool::new(false),
885 990 : inner,
886 990 : version: AtomicUsize::new(version),
887 990 : status: Some(tokio::sync::watch::channel(init_status).0),
888 990 : consecutive_failures: AtomicUsize::new(0),
889 990 : generation,
890 990 : shard,
891 990 : last_evicted_at: std::sync::Mutex::default(),
892 990 : #[cfg(test)]
893 990 : failpoints: Default::default(),
894 990 : }
895 990 : }
896 :
897 263 : fn delete_on_drop(&self) {
898 263 : let res =
899 263 : self.wanted_deleted
900 263 : .compare_exchange(false, true, Ordering::Release, Ordering::Relaxed);
901 :
902 263 : if res.is_ok() {
903 261 : LAYER_IMPL_METRICS.inc_started_deletes();
904 261 : }
905 263 : }
906 :
907 : /// Cancellation safe, however dropping the future and calling this method again might result
908 : /// in a new attempt to evict OR join the previously started attempt.
909 26 : #[tracing::instrument(level = tracing::Level::DEBUG, skip_all, ret, err(level = tracing::Level::DEBUG), fields(layer=%self))]
910 : pub(crate) async fn evict_and_wait(&self, timeout: Duration) -> Result<(), EvictionError> {
911 : let mut rx = self.status.as_ref().unwrap().subscribe();
912 :
913 : {
914 : let current = rx.borrow_and_update();
915 : match &*current {
916 : Status::Resident => {
917 : // we might get lucky and evict this; continue
918 : }
919 : Status::Evicted | Status::Downloading => {
920 : // it is already evicted
921 : return Err(EvictionError::NotFound);
922 : }
923 : }
924 : }
925 :
926 : let strong = {
927 : match self.inner.get() {
928 : Some(mut either) => either.downgrade(),
929 : None => {
930 : // we already have a scheduled eviction, which just has not gotten to run yet.
931 : // it might still race with a read access, but that could also get cancelled,
932 : // so let's say this is not evictable.
933 : return Err(EvictionError::NotFound);
934 : }
935 : }
936 : };
937 :
938 : if strong.is_some() {
939 : // drop the DownloadedLayer outside of the holding the guard
940 : drop(strong);
941 :
942 : // idea here is that only one evicter should ever get to witness a strong reference,
943 : // which means whenever get_or_maybe_download upgrades a weak, it must mark up a
944 : // cancelled eviction and signal us, like it currently does.
945 : //
946 : // a second concurrent evict_and_wait will not see a strong reference.
947 : LAYER_IMPL_METRICS.inc_started_evictions();
948 : }
949 :
950 : let changed = rx.changed();
951 : let changed = tokio::time::timeout(timeout, changed).await;
952 :
953 : let Ok(changed) = changed else {
954 : return Err(EvictionError::Timeout);
955 : };
956 :
957 : let _: () = changed.expect("cannot be closed, because we are holding a strong reference");
958 :
959 : let current = rx.borrow_and_update();
960 :
961 : match &*current {
962 : // the easiest case
963 : Status::Evicted => Ok(()),
964 : // it surely was evicted in between, but then there was a new access now; we can't know
965 : // if it'll succeed so lets just call it evicted
966 : Status::Downloading => Ok(()),
967 : // either the download which was started after eviction completed already, or it was
968 : // never evicted
969 : Status::Resident => Err(EvictionError::Downloaded),
970 : }
971 : }
972 :
973 : /// Cancellation safe.
974 139149 : async fn get_or_maybe_download(
975 139149 : self: &Arc<Self>,
976 139149 : allow_download: bool,
977 139149 : ctx: &RequestContext,
978 139149 : ) -> Result<Arc<DownloadedLayer>, DownloadError> {
979 139149 : let mut wait_for_download_recorder =
980 139149 : scopeguard::guard(utils::elapsed_accum::ElapsedAccum::default(), |accum| {
981 139149 : ctx.ondemand_download_wait_observe(accum.get());
982 139149 : });
983 12 : let (weak, permit) = {
984 : // get_or_init_detached can:
985 : // - be fast (mutex lock) OR uncontested semaphore permit acquire
986 : // - be slow (wait for semaphore permit or closing)
987 139149 : let init_cancelled = scopeguard::guard((), |_| LAYER_IMPL_METRICS.inc_init_cancelled());
988 :
989 139149 : let locked = self
990 139149 : .inner
991 139149 : .get_or_init_detached_measured(Some(&mut wait_for_download_recorder))
992 139149 : .await
993 139149 : .map(|mut guard| guard.get_and_upgrade().ok_or(guard));
994 :
995 139149 : scopeguard::ScopeGuard::into_inner(init_cancelled);
996 :
997 139137 : match locked {
998 : // this path could had been a RwLock::read
999 139137 : Ok(Ok((strong, upgraded))) if !upgraded => return Ok(strong),
1000 0 : Ok(Ok((strong, _))) => {
1001 : // when upgraded back, the Arc<DownloadedLayer> is still available, but
1002 : // previously a `evict_and_wait` was received. this is the only place when we
1003 : // send out an update without holding the InitPermit.
1004 : //
1005 : // note that we also have dropped the Guard; this is fine, because we just made
1006 : // a state change and are holding a strong reference to be returned.
1007 0 : self.status.as_ref().unwrap().send_replace(Status::Resident);
1008 0 : LAYER_IMPL_METRICS
1009 0 : .inc_eviction_cancelled(EvictionCancelled::UpgradedBackOnAccess);
1010 :
1011 0 : return Ok(strong);
1012 : }
1013 4 : Ok(Err(guard)) => {
1014 : // path to here: we won the eviction, the file should still be on the disk.
1015 4 : let (weak, permit) = guard.take_and_deinit();
1016 4 : (Some(weak), permit)
1017 : }
1018 8 : Err(permit) => (None, permit),
1019 : }
1020 : };
1021 12 : let _guard = wait_for_download_recorder.guard();
1022 :
1023 12 : if let Some(weak) = weak {
1024 : // only drop the weak after dropping the heavier_once_cell guard
1025 4 : assert!(
1026 4 : matches!(weak, ResidentOrWantedEvicted::WantedEvicted(..)),
1027 0 : "unexpected {weak:?}, ResidentOrWantedEvicted::get_and_upgrade has a bug"
1028 : );
1029 8 : }
1030 :
1031 12 : let timeline = self
1032 12 : .timeline
1033 12 : .upgrade()
1034 12 : .ok_or(DownloadError::TimelineShutdown)?;
1035 :
1036 : // count cancellations, which currently remain largely unexpected
1037 12 : let init_cancelled = scopeguard::guard((), |_| LAYER_IMPL_METRICS.inc_init_cancelled());
1038 :
1039 : // check if we really need to be downloaded: this can happen if a read access won the
1040 : // semaphore before eviction.
1041 : //
1042 : // if we are cancelled while doing this `stat` the `self.inner` will be uninitialized. a
1043 : // pending eviction will try to evict even upon finding an uninitialized `self.inner`.
1044 12 : let needs_download = self
1045 12 : .needs_download()
1046 12 : .await
1047 12 : .map_err(DownloadError::PreStatFailed);
1048 :
1049 12 : scopeguard::ScopeGuard::into_inner(init_cancelled);
1050 :
1051 12 : let needs_download = needs_download?;
1052 :
1053 12 : let Some(reason) = needs_download else {
1054 : // the file is present locally because eviction has not had a chance to run yet
1055 :
1056 : #[cfg(test)]
1057 4 : self.failpoint(failpoints::FailpointKind::AfterDeterminingLayerNeedsNoDownload)
1058 4 : .await?;
1059 :
1060 3 : LAYER_IMPL_METRICS.inc_init_needed_no_download();
1061 :
1062 3 : return Ok(self.initialize_after_layer_is_on_disk(permit));
1063 : };
1064 :
1065 : // we must download; getting cancelled before spawning the download is not an issue as
1066 : // any still running eviction would not find anything to evict.
1067 :
1068 8 : if let NeedsDownload::NotFile(ft) = reason {
1069 0 : return Err(DownloadError::NotFile(ft));
1070 8 : }
1071 :
1072 8 : self.check_expected_download(ctx)?;
1073 :
1074 8 : if !allow_download {
1075 : // this is only used from tests, but it is hard to test without the boolean
1076 1 : return Err(DownloadError::DownloadRequired);
1077 7 : }
1078 :
1079 7 : let ctx = RequestContextBuilder::from(ctx)
1080 7 : .perf_span(|crnt_perf_span| {
1081 0 : info_span!(
1082 : target: PERF_TRACE_TARGET,
1083 0 : parent: crnt_perf_span,
1084 : "DOWNLOAD_LAYER",
1085 : layer = %self,
1086 : reason = %reason,
1087 : )
1088 0 : })
1089 7 : .attached_child();
1090 :
1091 7 : async move {
1092 7 : tracing::info!(%reason, "downloading on-demand");
1093 :
1094 7 : let init_cancelled = scopeguard::guard((), |_| LAYER_IMPL_METRICS.inc_init_cancelled());
1095 7 : let res = self
1096 7 : .download_init_and_wait(timeline, permit, ctx.attached_child())
1097 7 : .maybe_perf_instrument(&ctx, |current_perf_span| current_perf_span.clone())
1098 7 : .await?;
1099 :
1100 7 : scopeguard::ScopeGuard::into_inner(init_cancelled);
1101 7 : Ok(res)
1102 7 : }
1103 7 : .instrument(tracing::info_span!("get_or_maybe_download", layer=%self))
1104 7 : .await
1105 139149 : }
1106 :
1107 : /// Nag or fail per RequestContext policy
1108 8 : fn check_expected_download(&self, ctx: &RequestContext) -> Result<(), DownloadError> {
1109 : use crate::context::DownloadBehavior::*;
1110 8 : let b = ctx.download_behavior();
1111 8 : match b {
1112 8 : Download => Ok(()),
1113 : Warn | Error => {
1114 0 : tracing::info!(
1115 0 : "unexpectedly on-demand downloading for task kind {:?}",
1116 0 : ctx.task_kind()
1117 : );
1118 0 : crate::metrics::UNEXPECTED_ONDEMAND_DOWNLOADS.inc();
1119 :
1120 0 : let really_error =
1121 0 : matches!(b, Error) && !self.conf.ondemand_download_behavior_treat_error_as_warn;
1122 :
1123 0 : if really_error {
1124 : // this check is only probablistic, seems like flakyness footgun
1125 0 : Err(DownloadError::ContextAndConfigReallyDeniesDownloads)
1126 : } else {
1127 0 : Ok(())
1128 : }
1129 : }
1130 : }
1131 8 : }
1132 :
1133 : /// Actual download, at most one is executed at the time.
1134 7 : async fn download_init_and_wait(
1135 7 : self: &Arc<Self>,
1136 7 : timeline: Arc<Timeline>,
1137 7 : permit: heavier_once_cell::InitPermit,
1138 7 : ctx: RequestContext,
1139 7 : ) -> Result<Arc<DownloadedLayer>, DownloadError> {
1140 7 : debug_assert_current_span_has_tenant_and_timeline_id();
1141 :
1142 7 : let (tx, rx) = tokio::sync::oneshot::channel();
1143 :
1144 7 : let this: Arc<Self> = self.clone();
1145 :
1146 7 : let guard = timeline
1147 7 : .gate
1148 7 : .enter()
1149 7 : .map_err(|_| DownloadError::DownloadCancelled)?;
1150 :
1151 7 : Self::spawn(
1152 7 : async move {
1153 7 : let _guard = guard;
1154 :
1155 : // now that we have commited to downloading, send out an update to:
1156 : // - unhang any pending eviction
1157 : // - break out of evict_and_wait
1158 7 : this.status
1159 7 : .as_ref()
1160 7 : .unwrap()
1161 7 : .send_replace(Status::Downloading);
1162 :
1163 : #[cfg(test)]
1164 7 : this.failpoint(failpoints::FailpointKind::WaitBeforeDownloading)
1165 7 : .await
1166 7 : .unwrap();
1167 :
1168 7 : let res = this.download_and_init(timeline, permit, &ctx).await;
1169 :
1170 7 : if let Err(res) = tx.send(res) {
1171 0 : match res {
1172 0 : Ok(_res) => {
1173 0 : tracing::debug!("layer initialized, but caller has been cancelled");
1174 0 : LAYER_IMPL_METRICS.inc_init_completed_without_requester();
1175 : }
1176 0 : Err(e) => {
1177 0 : tracing::info!(
1178 0 : "layer file download failed, and caller has been cancelled: {e:?}"
1179 : );
1180 0 : LAYER_IMPL_METRICS.inc_download_failed_without_requester();
1181 : }
1182 : }
1183 7 : }
1184 7 : }
1185 7 : .in_current_span(),
1186 : );
1187 :
1188 7 : match rx.await {
1189 7 : Ok(Ok(res)) => Ok(res),
1190 : Ok(Err(remote_storage::DownloadError::Cancelled)) => {
1191 0 : Err(DownloadError::DownloadCancelled)
1192 : }
1193 0 : Ok(Err(_)) => Err(DownloadError::DownloadFailed),
1194 0 : Err(_gone) => Err(DownloadError::DownloadCancelled),
1195 : }
1196 7 : }
1197 :
1198 7 : async fn download_and_init(
1199 7 : self: &Arc<LayerInner>,
1200 7 : timeline: Arc<Timeline>,
1201 7 : permit: heavier_once_cell::InitPermit,
1202 7 : ctx: &RequestContext,
1203 7 : ) -> Result<Arc<DownloadedLayer>, remote_storage::DownloadError> {
1204 7 : let start = std::time::Instant::now();
1205 7 : let result = timeline
1206 7 : .remote_client
1207 7 : .download_layer_file(
1208 7 : &self.desc.layer_name(),
1209 7 : &self.metadata(),
1210 7 : &self.path,
1211 7 : &timeline.gate,
1212 7 : &timeline.cancel,
1213 7 : ctx,
1214 7 : )
1215 7 : .await;
1216 7 : let latency = start.elapsed();
1217 7 : let latency_millis = u64::try_from(latency.as_millis()).unwrap();
1218 7 : match result {
1219 7 : Ok(size) => {
1220 7 : assert_eq!(size, self.desc.file_size);
1221 :
1222 7 : match self.needs_download().await {
1223 0 : Ok(Some(reason)) => {
1224 : // this is really a bug in needs_download or remote timeline client
1225 0 : panic!("post-condition failed: needs_download returned {reason:?}");
1226 : }
1227 7 : Ok(None) => {
1228 7 : // as expected
1229 7 : }
1230 0 : Err(e) => {
1231 0 : panic!("post-condition failed: needs_download errored: {e:?}");
1232 : }
1233 : };
1234 7 : tracing::info!(size=%self.desc.file_size, %latency_millis, "on-demand download successful");
1235 7 : timeline
1236 7 : .metrics
1237 7 : .resident_physical_size_add(self.desc.file_size);
1238 7 : self.consecutive_failures.store(0, Ordering::Relaxed);
1239 :
1240 7 : let since_last_eviction = self
1241 7 : .last_evicted_at
1242 7 : .lock()
1243 7 : .unwrap()
1244 7 : .take()
1245 7 : .map(|ts| ts.elapsed());
1246 7 : if let Some(since_last_eviction) = since_last_eviction {
1247 7 : LAYER_IMPL_METRICS.record_redownloaded_after(since_last_eviction);
1248 7 : }
1249 :
1250 7 : self.access_stats.record_residence_event();
1251 :
1252 7 : let task_kind: &'static str = ctx.task_kind().into();
1253 7 : ONDEMAND_DOWNLOAD_BYTES
1254 7 : .with_label_values(&[task_kind])
1255 7 : .inc_by(self.desc.file_size);
1256 7 : ONDEMAND_DOWNLOAD_COUNT
1257 7 : .with_label_values(&[task_kind])
1258 7 : .inc();
1259 :
1260 7 : Ok(self.initialize_after_layer_is_on_disk(permit))
1261 : }
1262 0 : Err(e) => {
1263 0 : let consecutive_failures =
1264 0 : 1 + self.consecutive_failures.fetch_add(1, Ordering::Relaxed);
1265 :
1266 0 : if timeline.cancel.is_cancelled() {
1267 : // If we're shutting down, drop out before logging the error
1268 0 : return Err(e);
1269 0 : }
1270 :
1271 0 : tracing::error!(consecutive_failures, %latency_millis, "layer file download failed: {e:#}");
1272 :
1273 0 : let backoff = utils::backoff::exponential_backoff_duration_seconds(
1274 0 : consecutive_failures.min(u32::MAX as usize) as u32,
1275 : 1.5,
1276 : 60.0,
1277 : );
1278 :
1279 0 : let backoff = std::time::Duration::from_secs_f64(backoff);
1280 :
1281 0 : tokio::select! {
1282 0 : _ = tokio::time::sleep(backoff) => {},
1283 0 : _ = timeline.cancel.cancelled() => {},
1284 : };
1285 :
1286 0 : Err(e)
1287 : }
1288 : }
1289 7 : }
1290 :
1291 : /// Initializes the `Self::inner` to a "resident" state.
1292 : ///
1293 : /// Callers are assumed to ensure that the file is actually on disk with `Self::needs_download`
1294 : /// before calling this method.
1295 : ///
1296 : /// If this method is ever made async, it needs to be cancellation safe so that no state
1297 : /// changes are made before we can write to the OnceCell in non-cancellable fashion.
1298 10 : fn initialize_after_layer_is_on_disk(
1299 10 : self: &Arc<LayerInner>,
1300 10 : permit: heavier_once_cell::InitPermit,
1301 10 : ) -> Arc<DownloadedLayer> {
1302 10 : debug_assert_current_span_has_tenant_and_timeline_id();
1303 :
1304 : // disable any scheduled but not yet running eviction deletions for this initialization
1305 10 : let next_version = 1 + self.version.fetch_add(1, Ordering::Relaxed);
1306 10 : self.status.as_ref().unwrap().send_replace(Status::Resident);
1307 :
1308 10 : let res = Arc::new(DownloadedLayer {
1309 10 : owner: Arc::downgrade(self),
1310 10 : kind: tokio::sync::OnceCell::default(),
1311 10 : version: next_version,
1312 10 : });
1313 :
1314 10 : let waiters = self.inner.initializer_count();
1315 10 : if waiters > 0 {
1316 0 : tracing::info!(waiters, "completing layer init for other tasks");
1317 10 : }
1318 :
1319 10 : let value = ResidentOrWantedEvicted::Resident(res.clone());
1320 :
1321 10 : self.inner.set(value, permit);
1322 :
1323 10 : res
1324 10 : }
1325 :
1326 178 : async fn needs_download(&self) -> Result<Option<NeedsDownload>, std::io::Error> {
1327 178 : match tokio::fs::metadata(&self.path).await {
1328 170 : Ok(m) => Ok(self.is_file_present_and_good_size(&m).err()),
1329 8 : Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(Some(NeedsDownload::NotFound)),
1330 0 : Err(e) => Err(e),
1331 : }
1332 178 : }
1333 :
1334 61 : fn needs_download_blocking(&self) -> Result<Option<NeedsDownload>, std::io::Error> {
1335 61 : match self.path.metadata() {
1336 61 : Ok(m) => Ok(self.is_file_present_and_good_size(&m).err()),
1337 0 : Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(Some(NeedsDownload::NotFound)),
1338 0 : Err(e) => Err(e),
1339 : }
1340 61 : }
1341 :
1342 231 : fn is_file_present_and_good_size(&self, m: &std::fs::Metadata) -> Result<(), NeedsDownload> {
1343 : // in future, this should include sha2-256 validation of the file.
1344 231 : if !m.is_file() {
1345 0 : Err(NeedsDownload::NotFile(m.file_type()))
1346 231 : } else if m.len() != self.desc.file_size {
1347 0 : Err(NeedsDownload::WrongSize {
1348 0 : actual: m.len(),
1349 0 : expected: self.desc.file_size,
1350 0 : })
1351 : } else {
1352 231 : Ok(())
1353 : }
1354 231 : }
1355 :
1356 0 : fn info(&self, reset: LayerAccessStatsReset) -> HistoricLayerInfo {
1357 0 : let layer_name = self.desc.layer_name().to_string();
1358 :
1359 0 : let resident = self
1360 0 : .inner
1361 0 : .get()
1362 0 : .map(|rowe| rowe.is_likely_resident())
1363 0 : .unwrap_or(false);
1364 :
1365 0 : let access_stats = self.access_stats.as_api_model(reset);
1366 :
1367 0 : if self.desc.is_delta {
1368 0 : let lsn_range = &self.desc.lsn_range;
1369 :
1370 0 : HistoricLayerInfo::Delta {
1371 0 : layer_file_name: layer_name,
1372 0 : layer_file_size: self.desc.file_size,
1373 0 : lsn_start: lsn_range.start,
1374 0 : lsn_end: lsn_range.end,
1375 0 : remote: !resident,
1376 0 : access_stats,
1377 0 : l0: crate::tenant::layer_map::LayerMap::is_l0(
1378 0 : &self.layer_desc().key_range,
1379 0 : self.layer_desc().is_delta,
1380 0 : ),
1381 0 : }
1382 : } else {
1383 0 : let lsn = self.desc.image_layer_lsn();
1384 :
1385 0 : HistoricLayerInfo::Image {
1386 0 : layer_file_name: layer_name,
1387 0 : layer_file_size: self.desc.file_size,
1388 0 : lsn_start: lsn,
1389 0 : remote: !resident,
1390 0 : access_stats,
1391 0 : }
1392 : }
1393 0 : }
1394 :
1395 : /// `DownloadedLayer` is being dropped, so it calls this method.
1396 20 : fn on_downloaded_layer_drop(self: Arc<LayerInner>, only_version: usize) {
1397 : // we cannot know without inspecting LayerInner::inner if we should evict or not, even
1398 : // though here it is very likely
1399 20 : let span = tracing::info_span!(parent: None, "layer_evict", tenant_id = %self.desc.tenant_shard_id.tenant_id, shard_id = %self.desc.tenant_shard_id.shard_slug(), timeline_id = %self.desc.timeline_id, layer=%self, version=%only_version);
1400 :
1401 : // NOTE: this scope *must* never call `self.inner.get` because evict_and_wait might
1402 : // drop while the `self.inner` is being locked, leading to a deadlock.
1403 :
1404 20 : let start_evicting = async move {
1405 : #[cfg(test)]
1406 20 : self.failpoint(failpoints::FailpointKind::WaitBeforeStartingEvicting)
1407 20 : .await
1408 20 : .expect("failpoint should not have errored");
1409 :
1410 20 : tracing::debug!("eviction started");
1411 :
1412 20 : let res = self.wait_for_turn_and_evict(only_version).await;
1413 : // metrics: ignore the Ok branch, it is not done yet
1414 20 : if let Err(e) = res {
1415 3 : tracing::debug!(res=?Err::<(), _>(&e), "eviction completed");
1416 3 : LAYER_IMPL_METRICS.inc_eviction_cancelled(e);
1417 17 : }
1418 20 : };
1419 :
1420 20 : Self::spawn(start_evicting.instrument(span));
1421 20 : }
1422 :
1423 20 : async fn wait_for_turn_and_evict(
1424 20 : self: Arc<LayerInner>,
1425 20 : only_version: usize,
1426 20 : ) -> Result<(), EvictionCancelled> {
1427 39 : fn is_good_to_continue(status: &Status) -> Result<(), EvictionCancelled> {
1428 : use Status::*;
1429 39 : match status {
1430 38 : Resident => Ok(()),
1431 1 : Evicted => Err(EvictionCancelled::UnexpectedEvictedState),
1432 0 : Downloading => Err(EvictionCancelled::LostToDownload),
1433 : }
1434 39 : }
1435 :
1436 20 : let timeline = self
1437 20 : .timeline
1438 20 : .upgrade()
1439 20 : .ok_or(EvictionCancelled::TimelineGone)?;
1440 :
1441 20 : let mut rx = self
1442 20 : .status
1443 20 : .as_ref()
1444 20 : .expect("LayerInner cannot be dropped, holding strong ref")
1445 20 : .subscribe();
1446 :
1447 20 : is_good_to_continue(&rx.borrow_and_update())?;
1448 :
1449 19 : let Ok(gate) = timeline.gate.enter() else {
1450 0 : return Err(EvictionCancelled::TimelineGone);
1451 : };
1452 :
1453 17 : let permit = {
1454 : // we cannot just `std::fs::remove_file` because there might already be an
1455 : // get_or_maybe_download which will inspect filesystem and reinitialize. filesystem
1456 : // operations must be done while holding the heavier_once_cell::InitPermit
1457 19 : let mut wait = std::pin::pin!(self.inner.get_or_init_detached());
1458 :
1459 19 : let waited = loop {
1460 : // we must race to the Downloading starting, otherwise we would have to wait until the
1461 : // completion of the download. waiting for download could be long and hinder our
1462 : // efforts to alert on "hanging" evictions.
1463 19 : tokio::select! {
1464 19 : res = &mut wait => break res,
1465 19 : _ = rx.changed() => {
1466 0 : is_good_to_continue(&rx.borrow_and_update())?;
1467 : // two possibilities for Status::Resident:
1468 : // - the layer was found locally from disk by a read
1469 : // - we missed a bunch of updates and now the layer is
1470 : // again downloaded -- assume we'll fail later on with
1471 : // version check or AlreadyReinitialized
1472 : }
1473 : }
1474 : };
1475 :
1476 : // re-check now that we have the guard or permit; all updates should have happened
1477 : // while holding the permit.
1478 19 : is_good_to_continue(&rx.borrow_and_update())?;
1479 :
1480 : // the term deinitialize is used here, because we clearing out the Weak will eventually
1481 : // lead to deallocating the reference counted value, and the value we
1482 : // `Guard::take_and_deinit` is likely to be the last because the Weak is never cloned.
1483 19 : let (_weak, permit) = match waited {
1484 18 : Ok(guard) => {
1485 18 : match &*guard {
1486 17 : ResidentOrWantedEvicted::WantedEvicted(_weak, version)
1487 17 : if *version == only_version =>
1488 : {
1489 16 : tracing::debug!(version, "deinitializing matching WantedEvicted");
1490 16 : let (weak, permit) = guard.take_and_deinit();
1491 16 : (Some(weak), permit)
1492 : }
1493 1 : ResidentOrWantedEvicted::WantedEvicted(_, version) => {
1494 : // if we were not doing the version check, we would need to try to
1495 : // upgrade the weak here to see if it really is dropped. version check
1496 : // is done instead assuming that it is cheaper.
1497 1 : tracing::debug!(
1498 : version,
1499 : only_version,
1500 0 : "version mismatch, not deinitializing"
1501 : );
1502 1 : return Err(EvictionCancelled::VersionCheckFailed);
1503 : }
1504 : ResidentOrWantedEvicted::Resident(_) => {
1505 1 : return Err(EvictionCancelled::AlreadyReinitialized);
1506 : }
1507 : }
1508 : }
1509 1 : Err(permit) => {
1510 1 : tracing::debug!("continuing after cancelled get_or_maybe_download or eviction");
1511 1 : (None, permit)
1512 : }
1513 : };
1514 :
1515 17 : permit
1516 : };
1517 :
1518 17 : let span = tracing::Span::current();
1519 :
1520 17 : let spawned_at = std::time::Instant::now();
1521 :
1522 : // this is on purpose a detached spawn; we don't need to wait for it
1523 : //
1524 : // eviction completion reporting is the only thing hinging on this, and it can be just as
1525 : // well from a spawn_blocking thread.
1526 : //
1527 : // important to note that now that we've acquired the permit we have made sure the evicted
1528 : // file is either the exact `WantedEvicted` we wanted to evict, or uninitialized in case
1529 : // there are multiple evictions. The rest is not cancellable, and we've now commited to
1530 : // evicting.
1531 : //
1532 : // If spawn_blocking has a queue and maximum number of threads are in use, we could stall
1533 : // reads. We will need to add cancellation for that if necessary.
1534 17 : Self::spawn_blocking(move || {
1535 17 : let _span = span.entered();
1536 :
1537 17 : let res = self.evict_blocking(&timeline, &gate, &permit);
1538 :
1539 17 : let waiters = self.inner.initializer_count();
1540 :
1541 17 : if waiters > 0 {
1542 0 : LAYER_IMPL_METRICS.inc_evicted_with_waiters();
1543 17 : }
1544 :
1545 17 : let completed_in = spawned_at.elapsed();
1546 17 : LAYER_IMPL_METRICS.record_time_to_evict(completed_in);
1547 :
1548 17 : match res {
1549 17 : Ok(()) => LAYER_IMPL_METRICS.inc_completed_evictions(),
1550 0 : Err(e) => LAYER_IMPL_METRICS.inc_eviction_cancelled(e),
1551 : }
1552 :
1553 17 : tracing::debug!(?res, elapsed_ms=%completed_in.as_millis(), %waiters, "eviction completed");
1554 17 : });
1555 :
1556 17 : Ok(())
1557 20 : }
1558 :
1559 : /// This is blocking only to do just one spawn_blocking hop compared to multiple via tokio::fs.
1560 17 : fn evict_blocking(
1561 17 : &self,
1562 17 : timeline: &Timeline,
1563 17 : _gate: &gate::GateGuard,
1564 17 : _permit: &heavier_once_cell::InitPermit,
1565 17 : ) -> Result<(), EvictionCancelled> {
1566 : // now accesses to `self.inner.get_or_init*` wait on the semaphore or the `_permit`
1567 :
1568 17 : match capture_mtime_and_remove(&self.path) {
1569 17 : Ok(local_layer_mtime) => {
1570 17 : let duration = SystemTime::now().duration_since(local_layer_mtime);
1571 17 : match duration {
1572 17 : Ok(elapsed) => {
1573 17 : let accessed_and_visible = self.access_stats.accessed()
1574 2 : && self.access_stats.visibility() == LayerVisibilityHint::Visible;
1575 17 : if accessed_and_visible {
1576 2 : // Only layers used for reads contribute to our "low residence" metric that is used
1577 2 : // to detect thrashing. Layers promoted for other reasons (e.g. compaction) are allowed
1578 2 : // to be rapidly evicted without contributing to this metric.
1579 2 : timeline
1580 2 : .metrics
1581 2 : .evictions_with_low_residence_duration
1582 2 : .read()
1583 2 : .unwrap()
1584 2 : .observe(elapsed);
1585 15 : }
1586 :
1587 17 : tracing::info!(
1588 0 : residence_millis = elapsed.as_millis(),
1589 : accessed_and_visible,
1590 0 : "evicted layer after known residence period"
1591 : );
1592 : }
1593 : Err(_) => {
1594 0 : tracing::info!("evicted layer after unknown residence period");
1595 : }
1596 : }
1597 17 : timeline.metrics.evictions.inc();
1598 17 : timeline
1599 17 : .metrics
1600 17 : .resident_physical_size_sub(self.desc.file_size);
1601 : }
1602 0 : Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
1603 0 : tracing::error!(
1604 : layer_size = %self.desc.file_size,
1605 0 : "failed to evict layer from disk, it was already gone"
1606 : );
1607 0 : return Err(EvictionCancelled::FileNotFound);
1608 : }
1609 0 : Err(e) => {
1610 : // FIXME: this should probably be an abort
1611 0 : tracing::error!("failed to evict file from disk: {e:#}");
1612 0 : return Err(EvictionCancelled::RemoveFailed);
1613 : }
1614 : }
1615 :
1616 17 : self.access_stats.record_residence_event();
1617 :
1618 17 : *self.last_evicted_at.lock().unwrap() = Some(std::time::Instant::now());
1619 :
1620 17 : self.status.as_ref().unwrap().send_replace(Status::Evicted);
1621 :
1622 17 : Ok(())
1623 17 : }
1624 :
1625 1605 : fn metadata(&self) -> LayerFileMetadata {
1626 1605 : LayerFileMetadata::new(self.desc.file_size, self.generation, self.shard)
1627 1605 : }
1628 :
1629 : /// Needed to use entered runtime in tests, but otherwise use BACKGROUND_RUNTIME.
1630 : ///
1631 : /// Synchronizing with spawned tasks is very complicated otherwise.
1632 27 : fn spawn<F>(fut: F)
1633 27 : where
1634 27 : F: std::future::Future<Output = ()> + Send + 'static,
1635 : {
1636 : #[cfg(test)]
1637 27 : tokio::task::spawn(fut);
1638 : #[cfg(not(test))]
1639 0 : crate::task_mgr::BACKGROUND_RUNTIME.spawn(fut);
1640 27 : }
1641 :
1642 : /// Needed to use entered runtime in tests, but otherwise use BACKGROUND_RUNTIME.
1643 278 : fn spawn_blocking<F>(f: F)
1644 278 : where
1645 278 : F: FnOnce() + Send + 'static,
1646 : {
1647 : #[cfg(test)]
1648 278 : tokio::task::spawn_blocking(f);
1649 : #[cfg(not(test))]
1650 0 : crate::task_mgr::BACKGROUND_RUNTIME.spawn_blocking(f);
1651 278 : }
1652 : }
1653 :
1654 17 : fn capture_mtime_and_remove(path: &Utf8Path) -> Result<SystemTime, std::io::Error> {
1655 17 : let m = path.metadata()?;
1656 17 : let local_layer_mtime = m.modified()?;
1657 17 : std::fs::remove_file(path)?;
1658 17 : Ok(local_layer_mtime)
1659 17 : }
1660 :
1661 : #[derive(Debug, thiserror::Error)]
1662 : pub(crate) enum EvictionError {
1663 : #[error("layer was already evicted")]
1664 : NotFound,
1665 :
1666 : /// Evictions must always lose to downloads in races, and this time it happened.
1667 : #[error("layer was downloaded instead")]
1668 : Downloaded,
1669 :
1670 : #[error("eviction did not happen within timeout")]
1671 : Timeout,
1672 : }
1673 :
1674 : /// Error internal to the [`LayerInner::get_or_maybe_download`]
1675 : #[derive(Debug, thiserror::Error)]
1676 : pub(crate) enum DownloadError {
1677 : #[error("timeline has already shutdown")]
1678 : TimelineShutdown,
1679 : #[error("context denies downloading")]
1680 : ContextAndConfigReallyDeniesDownloads,
1681 : #[error("downloading is really required but not allowed by this method")]
1682 : DownloadRequired,
1683 : #[error("layer path exists, but it is not a file: {0:?}")]
1684 : NotFile(std::fs::FileType),
1685 : /// Why no error here? Because it will be reported by page_service. We should had also done
1686 : /// retries already.
1687 : #[error("downloading evicted layer file failed")]
1688 : DownloadFailed,
1689 : #[error("downloading failed, possibly for shutdown")]
1690 : DownloadCancelled,
1691 : #[error("pre-condition: stat before download failed")]
1692 : PreStatFailed(#[source] std::io::Error),
1693 :
1694 : #[cfg(test)]
1695 : #[error("failpoint: {0:?}")]
1696 : Failpoint(failpoints::FailpointKind),
1697 : }
1698 :
1699 : impl DownloadError {
1700 0 : pub(crate) fn is_cancelled(&self) -> bool {
1701 0 : matches!(self, DownloadError::DownloadCancelled)
1702 0 : }
1703 : }
1704 :
1705 : #[derive(Debug, PartialEq, Copy, Clone)]
1706 : pub(crate) enum NeedsDownload {
1707 : NotFound,
1708 : NotFile(std::fs::FileType),
1709 : WrongSize { actual: u64, expected: u64 },
1710 : }
1711 :
1712 : impl std::fmt::Display for NeedsDownload {
1713 7 : fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1714 7 : match self {
1715 7 : NeedsDownload::NotFound => write!(f, "file was not found"),
1716 0 : NeedsDownload::NotFile(ft) => write!(f, "path is not a file; {ft:?}"),
1717 0 : NeedsDownload::WrongSize { actual, expected } => {
1718 0 : write!(f, "file size mismatch {actual} vs. {expected}")
1719 : }
1720 : }
1721 7 : }
1722 : }
1723 :
1724 : /// Existence of `DownloadedLayer` means that we have the file locally, and can later evict it.
1725 : pub(crate) struct DownloadedLayer {
1726 : owner: Weak<LayerInner>,
1727 : // Use tokio OnceCell as we do not need to deinitialize this, it'll just get dropped with the
1728 : // DownloadedLayer
1729 : kind: tokio::sync::OnceCell<anyhow::Result<LayerKind>>,
1730 : version: usize,
1731 : }
1732 :
1733 : impl std::fmt::Debug for DownloadedLayer {
1734 0 : fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1735 0 : f.debug_struct("DownloadedLayer")
1736 0 : // owner omitted because it is always "Weak"
1737 0 : .field("kind", &self.kind)
1738 0 : .field("version", &self.version)
1739 0 : .finish()
1740 0 : }
1741 : }
1742 :
1743 : impl Drop for DownloadedLayer {
1744 379 : fn drop(&mut self) {
1745 379 : if let Some(owner) = self.owner.upgrade() {
1746 20 : owner.on_downloaded_layer_drop(self.version);
1747 359 : } else {
1748 359 : // Layer::drop will handle cancelling the eviction; because of drop order and
1749 359 : // `DownloadedLayer` never leaking, we cannot know here if eviction was requested.
1750 359 : }
1751 379 : }
1752 : }
1753 :
1754 : impl DownloadedLayer {
1755 : /// Initializes the `DeltaLayerInner` or `ImageLayerInner` within [`LayerKind`].
1756 : /// Failure to load the layer is sticky, i.e., future `get()` calls will return
1757 : /// the initial load failure immediately.
1758 : ///
1759 : /// `owner` parameter is a strong reference at the same `LayerInner` as the
1760 : /// `DownloadedLayer::owner` would be when upgraded. Given how this method ends up called,
1761 : /// we will always have the LayerInner on the callstack, so we can just use it.
1762 139376 : async fn get<'a>(
1763 139376 : &'a self,
1764 139376 : owner: &Arc<LayerInner>,
1765 139376 : ctx: &RequestContext,
1766 139376 : ) -> anyhow::Result<&'a LayerKind> {
1767 139376 : let init = || async {
1768 635 : assert_eq!(
1769 635 : Weak::as_ptr(&self.owner),
1770 635 : Arc::as_ptr(owner),
1771 0 : "these are the same, just avoiding the upgrade"
1772 : );
1773 :
1774 635 : let res = if owner.desc.is_delta {
1775 560 : let ctx = RequestContextBuilder::from(ctx)
1776 560 : .page_content_kind(crate::context::PageContentKind::DeltaLayerSummary)
1777 560 : .attached_child();
1778 560 : let summary = Some(delta_layer::Summary::expected(
1779 560 : owner.desc.tenant_shard_id.tenant_id,
1780 560 : owner.desc.timeline_id,
1781 560 : owner.desc.key_range.clone(),
1782 560 : owner.desc.lsn_range.clone(),
1783 560 : ));
1784 560 : delta_layer::DeltaLayerInner::load(
1785 560 : &owner.path,
1786 560 : summary,
1787 560 : Some(owner.conf.max_vectored_read_bytes),
1788 560 : &ctx,
1789 560 : )
1790 560 : .await
1791 560 : .map(LayerKind::Delta)
1792 : } else {
1793 75 : let ctx = RequestContextBuilder::from(ctx)
1794 75 : .page_content_kind(crate::context::PageContentKind::ImageLayerSummary)
1795 75 : .attached_child();
1796 75 : let lsn = owner.desc.image_layer_lsn();
1797 75 : let summary = Some(image_layer::Summary::expected(
1798 75 : owner.desc.tenant_shard_id.tenant_id,
1799 75 : owner.desc.timeline_id,
1800 75 : owner.desc.key_range.clone(),
1801 75 : lsn,
1802 75 : ));
1803 75 : image_layer::ImageLayerInner::load(
1804 75 : &owner.path,
1805 75 : lsn,
1806 75 : summary,
1807 75 : Some(owner.conf.max_vectored_read_bytes),
1808 75 : &ctx,
1809 75 : )
1810 75 : .await
1811 75 : .map(LayerKind::Image)
1812 : };
1813 :
1814 635 : match res {
1815 635 : Ok(layer) => Ok(layer),
1816 0 : Err(err) => {
1817 0 : LAYER_IMPL_METRICS.inc_permanent_loading_failures();
1818 : // We log this message once over the lifetime of `Self`
1819 : // => Ok and good to log backtrace and path here.
1820 0 : tracing::error!(
1821 0 : "layer load failed, assuming permanent failure: {}: {err:?}",
1822 0 : owner.path
1823 : );
1824 0 : Err(err)
1825 : }
1826 : }
1827 1270 : };
1828 139376 : self.kind
1829 139376 : .get_or_init(init)
1830 139376 : .await
1831 139376 : .as_ref()
1832 : // We already logged the full backtrace above, once. Don't repeat that here.
1833 139376 : .map_err(|e| anyhow::anyhow!("layer load failed earlier: {e}"))
1834 139376 : }
1835 :
1836 138851 : async fn get_values_reconstruct_data(
1837 138851 : &self,
1838 138851 : this: ResidentLayer,
1839 138851 : keyspace: KeySpace,
1840 138851 : lsn_range: Range<Lsn>,
1841 138851 : reconstruct_data: &mut ValuesReconstructState,
1842 138851 : ctx: &RequestContext,
1843 138851 : ) -> Result<(), GetVectoredError> {
1844 : use LayerKind::*;
1845 :
1846 138851 : match self
1847 138851 : .get(&this.owner.0, ctx)
1848 138851 : .await
1849 138851 : .map_err(GetVectoredError::Other)?
1850 : {
1851 124723 : Delta(d) => {
1852 124723 : d.get_values_reconstruct_data(this, keyspace, lsn_range, reconstruct_data, ctx)
1853 124723 : .await
1854 : }
1855 14128 : Image(i) => {
1856 14128 : i.get_values_reconstruct_data(this, keyspace, reconstruct_data, ctx)
1857 14128 : .await
1858 : }
1859 : }
1860 138851 : }
1861 :
1862 2 : async fn dump(&self, owner: &Arc<LayerInner>, ctx: &RequestContext) -> anyhow::Result<()> {
1863 : use LayerKind::*;
1864 2 : match self.get(owner, ctx).await? {
1865 2 : Delta(d) => d.dump(ctx).await?,
1866 0 : Image(i) => i.dump(ctx).await?,
1867 : }
1868 :
1869 2 : Ok(())
1870 2 : }
1871 : }
1872 :
1873 : /// Wrapper around an actual layer implementation.
1874 : #[derive(Debug)]
1875 : enum LayerKind {
1876 : Delta(delta_layer::DeltaLayerInner),
1877 : Image(image_layer::ImageLayerInner),
1878 : }
1879 :
1880 : /// Guard for forcing a layer be resident while it exists.
1881 : #[derive(Clone)]
1882 : pub struct ResidentLayer {
1883 : owner: Layer,
1884 : downloaded: Arc<DownloadedLayer>,
1885 : }
1886 :
1887 : impl std::fmt::Display for ResidentLayer {
1888 1096 : fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1889 1096 : write!(f, "{}", self.owner)
1890 1096 : }
1891 : }
1892 :
1893 : impl std::fmt::Debug for ResidentLayer {
1894 0 : fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1895 0 : write!(f, "{}", self.owner)
1896 0 : }
1897 : }
1898 :
1899 : impl ResidentLayer {
1900 : /// Release the eviction guard, converting back into a plain [`Layer`].
1901 : ///
1902 : /// You can access the [`Layer`] also by using `as_ref`.
1903 210 : pub(crate) fn drop_eviction_guard(self) -> Layer {
1904 210 : self.into()
1905 210 : }
1906 :
1907 : /// Loads all keys stored in the layer. Returns key, lsn and value size.
1908 0 : #[tracing::instrument(level = tracing::Level::DEBUG, skip_all, fields(layer=%self))]
1909 : pub(crate) async fn load_keys<'a>(
1910 : &'a self,
1911 : ctx: &RequestContext,
1912 : ) -> anyhow::Result<Vec<pageserver_api::key::Key>> {
1913 : use LayerKind::*;
1914 :
1915 : let owner = &self.owner.0;
1916 : let inner = self.downloaded.get(owner, ctx).await?;
1917 :
1918 : // this is valid because the DownloadedLayer::kind is a OnceCell, not a
1919 : // Mutex<OnceCell>, so we cannot go and deinitialize the value with OnceCell::take
1920 : // while it's being held.
1921 : self.owner.record_access(ctx);
1922 :
1923 : let res = match inner {
1924 : Delta(d) => delta_layer::DeltaLayerInner::load_keys(d, ctx).await,
1925 : Image(i) => image_layer::ImageLayerInner::load_keys(i, ctx).await,
1926 : };
1927 0 : res.with_context(|| format!("Layer index is corrupted for {self}"))
1928 : }
1929 :
1930 : /// Read all they keys in this layer which match the ShardIdentity, and write them all to
1931 : /// the provided writer. Return the number of keys written.
1932 4 : #[tracing::instrument(level = tracing::Level::DEBUG, skip_all, fields(layer=%self))]
1933 : pub(crate) async fn filter(
1934 : &self,
1935 : shard_identity: &ShardIdentity,
1936 : writer: &mut ImageLayerWriter,
1937 : ctx: &RequestContext,
1938 : ) -> Result<usize, CompactionError> {
1939 : use LayerKind::*;
1940 :
1941 : match self
1942 : .downloaded
1943 : .get(&self.owner.0, ctx)
1944 : .await
1945 : .map_err(CompactionError::Other)?
1946 : {
1947 : Delta(_) => {
1948 : return Err(CompactionError::Other(anyhow::anyhow!(format!(
1949 : "cannot filter() on a delta layer {self}"
1950 : ))));
1951 : }
1952 : Image(i) => i
1953 : .filter(shard_identity, writer, ctx)
1954 : .await
1955 : .map_err(CompactionError::Other),
1956 : }
1957 : }
1958 :
1959 : /// Returns the amount of keys and values written to the writer.
1960 5 : pub(crate) async fn copy_delta_prefix(
1961 5 : &self,
1962 5 : writer: &mut super::delta_layer::DeltaLayerWriter,
1963 5 : until: Lsn,
1964 5 : ctx: &RequestContext,
1965 5 : ) -> anyhow::Result<usize> {
1966 : use LayerKind::*;
1967 :
1968 5 : let owner = &self.owner.0;
1969 :
1970 5 : match self.downloaded.get(owner, ctx).await? {
1971 5 : Delta(d) => d
1972 5 : .copy_prefix(writer, until, ctx)
1973 5 : .await
1974 5 : .with_context(|| format!("copy_delta_prefix until {until} of {self}")),
1975 0 : Image(_) => anyhow::bail!(format!("cannot copy_lsn_prefix of image layer {self}")),
1976 : }
1977 5 : }
1978 :
1979 976 : pub(crate) fn local_path(&self) -> &Utf8Path {
1980 976 : &self.owner.0.path
1981 976 : }
1982 :
1983 1130 : pub(crate) fn metadata(&self) -> LayerFileMetadata {
1984 1130 : self.owner.metadata()
1985 1130 : }
1986 :
1987 : /// Cast the layer to a delta, return an error if it is an image layer.
1988 478 : pub(crate) async fn get_as_delta(
1989 478 : &self,
1990 478 : ctx: &RequestContext,
1991 478 : ) -> anyhow::Result<&delta_layer::DeltaLayerInner> {
1992 : use LayerKind::*;
1993 478 : match self.downloaded.get(&self.owner.0, ctx).await? {
1994 478 : Delta(d) => Ok(d),
1995 0 : Image(_) => Err(anyhow::anyhow!("image layer")),
1996 : }
1997 478 : }
1998 :
1999 : /// Cast the layer to an image, return an error if it is a delta layer.
2000 36 : pub(crate) async fn get_as_image(
2001 36 : &self,
2002 36 : ctx: &RequestContext,
2003 36 : ) -> anyhow::Result<&image_layer::ImageLayerInner> {
2004 : use LayerKind::*;
2005 36 : match self.downloaded.get(&self.owner.0, ctx).await? {
2006 36 : Image(d) => Ok(d),
2007 0 : Delta(_) => Err(anyhow::anyhow!("delta layer")),
2008 : }
2009 36 : }
2010 : }
2011 :
2012 : impl AsLayerDesc for ResidentLayer {
2013 896327 : fn layer_desc(&self) -> &PersistentLayerDesc {
2014 896327 : self.owner.layer_desc()
2015 896327 : }
2016 : }
2017 :
2018 : impl AsRef<Layer> for ResidentLayer {
2019 1054 : fn as_ref(&self) -> &Layer {
2020 1054 : &self.owner
2021 1054 : }
2022 : }
2023 :
2024 : /// Drop the eviction guard.
2025 : impl From<ResidentLayer> for Layer {
2026 210 : fn from(value: ResidentLayer) -> Self {
2027 210 : value.owner
2028 210 : }
2029 : }
2030 :
2031 : use metrics::IntCounter;
2032 :
2033 : pub(crate) struct LayerImplMetrics {
2034 : started_evictions: IntCounter,
2035 : completed_evictions: IntCounter,
2036 : cancelled_evictions: enum_map::EnumMap<EvictionCancelled, IntCounter>,
2037 :
2038 : started_deletes: IntCounter,
2039 : completed_deletes: IntCounter,
2040 : failed_deletes: enum_map::EnumMap<DeleteFailed, IntCounter>,
2041 :
2042 : rare_counters: enum_map::EnumMap<RareEvent, IntCounter>,
2043 : inits_cancelled: metrics::core::GenericCounter<metrics::core::AtomicU64>,
2044 : redownload_after: metrics::Histogram,
2045 : time_to_evict: metrics::Histogram,
2046 : }
2047 :
2048 : impl Default for LayerImplMetrics {
2049 27 : fn default() -> Self {
2050 : use enum_map::Enum;
2051 :
2052 : // reminder: these will be pageserver_layer_* with "_total" suffix
2053 :
2054 27 : let started_evictions = metrics::register_int_counter!(
2055 : "pageserver_layer_started_evictions",
2056 : "Evictions started in the Layer implementation"
2057 : )
2058 27 : .unwrap();
2059 27 : let completed_evictions = metrics::register_int_counter!(
2060 : "pageserver_layer_completed_evictions",
2061 : "Evictions completed in the Layer implementation"
2062 : )
2063 27 : .unwrap();
2064 :
2065 27 : let cancelled_evictions = metrics::register_int_counter_vec!(
2066 : "pageserver_layer_cancelled_evictions_count",
2067 : "Different reasons for evictions to have been cancelled or failed",
2068 27 : &["reason"]
2069 : )
2070 27 : .unwrap();
2071 :
2072 243 : let cancelled_evictions = enum_map::EnumMap::from_array(std::array::from_fn(|i| {
2073 243 : let reason = EvictionCancelled::from_usize(i);
2074 243 : let s = reason.as_str();
2075 243 : cancelled_evictions.with_label_values(&[s])
2076 243 : }));
2077 :
2078 27 : let started_deletes = metrics::register_int_counter!(
2079 : "pageserver_layer_started_deletes",
2080 : "Deletions on drop pending in the Layer implementation"
2081 : )
2082 27 : .unwrap();
2083 27 : let completed_deletes = metrics::register_int_counter!(
2084 : "pageserver_layer_completed_deletes",
2085 : "Deletions on drop completed in the Layer implementation"
2086 : )
2087 27 : .unwrap();
2088 :
2089 27 : let failed_deletes = metrics::register_int_counter_vec!(
2090 : "pageserver_layer_failed_deletes_count",
2091 : "Different reasons for deletions on drop to have failed",
2092 27 : &["reason"]
2093 : )
2094 27 : .unwrap();
2095 :
2096 54 : let failed_deletes = enum_map::EnumMap::from_array(std::array::from_fn(|i| {
2097 54 : let reason = DeleteFailed::from_usize(i);
2098 54 : let s = reason.as_str();
2099 54 : failed_deletes.with_label_values(&[s])
2100 54 : }));
2101 :
2102 27 : let rare_counters = metrics::register_int_counter_vec!(
2103 : "pageserver_layer_assumed_rare_count",
2104 : "Times unexpected or assumed rare event happened",
2105 27 : &["event"]
2106 : )
2107 27 : .unwrap();
2108 :
2109 189 : let rare_counters = enum_map::EnumMap::from_array(std::array::from_fn(|i| {
2110 189 : let event = RareEvent::from_usize(i);
2111 189 : let s = event.as_str();
2112 189 : rare_counters.with_label_values(&[s])
2113 189 : }));
2114 :
2115 27 : let inits_cancelled = metrics::register_int_counter!(
2116 : "pageserver_layer_inits_cancelled_count",
2117 : "Times Layer initialization was cancelled",
2118 : )
2119 27 : .unwrap();
2120 :
2121 27 : let redownload_after = {
2122 27 : let minute = 60.0;
2123 27 : let hour = 60.0 * minute;
2124 27 : metrics::register_histogram!(
2125 : "pageserver_layer_redownloaded_after",
2126 : "Time between evicting and re-downloading.",
2127 27 : vec![
2128 : 10.0,
2129 : 30.0,
2130 27 : minute,
2131 27 : 5.0 * minute,
2132 27 : 15.0 * minute,
2133 27 : 30.0 * minute,
2134 27 : hour,
2135 27 : 12.0 * hour,
2136 : ]
2137 : )
2138 27 : .unwrap()
2139 : };
2140 :
2141 27 : let time_to_evict = metrics::register_histogram!(
2142 : "pageserver_layer_eviction_held_permit_seconds",
2143 : "Time eviction held the permit.",
2144 27 : vec![0.001, 0.010, 0.100, 0.500, 1.000, 5.000]
2145 : )
2146 27 : .unwrap();
2147 :
2148 27 : Self {
2149 27 : started_evictions,
2150 27 : completed_evictions,
2151 27 : cancelled_evictions,
2152 27 :
2153 27 : started_deletes,
2154 27 : completed_deletes,
2155 27 : failed_deletes,
2156 27 :
2157 27 : rare_counters,
2158 27 : inits_cancelled,
2159 27 : redownload_after,
2160 27 : time_to_evict,
2161 27 : }
2162 27 : }
2163 : }
2164 :
2165 : impl LayerImplMetrics {
2166 21 : fn inc_started_evictions(&self) {
2167 21 : self.started_evictions.inc();
2168 21 : }
2169 17 : fn inc_completed_evictions(&self) {
2170 17 : self.completed_evictions.inc();
2171 17 : }
2172 4 : fn inc_eviction_cancelled(&self, reason: EvictionCancelled) {
2173 4 : self.cancelled_evictions[reason].inc()
2174 4 : }
2175 :
2176 261 : fn inc_started_deletes(&self) {
2177 261 : self.started_deletes.inc();
2178 261 : }
2179 260 : fn inc_completed_deletes(&self) {
2180 260 : self.completed_deletes.inc();
2181 260 : }
2182 0 : fn inc_deletes_failed(&self, reason: DeleteFailed) {
2183 0 : self.failed_deletes[reason].inc();
2184 0 : }
2185 :
2186 : /// Counted separatedly from failed layer deletes because we will complete the layer deletion
2187 : /// attempt regardless of failure to delete local file.
2188 0 : fn inc_delete_removes_failed(&self) {
2189 0 : self.rare_counters[RareEvent::RemoveOnDropFailed].inc();
2190 0 : }
2191 :
2192 : /// Expected rare just as cancellations are rare, but we could have cancellations separate from
2193 : /// the single caller which can start the download, so use this counter to separte them.
2194 0 : fn inc_init_completed_without_requester(&self) {
2195 0 : self.rare_counters[RareEvent::InitCompletedWithoutRequester].inc();
2196 0 : }
2197 :
2198 : /// Expected rare because cancellations are unexpected, and failures are unexpected
2199 0 : fn inc_download_failed_without_requester(&self) {
2200 0 : self.rare_counters[RareEvent::DownloadFailedWithoutRequester].inc();
2201 0 : }
2202 :
2203 : /// The Weak in ResidentOrWantedEvicted::WantedEvicted was successfully upgraded.
2204 : ///
2205 : /// If this counter is always zero, we should replace ResidentOrWantedEvicted type with an
2206 : /// Option.
2207 0 : fn inc_raced_wanted_evicted_accesses(&self) {
2208 0 : self.rare_counters[RareEvent::UpgradedWantedEvicted].inc();
2209 0 : }
2210 :
2211 : /// These are only expected for [`Self::inc_init_cancelled`] amount when
2212 : /// running with remote storage.
2213 3 : fn inc_init_needed_no_download(&self) {
2214 3 : self.rare_counters[RareEvent::InitWithoutDownload].inc();
2215 3 : }
2216 :
2217 : /// Expected rare because all layer files should be readable and good
2218 0 : fn inc_permanent_loading_failures(&self) {
2219 0 : self.rare_counters[RareEvent::PermanentLoadingFailure].inc();
2220 0 : }
2221 :
2222 0 : fn inc_init_cancelled(&self) {
2223 0 : self.inits_cancelled.inc()
2224 0 : }
2225 :
2226 7 : fn record_redownloaded_after(&self, duration: std::time::Duration) {
2227 7 : self.redownload_after.observe(duration.as_secs_f64())
2228 7 : }
2229 :
2230 : /// This would be bad if it ever happened, or mean extreme disk pressure. We should probably
2231 : /// instead cancel eviction if we would have read waiters. We cannot however separate reads
2232 : /// from other evictions, so this could have noise as well.
2233 0 : fn inc_evicted_with_waiters(&self) {
2234 0 : self.rare_counters[RareEvent::EvictedWithWaiters].inc();
2235 0 : }
2236 :
2237 : /// Recorded at least initially as the permit is now acquired in async context before
2238 : /// spawn_blocking action.
2239 17 : fn record_time_to_evict(&self, duration: std::time::Duration) {
2240 17 : self.time_to_evict.observe(duration.as_secs_f64())
2241 17 : }
2242 : }
2243 :
2244 : #[derive(Debug, Clone, Copy, enum_map::Enum)]
2245 : enum EvictionCancelled {
2246 : LayerGone,
2247 : TimelineGone,
2248 : VersionCheckFailed,
2249 : FileNotFound,
2250 : RemoveFailed,
2251 : AlreadyReinitialized,
2252 : /// Not evicted because of a pending reinitialization
2253 : LostToDownload,
2254 : /// After eviction, there was a new layer access which cancelled the eviction.
2255 : UpgradedBackOnAccess,
2256 : UnexpectedEvictedState,
2257 : }
2258 :
2259 : impl EvictionCancelled {
2260 243 : fn as_str(&self) -> &'static str {
2261 243 : match self {
2262 27 : EvictionCancelled::LayerGone => "layer_gone",
2263 27 : EvictionCancelled::TimelineGone => "timeline_gone",
2264 27 : EvictionCancelled::VersionCheckFailed => "version_check_fail",
2265 27 : EvictionCancelled::FileNotFound => "file_not_found",
2266 27 : EvictionCancelled::RemoveFailed => "remove_failed",
2267 27 : EvictionCancelled::AlreadyReinitialized => "already_reinitialized",
2268 27 : EvictionCancelled::LostToDownload => "lost_to_download",
2269 27 : EvictionCancelled::UpgradedBackOnAccess => "upgraded_back_on_access",
2270 27 : EvictionCancelled::UnexpectedEvictedState => "unexpected_evicted_state",
2271 : }
2272 243 : }
2273 : }
2274 :
2275 : #[derive(enum_map::Enum)]
2276 : enum DeleteFailed {
2277 : TimelineGone,
2278 : DeleteSchedulingFailed,
2279 : }
2280 :
2281 : impl DeleteFailed {
2282 54 : fn as_str(&self) -> &'static str {
2283 54 : match self {
2284 27 : DeleteFailed::TimelineGone => "timeline_gone",
2285 27 : DeleteFailed::DeleteSchedulingFailed => "delete_scheduling_failed",
2286 : }
2287 54 : }
2288 : }
2289 :
2290 : #[derive(enum_map::Enum)]
2291 : enum RareEvent {
2292 : RemoveOnDropFailed,
2293 : InitCompletedWithoutRequester,
2294 : DownloadFailedWithoutRequester,
2295 : UpgradedWantedEvicted,
2296 : InitWithoutDownload,
2297 : PermanentLoadingFailure,
2298 : EvictedWithWaiters,
2299 : }
2300 :
2301 : impl RareEvent {
2302 189 : fn as_str(&self) -> &'static str {
2303 : use RareEvent::*;
2304 :
2305 189 : match self {
2306 27 : RemoveOnDropFailed => "remove_on_drop_failed",
2307 27 : InitCompletedWithoutRequester => "init_completed_without",
2308 27 : DownloadFailedWithoutRequester => "download_failed_without",
2309 27 : UpgradedWantedEvicted => "raced_wanted_evicted",
2310 27 : InitWithoutDownload => "init_needed_no_download",
2311 27 : PermanentLoadingFailure => "permanent_loading_failure",
2312 27 : EvictedWithWaiters => "evicted_with_waiters",
2313 : }
2314 189 : }
2315 : }
2316 :
2317 : pub(crate) static LAYER_IMPL_METRICS: once_cell::sync::Lazy<LayerImplMetrics> =
2318 : once_cell::sync::Lazy::new(LayerImplMetrics::default);
|
