Line data Source code
1 : //! A RemoteLayer is an in-memory placeholder for a layer file that exists
2 : //! in remote storage.
3 : //!
4 : use crate::config::PageServerConf;
5 : use crate::context::RequestContext;
6 : use crate::repository::Key;
7 : use crate::tenant::remote_timeline_client::index::LayerFileMetadata;
8 : use crate::tenant::storage_layer::{Layer, ValueReconstructResult, ValueReconstructState};
9 : use crate::tenant::timeline::layer_manager::LayerManager;
10 : use anyhow::{bail, Result};
11 : use pageserver_api::models::HistoricLayerInfo;
12 : use std::ops::Range;
13 : use std::path::PathBuf;
14 : use std::sync::Arc;
15 :
16 : use utils::{
17 : id::{TenantId, TimelineId},
18 : lsn::Lsn,
19 : };
20 :
21 : use super::filename::{DeltaFileName, ImageFileName};
22 : use super::{
23 : AsLayerDesc, DeltaLayer, ImageLayer, LayerAccessStats, LayerAccessStatsReset,
24 : LayerResidenceStatus, PersistentLayer, PersistentLayerDesc,
25 : };
26 :
27 : /// RemoteLayer is a not yet downloaded [`ImageLayer`] or
28 : /// [`DeltaLayer`](super::DeltaLayer).
29 : ///
30 : /// RemoteLayer might be downloaded on-demand during operations which are
31 : /// allowed download remote layers and during which, it gets replaced with a
32 : /// concrete `DeltaLayer` or `ImageLayer`.
33 : ///
34 : /// See: [`crate::context::RequestContext`] for authorization to download
35 : pub struct RemoteLayer {
36 : pub desc: PersistentLayerDesc,
37 :
38 : pub layer_metadata: LayerFileMetadata,
39 :
40 : access_stats: LayerAccessStats,
41 :
42 : pub(crate) ongoing_download: Arc<tokio::sync::Semaphore>,
43 :
44 : /// Has `LayerMap::replace` failed for this (true) or not (false).
45 : ///
46 : /// Used together with [`ongoing_download`] semaphore in `Timeline::download_remote_layer`.
47 : /// The field is used to mark a RemoteLayer permanently (until restart or ignore+load)
48 : /// unprocessable, because a LayerMap::replace failed.
49 : ///
50 : /// It is very unlikely to accumulate these in the Timeline's LayerMap, but having this avoids
51 : /// a possible fast loop between `Timeline::get_reconstruct_data` and
52 : /// `Timeline::download_remote_layer`, which also logs.
53 : ///
54 : /// [`ongoing_download`]: Self::ongoing_download
55 : pub(crate) download_replacement_failure: std::sync::atomic::AtomicBool,
56 : }
57 :
58 : impl std::fmt::Debug for RemoteLayer {
59 0 : fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
60 0 : f.debug_struct("RemoteLayer")
61 0 : .field("file_name", &self.desc.filename())
62 0 : .field("layer_metadata", &self.layer_metadata)
63 0 : .field("is_incremental", &self.desc.is_incremental())
64 0 : .finish()
65 0 : }
66 : }
67 :
68 : #[async_trait::async_trait]
69 : impl Layer for RemoteLayer {
70 0 : async fn get_value_reconstruct_data(
71 0 : &self,
72 0 : _key: Key,
73 0 : _lsn_range: Range<Lsn>,
74 0 : _reconstruct_state: &mut ValueReconstructState,
75 0 : _ctx: &RequestContext,
76 0 : ) -> Result<ValueReconstructResult> {
77 0 : bail!("layer {self} needs to be downloaded");
78 0 : }
79 : }
80 :
81 : /// Boilerplate to implement the Layer trait, always use layer_desc for persistent layers.
82 : impl std::fmt::Display for RemoteLayer {
83 3102 : fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
84 3102 : write!(f, "{}", self.layer_desc().short_id())
85 3102 : }
86 : }
87 :
88 : impl AsLayerDesc for RemoteLayer {
89 13574 : fn layer_desc(&self) -> &PersistentLayerDesc {
90 13574 : &self.desc
91 13574 : }
92 : }
93 :
94 : impl PersistentLayer for RemoteLayer {
95 993 : fn local_path(&self) -> Option<PathBuf> {
96 993 : None
97 993 : }
98 :
99 0 : fn delete_resident_layer_file(&self) -> Result<()> {
100 0 : bail!("remote layer has no layer file");
101 0 : }
102 :
103 1063 : fn downcast_remote_layer<'a>(self: Arc<Self>) -> Option<std::sync::Arc<RemoteLayer>> {
104 1063 : Some(self)
105 1063 : }
106 :
107 1606 : fn is_remote_layer(&self) -> bool {
108 1606 : true
109 1606 : }
110 :
111 398 : fn info(&self, reset: LayerAccessStatsReset) -> HistoricLayerInfo {
112 398 : let layer_file_name = self.layer_desc().filename().file_name();
113 398 : let lsn_range = self.layer_desc().lsn_range.clone();
114 398 :
115 398 : if self.desc.is_delta {
116 395 : HistoricLayerInfo::Delta {
117 395 : layer_file_name,
118 395 : layer_file_size: self.layer_metadata.file_size(),
119 395 : lsn_start: lsn_range.start,
120 395 : lsn_end: lsn_range.end,
121 395 : remote: true,
122 395 : access_stats: self.access_stats.as_api_model(reset),
123 395 : }
124 : } else {
125 3 : HistoricLayerInfo::Image {
126 3 : layer_file_name,
127 3 : layer_file_size: self.layer_metadata.file_size(),
128 3 : lsn_start: lsn_range.start,
129 3 : remote: true,
130 3 : access_stats: self.access_stats.as_api_model(reset),
131 3 : }
132 : }
133 398 : }
134 :
135 0 : fn access_stats(&self) -> &LayerAccessStats {
136 0 : &self.access_stats
137 0 : }
138 : }
139 :
140 : impl RemoteLayer {
141 954 : pub fn new_img(
142 954 : tenantid: TenantId,
143 954 : timelineid: TimelineId,
144 954 : fname: &ImageFileName,
145 954 : layer_metadata: &LayerFileMetadata,
146 954 : access_stats: LayerAccessStats,
147 954 : ) -> RemoteLayer {
148 954 : RemoteLayer {
149 954 : desc: PersistentLayerDesc::new_img(
150 954 : tenantid,
151 954 : timelineid,
152 954 : fname.key_range.clone(),
153 954 : fname.lsn,
154 954 : layer_metadata.file_size(),
155 954 : ),
156 954 : layer_metadata: layer_metadata.clone(),
157 954 : ongoing_download: Arc::new(tokio::sync::Semaphore::new(1)),
158 954 : download_replacement_failure: std::sync::atomic::AtomicBool::default(),
159 954 : access_stats,
160 954 : }
161 954 : }
162 :
163 1747 : pub fn new_delta(
164 1747 : tenantid: TenantId,
165 1747 : timelineid: TimelineId,
166 1747 : fname: &DeltaFileName,
167 1747 : layer_metadata: &LayerFileMetadata,
168 1747 : access_stats: LayerAccessStats,
169 1747 : ) -> RemoteLayer {
170 1747 : RemoteLayer {
171 1747 : desc: PersistentLayerDesc::new_delta(
172 1747 : tenantid,
173 1747 : timelineid,
174 1747 : fname.key_range.clone(),
175 1747 : fname.lsn_range.clone(),
176 1747 : layer_metadata.file_size(),
177 1747 : ),
178 1747 : layer_metadata: layer_metadata.clone(),
179 1747 : ongoing_download: Arc::new(tokio::sync::Semaphore::new(1)),
180 1747 : download_replacement_failure: std::sync::atomic::AtomicBool::default(),
181 1747 : access_stats,
182 1747 : }
183 1747 : }
184 :
185 : /// Create a Layer struct representing this layer, after it has been downloaded.
186 1008 : pub(crate) fn create_downloaded_layer(
187 1008 : &self,
188 1008 : _layer_map_lock_held_witness: &LayerManager,
189 1008 : conf: &'static PageServerConf,
190 1008 : file_size: u64,
191 1008 : ) -> Arc<dyn PersistentLayer> {
192 1008 : if self.desc.is_delta {
193 283 : let fname = self.desc.delta_file_name();
194 283 : Arc::new(DeltaLayer::new(
195 283 : conf,
196 283 : self.desc.timeline_id,
197 283 : self.desc.tenant_id,
198 283 : &fname,
199 283 : file_size,
200 283 : self.access_stats
201 283 : .clone_for_residence_change(LayerResidenceStatus::Resident),
202 283 : ))
203 : } else {
204 725 : let fname = self.desc.image_file_name();
205 725 : Arc::new(ImageLayer::new(
206 725 : conf,
207 725 : self.desc.timeline_id,
208 725 : self.desc.tenant_id,
209 725 : &fname,
210 725 : file_size,
211 725 : self.access_stats
212 725 : .clone_for_residence_change(LayerResidenceStatus::Resident),
213 725 : ))
214 : }
215 1008 : }
216 : }
|
