Line data Source code
1 : use std::fs::{self, File};
2 : use std::path::{Path, PathBuf};
3 :
4 : use anyhow::Result;
5 : use camino::{Utf8Path, Utf8PathBuf};
6 : use clap::Subcommand;
7 : use pageserver::context::{DownloadBehavior, RequestContext};
8 : use pageserver::task_mgr::TaskKind;
9 : use pageserver::tenant::storage_layer::{DeltaLayer, ImageLayer, delta_layer, image_layer};
10 : use pageserver::tenant::{TENANTS_SEGMENT_NAME, TIMELINES_SEGMENT_NAME};
11 : use pageserver::virtual_file::api::IoMode;
12 : use pageserver::{page_cache, virtual_file};
13 : use pageserver_api::key::Key;
14 : use utils::id::{TenantId, TimelineId};
15 :
16 : use crate::layer_map_analyzer::{LayerFile, parse_filename};
17 :
18 : #[derive(Subcommand)]
19 : pub(crate) enum LayerCmd {
20 : /// List all tenants and timelines under the pageserver path
21 : ///
22 : /// Example: `cargo run --bin pagectl layer list .neon/`
23 : List { path: PathBuf },
24 : /// List all layers of a given tenant and timeline
25 : ///
26 : /// Example: `cargo run --bin pagectl layer list .neon/`
27 : ListLayer {
28 : path: PathBuf,
29 : tenant: String,
30 : timeline: String,
31 : key: Option<Key>,
32 : },
33 : /// Dump all information of a layer file
34 : DumpLayer {
35 : path: PathBuf,
36 : tenant: String,
37 : timeline: String,
38 : /// The id from list-layer command
39 : id: usize,
40 : },
41 : /// Dump all information of a layer file locally
42 : DumpLayerLocal { path: PathBuf },
43 : RewriteSummary {
44 : layer_file_path: Utf8PathBuf,
45 : #[clap(long)]
46 : new_tenant_id: Option<TenantId>,
47 : #[clap(long)]
48 : new_timeline_id: Option<TimelineId>,
49 : },
50 : }
51 :
52 0 : async fn read_delta_file(path: impl AsRef<Path>, ctx: &RequestContext) -> Result<()> {
53 0 : virtual_file::init(
54 : 10,
55 0 : virtual_file::api::IoEngineKind::StdFs,
56 0 : IoMode::preferred(),
57 0 : virtual_file::SyncMode::Sync,
58 : );
59 0 : page_cache::init(100);
60 0 : let path = Utf8Path::from_path(path.as_ref()).expect("non-Unicode path");
61 0 : let file = File::open(path)?;
62 0 : let delta_layer = DeltaLayer::new_for_path(path, file)?;
63 0 : delta_layer.dump(true, ctx).await?;
64 0 : Ok(())
65 0 : }
66 :
67 0 : async fn read_image_file(path: impl AsRef<Path>, ctx: &RequestContext) -> Result<()> {
68 0 : virtual_file::init(
69 : 10,
70 0 : virtual_file::api::IoEngineKind::StdFs,
71 0 : IoMode::preferred(),
72 0 : virtual_file::SyncMode::Sync,
73 : );
74 0 : page_cache::init(100);
75 0 : let path = Utf8Path::from_path(path.as_ref()).expect("non-Unicode path");
76 0 : let file = File::open(path)?;
77 0 : let image_layer = ImageLayer::new_for_path(path, file)?;
78 0 : image_layer.dump(true, ctx).await?;
79 0 : Ok(())
80 0 : }
81 :
82 0 : pub(crate) async fn main(cmd: &LayerCmd) -> Result<()> {
83 0 : let ctx =
84 0 : RequestContext::new(TaskKind::DebugTool, DownloadBehavior::Error).with_scope_debug_tools();
85 0 : match cmd {
86 0 : LayerCmd::List { path } => {
87 0 : for tenant in fs::read_dir(path.join(TENANTS_SEGMENT_NAME))? {
88 0 : let tenant = tenant?;
89 0 : if !tenant.file_type()?.is_dir() {
90 0 : continue;
91 0 : }
92 0 : println!("tenant {}", tenant.file_name().to_string_lossy());
93 0 : for timeline in fs::read_dir(tenant.path().join(TIMELINES_SEGMENT_NAME))? {
94 0 : let timeline = timeline?;
95 0 : if !timeline.file_type()?.is_dir() {
96 0 : continue;
97 0 : }
98 0 : println!("- timeline {}", timeline.file_name().to_string_lossy());
99 : }
100 : }
101 0 : Ok(())
102 : }
103 : LayerCmd::ListLayer {
104 0 : path,
105 0 : tenant,
106 0 : timeline,
107 0 : key,
108 : } => {
109 0 : let timeline_path = path
110 0 : .join(TENANTS_SEGMENT_NAME)
111 0 : .join(tenant)
112 0 : .join(TIMELINES_SEGMENT_NAME)
113 0 : .join(timeline);
114 0 : let mut idx = 0;
115 0 : let mut to_print = Vec::default();
116 0 : for layer in fs::read_dir(timeline_path)? {
117 0 : let layer = layer?;
118 0 : if let Ok(layer_file) = parse_filename(&layer.file_name().into_string().unwrap()) {
119 0 : if let Some(key) = key {
120 0 : if layer_file.key_range.start <= *key && *key < layer_file.key_range.end {
121 0 : to_print.push((idx, layer_file));
122 0 : }
123 0 : } else {
124 0 : to_print.push((idx, layer_file));
125 0 : }
126 0 : idx += 1;
127 0 : }
128 : }
129 :
130 0 : if key.is_some() {
131 0 : to_print
132 0 : .sort_by_key(|(_idx, layer_file)| std::cmp::Reverse(layer_file.lsn_range.end));
133 0 : }
134 :
135 0 : for (idx, layer_file) in to_print {
136 0 : print_layer_file(idx, &layer_file);
137 0 : }
138 0 : Ok(())
139 : }
140 : LayerCmd::DumpLayer {
141 0 : path,
142 0 : tenant,
143 0 : timeline,
144 0 : id,
145 : } => {
146 0 : let timeline_path = path
147 0 : .join("tenants")
148 0 : .join(tenant)
149 0 : .join("timelines")
150 0 : .join(timeline);
151 0 : let mut idx = 0;
152 0 : for layer in fs::read_dir(timeline_path)? {
153 0 : let layer = layer?;
154 0 : if let Ok(layer_file) = parse_filename(&layer.file_name().into_string().unwrap()) {
155 0 : if *id == idx {
156 0 : print_layer_file(idx, &layer_file);
157 :
158 0 : if layer_file.is_delta {
159 0 : read_delta_file(layer.path(), &ctx).await?;
160 : } else {
161 0 : read_image_file(layer.path(), &ctx).await?;
162 : }
163 :
164 0 : break;
165 0 : }
166 0 : idx += 1;
167 0 : }
168 : }
169 0 : Ok(())
170 : }
171 0 : LayerCmd::DumpLayerLocal { path } => {
172 0 : if let Ok(layer_file) = parse_filename(path.file_name().unwrap().to_str().unwrap()) {
173 0 : print_layer_file(0, &layer_file);
174 :
175 0 : if layer_file.is_delta {
176 0 : read_delta_file(path, &ctx).await?;
177 : } else {
178 0 : read_image_file(path, &ctx).await?;
179 : }
180 0 : }
181 0 : Ok(())
182 : }
183 : LayerCmd::RewriteSummary {
184 0 : layer_file_path,
185 0 : new_tenant_id,
186 0 : new_timeline_id,
187 : } => {
188 0 : pageserver::virtual_file::init(
189 : 10,
190 0 : virtual_file::api::IoEngineKind::StdFs,
191 0 : IoMode::preferred(),
192 0 : virtual_file::SyncMode::Sync,
193 : );
194 0 : pageserver::page_cache::init(100);
195 :
196 0 : let ctx = RequestContext::new(TaskKind::DebugTool, DownloadBehavior::Error)
197 0 : .with_scope_debug_tools();
198 :
199 : macro_rules! rewrite_closure {
200 : ($($summary_ty:tt)*) => {{
201 : |summary| $($summary_ty)* {
202 : tenant_id: new_tenant_id.unwrap_or(summary.tenant_id),
203 : timeline_id: new_timeline_id.unwrap_or(summary.timeline_id),
204 : ..summary
205 0 : }
206 : }};
207 : }
208 :
209 0 : let res = ImageLayer::rewrite_summary(
210 0 : layer_file_path,
211 0 : rewrite_closure!(image_layer::Summary),
212 0 : &ctx,
213 0 : )
214 0 : .await;
215 0 : match res {
216 : Ok(()) => {
217 0 : println!("Successfully rewrote summary of image layer {layer_file_path}");
218 0 : return Ok(());
219 : }
220 0 : Err(image_layer::RewriteSummaryError::MagicMismatch) => (), // fallthrough
221 0 : Err(image_layer::RewriteSummaryError::Other(e)) => {
222 0 : return Err(e);
223 : }
224 : }
225 :
226 0 : let res = DeltaLayer::rewrite_summary(
227 0 : layer_file_path,
228 0 : rewrite_closure!(delta_layer::Summary),
229 0 : &ctx,
230 0 : )
231 0 : .await;
232 0 : match res {
233 : Ok(()) => {
234 0 : println!("Successfully rewrote summary of delta layer {layer_file_path}");
235 0 : return Ok(());
236 : }
237 0 : Err(delta_layer::RewriteSummaryError::MagicMismatch) => (), // fallthrough
238 0 : Err(delta_layer::RewriteSummaryError::Other(e)) => {
239 0 : return Err(e);
240 : }
241 : }
242 :
243 0 : anyhow::bail!("not an image or delta layer: {layer_file_path}");
244 : }
245 : }
246 0 : }
247 :
248 0 : fn print_layer_file(idx: usize, layer_file: &LayerFile) {
249 0 : println!(
250 0 : "[{:3}] key:{}-{}\n lsn:{}-{}\n delta:{}",
251 : idx,
252 : layer_file.key_range.start,
253 : layer_file.key_range.end,
254 : layer_file.lsn_range.start,
255 : layer_file.lsn_range.end,
256 : layer_file.is_delta,
257 : );
258 0 : }
|
