Line data Source code
1 : use std::path::{Path, PathBuf};
2 :
3 : use anyhow::Result;
4 : use camino::{Utf8Path, Utf8PathBuf};
5 : use clap::Subcommand;
6 : use pageserver::context::{DownloadBehavior, RequestContext};
7 : use pageserver::task_mgr::TaskKind;
8 : use pageserver::tenant::block_io::BlockCursor;
9 : use pageserver::tenant::disk_btree::DiskBtreeReader;
10 : use pageserver::tenant::storage_layer::delta_layer::{BlobRef, Summary};
11 : use pageserver::tenant::storage_layer::{delta_layer, image_layer};
12 : use pageserver::tenant::storage_layer::{DeltaLayer, ImageLayer};
13 : use pageserver::tenant::{TENANTS_SEGMENT_NAME, TIMELINES_SEGMENT_NAME};
14 : use pageserver::virtual_file::api::IoMode;
15 : use pageserver::{page_cache, virtual_file};
16 : use pageserver::{
17 : repository::{Key, KEY_SIZE},
18 : tenant::{
19 : block_io::FileBlockReader, disk_btree::VisitDirection,
20 : storage_layer::delta_layer::DELTA_KEY_SIZE,
21 : },
22 : virtual_file::VirtualFile,
23 : };
24 : use std::fs;
25 : use utils::bin_ser::BeSer;
26 : use utils::id::{TenantId, TimelineId};
27 :
28 : use crate::layer_map_analyzer::parse_filename;
29 :
30 0 : #[derive(Subcommand)]
31 : pub(crate) enum LayerCmd {
32 : /// List all tenants and timelines under the pageserver path
33 : ///
34 : /// Example: `cargo run --bin pagectl layer list .neon/`
35 0 : List { path: PathBuf },
36 : /// List all layers of a given tenant and timeline
37 : ///
38 : /// Example: `cargo run --bin pagectl layer list .neon/`
39 : ListLayer {
40 0 : path: PathBuf,
41 0 : tenant: String,
42 0 : timeline: String,
43 : },
44 : /// Dump all information of a layer file
45 : DumpLayer {
46 0 : path: PathBuf,
47 0 : tenant: String,
48 0 : timeline: String,
49 : /// The id from list-layer command
50 0 : id: usize,
51 : },
52 : RewriteSummary {
53 0 : layer_file_path: Utf8PathBuf,
54 : #[clap(long)]
55 : new_tenant_id: Option<TenantId>,
56 : #[clap(long)]
57 : new_timeline_id: Option<TimelineId>,
58 : },
59 : }
60 :
61 0 : async fn read_delta_file(path: impl AsRef<Path>, ctx: &RequestContext) -> Result<()> {
62 0 : let path = Utf8Path::from_path(path.as_ref()).expect("non-Unicode path");
63 0 : virtual_file::init(
64 0 : 10,
65 0 : virtual_file::api::IoEngineKind::StdFs,
66 0 : IoMode::preferred(),
67 0 : );
68 0 : page_cache::init(100);
69 0 : let file = VirtualFile::open(path, ctx).await?;
70 0 : let file_id = page_cache::next_file_id();
71 0 : let block_reader = FileBlockReader::new(&file, file_id);
72 0 : let summary_blk = block_reader.read_blk(0, ctx).await?;
73 0 : let actual_summary = Summary::des_prefix(summary_blk.as_ref())?;
74 0 : let tree_reader = DiskBtreeReader::<_, DELTA_KEY_SIZE>::new(
75 0 : actual_summary.index_start_blk,
76 0 : actual_summary.index_root_blk,
77 0 : &block_reader,
78 0 : );
79 0 : // TODO(chi): dedup w/ `delta_layer.rs` by exposing the API.
80 0 : let mut all = vec![];
81 0 : tree_reader
82 0 : .visit(
83 0 : &[0u8; DELTA_KEY_SIZE],
84 0 : VisitDirection::Forwards,
85 0 : |key, value_offset| {
86 0 : let curr = Key::from_slice(&key[..KEY_SIZE]);
87 0 : all.push((curr, BlobRef(value_offset)));
88 0 : true
89 0 : },
90 0 : ctx,
91 0 : )
92 0 : .await?;
93 0 : let cursor = BlockCursor::new_fileblockreader(&block_reader);
94 0 : for (k, v) in all {
95 0 : let value = cursor.read_blob(v.pos(), ctx).await?;
96 0 : println!("key:{} value_len:{}", k, value.len());
97 0 : assert!(k.is_i128_representable(), "invalid key: ");
98 : }
99 : // TODO(chi): special handling for last key?
100 0 : Ok(())
101 0 : }
102 :
103 0 : pub(crate) async fn main(cmd: &LayerCmd) -> Result<()> {
104 0 : let ctx = RequestContext::new(TaskKind::DebugTool, DownloadBehavior::Error);
105 0 : match cmd {
106 0 : LayerCmd::List { path } => {
107 0 : for tenant in fs::read_dir(path.join(TENANTS_SEGMENT_NAME))? {
108 0 : let tenant = tenant?;
109 0 : if !tenant.file_type()?.is_dir() {
110 0 : continue;
111 0 : }
112 0 : println!("tenant {}", tenant.file_name().to_string_lossy());
113 0 : for timeline in fs::read_dir(tenant.path().join(TIMELINES_SEGMENT_NAME))? {
114 0 : let timeline = timeline?;
115 0 : if !timeline.file_type()?.is_dir() {
116 0 : continue;
117 0 : }
118 0 : println!("- timeline {}", timeline.file_name().to_string_lossy());
119 : }
120 : }
121 0 : Ok(())
122 : }
123 : LayerCmd::ListLayer {
124 0 : path,
125 0 : tenant,
126 0 : timeline,
127 0 : } => {
128 0 : let timeline_path = path
129 0 : .join(TENANTS_SEGMENT_NAME)
130 0 : .join(tenant)
131 0 : .join(TIMELINES_SEGMENT_NAME)
132 0 : .join(timeline);
133 0 : let mut idx = 0;
134 0 : for layer in fs::read_dir(timeline_path)? {
135 0 : let layer = layer?;
136 0 : if let Some(layer_file) = parse_filename(&layer.file_name().into_string().unwrap())
137 0 : {
138 0 : println!(
139 0 : "[{:3}] key:{}-{}\n lsn:{}-{}\n delta:{}",
140 0 : idx,
141 0 : layer_file.key_range.start,
142 0 : layer_file.key_range.end,
143 0 : layer_file.lsn_range.start,
144 0 : layer_file.lsn_range.end,
145 0 : layer_file.is_delta,
146 0 : );
147 0 : idx += 1;
148 0 : }
149 : }
150 0 : Ok(())
151 : }
152 : LayerCmd::DumpLayer {
153 0 : path,
154 0 : tenant,
155 0 : timeline,
156 0 : id,
157 0 : } => {
158 0 : let timeline_path = path
159 0 : .join("tenants")
160 0 : .join(tenant)
161 0 : .join("timelines")
162 0 : .join(timeline);
163 0 : let mut idx = 0;
164 0 : for layer in fs::read_dir(timeline_path)? {
165 0 : let layer = layer?;
166 0 : if let Some(layer_file) = parse_filename(&layer.file_name().into_string().unwrap())
167 : {
168 0 : if *id == idx {
169 : // TODO(chi): dedup code
170 0 : println!(
171 0 : "[{:3}] key:{}-{}\n lsn:{}-{}\n delta:{}",
172 0 : idx,
173 0 : layer_file.key_range.start,
174 0 : layer_file.key_range.end,
175 0 : layer_file.lsn_range.start,
176 0 : layer_file.lsn_range.end,
177 0 : layer_file.is_delta,
178 0 : );
179 0 :
180 0 : if layer_file.is_delta {
181 0 : read_delta_file(layer.path(), &ctx).await?;
182 : } else {
183 0 : anyhow::bail!("not supported yet :(");
184 : }
185 :
186 0 : break;
187 0 : }
188 0 : idx += 1;
189 0 : }
190 : }
191 0 : Ok(())
192 : }
193 : LayerCmd::RewriteSummary {
194 0 : layer_file_path,
195 0 : new_tenant_id,
196 0 : new_timeline_id,
197 0 : } => {
198 0 : pageserver::virtual_file::init(
199 0 : 10,
200 0 : virtual_file::api::IoEngineKind::StdFs,
201 0 : IoMode::preferred(),
202 0 : );
203 0 : pageserver::page_cache::init(100);
204 0 :
205 0 : let ctx = RequestContext::new(TaskKind::DebugTool, DownloadBehavior::Error);
206 :
207 : macro_rules! rewrite_closure {
208 : ($($summary_ty:tt)*) => {{
209 : |summary| $($summary_ty)* {
210 : tenant_id: new_tenant_id.unwrap_or(summary.tenant_id),
211 : timeline_id: new_timeline_id.unwrap_or(summary.timeline_id),
212 : ..summary
213 0 : }
214 : }};
215 : }
216 :
217 0 : let res = ImageLayer::rewrite_summary(
218 0 : layer_file_path,
219 0 : rewrite_closure!(image_layer::Summary),
220 0 : &ctx,
221 0 : )
222 0 : .await;
223 0 : match res {
224 : Ok(()) => {
225 0 : println!("Successfully rewrote summary of image layer {layer_file_path}");
226 0 : return Ok(());
227 : }
228 0 : Err(image_layer::RewriteSummaryError::MagicMismatch) => (), // fallthrough
229 0 : Err(image_layer::RewriteSummaryError::Other(e)) => {
230 0 : return Err(e);
231 : }
232 : }
233 :
234 0 : let res = DeltaLayer::rewrite_summary(
235 0 : layer_file_path,
236 0 : rewrite_closure!(delta_layer::Summary),
237 0 : &ctx,
238 0 : )
239 0 : .await;
240 0 : match res {
241 : Ok(()) => {
242 0 : println!("Successfully rewrote summary of delta layer {layer_file_path}");
243 0 : return Ok(());
244 : }
245 0 : Err(delta_layer::RewriteSummaryError::MagicMismatch) => (), // fallthrough
246 0 : Err(delta_layer::RewriteSummaryError::Other(e)) => {
247 0 : return Err(e);
248 : }
249 : }
250 :
251 0 : anyhow::bail!("not an image or delta layer: {layer_file_path}");
252 : }
253 : }
254 0 : }
|
