Line data Source code
1 : use std::str::FromStr;
2 :
3 : use anyhow::{Context, Ok};
4 : use camino::Utf8PathBuf;
5 : use pageserver::tenant::{
6 : IndexPart,
7 : layer_map::{LayerMap, SearchResult},
8 : remote_timeline_client::{index::LayerFileMetadata, remote_layer_path},
9 : storage_layer::{LayerName, LayerVisibilityHint, PersistentLayerDesc, ReadableLayerWeak},
10 : };
11 : use pageserver_api::key::Key;
12 : use serde::Serialize;
13 : use std::collections::BTreeMap;
14 : use utils::{
15 : id::{TenantId, TimelineId},
16 : lsn::Lsn,
17 : shard::TenantShardId,
18 : };
19 :
20 : #[derive(clap::Subcommand)]
21 : pub(crate) enum IndexPartCmd {
22 : Dump {
23 : path: Utf8PathBuf,
24 : },
25 : /// Find all layers that need to be searched to construct the given page at the given LSN.
26 : Search {
27 : #[arg(long)]
28 : tenant_id: String,
29 : #[arg(long)]
30 : timeline_id: String,
31 : #[arg(long)]
32 : path: Utf8PathBuf,
33 : #[arg(long)]
34 : key: String,
35 : #[arg(long)]
36 : lsn: String,
37 : },
38 : /// List all visible delta and image layers at the latest LSN.
39 : ListVisibleLayers {
40 : #[arg(long)]
41 : path: Utf8PathBuf,
42 : },
43 : }
44 :
45 0 : fn create_layer_map_from_index_part(
46 0 : index_part: &IndexPart,
47 0 : tenant_shard_id: TenantShardId,
48 0 : timeline_id: TimelineId,
49 0 : ) -> LayerMap {
50 0 : let mut layer_map = LayerMap::default();
51 : {
52 0 : let mut updates = layer_map.batch_update();
53 0 : for (key, value) in index_part.layer_metadata.iter() {
54 0 : updates.insert_historic(PersistentLayerDesc::from_filename(
55 0 : tenant_shard_id,
56 0 : timeline_id,
57 0 : key.clone(),
58 0 : value.file_size,
59 0 : ));
60 0 : }
61 : }
62 0 : layer_map
63 0 : }
64 :
65 0 : async fn search_layers(
66 0 : tenant_id: &str,
67 0 : timeline_id: &str,
68 0 : path: &Utf8PathBuf,
69 0 : key: &str,
70 0 : lsn: &str,
71 0 : ) -> anyhow::Result<()> {
72 0 : let tenant_id = TenantId::from_str(tenant_id).unwrap();
73 0 : let tenant_shard_id = TenantShardId::unsharded(tenant_id);
74 0 : let timeline_id = TimelineId::from_str(timeline_id).unwrap();
75 0 : let index_json = {
76 0 : let bytes = tokio::fs::read(path).await?;
77 0 : IndexPart::from_json_bytes(&bytes).unwrap()
78 : };
79 0 : let layer_map = create_layer_map_from_index_part(&index_json, tenant_shard_id, timeline_id);
80 0 : let key = Key::from_hex(key)?;
81 :
82 0 : let lsn = Lsn::from_str(lsn).unwrap();
83 0 : let mut end_lsn = lsn;
84 : loop {
85 0 : let result = layer_map.search(key, end_lsn);
86 0 : match result {
87 0 : Some(SearchResult { layer, lsn_floor }) => {
88 0 : let disk_layer = match layer {
89 0 : ReadableLayerWeak::PersistentLayer(layer) => layer,
90 : ReadableLayerWeak::InMemoryLayer(_) => {
91 0 : anyhow::bail!("unexpected in-memory layer")
92 : }
93 : };
94 :
95 0 : let metadata = index_json
96 0 : .layer_metadata
97 0 : .get(&disk_layer.layer_name())
98 0 : .unwrap();
99 0 : println!(
100 0 : "{}",
101 0 : remote_layer_path(
102 0 : &tenant_id,
103 0 : &timeline_id,
104 0 : metadata.shard,
105 0 : &disk_layer.layer_name(),
106 0 : metadata.generation
107 : )
108 : );
109 0 : end_lsn = lsn_floor;
110 : }
111 0 : None => break,
112 : }
113 : }
114 0 : Ok(())
115 0 : }
116 :
117 : #[derive(Debug, Clone, Serialize)]
118 : struct VisibleLayers {
119 : pub total_images: u64,
120 : pub total_image_bytes: u64,
121 : pub total_deltas: u64,
122 : pub total_delta_bytes: u64,
123 : pub layer_metadata: BTreeMap<LayerName, LayerFileMetadata>,
124 : }
125 :
126 : impl VisibleLayers {
127 0 : pub fn new() -> Self {
128 0 : Self {
129 0 : layer_metadata: BTreeMap::new(),
130 0 : total_images: 0,
131 0 : total_image_bytes: 0,
132 0 : total_deltas: 0,
133 0 : total_delta_bytes: 0,
134 0 : }
135 0 : }
136 :
137 0 : pub fn add_layer(&mut self, name: LayerName, layer: LayerFileMetadata) {
138 0 : match name {
139 0 : LayerName::Image(_) => {
140 0 : self.total_images += 1;
141 0 : self.total_image_bytes += layer.file_size;
142 0 : }
143 0 : LayerName::Delta(_) => {
144 0 : self.total_deltas += 1;
145 0 : self.total_delta_bytes += layer.file_size;
146 0 : }
147 : }
148 0 : self.layer_metadata.insert(name, layer);
149 0 : }
150 : }
151 :
152 0 : async fn list_visible_layers(path: &Utf8PathBuf) -> anyhow::Result<()> {
153 0 : let tenant_id = TenantId::generate();
154 0 : let tenant_shard_id = TenantShardId::unsharded(tenant_id);
155 0 : let timeline_id = TimelineId::generate();
156 :
157 0 : let bytes = tokio::fs::read(path).await.context("read file")?;
158 0 : let index_part = IndexPart::from_json_bytes(&bytes).context("deserialize")?;
159 0 : let layer_map = create_layer_map_from_index_part(&index_part, tenant_shard_id, timeline_id);
160 0 : let mut visible_layers = VisibleLayers::new();
161 0 : let (layers, _key_space) = layer_map.get_visibility(Vec::new());
162 0 : for (layer, visibility) in layers {
163 0 : if visibility == LayerVisibilityHint::Visible {
164 0 : visible_layers.add_layer(
165 0 : layer.layer_name(),
166 0 : index_part
167 0 : .layer_metadata
168 0 : .get(&layer.layer_name())
169 0 : .unwrap()
170 0 : .clone(),
171 0 : );
172 0 : }
173 : }
174 0 : let output = serde_json::to_string_pretty(&visible_layers).context("serialize output")?;
175 0 : println!("{output}");
176 :
177 0 : Ok(())
178 0 : }
179 :
180 0 : pub(crate) async fn main(cmd: &IndexPartCmd) -> anyhow::Result<()> {
181 0 : match cmd {
182 0 : IndexPartCmd::Dump { path } => {
183 0 : let bytes = tokio::fs::read(path).await.context("read file")?;
184 0 : let des: IndexPart = IndexPart::from_json_bytes(&bytes).context("deserialize")?;
185 0 : let output = serde_json::to_string_pretty(&des).context("serialize output")?;
186 0 : println!("{output}");
187 0 : Ok(())
188 : }
189 : IndexPartCmd::Search {
190 0 : tenant_id,
191 0 : timeline_id,
192 0 : path,
193 0 : key,
194 0 : lsn,
195 0 : } => search_layers(tenant_id, timeline_id, path, key, lsn).await,
196 0 : IndexPartCmd::ListVisibleLayers { path } => list_visible_layers(path).await,
197 : }
198 0 : }
|
