Line data Source code
1 : //! Tool for extracting content-dependent metadata about layers. Useful for scanning real project layer files and evaluating the effectiveness of different heuristics on them.
2 : //!
3 : //! Currently it only analyzes holes, which are regions within the layer range that the layer contains no updates for. In the future it might do more analysis (maybe key quantiles?) but it should never return sensitive data.
4 :
5 : use anyhow::{anyhow, Result};
6 : use camino::{Utf8Path, Utf8PathBuf};
7 : use pageserver::context::{DownloadBehavior, RequestContext};
8 : use pageserver::task_mgr::TaskKind;
9 : use pageserver::tenant::{TENANTS_SEGMENT_NAME, TIMELINES_SEGMENT_NAME};
10 : use pageserver::virtual_file::api::IoMode;
11 : use std::cmp::Ordering;
12 : use std::collections::BinaryHeap;
13 : use std::ops::Range;
14 : use std::str::FromStr;
15 : use std::{fs, str};
16 :
17 : use pageserver::page_cache::{self, PAGE_SZ};
18 : use pageserver::tenant::block_io::FileBlockReader;
19 : use pageserver::tenant::disk_btree::{DiskBtreeReader, VisitDirection};
20 : use pageserver::tenant::storage_layer::delta_layer::{Summary, DELTA_KEY_SIZE};
21 : use pageserver::tenant::storage_layer::{range_overlaps, LayerName};
22 : use pageserver::virtual_file::{self, VirtualFile};
23 : use pageserver_api::key::{Key, KEY_SIZE};
24 :
25 : use utils::{bin_ser::BeSer, lsn::Lsn};
26 :
27 : use crate::AnalyzeLayerMapCmd;
28 :
29 : const MIN_HOLE_LENGTH: i128 = (128 * 1024 * 1024 / PAGE_SZ) as i128;
30 : const DEFAULT_MAX_HOLES: usize = 10;
31 :
32 : /// Wrapper for key range to provide reverse ordering by range length for BinaryHeap
33 : #[derive(PartialEq, Eq)]
34 : pub struct Hole(Range<Key>);
35 :
36 : impl Ord for Hole {
37 0 : fn cmp(&self, other: &Self) -> Ordering {
38 0 : let other_len = other.0.end.to_i128() - other.0.start.to_i128();
39 0 : let self_len = self.0.end.to_i128() - self.0.start.to_i128();
40 0 : other_len.cmp(&self_len)
41 0 : }
42 : }
43 :
44 : impl PartialOrd for Hole {
45 0 : fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
46 0 : Some(self.cmp(other))
47 0 : }
48 : }
49 :
50 : pub(crate) struct LayerFile {
51 : pub key_range: Range<Key>,
52 : pub lsn_range: Range<Lsn>,
53 : pub is_delta: bool,
54 : pub holes: Vec<Hole>,
55 : }
56 :
57 : impl LayerFile {
58 0 : fn skips(&self, key_range: &Range<Key>) -> bool {
59 0 : if !range_overlaps(&self.key_range, key_range) {
60 0 : return false;
61 0 : }
62 0 : let start = match self
63 0 : .holes
64 0 : .binary_search_by_key(&key_range.start, |hole| hole.0.start)
65 : {
66 0 : Ok(index) => index,
67 0 : Err(index) => {
68 0 : if index == 0 {
69 0 : return false;
70 0 : }
71 0 : index - 1
72 : }
73 : };
74 0 : self.holes[start].0.end >= key_range.end
75 0 : }
76 : }
77 :
78 0 : pub(crate) fn parse_filename(name: &str) -> anyhow::Result<LayerFile> {
79 0 : let layer_name =
80 0 : LayerName::from_str(name).map_err(|e| anyhow!("failed to parse layer name: {e}"))?;
81 :
82 0 : let holes = Vec::new();
83 0 : Ok(LayerFile {
84 0 : key_range: layer_name.key_range().clone(),
85 0 : lsn_range: layer_name.lsn_as_range(),
86 0 : is_delta: layer_name.is_delta(),
87 0 : holes,
88 0 : })
89 0 : }
90 :
91 : // Finds the max_holes largest holes, ignoring any that are smaller than MIN_HOLE_LENGTH"
92 0 : async fn get_holes(path: &Utf8Path, max_holes: usize, ctx: &RequestContext) -> Result<Vec<Hole>> {
93 0 : let file = VirtualFile::open(path, ctx).await?;
94 0 : let file_id = page_cache::next_file_id();
95 0 : let block_reader = FileBlockReader::new(&file, file_id);
96 0 : let summary_blk = block_reader.read_blk(0, ctx).await?;
97 0 : let actual_summary = Summary::des_prefix(summary_blk.as_ref())?;
98 0 : let tree_reader = DiskBtreeReader::<_, DELTA_KEY_SIZE>::new(
99 0 : actual_summary.index_start_blk,
100 0 : actual_summary.index_root_blk,
101 0 : block_reader,
102 0 : );
103 0 : // min-heap (reserve space for one more element added before eviction)
104 0 : let mut heap: BinaryHeap<Hole> = BinaryHeap::with_capacity(max_holes + 1);
105 0 : let mut prev_key: Option<Key> = None;
106 0 : tree_reader
107 0 : .visit(
108 0 : &[0u8; DELTA_KEY_SIZE],
109 0 : VisitDirection::Forwards,
110 0 : |key, _value| {
111 0 : let curr = Key::from_slice(&key[..KEY_SIZE]);
112 0 : if let Some(prev) = prev_key {
113 0 : if curr.to_i128() - prev.to_i128() >= MIN_HOLE_LENGTH {
114 0 : heap.push(Hole(prev..curr));
115 0 : if heap.len() > max_holes {
116 0 : heap.pop(); // remove smallest hole
117 0 : }
118 0 : }
119 0 : }
120 0 : prev_key = Some(curr.next());
121 0 : true
122 0 : },
123 0 : ctx,
124 0 : )
125 0 : .await?;
126 0 : let mut holes = heap.into_vec();
127 0 : holes.sort_by_key(|hole| hole.0.start);
128 0 : Ok(holes)
129 0 : }
130 :
131 0 : pub(crate) async fn main(cmd: &AnalyzeLayerMapCmd) -> Result<()> {
132 0 : let storage_path = &cmd.path;
133 0 : let max_holes = cmd.max_holes.unwrap_or(DEFAULT_MAX_HOLES);
134 0 : let ctx = RequestContext::new(TaskKind::DebugTool, DownloadBehavior::Error);
135 0 :
136 0 : // Initialize virtual_file (file desriptor cache) and page cache which are needed to access layer persistent B-Tree.
137 0 : pageserver::virtual_file::init(
138 0 : 10,
139 0 : virtual_file::api::IoEngineKind::StdFs,
140 0 : IoMode::preferred(),
141 0 : virtual_file::SyncMode::Sync,
142 0 : );
143 0 : pageserver::page_cache::init(100);
144 0 :
145 0 : let mut total_delta_layers = 0usize;
146 0 : let mut total_image_layers = 0usize;
147 0 : let mut total_excess_layers = 0usize;
148 0 : for tenant in fs::read_dir(storage_path.join(TENANTS_SEGMENT_NAME))? {
149 0 : let tenant = tenant?;
150 0 : if !tenant.file_type()?.is_dir() {
151 0 : continue;
152 0 : }
153 0 : for timeline in fs::read_dir(tenant.path().join(TIMELINES_SEGMENT_NAME))? {
154 0 : let timeline = timeline?;
155 0 : if !timeline.file_type()?.is_dir() {
156 0 : continue;
157 0 : }
158 0 : // Collect sorted vec of layers and count deltas
159 0 : let mut layers = Vec::new();
160 0 : let mut n_deltas = 0usize;
161 :
162 0 : for layer in fs::read_dir(timeline.path())? {
163 0 : let layer = layer?;
164 0 : if let Ok(mut layer_file) =
165 0 : parse_filename(&layer.file_name().into_string().unwrap())
166 : {
167 0 : if layer_file.is_delta {
168 0 : let layer_path =
169 0 : Utf8PathBuf::from_path_buf(layer.path()).expect("non-Unicode path");
170 0 : layer_file.holes = get_holes(&layer_path, max_holes, &ctx).await?;
171 0 : n_deltas += 1;
172 0 : }
173 0 : layers.push(layer_file);
174 0 : }
175 : }
176 0 : layers.sort_by_key(|layer| layer.lsn_range.end);
177 0 :
178 0 : // Count the number of holes and number of excess layers.
179 0 : // Excess layer is image layer generated when holes in delta layers are not considered.
180 0 : let mut n_excess_layers = 0usize;
181 0 : let mut n_holes = 0usize;
182 :
183 0 : for i in 0..layers.len() {
184 0 : if !layers[i].is_delta {
185 0 : let mut n_deltas_since_last_image = 0usize;
186 0 : let mut n_skipped = 0usize;
187 0 : let img_key_range = &layers[i].key_range;
188 0 : for j in (0..i).rev() {
189 0 : if range_overlaps(img_key_range, &layers[j].key_range) {
190 0 : if layers[j].is_delta {
191 0 : n_deltas_since_last_image += 1;
192 0 : if layers[j].skips(img_key_range) {
193 0 : n_skipped += 1;
194 0 : }
195 : } else {
196 : // Image layer is always dense, despite to the fact that it doesn't contain all possible
197 : // key values in the specified range: there are may be no keys in the storage belonging
198 : // to the image layer range but not present in the image layer.
199 0 : break;
200 : }
201 0 : }
202 : }
203 0 : if n_deltas_since_last_image >= 3 && n_deltas_since_last_image - n_skipped < 3 {
204 0 : // It is just approximation: it doesn't take in account all image coverage.
205 0 : // Moreover the new layer map doesn't count total deltas, but the max stack of overlapping deltas.
206 0 : n_excess_layers += 1;
207 0 : }
208 0 : n_holes += n_skipped;
209 0 : }
210 : }
211 0 : println!(
212 0 : "Tenant {} timeline {} delta layers {} image layers {} excess layers {} holes {}",
213 0 : tenant.file_name().into_string().unwrap(),
214 0 : timeline.file_name().into_string().unwrap(),
215 0 : n_deltas,
216 0 : layers.len() - n_deltas,
217 0 : n_excess_layers,
218 0 : n_holes
219 0 : );
220 0 : total_delta_layers += n_deltas;
221 0 : total_image_layers += layers.len() - n_deltas;
222 0 : total_excess_layers += n_excess_layers;
223 : }
224 : }
225 0 : println!(
226 0 : "Total delta layers {} image layers {} excess layers {}",
227 0 : total_delta_layers, total_image_layers, total_excess_layers
228 0 : );
229 0 : Ok(())
230 0 : }
|
