Line data Source code
1 : //! A tool for visualizing the arrangement of layerfiles within a timeline.
2 : //!
3 : //! It reads filenames from stdin and prints a svg on stdout. The image is a plot in
4 : //! page-lsn space, where every delta layer is a rectangle and every image layer is a
5 : //! thick line. Legend:
6 : //! - The x axis (left to right) represents page index.
7 : //! - The y axis represents LSN, growing upwards.
8 : //!
9 : //! Coordinates in both axis are compressed for better readability.
10 : //! (see <https://medium.com/algorithms-digest/coordinate-compression-2fff95326fb>)
11 : //!
12 : //! The plain text API was chosen so that we can easily work with filenames from various
13 : //! sources; see the Usage section below for examples.
14 : //!
15 : //! # Usage
16 : //!
17 : //! ## Producing the SVG
18 : //!
19 : //! ```bash
20 : //!
21 : //! # local timeline dir
22 : //! ls test_output/test_pgbench\[neon-45-684\]/repo/tenants/$TENANT/timelines/$TIMELINE | \
23 : //! grep "__" | cargo run --release --bin pagectl draw-timeline > out.svg
24 : //!
25 : //! # Layer map dump from `/v1/tenant/$TENANT/timeline/$TIMELINE/layer`
26 : //! (jq -r '.historic_layers[] | .layer_file_name' | cargo run -p pagectl draw-timeline) < layer-map.json > out.svg
27 : //!
28 : //! # From an `index_part.json` in S3
29 : //! (jq -r '.layer_metadata | keys[]' | cargo run -p pagectl draw-timeline ) < index_part.json-00000016 > out.svg
30 : //!
31 : //! # enrich with lines for gc_cutoff and a child branch point
32 : //! cat <(jq -r '.historic_layers[] | .layer_file_name' < layers.json) <(echo -e 'gc_cutoff:0000001CE3FE32C9\nbranch:0000001DE3FE32C9') | cargo run --bin pagectl draw-timeline >| out.svg
33 : //! ```
34 : //!
35 : //! ## Viewing
36 : //!
37 : //! **Inkscape** is better than the built-in viewers in browsers.
38 : //!
39 : //! After selecting a layer file rectangle, use "Open XML Editor" (Ctrl|Cmd + Shift + X)
40 : //! to see the layer file name in the comment field.
41 : //!
42 : //! ```bash
43 : //!
44 : //! # Linux
45 : //! inkscape out.svg
46 : //!
47 : //! # macOS
48 : //! /Applications/Inkscape.app/Contents/MacOS/inkscape out.svg
49 : //!
50 : //! ```
51 : //!
52 :
53 : use std::cmp::Ordering;
54 : use std::collections::{BTreeMap, BTreeSet};
55 : use std::io::{self, BufRead};
56 : use std::ops::Range;
57 : use std::path::PathBuf;
58 : use std::str::FromStr;
59 :
60 : use anyhow::{Context, Result};
61 : use pageserver_api::key::Key;
62 : use svg_fmt::{BeginSvg, EndSvg, Fill, Stroke, rectangle, rgb};
63 : use utils::lsn::Lsn;
64 : use utils::project_git_version;
65 :
66 : project_git_version!(GIT_VERSION);
67 :
68 : // Map values to their compressed coordinate - the index the value
69 : // would have in a sorted and deduplicated list of all values.
70 0 : fn build_coordinate_compression_map<T: Ord + Copy>(coords: Vec<T>) -> BTreeMap<T, usize> {
71 0 : let set: BTreeSet<T> = coords.into_iter().collect();
72 :
73 0 : let mut map: BTreeMap<T, usize> = BTreeMap::new();
74 0 : for (i, e) in set.iter().enumerate() {
75 0 : map.insert(*e, i);
76 0 : }
77 :
78 0 : map
79 0 : }
80 :
81 0 : fn parse_filename(name: &str) -> (Range<Key>, Range<Lsn>) {
82 0 : let split: Vec<&str> = name.split("__").collect();
83 0 : let keys: Vec<&str> = split[0].split('-').collect();
84 :
85 : // Remove the temporary file extension, e.g., remove the `.d20a.___temp` part from the following filename:
86 : // 000000067F000040490000404A00441B0000-000000067F000040490000404A00441B4000__000043483A34CE00.d20a.___temp
87 0 : let lsns = split[1].split('.').collect::<Vec<&str>>()[0];
88 0 : let mut lsns: Vec<&str> = lsns.split('-').collect();
89 :
90 : // The current format of the layer file name: 000000067F0000000400000B150100000000-000000067F0000000400000D350100000000__00000000014B7AC8-v1-00000001
91 :
92 : // Handle generation number `-00000001` part
93 0 : if lsns.last().expect("should").len() == 8 {
94 0 : lsns.pop();
95 0 : }
96 :
97 : // Handle version number `-v1` part
98 0 : if lsns.last().expect("should").starts_with('v') {
99 0 : lsns.pop();
100 0 : }
101 :
102 0 : if lsns.len() == 1 {
103 0 : lsns.push(lsns[0]);
104 0 : }
105 :
106 0 : let keys = Key::from_hex(keys[0]).unwrap()..Key::from_hex(keys[1]).unwrap();
107 0 : let lsns = Lsn::from_hex(lsns[0]).unwrap()..Lsn::from_hex(lsns[1]).unwrap();
108 0 : (keys, lsns)
109 0 : }
110 :
111 : #[derive(Clone, Copy)]
112 : enum LineKind {
113 : GcCutoff,
114 : Branch,
115 : }
116 :
117 : impl From<LineKind> for Fill {
118 0 : fn from(value: LineKind) -> Self {
119 0 : match value {
120 0 : LineKind::GcCutoff => Fill::Color(rgb(255, 0, 0)),
121 0 : LineKind::Branch => Fill::Color(rgb(0, 255, 0)),
122 : }
123 0 : }
124 : }
125 :
126 : impl FromStr for LineKind {
127 : type Err = anyhow::Error;
128 :
129 0 : fn from_str(s: &str) -> std::prelude::v1::Result<Self, Self::Err> {
130 0 : Ok(match s {
131 0 : "gc_cutoff" => LineKind::GcCutoff,
132 0 : "branch" => LineKind::Branch,
133 0 : _ => anyhow::bail!("unsupported linekind: {s}"),
134 : })
135 0 : }
136 : }
137 :
138 0 : pub fn main() -> Result<()> {
139 : // Parse layer filenames from stdin
140 : struct Layer {
141 : filename: String,
142 : key_range: Range<Key>,
143 : lsn_range: Range<Lsn>,
144 : }
145 0 : let mut files: Vec<Layer> = vec![];
146 0 : let stdin = io::stdin();
147 :
148 0 : let mut lines: Vec<(Lsn, LineKind)> = vec![];
149 :
150 0 : for (lineno, line) in stdin.lock().lines().enumerate() {
151 0 : let lineno = lineno + 1;
152 :
153 0 : let line = line.unwrap();
154 0 : if let Some((kind, lsn)) = line.split_once(':') {
155 0 : let (kind, lsn) = LineKind::from_str(kind)
156 0 : .context("parse kind")
157 0 : .and_then(|kind| {
158 0 : if lsn.contains('/') {
159 0 : Lsn::from_str(lsn)
160 : } else {
161 0 : Lsn::from_hex(lsn)
162 : }
163 0 : .map(|lsn| (kind, lsn))
164 0 : .context("parse lsn")
165 0 : })
166 0 : .with_context(|| format!("parse {line:?} on {lineno}"))?;
167 0 : lines.push((lsn, kind));
168 0 : continue;
169 0 : }
170 0 : let line = PathBuf::from_str(&line).unwrap();
171 0 : let filename = line.file_name().unwrap();
172 0 : let filename = filename.to_str().unwrap();
173 0 : let (key_range, lsn_range) = parse_filename(filename);
174 0 : files.push(Layer {
175 0 : filename: filename.to_owned(),
176 0 : key_range,
177 0 : lsn_range,
178 0 : });
179 : }
180 :
181 : // Collect all coordinates
182 0 : let mut keys: Vec<Key> = Vec::with_capacity(files.len());
183 0 : let mut lsns: Vec<Lsn> = Vec::with_capacity(files.len() + lines.len());
184 :
185 : for Layer {
186 0 : key_range: keyr,
187 0 : lsn_range: lsnr,
188 : ..
189 0 : } in &files
190 0 : {
191 0 : keys.push(keyr.start);
192 0 : keys.push(keyr.end);
193 0 : lsns.push(lsnr.start);
194 0 : lsns.push(lsnr.end);
195 0 : }
196 :
197 0 : lsns.extend(lines.iter().map(|(lsn, _)| *lsn));
198 :
199 : // Analyze
200 0 : let key_map = build_coordinate_compression_map(keys);
201 0 : let lsn_map = build_coordinate_compression_map(lsns);
202 :
203 : // Initialize stats
204 0 : let mut num_deltas = 0;
205 0 : let mut num_images = 0;
206 :
207 : // Draw
208 0 : let stretch = 3.0; // Stretch out vertically for better visibility
209 0 : println!(
210 0 : "{}",
211 0 : BeginSvg {
212 0 : w: (key_map.len() + 10) as f32,
213 0 : h: stretch * lsn_map.len() as f32
214 0 : }
215 : );
216 :
217 0 : let xmargin = 0.05; // Height-dependent margin to disambiguate overlapping deltas
218 :
219 : for Layer {
220 0 : filename,
221 0 : key_range: keyr,
222 0 : lsn_range: lsnr,
223 0 : } in &files
224 : {
225 0 : let key_start = *key_map.get(&keyr.start).unwrap();
226 0 : let key_end = *key_map.get(&keyr.end).unwrap();
227 0 : let key_diff = key_end - key_start;
228 0 : let lsn_max = lsn_map.len();
229 :
230 0 : if key_start >= key_end {
231 0 : panic!("Invalid key range {key_start}-{key_end}");
232 0 : }
233 :
234 0 : let lsn_start = *lsn_map.get(&lsnr.start).unwrap();
235 0 : let lsn_end = *lsn_map.get(&lsnr.end).unwrap();
236 :
237 0 : let mut lsn_diff = (lsn_end - lsn_start) as f32;
238 0 : let mut fill = Fill::None;
239 0 : let mut ymargin = 0.05 * lsn_diff; // Height-dependent margin to disambiguate overlapping deltas
240 0 : let mut lsn_offset = 0.0;
241 :
242 : // Fill in and thicken rectangle if it's an
243 : // image layer so that we can see it.
244 0 : match lsn_start.cmp(&lsn_end) {
245 0 : Ordering::Less => num_deltas += 1,
246 0 : Ordering::Equal => {
247 0 : num_images += 1;
248 0 : lsn_diff = 0.3;
249 0 : lsn_offset = -lsn_diff / 2.0;
250 0 : ymargin = 0.05;
251 0 : fill = Fill::Color(rgb(0, 0, 0));
252 0 : }
253 0 : Ordering::Greater => panic!("Invalid lsn range {lsn_start}-{lsn_end}"),
254 : }
255 :
256 0 : println!(
257 0 : " {}",
258 0 : rectangle(
259 0 : 5.0 + key_start as f32 + stretch * xmargin,
260 0 : stretch * (lsn_max as f32 - (lsn_end as f32 - ymargin - lsn_offset)),
261 0 : key_diff as f32 - stretch * 2.0 * xmargin,
262 0 : stretch * (lsn_diff - 2.0 * ymargin)
263 : )
264 0 : .fill(fill)
265 0 : .stroke(Stroke::Color(rgb(0, 0, 0), 0.1))
266 0 : .border_radius(0.4)
267 0 : .comment(filename)
268 : );
269 : }
270 :
271 0 : for (lsn, kind) in lines {
272 0 : let lsn_start = *lsn_map.get(&lsn).unwrap();
273 0 : let lsn_end = lsn_start;
274 0 : let stretch = 2.0;
275 0 : let lsn_diff = 0.3;
276 0 : let lsn_offset = -lsn_diff / 2.0;
277 0 : let ymargin = 0.05;
278 0 : println!(
279 0 : "{}",
280 0 : rectangle(
281 0 : 0.0f32 + stretch * xmargin,
282 0 : stretch * (lsn_map.len() as f32 - (lsn_end as f32 - ymargin - lsn_offset)),
283 0 : (key_map.len() + 10) as f32,
284 0 : stretch * (lsn_diff - 2.0 * ymargin)
285 0 : )
286 0 : .fill(kind)
287 0 : );
288 0 : }
289 :
290 0 : println!("{EndSvg}");
291 :
292 0 : eprintln!("num_images: {num_images}");
293 0 : eprintln!("num_deltas: {num_deltas}");
294 :
295 0 : Ok(())
296 0 : }
|