Line data Source code
1 : //! A tool for visualizing the arrangement of layerfiles within a timeline.
2 : //!
3 : //! It reads filenames from stdin and prints a svg on stdout. The image is a plot in
4 : //! page-lsn space, where every delta layer is a rectangle and every image layer is a
5 : //! thick line. Legend:
6 : //! - The x axis (left to right) represents page index.
7 : //! - The y axis represents LSN, growing upwards.
8 : //!
9 : //! Coordinates in both axis are compressed for better readability.
10 : //! (see <https://medium.com/algorithms-digest/coordinate-compression-2fff95326fb>)
11 : //!
12 : //! The plain text API was chosen so that we can easily work with filenames from various
13 : //! sources; see the Usage section below for examples.
14 : //!
15 : //! # Usage
16 : //!
17 : //! ## Producing the SVG
18 : //!
19 : //! ```bash
20 : //!
21 : //! # local timeline dir
22 : //! ls test_output/test_pgbench\[neon-45-684\]/repo/tenants/$TENANT/timelines/$TIMELINE | \
23 : //! grep "__" | cargo run --release --bin pagectl draw-timeline-dir > out.svg
24 : //!
25 : //! # Layer map dump from `/v1/tenant/$TENANT/timeline/$TIMELINE/layer`
26 : //! (jq -r '.historic_layers[] | .layer_file_name' | cargo run -p pagectl draw-timeline) < layer-map.json > out.svg
27 : //!
28 : //! # From an `index_part.json` in S3
29 : //! (jq -r '.layer_metadata | keys[]' | cargo run -p pagectl draw-timeline ) < index_part.json-00000016 > out.svg
30 : //!
31 : //! # enrich with lines for gc_cutoff and a child branch point
32 : //! cat <(jq -r '.historic_layers[] | .layer_file_name' < layers.json) <(echo -e 'gc_cutoff:0000001CE3FE32C9\nbranch:0000001DE3FE32C9') | cargo run --bin pagectl draw-timeline >| out.svg
33 : //! ```
34 : //!
35 : //! ## Viewing
36 : //!
37 : //! **Inkscape** is better than the built-in viewers in browsers.
38 : //!
39 : //! After selecting a layer file rectangle, use "Open XML Editor" (Ctrl|Cmd + Shift + X)
40 : //! to see the layer file name in the comment field.
41 : //!
42 : //! ```bash
43 : //!
44 : //! # Linux
45 : //! inkscape out.svg
46 : //!
47 : //! # macOS
48 : //! /Applications/Inkscape.app/Contents/MacOS/inkscape out.svg
49 : //!
50 : //! ```
51 : //!
52 :
53 : use anyhow::{Context, Result};
54 : use pageserver::repository::Key;
55 : use std::cmp::Ordering;
56 : use std::io::{self, BufRead};
57 : use std::path::PathBuf;
58 : use std::str::FromStr;
59 : use std::{
60 : collections::{BTreeMap, BTreeSet},
61 : ops::Range,
62 : };
63 : use svg_fmt::{rectangle, rgb, BeginSvg, EndSvg, Fill, Stroke};
64 : use utils::{lsn::Lsn, project_git_version};
65 :
66 : project_git_version!(GIT_VERSION);
67 :
68 : // Map values to their compressed coordinate - the index the value
69 : // would have in a sorted and deduplicated list of all values.
70 0 : fn build_coordinate_compression_map<T: Ord + Copy>(coords: Vec<T>) -> BTreeMap<T, usize> {
71 0 : let set: BTreeSet<T> = coords.into_iter().collect();
72 0 :
73 0 : let mut map: BTreeMap<T, usize> = BTreeMap::new();
74 0 : for (i, e) in set.iter().enumerate() {
75 0 : map.insert(*e, i);
76 0 : }
77 :
78 0 : map
79 0 : }
80 :
81 0 : fn parse_filename(name: &str) -> (Range<Key>, Range<Lsn>) {
82 0 : let split: Vec<&str> = name.split("__").collect();
83 0 : let keys: Vec<&str> = split[0].split('-').collect();
84 0 : let mut lsns: Vec<&str> = split[1].split('-').collect();
85 0 :
86 0 : // The current format of the layer file name: 000000067F0000000400000B150100000000-000000067F0000000400000D350100000000__00000000014B7AC8-v1-00000001
87 0 :
88 0 : // Handle generation number `-00000001` part
89 0 : if lsns.last().expect("should").len() == 8 {
90 0 : lsns.pop();
91 0 : }
92 :
93 : // Handle version number `-v1` part
94 0 : if lsns.last().expect("should").starts_with('v') {
95 0 : lsns.pop();
96 0 : }
97 :
98 0 : if lsns.len() == 1 {
99 0 : lsns.push(lsns[0]);
100 0 : }
101 :
102 0 : let keys = Key::from_hex(keys[0]).unwrap()..Key::from_hex(keys[1]).unwrap();
103 0 : let lsns = Lsn::from_hex(lsns[0]).unwrap()..Lsn::from_hex(lsns[1]).unwrap();
104 0 : (keys, lsns)
105 0 : }
106 :
107 : #[derive(Clone, Copy)]
108 : enum LineKind {
109 : GcCutoff,
110 : Branch,
111 : }
112 :
113 : impl From<LineKind> for Fill {
114 0 : fn from(value: LineKind) -> Self {
115 0 : match value {
116 0 : LineKind::GcCutoff => Fill::Color(rgb(255, 0, 0)),
117 0 : LineKind::Branch => Fill::Color(rgb(0, 255, 0)),
118 : }
119 0 : }
120 : }
121 :
122 : impl FromStr for LineKind {
123 : type Err = anyhow::Error;
124 :
125 0 : fn from_str(s: &str) -> std::prelude::v1::Result<Self, Self::Err> {
126 0 : Ok(match s {
127 0 : "gc_cutoff" => LineKind::GcCutoff,
128 0 : "branch" => LineKind::Branch,
129 0 : _ => anyhow::bail!("unsupported linekind: {s}"),
130 : })
131 0 : }
132 : }
133 :
134 0 : pub fn main() -> Result<()> {
135 0 : // Parse layer filenames from stdin
136 0 : struct Layer {
137 0 : filename: String,
138 0 : key_range: Range<Key>,
139 0 : lsn_range: Range<Lsn>,
140 0 : }
141 0 : let mut files: Vec<Layer> = vec![];
142 0 : let stdin = io::stdin();
143 0 :
144 0 : let mut lines: Vec<(Lsn, LineKind)> = vec![];
145 :
146 0 : for (lineno, line) in stdin.lock().lines().enumerate() {
147 0 : let lineno = lineno + 1;
148 0 :
149 0 : let line = line.unwrap();
150 0 : if let Some((kind, lsn)) = line.split_once(':') {
151 0 : let (kind, lsn) = LineKind::from_str(kind)
152 0 : .context("parse kind")
153 0 : .and_then(|kind| {
154 0 : if lsn.contains('/') {
155 0 : Lsn::from_str(lsn)
156 : } else {
157 0 : Lsn::from_hex(lsn)
158 : }
159 0 : .map(|lsn| (kind, lsn))
160 0 : .context("parse lsn")
161 0 : })
162 0 : .with_context(|| format!("parse {line:?} on {lineno}"))?;
163 0 : lines.push((lsn, kind));
164 0 : continue;
165 0 : }
166 0 : let line = PathBuf::from_str(&line).unwrap();
167 0 : let filename = line.file_name().unwrap();
168 0 : let filename = filename.to_str().unwrap();
169 0 : let (key_range, lsn_range) = parse_filename(filename);
170 0 : files.push(Layer {
171 0 : filename: filename.to_owned(),
172 0 : key_range,
173 0 : lsn_range,
174 0 : });
175 : }
176 :
177 : // Collect all coordinates
178 0 : let mut keys: Vec<Key> = Vec::with_capacity(files.len());
179 0 : let mut lsns: Vec<Lsn> = Vec::with_capacity(files.len() + lines.len());
180 :
181 : for Layer {
182 0 : key_range: keyr,
183 0 : lsn_range: lsnr,
184 : ..
185 0 : } in &files
186 0 : {
187 0 : keys.push(keyr.start);
188 0 : keys.push(keyr.end);
189 0 : lsns.push(lsnr.start);
190 0 : lsns.push(lsnr.end);
191 0 : }
192 :
193 0 : lsns.extend(lines.iter().map(|(lsn, _)| *lsn));
194 0 :
195 0 : // Analyze
196 0 : let key_map = build_coordinate_compression_map(keys);
197 0 : let lsn_map = build_coordinate_compression_map(lsns);
198 0 :
199 0 : // Initialize stats
200 0 : let mut num_deltas = 0;
201 0 : let mut num_images = 0;
202 0 :
203 0 : // Draw
204 0 : let stretch = 3.0; // Stretch out vertically for better visibility
205 0 : println!(
206 0 : "{}",
207 0 : BeginSvg {
208 0 : w: (key_map.len() + 10) as f32,
209 0 : h: stretch * lsn_map.len() as f32
210 0 : }
211 0 : );
212 0 :
213 0 : let xmargin = 0.05; // Height-dependent margin to disambiguate overlapping deltas
214 :
215 : for Layer {
216 0 : filename,
217 0 : key_range: keyr,
218 0 : lsn_range: lsnr,
219 0 : } in &files
220 : {
221 0 : let key_start = *key_map.get(&keyr.start).unwrap();
222 0 : let key_end = *key_map.get(&keyr.end).unwrap();
223 0 : let key_diff = key_end - key_start;
224 0 : let lsn_max = lsn_map.len();
225 0 :
226 0 : if key_start >= key_end {
227 0 : panic!("Invalid key range {}-{}", key_start, key_end);
228 0 : }
229 0 :
230 0 : let lsn_start = *lsn_map.get(&lsnr.start).unwrap();
231 0 : let lsn_end = *lsn_map.get(&lsnr.end).unwrap();
232 0 :
233 0 : let mut lsn_diff = (lsn_end - lsn_start) as f32;
234 0 : let mut fill = Fill::None;
235 0 : let mut ymargin = 0.05 * lsn_diff; // Height-dependent margin to disambiguate overlapping deltas
236 0 : let mut lsn_offset = 0.0;
237 0 :
238 0 : // Fill in and thicken rectangle if it's an
239 0 : // image layer so that we can see it.
240 0 : match lsn_start.cmp(&lsn_end) {
241 0 : Ordering::Less => num_deltas += 1,
242 0 : Ordering::Equal => {
243 0 : num_images += 1;
244 0 : lsn_diff = 0.3;
245 0 : lsn_offset = -lsn_diff / 2.0;
246 0 : ymargin = 0.05;
247 0 : fill = Fill::Color(rgb(0, 0, 0));
248 0 : }
249 0 : Ordering::Greater => panic!("Invalid lsn range {}-{}", lsn_start, lsn_end),
250 : }
251 :
252 0 : println!(
253 0 : " {}",
254 0 : rectangle(
255 0 : 5.0 + key_start as f32 + stretch * xmargin,
256 0 : stretch * (lsn_max as f32 - (lsn_end as f32 - ymargin - lsn_offset)),
257 0 : key_diff as f32 - stretch * 2.0 * xmargin,
258 0 : stretch * (lsn_diff - 2.0 * ymargin)
259 0 : )
260 0 : .fill(fill)
261 0 : .stroke(Stroke::Color(rgb(0, 0, 0), 0.1))
262 0 : .border_radius(0.4)
263 0 : .comment(filename)
264 0 : );
265 : }
266 :
267 0 : for (lsn, kind) in lines {
268 0 : let lsn_start = *lsn_map.get(&lsn).unwrap();
269 0 : let lsn_end = lsn_start;
270 0 : let stretch = 2.0;
271 0 : let lsn_diff = 0.3;
272 0 : let lsn_offset = -lsn_diff / 2.0;
273 0 : let ymargin = 0.05;
274 0 : println!(
275 0 : "{}",
276 0 : rectangle(
277 0 : 0.0f32 + stretch * xmargin,
278 0 : stretch * (lsn_map.len() as f32 - (lsn_end as f32 - ymargin - lsn_offset)),
279 0 : (key_map.len() + 10) as f32,
280 0 : stretch * (lsn_diff - 2.0 * ymargin)
281 0 : )
282 0 : .fill(kind)
283 0 : );
284 0 : }
285 :
286 0 : println!("{}", EndSvg);
287 0 :
288 0 : eprintln!("num_images: {}", num_images);
289 0 : eprintln!("num_deltas: {}", num_deltas);
290 0 :
291 0 : Ok(())
292 0 : }
|
