Line data Source code
1 : //! A tool for visualizing the arrangement of layerfiles within a timeline.
2 : //!
3 : //! It reads filenames from stdin and prints a svg on stdout. The image is a plot in
4 : //! page-lsn space, where every delta layer is a rectangle and every image layer is a
5 : //! thick line. Legend:
6 : //! - The x axis (left to right) represents page index.
7 : //! - The y axis represents LSN, growing upwards.
8 : //!
9 : //! Coordinates in both axis are compressed for better readability.
10 : //! (see <https://medium.com/algorithms-digest/coordinate-compression-2fff95326fb>)
11 : //!
12 : //! The plain text API was chosen so that we can easily work with filenames from various
13 : //! sources; see the Usage section below for examples.
14 : //!
15 : //! # Usage
16 : //!
17 : //! ## Producing the SVG
18 : //!
19 : //! ```bash
20 : //!
21 : //! # local timeline dir
22 : //! ls test_output/test_pgbench\[neon-45-684\]/repo/tenants/$TENANT/timelines/$TIMELINE | \
23 : //! grep "__" | cargo run --release --bin pagectl draw-timeline-dir > out.svg
24 : //!
25 : //! # Layer map dump from `/v1/tenant/$TENANT/timeline/$TIMELINE/layer`
26 : //! (jq -r '.historic_layers[] | .layer_file_name' | cargo run -p pagectl draw-timeline) < layer-map.json > out.svg
27 : //!
28 : //! # From an `index_part.json` in S3
29 : //! (jq -r '.layer_metadata | keys[]' | cargo run -p pagectl draw-timeline ) < index_part.json-00000016 > out.svg
30 : //!
31 : //! # enrich with lines for gc_cutoff and a child branch point
32 : //! cat <(jq -r '.historic_layers[] | .layer_file_name' < layers.json) <(echo -e 'gc_cutoff:0000001CE3FE32C9\nbranch:0000001DE3FE32C9') | cargo run --bin pagectl draw-timeline >| out.svg
33 : //! ```
34 : //!
35 : //! ## Viewing
36 : //!
37 : //! **Inkscape** is better than the built-in viewers in browsers.
38 : //!
39 : //! After selecting a layer file rectangle, use "Open XML Editor" (Ctrl|Cmd + Shift + X)
40 : //! to see the layer file name in the comment field.
41 : //!
42 : //! ```bash
43 : //!
44 : //! # Linux
45 : //! inkscape out.svg
46 : //!
47 : //! # macOS
48 : //! /Applications/Inkscape.app/Contents/MacOS/inkscape out.svg
49 : //!
50 : //! ```
51 : //!
52 :
53 : use anyhow::{Context, Result};
54 : use pageserver::repository::Key;
55 : use std::cmp::Ordering;
56 : use std::io::{self, BufRead};
57 : use std::path::PathBuf;
58 : use std::str::FromStr;
59 : use std::{
60 : collections::{BTreeMap, BTreeSet},
61 : ops::Range,
62 : };
63 : use svg_fmt::{rectangle, rgb, BeginSvg, EndSvg, Fill, Stroke};
64 : use utils::{lsn::Lsn, project_git_version};
65 :
66 : project_git_version!(GIT_VERSION);
67 :
68 : // Map values to their compressed coordinate - the index the value
69 : // would have in a sorted and deduplicated list of all values.
70 0 : fn build_coordinate_compression_map<T: Ord + Copy>(coords: Vec<T>) -> BTreeMap<T, usize> {
71 0 : let set: BTreeSet<T> = coords.into_iter().collect();
72 0 :
73 0 : let mut map: BTreeMap<T, usize> = BTreeMap::new();
74 0 : for (i, e) in set.iter().enumerate() {
75 0 : map.insert(*e, i);
76 0 : }
77 :
78 0 : map
79 0 : }
80 :
81 0 : fn parse_filename(name: &str) -> (Range<Key>, Range<Lsn>) {
82 0 : let split: Vec<&str> = name.split("__").collect();
83 0 : let keys: Vec<&str> = split[0].split('-').collect();
84 0 : let mut lsns: Vec<&str> = split[1].split('-').collect();
85 0 :
86 0 : if lsns.last().expect("should").len() == 8 {
87 0 : lsns.pop();
88 0 : }
89 :
90 0 : if lsns.len() == 1 {
91 0 : lsns.push(lsns[0]);
92 0 : }
93 :
94 0 : let keys = Key::from_hex(keys[0]).unwrap()..Key::from_hex(keys[1]).unwrap();
95 0 : let lsns = Lsn::from_hex(lsns[0]).unwrap()..Lsn::from_hex(lsns[1]).unwrap();
96 0 : (keys, lsns)
97 0 : }
98 :
99 : #[derive(Clone, Copy)]
100 : enum LineKind {
101 : GcCutoff,
102 : Branch,
103 : }
104 :
105 : impl From<LineKind> for Fill {
106 0 : fn from(value: LineKind) -> Self {
107 0 : match value {
108 0 : LineKind::GcCutoff => Fill::Color(rgb(255, 0, 0)),
109 0 : LineKind::Branch => Fill::Color(rgb(0, 255, 0)),
110 : }
111 0 : }
112 : }
113 :
114 : impl FromStr for LineKind {
115 : type Err = anyhow::Error;
116 :
117 0 : fn from_str(s: &str) -> std::prelude::v1::Result<Self, Self::Err> {
118 0 : Ok(match s {
119 0 : "gc_cutoff" => LineKind::GcCutoff,
120 0 : "branch" => LineKind::Branch,
121 0 : _ => anyhow::bail!("unsupported linekind: {s}"),
122 : })
123 0 : }
124 : }
125 :
126 0 : pub fn main() -> Result<()> {
127 0 : // Parse layer filenames from stdin
128 0 : struct Layer {
129 0 : filename: String,
130 0 : key_range: Range<Key>,
131 0 : lsn_range: Range<Lsn>,
132 0 : }
133 0 : let mut files: Vec<Layer> = vec![];
134 0 : let stdin = io::stdin();
135 0 :
136 0 : let mut lines: Vec<(Lsn, LineKind)> = vec![];
137 :
138 0 : for (lineno, line) in stdin.lock().lines().enumerate() {
139 0 : let lineno = lineno + 1;
140 0 :
141 0 : let line = line.unwrap();
142 0 : if let Some((kind, lsn)) = line.split_once(':') {
143 0 : let (kind, lsn) = LineKind::from_str(kind)
144 0 : .context("parse kind")
145 0 : .and_then(|kind| {
146 0 : if lsn.contains('/') {
147 0 : Lsn::from_str(lsn)
148 : } else {
149 0 : Lsn::from_hex(lsn)
150 : }
151 0 : .map(|lsn| (kind, lsn))
152 0 : .context("parse lsn")
153 0 : })
154 0 : .with_context(|| format!("parse {line:?} on {lineno}"))?;
155 0 : lines.push((lsn, kind));
156 0 : continue;
157 0 : }
158 0 : let line = PathBuf::from_str(&line).unwrap();
159 0 : let filename = line.file_name().unwrap();
160 0 : let filename = filename.to_str().unwrap();
161 0 : let (key_range, lsn_range) = parse_filename(filename);
162 0 : files.push(Layer {
163 0 : filename: filename.to_owned(),
164 0 : key_range,
165 0 : lsn_range,
166 0 : });
167 : }
168 :
169 : // Collect all coordinates
170 0 : let mut keys: Vec<Key> = Vec::with_capacity(files.len());
171 0 : let mut lsns: Vec<Lsn> = Vec::with_capacity(files.len() + lines.len());
172 :
173 : for Layer {
174 0 : key_range: keyr,
175 0 : lsn_range: lsnr,
176 : ..
177 0 : } in &files
178 0 : {
179 0 : keys.push(keyr.start);
180 0 : keys.push(keyr.end);
181 0 : lsns.push(lsnr.start);
182 0 : lsns.push(lsnr.end);
183 0 : }
184 :
185 0 : lsns.extend(lines.iter().map(|(lsn, _)| *lsn));
186 0 :
187 0 : // Analyze
188 0 : let key_map = build_coordinate_compression_map(keys);
189 0 : let lsn_map = build_coordinate_compression_map(lsns);
190 0 :
191 0 : // Initialize stats
192 0 : let mut num_deltas = 0;
193 0 : let mut num_images = 0;
194 0 :
195 0 : // Draw
196 0 : let stretch = 3.0; // Stretch out vertically for better visibility
197 0 : println!(
198 0 : "{}",
199 0 : BeginSvg {
200 0 : w: (key_map.len() + 10) as f32,
201 0 : h: stretch * lsn_map.len() as f32
202 0 : }
203 0 : );
204 0 :
205 0 : let xmargin = 0.05; // Height-dependent margin to disambiguate overlapping deltas
206 :
207 : for Layer {
208 0 : filename,
209 0 : key_range: keyr,
210 0 : lsn_range: lsnr,
211 0 : } in &files
212 : {
213 0 : let key_start = *key_map.get(&keyr.start).unwrap();
214 0 : let key_end = *key_map.get(&keyr.end).unwrap();
215 0 : let key_diff = key_end - key_start;
216 0 : let lsn_max = lsn_map.len();
217 0 :
218 0 : if key_start >= key_end {
219 0 : panic!("Invalid key range {}-{}", key_start, key_end);
220 0 : }
221 0 :
222 0 : let lsn_start = *lsn_map.get(&lsnr.start).unwrap();
223 0 : let lsn_end = *lsn_map.get(&lsnr.end).unwrap();
224 0 :
225 0 : let mut lsn_diff = (lsn_end - lsn_start) as f32;
226 0 : let mut fill = Fill::None;
227 0 : let mut ymargin = 0.05 * lsn_diff; // Height-dependent margin to disambiguate overlapping deltas
228 0 : let mut lsn_offset = 0.0;
229 0 :
230 0 : // Fill in and thicken rectangle if it's an
231 0 : // image layer so that we can see it.
232 0 : match lsn_start.cmp(&lsn_end) {
233 0 : Ordering::Less => num_deltas += 1,
234 0 : Ordering::Equal => {
235 0 : num_images += 1;
236 0 : lsn_diff = 0.3;
237 0 : lsn_offset = -lsn_diff / 2.0;
238 0 : ymargin = 0.05;
239 0 : fill = Fill::Color(rgb(0, 0, 0));
240 0 : }
241 0 : Ordering::Greater => panic!("Invalid lsn range {}-{}", lsn_start, lsn_end),
242 : }
243 :
244 0 : println!(
245 0 : " {}",
246 0 : rectangle(
247 0 : 5.0 + key_start as f32 + stretch * xmargin,
248 0 : stretch * (lsn_max as f32 - (lsn_end as f32 - ymargin - lsn_offset)),
249 0 : key_diff as f32 - stretch * 2.0 * xmargin,
250 0 : stretch * (lsn_diff - 2.0 * ymargin)
251 0 : )
252 0 : .fill(fill)
253 0 : .stroke(Stroke::Color(rgb(0, 0, 0), 0.1))
254 0 : .border_radius(0.4)
255 0 : .comment(filename)
256 0 : );
257 : }
258 :
259 0 : for (lsn, kind) in lines {
260 0 : let lsn_start = *lsn_map.get(&lsn).unwrap();
261 0 : let lsn_end = lsn_start;
262 0 : let stretch = 2.0;
263 0 : let lsn_diff = 0.3;
264 0 : let lsn_offset = -lsn_diff / 2.0;
265 0 : let ymargin = 0.05;
266 0 : println!(
267 0 : "{}",
268 0 : rectangle(
269 0 : 0.0f32 + stretch * xmargin,
270 0 : stretch * (lsn_map.len() as f32 - (lsn_end as f32 - ymargin - lsn_offset)),
271 0 : (key_map.len() + 10) as f32,
272 0 : stretch * (lsn_diff - 2.0 * ymargin)
273 0 : )
274 0 : .fill(kind)
275 0 : );
276 0 : }
277 :
278 0 : println!("{}", EndSvg);
279 0 :
280 0 : eprintln!("num_images: {}", num_images);
281 0 : eprintln!("num_deltas: {}", num_deltas);
282 0 :
283 0 : Ok(())
284 0 : }
|
