LCOV - code coverage report
Current view: top level - pageserver/ctl/src - draw_timeline_dir.rs (source / functions) Coverage Total Hit
Test: 1e20c4f2b28aa592527961bb32170ebbd2c9172f.info Lines: 0.0 % 152 0
Test Date: 2025-07-16 12:29:03 Functions: 0.0 % 9 0

            Line data    Source code
       1              : //! A tool for visualizing the arrangement of layerfiles within a timeline.
       2              : //!
       3              : //! It reads filenames from stdin and prints a svg on stdout. The image is a plot in
       4              : //! page-lsn space, where every delta layer is a rectangle and every image layer is a
       5              : //! thick line. Legend:
       6              : //! - The x axis (left to right) represents page index.
       7              : //! - The y axis represents LSN, growing upwards.
       8              : //!
       9              : //! Coordinates in both axis are compressed for better readability.
      10              : //! (see <https://medium.com/algorithms-digest/coordinate-compression-2fff95326fb>)
      11              : //!
      12              : //! The plain text API was chosen so that we can easily work with filenames from various
      13              : //! sources; see the Usage section below for examples.
      14              : //!
      15              : //! # Usage
      16              : //!
      17              : //! ## Producing the SVG
      18              : //!
      19              : //! ```bash
      20              : //!
      21              : //! # local timeline dir
      22              : //! ls test_output/test_pgbench\[neon-45-684\]/repo/tenants/$TENANT/timelines/$TIMELINE | \
      23              : //!     grep "__" | cargo run --release --bin pagectl draw-timeline > out.svg
      24              : //!
      25              : //! # Layer map dump from `/v1/tenant/$TENANT/timeline/$TIMELINE/layer`
      26              : //! (jq -r '.historic_layers[] | .layer_file_name' | cargo  run -p pagectl draw-timeline) < layer-map.json > out.svg
      27              : //!
      28              : //! # From an `index_part.json` in S3
      29              : //! (jq -r '.layer_metadata | keys[]' | cargo  run -p pagectl draw-timeline ) < index_part.json-00000016 > out.svg
      30              : //!
      31              : //! # enrich with lines for gc_cutoff and a child branch point
      32              : //! cat <(jq -r '.historic_layers[] | .layer_file_name' < layers.json) <(echo -e 'gc_cutoff:0000001CE3FE32C9\nbranch:0000001DE3FE32C9') | cargo run --bin pagectl draw-timeline >| out.svg
      33              : //! ```
      34              : //!
      35              : //! ## Viewing
      36              : //!
      37              : //! **Inkscape** is better than the built-in viewers in browsers.
      38              : //!
      39              : //! After selecting a layer file rectangle, use "Open XML Editor" (Ctrl|Cmd + Shift + X)
      40              : //! to see the layer file name in the comment field.
      41              : //!
      42              : //! ```bash
      43              : //!
      44              : //! # Linux
      45              : //! inkscape out.svg
      46              : //!
      47              : //! # macOS
      48              : //! /Applications/Inkscape.app/Contents/MacOS/inkscape out.svg
      49              : //!
      50              : //! ```
      51              : //!
      52              : 
      53              : use std::cmp::Ordering;
      54              : use std::collections::{BTreeMap, BTreeSet};
      55              : use std::io::{self, BufRead};
      56              : use std::ops::Range;
      57              : use std::path::PathBuf;
      58              : use std::str::FromStr;
      59              : 
      60              : use anyhow::{Context, Result};
      61              : use pageserver_api::key::Key;
      62              : use svg_fmt::{BeginSvg, EndSvg, Fill, Stroke, rectangle, rgb};
      63              : use utils::lsn::Lsn;
      64              : use utils::project_git_version;
      65              : 
      66              : project_git_version!(GIT_VERSION);
      67              : 
      68              : // Map values to their compressed coordinate - the index the value
      69              : // would have in a sorted and deduplicated list of all values.
      70            0 : fn build_coordinate_compression_map<T: Ord + Copy>(coords: Vec<T>) -> BTreeMap<T, usize> {
      71            0 :     let set: BTreeSet<T> = coords.into_iter().collect();
      72              : 
      73            0 :     let mut map: BTreeMap<T, usize> = BTreeMap::new();
      74            0 :     for (i, e) in set.iter().enumerate() {
      75            0 :         map.insert(*e, i);
      76            0 :     }
      77              : 
      78            0 :     map
      79            0 : }
      80              : 
      81            0 : fn parse_filename(name: &str) -> (Range<Key>, Range<Lsn>) {
      82            0 :     let split: Vec<&str> = name.split("__").collect();
      83            0 :     let keys: Vec<&str> = split[0].split('-').collect();
      84              : 
      85              :     // Remove the temporary file extension, e.g., remove the `.d20a.___temp` part from the following filename:
      86              :     // 000000067F000040490000404A00441B0000-000000067F000040490000404A00441B4000__000043483A34CE00.d20a.___temp
      87            0 :     let lsns = split[1].split('.').collect::<Vec<&str>>()[0];
      88            0 :     let mut lsns: Vec<&str> = lsns.split('-').collect();
      89              : 
      90              :     // The current format of the layer file name: 000000067F0000000400000B150100000000-000000067F0000000400000D350100000000__00000000014B7AC8-v1-00000001
      91              : 
      92              :     // Handle generation number `-00000001` part
      93            0 :     if lsns.last().expect("should").len() == 8 {
      94            0 :         lsns.pop();
      95            0 :     }
      96              : 
      97              :     // Handle version number `-v1` part
      98            0 :     if lsns.last().expect("should").starts_with('v') {
      99            0 :         lsns.pop();
     100            0 :     }
     101              : 
     102            0 :     if lsns.len() == 1 {
     103            0 :         lsns.push(lsns[0]);
     104            0 :     }
     105              : 
     106            0 :     let keys = Key::from_hex(keys[0]).unwrap()..Key::from_hex(keys[1]).unwrap();
     107            0 :     let lsns = Lsn::from_hex(lsns[0]).unwrap()..Lsn::from_hex(lsns[1]).unwrap();
     108            0 :     (keys, lsns)
     109            0 : }
     110              : 
     111              : #[derive(Clone, Copy)]
     112              : enum LineKind {
     113              :     GcCutoff,
     114              :     Branch,
     115              : }
     116              : 
     117              : impl From<LineKind> for Fill {
     118            0 :     fn from(value: LineKind) -> Self {
     119            0 :         match value {
     120            0 :             LineKind::GcCutoff => Fill::Color(rgb(255, 0, 0)),
     121            0 :             LineKind::Branch => Fill::Color(rgb(0, 255, 0)),
     122              :         }
     123            0 :     }
     124              : }
     125              : 
     126              : impl FromStr for LineKind {
     127              :     type Err = anyhow::Error;
     128              : 
     129            0 :     fn from_str(s: &str) -> std::prelude::v1::Result<Self, Self::Err> {
     130            0 :         Ok(match s {
     131            0 :             "gc_cutoff" => LineKind::GcCutoff,
     132            0 :             "branch" => LineKind::Branch,
     133            0 :             _ => anyhow::bail!("unsupported linekind: {s}"),
     134              :         })
     135            0 :     }
     136              : }
     137              : 
     138            0 : pub fn main() -> Result<()> {
     139              :     // Parse layer filenames from stdin
     140              :     struct Layer {
     141              :         filename: String,
     142              :         key_range: Range<Key>,
     143              :         lsn_range: Range<Lsn>,
     144              :     }
     145            0 :     let mut files: Vec<Layer> = vec![];
     146            0 :     let stdin = io::stdin();
     147              : 
     148            0 :     let mut lines: Vec<(Lsn, LineKind)> = vec![];
     149              : 
     150            0 :     for (lineno, line) in stdin.lock().lines().enumerate() {
     151            0 :         let lineno = lineno + 1;
     152              : 
     153            0 :         let line = line.unwrap();
     154            0 :         if let Some((kind, lsn)) = line.split_once(':') {
     155            0 :             let (kind, lsn) = LineKind::from_str(kind)
     156            0 :                 .context("parse kind")
     157            0 :                 .and_then(|kind| {
     158            0 :                     if lsn.contains('/') {
     159            0 :                         Lsn::from_str(lsn)
     160              :                     } else {
     161            0 :                         Lsn::from_hex(lsn)
     162              :                     }
     163            0 :                     .map(|lsn| (kind, lsn))
     164            0 :                     .context("parse lsn")
     165            0 :                 })
     166            0 :                 .with_context(|| format!("parse {line:?} on {lineno}"))?;
     167            0 :             lines.push((lsn, kind));
     168            0 :             continue;
     169            0 :         }
     170            0 :         let line = PathBuf::from_str(&line).unwrap();
     171            0 :         let filename = line.file_name().unwrap();
     172            0 :         let filename = filename.to_str().unwrap();
     173            0 :         let (key_range, lsn_range) = parse_filename(filename);
     174            0 :         files.push(Layer {
     175            0 :             filename: filename.to_owned(),
     176            0 :             key_range,
     177            0 :             lsn_range,
     178            0 :         });
     179              :     }
     180              : 
     181              :     // Collect all coordinates
     182            0 :     let mut keys: Vec<Key> = Vec::with_capacity(files.len());
     183            0 :     let mut lsns: Vec<Lsn> = Vec::with_capacity(files.len() + lines.len());
     184              : 
     185              :     for Layer {
     186            0 :         key_range: keyr,
     187            0 :         lsn_range: lsnr,
     188              :         ..
     189            0 :     } in &files
     190            0 :     {
     191            0 :         keys.push(keyr.start);
     192            0 :         keys.push(keyr.end);
     193            0 :         lsns.push(lsnr.start);
     194            0 :         lsns.push(lsnr.end);
     195            0 :     }
     196              : 
     197            0 :     lsns.extend(lines.iter().map(|(lsn, _)| *lsn));
     198              : 
     199              :     // Analyze
     200            0 :     let key_map = build_coordinate_compression_map(keys);
     201            0 :     let lsn_map = build_coordinate_compression_map(lsns);
     202              : 
     203              :     // Initialize stats
     204            0 :     let mut num_deltas = 0;
     205            0 :     let mut num_images = 0;
     206              : 
     207              :     // Draw
     208            0 :     let stretch = 3.0; // Stretch out vertically for better visibility
     209            0 :     println!(
     210            0 :         "{}",
     211            0 :         BeginSvg {
     212            0 :             w: (key_map.len() + 10) as f32,
     213            0 :             h: stretch * lsn_map.len() as f32
     214            0 :         }
     215              :     );
     216              : 
     217            0 :     let xmargin = 0.05; // Height-dependent margin to disambiguate overlapping deltas
     218              : 
     219              :     for Layer {
     220            0 :         filename,
     221            0 :         key_range: keyr,
     222            0 :         lsn_range: lsnr,
     223            0 :     } in &files
     224              :     {
     225            0 :         let key_start = *key_map.get(&keyr.start).unwrap();
     226            0 :         let key_end = *key_map.get(&keyr.end).unwrap();
     227            0 :         let key_diff = key_end - key_start;
     228            0 :         let lsn_max = lsn_map.len();
     229              : 
     230            0 :         if key_start >= key_end {
     231            0 :             panic!("Invalid key range {key_start}-{key_end}");
     232            0 :         }
     233              : 
     234            0 :         let lsn_start = *lsn_map.get(&lsnr.start).unwrap();
     235            0 :         let lsn_end = *lsn_map.get(&lsnr.end).unwrap();
     236              : 
     237            0 :         let mut lsn_diff = (lsn_end - lsn_start) as f32;
     238            0 :         let mut fill = Fill::None;
     239            0 :         let mut ymargin = 0.05 * lsn_diff; // Height-dependent margin to disambiguate overlapping deltas
     240            0 :         let mut lsn_offset = 0.0;
     241              : 
     242              :         // Fill in and thicken rectangle if it's an
     243              :         // image layer so that we can see it.
     244            0 :         match lsn_start.cmp(&lsn_end) {
     245            0 :             Ordering::Less => num_deltas += 1,
     246            0 :             Ordering::Equal => {
     247            0 :                 num_images += 1;
     248            0 :                 lsn_diff = 0.3;
     249            0 :                 lsn_offset = -lsn_diff / 2.0;
     250            0 :                 ymargin = 0.05;
     251            0 :                 fill = Fill::Color(rgb(0, 0, 0));
     252            0 :             }
     253            0 :             Ordering::Greater => panic!("Invalid lsn range {lsn_start}-{lsn_end}"),
     254              :         }
     255              : 
     256            0 :         println!(
     257            0 :             "    {}",
     258            0 :             rectangle(
     259            0 :                 5.0 + key_start as f32 + stretch * xmargin,
     260            0 :                 stretch * (lsn_max as f32 - (lsn_end as f32 - ymargin - lsn_offset)),
     261            0 :                 key_diff as f32 - stretch * 2.0 * xmargin,
     262            0 :                 stretch * (lsn_diff - 2.0 * ymargin)
     263              :             )
     264            0 :             .fill(fill)
     265            0 :             .stroke(Stroke::Color(rgb(0, 0, 0), 0.1))
     266            0 :             .border_radius(0.4)
     267            0 :             .comment(filename)
     268              :         );
     269              :     }
     270              : 
     271            0 :     for (lsn, kind) in lines {
     272            0 :         let lsn_start = *lsn_map.get(&lsn).unwrap();
     273            0 :         let lsn_end = lsn_start;
     274            0 :         let stretch = 2.0;
     275            0 :         let lsn_diff = 0.3;
     276            0 :         let lsn_offset = -lsn_diff / 2.0;
     277            0 :         let ymargin = 0.05;
     278            0 :         println!(
     279            0 :             "{}",
     280            0 :             rectangle(
     281            0 :                 0.0f32 + stretch * xmargin,
     282            0 :                 stretch * (lsn_map.len() as f32 - (lsn_end as f32 - ymargin - lsn_offset)),
     283            0 :                 (key_map.len() + 10) as f32,
     284            0 :                 stretch * (lsn_diff - 2.0 * ymargin)
     285            0 :             )
     286            0 :             .fill(kind)
     287            0 :         );
     288            0 :     }
     289              : 
     290            0 :     println!("{EndSvg}");
     291              : 
     292            0 :     eprintln!("num_images: {num_images}");
     293            0 :     eprintln!("num_deltas: {num_deltas}");
     294              : 
     295            0 :     Ok(())
     296            0 : }
        

Generated by: LCOV version 2.1-beta