Line data Source code
1 : use anyhow::Context;
2 : use clap::Parser;
3 : use pageserver_api::{
4 : key::Key,
5 : reltag::{BlockNumber, RelTag, SlruKind},
6 : shard::{ShardCount, ShardStripeSize},
7 : };
8 : use std::str::FromStr;
9 :
10 0 : #[derive(Parser)]
11 : pub(super) struct DescribeKeyCommand {
12 : /// Key material in one of the forms: hex, span attributes captured from log, reltag blocknum
13 0 : input: Vec<String>,
14 :
15 : /// The number of shards to calculate what Keys placement would be.
16 : #[arg(long)]
17 : shard_count: Option<CustomShardCount>,
18 :
19 : /// The sharding stripe size.
20 : ///
21 : /// The default is hardcoded. It makes no sense to provide this without providing
22 : /// `--shard-count`.
23 : #[arg(long, requires = "shard_count")]
24 : stripe_size: Option<u32>,
25 : }
26 :
27 : /// Sharded shard count without unsharded count, which the actual ShardCount supports.
28 : #[derive(Clone, Copy)]
29 : pub(super) struct CustomShardCount(std::num::NonZeroU8);
30 :
31 0 : #[derive(Debug, thiserror::Error)]
32 : pub(super) enum InvalidShardCount {
33 : #[error(transparent)]
34 : ParsingFailed(#[from] std::num::ParseIntError),
35 : #[error("too few shards")]
36 : TooFewShards,
37 : }
38 :
39 : impl FromStr for CustomShardCount {
40 : type Err = InvalidShardCount;
41 :
42 0 : fn from_str(s: &str) -> Result<Self, Self::Err> {
43 0 : let inner: std::num::NonZeroU8 = s.parse()?;
44 0 : if inner.get() < 2 {
45 0 : Err(InvalidShardCount::TooFewShards)
46 : } else {
47 0 : Ok(CustomShardCount(inner))
48 : }
49 0 : }
50 : }
51 :
52 : impl From<CustomShardCount> for ShardCount {
53 0 : fn from(value: CustomShardCount) -> Self {
54 0 : ShardCount::new(value.0.get())
55 0 : }
56 : }
57 :
58 : impl DescribeKeyCommand {
59 0 : pub(super) fn execute(self) {
60 0 : let DescribeKeyCommand {
61 0 : input,
62 0 : shard_count,
63 0 : stripe_size,
64 0 : } = self;
65 0 :
66 0 : let material = KeyMaterial::try_from(input.as_slice()).unwrap();
67 0 : let kind = material.kind();
68 0 : let key = Key::from(material);
69 0 :
70 0 : println!("parsed from {kind}: {key}:");
71 0 : println!();
72 0 : println!("{key:?}");
73 :
74 : macro_rules! kind_query {
75 : ([$($name:ident),*$(,)?]) => {{[$(kind_query!($name)),*]}};
76 : ($name:ident) => {{
77 : let s: &'static str = stringify!($name);
78 : let s = s.strip_prefix("is_").unwrap_or(s);
79 : let s = s.strip_suffix("_key").unwrap_or(s);
80 :
81 : #[allow(clippy::needless_borrow)]
82 : (s, key.$name())
83 : }};
84 : }
85 :
86 : // the current characterization is a mess of these boolean queries and separate
87 : // "recognization". I think it accurately represents how strictly we model the Key
88 : // right now, but could of course be made less confusing.
89 :
90 0 : let queries = kind_query!([
91 0 : is_rel_block_key,
92 0 : is_rel_vm_block_key,
93 0 : is_rel_fsm_block_key,
94 0 : is_slru_block_key,
95 0 : is_inherited_key,
96 0 : is_rel_size_key,
97 0 : is_slru_segment_size_key,
98 0 : ]);
99 0 :
100 0 : let recognized_kind = "recognized kind";
101 0 : let metadata_key = "metadata key";
102 0 : let shard_placement = "shard placement";
103 0 :
104 0 : let longest = queries
105 0 : .iter()
106 0 : .map(|t| t.0)
107 0 : .chain([recognized_kind, metadata_key, shard_placement])
108 0 : .map(|s| s.len())
109 0 : .max()
110 0 : .unwrap();
111 0 :
112 0 : let colon = 1;
113 0 : let padding = 1;
114 :
115 0 : for (name, is) in queries {
116 0 : let width = longest - name.len() + colon + padding;
117 0 : println!("{}{:width$}{}", name, ":", is);
118 0 : }
119 :
120 0 : let width = longest - recognized_kind.len() + colon + padding;
121 0 : println!(
122 0 : "{}{:width$}{:?}",
123 0 : recognized_kind,
124 0 : ":",
125 0 : RecognizedKeyKind::new(key),
126 0 : );
127 :
128 0 : if let Some(shard_count) = shard_count {
129 0 : // seeing the sharding placement might be confusing, so leave it out unless shard
130 0 : // count was given.
131 0 :
132 0 : let stripe_size = stripe_size.map(ShardStripeSize).unwrap_or_default();
133 0 : println!(
134 0 : "# placement with shard_count: {} and stripe_size: {}:",
135 0 : shard_count.0, stripe_size.0
136 0 : );
137 0 : let width = longest - shard_placement.len() + colon + padding;
138 0 : println!(
139 0 : "{}{:width$}{:?}",
140 0 : shard_placement,
141 0 : ":",
142 0 : pageserver_api::shard::describe(&key, shard_count.into(), stripe_size)
143 0 : );
144 0 : }
145 0 : }
146 : }
147 :
148 : /// Hand-wavy "inputs we accept" for a key.
149 : #[derive(Debug)]
150 : pub(super) enum KeyMaterial {
151 : Hex(Key),
152 : String(SpanAttributesFromLogs),
153 : Split(RelTag, BlockNumber),
154 : }
155 :
156 : impl KeyMaterial {
157 0 : fn kind(&self) -> &'static str {
158 0 : match self {
159 0 : KeyMaterial::Hex(_) => "hex",
160 0 : KeyMaterial::String(_) | KeyMaterial::Split(_, _) => "split",
161 : }
162 0 : }
163 : }
164 :
165 : impl From<KeyMaterial> for Key {
166 6 : fn from(value: KeyMaterial) -> Self {
167 6 : match value {
168 0 : KeyMaterial::Hex(key) => key,
169 3 : KeyMaterial::String(SpanAttributesFromLogs(rt, blocknum))
170 3 : | KeyMaterial::Split(rt, blocknum) => {
171 6 : pageserver_api::key::rel_block_to_key(rt, blocknum)
172 : }
173 : }
174 6 : }
175 : }
176 :
177 : impl<S: AsRef<str>> TryFrom<&[S]> for KeyMaterial {
178 : type Error = anyhow::Error;
179 :
180 8 : fn try_from(value: &[S]) -> Result<Self, Self::Error> {
181 8 : match value {
182 8 : [] => anyhow::bail!(
183 0 : "need 1..N positional arguments describing the key, try hex or a log line"
184 0 : ),
185 5 : [one] => {
186 5 : let one = one.as_ref();
187 5 :
188 5 : let key = Key::from_hex(one).map(KeyMaterial::Hex);
189 5 :
190 5 : let attrs = SpanAttributesFromLogs::from_str(one).map(KeyMaterial::String);
191 5 :
192 5 : match (key, attrs) {
193 1 : (Ok(key), _) => Ok(key),
194 3 : (_, Ok(s)) => Ok(s),
195 1 : (Err(e1), Err(e2)) => anyhow::bail!(
196 1 : "failed to parse {one:?} as hex or span attributes:\n- {e1:#}\n- {e2:#}"
197 1 : ),
198 : }
199 : }
200 3 : more => {
201 : // assume going left to right one of these is a reltag and then we find a blocknum
202 : // this works, because we don't have plain numbers at least right after reltag in
203 : // logs. for some definition of "works".
204 :
205 3 : let Some((reltag_at, reltag)) = more
206 3 : .iter()
207 3 : .map(AsRef::as_ref)
208 3 : .enumerate()
209 4 : .find_map(|(i, s)| {
210 4 : s.split_once("rel=")
211 4 : .map(|(_garbage, actual)| actual)
212 4 : .unwrap_or(s)
213 4 : .parse::<RelTag>()
214 4 : .ok()
215 4 : .map(|rt| (i, rt))
216 4 : })
217 : else {
218 0 : anyhow::bail!("found no RelTag in arguments");
219 : };
220 :
221 3 : let Some(blocknum) = more
222 3 : .iter()
223 3 : .map(AsRef::as_ref)
224 3 : .skip(reltag_at)
225 6 : .find_map(|s| {
226 6 : s.split_once("blkno=")
227 6 : .map(|(_garbage, actual)| actual)
228 6 : .unwrap_or(s)
229 6 : .parse::<BlockNumber>()
230 6 : .ok()
231 6 : })
232 : else {
233 0 : anyhow::bail!("found no blocknum in arguments");
234 : };
235 :
236 3 : Ok(KeyMaterial::Split(reltag, blocknum))
237 : }
238 : }
239 8 : }
240 : }
241 :
242 : #[derive(Debug)]
243 : pub(super) struct SpanAttributesFromLogs(RelTag, BlockNumber);
244 :
245 : impl std::str::FromStr for SpanAttributesFromLogs {
246 : type Err = anyhow::Error;
247 :
248 5 : fn from_str(s: &str) -> Result<Self, Self::Err> {
249 : // accept the span separator but do not require or fail if either is missing
250 : // "whatever{rel=1663/16389/24615 blkno=1052204 req_lsn=FFFFFFFF/FFFFFFFF}"
251 5 : let (_, reltag) = s
252 5 : .split_once("rel=")
253 5 : .ok_or_else(|| anyhow::anyhow!("cannot find 'rel='"))?;
254 3 : let reltag = reltag.split_whitespace().next().unwrap();
255 :
256 3 : let (_, blocknum) = s
257 3 : .split_once("blkno=")
258 3 : .ok_or_else(|| anyhow::anyhow!("cannot find 'blkno='"))?;
259 3 : let blocknum = blocknum.split_whitespace().next().unwrap();
260 :
261 3 : let reltag = reltag
262 3 : .parse()
263 3 : .with_context(|| format!("parse reltag from {reltag:?}"))?;
264 3 : let blocknum = blocknum
265 3 : .parse()
266 3 : .with_context(|| format!("parse blocknum from {blocknum:?}"))?;
267 :
268 3 : Ok(Self(reltag, blocknum))
269 5 : }
270 : }
271 :
272 : #[derive(Debug)]
273 : #[allow(dead_code)] // debug print is used
274 : enum RecognizedKeyKind {
275 : DbDir,
276 : ControlFile,
277 : Checkpoint,
278 : AuxFilesV1,
279 : SlruDir(Result<SlruKind, u32>),
280 : RelMap(RelTagish<2>),
281 : RelDir(RelTagish<2>),
282 : AuxFileV2(Result<AuxFileV2, utils::Hex<[u8; 16]>>),
283 : }
284 :
285 : #[derive(Debug, PartialEq)]
286 : #[allow(unused)]
287 : enum AuxFileV2 {
288 : Recognized(&'static str, utils::Hex<[u8; 13]>),
289 : OtherWithPrefix(&'static str, utils::Hex<[u8; 13]>),
290 : Other(utils::Hex<[u8; 13]>),
291 : }
292 :
293 : impl RecognizedKeyKind {
294 0 : fn new(key: Key) -> Option<Self> {
295 : use RecognizedKeyKind::{
296 : AuxFilesV1, Checkpoint, ControlFile, DbDir, RelDir, RelMap, SlruDir,
297 : };
298 :
299 0 : let slru_dir_kind = pageserver_api::key::slru_dir_kind(&key);
300 :
301 0 : Some(match key {
302 0 : pageserver_api::key::DBDIR_KEY => DbDir,
303 0 : pageserver_api::key::CONTROLFILE_KEY => ControlFile,
304 0 : pageserver_api::key::CHECKPOINT_KEY => Checkpoint,
305 0 : pageserver_api::key::AUX_FILES_KEY => AuxFilesV1,
306 0 : _ if slru_dir_kind.is_some() => SlruDir(slru_dir_kind.unwrap()),
307 0 : _ if key.field1 == 0 && key.field4 == 0 && key.field5 == 0 && key.field6 == 0 => {
308 0 : RelMap([key.field2, key.field3].into())
309 : }
310 0 : _ if key.field1 == 0 && key.field4 == 0 && key.field5 == 0 && key.field6 == 1 => {
311 0 : RelDir([key.field2, key.field3].into())
312 : }
313 0 : _ if key.is_metadata_key() => RecognizedKeyKind::AuxFileV2(
314 0 : AuxFileV2::new(key).ok_or_else(|| utils::Hex(key.to_i128().to_be_bytes())),
315 0 : ),
316 0 : _ => return None,
317 : })
318 0 : }
319 : }
320 :
321 : impl AuxFileV2 {
322 7 : fn new(key: Key) -> Option<AuxFileV2> {
323 : const EMPTY_HASH: [u8; 13] = {
324 : let mut out = [0u8; 13];
325 : let hash = pageserver::aux_file::fnv_hash(b"").to_be_bytes();
326 : let mut i = 3;
327 : while i < 16 {
328 : out[i - 3] = hash[i];
329 : i += 1;
330 : }
331 : out
332 : };
333 :
334 7 : let bytes = key.to_i128().to_be_bytes();
335 7 : let hash = utils::Hex(<[u8; 13]>::try_from(&bytes[3..]).unwrap());
336 7 :
337 7 : assert_eq!(EMPTY_HASH.len(), hash.0.len());
338 :
339 : // TODO: we could probably find the preimages for the hashes
340 :
341 7 : Some(match (bytes[1], bytes[2]) {
342 1 : (1, 1) => AuxFileV2::Recognized("pg_logical/mappings/", hash),
343 1 : (1, 2) => AuxFileV2::Recognized("pg_logical/snapshots/", hash),
344 1 : (1, 3) if hash.0 == EMPTY_HASH => {
345 1 : AuxFileV2::Recognized("pg_logical/replorigin_checkpoint", hash)
346 : }
347 1 : (2, 1) => AuxFileV2::Recognized("pg_replslot/", hash),
348 1 : (1, 0xff) => AuxFileV2::OtherWithPrefix("pg_logical/", hash),
349 1 : (0xff, 0xff) => AuxFileV2::Other(hash),
350 1 : _ => return None,
351 : })
352 7 : }
353 : }
354 :
355 : /// Prefix of RelTag, currently only known use cases are the two item versions.
356 : ///
357 : /// Renders like a reltag with `/`, nothing else.
358 : struct RelTagish<const N: usize>([u32; N]);
359 :
360 : impl<const N: usize> From<[u32; N]> for RelTagish<N> {
361 0 : fn from(val: [u32; N]) -> Self {
362 0 : RelTagish(val)
363 0 : }
364 : }
365 :
366 : impl<const N: usize> std::fmt::Debug for RelTagish<N> {
367 0 : fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
368 : use std::fmt::Write as _;
369 0 : let mut first = true;
370 0 : self.0.iter().try_for_each(|x| {
371 0 : if !first {
372 0 : f.write_char('/')?;
373 0 : }
374 0 : first = false;
375 0 : write!(f, "{}", x)
376 0 : })
377 0 : }
378 : }
379 :
380 : #[cfg(test)]
381 : mod tests {
382 : use pageserver::aux_file::encode_aux_file_key;
383 :
384 : use super::*;
385 :
386 : #[test]
387 1 : fn hex_is_key_material() {
388 1 : let m = KeyMaterial::try_from(&["000000067F0000400200DF927900FFFFFFFF"][..]).unwrap();
389 1 : assert!(matches!(m, KeyMaterial::Hex(_)), "{m:?}");
390 1 : }
391 :
392 : #[test]
393 1 : fn single_positional_spanalike_is_key_material() {
394 1 : // why is this needed? if you are checking many, then copypaste starts to appeal
395 1 : let strings = [
396 1 : (line!(), "2024-05-15T15:33:49.873906Z ERROR page_service_conn_main{peer_addr=A:B}:process_query{tenant_id=C timeline_id=D}:handle_pagerequests:handle_get_page_at_lsn_request{rel=1663/208101/2620_fsm blkno=2 req_lsn=0/238D98C8}: error reading relation or page version: Read error: could not find data for key 000000067F00032CE5000000000000000001 (shard ShardNumber(0)) at LSN 0/1D0A16C1, request LSN 0/238D98C8, ancestor 0/0"),
397 1 : (line!(), "rel=1663/208101/2620_fsm blkno=2"),
398 1 : (line!(), "rel=1663/208101/2620.1 blkno=2"),
399 1 : ];
400 1 :
401 1 : let mut first: Option<Key> = None;
402 :
403 4 : for (line, example) in strings {
404 3 : let m = KeyMaterial::try_from(&[example][..])
405 3 : .unwrap_or_else(|e| panic!("failed to parse example from line {line}: {e:?}"));
406 3 : let key = Key::from(m);
407 3 : if let Some(first) = first {
408 2 : assert_eq!(first, key);
409 1 : } else {
410 1 : first = Some(key);
411 1 : }
412 : }
413 :
414 : // not supporting this is rather accidential, but I think the input parsing is lenient
415 : // enough already
416 1 : KeyMaterial::try_from(&["1663/208101/2620_fsm 2"][..]).unwrap_err();
417 1 : }
418 :
419 : #[test]
420 1 : fn multiple_spanlike_args() {
421 1 : let strings = [
422 1 : (line!(), &["process_query{tenant_id=C", "timeline_id=D}:handle_pagerequests:handle_get_page_at_lsn_request{rel=1663/208101/2620_fsm", "blkno=2", "req_lsn=0/238D98C8}"][..]),
423 1 : (line!(), &["rel=1663/208101/2620_fsm", "blkno=2"][..]),
424 1 : (line!(), &["1663/208101/2620_fsm", "2"][..]),
425 1 : ];
426 1 :
427 1 : let mut first: Option<Key> = None;
428 :
429 4 : for (line, example) in strings {
430 3 : let m = KeyMaterial::try_from(example)
431 3 : .unwrap_or_else(|e| panic!("failed to parse example from line {line}: {e:?}"));
432 3 : let key = Key::from(m);
433 3 : if let Some(first) = first {
434 2 : assert_eq!(first, key);
435 1 : } else {
436 1 : first = Some(key);
437 1 : }
438 : }
439 1 : }
440 : #[test]
441 1 : fn recognized_auxfiles() {
442 : use AuxFileV2::*;
443 :
444 1 : let empty = [
445 1 : 0x2e, 0x07, 0xbb, 0x01, 0x42, 0x62, 0xb8, 0x21, 0x75, 0x62, 0x95, 0xc5, 0x8d,
446 1 : ];
447 1 : let foobar = [
448 1 : 0x62, 0x79, 0x3c, 0x64, 0xbf, 0x6f, 0x0d, 0x35, 0x97, 0xba, 0x44, 0x6f, 0x18,
449 1 : ];
450 1 :
451 1 : #[rustfmt::skip]
452 1 : let examples = [
453 1 : (line!(), "pg_logical/mappings/foobar", Recognized("pg_logical/mappings/", utils::Hex(foobar))),
454 1 : (line!(), "pg_logical/snapshots/foobar", Recognized("pg_logical/snapshots/", utils::Hex(foobar))),
455 1 : (line!(), "pg_logical/replorigin_checkpoint", Recognized("pg_logical/replorigin_checkpoint", utils::Hex(empty))),
456 1 : (line!(), "pg_logical/foobar", OtherWithPrefix("pg_logical/", utils::Hex(foobar))),
457 1 : (line!(), "pg_replslot/foobar", Recognized("pg_replslot/", utils::Hex(foobar))),
458 1 : (line!(), "foobar", Other(utils::Hex(foobar))),
459 1 : ];
460 :
461 7 : for (line, path, expected) in examples {
462 6 : let key = encode_aux_file_key(path);
463 6 : let recognized =
464 6 : AuxFileV2::new(key).unwrap_or_else(|| panic!("line {line} example failed"));
465 6 :
466 6 : assert_eq!(recognized, expected);
467 : }
468 :
469 1 : assert_eq!(
470 1 : AuxFileV2::new(Key::from_hex("600000102000000000000000000000000000").unwrap()),
471 : None,
472 0 : "example key has one too few 0 after 6 before 1"
473 : );
474 1 : }
475 : }
|
