Line data Source code
1 : use anyhow::{bail, Result};
2 : use byteorder::{ByteOrder, BE};
3 : use postgres_ffi::relfile_utils::{FSM_FORKNUM, VISIBILITYMAP_FORKNUM};
4 : use postgres_ffi::{Oid, TransactionId};
5 : use serde::{Deserialize, Serialize};
6 : use std::{fmt, ops::Range};
7 :
8 : use crate::reltag::{BlockNumber, RelTag, SlruKind};
9 :
10 : /// Key used in the Repository kv-store.
11 : ///
12 : /// The Repository treats this as an opaque struct, but see the code in pgdatadir_mapping.rs
13 : /// for what we actually store in these fields.
14 10690739 : #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize)]
15 : pub struct Key {
16 : pub field1: u8,
17 : pub field2: u32,
18 : pub field3: u32,
19 : pub field4: u32,
20 : pub field5: u8,
21 : pub field6: u32,
22 : }
23 :
24 : pub const KEY_SIZE: usize = 18;
25 :
26 : impl Key {
27 : /// 'field2' is used to store tablespaceid for relations and small enum numbers for other relish.
28 : /// As long as Neon does not support tablespace (because of lack of access to local file system),
29 : /// we can assume that only some predefined namespace OIDs are used which can fit in u16
30 6749428 : pub fn to_i128(&self) -> i128 {
31 6749428 : assert!(self.field2 < 0xFFFF || self.field2 == 0xFFFFFFFF || self.field2 == 0x22222222);
32 6749428 : (((self.field1 & 0xf) as i128) << 120)
33 6749428 : | (((self.field2 & 0xFFFF) as i128) << 104)
34 6749428 : | ((self.field3 as i128) << 72)
35 6749428 : | ((self.field4 as i128) << 40)
36 6749428 : | ((self.field5 as i128) << 32)
37 6749428 : | self.field6 as i128
38 6749428 : }
39 :
40 2162774 : pub const fn from_i128(x: i128) -> Self {
41 2162774 : Key {
42 2162774 : field1: ((x >> 120) & 0xf) as u8,
43 2162774 : field2: ((x >> 104) & 0xFFFF) as u32,
44 2162774 : field3: (x >> 72) as u32,
45 2162774 : field4: (x >> 40) as u32,
46 2162774 : field5: (x >> 32) as u8,
47 2162774 : field6: x as u32,
48 2162774 : }
49 2162774 : }
50 :
51 4255204 : pub fn next(&self) -> Key {
52 4255204 : self.add(1)
53 4255204 : }
54 :
55 4255214 : pub fn add(&self, x: u32) -> Key {
56 4255214 : let mut key = *self;
57 4255214 :
58 4255214 : let r = key.field6.overflowing_add(x);
59 4255214 : key.field6 = r.0;
60 4255214 : if r.1 {
61 0 : let r = key.field5.overflowing_add(1);
62 0 : key.field5 = r.0;
63 0 : if r.1 {
64 0 : let r = key.field4.overflowing_add(1);
65 0 : key.field4 = r.0;
66 0 : if r.1 {
67 0 : let r = key.field3.overflowing_add(1);
68 0 : key.field3 = r.0;
69 0 : if r.1 {
70 0 : let r = key.field2.overflowing_add(1);
71 0 : key.field2 = r.0;
72 0 : if r.1 {
73 0 : let r = key.field1.overflowing_add(1);
74 0 : key.field1 = r.0;
75 0 : assert!(!r.1);
76 0 : }
77 0 : }
78 0 : }
79 0 : }
80 4255214 : }
81 4255214 : key
82 4255214 : }
83 :
84 2102356 : pub fn from_slice(b: &[u8]) -> Self {
85 2102356 : Key {
86 2102356 : field1: b[0],
87 2102356 : field2: u32::from_be_bytes(b[1..5].try_into().unwrap()),
88 2102356 : field3: u32::from_be_bytes(b[5..9].try_into().unwrap()),
89 2102356 : field4: u32::from_be_bytes(b[9..13].try_into().unwrap()),
90 2102356 : field5: b[13],
91 2102356 : field6: u32::from_be_bytes(b[14..18].try_into().unwrap()),
92 2102356 : }
93 2102356 : }
94 :
95 4436096 : pub fn write_to_byte_slice(&self, buf: &mut [u8]) {
96 4436096 : buf[0] = self.field1;
97 4436096 : BE::write_u32(&mut buf[1..5], self.field2);
98 4436096 : BE::write_u32(&mut buf[5..9], self.field3);
99 4436096 : BE::write_u32(&mut buf[9..13], self.field4);
100 4436096 : buf[13] = self.field5;
101 4436096 : BE::write_u32(&mut buf[14..18], self.field6);
102 4436096 : }
103 : }
104 :
105 : impl fmt::Display for Key {
106 10842 : fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
107 10842 : write!(
108 10842 : f,
109 10842 : "{:02X}{:08X}{:08X}{:08X}{:02X}{:08X}",
110 10842 : self.field1, self.field2, self.field3, self.field4, self.field5, self.field6
111 10842 : )
112 10842 : }
113 : }
114 :
115 : impl Key {
116 : pub const MIN: Key = Key {
117 : field1: u8::MIN,
118 : field2: u32::MIN,
119 : field3: u32::MIN,
120 : field4: u32::MIN,
121 : field5: u8::MIN,
122 : field6: u32::MIN,
123 : };
124 : pub const MAX: Key = Key {
125 : field1: u8::MAX,
126 : field2: u32::MAX,
127 : field3: u32::MAX,
128 : field4: u32::MAX,
129 : field5: u8::MAX,
130 : field6: u32::MAX,
131 : };
132 :
133 332 : pub fn from_hex(s: &str) -> Result<Self> {
134 332 : if s.len() != 36 {
135 0 : bail!("parse error");
136 332 : }
137 332 : Ok(Key {
138 332 : field1: u8::from_str_radix(&s[0..2], 16)?,
139 332 : field2: u32::from_str_radix(&s[2..10], 16)?,
140 332 : field3: u32::from_str_radix(&s[10..18], 16)?,
141 332 : field4: u32::from_str_radix(&s[18..26], 16)?,
142 332 : field5: u8::from_str_radix(&s[26..28], 16)?,
143 332 : field6: u32::from_str_radix(&s[28..36], 16)?,
144 : })
145 332 : }
146 : }
147 :
148 : // Layout of the Key address space
149 : //
150 : // The Key struct, used to address the underlying key-value store, consists of
151 : // 18 bytes, split into six fields. See 'Key' in repository.rs. We need to map
152 : // all the data and metadata keys into those 18 bytes.
153 : //
154 : // Principles for the mapping:
155 : //
156 : // - Things that are often accessed or modified together, should be close to
157 : // each other in the key space. For example, if a relation is extended by one
158 : // block, we create a new key-value pair for the block data, and update the
159 : // relation size entry. Because of that, the RelSize key comes after all the
160 : // RelBlocks of a relation: the RelSize and the last RelBlock are always next
161 : // to each other.
162 : //
163 : // The key space is divided into four major sections, identified by the first
164 : // byte, and the form a hierarchy:
165 : //
166 : // 00 Relation data and metadata
167 : //
168 : // DbDir () -> (dbnode, spcnode)
169 : // Filenodemap
170 : // RelDir -> relnode forknum
171 : // RelBlocks
172 : // RelSize
173 : //
174 : // 01 SLRUs
175 : //
176 : // SlruDir kind
177 : // SlruSegBlocks segno
178 : // SlruSegSize
179 : //
180 : // 02 pg_twophase
181 : //
182 : // 03 misc
183 : // Controlfile
184 : // checkpoint
185 : // pg_version
186 : //
187 : // 04 aux files
188 : //
189 : // Below is a full list of the keyspace allocation:
190 : //
191 : // DbDir:
192 : // 00 00000000 00000000 00000000 00 00000000
193 : //
194 : // Filenodemap:
195 : // 00 SPCNODE DBNODE 00000000 00 00000000
196 : //
197 : // RelDir:
198 : // 00 SPCNODE DBNODE 00000000 00 00000001 (Postgres never uses relfilenode 0)
199 : //
200 : // RelBlock:
201 : // 00 SPCNODE DBNODE RELNODE FORK BLKNUM
202 : //
203 : // RelSize:
204 : // 00 SPCNODE DBNODE RELNODE FORK FFFFFFFF
205 : //
206 : // SlruDir:
207 : // 01 kind 00000000 00000000 00 00000000
208 : //
209 : // SlruSegBlock:
210 : // 01 kind 00000001 SEGNO 00 BLKNUM
211 : //
212 : // SlruSegSize:
213 : // 01 kind 00000001 SEGNO 00 FFFFFFFF
214 : //
215 : // TwoPhaseDir:
216 : // 02 00000000 00000000 00000000 00 00000000
217 : //
218 : // TwoPhaseFile:
219 : // 02 00000000 00000000 00000000 00 XID
220 : //
221 : // ControlFile:
222 : // 03 00000000 00000000 00000000 00 00000000
223 : //
224 : // Checkpoint:
225 : // 03 00000000 00000000 00000000 00 00000001
226 : //
227 : // AuxFiles:
228 : // 03 00000000 00000000 00000000 00 00000002
229 : //
230 :
231 : //-- Section 01: relation data and metadata
232 :
233 : pub const DBDIR_KEY: Key = Key {
234 : field1: 0x00,
235 : field2: 0,
236 : field3: 0,
237 : field4: 0,
238 : field5: 0,
239 : field6: 0,
240 : };
241 :
242 : #[inline(always)]
243 0 : pub fn dbdir_key_range(spcnode: Oid, dbnode: Oid) -> Range<Key> {
244 0 : Key {
245 0 : field1: 0x00,
246 0 : field2: spcnode,
247 0 : field3: dbnode,
248 0 : field4: 0,
249 0 : field5: 0,
250 0 : field6: 0,
251 0 : }..Key {
252 0 : field1: 0x00,
253 0 : field2: spcnode,
254 0 : field3: dbnode,
255 0 : field4: 0xffffffff,
256 0 : field5: 0xff,
257 0 : field6: 0xffffffff,
258 0 : }
259 0 : }
260 :
261 : #[inline(always)]
262 16 : pub fn relmap_file_key(spcnode: Oid, dbnode: Oid) -> Key {
263 16 : Key {
264 16 : field1: 0x00,
265 16 : field2: spcnode,
266 16 : field3: dbnode,
267 16 : field4: 0,
268 16 : field5: 0,
269 16 : field6: 0,
270 16 : }
271 16 : }
272 :
273 : #[inline(always)]
274 1948 : pub fn rel_dir_to_key(spcnode: Oid, dbnode: Oid) -> Key {
275 1948 : Key {
276 1948 : field1: 0x00,
277 1948 : field2: spcnode,
278 1948 : field3: dbnode,
279 1948 : field4: 0,
280 1948 : field5: 0,
281 1948 : field6: 1,
282 1948 : }
283 1948 : }
284 :
285 : #[inline(always)]
286 865308 : pub fn rel_block_to_key(rel: RelTag, blknum: BlockNumber) -> Key {
287 865308 : Key {
288 865308 : field1: 0x00,
289 865308 : field2: rel.spcnode,
290 865308 : field3: rel.dbnode,
291 865308 : field4: rel.relnode,
292 865308 : field5: rel.forknum,
293 865308 : field6: blknum,
294 865308 : }
295 865308 : }
296 :
297 : #[inline(always)]
298 289754 : pub fn rel_size_to_key(rel: RelTag) -> Key {
299 289754 : Key {
300 289754 : field1: 0x00,
301 289754 : field2: rel.spcnode,
302 289754 : field3: rel.dbnode,
303 289754 : field4: rel.relnode,
304 289754 : field5: rel.forknum,
305 289754 : field6: 0xffffffff,
306 289754 : }
307 289754 : }
308 :
309 : #[inline(always)]
310 2 : pub fn rel_key_range(rel: RelTag) -> Range<Key> {
311 2 : Key {
312 2 : field1: 0x00,
313 2 : field2: rel.spcnode,
314 2 : field3: rel.dbnode,
315 2 : field4: rel.relnode,
316 2 : field5: rel.forknum,
317 2 : field6: 0,
318 2 : }..Key {
319 2 : field1: 0x00,
320 2 : field2: rel.spcnode,
321 2 : field3: rel.dbnode,
322 2 : field4: rel.relnode,
323 2 : field5: rel.forknum + 1,
324 2 : field6: 0,
325 2 : }
326 2 : }
327 :
328 : //-- Section 02: SLRUs
329 :
330 : #[inline(always)]
331 756 : pub fn slru_dir_to_key(kind: SlruKind) -> Key {
332 756 : Key {
333 756 : field1: 0x01,
334 756 : field2: match kind {
335 252 : SlruKind::Clog => 0x00,
336 252 : SlruKind::MultiXactMembers => 0x01,
337 252 : SlruKind::MultiXactOffsets => 0x02,
338 : },
339 : field3: 0,
340 : field4: 0,
341 : field5: 0,
342 : field6: 0,
343 : }
344 756 : }
345 :
346 : #[inline(always)]
347 14 : pub fn slru_block_to_key(kind: SlruKind, segno: u32, blknum: BlockNumber) -> Key {
348 14 : Key {
349 14 : field1: 0x01,
350 14 : field2: match kind {
351 10 : SlruKind::Clog => 0x00,
352 2 : SlruKind::MultiXactMembers => 0x01,
353 2 : SlruKind::MultiXactOffsets => 0x02,
354 : },
355 : field3: 1,
356 14 : field4: segno,
357 14 : field5: 0,
358 14 : field6: blknum,
359 14 : }
360 14 : }
361 :
362 : #[inline(always)]
363 6 : pub fn slru_segment_size_to_key(kind: SlruKind, segno: u32) -> Key {
364 6 : Key {
365 6 : field1: 0x01,
366 6 : field2: match kind {
367 2 : SlruKind::Clog => 0x00,
368 2 : SlruKind::MultiXactMembers => 0x01,
369 2 : SlruKind::MultiXactOffsets => 0x02,
370 : },
371 : field3: 1,
372 6 : field4: segno,
373 6 : field5: 0,
374 6 : field6: 0xffffffff,
375 6 : }
376 6 : }
377 :
378 : #[inline(always)]
379 0 : pub fn slru_segment_key_range(kind: SlruKind, segno: u32) -> Range<Key> {
380 0 : let field2 = match kind {
381 0 : SlruKind::Clog => 0x00,
382 0 : SlruKind::MultiXactMembers => 0x01,
383 0 : SlruKind::MultiXactOffsets => 0x02,
384 : };
385 :
386 0 : Key {
387 0 : field1: 0x01,
388 0 : field2,
389 0 : field3: 1,
390 0 : field4: segno,
391 0 : field5: 0,
392 0 : field6: 0,
393 0 : }..Key {
394 0 : field1: 0x01,
395 0 : field2,
396 0 : field3: 1,
397 0 : field4: segno,
398 0 : field5: 1,
399 0 : field6: 0,
400 0 : }
401 0 : }
402 :
403 : //-- Section 03: pg_twophase
404 :
405 : pub const TWOPHASEDIR_KEY: Key = Key {
406 : field1: 0x02,
407 : field2: 0,
408 : field3: 0,
409 : field4: 0,
410 : field5: 0,
411 : field6: 0,
412 : };
413 :
414 : #[inline(always)]
415 0 : pub fn twophase_file_key(xid: TransactionId) -> Key {
416 0 : Key {
417 0 : field1: 0x02,
418 0 : field2: 0,
419 0 : field3: 0,
420 0 : field4: 0,
421 0 : field5: 0,
422 0 : field6: xid,
423 0 : }
424 0 : }
425 :
426 : #[inline(always)]
427 0 : pub fn twophase_key_range(xid: TransactionId) -> Range<Key> {
428 0 : let (next_xid, overflowed) = xid.overflowing_add(1);
429 0 :
430 0 : Key {
431 0 : field1: 0x02,
432 0 : field2: 0,
433 0 : field3: 0,
434 0 : field4: 0,
435 0 : field5: 0,
436 0 : field6: xid,
437 0 : }..Key {
438 0 : field1: 0x02,
439 0 : field2: 0,
440 0 : field3: 0,
441 0 : field4: 0,
442 0 : field5: u8::from(overflowed),
443 0 : field6: next_xid,
444 0 : }
445 0 : }
446 :
447 : //-- Section 03: Control file
448 : pub const CONTROLFILE_KEY: Key = Key {
449 : field1: 0x03,
450 : field2: 0,
451 : field3: 0,
452 : field4: 0,
453 : field5: 0,
454 : field6: 0,
455 : };
456 :
457 : pub const CHECKPOINT_KEY: Key = Key {
458 : field1: 0x03,
459 : field2: 0,
460 : field3: 0,
461 : field4: 0,
462 : field5: 0,
463 : field6: 1,
464 : };
465 :
466 : pub const AUX_FILES_KEY: Key = Key {
467 : field1: 0x03,
468 : field2: 0,
469 : field3: 0,
470 : field4: 0,
471 : field5: 0,
472 : field6: 2,
473 : };
474 :
475 : // Reverse mappings for a few Keys.
476 : // These are needed by WAL redo manager.
477 :
478 : // AUX_FILES currently stores only data for logical replication (slots etc), and
479 : // we don't preserve these on a branch because safekeepers can't follow timeline
480 : // switch (and generally it likely should be optional), so ignore these.
481 : #[inline(always)]
482 854144 : pub fn is_inherited_key(key: Key) -> bool {
483 854144 : key != AUX_FILES_KEY
484 854144 : }
485 :
486 : #[inline(always)]
487 0 : pub fn is_rel_fsm_block_key(key: Key) -> bool {
488 0 : key.field1 == 0x00 && key.field4 != 0 && key.field5 == FSM_FORKNUM && key.field6 != 0xffffffff
489 0 : }
490 :
491 : #[inline(always)]
492 0 : pub fn is_rel_vm_block_key(key: Key) -> bool {
493 0 : key.field1 == 0x00
494 0 : && key.field4 != 0
495 0 : && key.field5 == VISIBILITYMAP_FORKNUM
496 0 : && key.field6 != 0xffffffff
497 0 : }
498 :
499 : #[inline(always)]
500 0 : pub fn key_to_slru_block(key: Key) -> anyhow::Result<(SlruKind, u32, BlockNumber)> {
501 0 : Ok(match key.field1 {
502 : 0x01 => {
503 0 : let kind = match key.field2 {
504 0 : 0x00 => SlruKind::Clog,
505 0 : 0x01 => SlruKind::MultiXactMembers,
506 0 : 0x02 => SlruKind::MultiXactOffsets,
507 0 : _ => anyhow::bail!("unrecognized slru kind 0x{:02x}", key.field2),
508 : };
509 0 : let segno = key.field4;
510 0 : let blknum = key.field6;
511 0 :
512 0 : (kind, segno, blknum)
513 : }
514 0 : _ => anyhow::bail!("unexpected value kind 0x{:02x}", key.field1),
515 : })
516 0 : }
517 :
518 : #[inline(always)]
519 0 : pub fn is_slru_block_key(key: Key) -> bool {
520 0 : key.field1 == 0x01 // SLRU-related
521 0 : && key.field3 == 0x00000001 // but not SlruDir
522 0 : && key.field6 != 0xffffffff // and not SlruSegSize
523 0 : }
524 :
525 : #[inline(always)]
526 2605710 : pub fn is_rel_block_key(key: &Key) -> bool {
527 2605710 : key.field1 == 0x00 && key.field4 != 0 && key.field6 != 0xffffffff
528 2605710 : }
529 :
530 : /// Guaranteed to return `Ok()` if [[is_rel_block_key]] returns `true` for `key`.
531 : #[inline(always)]
532 6 : pub fn key_to_rel_block(key: Key) -> anyhow::Result<(RelTag, BlockNumber)> {
533 6 : Ok(match key.field1 {
534 6 : 0x00 => (
535 6 : RelTag {
536 6 : spcnode: key.field2,
537 6 : dbnode: key.field3,
538 6 : relnode: key.field4,
539 6 : forknum: key.field5,
540 6 : },
541 6 : key.field6,
542 6 : ),
543 0 : _ => anyhow::bail!("unexpected value kind 0x{:02x}", key.field1),
544 : })
545 6 : }
546 :
547 : impl std::str::FromStr for Key {
548 : type Err = anyhow::Error;
549 :
550 14 : fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
551 14 : Self::from_hex(s)
552 14 : }
553 : }
554 :
555 : #[cfg(test)]
556 : mod tests {
557 : use std::str::FromStr;
558 :
559 : use crate::key::Key;
560 :
561 : use rand::Rng;
562 : use rand::SeedableRng;
563 :
564 2 : #[test]
565 2 : fn display_fromstr_bijection() {
566 2 : let mut rng = rand::rngs::StdRng::seed_from_u64(42);
567 2 :
568 2 : let key = Key {
569 2 : field1: rng.gen(),
570 2 : field2: rng.gen(),
571 2 : field3: rng.gen(),
572 2 : field4: rng.gen(),
573 2 : field5: rng.gen(),
574 2 : field6: rng.gen(),
575 2 : };
576 2 :
577 2 : assert_eq!(key, Key::from_str(&format!("{key}")).unwrap());
578 2 : }
579 : }
|
