TLA Line data Source code
1 : use std::cmp;
2 : use std::collections::hash_map::Entry;
3 : use std::collections::{HashMap, HashSet};
4 : use std::sync::Arc;
5 :
6 : use anyhow::{bail, Context};
7 : use tokio::sync::oneshot::error::RecvError;
8 : use tokio::sync::Semaphore;
9 : use tokio_util::sync::CancellationToken;
10 :
11 : use crate::context::RequestContext;
12 : use crate::pgdatadir_mapping::CalculateLogicalSizeError;
13 :
14 : use super::{LogicalSizeCalculationCause, Tenant};
15 : use crate::tenant::Timeline;
16 : use utils::id::TimelineId;
17 : use utils::lsn::Lsn;
18 :
19 : use tracing::*;
20 :
21 : use tenant_size_model::{Segment, StorageModel};
22 :
23 : /// Inputs to the actual tenant sizing model
24 : ///
25 : /// Implements [`serde::Serialize`] but is not meant to be part of the public API, instead meant to
26 : /// be a transferrable format between execution environments and developer.
27 : ///
28 : /// This tracks more information than the actual StorageModel that calculation
29 : /// needs. We will convert this into a StorageModel when it's time to perform
30 : /// the calculation.
31 : ///
32 CBC 55 : #[derive(Debug, serde::Serialize, serde::Deserialize)]
33 : pub struct ModelInputs {
34 : pub segments: Vec<SegmentMeta>,
35 : pub timeline_inputs: Vec<TimelineInputs>,
36 : }
37 :
38 : /// A [`Segment`], with some extra information for display purposes
39 261 : #[derive(Debug, serde::Serialize, serde::Deserialize)]
40 : pub struct SegmentMeta {
41 : pub segment: Segment,
42 : pub timeline_id: TimelineId,
43 : pub kind: LsnKind,
44 : }
45 :
46 : impl SegmentMeta {
47 651 : fn size_needed(&self) -> bool {
48 651 : match self.kind {
49 : LsnKind::BranchStart => {
50 : // If we don't have a later GcCutoff point on this branch, and
51 : // no ancestor, calculate size for the branch start point.
52 256 : self.segment.needed && self.segment.parent.is_none()
53 : }
54 82 : LsnKind::BranchPoint => true,
55 57 : LsnKind::GcCutOff => true,
56 256 : LsnKind::BranchEnd => false,
57 : }
58 651 : }
59 : }
60 :
61 : #[derive(
62 261 : Debug, Clone, Copy, Eq, Ord, PartialEq, PartialOrd, serde::Serialize, serde::Deserialize,
63 : )]
64 : pub enum LsnKind {
65 : /// A timeline starting here
66 : BranchStart,
67 : /// A child timeline branches off from here
68 : BranchPoint,
69 : /// GC cutoff point
70 : GcCutOff,
71 : /// Last record LSN
72 : BranchEnd,
73 : }
74 :
75 : /// Collect all relevant LSNs to the inputs. These will only be helpful in the serialized form as
76 : /// part of [`ModelInputs`] from the HTTP api, explaining the inputs.
77 96 : #[derive(Debug, serde::Serialize, serde::Deserialize)]
78 : pub struct TimelineInputs {
79 : pub timeline_id: TimelineId,
80 :
81 : pub ancestor_id: Option<TimelineId>,
82 :
83 : ancestor_lsn: Lsn,
84 : last_record: Lsn,
85 : latest_gc_cutoff: Lsn,
86 : horizon_cutoff: Lsn,
87 : pitr_cutoff: Lsn,
88 :
89 : /// Cutoff point based on GC settings
90 : next_gc_cutoff: Lsn,
91 :
92 : /// Cutoff point calculated from the user-supplied 'max_retention_period'
93 : retention_param_cutoff: Option<Lsn>,
94 : }
95 :
96 : /// Gathers the inputs for the tenant sizing model.
97 : ///
98 : /// Tenant size does not consider the latest state, but only the state until next_gc_cutoff, which
99 : /// is updated on-demand, during the start of this calculation and separate from the
100 : /// [`TimelineInputs::latest_gc_cutoff`].
101 : ///
102 : /// For timelines in general:
103 : ///
104 : /// ```text
105 : /// 0-----|---------|----|------------| · · · · · |·> lsn
106 : /// initdb_lsn branchpoints* next_gc_cutoff latest
107 : /// ```
108 : ///
109 : /// Until gc_horizon_cutoff > `Timeline::last_record_lsn` for any of the tenant's timelines, the
110 : /// tenant size will be zero.
111 92 : pub(super) async fn gather_inputs(
112 92 : tenant: &Tenant,
113 92 : limit: &Arc<Semaphore>,
114 92 : max_retention_period: Option<u64>,
115 92 : logical_size_cache: &mut HashMap<(TimelineId, Lsn), u64>,
116 92 : cause: LogicalSizeCalculationCause,
117 92 : cancel: &CancellationToken,
118 92 : ctx: &RequestContext,
119 92 : ) -> anyhow::Result<ModelInputs> {
120 92 : // refresh is needed to update gc related pitr_cutoff and horizon_cutoff
121 92 : tenant
122 92 : .refresh_gc_info(cancel, ctx)
123 20 : .await
124 92 : .context("Failed to refresh gc_info before gathering inputs")?;
125 :
126 : // Collect information about all the timelines
127 92 : let mut timelines = tenant.list_timelines();
128 92 :
129 92 : if timelines.is_empty() {
130 : // perhaps the tenant has just been created, and as such doesn't have any data yet
131 4 : return Ok(ModelInputs {
132 4 : segments: vec![],
133 4 : timeline_inputs: Vec::new(),
134 4 : });
135 88 : }
136 88 :
137 88 : // Filter out timelines that are not active
138 88 : //
139 88 : // There may be a race when a timeline is dropped,
140 88 : // but it is unlikely to cause any issues. In the worst case,
141 88 : // the calculation will error out.
142 129 : timelines.retain(|t| t.is_active());
143 88 :
144 88 : // Build a map of branch points.
145 88 : let mut branchpoints: HashMap<TimelineId, HashSet<Lsn>> = HashMap::new();
146 129 : for timeline in timelines.iter() {
147 129 : if let Some(ancestor_id) = timeline.get_ancestor_timeline_id() {
148 41 : branchpoints
149 41 : .entry(ancestor_id)
150 41 : .or_default()
151 41 : .insert(timeline.get_ancestor_lsn());
152 88 : }
153 : }
154 :
155 : // These become the final result.
156 88 : let mut timeline_inputs = Vec::with_capacity(timelines.len());
157 88 : let mut segments: Vec<SegmentMeta> = Vec::new();
158 88 :
159 88 : //
160 88 : // Build Segments representing each timeline. As we do that, also remember
161 88 : // the branchpoints and branch startpoints in 'branchpoint_segments' and
162 88 : // 'branchstart_segments'
163 88 : //
164 88 :
165 88 : // BranchPoint segments of each timeline
166 88 : // (timeline, branchpoint LSN) -> segment_id
167 88 : let mut branchpoint_segments: HashMap<(TimelineId, Lsn), usize> = HashMap::new();
168 88 :
169 88 : // timeline, Branchpoint seg id, (ancestor, ancestor LSN)
170 88 : type BranchStartSegment = (TimelineId, usize, Option<(TimelineId, Lsn)>);
171 88 : let mut branchstart_segments: Vec<BranchStartSegment> = Vec::new();
172 :
173 129 : for timeline in timelines.iter() {
174 129 : let timeline_id = timeline.timeline_id;
175 129 : let last_record_lsn = timeline.get_last_record_lsn();
176 129 : let ancestor_lsn = timeline.get_ancestor_lsn();
177 129 :
178 129 : // there's a race between the update (holding tenant.gc_lock) and this read but it
179 129 : // might not be an issue, because it's not for Timeline::gc
180 129 : let gc_info = timeline.gc_info.read().unwrap();
181 129 :
182 129 : // similar to gc, but Timeline::get_latest_gc_cutoff_lsn() will not be updated before a
183 129 : // new gc run, which we have no control over. however differently from `Timeline::gc`
184 129 : // we don't consider the `Timeline::disk_consistent_lsn` at all, because we are not
185 129 : // actually removing files.
186 129 : let mut next_gc_cutoff = cmp::min(gc_info.horizon_cutoff, gc_info.pitr_cutoff);
187 :
188 : // If the caller provided a shorter retention period, use that instead of the GC cutoff.
189 129 : let retention_param_cutoff = if let Some(max_retention_period) = max_retention_period {
190 UBC 0 : let param_cutoff = Lsn(last_record_lsn.0.saturating_sub(max_retention_period));
191 0 : if next_gc_cutoff < param_cutoff {
192 0 : next_gc_cutoff = param_cutoff;
193 0 : }
194 0 : Some(param_cutoff)
195 : } else {
196 CBC 129 : None
197 : };
198 :
199 : // next_gc_cutoff in parent branch are not of interest (right now at least), nor do we
200 : // want to query any logical size before initdb_lsn.
201 129 : let branch_start_lsn = cmp::max(ancestor_lsn, timeline.initdb_lsn);
202 129 :
203 129 : // Build "interesting LSNs" on this timeline
204 129 : let mut lsns: Vec<(Lsn, LsnKind)> = gc_info
205 129 : .retain_lsns
206 129 : .iter()
207 129 : .filter(|&&lsn| lsn > ancestor_lsn)
208 129 : .copied()
209 129 : // this assumes there are no other retain_lsns than the branchpoints
210 129 : .map(|lsn| (lsn, LsnKind::BranchPoint))
211 129 : .collect::<Vec<_>>();
212 :
213 : // Add branch points we collected earlier, just in case there were any that were
214 : // not present in retain_lsns. We will remove any duplicates below later.
215 129 : if let Some(this_branchpoints) = branchpoints.get(&timeline_id) {
216 36 : lsns.extend(
217 36 : this_branchpoints
218 36 : .iter()
219 41 : .map(|lsn| (*lsn, LsnKind::BranchPoint)),
220 36 : )
221 93 : }
222 :
223 : // Add a point for the GC cutoff
224 129 : let branch_start_needed = next_gc_cutoff <= branch_start_lsn;
225 129 : if !branch_start_needed {
226 29 : lsns.push((next_gc_cutoff, LsnKind::GcCutOff));
227 100 : }
228 :
229 129 : lsns.sort_unstable();
230 129 : lsns.dedup();
231 129 :
232 129 : //
233 129 : // Create Segments for the interesting points.
234 129 : //
235 129 :
236 129 : // Timeline start point
237 129 : let ancestor = timeline
238 129 : .get_ancestor_timeline_id()
239 129 : .map(|ancestor_id| (ancestor_id, ancestor_lsn));
240 129 : branchstart_segments.push((timeline_id, segments.len(), ancestor));
241 129 : segments.push(SegmentMeta {
242 129 : segment: Segment {
243 129 : parent: None, // filled in later
244 129 : lsn: branch_start_lsn.0,
245 129 : size: None, // filled in later
246 129 : needed: branch_start_needed,
247 129 : },
248 129 : timeline_id: timeline.timeline_id,
249 129 : kind: LsnKind::BranchStart,
250 129 : });
251 129 :
252 129 : // GC cutoff point, and any branch points, i.e. points where
253 129 : // other timelines branch off from this timeline.
254 129 : let mut parent = segments.len() - 1;
255 199 : for (lsn, kind) in lsns {
256 70 : if kind == LsnKind::BranchPoint {
257 41 : branchpoint_segments.insert((timeline_id, lsn), segments.len());
258 41 : }
259 70 : segments.push(SegmentMeta {
260 70 : segment: Segment {
261 70 : parent: Some(parent),
262 70 : lsn: lsn.0,
263 70 : size: None,
264 70 : needed: lsn > next_gc_cutoff,
265 70 : },
266 70 : timeline_id: timeline.timeline_id,
267 70 : kind,
268 70 : });
269 70 : parent += 1;
270 : }
271 :
272 : // Current end of the timeline
273 129 : segments.push(SegmentMeta {
274 129 : segment: Segment {
275 129 : parent: Some(parent),
276 129 : lsn: last_record_lsn.0,
277 129 : size: None, // Filled in later, if necessary
278 129 : needed: true,
279 129 : },
280 129 : timeline_id: timeline.timeline_id,
281 129 : kind: LsnKind::BranchEnd,
282 129 : });
283 129 :
284 129 : timeline_inputs.push(TimelineInputs {
285 129 : timeline_id: timeline.timeline_id,
286 129 : ancestor_id: timeline.get_ancestor_timeline_id(),
287 129 : ancestor_lsn,
288 129 : last_record: last_record_lsn,
289 129 : // this is not used above, because it might not have updated recently enough
290 129 : latest_gc_cutoff: *timeline.get_latest_gc_cutoff_lsn(),
291 129 : horizon_cutoff: gc_info.horizon_cutoff,
292 129 : pitr_cutoff: gc_info.pitr_cutoff,
293 129 : next_gc_cutoff,
294 129 : retention_param_cutoff,
295 129 : });
296 : }
297 :
298 : // We now have all segments from the timelines in 'segments'. The timelines
299 : // haven't been linked to each other yet, though. Do that.
300 217 : for (_timeline_id, seg_id, ancestor) in branchstart_segments {
301 : // Look up the branch point
302 129 : if let Some(ancestor) = ancestor {
303 41 : let parent_id = *branchpoint_segments.get(&ancestor).unwrap();
304 41 : segments[seg_id].segment.parent = Some(parent_id);
305 88 : }
306 : }
307 :
308 : // We left the 'size' field empty in all of the Segments so far.
309 : // Now find logical sizes for all of the points that might need or benefit from them.
310 88 : fill_logical_sizes(
311 88 : &timelines,
312 88 : &mut segments,
313 88 : limit,
314 88 : logical_size_cache,
315 88 : cause,
316 88 : ctx,
317 88 : )
318 39 : .await?;
319 :
320 86 : Ok(ModelInputs {
321 86 : segments,
322 86 : timeline_inputs,
323 86 : })
324 92 : }
325 :
326 : /// Augment 'segments' with logical sizes
327 : ///
328 : /// this will probably conflict with on-demand downloaded layers, or at least force them all
329 : /// to be downloaded
330 : ///
331 88 : async fn fill_logical_sizes(
332 88 : timelines: &[Arc<Timeline>],
333 88 : segments: &mut [SegmentMeta],
334 88 : limit: &Arc<Semaphore>,
335 88 : logical_size_cache: &mut HashMap<(TimelineId, Lsn), u64>,
336 88 : cause: LogicalSizeCalculationCause,
337 88 : ctx: &RequestContext,
338 88 : ) -> anyhow::Result<()> {
339 88 : let timeline_hash: HashMap<TimelineId, Arc<Timeline>> = HashMap::from_iter(
340 88 : timelines
341 88 : .iter()
342 129 : .map(|timeline| (timeline.timeline_id, Arc::clone(timeline))),
343 88 : );
344 88 :
345 88 : // record the used/inserted cache keys here, to remove extras not to start leaking
346 88 : // after initial run the cache should be quite stable, but live timelines will eventually
347 88 : // require new lsns to be inspected.
348 88 : let mut sizes_needed = HashMap::<(TimelineId, Lsn), Option<u64>>::new();
349 88 :
350 88 : // with joinset, on drop, all of the tasks will just be de-scheduled, which we can use to
351 88 : // our advantage with `?` error handling.
352 88 : let mut joinset = tokio::task::JoinSet::new();
353 :
354 : // For each point that would benefit from having a logical size available,
355 : // spawn a Task to fetch it, unless we have it cached already.
356 328 : for seg in segments.iter() {
357 328 : if !seg.size_needed() {
358 187 : continue;
359 141 : }
360 141 :
361 141 : let timeline_id = seg.timeline_id;
362 141 : let lsn = Lsn(seg.segment.lsn);
363 :
364 141 : if let Entry::Vacant(e) = sizes_needed.entry((timeline_id, lsn)) {
365 141 : let cached_size = logical_size_cache.get(&(timeline_id, lsn)).cloned();
366 141 : if cached_size.is_none() {
367 39 : let timeline = Arc::clone(timeline_hash.get(&timeline_id).unwrap());
368 39 : let parallel_size_calcs = Arc::clone(limit);
369 39 : let ctx = ctx.attached_child();
370 39 : joinset.spawn(
371 39 : calculate_logical_size(parallel_size_calcs, timeline, lsn, cause, ctx)
372 39 : .in_current_span(),
373 39 : );
374 102 : }
375 141 : e.insert(cached_size);
376 UBC 0 : }
377 : }
378 :
379 : // Perform the size lookups
380 CBC 88 : let mut have_any_error = false;
381 127 : while let Some(res) = joinset.join_next().await {
382 : // each of these come with Result<anyhow::Result<_>, JoinError>
383 : // because of spawn + spawn_blocking
384 39 : match res {
385 UBC 0 : Err(join_error) if join_error.is_cancelled() => {
386 0 : unreachable!("we are not cancelling any of the futures, nor should be");
387 : }
388 0 : Err(join_error) => {
389 : // cannot really do anything, as this panic is likely a bug
390 0 : error!("task that calls spawn_ondemand_logical_size_calculation panicked: {join_error:#}");
391 0 : have_any_error = true;
392 : }
393 0 : Ok(Err(recv_result_error)) => {
394 : // cannot really do anything, as this panic is likely a bug
395 0 : error!("failed to receive logical size query result: {recv_result_error:#}");
396 0 : have_any_error = true;
397 : }
398 CBC 2 : Ok(Ok(TimelineAtLsnSizeResult(timeline, lsn, Err(error)))) => {
399 2 : if !matches!(error, CalculateLogicalSizeError::Cancelled) {
400 UBC 0 : warn!(
401 0 : timeline_id=%timeline.timeline_id,
402 0 : "failed to calculate logical size at {lsn}: {error:#}"
403 0 : );
404 CBC 2 : }
405 2 : have_any_error = true;
406 : }
407 37 : Ok(Ok(TimelineAtLsnSizeResult(timeline, lsn, Ok(size)))) => {
408 UBC 0 : debug!(timeline_id=%timeline.timeline_id, %lsn, size, "size calculated");
409 :
410 CBC 37 : logical_size_cache.insert((timeline.timeline_id, lsn), size);
411 37 : sizes_needed.insert((timeline.timeline_id, lsn), Some(size));
412 : }
413 : }
414 : }
415 :
416 : // prune any keys not needed anymore; we record every used key and added key.
417 148 : logical_size_cache.retain(|key, _| sizes_needed.contains_key(key));
418 88 :
419 88 : if have_any_error {
420 : // we cannot complete this round, because we are missing data.
421 : // we have however cached all we were able to request calculation on.
422 2 : anyhow::bail!("failed to calculate some logical_sizes");
423 86 : }
424 :
425 : // Insert the looked up sizes to the Segments
426 323 : for seg in segments.iter_mut() {
427 323 : if !seg.size_needed() {
428 184 : continue;
429 139 : }
430 139 :
431 139 : let timeline_id = seg.timeline_id;
432 139 : let lsn = Lsn(seg.segment.lsn);
433 :
434 139 : if let Some(Some(size)) = sizes_needed.get(&(timeline_id, lsn)) {
435 139 : seg.segment.size = Some(*size);
436 139 : } else {
437 UBC 0 : bail!("could not find size at {} in timeline {}", lsn, timeline_id);
438 : }
439 : }
440 CBC 86 : Ok(())
441 88 : }
442 :
443 : impl ModelInputs {
444 84 : pub fn calculate_model(&self) -> anyhow::Result<tenant_size_model::StorageModel> {
445 84 : // Convert SegmentMetas into plain Segments
446 84 : let storage = StorageModel {
447 84 : segments: self
448 84 : .segments
449 84 : .iter()
450 321 : .map(|seg| seg.segment.clone())
451 84 : .collect(),
452 84 : };
453 84 :
454 84 : Ok(storage)
455 84 : }
456 :
457 : // calculate total project size
458 28 : pub fn calculate(&self) -> anyhow::Result<u64> {
459 28 : let storage = self.calculate_model()?;
460 28 : let sizes = storage.calculate();
461 28 :
462 28 : Ok(sizes.total_size)
463 28 : }
464 : }
465 :
466 : /// Newtype around the tuple that carries the timeline at lsn logical size calculation.
467 : struct TimelineAtLsnSizeResult(
468 : Arc<crate::tenant::Timeline>,
469 : utils::lsn::Lsn,
470 : Result<u64, CalculateLogicalSizeError>,
471 : );
472 :
473 39 : #[instrument(skip_all, fields(timeline_id=%timeline.timeline_id, lsn=%lsn))]
474 : async fn calculate_logical_size(
475 : limit: Arc<tokio::sync::Semaphore>,
476 : timeline: Arc<crate::tenant::Timeline>,
477 : lsn: utils::lsn::Lsn,
478 : cause: LogicalSizeCalculationCause,
479 : ctx: RequestContext,
480 : ) -> Result<TimelineAtLsnSizeResult, RecvError> {
481 : let _permit = tokio::sync::Semaphore::acquire_owned(limit)
482 : .await
483 : .expect("global semaphore should not had been closed");
484 :
485 : let size_res = timeline
486 : .spawn_ondemand_logical_size_calculation(lsn, cause, ctx)
487 : .instrument(info_span!("spawn_ondemand_logical_size_calculation"))
488 : .await?;
489 : Ok(TimelineAtLsnSizeResult(timeline, lsn, size_res))
490 : }
491 :
492 1 : #[test]
493 1 : fn verify_size_for_multiple_branches() {
494 1 : // this is generated from integration test test_tenant_size_with_multiple_branches, but this way
495 1 : // it has the stable lsn's
496 1 : //
497 1 : // The timeline_inputs don't participate in the size calculation, and are here just to explain
498 1 : // the inputs.
499 1 : let doc = r#"
500 1 : {
501 1 : "segments": [
502 1 : {
503 1 : "segment": {
504 1 : "parent": 9,
505 1 : "lsn": 26033560,
506 1 : "size": null,
507 1 : "needed": false
508 1 : },
509 1 : "timeline_id": "20b129c9b50cff7213e6503a31b2a5ce",
510 1 : "kind": "BranchStart"
511 1 : },
512 1 : {
513 1 : "segment": {
514 1 : "parent": 0,
515 1 : "lsn": 35720400,
516 1 : "size": 25206784,
517 1 : "needed": false
518 1 : },
519 1 : "timeline_id": "20b129c9b50cff7213e6503a31b2a5ce",
520 1 : "kind": "GcCutOff"
521 1 : },
522 1 : {
523 1 : "segment": {
524 1 : "parent": 1,
525 1 : "lsn": 35851472,
526 1 : "size": null,
527 1 : "needed": true
528 1 : },
529 1 : "timeline_id": "20b129c9b50cff7213e6503a31b2a5ce",
530 1 : "kind": "BranchEnd"
531 1 : },
532 1 : {
533 1 : "segment": {
534 1 : "parent": 7,
535 1 : "lsn": 24566168,
536 1 : "size": null,
537 1 : "needed": false
538 1 : },
539 1 : "timeline_id": "454626700469f0a9914949b9d018e876",
540 1 : "kind": "BranchStart"
541 1 : },
542 1 : {
543 1 : "segment": {
544 1 : "parent": 3,
545 1 : "lsn": 25261936,
546 1 : "size": 26050560,
547 1 : "needed": false
548 1 : },
549 1 : "timeline_id": "454626700469f0a9914949b9d018e876",
550 1 : "kind": "GcCutOff"
551 1 : },
552 1 : {
553 1 : "segment": {
554 1 : "parent": 4,
555 1 : "lsn": 25393008,
556 1 : "size": null,
557 1 : "needed": true
558 1 : },
559 1 : "timeline_id": "454626700469f0a9914949b9d018e876",
560 1 : "kind": "BranchEnd"
561 1 : },
562 1 : {
563 1 : "segment": {
564 1 : "parent": null,
565 1 : "lsn": 23694408,
566 1 : "size": null,
567 1 : "needed": false
568 1 : },
569 1 : "timeline_id": "cb5e3cbe60a4afc00d01880e1a37047f",
570 1 : "kind": "BranchStart"
571 1 : },
572 1 : {
573 1 : "segment": {
574 1 : "parent": 6,
575 1 : "lsn": 24566168,
576 1 : "size": 25739264,
577 1 : "needed": false
578 1 : },
579 1 : "timeline_id": "cb5e3cbe60a4afc00d01880e1a37047f",
580 1 : "kind": "BranchPoint"
581 1 : },
582 1 : {
583 1 : "segment": {
584 1 : "parent": 7,
585 1 : "lsn": 25902488,
586 1 : "size": 26402816,
587 1 : "needed": false
588 1 : },
589 1 : "timeline_id": "cb5e3cbe60a4afc00d01880e1a37047f",
590 1 : "kind": "GcCutOff"
591 1 : },
592 1 : {
593 1 : "segment": {
594 1 : "parent": 8,
595 1 : "lsn": 26033560,
596 1 : "size": 26468352,
597 1 : "needed": true
598 1 : },
599 1 : "timeline_id": "cb5e3cbe60a4afc00d01880e1a37047f",
600 1 : "kind": "BranchPoint"
601 1 : },
602 1 : {
603 1 : "segment": {
604 1 : "parent": 9,
605 1 : "lsn": 26033560,
606 1 : "size": null,
607 1 : "needed": true
608 1 : },
609 1 : "timeline_id": "cb5e3cbe60a4afc00d01880e1a37047f",
610 1 : "kind": "BranchEnd"
611 1 : }
612 1 : ],
613 1 : "timeline_inputs": [
614 1 : {
615 1 : "timeline_id": "20b129c9b50cff7213e6503a31b2a5ce",
616 1 : "ancestor_lsn": "0/18D3D98",
617 1 : "last_record": "0/2230CD0",
618 1 : "latest_gc_cutoff": "0/1698C48",
619 1 : "horizon_cutoff": "0/2210CD0",
620 1 : "pitr_cutoff": "0/2210CD0",
621 1 : "next_gc_cutoff": "0/2210CD0",
622 1 : "retention_param_cutoff": null
623 1 : },
624 1 : {
625 1 : "timeline_id": "454626700469f0a9914949b9d018e876",
626 1 : "ancestor_lsn": "0/176D998",
627 1 : "last_record": "0/1837770",
628 1 : "latest_gc_cutoff": "0/1698C48",
629 1 : "horizon_cutoff": "0/1817770",
630 1 : "pitr_cutoff": "0/1817770",
631 1 : "next_gc_cutoff": "0/1817770",
632 1 : "retention_param_cutoff": null
633 1 : },
634 1 : {
635 1 : "timeline_id": "cb5e3cbe60a4afc00d01880e1a37047f",
636 1 : "ancestor_lsn": "0/0",
637 1 : "last_record": "0/18D3D98",
638 1 : "latest_gc_cutoff": "0/1698C48",
639 1 : "horizon_cutoff": "0/18B3D98",
640 1 : "pitr_cutoff": "0/18B3D98",
641 1 : "next_gc_cutoff": "0/18B3D98",
642 1 : "retention_param_cutoff": null
643 1 : }
644 1 : ]
645 1 : }
646 1 : "#;
647 1 : let inputs: ModelInputs = serde_json::from_str(doc).unwrap();
648 1 :
649 1 : assert_eq!(inputs.calculate().unwrap(), 37_851_408);
650 1 : }
651 :
652 1 : #[test]
653 1 : fn verify_size_for_one_branch() {
654 1 : let doc = r#"
655 1 : {
656 1 : "segments": [
657 1 : {
658 1 : "segment": {
659 1 : "parent": null,
660 1 : "lsn": 0,
661 1 : "size": null,
662 1 : "needed": false
663 1 : },
664 1 : "timeline_id": "f15ae0cf21cce2ba27e4d80c6709a6cd",
665 1 : "kind": "BranchStart"
666 1 : },
667 1 : {
668 1 : "segment": {
669 1 : "parent": 0,
670 1 : "lsn": 305547335776,
671 1 : "size": 220054675456,
672 1 : "needed": false
673 1 : },
674 1 : "timeline_id": "f15ae0cf21cce2ba27e4d80c6709a6cd",
675 1 : "kind": "GcCutOff"
676 1 : },
677 1 : {
678 1 : "segment": {
679 1 : "parent": 1,
680 1 : "lsn": 305614444640,
681 1 : "size": null,
682 1 : "needed": true
683 1 : },
684 1 : "timeline_id": "f15ae0cf21cce2ba27e4d80c6709a6cd",
685 1 : "kind": "BranchEnd"
686 1 : }
687 1 : ],
688 1 : "timeline_inputs": [
689 1 : {
690 1 : "timeline_id": "f15ae0cf21cce2ba27e4d80c6709a6cd",
691 1 : "ancestor_lsn": "0/0",
692 1 : "last_record": "47/280A5860",
693 1 : "latest_gc_cutoff": "47/240A5860",
694 1 : "horizon_cutoff": "47/240A5860",
695 1 : "pitr_cutoff": "47/240A5860",
696 1 : "next_gc_cutoff": "47/240A5860",
697 1 : "retention_param_cutoff": "0/0"
698 1 : }
699 1 : ]
700 1 : }"#;
701 1 :
702 1 : let model: ModelInputs = serde_json::from_str(doc).unwrap();
703 1 :
704 1 : let res = model.calculate_model().unwrap().calculate();
705 1 :
706 1 : println!("calculated synthetic size: {}", res.total_size);
707 1 : println!("result: {:?}", serde_json::to_string(&res.segments));
708 1 :
709 1 : use utils::lsn::Lsn;
710 1 : let latest_gc_cutoff_lsn: Lsn = "47/240A5860".parse().unwrap();
711 1 : let last_lsn: Lsn = "47/280A5860".parse().unwrap();
712 1 : println!(
713 1 : "latest_gc_cutoff lsn 47/240A5860 is {}, last_lsn lsn 47/280A5860 is {}",
714 1 : u64::from(latest_gc_cutoff_lsn),
715 1 : u64::from(last_lsn)
716 1 : );
717 1 : assert_eq!(res.total_size, 220121784320);
718 1 : }
|
