Line data Source code
1 : use std::cmp;
2 : use std::collections::hash_map::Entry;
3 : use std::collections::{HashMap, HashSet};
4 : use std::sync::Arc;
5 :
6 : use anyhow::{bail, Context};
7 : use tokio::sync::oneshot::error::RecvError;
8 : use tokio::sync::Semaphore;
9 : use tokio_util::sync::CancellationToken;
10 :
11 : use crate::context::RequestContext;
12 : use crate::pgdatadir_mapping::CalculateLogicalSizeError;
13 :
14 : use super::{LogicalSizeCalculationCause, Tenant};
15 : use crate::tenant::Timeline;
16 : use utils::id::TimelineId;
17 : use utils::lsn::Lsn;
18 :
19 : use tracing::*;
20 :
21 : use tenant_size_model::{Segment, StorageModel};
22 :
23 : /// Inputs to the actual tenant sizing model
24 : ///
25 : /// Implements [`serde::Serialize`] but is not meant to be part of the public API, instead meant to
26 : /// be a transferrable format between execution environments and developer.
27 : ///
28 : /// This tracks more information than the actual StorageModel that calculation
29 : /// needs. We will convert this into a StorageModel when it's time to perform
30 : /// the calculation.
31 : ///
32 12 : #[derive(Debug, serde::Serialize, serde::Deserialize)]
33 : pub struct ModelInputs {
34 : pub segments: Vec<SegmentMeta>,
35 : pub timeline_inputs: Vec<TimelineInputs>,
36 : }
37 :
38 : /// A [`Segment`], with some extra information for display purposes
39 112 : #[derive(Debug, serde::Serialize, serde::Deserialize)]
40 : pub struct SegmentMeta {
41 : pub segment: Segment,
42 : pub timeline_id: TimelineId,
43 : pub kind: LsnKind,
44 : }
45 :
46 : impl SegmentMeta {
47 0 : fn size_needed(&self) -> bool {
48 0 : match self.kind {
49 : LsnKind::BranchStart => {
50 : // If we don't have a later GcCutoff point on this branch, and
51 : // no ancestor, calculate size for the branch start point.
52 0 : self.segment.needed && self.segment.parent.is_none()
53 : }
54 0 : LsnKind::BranchPoint => true,
55 0 : LsnKind::GcCutOff => true,
56 0 : LsnKind::BranchEnd => false,
57 : }
58 0 : }
59 : }
60 :
61 : #[derive(
62 56 : Debug, Clone, Copy, Eq, Ord, PartialEq, PartialOrd, serde::Serialize, serde::Deserialize,
63 : )]
64 : pub enum LsnKind {
65 : /// A timeline starting here
66 : BranchStart,
67 : /// A child timeline branches off from here
68 : BranchPoint,
69 : /// GC cutoff point
70 : GcCutOff,
71 : /// Last record LSN
72 : BranchEnd,
73 : }
74 :
75 : /// Collect all relevant LSNs to the inputs. These will only be helpful in the serialized form as
76 : /// part of [`ModelInputs`] from the HTTP api, explaining the inputs.
77 72 : #[derive(Debug, serde::Serialize, serde::Deserialize)]
78 : pub struct TimelineInputs {
79 : pub timeline_id: TimelineId,
80 :
81 : pub ancestor_id: Option<TimelineId>,
82 :
83 : ancestor_lsn: Lsn,
84 : last_record: Lsn,
85 : latest_gc_cutoff: Lsn,
86 : horizon_cutoff: Lsn,
87 : pitr_cutoff: Lsn,
88 :
89 : /// Cutoff point based on GC settings
90 : next_gc_cutoff: Lsn,
91 :
92 : /// Cutoff point calculated from the user-supplied 'max_retention_period'
93 : retention_param_cutoff: Option<Lsn>,
94 : }
95 :
96 : /// Gathers the inputs for the tenant sizing model.
97 : ///
98 : /// Tenant size does not consider the latest state, but only the state until next_gc_cutoff, which
99 : /// is updated on-demand, during the start of this calculation and separate from the
100 : /// [`TimelineInputs::latest_gc_cutoff`].
101 : ///
102 : /// For timelines in general:
103 : ///
104 : /// ```text
105 : /// 0-----|---------|----|------------| · · · · · |·> lsn
106 : /// initdb_lsn branchpoints* next_gc_cutoff latest
107 : /// ```
108 : ///
109 : /// Until gc_horizon_cutoff > `Timeline::last_record_lsn` for any of the tenant's timelines, the
110 : /// tenant size will be zero.
111 0 : pub(super) async fn gather_inputs(
112 0 : tenant: &Tenant,
113 0 : limit: &Arc<Semaphore>,
114 0 : max_retention_period: Option<u64>,
115 0 : logical_size_cache: &mut HashMap<(TimelineId, Lsn), u64>,
116 0 : cause: LogicalSizeCalculationCause,
117 0 : cancel: &CancellationToken,
118 0 : ctx: &RequestContext,
119 0 : ) -> anyhow::Result<ModelInputs> {
120 0 : // refresh is needed to update gc related pitr_cutoff and horizon_cutoff
121 0 : tenant
122 0 : .refresh_gc_info(cancel, ctx)
123 0 : .await
124 0 : .context("Failed to refresh gc_info before gathering inputs")?;
125 :
126 : // Collect information about all the timelines
127 0 : let mut timelines = tenant.list_timelines();
128 0 :
129 0 : if timelines.is_empty() {
130 : // perhaps the tenant has just been created, and as such doesn't have any data yet
131 0 : return Ok(ModelInputs {
132 0 : segments: vec![],
133 0 : timeline_inputs: Vec::new(),
134 0 : });
135 0 : }
136 0 :
137 0 : // Filter out timelines that are not active
138 0 : //
139 0 : // There may be a race when a timeline is dropped,
140 0 : // but it is unlikely to cause any issues. In the worst case,
141 0 : // the calculation will error out.
142 0 : timelines.retain(|t| t.is_active());
143 0 :
144 0 : // Build a map of branch points.
145 0 : let mut branchpoints: HashMap<TimelineId, HashSet<Lsn>> = HashMap::new();
146 0 : for timeline in timelines.iter() {
147 0 : if let Some(ancestor_id) = timeline.get_ancestor_timeline_id() {
148 0 : branchpoints
149 0 : .entry(ancestor_id)
150 0 : .or_default()
151 0 : .insert(timeline.get_ancestor_lsn());
152 0 : }
153 : }
154 :
155 : // These become the final result.
156 0 : let mut timeline_inputs = Vec::with_capacity(timelines.len());
157 0 : let mut segments: Vec<SegmentMeta> = Vec::new();
158 0 :
159 0 : //
160 0 : // Build Segments representing each timeline. As we do that, also remember
161 0 : // the branchpoints and branch startpoints in 'branchpoint_segments' and
162 0 : // 'branchstart_segments'
163 0 : //
164 0 :
165 0 : // BranchPoint segments of each timeline
166 0 : // (timeline, branchpoint LSN) -> segment_id
167 0 : let mut branchpoint_segments: HashMap<(TimelineId, Lsn), usize> = HashMap::new();
168 0 :
169 0 : // timeline, Branchpoint seg id, (ancestor, ancestor LSN)
170 0 : type BranchStartSegment = (TimelineId, usize, Option<(TimelineId, Lsn)>);
171 0 : let mut branchstart_segments: Vec<BranchStartSegment> = Vec::new();
172 :
173 0 : for timeline in timelines.iter() {
174 0 : let timeline_id = timeline.timeline_id;
175 0 : let last_record_lsn = timeline.get_last_record_lsn();
176 0 : let ancestor_lsn = timeline.get_ancestor_lsn();
177 0 :
178 0 : // there's a race between the update (holding tenant.gc_lock) and this read but it
179 0 : // might not be an issue, because it's not for Timeline::gc
180 0 : let gc_info = timeline.gc_info.read().unwrap();
181 0 :
182 0 : // similar to gc, but Timeline::get_latest_gc_cutoff_lsn() will not be updated before a
183 0 : // new gc run, which we have no control over. however differently from `Timeline::gc`
184 0 : // we don't consider the `Timeline::disk_consistent_lsn` at all, because we are not
185 0 : // actually removing files.
186 0 : //
187 0 : // We only consider [`GcInfo::pitr_cutoff`], and not [`GcInfo::horizon_cutoff`], because from
188 0 : // a user's perspective they have only requested retention up to the time bound (pitr_cutoff), rather
189 0 : // than a space bound (horizon cutoff). This means that if someone drops a database and waits for their
190 0 : // PITR interval, they will see synthetic size decrease, even if we are still storing data inside
191 0 : // horizon_cutoff.
192 0 : let pitr_cutoff = gc_info.cutoffs.pitr;
193 0 : let horizon_cutoff = gc_info.cutoffs.horizon;
194 0 : let mut next_gc_cutoff = pitr_cutoff;
195 :
196 : // If the caller provided a shorter retention period, use that instead of the GC cutoff.
197 0 : let retention_param_cutoff = if let Some(max_retention_period) = max_retention_period {
198 0 : let param_cutoff = Lsn(last_record_lsn.0.saturating_sub(max_retention_period));
199 0 : if next_gc_cutoff < param_cutoff {
200 0 : next_gc_cutoff = param_cutoff;
201 0 : }
202 0 : Some(param_cutoff)
203 : } else {
204 0 : None
205 : };
206 :
207 : // next_gc_cutoff in parent branch are not of interest (right now at least), nor do we
208 : // want to query any logical size before initdb_lsn.
209 0 : let branch_start_lsn = cmp::max(ancestor_lsn, timeline.initdb_lsn);
210 0 :
211 0 : // Build "interesting LSNs" on this timeline
212 0 : let mut lsns: Vec<(Lsn, LsnKind)> = gc_info
213 0 : .retain_lsns
214 0 : .iter()
215 0 : .filter(|&&lsn| lsn > ancestor_lsn)
216 0 : .copied()
217 0 : // this assumes there are no other retain_lsns than the branchpoints
218 0 : .map(|lsn| (lsn, LsnKind::BranchPoint))
219 0 : .collect::<Vec<_>>();
220 0 :
221 0 : drop(gc_info);
222 :
223 : // Add branch points we collected earlier, just in case there were any that were
224 : // not present in retain_lsns. We will remove any duplicates below later.
225 0 : if let Some(this_branchpoints) = branchpoints.get(&timeline_id) {
226 0 : lsns.extend(
227 0 : this_branchpoints
228 0 : .iter()
229 0 : .map(|lsn| (*lsn, LsnKind::BranchPoint)),
230 0 : )
231 0 : }
232 :
233 : // Add a point for the GC cutoff
234 0 : let branch_start_needed = next_gc_cutoff <= branch_start_lsn;
235 0 : if !branch_start_needed {
236 0 : lsns.push((next_gc_cutoff, LsnKind::GcCutOff));
237 0 : }
238 :
239 0 : lsns.sort_unstable();
240 0 : lsns.dedup();
241 0 :
242 0 : //
243 0 : // Create Segments for the interesting points.
244 0 : //
245 0 :
246 0 : // Timeline start point
247 0 : let ancestor = timeline
248 0 : .get_ancestor_timeline_id()
249 0 : .map(|ancestor_id| (ancestor_id, ancestor_lsn));
250 0 : branchstart_segments.push((timeline_id, segments.len(), ancestor));
251 0 : segments.push(SegmentMeta {
252 0 : segment: Segment {
253 0 : parent: None, // filled in later
254 0 : lsn: branch_start_lsn.0,
255 0 : size: None, // filled in later
256 0 : needed: branch_start_needed,
257 0 : },
258 0 : timeline_id: timeline.timeline_id,
259 0 : kind: LsnKind::BranchStart,
260 0 : });
261 0 :
262 0 : // GC cutoff point, and any branch points, i.e. points where
263 0 : // other timelines branch off from this timeline.
264 0 : let mut parent = segments.len() - 1;
265 0 : for (lsn, kind) in lsns {
266 0 : if kind == LsnKind::BranchPoint {
267 0 : branchpoint_segments.insert((timeline_id, lsn), segments.len());
268 0 : }
269 0 : segments.push(SegmentMeta {
270 0 : segment: Segment {
271 0 : parent: Some(parent),
272 0 : lsn: lsn.0,
273 0 : size: None,
274 0 : needed: lsn > next_gc_cutoff,
275 0 : },
276 0 : timeline_id: timeline.timeline_id,
277 0 : kind,
278 0 : });
279 0 : parent += 1;
280 : }
281 :
282 : // Current end of the timeline
283 0 : segments.push(SegmentMeta {
284 0 : segment: Segment {
285 0 : parent: Some(parent),
286 0 : lsn: last_record_lsn.0,
287 0 : size: None, // Filled in later, if necessary
288 0 : needed: true,
289 0 : },
290 0 : timeline_id: timeline.timeline_id,
291 0 : kind: LsnKind::BranchEnd,
292 0 : });
293 0 :
294 0 : timeline_inputs.push(TimelineInputs {
295 0 : timeline_id: timeline.timeline_id,
296 0 : ancestor_id: timeline.get_ancestor_timeline_id(),
297 0 : ancestor_lsn,
298 0 : last_record: last_record_lsn,
299 0 : // this is not used above, because it might not have updated recently enough
300 0 : latest_gc_cutoff: *timeline.get_latest_gc_cutoff_lsn(),
301 0 : horizon_cutoff,
302 0 : pitr_cutoff,
303 0 : next_gc_cutoff,
304 0 : retention_param_cutoff,
305 0 : });
306 : }
307 :
308 : // We now have all segments from the timelines in 'segments'. The timelines
309 : // haven't been linked to each other yet, though. Do that.
310 0 : for (_timeline_id, seg_id, ancestor) in branchstart_segments {
311 : // Look up the branch point
312 0 : if let Some(ancestor) = ancestor {
313 0 : let parent_id = *branchpoint_segments.get(&ancestor).unwrap();
314 0 : segments[seg_id].segment.parent = Some(parent_id);
315 0 : }
316 : }
317 :
318 : // We left the 'size' field empty in all of the Segments so far.
319 : // Now find logical sizes for all of the points that might need or benefit from them.
320 0 : fill_logical_sizes(
321 0 : &timelines,
322 0 : &mut segments,
323 0 : limit,
324 0 : logical_size_cache,
325 0 : cause,
326 0 : ctx,
327 0 : )
328 0 : .await?;
329 :
330 0 : Ok(ModelInputs {
331 0 : segments,
332 0 : timeline_inputs,
333 0 : })
334 0 : }
335 :
336 : /// Augment 'segments' with logical sizes
337 : ///
338 : /// this will probably conflict with on-demand downloaded layers, or at least force them all
339 : /// to be downloaded
340 : ///
341 0 : async fn fill_logical_sizes(
342 0 : timelines: &[Arc<Timeline>],
343 0 : segments: &mut [SegmentMeta],
344 0 : limit: &Arc<Semaphore>,
345 0 : logical_size_cache: &mut HashMap<(TimelineId, Lsn), u64>,
346 0 : cause: LogicalSizeCalculationCause,
347 0 : ctx: &RequestContext,
348 0 : ) -> anyhow::Result<()> {
349 0 : let timeline_hash: HashMap<TimelineId, Arc<Timeline>> = HashMap::from_iter(
350 0 : timelines
351 0 : .iter()
352 0 : .map(|timeline| (timeline.timeline_id, Arc::clone(timeline))),
353 0 : );
354 0 :
355 0 : // record the used/inserted cache keys here, to remove extras not to start leaking
356 0 : // after initial run the cache should be quite stable, but live timelines will eventually
357 0 : // require new lsns to be inspected.
358 0 : let mut sizes_needed = HashMap::<(TimelineId, Lsn), Option<u64>>::new();
359 0 :
360 0 : // with joinset, on drop, all of the tasks will just be de-scheduled, which we can use to
361 0 : // our advantage with `?` error handling.
362 0 : let mut joinset = tokio::task::JoinSet::new();
363 :
364 : // For each point that would benefit from having a logical size available,
365 : // spawn a Task to fetch it, unless we have it cached already.
366 0 : for seg in segments.iter() {
367 0 : if !seg.size_needed() {
368 0 : continue;
369 0 : }
370 0 :
371 0 : let timeline_id = seg.timeline_id;
372 0 : let lsn = Lsn(seg.segment.lsn);
373 :
374 0 : if let Entry::Vacant(e) = sizes_needed.entry((timeline_id, lsn)) {
375 0 : let cached_size = logical_size_cache.get(&(timeline_id, lsn)).cloned();
376 0 : if cached_size.is_none() {
377 0 : let timeline = Arc::clone(timeline_hash.get(&timeline_id).unwrap());
378 0 : let parallel_size_calcs = Arc::clone(limit);
379 0 : let ctx = ctx.attached_child();
380 0 : joinset.spawn(
381 0 : calculate_logical_size(parallel_size_calcs, timeline, lsn, cause, ctx)
382 0 : .in_current_span(),
383 0 : );
384 0 : }
385 0 : e.insert(cached_size);
386 0 : }
387 : }
388 :
389 : // Perform the size lookups
390 0 : let mut have_any_error = false;
391 0 : while let Some(res) = joinset.join_next().await {
392 : // each of these come with Result<anyhow::Result<_>, JoinError>
393 : // because of spawn + spawn_blocking
394 0 : match res {
395 0 : Err(join_error) if join_error.is_cancelled() => {
396 0 : unreachable!("we are not cancelling any of the futures, nor should be");
397 : }
398 0 : Err(join_error) => {
399 0 : // cannot really do anything, as this panic is likely a bug
400 0 : error!("task that calls spawn_ondemand_logical_size_calculation panicked: {join_error:#}");
401 0 : have_any_error = true;
402 : }
403 0 : Ok(Err(recv_result_error)) => {
404 0 : // cannot really do anything, as this panic is likely a bug
405 0 : error!("failed to receive logical size query result: {recv_result_error:#}");
406 0 : have_any_error = true;
407 : }
408 0 : Ok(Ok(TimelineAtLsnSizeResult(timeline, lsn, Err(error)))) => {
409 0 : if !matches!(error, CalculateLogicalSizeError::Cancelled) {
410 0 : warn!(
411 0 : timeline_id=%timeline.timeline_id,
412 0 : "failed to calculate logical size at {lsn}: {error:#}"
413 : );
414 0 : }
415 0 : have_any_error = true;
416 : }
417 0 : Ok(Ok(TimelineAtLsnSizeResult(timeline, lsn, Ok(size)))) => {
418 0 : debug!(timeline_id=%timeline.timeline_id, %lsn, size, "size calculated");
419 :
420 0 : logical_size_cache.insert((timeline.timeline_id, lsn), size);
421 0 : sizes_needed.insert((timeline.timeline_id, lsn), Some(size));
422 : }
423 : }
424 : }
425 :
426 : // prune any keys not needed anymore; we record every used key and added key.
427 0 : logical_size_cache.retain(|key, _| sizes_needed.contains_key(key));
428 0 :
429 0 : if have_any_error {
430 : // we cannot complete this round, because we are missing data.
431 : // we have however cached all we were able to request calculation on.
432 0 : anyhow::bail!("failed to calculate some logical_sizes");
433 0 : }
434 :
435 : // Insert the looked up sizes to the Segments
436 0 : for seg in segments.iter_mut() {
437 0 : if !seg.size_needed() {
438 0 : continue;
439 0 : }
440 0 :
441 0 : let timeline_id = seg.timeline_id;
442 0 : let lsn = Lsn(seg.segment.lsn);
443 :
444 0 : if let Some(Some(size)) = sizes_needed.get(&(timeline_id, lsn)) {
445 0 : seg.segment.size = Some(*size);
446 0 : } else {
447 0 : bail!("could not find size at {} in timeline {}", lsn, timeline_id);
448 : }
449 : }
450 0 : Ok(())
451 0 : }
452 :
453 : impl ModelInputs {
454 4 : pub fn calculate_model(&self) -> anyhow::Result<tenant_size_model::StorageModel> {
455 4 : // Convert SegmentMetas into plain Segments
456 4 : let storage = StorageModel {
457 4 : segments: self
458 4 : .segments
459 4 : .iter()
460 28 : .map(|seg| seg.segment.clone())
461 4 : .collect(),
462 4 : };
463 4 :
464 4 : Ok(storage)
465 4 : }
466 :
467 : // calculate total project size
468 2 : pub fn calculate(&self) -> anyhow::Result<u64> {
469 2 : let storage = self.calculate_model()?;
470 2 : let sizes = storage.calculate();
471 2 :
472 2 : Ok(sizes.total_size)
473 2 : }
474 : }
475 :
476 : /// Newtype around the tuple that carries the timeline at lsn logical size calculation.
477 : struct TimelineAtLsnSizeResult(
478 : Arc<crate::tenant::Timeline>,
479 : utils::lsn::Lsn,
480 : Result<u64, CalculateLogicalSizeError>,
481 : );
482 :
483 0 : #[instrument(skip_all, fields(timeline_id=%timeline.timeline_id, lsn=%lsn))]
484 : async fn calculate_logical_size(
485 : limit: Arc<tokio::sync::Semaphore>,
486 : timeline: Arc<crate::tenant::Timeline>,
487 : lsn: utils::lsn::Lsn,
488 : cause: LogicalSizeCalculationCause,
489 : ctx: RequestContext,
490 : ) -> Result<TimelineAtLsnSizeResult, RecvError> {
491 : let _permit = tokio::sync::Semaphore::acquire_owned(limit)
492 : .await
493 : .expect("global semaphore should not had been closed");
494 :
495 : let size_res = timeline
496 : .spawn_ondemand_logical_size_calculation(lsn, cause, ctx)
497 : .instrument(info_span!("spawn_ondemand_logical_size_calculation"))
498 : .await?;
499 : Ok(TimelineAtLsnSizeResult(timeline, lsn, size_res))
500 : }
501 :
502 : #[test]
503 2 : fn verify_size_for_multiple_branches() {
504 2 : // this is generated from integration test test_tenant_size_with_multiple_branches, but this way
505 2 : // it has the stable lsn's
506 2 : //
507 2 : // The timeline_inputs don't participate in the size calculation, and are here just to explain
508 2 : // the inputs.
509 2 : let doc = r#"
510 2 : {
511 2 : "segments": [
512 2 : {
513 2 : "segment": {
514 2 : "parent": 9,
515 2 : "lsn": 26033560,
516 2 : "size": null,
517 2 : "needed": false
518 2 : },
519 2 : "timeline_id": "20b129c9b50cff7213e6503a31b2a5ce",
520 2 : "kind": "BranchStart"
521 2 : },
522 2 : {
523 2 : "segment": {
524 2 : "parent": 0,
525 2 : "lsn": 35720400,
526 2 : "size": 25206784,
527 2 : "needed": false
528 2 : },
529 2 : "timeline_id": "20b129c9b50cff7213e6503a31b2a5ce",
530 2 : "kind": "GcCutOff"
531 2 : },
532 2 : {
533 2 : "segment": {
534 2 : "parent": 1,
535 2 : "lsn": 35851472,
536 2 : "size": null,
537 2 : "needed": true
538 2 : },
539 2 : "timeline_id": "20b129c9b50cff7213e6503a31b2a5ce",
540 2 : "kind": "BranchEnd"
541 2 : },
542 2 : {
543 2 : "segment": {
544 2 : "parent": 7,
545 2 : "lsn": 24566168,
546 2 : "size": null,
547 2 : "needed": false
548 2 : },
549 2 : "timeline_id": "454626700469f0a9914949b9d018e876",
550 2 : "kind": "BranchStart"
551 2 : },
552 2 : {
553 2 : "segment": {
554 2 : "parent": 3,
555 2 : "lsn": 25261936,
556 2 : "size": 26050560,
557 2 : "needed": false
558 2 : },
559 2 : "timeline_id": "454626700469f0a9914949b9d018e876",
560 2 : "kind": "GcCutOff"
561 2 : },
562 2 : {
563 2 : "segment": {
564 2 : "parent": 4,
565 2 : "lsn": 25393008,
566 2 : "size": null,
567 2 : "needed": true
568 2 : },
569 2 : "timeline_id": "454626700469f0a9914949b9d018e876",
570 2 : "kind": "BranchEnd"
571 2 : },
572 2 : {
573 2 : "segment": {
574 2 : "parent": null,
575 2 : "lsn": 23694408,
576 2 : "size": null,
577 2 : "needed": false
578 2 : },
579 2 : "timeline_id": "cb5e3cbe60a4afc00d01880e1a37047f",
580 2 : "kind": "BranchStart"
581 2 : },
582 2 : {
583 2 : "segment": {
584 2 : "parent": 6,
585 2 : "lsn": 24566168,
586 2 : "size": 25739264,
587 2 : "needed": false
588 2 : },
589 2 : "timeline_id": "cb5e3cbe60a4afc00d01880e1a37047f",
590 2 : "kind": "BranchPoint"
591 2 : },
592 2 : {
593 2 : "segment": {
594 2 : "parent": 7,
595 2 : "lsn": 25902488,
596 2 : "size": 26402816,
597 2 : "needed": false
598 2 : },
599 2 : "timeline_id": "cb5e3cbe60a4afc00d01880e1a37047f",
600 2 : "kind": "GcCutOff"
601 2 : },
602 2 : {
603 2 : "segment": {
604 2 : "parent": 8,
605 2 : "lsn": 26033560,
606 2 : "size": 26468352,
607 2 : "needed": true
608 2 : },
609 2 : "timeline_id": "cb5e3cbe60a4afc00d01880e1a37047f",
610 2 : "kind": "BranchPoint"
611 2 : },
612 2 : {
613 2 : "segment": {
614 2 : "parent": 9,
615 2 : "lsn": 26033560,
616 2 : "size": null,
617 2 : "needed": true
618 2 : },
619 2 : "timeline_id": "cb5e3cbe60a4afc00d01880e1a37047f",
620 2 : "kind": "BranchEnd"
621 2 : }
622 2 : ],
623 2 : "timeline_inputs": [
624 2 : {
625 2 : "timeline_id": "20b129c9b50cff7213e6503a31b2a5ce",
626 2 : "ancestor_lsn": "0/18D3D98",
627 2 : "last_record": "0/2230CD0",
628 2 : "latest_gc_cutoff": "0/1698C48",
629 2 : "horizon_cutoff": "0/2210CD0",
630 2 : "pitr_cutoff": "0/2210CD0",
631 2 : "next_gc_cutoff": "0/2210CD0",
632 2 : "retention_param_cutoff": null
633 2 : },
634 2 : {
635 2 : "timeline_id": "454626700469f0a9914949b9d018e876",
636 2 : "ancestor_lsn": "0/176D998",
637 2 : "last_record": "0/1837770",
638 2 : "latest_gc_cutoff": "0/1698C48",
639 2 : "horizon_cutoff": "0/1817770",
640 2 : "pitr_cutoff": "0/1817770",
641 2 : "next_gc_cutoff": "0/1817770",
642 2 : "retention_param_cutoff": null
643 2 : },
644 2 : {
645 2 : "timeline_id": "cb5e3cbe60a4afc00d01880e1a37047f",
646 2 : "ancestor_lsn": "0/0",
647 2 : "last_record": "0/18D3D98",
648 2 : "latest_gc_cutoff": "0/1698C48",
649 2 : "horizon_cutoff": "0/18B3D98",
650 2 : "pitr_cutoff": "0/18B3D98",
651 2 : "next_gc_cutoff": "0/18B3D98",
652 2 : "retention_param_cutoff": null
653 2 : }
654 2 : ]
655 2 : }
656 2 : "#;
657 2 : let inputs: ModelInputs = serde_json::from_str(doc).unwrap();
658 2 :
659 2 : assert_eq!(inputs.calculate().unwrap(), 37_851_408);
660 2 : }
661 :
662 : #[test]
663 2 : fn verify_size_for_one_branch() {
664 2 : let doc = r#"
665 2 : {
666 2 : "segments": [
667 2 : {
668 2 : "segment": {
669 2 : "parent": null,
670 2 : "lsn": 0,
671 2 : "size": null,
672 2 : "needed": false
673 2 : },
674 2 : "timeline_id": "f15ae0cf21cce2ba27e4d80c6709a6cd",
675 2 : "kind": "BranchStart"
676 2 : },
677 2 : {
678 2 : "segment": {
679 2 : "parent": 0,
680 2 : "lsn": 305547335776,
681 2 : "size": 220054675456,
682 2 : "needed": false
683 2 : },
684 2 : "timeline_id": "f15ae0cf21cce2ba27e4d80c6709a6cd",
685 2 : "kind": "GcCutOff"
686 2 : },
687 2 : {
688 2 : "segment": {
689 2 : "parent": 1,
690 2 : "lsn": 305614444640,
691 2 : "size": null,
692 2 : "needed": true
693 2 : },
694 2 : "timeline_id": "f15ae0cf21cce2ba27e4d80c6709a6cd",
695 2 : "kind": "BranchEnd"
696 2 : }
697 2 : ],
698 2 : "timeline_inputs": [
699 2 : {
700 2 : "timeline_id": "f15ae0cf21cce2ba27e4d80c6709a6cd",
701 2 : "ancestor_lsn": "0/0",
702 2 : "last_record": "47/280A5860",
703 2 : "latest_gc_cutoff": "47/240A5860",
704 2 : "horizon_cutoff": "47/240A5860",
705 2 : "pitr_cutoff": "47/240A5860",
706 2 : "next_gc_cutoff": "47/240A5860",
707 2 : "retention_param_cutoff": "0/0"
708 2 : }
709 2 : ]
710 2 : }"#;
711 2 :
712 2 : let model: ModelInputs = serde_json::from_str(doc).unwrap();
713 2 :
714 2 : let res = model.calculate_model().unwrap().calculate();
715 2 :
716 2 : println!("calculated synthetic size: {}", res.total_size);
717 2 : println!("result: {:?}", serde_json::to_string(&res.segments));
718 2 :
719 2 : use utils::lsn::Lsn;
720 2 : let latest_gc_cutoff_lsn: Lsn = "47/240A5860".parse().unwrap();
721 2 : let last_lsn: Lsn = "47/280A5860".parse().unwrap();
722 2 : println!(
723 2 : "latest_gc_cutoff lsn 47/240A5860 is {}, last_lsn lsn 47/280A5860 is {}",
724 2 : u64::from(latest_gc_cutoff_lsn),
725 2 : u64::from(last_lsn)
726 2 : );
727 2 : assert_eq!(res.total_size, 220121784320);
728 2 : }
|
