Line data Source code
1 : //! Defines [`RequestContext`].
2 : //!
3 : //! It is a structure that we use throughout the pageserver to propagate
4 : //! high-level context from places that _originate_ activity down to the
5 : //! shared code paths at the heart of the pageserver. It's inspired by
6 : //! Golang's `context.Context`.
7 : //!
8 : //! For example, in `Timeline::get(page_nr, lsn)` we need to answer the following questions:
9 : //! 1. What high-level activity ([`TaskKind`]) needs this page?
10 : //! We need that information as a categorical dimension for page access
11 : //! statistics, which we, in turn, need to guide layer eviction policy design.
12 : //! 2. How should we behave if, to produce the page image, we need to
13 : //! on-demand download a layer file ([`DownloadBehavior`]).
14 : //!
15 : //! [`RequestContext`] satisfies those needs.
16 : //! The current implementation is a small `struct` that is passed through
17 : //! the call chain by reference.
18 : //!
19 : //! ### Future Work
20 : //!
21 : //! However, we do not intend to stop here, since there are other needs that
22 : //! require carrying information from high to low levels of the app.
23 : //!
24 : //! Most importantly, **cancellation signaling** in response to
25 : //! 1. timeouts (page_service max response time) and
26 : //! 2. lifecycle requests (detach tenant, delete timeline).
27 : //!
28 : //! Related to that, there is sometimes a need to ensure that all tokio tasks spawned
29 : //! by the transitive callees of a request have finished. The keyword here
30 : //! is **Structured Concurrency**, and right now, we use `task_mgr` in most places,
31 : //! `TaskHandle` in some places, and careful code review around `FuturesUnordered`
32 : //! or `JoinSet` in other places.
33 : //!
34 : //! We do not yet have a systematic cancellation story in pageserver, and it is
35 : //! pretty clear that [`RequestContext`] will be responsible for that.
36 : //! So, the API already prepares for this role through the
37 : //! [`RequestContext::detached_child`] and [`RequestContext::attached_child`] methods.
38 : //! See their doc comments for details on how we will use them in the future.
39 : //!
40 : //! It is not clear whether or how we will enforce Structured Concurrency, and
41 : //! what role [`RequestContext`] will play there.
42 : //! So, the API doesn't prepare us for this topic.
43 : //!
44 : //! Other future uses of `RequestContext`:
45 : //! - Communicate compute & IO priorities (user-initiated request vs. background-loop)
46 : //! - Request IDs for distributed tracing
47 : //! - Request/Timeline/Tenant-scoped log levels
48 : //!
49 : //! RequestContext might look quite different once it supports those features.
50 : //! Likely, it will have a shape similar to Golang's `context.Context`.
51 : //!
52 : //! ### Why A Struct Instead Of Method Parameters
53 : //!
54 : //! What's typical about such information is that it needs to be passed down
55 : //! along the call chain from high level to low level, but few of the functions
56 : //! in the middle need to understand it.
57 : //! Further, it is to be expected that we will need to propagate more data
58 : //! in the future (see the earlier section on future work).
59 : //! Hence, for functions in the middle of the call chain, we have the following
60 : //! requirements:
61 : //! 1. It should be easy to forward the context to callees.
62 : //! 2. To propagate more data from high-level to low-level code, the functions in
63 : //! the middle should not need to be modified.
64 : //!
65 : //! The solution is to have a container structure ([`RequestContext`]) that
66 : //! carries the information. Functions that don't care about what's in it
67 : //! pass it along to callees.
68 : //!
69 : //! ### Why Not Task-Local Variables
70 : //!
71 : //! One could use task-local variables (the equivalent of thread-local variables)
72 : //! to address the immediate needs outlined above.
73 : //! However, we reject task-local variables because:
74 : //! 1. they are implicit, thereby making it harder to trace the data flow in code
75 : //! reviews and during debugging,
76 : //! 2. they can be mutable, which enables implicit return data flow,
77 : //! 3. they are restrictive in that code which fans out into multiple tasks,
78 : //! or even threads, needs to carefully propagate the state.
79 : //!
80 : //! In contrast, information flow with [`RequestContext`] is
81 : //! 1. always explicit,
82 : //! 2. strictly uni-directional because RequestContext is immutable,
83 : //! 3. tangible because a [`RequestContext`] is just a value.
84 : //! When creating child activities, regardless of whether it's a task,
85 : //! thread, or even an RPC to another service, the value can
86 : //! be used like any other argument.
87 : //!
88 : //! The solution is that all code paths are infected with precisely one
89 : //! [`RequestContext`] argument. Functions in the middle of the call chain
90 : //! only need to pass it on.
91 :
92 : use std::sync::Arc;
93 :
94 : use once_cell::sync::Lazy;
95 : use tracing::warn;
96 : use utils::{id::TimelineId, shard::TenantShardId};
97 :
98 : use crate::{
99 : metrics::{StorageIoSizeMetrics, TimelineMetrics},
100 : task_mgr::TaskKind,
101 : tenant::Timeline,
102 : };
103 :
104 : // The main structure of this module, see module-level comment.
105 : pub struct RequestContext {
106 : task_kind: TaskKind,
107 : download_behavior: DownloadBehavior,
108 : access_stats_behavior: AccessStatsBehavior,
109 : page_content_kind: PageContentKind,
110 : read_path_debug: bool,
111 : scope: Scope,
112 : }
113 :
114 : #[derive(Clone)]
115 : pub(crate) enum Scope {
116 : Global {
117 : io_size_metrics: &'static crate::metrics::StorageIoSizeMetrics,
118 : },
119 : SecondaryTenant {
120 : io_size_metrics: &'static crate::metrics::StorageIoSizeMetrics,
121 : },
122 : SecondaryTimeline {
123 : io_size_metrics: crate::metrics::StorageIoSizeMetrics,
124 : },
125 : Timeline {
126 : // We wrap the `Arc<TimelineMetrics>`s inside another Arc to avoid child
127 : // context creation contending for the ref counters of the Arc<TimelineMetrics>,
128 : // which are shared among all tasks that operate on the timeline, especially
129 : // concurrent page_service connections.
130 : #[allow(clippy::redundant_allocation)]
131 : arc_arc: Arc<Arc<TimelineMetrics>>,
132 : },
133 : #[cfg(test)]
134 : UnitTest {
135 : io_size_metrics: &'static crate::metrics::StorageIoSizeMetrics,
136 : },
137 : }
138 :
139 : static GLOBAL_IO_SIZE_METRICS: Lazy<crate::metrics::StorageIoSizeMetrics> =
140 548 : Lazy::new(|| crate::metrics::StorageIoSizeMetrics::new("*", "*", "*"));
141 :
142 : impl Scope {
143 2112 : pub(crate) fn new_global() -> Self {
144 2112 : Scope::Global {
145 2112 : io_size_metrics: &GLOBAL_IO_SIZE_METRICS,
146 2112 : }
147 2112 : }
148 : /// NB: this allocates, so, use only at relatively long-lived roots, e.g., at start
149 : /// of a compaction iteration.
150 1836 : pub(crate) fn new_timeline(timeline: &Timeline) -> Self {
151 1836 : Scope::Timeline {
152 1836 : arc_arc: Arc::new(Arc::clone(&timeline.metrics)),
153 1836 : }
154 1836 : }
155 0 : pub(crate) fn new_page_service_pagestream(
156 0 : timeline_handle: &crate::tenant::timeline::handle::Handle<
157 0 : crate::page_service::TenantManagerTypes,
158 0 : >,
159 0 : ) -> Self {
160 0 : Scope::Timeline {
161 0 : arc_arc: Arc::clone(&timeline_handle.metrics),
162 0 : }
163 0 : }
164 0 : pub(crate) fn new_secondary_timeline(
165 0 : tenant_shard_id: &TenantShardId,
166 0 : timeline_id: &TimelineId,
167 0 : ) -> Self {
168 0 : // TODO(https://github.com/neondatabase/neon/issues/11156): secondary timelines have no infrastructure for metrics lifecycle.
169 0 :
170 0 : let tenant_id = tenant_shard_id.tenant_id.to_string();
171 0 : let shard_id = tenant_shard_id.shard_slug().to_string();
172 0 : let timeline_id = timeline_id.to_string();
173 0 :
174 0 : let io_size_metrics =
175 0 : crate::metrics::StorageIoSizeMetrics::new(&tenant_id, &shard_id, &timeline_id);
176 0 : Scope::SecondaryTimeline { io_size_metrics }
177 0 : }
178 0 : pub(crate) fn new_secondary_tenant(_tenant_shard_id: &TenantShardId) -> Self {
179 0 : // Before propagating metrics via RequestContext, the labels were inferred from file path.
180 0 : // The only user of VirtualFile at tenant scope is the heatmap download & read.
181 0 : // The inferred labels for the path of the heatmap file on local disk were that of the global metric (*,*,*).
182 0 : // Thus, we do the same here, and extend that for anything secondary-tenant scoped.
183 0 : //
184 0 : // If we want to have (tenant_id, shard_id, '*') labels for secondary tenants in the future,
185 0 : // we will need to think about the metric lifecycle, i.e., remove them during secondary tenant shutdown,
186 0 : // like we do for attached timelines. (We don't have attached-tenant-scoped usage of VirtualFile
187 0 : // at this point, so, we were able to completely side-step tenant-scoped stuff there).
188 0 : Scope::SecondaryTenant {
189 0 : io_size_metrics: &GLOBAL_IO_SIZE_METRICS,
190 0 : }
191 0 : }
192 : #[cfg(test)]
193 580 : pub(crate) fn new_unit_test() -> Self {
194 580 : Scope::UnitTest {
195 580 : io_size_metrics: &GLOBAL_IO_SIZE_METRICS,
196 580 : }
197 580 : }
198 : }
199 :
200 : /// The kind of access to the page cache.
201 : #[derive(Clone, Copy, PartialEq, Eq, Debug, enum_map::Enum, strum_macros::IntoStaticStr)]
202 : pub enum PageContentKind {
203 : Unknown,
204 : DeltaLayerSummary,
205 : DeltaLayerBtreeNode,
206 : DeltaLayerValue,
207 : ImageLayerSummary,
208 : ImageLayerBtreeNode,
209 : ImageLayerValue,
210 : InMemoryLayer,
211 : }
212 :
213 : /// Desired behavior if the operation requires an on-demand download
214 : /// to proceed.
215 : #[derive(Clone, Copy, PartialEq, Eq, Debug)]
216 : pub enum DownloadBehavior {
217 : /// Download the layer file. It can take a while.
218 : Download,
219 :
220 : /// Download the layer file, but print a warning to the log. This should be used
221 : /// in code where the layer file is expected to already exist locally.
222 : Warn,
223 :
224 : /// Return a PageReconstructError::NeedsDownload error
225 : Error,
226 : }
227 :
228 : /// Whether this request should update access times used in LRU eviction
229 : #[derive(Clone, Copy, PartialEq, Eq, Debug)]
230 : pub(crate) enum AccessStatsBehavior {
231 : /// Update access times: this request's access to data should be taken
232 : /// as a hint that the accessed layer is likely to be accessed again
233 : Update,
234 :
235 : /// Do not update access times: this request is accessing the layer
236 : /// but does not want to indicate that the layer should be retained in cache,
237 : /// perhaps because the requestor is a compaction routine that will soon cover
238 : /// this layer with another.
239 : Skip,
240 : }
241 :
242 : pub struct RequestContextBuilder {
243 : inner: RequestContext,
244 : }
245 :
246 : impl RequestContextBuilder {
247 : /// A new builder with default settings
248 2112 : pub fn new(task_kind: TaskKind) -> Self {
249 2112 : Self {
250 2112 : inner: RequestContext {
251 2112 : task_kind,
252 2112 : download_behavior: DownloadBehavior::Download,
253 2112 : access_stats_behavior: AccessStatsBehavior::Update,
254 2112 : page_content_kind: PageContentKind::Unknown,
255 2112 : read_path_debug: false,
256 2112 : scope: Scope::new_global(),
257 2112 : },
258 2112 : }
259 2112 : }
260 :
261 3230253 : pub fn extend(original: &RequestContext) -> Self {
262 3230253 : Self {
263 3230253 : // This is like a Copy, but avoid implementing Copy because ordinary users of
264 3230253 : // RequestContext should always move or ref it.
265 3230253 : inner: RequestContext {
266 3230253 : task_kind: original.task_kind,
267 3230253 : download_behavior: original.download_behavior,
268 3230253 : access_stats_behavior: original.access_stats_behavior,
269 3230253 : page_content_kind: original.page_content_kind,
270 3230253 : read_path_debug: original.read_path_debug,
271 3230253 : scope: original.scope.clone(),
272 3230253 : },
273 3230253 : }
274 3230253 : }
275 :
276 1531152 : pub fn task_kind(mut self, k: TaskKind) -> Self {
277 1531152 : self.inner.task_kind = k;
278 1531152 : self
279 1531152 : }
280 :
281 : /// Configure the DownloadBehavior of the context: whether to
282 : /// download missing layers, and/or warn on the download.
283 1532708 : pub fn download_behavior(mut self, b: DownloadBehavior) -> Self {
284 1532708 : self.inner.download_behavior = b;
285 1532708 : self
286 1532708 : }
287 :
288 : /// Configure the AccessStatsBehavior of the context: whether layer
289 : /// accesses should update the access time of the layer.
290 728 : pub(crate) fn access_stats_behavior(mut self, b: AccessStatsBehavior) -> Self {
291 728 : self.inner.access_stats_behavior = b;
292 728 : self
293 728 : }
294 :
295 1696513 : pub(crate) fn page_content_kind(mut self, k: PageContentKind) -> Self {
296 1696513 : self.inner.page_content_kind = k;
297 1696513 : self
298 1696513 : }
299 :
300 0 : pub(crate) fn read_path_debug(mut self, b: bool) -> Self {
301 0 : self.inner.read_path_debug = b;
302 0 : self
303 0 : }
304 :
305 2416 : pub(crate) fn scope(mut self, s: Scope) -> Self {
306 2416 : self.inner.scope = s;
307 2416 : self
308 2416 : }
309 :
310 3232365 : pub fn build(self) -> RequestContext {
311 3232365 : self.inner
312 3232365 : }
313 : }
314 :
315 : impl RequestContext {
316 : /// Create a new RequestContext that has no parent.
317 : ///
318 : /// The function is called `new` because, once we add children
319 : /// to it using `detached_child` or `attached_child`, the context
320 : /// form a tree (not implemented yet since cancellation will be
321 : /// the first feature that requires a tree).
322 : ///
323 : /// # Future: Cancellation
324 : ///
325 : /// The only reason why a context like this one can be canceled is
326 : /// because someone explicitly canceled it.
327 : /// It has no parent, so it cannot inherit cancellation from there.
328 1532 : pub fn new(task_kind: TaskKind, download_behavior: DownloadBehavior) -> Self {
329 1532 : RequestContextBuilder::new(task_kind)
330 1532 : .download_behavior(download_behavior)
331 1532 : .build()
332 1532 : }
333 :
334 : /// Create a detached child context for a task that may outlive `self`.
335 : ///
336 : /// Use this when spawning new background activity that should complete
337 : /// even if the current request is canceled.
338 : ///
339 : /// # Future: Cancellation
340 : ///
341 : /// Cancellation of `self` will not propagate to the child context returned
342 : /// by this method.
343 : ///
344 : /// # Future: Structured Concurrency
345 : ///
346 : /// We could add the Future as a parameter to this function, spawn it as a task,
347 : /// and pass to the new task the child context as an argument.
348 : /// That would be an ergonomic improvement.
349 : ///
350 : /// We could make new calls to this function fail if `self` is already canceled.
351 456 : pub fn detached_child(&self, task_kind: TaskKind, download_behavior: DownloadBehavior) -> Self {
352 456 : self.child_impl(task_kind, download_behavior)
353 456 : }
354 :
355 : /// Create a child of context `self` for a task that shall not outlive `self`.
356 : ///
357 : /// Use this when fanning-out work to other async tasks.
358 : ///
359 : /// # Future: Cancellation
360 : ///
361 : /// Cancelling a context will propagate to its attached children.
362 : ///
363 : /// # Future: Structured Concurrency
364 : ///
365 : /// We could add the Future as a parameter to this function, spawn it as a task,
366 : /// and track its `JoinHandle` inside the `RequestContext`.
367 : ///
368 : /// We could then provide another method to allow waiting for all child tasks
369 : /// to finish.
370 : ///
371 : /// We could make new calls to this function fail if `self` is already canceled.
372 : /// Alternatively, we could allow the creation but not spawn the task.
373 : /// The method to wait for child tasks would return an error, indicating
374 : /// that the child task was not started because the context was canceled.
375 1530696 : pub fn attached_child(&self) -> Self {
376 1530696 : self.child_impl(self.task_kind(), self.download_behavior())
377 1530696 : }
378 :
379 : /// Use this function when you should be creating a child context using
380 : /// [`attached_child`] or [`detached_child`], but your caller doesn't provide
381 : /// a context and you are unwilling to change all callers to provide one.
382 : ///
383 : /// Before we add cancellation, we should get rid of this method.
384 : ///
385 : /// [`attached_child`]: Self::attached_child
386 : /// [`detached_child`]: Self::detached_child
387 892 : pub fn todo_child(task_kind: TaskKind, download_behavior: DownloadBehavior) -> Self {
388 892 : Self::new(task_kind, download_behavior)
389 892 : }
390 :
391 1531152 : fn child_impl(&self, task_kind: TaskKind, download_behavior: DownloadBehavior) -> Self {
392 1531152 : RequestContextBuilder::extend(self)
393 1531152 : .task_kind(task_kind)
394 1531152 : .download_behavior(download_behavior)
395 1531152 : .build()
396 1531152 : }
397 :
398 932 : pub fn with_scope_timeline(&self, timeline: &Arc<Timeline>) -> Self {
399 932 : RequestContextBuilder::extend(self)
400 932 : .scope(Scope::new_timeline(timeline))
401 932 : .build()
402 932 : }
403 :
404 0 : pub(crate) fn with_scope_page_service_pagestream(
405 0 : &self,
406 0 : timeline_handle: &crate::tenant::timeline::handle::Handle<
407 0 : crate::page_service::TenantManagerTypes,
408 0 : >,
409 0 : ) -> Self {
410 0 : RequestContextBuilder::extend(self)
411 0 : .scope(Scope::new_page_service_pagestream(timeline_handle))
412 0 : .build()
413 0 : }
414 :
415 0 : pub fn with_scope_secondary_timeline(
416 0 : &self,
417 0 : tenant_shard_id: &TenantShardId,
418 0 : timeline_id: &TimelineId,
419 0 : ) -> Self {
420 0 : RequestContextBuilder::extend(self)
421 0 : .scope(Scope::new_secondary_timeline(tenant_shard_id, timeline_id))
422 0 : .build()
423 0 : }
424 :
425 0 : pub fn with_scope_secondary_tenant(&self, tenant_shard_id: &TenantShardId) -> Self {
426 0 : RequestContextBuilder::extend(self)
427 0 : .scope(Scope::new_secondary_tenant(tenant_shard_id))
428 0 : .build()
429 0 : }
430 :
431 : #[cfg(test)]
432 580 : pub fn with_scope_unit_test(&self) -> Self {
433 580 : RequestContextBuilder::new(TaskKind::UnitTest)
434 580 : .scope(Scope::new_unit_test())
435 580 : .build()
436 580 : }
437 :
438 3964073 : pub fn task_kind(&self) -> TaskKind {
439 3964073 : self.task_kind
440 3964073 : }
441 :
442 1530756 : pub fn download_behavior(&self) -> DownloadBehavior {
443 1530756 : self.download_behavior
444 1530756 : }
445 :
446 479379 : pub(crate) fn access_stats_behavior(&self) -> AccessStatsBehavior {
447 479379 : self.access_stats_behavior
448 479379 : }
449 :
450 1944556 : pub(crate) fn page_content_kind(&self) -> PageContentKind {
451 1944556 : self.page_content_kind
452 1944556 : }
453 :
454 0 : pub(crate) fn read_path_debug(&self) -> bool {
455 0 : self.read_path_debug
456 0 : }
457 :
458 3241370 : pub(crate) fn io_size_metrics(&self) -> &StorageIoSizeMetrics {
459 3241370 : match &self.scope {
460 0 : Scope::Global { io_size_metrics } => {
461 0 : let is_unit_test = cfg!(test);
462 0 : let is_regress_test_build = cfg!(feature = "testing");
463 0 : if is_unit_test || is_regress_test_build {
464 0 : panic!("all VirtualFile instances are timeline-scoped");
465 : } else {
466 3241370 : use once_cell::sync::Lazy;
467 3241370 : use std::sync::Mutex;
468 3241370 : use std::time::Duration;
469 3241370 : use utils::rate_limit::RateLimit;
470 3241370 : static LIMIT: Lazy<Mutex<RateLimit>> =
471 0 : Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(1))));
472 0 : let mut guard = LIMIT.lock().unwrap();
473 0 : guard.call2(|rate_limit_stats| {
474 0 : warn!(
475 : %rate_limit_stats,
476 0 : backtrace=%std::backtrace::Backtrace::force_capture(),
477 0 : "all VirtualFile instances are timeline-scoped",
478 : );
479 0 : });
480 0 :
481 0 : io_size_metrics
482 : }
483 : }
484 42849 : Scope::Timeline { arc_arc } => &arc_arc.storage_io_size,
485 0 : Scope::SecondaryTimeline { io_size_metrics } => io_size_metrics,
486 0 : Scope::SecondaryTenant { io_size_metrics } => io_size_metrics,
487 : #[cfg(test)]
488 3198521 : Scope::UnitTest { io_size_metrics } => io_size_metrics,
489 : }
490 3241370 : }
491 : }
|
