Line data Source code
1 : use std::ops::ControlFlow;
2 : use std::sync::Arc;
3 :
4 : use tokio_util::sync::CancellationToken;
5 : use tracing::{Instrument, info, info_span, warn};
6 : use utils::sync::duplex;
7 :
8 : use super::{Buffer, CheapCloneForRead, OwnedAsyncWriter};
9 : use crate::context::RequestContext;
10 : use crate::virtual_file::MaybeFatalIo;
11 : use crate::virtual_file::owned_buffers_io::io_buf_aligned::IoBufAligned;
12 : use crate::virtual_file::owned_buffers_io::io_buf_ext::FullSlice;
13 :
14 : /// A handle to the flush task.
15 : pub struct FlushHandle<Buf, W> {
16 : inner: Option<FlushHandleInner<Buf, W>>,
17 : }
18 :
19 : pub struct FlushHandleInner<Buf, W> {
20 : /// A bi-directional channel that sends (buffer, offset) for writes,
21 : /// and receives recyled buffer.
22 : channel: duplex::mpsc::Duplex<FlushRequest<Buf>, FullSlice<Buf>>,
23 : /// Join handle for the background flush task.
24 : join_handle: tokio::task::JoinHandle<Result<Arc<W>, FlushTaskError>>,
25 : }
26 :
27 : struct FlushRequest<Buf> {
28 : slice: FullSlice<Buf>,
29 : offset: u64,
30 : #[cfg(test)]
31 : ready_to_flush_rx: tokio::sync::oneshot::Receiver<()>,
32 : #[cfg(test)]
33 : done_flush_tx: tokio::sync::oneshot::Sender<()>,
34 : }
35 :
36 : /// Constructs a request and a control object for a new flush operation.
37 : #[cfg(not(test))]
38 0 : fn new_flush_op<Buf>(slice: FullSlice<Buf>, offset: u64) -> (FlushRequest<Buf>, FlushControl) {
39 0 : let request = FlushRequest { slice, offset };
40 0 : let control = FlushControl::untracked();
41 0 :
42 0 : (request, control)
43 0 : }
44 :
45 : /// Constructs a request and a control object for a new flush operation.
46 : #[cfg(test)]
47 39978 : fn new_flush_op<Buf>(slice: FullSlice<Buf>, offset: u64) -> (FlushRequest<Buf>, FlushControl) {
48 39978 : let (ready_to_flush_tx, ready_to_flush_rx) = tokio::sync::oneshot::channel();
49 39978 : let (done_flush_tx, done_flush_rx) = tokio::sync::oneshot::channel();
50 39978 : let control = FlushControl::not_started(ready_to_flush_tx, done_flush_rx);
51 39978 :
52 39978 : let request = FlushRequest {
53 39978 : slice,
54 39978 : offset,
55 39978 : ready_to_flush_rx,
56 39978 : done_flush_tx,
57 39978 : };
58 39978 : (request, control)
59 39978 : }
60 :
61 : /// A handle to a `FlushRequest` that allows unit tests precise control over flush behavior.
62 : #[cfg(test)]
63 : pub(crate) struct FlushControl {
64 : not_started: FlushNotStarted,
65 : }
66 :
67 : #[cfg(not(test))]
68 : pub(crate) struct FlushControl;
69 :
70 : impl FlushControl {
71 : #[cfg(test)]
72 39978 : fn not_started(
73 39978 : ready_to_flush_tx: tokio::sync::oneshot::Sender<()>,
74 39978 : done_flush_rx: tokio::sync::oneshot::Receiver<()>,
75 39978 : ) -> Self {
76 39978 : FlushControl {
77 39978 : not_started: FlushNotStarted {
78 39978 : ready_to_flush_tx,
79 39978 : done_flush_rx,
80 39978 : },
81 39978 : }
82 39978 : }
83 :
84 : #[cfg(not(test))]
85 0 : fn untracked() -> Self {
86 0 : FlushControl
87 0 : }
88 :
89 : /// In tests, turn flush control into a not started state.
90 : #[cfg(test)]
91 12 : pub(crate) fn into_not_started(self) -> FlushNotStarted {
92 12 : self.not_started
93 12 : }
94 :
95 : /// Release control to the submitted buffer.
96 : ///
97 : /// In `cfg(test)` environment, the buffer is guranteed to be flushed to disk after [`FlushControl::release`] is finishes execution.
98 39912 : pub async fn release(self) {
99 39912 : #[cfg(test)]
100 39912 : {
101 39912 : self.not_started
102 39912 : .ready_to_flush()
103 39912 : .wait_until_flush_is_done()
104 39912 : .await;
105 0 : }
106 39912 : }
107 : }
108 :
109 : impl<Buf, W> FlushHandle<Buf, W>
110 : where
111 : Buf: IoBufAligned + Send + Sync + CheapCloneForRead,
112 : W: OwnedAsyncWriter + Send + Sync + 'static + std::fmt::Debug,
113 : {
114 : /// Spawns a new background flush task and obtains a handle.
115 : ///
116 : /// Handle and background task are connected through a duplex channel.
117 : /// Dirty buffers are sent to the background task for flushing.
118 : /// Clean buffers are sent back to the handle for reuse.
119 : ///
120 : /// The queue depth is 1, and the passed-in `buf` seeds the queue depth.
121 : /// I.e., the passed-in buf is immediately available to the handle as a recycled buffer.
122 8010 : pub fn spawn_new<B>(
123 8010 : file: Arc<W>,
124 8010 : buf: B,
125 8010 : gate_guard: utils::sync::gate::GateGuard,
126 8010 : cancel: CancellationToken,
127 8010 : ctx: RequestContext,
128 8010 : span: tracing::Span,
129 8010 : ) -> Self
130 8010 : where
131 8010 : B: Buffer<IoBuf = Buf> + Send + 'static,
132 8010 : {
133 8010 : let (front, back) = duplex::mpsc::channel(1);
134 8010 : back.try_send(buf.flush())
135 8010 : .expect("we just created it with capacity 1");
136 8010 :
137 8010 : let join_handle = tokio::spawn(
138 8010 : FlushBackgroundTask::new(back, file, gate_guard, cancel, ctx)
139 8010 : .run()
140 8010 : .instrument(span),
141 8010 : );
142 8010 :
143 8010 : FlushHandle {
144 8010 : inner: Some(FlushHandleInner {
145 8010 : channel: front,
146 8010 : join_handle,
147 8010 : }),
148 8010 : }
149 8010 : }
150 :
151 : /// Submits a buffer to be flushed in the background task.
152 : /// Returns a buffer that completed flushing for re-use, length reset to 0, capacity unchanged.
153 : /// If `save_buf_for_read` is true, then we save the buffer in `Self::maybe_flushed`, otherwise
154 : /// clear `maybe_flushed`.
155 39978 : pub async fn flush(
156 39978 : &mut self,
157 39978 : slice: FullSlice<Buf>,
158 39978 : offset: u64,
159 39978 : ) -> Result<(FullSlice<Buf>, FlushControl), FlushTaskError> {
160 39978 : let (request, flush_control) = new_flush_op(slice, offset);
161 :
162 : // Submits the buffer to the background task.
163 39978 : let submit = self.inner_mut().channel.send(request).await;
164 39978 : if submit.is_err() {
165 0 : return self.handle_error().await;
166 39978 : }
167 :
168 : // Wait for an available buffer from the background flush task.
169 : // This is the BACKPRESSURE mechanism: if the flush task can't keep up,
170 : // then the write path will eventually wait for it here.
171 39978 : let Some(recycled) = self.inner_mut().channel.recv().await else {
172 0 : return self.handle_error().await;
173 : };
174 :
175 39978 : Ok((recycled, flush_control))
176 39978 : }
177 :
178 0 : async fn handle_error<T>(&mut self) -> Result<T, FlushTaskError> {
179 0 : Err(self
180 0 : .shutdown()
181 0 : .await
182 0 : .expect_err("flush task only disconnects duplex if it exits with an error"))
183 0 : }
184 :
185 : /// Cleans up the channel, join the flush task.
186 54 : pub async fn shutdown(&mut self) -> Result<Arc<W>, FlushTaskError> {
187 54 : let handle = self
188 54 : .inner
189 54 : .take()
190 54 : .expect("must not use after we returned an error");
191 54 : drop(handle.channel.tx);
192 54 : handle.join_handle.await.unwrap()
193 54 : }
194 :
195 : /// Gets a mutable reference to the inner handle. Panics if [`Self::inner`] is `None`.
196 : /// This only happens if the handle is used after an error.
197 79956 : fn inner_mut(&mut self) -> &mut FlushHandleInner<Buf, W> {
198 79956 : self.inner
199 79956 : .as_mut()
200 79956 : .expect("must not use after we returned an error")
201 79956 : }
202 : }
203 :
204 : /// A background task for flushing data to disk.
205 : pub struct FlushBackgroundTask<Buf, W> {
206 : /// A bi-directional channel that receives (buffer, offset) for writes,
207 : /// and send back recycled buffer.
208 : channel: duplex::mpsc::Duplex<FullSlice<Buf>, FlushRequest<Buf>>,
209 : /// A writter for persisting data to disk.
210 : writer: Arc<W>,
211 : ctx: RequestContext,
212 : cancel: CancellationToken,
213 : /// Prevent timeline from shuting down until the flush background task finishes flushing all remaining buffers to disk.
214 : _gate_guard: utils::sync::gate::GateGuard,
215 : }
216 :
217 : #[derive(Debug, thiserror::Error)]
218 : pub enum FlushTaskError {
219 : #[error("flush task cancelled")]
220 : Cancelled,
221 : }
222 :
223 : impl<Buf, W> FlushBackgroundTask<Buf, W>
224 : where
225 : Buf: IoBufAligned + Send + Sync,
226 : W: OwnedAsyncWriter + Sync + 'static,
227 : {
228 : /// Creates a new background flush task.
229 8010 : fn new(
230 8010 : channel: duplex::mpsc::Duplex<FullSlice<Buf>, FlushRequest<Buf>>,
231 8010 : file: Arc<W>,
232 8010 : gate_guard: utils::sync::gate::GateGuard,
233 8010 : cancel: CancellationToken,
234 8010 : ctx: RequestContext,
235 8010 : ) -> Self {
236 8010 : FlushBackgroundTask {
237 8010 : channel,
238 8010 : writer: file,
239 8010 : _gate_guard: gate_guard,
240 8010 : cancel,
241 8010 : ctx,
242 8010 : }
243 8010 : }
244 :
245 : /// Runs the background flush task.
246 8010 : async fn run(mut self) -> Result<Arc<W>, FlushTaskError> {
247 : // Exit condition: channel is closed and there is no remaining buffer to be flushed
248 47920 : while let Some(request) = self.channel.recv().await {
249 : #[cfg(test)]
250 : {
251 : // In test, wait for control to signal that we are ready to flush.
252 39978 : if request.ready_to_flush_rx.await.is_err() {
253 54 : tracing::debug!("control dropped");
254 39924 : }
255 : }
256 :
257 : // Write slice to disk at `offset`.
258 : //
259 : // Error handling happens according to the current policy of crashing
260 : // on fatal IO errors and retrying in place otherwise (deeming all other errors retryable).
261 : // (The upper layers of the Pageserver write path are not equipped to retry write errors
262 : // becasuse they often deallocate the buffers that were already written).
263 : //
264 : // TODO: use utils::backoff::retry once async closures are actually usable
265 : //
266 39978 : let mut slice_storage = Some(request.slice);
267 39978 : for attempt in 1.. {
268 39978 : if self.cancel.is_cancelled() {
269 0 : return Err(FlushTaskError::Cancelled);
270 39978 : }
271 39978 : let result = async {
272 39978 : if attempt > 1 {
273 0 : info!("retrying flush");
274 39978 : }
275 39978 : let slice = slice_storage.take().expect(
276 39978 : "likely previous invocation of this future didn't get polled to completion",
277 39978 : );
278 : // Don't cancel this write by doing tokio::select with self.cancel.cancelled().
279 : // The underlying tokio-epoll-uring slot / kernel operation is still ongoing and occupies resources.
280 : // If we retry indefinitely, we'll deplete those resources.
281 : // Future: teach tokio-epoll-uring io_uring operation cancellation, but still,
282 : // wait for cancelled ops to complete and discard their error.
283 39978 : let (slice, res) = self.writer.write_all_at(slice, request.offset, &self.ctx).await;
284 39978 : slice_storage = Some(slice);
285 39978 : let res = res.maybe_fatal_err("owned_buffers_io flush");
286 39978 : let Err(err) = res else {
287 39978 : return ControlFlow::Break(());
288 : };
289 0 : warn!(%err, "error flushing buffered writer buffer to disk, retrying after backoff");
290 0 : utils::backoff::exponential_backoff(attempt, 1.0, 10.0, &self.cancel).await;
291 0 : ControlFlow::Continue(())
292 39978 : }
293 39978 : .instrument(info_span!("flush_attempt", %attempt))
294 39978 : .await;
295 39978 : match result {
296 39978 : ControlFlow::Break(()) => break,
297 0 : ControlFlow::Continue(()) => continue,
298 : }
299 : }
300 39978 : let slice = slice_storage.expect("loop must have run at least once");
301 39978 :
302 39978 : #[cfg(test)]
303 39978 : {
304 39978 : // In test, tell control we are done flushing buffer.
305 39978 : if request.done_flush_tx.send(()).is_err() {
306 54 : tracing::debug!("control dropped");
307 39924 : }
308 0 : }
309 0 :
310 0 : // Sends the buffer back to the handle for reuse. The handle is in charged of cleaning the buffer.
311 39978 : if self.channel.send(slice).await.is_err() {
312 : // Although channel is closed. Still need to finish flushing the remaining buffers.
313 0 : continue;
314 39978 : }
315 : }
316 :
317 7149 : Ok(self.writer)
318 7149 : }
319 : }
320 :
321 : #[cfg(test)]
322 : pub(crate) struct FlushNotStarted {
323 : ready_to_flush_tx: tokio::sync::oneshot::Sender<()>,
324 : done_flush_rx: tokio::sync::oneshot::Receiver<()>,
325 : }
326 :
327 : #[cfg(test)]
328 : pub(crate) struct FlushInProgress {
329 : done_flush_rx: tokio::sync::oneshot::Receiver<()>,
330 : }
331 :
332 : #[cfg(test)]
333 : pub(crate) struct FlushDone;
334 :
335 : #[cfg(test)]
336 : impl FlushNotStarted {
337 : /// Signals the background task the buffer is ready to flush to disk.
338 39924 : pub fn ready_to_flush(self) -> FlushInProgress {
339 39924 : self.ready_to_flush_tx
340 39924 : .send(())
341 39924 : .map(|_| FlushInProgress {
342 39924 : done_flush_rx: self.done_flush_rx,
343 39924 : })
344 39924 : .unwrap()
345 39924 : }
346 : }
347 :
348 : #[cfg(test)]
349 : impl FlushInProgress {
350 : /// Waits until background flush is done.
351 39924 : pub async fn wait_until_flush_is_done(self) -> FlushDone {
352 39924 : self.done_flush_rx.await.unwrap();
353 39924 : FlushDone
354 39924 : }
355 : }
|
