Line data    Source code

       1              : //! failpoints for unit tests, implying `#[cfg(test)]`.
       2              : //!
       3              : //! These are not accessible over http.
       4              : 
       5              : use super::*;
       6              : 
       7              : impl Layer {
       8              :     /// Enable a failpoint from a unit test.
       9          108 :     pub(super) fn enable_failpoint(&self, failpoint: Failpoint) {
      10          108 :         self.0.failpoints.lock().unwrap().push(failpoint);
      11          108 :     }
      12              : }
      13              : 
      14              : impl LayerInner {
      15              :     /// Query if this failpoint is enabled, as in, arrive at a failpoint.
      16              :     ///
      17              :     /// Calls to this method need to be `#[cfg(test)]` guarded.
      18          372 :     pub(super) async fn failpoint(&self, kind: FailpointKind) -> Result<(), FailpointHit> {
      19          120 :         let fut = {
      20          372 :             let mut fps = self.failpoints.lock().unwrap();
      21          372 :             // find the *last* failpoint for cases in which we need to use multiple for the same
      22          372 :             // thing (two blocked evictions)
      23          372 :             let fp = fps.iter_mut().rfind(|x| x.kind() == kind);
      24              : 
      25          372 :             let Some(fp) = fp else {
      26          252 :                 return Ok(());
      27              :             };
      28              : 
      29          120 :             fp.hit()
      30          120 :         };
      31          120 : 
      32          120 :         fut.await
      33          372 :     }
      34              : }
      35              : 
      36              : #[derive(Debug, PartialEq, Eq)]
      37              : pub(crate) enum FailpointKind {
      38              :     /// Failpoint acts as an accurate cancelled by drop here; see the only site of use.
      39              :     AfterDeterminingLayerNeedsNoDownload,
      40              :     /// Failpoint for stalling eviction starting
      41              :     WaitBeforeStartingEvicting,
      42              :     /// Failpoint hit in the spawned task
      43              :     WaitBeforeDownloading,
      44              : }
      45              : 
      46              : pub(crate) enum Failpoint {
      47              :     AfterDeterminingLayerNeedsNoDownload,
      48              :     WaitBeforeStartingEvicting(
      49              :         Option<utils::completion::Completion>,
      50              :         utils::completion::Barrier,
      51              :     ),
      52              :     WaitBeforeDownloading(
      53              :         Option<utils::completion::Completion>,
      54              :         utils::completion::Barrier,
      55              :     ),
      56              : }
      57              : 
      58              : impl Failpoint {
      59          192 :     fn kind(&self) -> FailpointKind {
      60          192 :         match self {
      61              :             Failpoint::AfterDeterminingLayerNeedsNoDownload => {
      62           24 :                 FailpointKind::AfterDeterminingLayerNeedsNoDownload
      63              :             }
      64          144 :             Failpoint::WaitBeforeStartingEvicting(..) => FailpointKind::WaitBeforeStartingEvicting,
      65           24 :             Failpoint::WaitBeforeDownloading(..) => FailpointKind::WaitBeforeDownloading,
      66              :         }
      67          192 :     }
      68              : 
      69          120 :     fn hit(&mut self) -> impl std::future::Future<Output = Result<(), FailpointHit>> + 'static {
      70              :         use futures::future::FutureExt;
      71              : 
      72              :         // use boxed futures to avoid Either hurdles
      73          120 :         match self {
      74              :             Failpoint::AfterDeterminingLayerNeedsNoDownload => {
      75           12 :                 let kind = self.kind();
      76              : 
      77           12 :                 async move { Err(FailpointHit(kind)) }.boxed()
      78              :             }
      79           96 :             Failpoint::WaitBeforeStartingEvicting(arrival, b)
      80           12 :             | Failpoint::WaitBeforeDownloading(arrival, b) => {
      81              :                 // first one signals arrival
      82          108 :                 drop(arrival.take());
      83          108 : 
      84          108 :                 let b = b.clone();
      85              : 
      86          108 :                 async move {
      87          108 :                     tracing::trace!("waiting on a failpoint barrier");
      88          108 :                     b.wait().await;
      89          108 :                     tracing::trace!("done waiting on a failpoint barrier");
      90          108 :                     Ok(())
      91          108 :                 }
      92          108 :                 .boxed()
      93              :             }
      94              :         }
      95          120 :     }
      96              : }
      97              : 
      98              : impl std::fmt::Display for FailpointKind {
      99            0 :     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
     100            0 :         std::fmt::Debug::fmt(self, f)
     101            0 :     }
     102              : }
     103              : 
     104              : #[derive(Debug)]
     105              : pub(crate) struct FailpointHit(FailpointKind);
     106              : 
     107              : impl std::fmt::Display for FailpointHit {
     108            0 :     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
     109            0 :         std::fmt::Debug::fmt(self, f)
     110            0 :     }
     111              : }
     112              : 
     113              : impl std::error::Error for FailpointHit {}
     114              : 
     115              : impl From<FailpointHit> for DownloadError {
     116           12 :     fn from(value: FailpointHit) -> Self {
     117           12 :         DownloadError::Failpoint(value.0)
     118           12 :     }
     119              : }