Line data    Source code

       1              : //! failpoints for unit tests, implying `#[cfg(test)]`.
       2              : //!
       3              : //! These are not accessible over http.
       4              : 
       5              : use super::*;
       6              : 
       7              : impl Layer {
       8              :     /// Enable a failpoint from a unit test.
       9           54 :     pub(super) fn enable_failpoint(&self, failpoint: Failpoint) {
      10           54 :         self.0.failpoints.lock().unwrap().push(failpoint);
      11           54 :     }
      12              : }
      13              : 
      14              : impl LayerInner {
      15              :     /// Query if this failpoint is enabled, as in, arrive at a failpoint.
      16              :     ///
      17              :     /// Calls to this method need to be `#[cfg(test)]` guarded.
      18          114 :     pub(super) async fn failpoint(&self, kind: FailpointKind) -> Result<(), FailpointHit> {
      19           60 :         let fut = {
      20          114 :             let mut fps = self.failpoints.lock().unwrap();
      21          114 :             // find the *last* failpoint for cases in which we need to use multiple for the same
      22          114 :             // thing (two blocked evictions)
      23          114 :             let fp = fps.iter_mut().rfind(|x| x.kind() == kind);
      24              : 
      25          114 :             let Some(fp) = fp else {
      26           54 :                 return Ok(());
      27              :             };
      28              : 
      29           60 :             fp.hit()
      30           60 :         };
      31           60 : 
      32           60 :         fut.await
      33          114 :     }
      34              : }
      35              : 
      36              : #[derive(Debug, PartialEq, Eq)]
      37              : pub(crate) enum FailpointKind {
      38              :     /// Failpoint acts as an accurate cancelled by drop here; see the only site of use.
      39              :     AfterDeterminingLayerNeedsNoDownload,
      40              :     /// Failpoint for stalling eviction starting
      41              :     WaitBeforeStartingEvicting,
      42              :     /// Failpoint hit in the spawned task
      43              :     WaitBeforeDownloading,
      44              : }
      45              : 
      46              : pub(crate) enum Failpoint {
      47              :     AfterDeterminingLayerNeedsNoDownload,
      48              :     WaitBeforeStartingEvicting(
      49              :         Option<utils::completion::Completion>,
      50              :         utils::completion::Barrier,
      51              :     ),
      52              :     WaitBeforeDownloading(
      53              :         Option<utils::completion::Completion>,
      54              :         utils::completion::Barrier,
      55              :     ),
      56              : }
      57              : 
      58              : impl Failpoint {
      59           96 :     fn kind(&self) -> FailpointKind {
      60           96 :         match self {
      61              :             Failpoint::AfterDeterminingLayerNeedsNoDownload => {
      62           12 :                 FailpointKind::AfterDeterminingLayerNeedsNoDownload
      63              :             }
      64           72 :             Failpoint::WaitBeforeStartingEvicting(..) => FailpointKind::WaitBeforeStartingEvicting,
      65           12 :             Failpoint::WaitBeforeDownloading(..) => FailpointKind::WaitBeforeDownloading,
      66              :         }
      67           96 :     }
      68              : 
      69           60 :     fn hit(&mut self) -> impl std::future::Future<Output = Result<(), FailpointHit>> + 'static {
      70              :         use futures::future::FutureExt;
      71              : 
      72              :         // use boxed futures to avoid Either hurdles
      73           60 :         match self {
      74              :             Failpoint::AfterDeterminingLayerNeedsNoDownload => {
      75            6 :                 let kind = self.kind();
      76              : 
      77            6 :                 async move { Err(FailpointHit(kind)) }.boxed()
      78              :             }
      79           48 :             Failpoint::WaitBeforeStartingEvicting(arrival, b)
      80            6 :             | Failpoint::WaitBeforeDownloading(arrival, b) => {
      81              :                 // first one signals arrival
      82           54 :                 drop(arrival.take());
      83           54 : 
      84           54 :                 let b = b.clone();
      85              : 
      86           54 :                 async move {
      87           54 :                     tracing::trace!("waiting on a failpoint barrier");
      88           54 :                     b.wait().await;
      89           54 :                     tracing::trace!("done waiting on a failpoint barrier");
      90           54 :                     Ok(())
      91           54 :                 }
      92           54 :                 .boxed()
      93              :             }
      94              :         }
      95           60 :     }
      96              : }
      97              : 
      98              : impl std::fmt::Display for FailpointKind {
      99            0 :     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
     100            0 :         std::fmt::Debug::fmt(self, f)
     101            0 :     }
     102              : }
     103              : 
     104              : #[derive(Debug)]
     105              : pub(crate) struct FailpointHit(FailpointKind);
     106              : 
     107              : impl std::fmt::Display for FailpointHit {
     108            0 :     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
     109            0 :         std::fmt::Debug::fmt(self, f)
     110            0 :     }
     111              : }
     112              : 
     113              : impl std::error::Error for FailpointHit {}
     114              : 
     115              : impl From<FailpointHit> for DownloadError {
     116            6 :     fn from(value: FailpointHit) -> Self {
     117            6 :         DownloadError::Failpoint(value.0)
     118            6 :     }
     119              : }