Line data    Source code

       1              : //! failpoints for unit tests, implying `#[cfg(test)]`.
       2              : //!
       3              : //! These are not accessible over http.
       4              : 
       5              : use super::*;
       6              : 
       7              : impl Layer {
       8              :     /// Enable a failpoint from a unit test.
       9           18 :     pub(super) fn enable_failpoint(&self, failpoint: Failpoint) {
      10           18 :         self.0.failpoints.lock().unwrap().push(failpoint);
      11           18 :     }
      12              : }
      13              : 
      14              : impl LayerInner {
      15              :     /// Query if this failpoint is enabled, as in, arrive at a failpoint.
      16              :     ///
      17              :     /// Calls to this method need to be `#[cfg(test)]` guarded.
      18           38 :     pub(super) async fn failpoint(&self, kind: FailpointKind) -> Result<(), FailpointHit> {
      19           20 :         let fut = {
      20           38 :             let mut fps = self.failpoints.lock().unwrap();
      21           38 :             // find the *last* failpoint for cases in which we need to use multiple for the same
      22           38 :             // thing (two blocked evictions)
      23           38 :             let fp = fps.iter_mut().rfind(|x| x.kind() == kind);
      24              : 
      25           38 :             let Some(fp) = fp else {
      26           18 :                 return Ok(());
      27              :             };
      28              : 
      29           20 :             fp.hit()
      30           20 :         };
      31           20 : 
      32           20 :         fut.await
      33           38 :     }
      34              : }
      35              : 
      36              : #[derive(Debug, PartialEq, Eq)]
      37              : pub(crate) enum FailpointKind {
      38              :     /// Failpoint acts as an accurate cancelled by drop here; see the only site of use.
      39              :     AfterDeterminingLayerNeedsNoDownload,
      40              :     /// Failpoint for stalling eviction starting
      41              :     WaitBeforeStartingEvicting,
      42              :     /// Failpoint hit in the spawned task
      43              :     WaitBeforeDownloading,
      44              : }
      45              : 
      46              : pub(crate) enum Failpoint {
      47              :     AfterDeterminingLayerNeedsNoDownload,
      48              :     WaitBeforeStartingEvicting(
      49              :         Option<utils::completion::Completion>,
      50              :         utils::completion::Barrier,
      51              :     ),
      52              :     WaitBeforeDownloading(
      53              :         Option<utils::completion::Completion>,
      54              :         utils::completion::Barrier,
      55              :     ),
      56              : }
      57              : 
      58              : impl Failpoint {
      59           32 :     fn kind(&self) -> FailpointKind {
      60           32 :         match self {
      61              :             Failpoint::AfterDeterminingLayerNeedsNoDownload => {
      62            4 :                 FailpointKind::AfterDeterminingLayerNeedsNoDownload
      63              :             }
      64           24 :             Failpoint::WaitBeforeStartingEvicting(..) => FailpointKind::WaitBeforeStartingEvicting,
      65            4 :             Failpoint::WaitBeforeDownloading(..) => FailpointKind::WaitBeforeDownloading,
      66              :         }
      67           32 :     }
      68              : 
      69           20 :     fn hit(&mut self) -> impl std::future::Future<Output = Result<(), FailpointHit>> + 'static {
      70           20 :         use futures::future::FutureExt;
      71           20 : 
      72           20 :         // use boxed futures to avoid Either hurdles
      73           20 :         match self {
      74              :             Failpoint::AfterDeterminingLayerNeedsNoDownload => {
      75            2 :                 let kind = self.kind();
      76              : 
      77            2 :                 async move { Err(FailpointHit(kind)) }.boxed()
      78              :             }
      79           16 :             Failpoint::WaitBeforeStartingEvicting(arrival, b)
      80            2 :             | Failpoint::WaitBeforeDownloading(arrival, b) => {
      81              :                 // first one signals arrival
      82           18 :                 drop(arrival.take());
      83           18 : 
      84           18 :                 let b = b.clone();
      85           18 : 
      86           18 :                 async move {
      87           18 :                     tracing::trace!("waiting on a failpoint barrier");
      88           18 :                     b.wait().await;
      89           18 :                     tracing::trace!("done waiting on a failpoint barrier");
      90           18 :                     Ok(())
      91           18 :                 }
      92           18 :                 .boxed()
      93              :             }
      94              :         }
      95           20 :     }
      96              : }
      97              : 
      98              : impl std::fmt::Display for FailpointKind {
      99            0 :     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
     100            0 :         std::fmt::Debug::fmt(self, f)
     101            0 :     }
     102              : }
     103              : 
     104              : #[derive(Debug)]
     105              : pub(crate) struct FailpointHit(FailpointKind);
     106              : 
     107              : impl std::fmt::Display for FailpointHit {
     108            0 :     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
     109            0 :         std::fmt::Debug::fmt(self, f)
     110            0 :     }
     111              : }
     112              : 
     113              : impl std::error::Error for FailpointHit {}
     114              : 
     115              : impl From<FailpointHit> for DownloadError {
     116            2 :     fn from(value: FailpointHit) -> Self {
     117            2 :         DownloadError::Failpoint(value.0)
     118            2 :     }
     119              : }