Line data Source code
1 : use std::{
2 : borrow::Cow,
3 : fs::{self, File},
4 : io::{self, Write},
5 : };
6 :
7 : use camino::{Utf8Path, Utf8PathBuf};
8 :
9 : /// Similar to [`std::fs::create_dir`], except we fsync the
10 : /// created directory and its parent.
11 1221 : pub fn create_dir(path: impl AsRef<Utf8Path>) -> io::Result<()> {
12 1221 : let path = path.as_ref();
13 1221 :
14 1221 : fs::create_dir(path)?;
15 1219 : fsync_file_and_parent(path)?;
16 1219 : Ok(())
17 1221 : }
18 :
19 : /// Similar to [`std::fs::create_dir_all`], except we fsync all
20 : /// newly created directories and the pre-existing parent.
21 5 : pub fn create_dir_all(path: impl AsRef<Utf8Path>) -> io::Result<()> {
22 5 : let mut path = path.as_ref();
23 5 :
24 5 : let mut dirs_to_create = Vec::new();
25 :
26 : // Figure out which directories we need to create.
27 : loop {
28 8 : match path.metadata() {
29 4 : Ok(metadata) if metadata.is_dir() => break,
30 : Ok(_) => {
31 1 : return Err(io::Error::new(
32 1 : io::ErrorKind::AlreadyExists,
33 1 : format!("non-directory found in path: {path}"),
34 1 : ));
35 : }
36 4 : Err(ref e) if e.kind() == io::ErrorKind::NotFound => {}
37 1 : Err(e) => return Err(e),
38 : }
39 :
40 3 : dirs_to_create.push(path);
41 3 :
42 3 : match path.parent() {
43 3 : Some(parent) => path = parent,
44 : None => {
45 0 : return Err(io::Error::new(
46 0 : io::ErrorKind::InvalidInput,
47 0 : format!("can't find parent of path '{path}'"),
48 0 : ));
49 : }
50 : }
51 : }
52 :
53 : // Create directories from parent to child.
54 3 : for &path in dirs_to_create.iter().rev() {
55 3 : fs::create_dir(path)?;
56 : }
57 :
58 : // Fsync the created directories from child to parent.
59 3 : for &path in dirs_to_create.iter() {
60 3 : fsync(path)?;
61 : }
62 :
63 : // If we created any new directories, fsync the parent.
64 3 : if !dirs_to_create.is_empty() {
65 2 : fsync(path)?;
66 1 : }
67 :
68 3 : Ok(())
69 5 : }
70 :
71 : /// Adds a suffix to the file(directory) name, either appending the suffix to the end of its extension,
72 : /// or if there's no extension, creates one and puts a suffix there.
73 8481 : pub fn path_with_suffix_extension(
74 8481 : original_path: impl AsRef<Utf8Path>,
75 8481 : suffix: &str,
76 8481 : ) -> Utf8PathBuf {
77 8481 : let new_extension = match original_path.as_ref().extension() {
78 4366 : Some(extension) => Cow::Owned(format!("{extension}.{suffix}")),
79 4115 : None => Cow::Borrowed(suffix),
80 : };
81 8481 : original_path.as_ref().with_extension(new_extension)
82 8481 : }
83 :
84 1219 : pub fn fsync_file_and_parent(file_path: &Utf8Path) -> io::Result<()> {
85 1219 : let parent = file_path.parent().ok_or_else(|| {
86 0 : io::Error::new(
87 0 : io::ErrorKind::Other,
88 0 : format!("File {file_path:?} has no parent"),
89 0 : )
90 1219 : })?;
91 :
92 1219 : fsync(file_path)?;
93 1219 : fsync(parent)?;
94 1219 : Ok(())
95 1219 : }
96 :
97 2443 : pub fn fsync(path: &Utf8Path) -> io::Result<()> {
98 2443 : File::open(path)
99 2443 : .map_err(|e| io::Error::new(e.kind(), format!("Failed to open the file {path:?}: {e}")))
100 2443 : .and_then(|file| {
101 2443 : file.sync_all().map_err(|e| {
102 0 : io::Error::new(
103 0 : e.kind(),
104 0 : format!("Failed to sync file {path:?} data and metadata: {e}"),
105 0 : )
106 2443 : })
107 2443 : })
108 2443 : .map_err(|e| io::Error::new(e.kind(), format!("Failed to fsync file {path:?}: {e}")))
109 2443 : }
110 :
111 6 : pub async fn fsync_async(path: impl AsRef<Utf8Path>) -> Result<(), std::io::Error> {
112 6 : tokio::fs::File::open(path.as_ref()).await?.sync_all().await
113 6 : }
114 :
115 6 : pub async fn fsync_async_opt(
116 6 : path: impl AsRef<Utf8Path>,
117 6 : do_fsync: bool,
118 6 : ) -> Result<(), std::io::Error> {
119 6 : if do_fsync {
120 12 : fsync_async(path.as_ref()).await?;
121 0 : }
122 6 : Ok(())
123 6 : }
124 :
125 : /// Like postgres' durable_rename, renames file issuing fsyncs do make it
126 : /// durable. After return, file and rename are guaranteed to be persisted.
127 : ///
128 : /// Unlike postgres, it only does fsyncs to 1) file to be renamed to make
129 : /// contents durable; 2) its directory entry to make rename durable 3) again to
130 : /// already renamed file, which is not required by standards but postgres does
131 : /// it, let's stick to that. Postgres additionally fsyncs newpath *before*
132 : /// rename if it exists to ensure that at least one of the files survives, but
133 : /// current callers don't need that.
134 : ///
135 : /// virtual_file.rs has similar code, but it doesn't use vfs.
136 : ///
137 : /// Useful links: <https://lwn.net/Articles/457667/>
138 : /// <https://www.postgresql.org/message-id/flat/56583BDD.9060302%402ndquadrant.com>
139 : /// <https://thunk.org/tytso/blog/2009/03/15/dont-fear-the-fsync/>
140 2 : pub async fn durable_rename(
141 2 : old_path: impl AsRef<Utf8Path>,
142 2 : new_path: impl AsRef<Utf8Path>,
143 2 : do_fsync: bool,
144 2 : ) -> io::Result<()> {
145 2 : // first fsync the file
146 4 : fsync_async_opt(old_path.as_ref(), do_fsync).await?;
147 :
148 : // Time to do the real deal.
149 2 : tokio::fs::rename(old_path.as_ref(), new_path.as_ref()).await?;
150 :
151 : // Postgres'ish fsync of renamed file.
152 4 : fsync_async_opt(new_path.as_ref(), do_fsync).await?;
153 :
154 : // Now fsync the parent
155 2 : let parent = match new_path.as_ref().parent() {
156 2 : Some(p) => p,
157 0 : None => Utf8Path::new("./"), // assume current dir if there is no parent
158 : };
159 4 : fsync_async_opt(parent, do_fsync).await?;
160 :
161 2 : Ok(())
162 2 : }
163 :
164 : /// Writes a file to the specified `final_path` in a crash safe fasion, using [`std::fs`].
165 : ///
166 : /// The file is first written to the specified `tmp_path`, and in a second
167 : /// step, the `tmp_path` is renamed to the `final_path`. Intermediary fsync
168 : /// and atomic rename guarantee that, if we crash at any point, there will never
169 : /// be a partially written file at `final_path` (but maybe at `tmp_path`).
170 : ///
171 : /// Callers are responsible for serializing calls of this function for a given `final_path`.
172 : /// If they don't, there may be an error due to conflicting `tmp_path`, or there will
173 : /// be no error and the content of `final_path` will be the "winner" caller's `content`.
174 : /// I.e., the atomticity guarantees still hold.
175 84 : pub fn overwrite(
176 84 : final_path: &Utf8Path,
177 84 : tmp_path: &Utf8Path,
178 84 : content: &[u8],
179 84 : ) -> std::io::Result<()> {
180 84 : let Some(final_path_parent) = final_path.parent() else {
181 0 : return Err(std::io::Error::from_raw_os_error(
182 0 : nix::errno::Errno::EINVAL as i32,
183 0 : ));
184 : };
185 84 : std::fs::remove_file(tmp_path).or_else(crate::fs_ext::ignore_not_found)?;
186 84 : let mut file = std::fs::OpenOptions::new()
187 84 : .write(true)
188 84 : // Use `create_new` so that, if we race with ourselves or something else,
189 84 : // we bail out instead of causing damage.
190 84 : .create_new(true)
191 84 : .open(tmp_path)?;
192 84 : file.write_all(content)?;
193 84 : file.sync_all()?;
194 84 : drop(file); // don't keep the fd open for longer than we have to
195 84 :
196 84 : std::fs::rename(tmp_path, final_path)?;
197 :
198 84 : let final_parent_dirfd = std::fs::OpenOptions::new()
199 84 : .read(true)
200 84 : .open(final_path_parent)?;
201 :
202 84 : final_parent_dirfd.sync_all()?;
203 84 : Ok(())
204 84 : }
205 :
206 : #[cfg(test)]
207 : mod tests {
208 :
209 : use super::*;
210 :
211 : #[test]
212 1 : fn test_create_dir_fsyncd() {
213 1 : let dir = camino_tempfile::tempdir().unwrap();
214 1 :
215 1 : let existing_dir_path = dir.path();
216 1 : let err = create_dir(existing_dir_path).unwrap_err();
217 1 : assert_eq!(err.kind(), io::ErrorKind::AlreadyExists);
218 :
219 1 : let child_dir = existing_dir_path.join("child");
220 1 : create_dir(child_dir).unwrap();
221 1 :
222 1 : let nested_child_dir = existing_dir_path.join("child1").join("child2");
223 1 : let err = create_dir(nested_child_dir).unwrap_err();
224 1 : assert_eq!(err.kind(), io::ErrorKind::NotFound);
225 1 : }
226 :
227 : #[test]
228 1 : fn test_create_dir_all_fsyncd() {
229 1 : let dir = camino_tempfile::tempdir().unwrap();
230 1 :
231 1 : let existing_dir_path = dir.path();
232 1 : create_dir_all(existing_dir_path).unwrap();
233 1 :
234 1 : let child_dir = existing_dir_path.join("child");
235 1 : assert!(!child_dir.exists());
236 1 : create_dir_all(&child_dir).unwrap();
237 1 : assert!(child_dir.exists());
238 :
239 1 : let nested_child_dir = existing_dir_path.join("child1").join("child2");
240 1 : assert!(!nested_child_dir.exists());
241 1 : create_dir_all(&nested_child_dir).unwrap();
242 1 : assert!(nested_child_dir.exists());
243 :
244 1 : let file_path = existing_dir_path.join("file");
245 1 : std::fs::write(&file_path, b"").unwrap();
246 1 :
247 1 : let err = create_dir_all(&file_path).unwrap_err();
248 1 : assert_eq!(err.kind(), io::ErrorKind::AlreadyExists);
249 :
250 1 : let invalid_dir_path = file_path.join("folder");
251 1 : create_dir_all(invalid_dir_path).unwrap_err();
252 1 : }
253 :
254 : #[test]
255 1 : fn test_path_with_suffix_extension() {
256 1 : let p = Utf8PathBuf::from("/foo/bar");
257 1 : assert_eq!(
258 1 : &path_with_suffix_extension(p, "temp").to_string(),
259 1 : "/foo/bar.temp"
260 1 : );
261 1 : let p = Utf8PathBuf::from("/foo/bar");
262 1 : assert_eq!(
263 1 : &path_with_suffix_extension(p, "temp.temp").to_string(),
264 1 : "/foo/bar.temp.temp"
265 1 : );
266 1 : let p = Utf8PathBuf::from("/foo/bar.baz");
267 1 : assert_eq!(
268 1 : &path_with_suffix_extension(p, "temp.temp").to_string(),
269 1 : "/foo/bar.baz.temp.temp"
270 1 : );
271 1 : let p = Utf8PathBuf::from("/foo/bar.baz");
272 1 : assert_eq!(
273 1 : &path_with_suffix_extension(p, ".temp").to_string(),
274 1 : "/foo/bar.baz..temp"
275 1 : );
276 1 : let p = Utf8PathBuf::from("/foo/bar/dir/");
277 1 : assert_eq!(
278 1 : &path_with_suffix_extension(p, ".temp").to_string(),
279 1 : "/foo/bar/dir..temp"
280 1 : );
281 1 : }
282 : }
|
