Line data Source code
1 : use std::{
2 : borrow::Cow,
3 : fs::{self, File},
4 : io::{self, Write},
5 : };
6 :
7 : use camino::{Utf8Path, Utf8PathBuf};
8 :
9 : /// Similar to [`std::fs::create_dir`], except we fsync the
10 : /// created directory and its parent.
11 1236 : pub fn create_dir(path: impl AsRef<Utf8Path>) -> io::Result<()> {
12 1236 : let path = path.as_ref();
13 1236 :
14 1236 : fs::create_dir(path)?;
15 1224 : fsync_file_and_parent(path)?;
16 1224 : Ok(())
17 1236 : }
18 :
19 : /// Similar to [`std::fs::create_dir_all`], except we fsync all
20 : /// newly created directories and the pre-existing parent.
21 30 : pub fn create_dir_all(path: impl AsRef<Utf8Path>) -> io::Result<()> {
22 30 : let mut path = path.as_ref();
23 30 :
24 30 : let mut dirs_to_create = Vec::new();
25 :
26 : // Figure out which directories we need to create.
27 : loop {
28 48 : match path.metadata() {
29 24 : Ok(metadata) if metadata.is_dir() => break,
30 : Ok(_) => {
31 6 : return Err(io::Error::new(
32 6 : io::ErrorKind::AlreadyExists,
33 6 : format!("non-directory found in path: {path}"),
34 6 : ));
35 : }
36 24 : Err(ref e) if e.kind() == io::ErrorKind::NotFound => {}
37 6 : Err(e) => return Err(e),
38 : }
39 :
40 18 : dirs_to_create.push(path);
41 18 :
42 18 : match path.parent() {
43 18 : Some(parent) => path = parent,
44 : None => {
45 0 : return Err(io::Error::new(
46 0 : io::ErrorKind::InvalidInput,
47 0 : format!("can't find parent of path '{path}'"),
48 0 : ));
49 : }
50 : }
51 : }
52 :
53 : // Create directories from parent to child.
54 18 : for &path in dirs_to_create.iter().rev() {
55 18 : fs::create_dir(path)?;
56 : }
57 :
58 : // Fsync the created directories from child to parent.
59 18 : for &path in dirs_to_create.iter() {
60 18 : fsync(path)?;
61 : }
62 :
63 : // If we created any new directories, fsync the parent.
64 18 : if !dirs_to_create.is_empty() {
65 12 : fsync(path)?;
66 6 : }
67 :
68 18 : Ok(())
69 30 : }
70 :
71 : /// Adds a suffix to the file(directory) name, either appending the suffix to the end of its extension,
72 : /// or if there's no extension, creates one and puts a suffix there.
73 8869 : pub fn path_with_suffix_extension(
74 8869 : original_path: impl AsRef<Utf8Path>,
75 8869 : suffix: &str,
76 8869 : ) -> Utf8PathBuf {
77 8869 : let new_extension = match original_path.as_ref().extension() {
78 4539 : Some(extension) => Cow::Owned(format!("{extension}.{suffix}")),
79 4330 : None => Cow::Borrowed(suffix),
80 : };
81 8869 : original_path.as_ref().with_extension(new_extension)
82 8869 : }
83 :
84 1224 : pub fn fsync_file_and_parent(file_path: &Utf8Path) -> io::Result<()> {
85 1224 : let parent = file_path.parent().ok_or_else(|| {
86 0 : io::Error::new(
87 0 : io::ErrorKind::Other,
88 0 : format!("File {file_path:?} has no parent"),
89 0 : )
90 1224 : })?;
91 :
92 1224 : fsync(file_path)?;
93 1224 : fsync(parent)?;
94 1224 : Ok(())
95 1224 : }
96 :
97 2478 : pub fn fsync(path: &Utf8Path) -> io::Result<()> {
98 2478 : File::open(path)
99 2478 : .map_err(|e| io::Error::new(e.kind(), format!("Failed to open the file {path:?}: {e}")))
100 2478 : .and_then(|file| {
101 2478 : file.sync_all().map_err(|e| {
102 0 : io::Error::new(
103 0 : e.kind(),
104 0 : format!("Failed to sync file {path:?} data and metadata: {e}"),
105 0 : )
106 2478 : })
107 2478 : })
108 2478 : .map_err(|e| io::Error::new(e.kind(), format!("Failed to fsync file {path:?}: {e}")))
109 2478 : }
110 :
111 36 : pub async fn fsync_async(path: impl AsRef<Utf8Path>) -> Result<(), std::io::Error> {
112 36 : tokio::fs::File::open(path.as_ref()).await?.sync_all().await
113 36 : }
114 :
115 36 : pub async fn fsync_async_opt(
116 36 : path: impl AsRef<Utf8Path>,
117 36 : do_fsync: bool,
118 36 : ) -> Result<(), std::io::Error> {
119 36 : if do_fsync {
120 65 : fsync_async(path.as_ref()).await?;
121 0 : }
122 36 : Ok(())
123 36 : }
124 :
125 : /// Like postgres' durable_rename, renames file issuing fsyncs do make it
126 : /// durable. After return, file and rename are guaranteed to be persisted.
127 : ///
128 : /// Unlike postgres, it only does fsyncs to 1) file to be renamed to make
129 : /// contents durable; 2) its directory entry to make rename durable 3) again to
130 : /// already renamed file, which is not required by standards but postgres does
131 : /// it, let's stick to that. Postgres additionally fsyncs newpath *before*
132 : /// rename if it exists to ensure that at least one of the files survives, but
133 : /// current callers don't need that.
134 : ///
135 : /// virtual_file.rs has similar code, but it doesn't use vfs.
136 : ///
137 : /// Useful links: <https://lwn.net/Articles/457667/>
138 : /// <https://www.postgresql.org/message-id/flat/56583BDD.9060302%402ndquadrant.com>
139 : /// <https://thunk.org/tytso/blog/2009/03/15/dont-fear-the-fsync/>
140 12 : pub async fn durable_rename(
141 12 : old_path: impl AsRef<Utf8Path>,
142 12 : new_path: impl AsRef<Utf8Path>,
143 12 : do_fsync: bool,
144 12 : ) -> io::Result<()> {
145 12 : // first fsync the file
146 22 : fsync_async_opt(old_path.as_ref(), do_fsync).await?;
147 :
148 : // Time to do the real deal.
149 12 : tokio::fs::rename(old_path.as_ref(), new_path.as_ref()).await?;
150 :
151 : // Postgres'ish fsync of renamed file.
152 22 : fsync_async_opt(new_path.as_ref(), do_fsync).await?;
153 :
154 : // Now fsync the parent
155 12 : let parent = match new_path.as_ref().parent() {
156 12 : Some(p) => p,
157 0 : None => Utf8Path::new("./"), // assume current dir if there is no parent
158 : };
159 21 : fsync_async_opt(parent, do_fsync).await?;
160 :
161 12 : Ok(())
162 12 : }
163 :
164 : /// Writes a file to the specified `final_path` in a crash safe fasion, using [`std::fs`].
165 : ///
166 : /// The file is first written to the specified `tmp_path`, and in a second
167 : /// step, the `tmp_path` is renamed to the `final_path`. Intermediary fsync
168 : /// and atomic rename guarantee that, if we crash at any point, there will never
169 : /// be a partially written file at `final_path` (but maybe at `tmp_path`).
170 : ///
171 : /// Callers are responsible for serializing calls of this function for a given `final_path`.
172 : /// If they don't, there may be an error due to conflicting `tmp_path`, or there will
173 : /// be no error and the content of `final_path` will be the "winner" caller's `content`.
174 : /// I.e., the atomticity guarantees still hold.
175 84 : pub fn overwrite(
176 84 : final_path: &Utf8Path,
177 84 : tmp_path: &Utf8Path,
178 84 : content: &[u8],
179 84 : ) -> std::io::Result<()> {
180 84 : let Some(final_path_parent) = final_path.parent() else {
181 0 : return Err(std::io::Error::from_raw_os_error(
182 0 : nix::errno::Errno::EINVAL as i32,
183 0 : ));
184 : };
185 84 : std::fs::remove_file(tmp_path).or_else(crate::fs_ext::ignore_not_found)?;
186 84 : let mut file = std::fs::OpenOptions::new()
187 84 : .write(true)
188 84 : // Use `create_new` so that, if we race with ourselves or something else,
189 84 : // we bail out instead of causing damage.
190 84 : .create_new(true)
191 84 : .open(tmp_path)?;
192 84 : file.write_all(content)?;
193 84 : file.sync_all()?;
194 84 : drop(file); // don't keep the fd open for longer than we have to
195 84 :
196 84 : std::fs::rename(tmp_path, final_path)?;
197 :
198 84 : let final_parent_dirfd = std::fs::OpenOptions::new()
199 84 : .read(true)
200 84 : .open(final_path_parent)?;
201 :
202 84 : final_parent_dirfd.sync_all()?;
203 84 : Ok(())
204 84 : }
205 :
206 : #[cfg(test)]
207 : mod tests {
208 :
209 : use super::*;
210 :
211 : #[test]
212 6 : fn test_create_dir_fsyncd() {
213 6 : let dir = camino_tempfile::tempdir().unwrap();
214 6 :
215 6 : let existing_dir_path = dir.path();
216 6 : let err = create_dir(existing_dir_path).unwrap_err();
217 6 : assert_eq!(err.kind(), io::ErrorKind::AlreadyExists);
218 :
219 6 : let child_dir = existing_dir_path.join("child");
220 6 : create_dir(child_dir).unwrap();
221 6 :
222 6 : let nested_child_dir = existing_dir_path.join("child1").join("child2");
223 6 : let err = create_dir(nested_child_dir).unwrap_err();
224 6 : assert_eq!(err.kind(), io::ErrorKind::NotFound);
225 6 : }
226 :
227 : #[test]
228 6 : fn test_create_dir_all_fsyncd() {
229 6 : let dir = camino_tempfile::tempdir().unwrap();
230 6 :
231 6 : let existing_dir_path = dir.path();
232 6 : create_dir_all(existing_dir_path).unwrap();
233 6 :
234 6 : let child_dir = existing_dir_path.join("child");
235 6 : assert!(!child_dir.exists());
236 6 : create_dir_all(&child_dir).unwrap();
237 6 : assert!(child_dir.exists());
238 :
239 6 : let nested_child_dir = existing_dir_path.join("child1").join("child2");
240 6 : assert!(!nested_child_dir.exists());
241 6 : create_dir_all(&nested_child_dir).unwrap();
242 6 : assert!(nested_child_dir.exists());
243 :
244 6 : let file_path = existing_dir_path.join("file");
245 6 : std::fs::write(&file_path, b"").unwrap();
246 6 :
247 6 : let err = create_dir_all(&file_path).unwrap_err();
248 6 : assert_eq!(err.kind(), io::ErrorKind::AlreadyExists);
249 :
250 6 : let invalid_dir_path = file_path.join("folder");
251 6 : create_dir_all(invalid_dir_path).unwrap_err();
252 6 : }
253 :
254 : #[test]
255 6 : fn test_path_with_suffix_extension() {
256 6 : let p = Utf8PathBuf::from("/foo/bar");
257 6 : assert_eq!(
258 6 : &path_with_suffix_extension(p, "temp").to_string(),
259 6 : "/foo/bar.temp"
260 6 : );
261 6 : let p = Utf8PathBuf::from("/foo/bar");
262 6 : assert_eq!(
263 6 : &path_with_suffix_extension(p, "temp.temp").to_string(),
264 6 : "/foo/bar.temp.temp"
265 6 : );
266 6 : let p = Utf8PathBuf::from("/foo/bar.baz");
267 6 : assert_eq!(
268 6 : &path_with_suffix_extension(p, "temp.temp").to_string(),
269 6 : "/foo/bar.baz.temp.temp"
270 6 : );
271 6 : let p = Utf8PathBuf::from("/foo/bar.baz");
272 6 : assert_eq!(
273 6 : &path_with_suffix_extension(p, ".temp").to_string(),
274 6 : "/foo/bar.baz..temp"
275 6 : );
276 6 : let p = Utf8PathBuf::from("/foo/bar/dir/");
277 6 : assert_eq!(
278 6 : &path_with_suffix_extension(p, ".temp").to_string(),
279 6 : "/foo/bar/dir..temp"
280 6 : );
281 6 : }
282 : }
|
