Line data Source code
1 : use std::{
2 : borrow::Cow,
3 : fs::{self, File},
4 : io,
5 : };
6 :
7 : use camino::{Utf8Path, Utf8PathBuf};
8 :
9 : /// Similar to [`std::fs::create_dir`], except we fsync the
10 : /// created directory and its parent.
11 1151 : pub fn create_dir(path: impl AsRef<Utf8Path>) -> io::Result<()> {
12 1151 : let path = path.as_ref();
13 1151 :
14 1151 : fs::create_dir(path)?;
15 1147 : fsync_file_and_parent(path)?;
16 1147 : Ok(())
17 1151 : }
18 :
19 : /// Similar to [`std::fs::create_dir_all`], except we fsync all
20 : /// newly created directories and the pre-existing parent.
21 408 : pub fn create_dir_all(path: impl AsRef<Utf8Path>) -> io::Result<()> {
22 408 : let mut path = path.as_ref();
23 408 :
24 408 : let mut dirs_to_create = Vec::new();
25 :
26 : // Figure out which directories we need to create.
27 : loop {
28 812 : match path.metadata() {
29 406 : Ok(metadata) if metadata.is_dir() => break,
30 : Ok(_) => {
31 2 : return Err(io::Error::new(
32 2 : io::ErrorKind::AlreadyExists,
33 2 : format!("non-directory found in path: {path}"),
34 2 : ));
35 : }
36 406 : Err(ref e) if e.kind() == io::ErrorKind::NotFound => {}
37 2 : Err(e) => return Err(e),
38 : }
39 :
40 404 : dirs_to_create.push(path);
41 404 :
42 404 : match path.parent() {
43 404 : Some(parent) => path = parent,
44 : None => {
45 0 : return Err(io::Error::new(
46 0 : io::ErrorKind::InvalidInput,
47 0 : format!("can't find parent of path '{path}'"),
48 0 : ));
49 : }
50 : }
51 : }
52 :
53 : // Create directories from parent to child.
54 404 : for &path in dirs_to_create.iter().rev() {
55 404 : fs::create_dir(path)?;
56 : }
57 :
58 : // Fsync the created directories from child to parent.
59 404 : for &path in dirs_to_create.iter() {
60 404 : fsync(path)?;
61 : }
62 :
63 : // If we created any new directories, fsync the parent.
64 404 : if !dirs_to_create.is_empty() {
65 402 : fsync(path)?;
66 2 : }
67 :
68 404 : Ok(())
69 408 : }
70 :
71 : /// Adds a suffix to the file(directory) name, either appending the suffix to the end of its extension,
72 : /// or if there's no extension, creates one and puts a suffix there.
73 35493 : pub fn path_with_suffix_extension(
74 35493 : original_path: impl AsRef<Utf8Path>,
75 35493 : suffix: &str,
76 35493 : ) -> Utf8PathBuf {
77 35493 : let new_extension = match original_path.as_ref().extension() {
78 4694 : Some(extension) => Cow::Owned(format!("{extension}.{suffix}")),
79 30799 : None => Cow::Borrowed(suffix),
80 : };
81 35493 : original_path.as_ref().with_extension(new_extension)
82 35493 : }
83 :
84 3543 : pub fn fsync_file_and_parent(file_path: &Utf8Path) -> io::Result<()> {
85 3543 : let parent = file_path.parent().ok_or_else(|| {
86 0 : io::Error::new(
87 0 : io::ErrorKind::Other,
88 0 : format!("File {file_path:?} has no parent"),
89 0 : )
90 3543 : })?;
91 :
92 3543 : fsync(file_path)?;
93 3543 : fsync(parent)?;
94 3543 : Ok(())
95 3543 : }
96 :
97 9055 : pub fn fsync(path: &Utf8Path) -> io::Result<()> {
98 9055 : File::open(path)
99 9055 : .map_err(|e| io::Error::new(e.kind(), format!("Failed to open the file {path:?}: {e}")))
100 9055 : .and_then(|file| {
101 9055 : file.sync_all().map_err(|e| {
102 0 : io::Error::new(
103 0 : e.kind(),
104 0 : format!("Failed to sync file {path:?} data and metadata: {e}"),
105 0 : )
106 9055 : })
107 9055 : })
108 9055 : .map_err(|e| io::Error::new(e.kind(), format!("Failed to fsync file {path:?}: {e}")))
109 9055 : }
110 :
111 11137 : pub async fn fsync_async(path: impl AsRef<Utf8Path>) -> Result<(), std::io::Error> {
112 11137 : tokio::fs::File::open(path.as_ref()).await?.sync_all().await
113 11137 : }
114 :
115 18753 : pub async fn fsync_async_opt(
116 18753 : path: impl AsRef<Utf8Path>,
117 18753 : do_fsync: bool,
118 18753 : ) -> Result<(), std::io::Error> {
119 18753 : if do_fsync {
120 21 : fsync_async(path.as_ref()).await?;
121 18741 : }
122 18753 : Ok(())
123 18753 : }
124 :
125 : /// Like postgres' durable_rename, renames file issuing fsyncs do make it
126 : /// durable. After return, file and rename are guaranteed to be persisted.
127 : ///
128 : /// Unlike postgres, it only does fsyncs to 1) file to be renamed to make
129 : /// contents durable; 2) its directory entry to make rename durable 3) again to
130 : /// already renamed file, which is not required by standards but postgres does
131 : /// it, let's stick to that. Postgres additionally fsyncs newpath *before*
132 : /// rename if it exists to ensure that at least one of the files survives, but
133 : /// current callers don't need that.
134 : ///
135 : /// virtual_file.rs has similar code, but it doesn't use vfs.
136 : ///
137 : /// Useful links: <https://lwn.net/Articles/457667/>
138 : /// <https://www.postgresql.org/message-id/flat/56583BDD.9060302%402ndquadrant.com>
139 : /// <https://thunk.org/tytso/blog/2009/03/15/dont-fear-the-fsync/>
140 6251 : pub async fn durable_rename(
141 6251 : old_path: impl AsRef<Utf8Path>,
142 6251 : new_path: impl AsRef<Utf8Path>,
143 6251 : do_fsync: bool,
144 6251 : ) -> io::Result<()> {
145 6251 : // first fsync the file
146 6251 : fsync_async_opt(old_path.as_ref(), do_fsync).await?;
147 :
148 : // Time to do the real deal.
149 6251 : tokio::fs::rename(old_path.as_ref(), new_path.as_ref()).await?;
150 :
151 : // Postgres'ish fsync of renamed file.
152 6251 : fsync_async_opt(new_path.as_ref(), do_fsync).await?;
153 :
154 : // Now fsync the parent
155 6251 : let parent = match new_path.as_ref().parent() {
156 6251 : Some(p) => p,
157 0 : None => Utf8Path::new("./"), // assume current dir if there is no parent
158 : };
159 6251 : fsync_async_opt(parent, do_fsync).await?;
160 :
161 6251 : Ok(())
162 6251 : }
163 :
164 : #[cfg(test)]
165 : mod tests {
166 :
167 : use super::*;
168 :
169 2 : #[test]
170 2 : fn test_create_dir_fsyncd() {
171 2 : let dir = camino_tempfile::tempdir().unwrap();
172 2 :
173 2 : let existing_dir_path = dir.path();
174 2 : let err = create_dir(existing_dir_path).unwrap_err();
175 2 : assert_eq!(err.kind(), io::ErrorKind::AlreadyExists);
176 :
177 2 : let child_dir = existing_dir_path.join("child");
178 2 : create_dir(child_dir).unwrap();
179 2 :
180 2 : let nested_child_dir = existing_dir_path.join("child1").join("child2");
181 2 : let err = create_dir(nested_child_dir).unwrap_err();
182 2 : assert_eq!(err.kind(), io::ErrorKind::NotFound);
183 2 : }
184 :
185 2 : #[test]
186 2 : fn test_create_dir_all_fsyncd() {
187 2 : let dir = camino_tempfile::tempdir().unwrap();
188 2 :
189 2 : let existing_dir_path = dir.path();
190 2 : create_dir_all(existing_dir_path).unwrap();
191 2 :
192 2 : let child_dir = existing_dir_path.join("child");
193 2 : assert!(!child_dir.exists());
194 2 : create_dir_all(&child_dir).unwrap();
195 2 : assert!(child_dir.exists());
196 :
197 2 : let nested_child_dir = existing_dir_path.join("child1").join("child2");
198 2 : assert!(!nested_child_dir.exists());
199 2 : create_dir_all(&nested_child_dir).unwrap();
200 2 : assert!(nested_child_dir.exists());
201 :
202 2 : let file_path = existing_dir_path.join("file");
203 2 : std::fs::write(&file_path, b"").unwrap();
204 2 :
205 2 : let err = create_dir_all(&file_path).unwrap_err();
206 2 : assert_eq!(err.kind(), io::ErrorKind::AlreadyExists);
207 :
208 2 : let invalid_dir_path = file_path.join("folder");
209 2 : create_dir_all(invalid_dir_path).unwrap_err();
210 2 : }
211 :
212 2 : #[test]
213 2 : fn test_path_with_suffix_extension() {
214 2 : let p = Utf8PathBuf::from("/foo/bar");
215 2 : assert_eq!(
216 2 : &path_with_suffix_extension(p, "temp").to_string(),
217 2 : "/foo/bar.temp"
218 2 : );
219 2 : let p = Utf8PathBuf::from("/foo/bar");
220 2 : assert_eq!(
221 2 : &path_with_suffix_extension(p, "temp.temp").to_string(),
222 2 : "/foo/bar.temp.temp"
223 2 : );
224 2 : let p = Utf8PathBuf::from("/foo/bar.baz");
225 2 : assert_eq!(
226 2 : &path_with_suffix_extension(p, "temp.temp").to_string(),
227 2 : "/foo/bar.baz.temp.temp"
228 2 : );
229 2 : let p = Utf8PathBuf::from("/foo/bar.baz");
230 2 : assert_eq!(
231 2 : &path_with_suffix_extension(p, ".temp").to_string(),
232 2 : "/foo/bar.baz..temp"
233 2 : );
234 2 : let p = Utf8PathBuf::from("/foo/bar/dir/");
235 2 : assert_eq!(
236 2 : &path_with_suffix_extension(p, ".temp").to_string(),
237 2 : "/foo/bar/dir..temp"
238 2 : );
239 2 : }
240 : }
|
