Line data Source code
1 : use std::os::fd::AsRawFd;
2 : use std::{
3 : borrow::Cow,
4 : fs::{self, File},
5 : io::{self, Write},
6 : };
7 :
8 : use camino::{Utf8Path, Utf8PathBuf};
9 :
10 : /// Similar to [`std::fs::create_dir`], except we fsync the
11 : /// created directory and its parent.
12 1221 : pub fn create_dir(path: impl AsRef<Utf8Path>) -> io::Result<()> {
13 1221 : let path = path.as_ref();
14 1221 :
15 1221 : fs::create_dir(path)?;
16 1219 : fsync_file_and_parent(path)?;
17 1219 : Ok(())
18 1221 : }
19 :
20 : /// Similar to [`std::fs::create_dir_all`], except we fsync all
21 : /// newly created directories and the pre-existing parent.
22 5 : pub fn create_dir_all(path: impl AsRef<Utf8Path>) -> io::Result<()> {
23 5 : let mut path = path.as_ref();
24 5 :
25 5 : let mut dirs_to_create = Vec::new();
26 :
27 : // Figure out which directories we need to create.
28 : loop {
29 8 : match path.metadata() {
30 4 : Ok(metadata) if metadata.is_dir() => break,
31 : Ok(_) => {
32 1 : return Err(io::Error::new(
33 1 : io::ErrorKind::AlreadyExists,
34 1 : format!("non-directory found in path: {path}"),
35 1 : ));
36 : }
37 4 : Err(ref e) if e.kind() == io::ErrorKind::NotFound => {}
38 1 : Err(e) => return Err(e),
39 : }
40 :
41 3 : dirs_to_create.push(path);
42 3 :
43 3 : match path.parent() {
44 3 : Some(parent) => path = parent,
45 : None => {
46 0 : return Err(io::Error::new(
47 0 : io::ErrorKind::InvalidInput,
48 0 : format!("can't find parent of path '{path}'"),
49 0 : ));
50 : }
51 : }
52 : }
53 :
54 : // Create directories from parent to child.
55 3 : for &path in dirs_to_create.iter().rev() {
56 3 : fs::create_dir(path)?;
57 : }
58 :
59 : // Fsync the created directories from child to parent.
60 3 : for &path in dirs_to_create.iter() {
61 3 : fsync(path)?;
62 : }
63 :
64 : // If we created any new directories, fsync the parent.
65 3 : if !dirs_to_create.is_empty() {
66 2 : fsync(path)?;
67 1 : }
68 :
69 3 : Ok(())
70 5 : }
71 :
72 : /// Adds a suffix to the file(directory) name, either appending the suffix to the end of its extension,
73 : /// or if there's no extension, creates one and puts a suffix there.
74 8498 : pub fn path_with_suffix_extension(
75 8498 : original_path: impl AsRef<Utf8Path>,
76 8498 : suffix: &str,
77 8498 : ) -> Utf8PathBuf {
78 8498 : let new_extension = match original_path.as_ref().extension() {
79 4360 : Some(extension) => Cow::Owned(format!("{extension}.{suffix}")),
80 4138 : None => Cow::Borrowed(suffix),
81 : };
82 8498 : original_path.as_ref().with_extension(new_extension)
83 8498 : }
84 :
85 1219 : pub fn fsync_file_and_parent(file_path: &Utf8Path) -> io::Result<()> {
86 1219 : let parent = file_path.parent().ok_or_else(|| {
87 0 : io::Error::new(
88 0 : io::ErrorKind::Other,
89 0 : format!("File {file_path:?} has no parent"),
90 0 : )
91 1219 : })?;
92 :
93 1219 : fsync(file_path)?;
94 1219 : fsync(parent)?;
95 1219 : Ok(())
96 1219 : }
97 :
98 2443 : pub fn fsync(path: &Utf8Path) -> io::Result<()> {
99 2443 : File::open(path)
100 2443 : .map_err(|e| io::Error::new(e.kind(), format!("Failed to open the file {path:?}: {e}")))
101 2443 : .and_then(|file| {
102 2443 : file.sync_all().map_err(|e| {
103 0 : io::Error::new(
104 0 : e.kind(),
105 0 : format!("Failed to sync file {path:?} data and metadata: {e}"),
106 0 : )
107 2443 : })
108 2443 : })
109 2443 : .map_err(|e| io::Error::new(e.kind(), format!("Failed to fsync file {path:?}: {e}")))
110 2443 : }
111 :
112 6 : pub async fn fsync_async(path: impl AsRef<Utf8Path>) -> Result<(), std::io::Error> {
113 6 : tokio::fs::File::open(path.as_ref()).await?.sync_all().await
114 6 : }
115 :
116 6 : pub async fn fsync_async_opt(
117 6 : path: impl AsRef<Utf8Path>,
118 6 : do_fsync: bool,
119 6 : ) -> Result<(), std::io::Error> {
120 6 : if do_fsync {
121 12 : fsync_async(path.as_ref()).await?;
122 0 : }
123 6 : Ok(())
124 6 : }
125 :
126 : /// Like postgres' durable_rename, renames file issuing fsyncs do make it
127 : /// durable. After return, file and rename are guaranteed to be persisted.
128 : ///
129 : /// Unlike postgres, it only does fsyncs to 1) file to be renamed to make
130 : /// contents durable; 2) its directory entry to make rename durable 3) again to
131 : /// already renamed file, which is not required by standards but postgres does
132 : /// it, let's stick to that. Postgres additionally fsyncs newpath *before*
133 : /// rename if it exists to ensure that at least one of the files survives, but
134 : /// current callers don't need that.
135 : ///
136 : /// virtual_file.rs has similar code, but it doesn't use vfs.
137 : ///
138 : /// Useful links: <https://lwn.net/Articles/457667/>
139 : /// <https://www.postgresql.org/message-id/flat/56583BDD.9060302%402ndquadrant.com>
140 : /// <https://thunk.org/tytso/blog/2009/03/15/dont-fear-the-fsync/>
141 2 : pub async fn durable_rename(
142 2 : old_path: impl AsRef<Utf8Path>,
143 2 : new_path: impl AsRef<Utf8Path>,
144 2 : do_fsync: bool,
145 2 : ) -> io::Result<()> {
146 2 : // first fsync the file
147 4 : fsync_async_opt(old_path.as_ref(), do_fsync).await?;
148 :
149 : // Time to do the real deal.
150 2 : tokio::fs::rename(old_path.as_ref(), new_path.as_ref()).await?;
151 :
152 : // Postgres'ish fsync of renamed file.
153 4 : fsync_async_opt(new_path.as_ref(), do_fsync).await?;
154 :
155 : // Now fsync the parent
156 2 : let parent = match new_path.as_ref().parent() {
157 2 : Some(p) => p,
158 0 : None => Utf8Path::new("./"), // assume current dir if there is no parent
159 : };
160 4 : fsync_async_opt(parent, do_fsync).await?;
161 :
162 2 : Ok(())
163 2 : }
164 :
165 : /// Writes a file to the specified `final_path` in a crash safe fasion, using [`std::fs`].
166 : ///
167 : /// The file is first written to the specified `tmp_path`, and in a second
168 : /// step, the `tmp_path` is renamed to the `final_path`. Intermediary fsync
169 : /// and atomic rename guarantee that, if we crash at any point, there will never
170 : /// be a partially written file at `final_path` (but maybe at `tmp_path`).
171 : ///
172 : /// Callers are responsible for serializing calls of this function for a given `final_path`.
173 : /// If they don't, there may be an error due to conflicting `tmp_path`, or there will
174 : /// be no error and the content of `final_path` will be the "winner" caller's `content`.
175 : /// I.e., the atomticity guarantees still hold.
176 84 : pub fn overwrite(
177 84 : final_path: &Utf8Path,
178 84 : tmp_path: &Utf8Path,
179 84 : content: &[u8],
180 84 : ) -> std::io::Result<()> {
181 84 : let Some(final_path_parent) = final_path.parent() else {
182 0 : return Err(std::io::Error::from_raw_os_error(
183 0 : nix::errno::Errno::EINVAL as i32,
184 0 : ));
185 : };
186 84 : std::fs::remove_file(tmp_path).or_else(crate::fs_ext::ignore_not_found)?;
187 84 : let mut file = std::fs::OpenOptions::new()
188 84 : .write(true)
189 84 : // Use `create_new` so that, if we race with ourselves or something else,
190 84 : // we bail out instead of causing damage.
191 84 : .create_new(true)
192 84 : .open(tmp_path)?;
193 84 : file.write_all(content)?;
194 84 : file.sync_all()?;
195 84 : drop(file); // don't keep the fd open for longer than we have to
196 84 :
197 84 : std::fs::rename(tmp_path, final_path)?;
198 :
199 84 : let final_parent_dirfd = std::fs::OpenOptions::new()
200 84 : .read(true)
201 84 : .open(final_path_parent)?;
202 :
203 84 : final_parent_dirfd.sync_all()?;
204 84 : Ok(())
205 84 : }
206 :
207 : /// Syncs the filesystem for the given file descriptor.
208 : #[cfg_attr(target_os = "macos", allow(unused_variables))]
209 0 : pub fn syncfs(fd: impl AsRawFd) -> anyhow::Result<()> {
210 : // Linux guarantees durability for syncfs.
211 : // POSIX doesn't have syncfs, and further does not actually guarantee durability of sync().
212 : #[cfg(target_os = "linux")]
213 0 : {
214 0 : use anyhow::Context;
215 0 : nix::unistd::syncfs(fd.as_raw_fd()).context("syncfs")?;
216 : }
217 : #[cfg(target_os = "macos")]
218 : {
219 : // macOS is not a production platform for Neon, don't even bother.
220 : }
221 : #[cfg(not(any(target_os = "linux", target_os = "macos")))]
222 : {
223 : compile_error!("Unsupported OS");
224 : }
225 0 : Ok(())
226 0 : }
227 :
228 : #[cfg(test)]
229 : mod tests {
230 :
231 : use super::*;
232 :
233 : #[test]
234 1 : fn test_create_dir_fsyncd() {
235 1 : let dir = camino_tempfile::tempdir().unwrap();
236 1 :
237 1 : let existing_dir_path = dir.path();
238 1 : let err = create_dir(existing_dir_path).unwrap_err();
239 1 : assert_eq!(err.kind(), io::ErrorKind::AlreadyExists);
240 :
241 1 : let child_dir = existing_dir_path.join("child");
242 1 : create_dir(child_dir).unwrap();
243 1 :
244 1 : let nested_child_dir = existing_dir_path.join("child1").join("child2");
245 1 : let err = create_dir(nested_child_dir).unwrap_err();
246 1 : assert_eq!(err.kind(), io::ErrorKind::NotFound);
247 1 : }
248 :
249 : #[test]
250 1 : fn test_create_dir_all_fsyncd() {
251 1 : let dir = camino_tempfile::tempdir().unwrap();
252 1 :
253 1 : let existing_dir_path = dir.path();
254 1 : create_dir_all(existing_dir_path).unwrap();
255 1 :
256 1 : let child_dir = existing_dir_path.join("child");
257 1 : assert!(!child_dir.exists());
258 1 : create_dir_all(&child_dir).unwrap();
259 1 : assert!(child_dir.exists());
260 :
261 1 : let nested_child_dir = existing_dir_path.join("child1").join("child2");
262 1 : assert!(!nested_child_dir.exists());
263 1 : create_dir_all(&nested_child_dir).unwrap();
264 1 : assert!(nested_child_dir.exists());
265 :
266 1 : let file_path = existing_dir_path.join("file");
267 1 : std::fs::write(&file_path, b"").unwrap();
268 1 :
269 1 : let err = create_dir_all(&file_path).unwrap_err();
270 1 : assert_eq!(err.kind(), io::ErrorKind::AlreadyExists);
271 :
272 1 : let invalid_dir_path = file_path.join("folder");
273 1 : create_dir_all(invalid_dir_path).unwrap_err();
274 1 : }
275 :
276 : #[test]
277 1 : fn test_path_with_suffix_extension() {
278 1 : let p = Utf8PathBuf::from("/foo/bar");
279 1 : assert_eq!(
280 1 : &path_with_suffix_extension(p, "temp").to_string(),
281 1 : "/foo/bar.temp"
282 1 : );
283 1 : let p = Utf8PathBuf::from("/foo/bar");
284 1 : assert_eq!(
285 1 : &path_with_suffix_extension(p, "temp.temp").to_string(),
286 1 : "/foo/bar.temp.temp"
287 1 : );
288 1 : let p = Utf8PathBuf::from("/foo/bar.baz");
289 1 : assert_eq!(
290 1 : &path_with_suffix_extension(p, "temp.temp").to_string(),
291 1 : "/foo/bar.baz.temp.temp"
292 1 : );
293 1 : let p = Utf8PathBuf::from("/foo/bar.baz");
294 1 : assert_eq!(
295 1 : &path_with_suffix_extension(p, ".temp").to_string(),
296 1 : "/foo/bar.baz..temp"
297 1 : );
298 1 : let p = Utf8PathBuf::from("/foo/bar/dir/");
299 1 : assert_eq!(
300 1 : &path_with_suffix_extension(p, ".temp").to_string(),
301 1 : "/foo/bar/dir..temp"
302 1 : );
303 1 : }
304 : }
|
