feat(cache): let a hit rebuild folders that hold only the task's outputs

CHANGELOG.md

@@ -1,5 +1,6 @@
 # Changelog
 
+- **Changed** A task that checks or lists a folder holding only files it wrote, like `tsc` checking its `dist` output folder, is now a cache hit when the folder is missing, as on a fresh clone, and the hit restores it. Previously the missing folder was a cache miss ([#914](https://github.com/voidzero-dev/vite-task/pull/914)).
 - **Changed** Tasks that write a file they also read are now cached, instead of being reported as `not cached because it modified its inputs`. For example, `tsc --incremental` reads and rewrites `tsconfig.tsbuildinfo`: the next run is a cache hit if the file is as `tsc` left it, or if it is missing, as on a fresh clone, in which case the hit restores it ([#913](https://github.com/voidzero-dev/vite-task/pull/913)).
 - **Fixed** A cache hit restores the symbolic links a task's `output` globs match as links, instead of dropping them, so a replayed output that links a directory, such as a bundler's `node_modules/<package>` link, is complete ([#885](https://github.com/voidzero-dev/vite-task/pull/885)).
 - **Fixed** On Linux and macOS, programs using automatic file-access tracking no longer abort when they call `execve`, `execv`, `execl`, `execle`, or `posix_spawn` with an empty path; the call fails with `ENOENT`, as it does without tracking. On Linux, `execveat` with an empty path and without `AT_EMPTY_PATH` now also fails with `ENOENT`, instead of trying to run what its directory descriptor refers to ([#896](https://github.com/voidzero-dev/vite-task/pull/896)).

crates/vt/docs/task-cache.md

@@ -197,6 +197,7 @@ Vite Task uses `fspy` to monitor file system access during task execution:
 │    • Check if file exists                                    │
 │    • If file: Hash contents with xxHash3                     │
 │    • If directory: Record structure                          │
+│      (skipped if it holds only files the task wrote)         │
 │    • If missing: Mark as NotFound                            │
 │         │                                                    │
 │         ▼                                                    │

crates/vt/src/session/execute/cache_update.rs

@@ -33,8 +33,8 @@ struct TrackingOutcome {
     /// Auto-output writes after output exclusions are applied. Empty when
     /// `output_config.includes_auto` is false.
     path_writes: FxHashSet<RelativePathBuf>,
-    /// Every write fspy recorded, before output exclusions and `ignoreOutput`
-    /// paths are applied.
+    /// What the task can rebuild, from every write fspy recorded before output
+    /// exclusions and `ignoreOutput` paths are applied.
     rebuildable: Rebuildable,
 }
 
@@ -254,7 +254,7 @@ fn observe_fspy(
             TrackingOutcome {
                 path_reads: filtered_path_reads,
                 path_writes: filtered_path_writes,
-                rebuildable: Rebuildable::new(tracked.path_writes),
+                rebuildable: Rebuildable::new(tracked.path_writes, workspace_root),
             }
         })
     }

crates/vt/src/session/execute/fingerprint.rs

@@ -117,8 +117,10 @@ impl PostRunFingerprint {
     /// Negative glob filtering is done upstream (see
     /// [`super::tracked_accesses::TrackedPathAccesses::from_raw`]).
     /// Paths already present in `globbed_inputs` are skipped — they are
-    /// already tracked by the prerun glob fingerprint. Every other path is
-    /// fingerprinted as the task left it, including a file the task rewrote.
+    /// already tracked by the prerun glob fingerprint. Folders holding only
+    /// what the task can rebuild are skipped too: a hit restores the files in
+    /// them. Every other path is fingerprinted as the task left it, including
+    /// a file the task rewrote.
     ///
     /// # Arguments
     /// * `inferred_path_reads` - Map of paths that were read during execution (from fspy)
@@ -138,7 +140,9 @@ impl PostRunFingerprint {
     ) -> anyhow::Result<Self> {
         let inferred_inputs = inferred_path_reads
             .par_iter()
-            .filter(|(path, _)| !globbed_inputs.contains_key(*path))
+            .filter(|(path, _)| {
+                !globbed_inputs.contains_key(*path) && !rebuildable.is_rebuildable_folder(path)
+            })
             .map(|(relative_path, path_read)| {
                 let full_path = Arc::<AbsolutePath>::from(base_dir.join(relative_path));
                 let fingerprint = fingerprint_path(&full_path, *path_read)?;
@@ -681,7 +685,7 @@ mod tests {
         let create = |written| {
             PostRunFingerprint::create(
                 &path_reads,
-                &Rebuildable::new(written),
+                &Rebuildable::new(written, &workspace_root),
                 &workspace_root,
                 &BTreeMap::new(),
                 BTreeMap::new(),
@@ -708,4 +712,55 @@ mod tests {
             Some(PostRunMismatch::Input { kind: InputChangeKind::ContentModified, .. })
         ));
     }
+
+    #[test]
+    fn create_skips_folders_holding_only_written_files() {
+        let tmp = tempfile::TempDir::new().unwrap();
+        let workspace_root = AbsolutePathBuf::new(tmp.path().to_path_buf()).unwrap();
+        let rel = |path: &str| RelativePathBuf::new(path).unwrap();
+        for dir in ["out/sub", "mixed", "holds_empty/empty"] {
+            std::fs::create_dir_all(workspace_root.join(dir).as_path()).unwrap();
+        }
+        let written = ["out/a.txt", "out/sub/b.txt", "mixed/a.txt", "holds_empty/a.txt"];
+        for file in written.iter().copied().chain(["out/.DS_Store", "mixed/notes.txt"]) {
+            std::fs::write(workspace_root.join(file).as_path(), "x").unwrap();
+        }
+
+        let path_reads = ["out", "out/sub", "mixed", "holds_empty"]
+            .into_iter()
+            .map(|dir| (rel(dir), PathRead { read_dir_entries: true }))
+            .collect();
+        let fingerprint = PostRunFingerprint::create(
+            &path_reads,
+            &Rebuildable::new(written.into_iter().map(rel).collect(), &workspace_root),
+            &workspace_root,
+            &BTreeMap::new(),
+            BTreeMap::new(),
+            BTreeMap::new(),
+        )
+        .unwrap();
+
+        let mut fingerprinted: Vec<&str> =
+            fingerprint.inferred_inputs.keys().map(|path| path.as_str()).collect();
+        fingerprinted.sort_unstable();
+        assert_eq!(fingerprinted, ["holds_empty", "mixed"]);
+    }
+
+    #[test]
+    fn folders_gone_after_the_run_play_no_part() {
+        let tmp = tempfile::TempDir::new().unwrap();
+        let workspace_root = AbsolutePathBuf::new(tmp.path().to_path_buf()).unwrap();
+        let rel = |path: &str| RelativePathBuf::new(path).unwrap();
+        std::fs::create_dir(workspace_root.join("out").as_path()).unwrap();
+        std::fs::write(workspace_root.join("out/a.txt").as_path(), "x").unwrap();
+
+        // `out/tmp` and the file the task wrote in it are gone after the run,
+        // like Astro's `dist/.prerender`.
+        let rebuildable = Rebuildable::new(
+            ["out/a.txt", "out/tmp/b.txt"].into_iter().map(rel).collect(),
+            &workspace_root,
+        );
+        assert!(rebuildable.is_rebuildable_folder(&rel("out")));
+        assert!(!rebuildable.is_rebuildable_folder(&rel("out/tmp")));
+    }
 }

crates/vt/src/session/execute/rebuildable.rs

@@ -6,26 +6,140 @@
 //! would write them again, so a later run may find them missing. A file that
 //! is there must still be as the task left it.
 //!
+//! The same goes for a folder that holds only what the task can rebuild:
+//! every file (or symlink) in it is one the task wrote, and every folder in
+//! it holds only what the task can rebuild too. A hit restores the files,
+//! which recreates the folder, so it isn't fingerprinted at all. Such folders
+//! are found after the run by climbing from the written paths through their
+//! parent folders, listing each one in the real file system. A written path
+//! or folder that is gone by then plays no part.
+//!
 //! The written paths are fspy's records before output exclusions and
 //! tool-reported `ignoreOutput` paths are applied: those decide what is
 //! archived, not what the task can rebuild.
 
+use std::{cmp::Reverse, fs, io};
+
 use rustc_hash::FxHashSet;
-use vt_path::{RelativePath, RelativePathBuf};
+use vt_path::{AbsolutePath, RelativePath, RelativePathBuf};
 
-/// The paths a task wrote, relative to the workspace root.
+/// The paths a task wrote and the folders holding only what it can rebuild,
+/// relative to the workspace root.
 #[derive(Debug, Default)]
 pub struct Rebuildable {
     written: FxHashSet<RelativePathBuf>,
+    folders: FxHashSet<RelativePathBuf>,
 }
 
 impl Rebuildable {
-    pub const fn new(written: FxHashSet<RelativePathBuf>) -> Self {
-        Self { written }
+    /// Finds the rebuildable folders above the `written` paths by listing them
+    /// under `workspace_root`, which must be done after the run.
+    pub fn new(written: FxHashSet<RelativePathBuf>, workspace_root: &AbsolutePath) -> Self {
+        let folders = find_folders(&written, workspace_root);
+        Self { written, folders }
     }
 
     /// Whether a later run may find `path` missing: the task wrote it.
     pub fn may_be_missing(&self, path: &RelativePath) -> bool {
         self.written.contains(path)
     }
+
+    /// Whether `path` is a folder holding only what the task can rebuild.
+    pub fn is_rebuildable_folder(&self, path: &RelativePath) -> bool {
+        self.folders.contains(path)
+    }
+}
+
+/// Finds the folders holding only what the task can rebuild. Each folder above
+/// a written path is listed once, deepest first, so a folder's subfolders are
+/// decided before it. Climbing stops at the first folder that isn't
+/// rebuildable, since no folder above it can be, and at the workspace root. A
+/// folder that is gone plays no part: its parent is decided by the entries it
+/// holds.
+fn find_folders(
+    written: &FxHashSet<RelativePathBuf>,
+    workspace_root: &AbsolutePath,
+) -> FxHashSet<RelativePathBuf> {
+    let mut candidates = FxHashSet::default();
+    for path in written {
+        let mut dir = path.parent();
+        // A folder already reached from another path has its parents too.
+        while let Some(parent) = dir
+            && !parent.as_str().is_empty()
+            && candidates.insert(parent)
+        {
+            dir = parent.parent();
+        }
+    }
+    let mut candidates: Vec<&RelativePath> = candidates.into_iter().collect();
+    candidates.sort_unstable_by_key(|dir| Reverse(dir.as_str().matches('/').count()));
+
+    let mut folders = FxHashSet::default();
+    let mut above_unrebuildable = FxHashSet::default();
+    for dir in candidates {
+        let rebuildable = if above_unrebuildable.contains(dir) {
+            Some(false)
+        } else {
+            holds_only_rebuildable(dir, workspace_root, written, &folders)
+        };
+        match rebuildable {
+            Some(true) => {
+                folders.insert(dir.to_relative_path_buf());
+            }
+            Some(false) => {
+                if let Some(parent) = dir.parent() {
+                    above_unrebuildable.insert(parent);
+                }
+            }
+            None => {}
+        }
+    }
+    folders
+}
+
+/// Whether `dir` holds at least one entry, and every entry is a file (or
+/// symlink) in `written` or a folder in `folders`, or `None` when `dir` isn't
+/// a folder anymore. `.DS_Store` doesn't count, as in directory listing
+/// fingerprints. An empty folder isn't rebuildable: there's no written file to
+/// recreate it.
+fn holds_only_rebuildable(
+    dir: &RelativePath,
+    workspace_root: &AbsolutePath,
+    written: &FxHashSet<RelativePathBuf>,
+    folders: &FxHashSet<RelativePathBuf>,
+) -> Option<bool> {
+    let entries = match fs::read_dir(workspace_root.join(dir)) {
+        Ok(entries) => entries,
+        // Gone, or not a folder anymore: it plays no part.
+        Err(err)
+            if matches!(err.kind(), io::ErrorKind::NotFound | io::ErrorKind::NotADirectory) =>
+        {
+            return None;
+        }
+        Err(_) => return Some(false),
+    };
+    let mut is_empty = true;
+    for entry in entries {
+        let Ok(entry) = entry else {
+            return Some(false);
+        };
+        let name = entry.file_name();
+        if name == ".DS_Store" {
+            continue;
+        }
+        let (Some(name), Ok(file_type)) = (name.to_str(), entry.file_type()) else {
+            return Some(false);
+        };
+        let Ok(name) = RelativePathBuf::new(name) else {
+            return Some(false);
+        };
+        let path = dir.join(name);
+        let rebuildable =
+            if file_type.is_dir() { folders.contains(&path) } else { written.contains(&path) };
+        if !rebuildable {
+            return Some(false);
+        }
+        is_empty = false;
+    }
+    Some(!is_empty)
 }

crates/vt_bin/tests/e2e_snapshots/fixtures/stat_own_output_folder/package.json

@@ -0,0 +1,4 @@
+{
+  "name": "stat-own-output-folder",
+  "private": true
+}

crates/vt_bin/tests/e2e_snapshots/fixtures/stat_own_output_folder/snapshots.toml

@@ -0,0 +1,28 @@
+[[e2e]]
+name = "stat_own_output_folder"
+comment = """
+The task makes `out`, writes `out/a.txt` and stats `out`, like a build tool checking its output folder.
+"""
+steps = [
+  [
+    "vt",
+    "run",
+    "build",
+  ],
+  { argv = [
+    "vtt",
+    "rm",
+    "-r",
+    "out",
+  ], comment = "remove the output folder, as on a clean checkout" },
+  [
+    "vt",
+    "run",
+    "build",
+  ],
+  [
+    "vtt",
+    "stat-file",
+    "out/a.txt",
+  ],
+]

crates/vt_bin/tests/e2e_snapshots/fixtures/stat_own_output_folder/snapshots/stat_own_output_folder.md

@@ -0,0 +1,31 @@
+# stat_own_output_folder
+
+The task makes `out`, writes `out/a.txt` and stats `out`, like a build tool checking its output folder.
+
+## `vt run build`
+
+```
+$ vtt fs mkdir out write out/a.txt hi stat out
+```
+
+## `vtt rm -r out`
+
+remove the output folder, as on a clean checkout
+
+```
+```
+
+## `vt run build`
+
+```
+$ vtt fs mkdir out write out/a.txt hi stat out ◉ cache hit, replaying
+
+---
+vt run: cache hit.
+```
+
+## `vtt stat-file out/a.txt`
+
+```
+out/a.txt: exists
+```

crates/vt_bin/tests/e2e_snapshots/fixtures/stat_own_output_folder/vite-task.json

@@ -0,0 +1,8 @@
+{
+  "tasks": {
+    "build": {
+      "command": "vtt fs mkdir out write out/a.txt hi stat out",
+      "cache": true
+    }
+  }
+}

crates/vt_bin/tests/e2e_snapshots/fixtures/typescript_out_dir/package.json

@@ -0,0 +1,4 @@
+{
+  "name": "typescript-out-dir",
+  "private": true
+}

crates/vt_bin/tests/e2e_snapshots/fixtures/typescript_out_dir/snapshots.toml

@@ -0,0 +1,29 @@
+[[e2e]]
+name = "typescript_out_dir"
+comment = """
+TypeScript 7's `tsc` compiles `src` into `dist`. It stats `dist` and lists the package root.
+"""
+ignore = true
+steps = [
+  [
+    "vt",
+    "run",
+    "build",
+  ],
+  { argv = [
+    "vtt",
+    "rm",
+    "-r",
+    "dist",
+  ], comment = "remove the output folder, as on a clean checkout" },
+  [
+    "vt",
+    "run",
+    "build",
+  ],
+  [
+    "vtt",
+    "stat-file",
+    "dist/main.js",
+  ],
+]