Skip to content

prototype cache with cas backend #33

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ranger-ross wants to merge 9 commits into
base: master
Choose a base branch
from
Open

prototype cache with cas backend #33

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
ea548b5
added a cas cache
ranger-ross Sep 9, 2026
4895cc3
refactor content create to layout prepare
ranger-ross Sep 9, 2026
47f41a4
properly consider if a unit is cachable during fingerprint checking
ranger-ross Sep 10, 2026
5b2d1eb
share the build cache
ranger-ross Sep 10, 2026
1a54874
Handle arbitrary OUT_DIR files
ranger-ross Sep 20, 2026
adfc398
vibecode: Added support for build scripts
ranger-ross Sep 20, 2026
c231ae4
clean up
ranger-ross Sep 20, 2026
e2a0cfc
use blake3
ranger-ross Sep 20, 2026
59b40b3
Added 'Cached' user text
ranger-ross Sep 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions crates/cargo-util-terminal/src/style.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ use anstyle::*;

pub const NOP: Style = Style::new();
pub const HEADER: Style = AnsiColor::BrightGreen.on_default().effects(Effects::BOLD);
pub const CACHE: Style = AnsiColor::BrightMagenta.on_default().effects(Effects::BOLD);
pub const USAGE: Style = AnsiColor::BrightGreen.on_default().effects(Effects::BOLD);
pub const LITERAL: Style = AnsiColor::BrightCyan.on_default().effects(Effects::BOLD);
pub const PLACEHOLDER: Style = AnsiColor::Cyan.on_default();
Expand Down
11 changes: 10 additions & 1 deletion src/compiler/build_runner/compilation_files.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
//! See [`CompilationFiles`].

use crate::compiler::cache::BuildCache;
use crate::util::data_structures::HashMap;
use std::cell::OnceCell;
use std::fmt;
Expand Down Expand Up @@ -136,6 +137,8 @@ pub struct CompilationFiles<'a, 'gctx> {
metas: HashMap<Unit, Metadata>,
/// For each Unit, a list all files produced.
outputs: HashMap<Unit, OnceCell<Arc<Vec<OutputFile>>>>,
/// The shared build cache
build_cache: Arc<BuildCache>,
}

/// Info about a single file emitted by the compiler.
Expand Down Expand Up @@ -177,6 +180,7 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> {
.cloned()
.map(|unit| (unit, OnceCell::new()))
.collect();
let build_cache = Arc::new(BuildCache::new(host.build_cache().clone()));
CompilationFiles {
ws: build_runner.bcx.ws,
host,
Expand All @@ -185,6 +189,7 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> {
roots: build_runner.bcx.roots.clone(),
metas,
outputs,
build_cache,
}
}

Expand Down Expand Up @@ -245,7 +250,7 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> {
///
/// Note that some units may share the same directory, so care should be
/// taken in those cases!
fn pkg_dir(&self, unit: &Unit) -> String {
pub fn pkg_dir(&self, unit: &Unit) -> String {
let separator = match self.ws.gctx().cli_unstable().build_dir_new_layout {
true => "/",
false => "-",
Expand Down Expand Up @@ -647,6 +652,10 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> {
Ok(Arc::new(ret))
}

pub fn build_cache(&self) -> Arc<BuildCache> {
self.build_cache.clone()
}

/// Append the SBOM suffix to the file name.
fn append_sbom_suffix(link: &PathBuf) -> PathBuf {
const SBOM_FILE_EXTENSION: &str = ".cargo-sbom.json";
Expand Down
32 changes: 32 additions & 0 deletions src/compiler/build_runner/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -314,6 +314,38 @@
Ok(self.compilation)
}

/// TODO: We should probably compute this upfront in unit_dependencies.rs
/// I wanted to include it on Unit but couldn't easily as the graph creation is topdown
/// but is_cachable is bottom up. Since units are intern, we can't easily edit them.

Check warning on line 319 in src/compiler/build_runner/mod.rs

View workflow job for this annotation

GitHub Actions / Spell Check with Typos 

"cachable" should be "cacheable".
/// We can probably create a seperate "cachability map" for easy lookups

Check warning on line 320 in src/compiler/build_runner/mod.rs

View workflow job for this annotation

GitHub Actions / Spell Check with Typos 

"seperate" should be "separate".
pub fn is_cacheable(&self, unit: &Unit) -> bool {
let mut visited = HashSet::default();
self.is_cacheable_recursive(unit, &mut visited)
}

fn is_cacheable_recursive(&self, unit: &Unit, visited: &mut HashSet<Unit>) -> bool {
if !visited.insert(unit.clone()) {
return true;
}
if !self.is_unit_cacheable(unit) {
return false;
}
self.unit_deps(unit)
.iter()
.all(|dep| self.is_cacheable_recursive(&dep.unit, visited))
}

fn is_unit_cacheable(&self, unit: &Unit) -> bool {
self.bcx.gctx.cli_unstable().build_dir_new_layout && !unit.is_local()
// && !unit.pkg.has_custom_build()
// && !unit.target.is_custom_build()
// && !unit.target.is_bin()
// && !unit.mode.is_doc()
// && !unit.mode.is_doc_scrape()
// && !unit.mode.is_any_test()
// && !unit.artifact.is_true()
}

fn collect_tests_and_executables(&mut self, unit: &Unit) -> CargoResult<()> {
for output in self.outputs(unit)?.iter() {
if matches!(
Expand Down
173 changes: 173 additions & 0 deletions src/compiler/cache.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,173 @@
use std::{
collections::BTreeMap,
path::{Path, PathBuf},
};

use anyhow::Context;
use cargo_util::paths;
use cargo_util::paths::link_or_copy;
use serde::{Deserialize, Serialize};

use crate::{CargoResult, compiler::layout::BuildCacheLayout};

pub struct BuildCache {
layout: BuildCacheLayout,
}

impl BuildCache {
pub fn new(layout: BuildCacheLayout) -> Self {
Self { layout }
}

pub fn get(&self, pkg_dir: &str) -> Option<CacheEntry> {
let path = self.layout.entries_dir().join(pkg_dir);
let content = std::fs::read_to_string(&path).ok()?;
let entry = serde_json::from_str(&content).ok()?;
Some(entry)
}

pub fn publish_entry(&self, pkg_dir: &str, out_dir: &Path) -> CargoResult<()> {
let content = self.layout.content_dir();

let mut files = BTreeMap::new();
for entry in walkdir::WalkDir::new(out_dir) {
let entry = entry?;
let src = entry.path();
if !src.is_file() {
continue;
}
let rel = src
.strip_prefix(out_dir)
.expect("walked path under out dir");
if rel.as_os_str().is_empty() {
continue;
}
let hash = Self::hash(src)?;
let dest = content.join(&hash);

link_or_copy(src, &dest)?;

files.insert(rel.to_path_buf(), hash);
}

let entry = CacheEntry { files };

let json = serde_json::to_string(&entry)?;
let entry_path = self.layout.entries_dir().join(pkg_dir);
cargo_util::paths::create_dir_all(entry_path.parent().unwrap())?;

cargo_util::paths::write_atomic(&entry_path, &json).context("writing cache entry")?;

Ok(())
}

pub fn restore_from_cache(&self, entry: CacheEntry, out_dir: &Path) -> CargoResult<()> {
paths::create_dir_all(out_dir)?;

let content_dir = self.layout.content_dir();
for (path_in_out_dir, hash) in entry.files {
if path_in_out_dir.as_os_str().is_empty() {
continue;
}
let dest = out_dir.join(&path_in_out_dir);
if let Some(parent) = dest.parent() {
paths::create_dir_all(parent)?;
}
let file_in_cache = content_dir.join(hash);
paths::link_or_copy(&file_in_cache, &dest)?;
}

Ok(())
}

fn restore_files(&self, files: BTreeMap<PathBuf, String>, dest_root: &Path) -> CargoResult<()> {
paths::create_dir_all(dest_root)?;
let content_dir = self.layout.content_dir();
for (rel, hash) in files {
if rel.as_os_str().is_empty() || rel.is_absolute() {
continue;
}
if rel
.components()
.any(|c| matches!(c, std::path::Component::ParentDir))
{
continue;
}
let dest = dest_root.join(rel);
if let Some(parent) = dest.parent() {
paths::create_dir_all(parent)?;
}
paths::link_or_copy(&content_dir.join(hash), &dest)?;
}
Ok(())
}

pub fn publish_build_script(
&self,
pkg_dir: &str,
run_root: &Path,
output: &crate::compiler::custom_build::BuildOutput,
) -> CargoResult<()> {
let entry = ScriptEntry {
files: self.dedup_dir(run_root)?,
output: output.clone(),
};
let json = serde_json::to_string(&entry)?;
let entry_path = self.layout.entries_dir().join(pkg_dir).join("script.json");
cargo_util::paths::create_dir_all(entry_path.parent().unwrap())?;
cargo_util::paths::write_atomic(&entry_path, &json)
.context("writing script cache entry")?;
Ok(())
}

pub fn restore_build_script(
&self,
pkg_dir: &str,
run_root: &Path,
) -> Option<crate::compiler::custom_build::BuildOutput> {
let path = self.layout.entries_dir().join(pkg_dir).join("script.json");
let content = std::fs::read_to_string(&path).ok()?;
let entry: ScriptEntry = serde_json::from_str(&content).ok()?;
self.restore_files(entry.files, run_root).ok()?;
Some(entry.output)
}

fn dedup_dir(&self, dir: &Path) -> CargoResult<BTreeMap<PathBuf, String>> {
let content = self.layout.content_dir();
let mut files = BTreeMap::new();
for entry in walkdir::WalkDir::new(dir) {
let entry = entry?;
let src = entry.path();
if !src.is_file() {
continue;
}
let rel = src.strip_prefix(dir).expect("walked path under cached dir");
if rel.as_os_str().is_empty() {
continue;
}
let hash = Self::hash(src)?;
link_or_copy(src, &content.join(&hash))?;
files.insert(rel.to_path_buf(), hash);
}
Ok(files)
}

fn hash(path: &Path) -> CargoResult<String> {
let mut hasher = blake3::Hasher::new();
let mut file = std::fs::File::open(path)?;
std::io::copy(&mut file, &mut hasher)?;
Ok(hasher.finalize().to_hex().to_string())
}
}

#[derive(Debug, Serialize, Deserialize)]
struct ScriptEntry {
files: BTreeMap<PathBuf, String>,
output: crate::compiler::custom_build::BuildOutput,
}

#[derive(Debug, Serialize, Deserialize)]
pub struct CacheEntry {
// A map of paths relative to the OUT_DIR and their hashes.
pub files: BTreeMap<PathBuf, String>,
}
Loading
Loading