[M7] cargoxx vendor + build --offline + path: store-path codegen

This commit is contained in:
2026-05-16 00:27:45 +00:00
parent 43a7d1f09d
commit 85417f317c
11 changed files with 444 additions and 22 deletions

View File

@@ -1,3 +1,5 @@
flake_utils_rev = '11707dc2f618dd54ca8739b309ec4fc024de578b'
nixpkgs_rev = 'da5ad661ba4e5ef59ba743f0d112cbc30e474f32'
version = 1
[[package]]
@@ -6,19 +8,25 @@ name = 'cargoxx'
version = '0.1.0'
[[package]]
find_package = 'reproc CONFIG REQUIRED'
linkdb_source = 'nix-probe'
name = 'reproc'
nixpkgs_attr = 'reproc'
targets = [ 'reproc' ]
version = '*'
[[package]]
find_package = 'SQLite3 REQUIRED'
linkdb_source = 'cmake-findmodule'
name = 'sqlite'
nixpkgs_attr = 'sqlite'
targets = [ 'SQLite::SQLite3' ]
version = '*'
[[package]]
find_package = 'Catch2 CONFIG REQUIRED'
linkdb_source = 'nix-probe'
name = 'catch2_3'
nixpkgs_attr = 'catch2_3'
targets = [ 'Catch2::Catch2', 'Catch2::Catch2WithMain' ]
version = '*'

View File

@@ -46,9 +46,11 @@ target_sources(cargoxx
../src/cli/cmd_remove.cpp
../src/cli/cmd_run.cpp
../src/cli/cmd_test.cpp
../src/cli/cmd_vendor.cpp
../src/cli/run.cpp
../src/codegen/cmake.cpp
../src/codegen/flake.cpp
../src/codegen/vendor.cpp
../src/exec/subprocess.cpp
../src/layout/layout.cpp
../src/linkdb/database.cpp

View File

@@ -20,7 +20,13 @@ auto cmd_new(const std::string& name, bool lib_only,
// `overlay_path` lets tests redirect the linkdb overlay away from ~/.cache.
auto cmd_build(const std::filesystem::path& project_root, bool no_build, bool release,
std::optional<std::string> target = std::nullopt,
std::optional<std::filesystem::path> overlay_path = std::nullopt)
std::optional<std::filesystem::path> overlay_path = std::nullopt,
bool offline = false,
std::optional<std::filesystem::path> vendor = std::nullopt)
-> util::Result<void>;
auto cmd_vendor(const std::filesystem::path& project_root,
const std::filesystem::path& output)
-> util::Result<void>;
// Builds the project, picks a binary target, and execs it with `args`.

View File

@@ -177,7 +177,8 @@ auto run_nix_cmake(const fs::path& project_root, const std::vector<std::string>&
auto cmd_build(const fs::path& project_root, bool no_build, bool release,
std::optional<std::string> target,
std::optional<fs::path> overlay_path) -> util::Result<void> {
std::optional<fs::path> overlay_path, bool offline,
std::optional<fs::path> vendor) -> util::Result<void> {
auto manifest_path = project_root / "Cargoxx.toml";
auto m = manifest::parse(manifest_path);
if (!m) {
@@ -283,6 +284,24 @@ auto cmd_build(const fs::path& project_root, bool no_build, bool release,
}
auto lock = merge_lockfile(*m, *recipes, *dev_recipes, prior);
std::optional<codegen::VendorIndex> vendor_index;
if (offline) {
auto vendor_path = vendor.value_or(project_root / "vendor.toml");
if (std::error_code v_ec; !fs::exists(vendor_path, v_ec)) {
return std::unexpected(io_error(
std::format("--offline requires vendor.toml; expected at '{}'",
vendor_path.string()),
vendor_path));
}
std::ifstream in_file{vendor_path};
std::string body{std::istreambuf_iterator<char>(in_file), {}};
auto parsed = codegen::parse_vendor_toml(body);
if (!parsed) {
return std::unexpected(parsed.error());
}
vendor_index = std::move(*parsed);
}
codegen::GenerateInputs in{
.manifest = *m,
.layout = *layout_result,
@@ -290,6 +309,7 @@ auto cmd_build(const fs::path& project_root, bool no_build, bool release,
.recipes = *recipes,
.dev_recipes = *dev_recipes,
.project_root = project_root,
.vendor = vendor_index,
};
auto flake_text = codegen::flake_nix(in);
auto cmake_text = codegen::cmake_lists(in);
@@ -326,16 +346,41 @@ auto cmd_build(const fs::path& project_root, bool no_build, bool release,
"-G", "Ninja",
std::format("-DCMAKE_BUILD_TYPE={}", profile_cap),
};
if (auto r = run_nix_cmake(project_root, configure_args, "configure"); !r) {
return std::unexpected(r.error());
}
std::vector<std::string> build_args{"--build", build_dir};
if (target) {
build_args.push_back("--target");
build_args.push_back(*target);
}
if (auto r = run_nix_cmake(project_root, build_args, "build"); !r) {
auto run_cmake = [&](const std::vector<std::string>& args,
std::string_view phase) -> util::Result<void> {
if (offline) {
auto r = exec::run("cmake", args,
exec::ExecOptions{
.cwd = project_root,
.env_overrides = {},
.timeout = std::nullopt,
.inherit_stdio = true,
});
if (!r) {
return std::unexpected(r.error());
}
if (r->exit_code != 0) {
return std::unexpected(util::Error{
util::ErrorCode::BuildCmakeFailed,
std::format("cmake {} failed (exit {})", phase, r->exit_code),
"", std::nullopt, std::nullopt,
});
}
return {};
}
return run_nix_cmake(project_root, args, phase);
};
if (auto r = run_cmake(configure_args, "configure"); !r) {
return std::unexpected(r.error());
}
if (auto r = run_cmake(build_args, "build"); !r) {
return std::unexpected(r.error());
}

128
src/cli/cmd_vendor.cpp Normal file
View File

@@ -0,0 +1,128 @@
module cargoxx.cli;
import std;
import cargoxx.lockfile;
import cargoxx.resolver;
import cargoxx.util;
namespace cargoxx::cli {
namespace fs = std::filesystem;
namespace {
auto error(util::ErrorCode code, std::string msg, fs::path path) -> util::Error {
return util::Error{code, std::move(msg), "", std::move(path), std::nullopt};
}
auto escape_toml(std::string_view s) -> std::string {
std::string out;
out.reserve(s.size() + 2);
out += '"';
for (char c : s) {
if (c == '"' || c == '\\') {
out += '\\';
}
out += c;
}
out += '"';
return out;
}
} // namespace
auto cmd_vendor(const fs::path& project_root, const fs::path& output)
-> util::Result<void> {
auto lock_path = project_root / "Cargoxx.lock";
auto lock = lockfile::parse(lock_path);
if (!lock) {
return std::unexpected(lock.error());
}
if (!lock->nixpkgs_rev_pin || lock->nixpkgs_rev_pin->empty()) {
return std::unexpected(error(
util::ErrorCode::ManifestInvalidField,
"Cargoxx.lock has no top-level nixpkgs_rev — run cargoxx build "
"online first to pin it",
lock_path));
}
if (!lock->flake_utils_rev_pin || lock->flake_utils_rev_pin->empty()) {
return std::unexpected(error(
util::ErrorCode::ManifestInvalidField,
"Cargoxx.lock has no top-level flake_utils_rev",
lock_path));
}
auto nixpkgs_src = resolver::realize_flake_source(
std::format("github:NixOS/nixpkgs/{}", *lock->nixpkgs_rev_pin));
if (!nixpkgs_src) {
return std::unexpected(nixpkgs_src.error());
}
auto flake_utils_src = resolver::realize_flake_source(
std::format("github:numtide/flake-utils/{}", *lock->flake_utils_rev_pin));
if (!flake_utils_src) {
return std::unexpected(flake_utils_src.error());
}
std::string body;
body += "schema = 1\n\n";
body += "[nixpkgs]\n";
body += std::format("rev = {}\n", escape_toml(*lock->nixpkgs_rev_pin));
body += std::format("store_path = {}\n\n", escape_toml(*nixpkgs_src));
body += "[flake_utils]\n";
body += std::format("rev = {}\n", escape_toml(*lock->flake_utils_rev_pin));
body += std::format("store_path = {}\n", escape_toml(*flake_utils_src));
for (const auto& p : lock->packages) {
if (!p.linkdb_source) {
continue;
}
if (!p.nixpkgs_attr) {
return std::unexpected(error(
util::ErrorCode::ManifestInvalidField,
std::format("lockfile dep '{}' has no nixpkgs_attr", p.name),
lock_path));
}
auto rev = p.nixpkgs_rev.has_value() && !p.nixpkgs_rev->empty()
? *p.nixpkgs_rev
: *lock->nixpkgs_rev_pin;
auto store = resolver::realize_path_at_rev(rev, *p.nixpkgs_attr);
if (!store) {
return std::unexpected(store.error());
}
std::optional<std::string> dev_path;
if (auto d = resolver::realize_path_at_rev(
rev, std::format("{}.dev", *p.nixpkgs_attr));
d) {
dev_path = *d;
}
body += "\n[[dep]]\n";
body += std::format("name = {}\n", escape_toml(p.name));
body += std::format("nixpkgs_attr = {}\n", escape_toml(*p.nixpkgs_attr));
body += std::format("nixpkgs_rev = {}\n", escape_toml(rev));
body += std::format("store_path = {}\n", escape_toml(*store));
if (dev_path) {
body += std::format("dev_path = {}\n", escape_toml(*dev_path));
}
}
std::error_code ec;
fs::create_directories(output.parent_path(), ec);
std::ofstream out{output};
if (!out) {
return std::unexpected(error(
util::ErrorCode::Internal,
std::format("cannot open vendor file for writing: {}",
output.string()),
output));
}
out << body;
if (!out) {
return std::unexpected(error(
util::ErrorCode::Internal,
std::format("write failed: {}", output.string()), output));
}
return {};
}
} // namespace cargoxx::cli

View File

@@ -23,13 +23,26 @@ auto run(int argc, char** argv) -> int {
"build", "Generate flake.nix and build/CMakeLists.txt; build with nix+cmake");
bool build_no_build = false;
bool build_release = false;
bool build_offline = false;
std::string build_target;
std::string build_vendor;
build_cmd->add_flag("--no-build", build_no_build,
"Generate files only; do not invoke nix/cmake");
build_cmd->add_flag("--release", build_release, "Build the release profile");
build_cmd->add_flag("--offline", build_offline,
"Skip network probes and nix develop wrappers. "
"Reads vendor.toml for store-path inputs.");
build_cmd->add_option("--vendor", build_vendor,
"Path to vendor.toml (used with --offline; default ./vendor.toml)");
build_cmd->add_option("--target", build_target,
"Build a specific target (passed to cmake --build)");
auto* vendor_cmd = app.add_subcommand(
"vendor", "Resolve every locked dependency into /nix/store and write vendor.toml");
std::string vendor_output;
vendor_cmd->add_option("--output", vendor_output,
"Path to write vendor.toml (default ./vendor.toml)");
auto* run_cmd = app.add_subcommand("run", "Build and run a binary target");
bool run_release = false;
std::string run_bin;
@@ -109,7 +122,12 @@ auto run(int argc, char** argv) -> int {
if (!build_target.empty()) {
target = build_target;
}
auto r = cmd_build(cwd, build_no_build, build_release, target);
std::optional<std::filesystem::path> vendor_path;
if (!build_vendor.empty()) {
vendor_path = build_vendor;
}
auto r = cmd_build(cwd, build_no_build, build_release, target,
std::nullopt, build_offline, vendor_path);
if (!r) {
std::cerr << util::format(r.error());
return 1;
@@ -122,6 +140,18 @@ auto run(int argc, char** argv) -> int {
return 0;
}
if (*vendor_cmd) {
auto out = vendor_output.empty() ? cwd / "vendor.toml"
: std::filesystem::path{vendor_output};
auto r = cmd_vendor(cwd, out);
if (!r) {
std::cerr << util::format(r.error());
return 1;
}
std::cout << std::format(" Wrote {}\n", out.string());
return 0;
}
if (*run_cmd) {
std::optional<std::string> bin;
if (!run_bin.empty()) {

View File

@@ -9,6 +9,16 @@ import cargoxx.lockfile;
export namespace cargoxx::codegen {
// When set, codegen emits the generated flake's nixpkgs / flake-utils
// inputs as `path:/nix/store/...` references (already-realized source
// paths) instead of `github:NixOS/nixpkgs/<rev>` URLs. This makes the
// inner build hermetic — no network and no nix daemon access required.
struct VendorIndex {
std::string nixpkgs_store_path;
std::string flake_utils_store_path;
std::unordered_map<std::string, std::string> dep_store_paths; // by attr
};
// All inputs the generators need. Held by const reference; the caller owns
// the underlying objects. Not copyable.
struct GenerateInputs {
@@ -18,9 +28,14 @@ struct GenerateInputs {
std::vector<linkdb::Recipe> recipes; // one per manifest dep, same order
std::vector<linkdb::Recipe> dev_recipes; // one per dev_dependency, same order
std::filesystem::path project_root;
std::optional<VendorIndex> vendor;
};
auto flake_nix(const GenerateInputs& in) -> std::string;
auto cmake_lists(const GenerateInputs& in) -> std::string;
// Pure: parses a vendor.toml (see cmd_vendor) into a VendorIndex.
auto parse_vendor_toml(std::string_view body)
-> util::Result<VendorIndex>;
} // namespace cargoxx::codegen

View File

@@ -100,22 +100,36 @@ auto pinned_inputs_dedup(const std::vector<DepBinding>& bindings)
}
auto emit_inputs_block(const std::vector<const DepBinding*>& pinned,
const lockfile::Lockfile& lock) -> std::string {
auto nixpkgs_url =
lock.nixpkgs_rev_pin && !lock.nixpkgs_rev_pin->empty()
? std::format("github:NixOS/nixpkgs/{}", *lock.nixpkgs_rev_pin)
: std::string{"github:NixOS/nixpkgs/nixos-unstable"};
auto flake_utils_url =
lock.flake_utils_rev_pin && !lock.flake_utils_rev_pin->empty()
? std::format("github:numtide/flake-utils/{}",
*lock.flake_utils_rev_pin)
: std::string{"github:numtide/flake-utils"};
const lockfile::Lockfile& lock,
const std::optional<VendorIndex>& vendor)
-> std::string {
auto nixpkgs_url = [&]() -> std::string {
if (vendor && !vendor->nixpkgs_store_path.empty()) {
return std::format("path:{}", vendor->nixpkgs_store_path);
}
if (lock.nixpkgs_rev_pin && !lock.nixpkgs_rev_pin->empty()) {
return std::format("github:NixOS/nixpkgs/{}", *lock.nixpkgs_rev_pin);
}
return "github:NixOS/nixpkgs/nixos-unstable";
}();
auto flake_utils_url = [&]() -> std::string {
if (vendor && !vendor->flake_utils_store_path.empty()) {
return std::format("path:{}", vendor->flake_utils_store_path);
}
if (lock.flake_utils_rev_pin && !lock.flake_utils_rev_pin->empty()) {
return std::format("github:numtide/flake-utils/{}",
*lock.flake_utils_rev_pin);
}
return "github:numtide/flake-utils";
}();
std::string out =
" inputs = {\n"
+ std::format(" nixpkgs.url = \"{}\";\n", nixpkgs_url);
for (const auto* b : pinned) {
out += std::format(" {}.url = \"github:NixOS/nixpkgs/{}\";\n",
b->sanitized, *b->rev);
if (!vendor) {
for (const auto* b : pinned) {
out += std::format(" {}.url = \"github:NixOS/nixpkgs/{}\";\n",
b->sanitized, *b->rev);
}
}
out += std::format(" flake-utils.url = \"{}\";\n", flake_utils_url);
out += " };\n";
@@ -171,7 +185,7 @@ auto flake_nix(const GenerateInputs& in) -> std::string {
out += "{\n";
out += std::format(" description = \"{}\";\n\n", in.manifest.package.name);
out += emit_inputs_block(pinned, in.lock);
out += emit_inputs_block(pinned, in.lock, in.vendor);
const bool any_pkg_config =
std::ranges::any_of(in.recipes,

69
src/codegen/vendor.cpp Normal file
View File

@@ -0,0 +1,69 @@
module;
#include <toml.hpp>
module cargoxx.codegen;
import std;
import cargoxx.util;
namespace cargoxx::codegen {
auto parse_vendor_toml(std::string_view body) -> util::Result<VendorIndex> {
toml::table root;
try {
root = toml::parse(body);
} catch (const toml::parse_error& e) {
return std::unexpected(util::Error{
util::ErrorCode::ManifestParseError,
std::format("vendor.toml is not valid TOML: {}", e.description()),
"", std::nullopt, std::nullopt,
});
}
auto missing = [](std::string msg) {
return util::Error{
util::ErrorCode::ManifestInvalidField, std::move(msg), "",
std::nullopt, std::nullopt,
};
};
VendorIndex out;
if (const auto* tbl = root["nixpkgs"].as_table()) {
if (auto v = (*tbl)["store_path"].value<std::string>()) {
out.nixpkgs_store_path = *v;
} else {
return std::unexpected(missing("vendor.toml: [nixpkgs].store_path is required"));
}
} else {
return std::unexpected(missing("vendor.toml: [nixpkgs] table is required"));
}
if (const auto* tbl = root["flake_utils"].as_table()) {
if (auto v = (*tbl)["store_path"].value<std::string>()) {
out.flake_utils_store_path = *v;
} else {
return std::unexpected(missing("vendor.toml: [flake_utils].store_path is required"));
}
} else {
return std::unexpected(missing("vendor.toml: [flake_utils] table is required"));
}
if (const auto* arr = root["dep"].as_array()) {
for (const auto& el : *arr) {
const auto* tbl = el.as_table();
if (!tbl) {
return std::unexpected(missing("vendor.toml: [[dep]] entries must be tables"));
}
auto attr = (*tbl)["nixpkgs_attr"].value<std::string>();
auto path = (*tbl)["store_path"].value<std::string>();
if (!attr || !path) {
return std::unexpected(missing(
"vendor.toml: each [[dep]] needs nixpkgs_attr and store_path"));
}
out.dep_store_paths.emplace(*attr, *path);
}
}
return out;
}
} // namespace cargoxx::codegen

View File

@@ -162,4 +162,95 @@ auto realize_path(const std::string& flake_attr) -> util::Result<std::string> {
return path;
}
auto realize_path_at_rev(const std::string& rev, const std::string& attr)
-> util::Result<std::string> {
if (rev.empty() || attr.empty()) {
return std::unexpected(make_error(
util::ErrorCode::ResolutionUnknownPackage,
"realize_path_at_rev: rev and attr must be non-empty"));
}
std::vector<std::string> args{
"--extra-experimental-features", "nix-command flakes",
"build", std::format("github:NixOS/nixpkgs/{}#{}", rev, attr),
"--no-link", "--print-out-paths",
};
auto r = exec::run("nix", args,
exec::ExecOptions{
.cwd = {},
.env_overrides = {},
.timeout = std::chrono::seconds{600},
.inherit_stdio = false,
});
if (!r) {
return std::unexpected(r.error());
}
if (r->exit_code != 0) {
if (looks_like_missing_attribute(r->stderr_text)) {
return std::unexpected(make_error(
util::ErrorCode::ResolutionUnknownPackage,
std::format("nixpkgs/{} has no attribute '{}'", rev, attr)));
}
return std::unexpected(make_error(
util::ErrorCode::ResolutionNetworkError,
std::format("nix build failed (exit {}): {}", r->exit_code,
r->stderr_text)));
}
auto path = r->stdout_text;
while (!path.empty() && (path.back() == '\n' || path.back() == ' ')) {
path.pop_back();
}
if (path.empty()) {
return std::unexpected(make_error(
util::ErrorCode::ResolutionNetworkError,
std::format("nix build emitted no path for '{}#{}'", rev, attr)));
}
return path;
}
auto realize_flake_source(const std::string& flake_ref)
-> util::Result<std::string> {
if (flake_ref.empty()) {
return std::unexpected(make_error(
util::ErrorCode::ResolutionUnknownPackage,
"realize_flake_source: flake_ref is empty"));
}
std::vector<std::string> args{
"--extra-experimental-features", "nix-command flakes",
"flake", "prefetch", flake_ref, "--json",
};
auto r = exec::run("nix", args,
exec::ExecOptions{
.cwd = {},
.env_overrides = {},
.timeout = std::chrono::seconds{300},
.inherit_stdio = false,
});
if (!r) {
return std::unexpected(r.error());
}
if (r->exit_code != 0) {
return std::unexpected(make_error(
util::ErrorCode::ResolutionNetworkError,
std::format("nix flake prefetch failed (exit {}): {}",
r->exit_code, r->stderr_text)));
}
std::string_view body = r->stdout_text;
constexpr std::string_view key = "\"storePath\":\"";
auto pos = body.find(key);
if (pos == std::string_view::npos) {
return std::unexpected(make_error(
util::ErrorCode::ResolutionNetworkError,
std::format("nix flake prefetch emitted no storePath for '{}'",
flake_ref)));
}
pos += key.size();
auto end = body.find('"', pos);
if (end == std::string_view::npos) {
return std::unexpected(make_error(
util::ErrorCode::ResolutionNetworkError,
"nix flake prefetch JSON malformed"));
}
return std::string{body.substr(pos, end - pos)};
}
} // namespace cargoxx::resolver

View File

@@ -45,6 +45,20 @@ auto nixpkgs_probe(const std::string& attr) -> util::Result<NixpkgsInfo>;
// for build / network errors.
auto realize_path(const std::string& flake_attr) -> util::Result<std::string>;
// Like `realize_path`, but pins the nixpkgs revision instead of using the
// registry alias. Builds `github:NixOS/nixpkgs/<rev>#<attr>` and returns
// the resulting `/nix/store/...` path. Used by the vendor subcommand to
// materialize each lockfile-pinned dep without depending on the user's
// flake registry.
auto realize_path_at_rev(const std::string& rev, const std::string& attr)
-> util::Result<std::string>;
// Returns the source store path for `github:NixOS/nixpkgs/<rev>` (the
// path Nix would set as `nixpkgs.outPath` when imported). Used by the
// vendor subcommand to record nixpkgs/flake-utils source locations.
auto realize_flake_source(const std::string& flake_ref)
-> util::Result<std::string>;
// One CMake config-file's IMPORTED targets together with the find_package
// expression derived from its filename stem.
struct NixCmakeCandidate {