From 85417f317ca4c836e120de311607c5d601b609fe Mon Sep 17 00:00:00 2001 From: Amadey Vorontsov Date: Sat, 16 May 2026 00:27:45 +0000 Subject: [PATCH] [M7] cargoxx vendor + build --offline + path: store-path codegen --- Cargoxx.lock | 8 +++ build/CMakeLists.txt | 2 + src/cli/cli.cppm | 8 ++- src/cli/cmd_build.cpp | 57 +++++++++++++-- src/cli/cmd_vendor.cpp | 128 +++++++++++++++++++++++++++++++++ src/cli/run.cpp | 32 ++++++++- src/codegen/codegen.cppm | 15 ++++ src/codegen/flake.cpp | 42 +++++++---- src/codegen/vendor.cpp | 69 ++++++++++++++++++ src/resolver/nixpkgs_probe.cpp | 91 +++++++++++++++++++++++ src/resolver/resolver.cppm | 14 ++++ 11 files changed, 444 insertions(+), 22 deletions(-) create mode 100644 src/cli/cmd_vendor.cpp create mode 100644 src/codegen/vendor.cpp diff --git a/Cargoxx.lock b/Cargoxx.lock index 86fa894..9caa47d 100644 --- a/Cargoxx.lock +++ b/Cargoxx.lock @@ -1,3 +1,5 @@ +flake_utils_rev = '11707dc2f618dd54ca8739b309ec4fc024de578b' +nixpkgs_rev = 'da5ad661ba4e5ef59ba743f0d112cbc30e474f32' version = 1 [[package]] @@ -6,19 +8,25 @@ name = 'cargoxx' version = '0.1.0' [[package]] +find_package = 'reproc CONFIG REQUIRED' linkdb_source = 'nix-probe' name = 'reproc' nixpkgs_attr = 'reproc' +targets = [ 'reproc' ] version = '*' [[package]] +find_package = 'SQLite3 REQUIRED' linkdb_source = 'cmake-findmodule' name = 'sqlite' nixpkgs_attr = 'sqlite' +targets = [ 'SQLite::SQLite3' ] version = '*' [[package]] +find_package = 'Catch2 CONFIG REQUIRED' linkdb_source = 'nix-probe' name = 'catch2_3' nixpkgs_attr = 'catch2_3' +targets = [ 'Catch2::Catch2', 'Catch2::Catch2WithMain' ] version = '*' diff --git a/build/CMakeLists.txt b/build/CMakeLists.txt index 1eb12b2..601861d 100644 --- a/build/CMakeLists.txt +++ b/build/CMakeLists.txt @@ -46,9 +46,11 @@ target_sources(cargoxx ../src/cli/cmd_remove.cpp ../src/cli/cmd_run.cpp ../src/cli/cmd_test.cpp + ../src/cli/cmd_vendor.cpp ../src/cli/run.cpp ../src/codegen/cmake.cpp ../src/codegen/flake.cpp + ../src/codegen/vendor.cpp ../src/exec/subprocess.cpp ../src/layout/layout.cpp ../src/linkdb/database.cpp diff --git a/src/cli/cli.cppm b/src/cli/cli.cppm index 410e1b4..a742769 100644 --- a/src/cli/cli.cppm +++ b/src/cli/cli.cppm @@ -20,7 +20,13 @@ auto cmd_new(const std::string& name, bool lib_only, // `overlay_path` lets tests redirect the linkdb overlay away from ~/.cache. auto cmd_build(const std::filesystem::path& project_root, bool no_build, bool release, std::optional target = std::nullopt, - std::optional overlay_path = std::nullopt) + std::optional overlay_path = std::nullopt, + bool offline = false, + std::optional vendor = std::nullopt) + -> util::Result; + +auto cmd_vendor(const std::filesystem::path& project_root, + const std::filesystem::path& output) -> util::Result; // Builds the project, picks a binary target, and execs it with `args`. diff --git a/src/cli/cmd_build.cpp b/src/cli/cmd_build.cpp index 9294cba..85aad4c 100644 --- a/src/cli/cmd_build.cpp +++ b/src/cli/cmd_build.cpp @@ -177,7 +177,8 @@ auto run_nix_cmake(const fs::path& project_root, const std::vector& auto cmd_build(const fs::path& project_root, bool no_build, bool release, std::optional target, - std::optional overlay_path) -> util::Result { + std::optional overlay_path, bool offline, + std::optional vendor) -> util::Result { auto manifest_path = project_root / "Cargoxx.toml"; auto m = manifest::parse(manifest_path); if (!m) { @@ -283,6 +284,24 @@ auto cmd_build(const fs::path& project_root, bool no_build, bool release, } auto lock = merge_lockfile(*m, *recipes, *dev_recipes, prior); + std::optional vendor_index; + if (offline) { + auto vendor_path = vendor.value_or(project_root / "vendor.toml"); + if (std::error_code v_ec; !fs::exists(vendor_path, v_ec)) { + return std::unexpected(io_error( + std::format("--offline requires vendor.toml; expected at '{}'", + vendor_path.string()), + vendor_path)); + } + std::ifstream in_file{vendor_path}; + std::string body{std::istreambuf_iterator(in_file), {}}; + auto parsed = codegen::parse_vendor_toml(body); + if (!parsed) { + return std::unexpected(parsed.error()); + } + vendor_index = std::move(*parsed); + } + codegen::GenerateInputs in{ .manifest = *m, .layout = *layout_result, @@ -290,6 +309,7 @@ auto cmd_build(const fs::path& project_root, bool no_build, bool release, .recipes = *recipes, .dev_recipes = *dev_recipes, .project_root = project_root, + .vendor = vendor_index, }; auto flake_text = codegen::flake_nix(in); auto cmake_text = codegen::cmake_lists(in); @@ -326,16 +346,41 @@ auto cmd_build(const fs::path& project_root, bool no_build, bool release, "-G", "Ninja", std::format("-DCMAKE_BUILD_TYPE={}", profile_cap), }; - if (auto r = run_nix_cmake(project_root, configure_args, "configure"); !r) { - return std::unexpected(r.error()); - } - std::vector build_args{"--build", build_dir}; if (target) { build_args.push_back("--target"); build_args.push_back(*target); } - if (auto r = run_nix_cmake(project_root, build_args, "build"); !r) { + + auto run_cmake = [&](const std::vector& args, + std::string_view phase) -> util::Result { + if (offline) { + auto r = exec::run("cmake", args, + exec::ExecOptions{ + .cwd = project_root, + .env_overrides = {}, + .timeout = std::nullopt, + .inherit_stdio = true, + }); + if (!r) { + return std::unexpected(r.error()); + } + if (r->exit_code != 0) { + return std::unexpected(util::Error{ + util::ErrorCode::BuildCmakeFailed, + std::format("cmake {} failed (exit {})", phase, r->exit_code), + "", std::nullopt, std::nullopt, + }); + } + return {}; + } + return run_nix_cmake(project_root, args, phase); + }; + + if (auto r = run_cmake(configure_args, "configure"); !r) { + return std::unexpected(r.error()); + } + if (auto r = run_cmake(build_args, "build"); !r) { return std::unexpected(r.error()); } diff --git a/src/cli/cmd_vendor.cpp b/src/cli/cmd_vendor.cpp new file mode 100644 index 0000000..15b7a44 --- /dev/null +++ b/src/cli/cmd_vendor.cpp @@ -0,0 +1,128 @@ +module cargoxx.cli; + +import std; +import cargoxx.lockfile; +import cargoxx.resolver; +import cargoxx.util; + +namespace cargoxx::cli { + +namespace fs = std::filesystem; + +namespace { + +auto error(util::ErrorCode code, std::string msg, fs::path path) -> util::Error { + return util::Error{code, std::move(msg), "", std::move(path), std::nullopt}; +} + +auto escape_toml(std::string_view s) -> std::string { + std::string out; + out.reserve(s.size() + 2); + out += '"'; + for (char c : s) { + if (c == '"' || c == '\\') { + out += '\\'; + } + out += c; + } + out += '"'; + return out; +} + +} // namespace + +auto cmd_vendor(const fs::path& project_root, const fs::path& output) + -> util::Result { + auto lock_path = project_root / "Cargoxx.lock"; + auto lock = lockfile::parse(lock_path); + if (!lock) { + return std::unexpected(lock.error()); + } + if (!lock->nixpkgs_rev_pin || lock->nixpkgs_rev_pin->empty()) { + return std::unexpected(error( + util::ErrorCode::ManifestInvalidField, + "Cargoxx.lock has no top-level nixpkgs_rev — run cargoxx build " + "online first to pin it", + lock_path)); + } + if (!lock->flake_utils_rev_pin || lock->flake_utils_rev_pin->empty()) { + return std::unexpected(error( + util::ErrorCode::ManifestInvalidField, + "Cargoxx.lock has no top-level flake_utils_rev", + lock_path)); + } + + auto nixpkgs_src = resolver::realize_flake_source( + std::format("github:NixOS/nixpkgs/{}", *lock->nixpkgs_rev_pin)); + if (!nixpkgs_src) { + return std::unexpected(nixpkgs_src.error()); + } + auto flake_utils_src = resolver::realize_flake_source( + std::format("github:numtide/flake-utils/{}", *lock->flake_utils_rev_pin)); + if (!flake_utils_src) { + return std::unexpected(flake_utils_src.error()); + } + + std::string body; + body += "schema = 1\n\n"; + body += "[nixpkgs]\n"; + body += std::format("rev = {}\n", escape_toml(*lock->nixpkgs_rev_pin)); + body += std::format("store_path = {}\n\n", escape_toml(*nixpkgs_src)); + body += "[flake_utils]\n"; + body += std::format("rev = {}\n", escape_toml(*lock->flake_utils_rev_pin)); + body += std::format("store_path = {}\n", escape_toml(*flake_utils_src)); + + for (const auto& p : lock->packages) { + if (!p.linkdb_source) { + continue; + } + if (!p.nixpkgs_attr) { + return std::unexpected(error( + util::ErrorCode::ManifestInvalidField, + std::format("lockfile dep '{}' has no nixpkgs_attr", p.name), + lock_path)); + } + auto rev = p.nixpkgs_rev.has_value() && !p.nixpkgs_rev->empty() + ? *p.nixpkgs_rev + : *lock->nixpkgs_rev_pin; + auto store = resolver::realize_path_at_rev(rev, *p.nixpkgs_attr); + if (!store) { + return std::unexpected(store.error()); + } + std::optional dev_path; + if (auto d = resolver::realize_path_at_rev( + rev, std::format("{}.dev", *p.nixpkgs_attr)); + d) { + dev_path = *d; + } + + body += "\n[[dep]]\n"; + body += std::format("name = {}\n", escape_toml(p.name)); + body += std::format("nixpkgs_attr = {}\n", escape_toml(*p.nixpkgs_attr)); + body += std::format("nixpkgs_rev = {}\n", escape_toml(rev)); + body += std::format("store_path = {}\n", escape_toml(*store)); + if (dev_path) { + body += std::format("dev_path = {}\n", escape_toml(*dev_path)); + } + } + + std::error_code ec; + fs::create_directories(output.parent_path(), ec); + std::ofstream out{output}; + if (!out) { + return std::unexpected(error( + util::ErrorCode::Internal, + std::format("cannot open vendor file for writing: {}", + output.string()), + output)); + } + out << body; + if (!out) { + return std::unexpected(error( + util::ErrorCode::Internal, + std::format("write failed: {}", output.string()), output)); + } + return {}; +} + +} // namespace cargoxx::cli diff --git a/src/cli/run.cpp b/src/cli/run.cpp index 5920047..1c8c10b 100644 --- a/src/cli/run.cpp +++ b/src/cli/run.cpp @@ -23,13 +23,26 @@ auto run(int argc, char** argv) -> int { "build", "Generate flake.nix and build/CMakeLists.txt; build with nix+cmake"); bool build_no_build = false; bool build_release = false; + bool build_offline = false; std::string build_target; + std::string build_vendor; build_cmd->add_flag("--no-build", build_no_build, "Generate files only; do not invoke nix/cmake"); build_cmd->add_flag("--release", build_release, "Build the release profile"); + build_cmd->add_flag("--offline", build_offline, + "Skip network probes and nix develop wrappers. " + "Reads vendor.toml for store-path inputs."); + build_cmd->add_option("--vendor", build_vendor, + "Path to vendor.toml (used with --offline; default ./vendor.toml)"); build_cmd->add_option("--target", build_target, "Build a specific target (passed to cmake --build)"); + auto* vendor_cmd = app.add_subcommand( + "vendor", "Resolve every locked dependency into /nix/store and write vendor.toml"); + std::string vendor_output; + vendor_cmd->add_option("--output", vendor_output, + "Path to write vendor.toml (default ./vendor.toml)"); + auto* run_cmd = app.add_subcommand("run", "Build and run a binary target"); bool run_release = false; std::string run_bin; @@ -109,7 +122,12 @@ auto run(int argc, char** argv) -> int { if (!build_target.empty()) { target = build_target; } - auto r = cmd_build(cwd, build_no_build, build_release, target); + std::optional vendor_path; + if (!build_vendor.empty()) { + vendor_path = build_vendor; + } + auto r = cmd_build(cwd, build_no_build, build_release, target, + std::nullopt, build_offline, vendor_path); if (!r) { std::cerr << util::format(r.error()); return 1; @@ -122,6 +140,18 @@ auto run(int argc, char** argv) -> int { return 0; } + if (*vendor_cmd) { + auto out = vendor_output.empty() ? cwd / "vendor.toml" + : std::filesystem::path{vendor_output}; + auto r = cmd_vendor(cwd, out); + if (!r) { + std::cerr << util::format(r.error()); + return 1; + } + std::cout << std::format(" Wrote {}\n", out.string()); + return 0; + } + if (*run_cmd) { std::optional bin; if (!run_bin.empty()) { diff --git a/src/codegen/codegen.cppm b/src/codegen/codegen.cppm index 23e6783..b28e465 100644 --- a/src/codegen/codegen.cppm +++ b/src/codegen/codegen.cppm @@ -9,6 +9,16 @@ import cargoxx.lockfile; export namespace cargoxx::codegen { +// When set, codegen emits the generated flake's nixpkgs / flake-utils +// inputs as `path:/nix/store/...` references (already-realized source +// paths) instead of `github:NixOS/nixpkgs/` URLs. This makes the +// inner build hermetic — no network and no nix daemon access required. +struct VendorIndex { + std::string nixpkgs_store_path; + std::string flake_utils_store_path; + std::unordered_map dep_store_paths; // by attr +}; + // All inputs the generators need. Held by const reference; the caller owns // the underlying objects. Not copyable. struct GenerateInputs { @@ -18,9 +28,14 @@ struct GenerateInputs { std::vector recipes; // one per manifest dep, same order std::vector dev_recipes; // one per dev_dependency, same order std::filesystem::path project_root; + std::optional vendor; }; auto flake_nix(const GenerateInputs& in) -> std::string; auto cmake_lists(const GenerateInputs& in) -> std::string; +// Pure: parses a vendor.toml (see cmd_vendor) into a VendorIndex. +auto parse_vendor_toml(std::string_view body) + -> util::Result; + } // namespace cargoxx::codegen diff --git a/src/codegen/flake.cpp b/src/codegen/flake.cpp index 3b07657..394e407 100644 --- a/src/codegen/flake.cpp +++ b/src/codegen/flake.cpp @@ -100,22 +100,36 @@ auto pinned_inputs_dedup(const std::vector& bindings) } auto emit_inputs_block(const std::vector& pinned, - const lockfile::Lockfile& lock) -> std::string { - auto nixpkgs_url = - lock.nixpkgs_rev_pin && !lock.nixpkgs_rev_pin->empty() - ? std::format("github:NixOS/nixpkgs/{}", *lock.nixpkgs_rev_pin) - : std::string{"github:NixOS/nixpkgs/nixos-unstable"}; - auto flake_utils_url = - lock.flake_utils_rev_pin && !lock.flake_utils_rev_pin->empty() - ? std::format("github:numtide/flake-utils/{}", - *lock.flake_utils_rev_pin) - : std::string{"github:numtide/flake-utils"}; + const lockfile::Lockfile& lock, + const std::optional& vendor) + -> std::string { + auto nixpkgs_url = [&]() -> std::string { + if (vendor && !vendor->nixpkgs_store_path.empty()) { + return std::format("path:{}", vendor->nixpkgs_store_path); + } + if (lock.nixpkgs_rev_pin && !lock.nixpkgs_rev_pin->empty()) { + return std::format("github:NixOS/nixpkgs/{}", *lock.nixpkgs_rev_pin); + } + return "github:NixOS/nixpkgs/nixos-unstable"; + }(); + auto flake_utils_url = [&]() -> std::string { + if (vendor && !vendor->flake_utils_store_path.empty()) { + return std::format("path:{}", vendor->flake_utils_store_path); + } + if (lock.flake_utils_rev_pin && !lock.flake_utils_rev_pin->empty()) { + return std::format("github:numtide/flake-utils/{}", + *lock.flake_utils_rev_pin); + } + return "github:numtide/flake-utils"; + }(); std::string out = " inputs = {\n" + std::format(" nixpkgs.url = \"{}\";\n", nixpkgs_url); - for (const auto* b : pinned) { - out += std::format(" {}.url = \"github:NixOS/nixpkgs/{}\";\n", - b->sanitized, *b->rev); + if (!vendor) { + for (const auto* b : pinned) { + out += std::format(" {}.url = \"github:NixOS/nixpkgs/{}\";\n", + b->sanitized, *b->rev); + } } out += std::format(" flake-utils.url = \"{}\";\n", flake_utils_url); out += " };\n"; @@ -171,7 +185,7 @@ auto flake_nix(const GenerateInputs& in) -> std::string { out += "{\n"; out += std::format(" description = \"{}\";\n\n", in.manifest.package.name); - out += emit_inputs_block(pinned, in.lock); + out += emit_inputs_block(pinned, in.lock, in.vendor); const bool any_pkg_config = std::ranges::any_of(in.recipes, diff --git a/src/codegen/vendor.cpp b/src/codegen/vendor.cpp new file mode 100644 index 0000000..7f1483c --- /dev/null +++ b/src/codegen/vendor.cpp @@ -0,0 +1,69 @@ +module; + +#include + +module cargoxx.codegen; + +import std; +import cargoxx.util; + +namespace cargoxx::codegen { + +auto parse_vendor_toml(std::string_view body) -> util::Result { + toml::table root; + try { + root = toml::parse(body); + } catch (const toml::parse_error& e) { + return std::unexpected(util::Error{ + util::ErrorCode::ManifestParseError, + std::format("vendor.toml is not valid TOML: {}", e.description()), + "", std::nullopt, std::nullopt, + }); + } + + auto missing = [](std::string msg) { + return util::Error{ + util::ErrorCode::ManifestInvalidField, std::move(msg), "", + std::nullopt, std::nullopt, + }; + }; + + VendorIndex out; + if (const auto* tbl = root["nixpkgs"].as_table()) { + if (auto v = (*tbl)["store_path"].value()) { + out.nixpkgs_store_path = *v; + } else { + return std::unexpected(missing("vendor.toml: [nixpkgs].store_path is required")); + } + } else { + return std::unexpected(missing("vendor.toml: [nixpkgs] table is required")); + } + if (const auto* tbl = root["flake_utils"].as_table()) { + if (auto v = (*tbl)["store_path"].value()) { + out.flake_utils_store_path = *v; + } else { + return std::unexpected(missing("vendor.toml: [flake_utils].store_path is required")); + } + } else { + return std::unexpected(missing("vendor.toml: [flake_utils] table is required")); + } + + if (const auto* arr = root["dep"].as_array()) { + for (const auto& el : *arr) { + const auto* tbl = el.as_table(); + if (!tbl) { + return std::unexpected(missing("vendor.toml: [[dep]] entries must be tables")); + } + auto attr = (*tbl)["nixpkgs_attr"].value(); + auto path = (*tbl)["store_path"].value(); + if (!attr || !path) { + return std::unexpected(missing( + "vendor.toml: each [[dep]] needs nixpkgs_attr and store_path")); + } + out.dep_store_paths.emplace(*attr, *path); + } + } + return out; +} + +} // namespace cargoxx::codegen diff --git a/src/resolver/nixpkgs_probe.cpp b/src/resolver/nixpkgs_probe.cpp index 5358a58..7b05762 100644 --- a/src/resolver/nixpkgs_probe.cpp +++ b/src/resolver/nixpkgs_probe.cpp @@ -162,4 +162,95 @@ auto realize_path(const std::string& flake_attr) -> util::Result { return path; } +auto realize_path_at_rev(const std::string& rev, const std::string& attr) + -> util::Result { + if (rev.empty() || attr.empty()) { + return std::unexpected(make_error( + util::ErrorCode::ResolutionUnknownPackage, + "realize_path_at_rev: rev and attr must be non-empty")); + } + std::vector args{ + "--extra-experimental-features", "nix-command flakes", + "build", std::format("github:NixOS/nixpkgs/{}#{}", rev, attr), + "--no-link", "--print-out-paths", + }; + auto r = exec::run("nix", args, + exec::ExecOptions{ + .cwd = {}, + .env_overrides = {}, + .timeout = std::chrono::seconds{600}, + .inherit_stdio = false, + }); + if (!r) { + return std::unexpected(r.error()); + } + if (r->exit_code != 0) { + if (looks_like_missing_attribute(r->stderr_text)) { + return std::unexpected(make_error( + util::ErrorCode::ResolutionUnknownPackage, + std::format("nixpkgs/{} has no attribute '{}'", rev, attr))); + } + return std::unexpected(make_error( + util::ErrorCode::ResolutionNetworkError, + std::format("nix build failed (exit {}): {}", r->exit_code, + r->stderr_text))); + } + auto path = r->stdout_text; + while (!path.empty() && (path.back() == '\n' || path.back() == ' ')) { + path.pop_back(); + } + if (path.empty()) { + return std::unexpected(make_error( + util::ErrorCode::ResolutionNetworkError, + std::format("nix build emitted no path for '{}#{}'", rev, attr))); + } + return path; +} + +auto realize_flake_source(const std::string& flake_ref) + -> util::Result { + if (flake_ref.empty()) { + return std::unexpected(make_error( + util::ErrorCode::ResolutionUnknownPackage, + "realize_flake_source: flake_ref is empty")); + } + std::vector args{ + "--extra-experimental-features", "nix-command flakes", + "flake", "prefetch", flake_ref, "--json", + }; + auto r = exec::run("nix", args, + exec::ExecOptions{ + .cwd = {}, + .env_overrides = {}, + .timeout = std::chrono::seconds{300}, + .inherit_stdio = false, + }); + if (!r) { + return std::unexpected(r.error()); + } + if (r->exit_code != 0) { + return std::unexpected(make_error( + util::ErrorCode::ResolutionNetworkError, + std::format("nix flake prefetch failed (exit {}): {}", + r->exit_code, r->stderr_text))); + } + std::string_view body = r->stdout_text; + constexpr std::string_view key = "\"storePath\":\""; + auto pos = body.find(key); + if (pos == std::string_view::npos) { + return std::unexpected(make_error( + util::ErrorCode::ResolutionNetworkError, + std::format("nix flake prefetch emitted no storePath for '{}'", + flake_ref))); + } + pos += key.size(); + auto end = body.find('"', pos); + if (end == std::string_view::npos) { + return std::unexpected(make_error( + util::ErrorCode::ResolutionNetworkError, + "nix flake prefetch JSON malformed")); + } + return std::string{body.substr(pos, end - pos)}; +} + } // namespace cargoxx::resolver diff --git a/src/resolver/resolver.cppm b/src/resolver/resolver.cppm index abf5b16..db067ab 100644 --- a/src/resolver/resolver.cppm +++ b/src/resolver/resolver.cppm @@ -45,6 +45,20 @@ auto nixpkgs_probe(const std::string& attr) -> util::Result; // for build / network errors. auto realize_path(const std::string& flake_attr) -> util::Result; +// Like `realize_path`, but pins the nixpkgs revision instead of using the +// registry alias. Builds `github:NixOS/nixpkgs/#` and returns +// the resulting `/nix/store/...` path. Used by the vendor subcommand to +// materialize each lockfile-pinned dep without depending on the user's +// flake registry. +auto realize_path_at_rev(const std::string& rev, const std::string& attr) + -> util::Result; + +// Returns the source store path for `github:NixOS/nixpkgs/` (the +// path Nix would set as `nixpkgs.outPath` when imported). Used by the +// vendor subcommand to record nixpkgs/flake-utils source locations. +auto realize_flake_source(const std::string& flake_ref) + -> util::Result; + // One CMake config-file's IMPORTED targets together with the find_package // expression derived from its filename stem. struct NixCmakeCandidate {