From 444bbb6ac5ab9f4d9a8446e74f64f1e354e6af6a Mon Sep 17 00:00:00 2001 From: vhyrro Date: Tue, 22 Sep 2026 18:47:04 +0200 Subject: [PATCH 01/39] temp: extra tests --- lux-lib/src/build/mod.rs | 146 ++++++++++++++++++++ lux-lib/src/luarocks/install_binary_rock.rs | 7 + lux-lib/src/manifest/metadata.rs | 42 ++++++ lux-lib/tests/install.rs | 50 ++++++- 4 files changed, 243 insertions(+), 2 deletions(-) diff --git a/lux-lib/src/build/mod.rs b/lux-lib/src/build/mod.rs index f3f2b4753..fa14fd0c5 100644 --- a/lux-lib/src/build/mod.rs +++ b/lux-lib/src/build/mod.rs @@ -540,8 +540,10 @@ mod tests { config::ConfigBuilder, lua_installation::{detect_installed_lua_version, LuaInstallation}, lua_version::LuaVersion, + operations::{unpack_rockspec, DownloadedPackedRockBytes}, project::Project, rockspec::RockBinaries, + tree::Tree, }; #[tokio::test] @@ -624,4 +626,148 @@ mod tests { bin_file.assert(predicate::str::contains("#!/usr/bin/env bash")); bin_file.assert(predicate::str::contains("echo \"Hello\"")); } + + const LUATEST_SRC_ROCK: &str = "resources/test/luatest-0.2-1.src.rock"; + /// `sha256` of the checked-in `.src.rock` archive, i.e. what `Materialize` must + /// record as the `source` integrity. + const LUATEST_SRC_ROCK_SOURCE_HASH: &str = "sha256-2jS0XOq0iIVhsZJ3BVqXSlKsx2vsqCAaaYyqcBEb7RI="; + /// `sha256` of the rockspec text embedded in the `.src.rock`, i.e. the `rockspec` integrity. + const LUATEST_ROCKSPEC_HASH: &str = "sha256-NljJ20A+VadUyhhBjrRnojeQjSqRpQy7FWcgjUt2Fdc="; + + fn luatest_src_rock_bytes() -> Bytes { + Bytes::copy_from_slice( + &std::fs::read(PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(LUATEST_SRC_ROCK)) + .unwrap(), + ) + } + + fn luatest_config(dir: &assert_fs::TempDir) -> Config { + ConfigBuilder::new() + .unwrap() + .user_tree(Some(dir.to_path_buf())) + .lua_version(Some(LuaVersion::Lua51)) + .build() + .unwrap() + } + + /// Builds `luatest` from the checked-in `.src.rock` fixture, entirely offline. + /// This exercises the fetch + hash + build + deploy path end to end. + async fn build_luatest( + config: &Config, + tree: &Tree, + entry_type: EntryType, + behaviour: BuildBehaviour, + ) -> LocalPackage { + let bytes = luatest_src_rock_bytes(); + let rock = DownloadedPackedRockBytes { + name: "luatest".into(), + version: "0.2-1".parse().unwrap(), + bytes: bytes.clone(), + file_name: "luatest-0.2-1.src.rock".into(), + url: "https://example.org/luatest-0.2-1.src.rock".parse().unwrap(), + }; + let rockspec = unpack_rockspec(&rock).await.unwrap(); + let lua = LuaInstallation::new_from_config(config).await.unwrap(); + Build::new() + .rockspec(&rockspec) + .lua(&lua) + .tree(tree) + .entry_type(entry_type) + .config(config) + .behaviour(behaviour) + .source_spec(RemotePackageSourceSpec::SrcRock(SrcRockSource { + bytes, + source_url: RemotePackageSourceUrl::Url { + url: "https://example.org/luatest-0.2-1.src.rock".parse().unwrap(), + }, + })) + .build() + .await + .unwrap() + } + + /// The materialized `LocalPackage` must carry the exact source and rockspec + /// integrities, derived from the fetched bytes. This is the contract that + /// `lx generate-lockfile` and the manifest cache depend on. + #[tokio::test] + async fn materialize_src_rock_produces_expected_hashes() { + let dir = assert_fs::TempDir::new().unwrap(); + let config = luatest_config(&dir); + let tree = config.user_tree(LuaVersion::Lua51).unwrap(); + + let package = + build_luatest(&config, &tree, EntryType::Entrypoint, BuildBehaviour::Force).await; + + assert_eq!( + package.hashes().source, + LUATEST_SRC_ROCK_SOURCE_HASH.parse().unwrap() + ); + assert_eq!( + package.hashes().rockspec, + LUATEST_ROCKSPEC_HASH.parse().unwrap() + ); + // The source integrity is the hash of the fetched artifact. + assert_eq!( + package.hashes().source, + luatest_src_rock_bytes().hash().await.unwrap() + ); + // Entrypoints expose their binaries. + assert!(tree.bin().join("luatest").is_file()); + } + + /// Two independent forced builds of the same rockspec must produce identical + /// lockfile entries (id, source, source_url, hashes). This guards the + /// determinism that reproducible lockfiles depend on. + #[tokio::test] + async fn force_build_is_deterministic() { + let dir1 = assert_fs::TempDir::new().unwrap(); + let config1 = luatest_config(&dir1); + let tree1 = config1.user_tree(LuaVersion::Lua51).unwrap(); + + let dir2 = assert_fs::TempDir::new().unwrap(); + let config2 = luatest_config(&dir2); + let tree2 = config2.user_tree(LuaVersion::Lua51).unwrap(); + + let package1 = + build_luatest(&config1, &tree1, EntryType::Entrypoint, BuildBehaviour::Force).await; + let package2 = + build_luatest(&config2, &tree2, EntryType::Entrypoint, BuildBehaviour::Force).await; + + assert_eq!(package1, package2); + assert_eq!(package1.hashes(), package2.hashes()); + } + + /// `NoForce` must short-circuit on the tree lockfile without touching the + /// tree, while `Force` must rebuild and redeploy. + #[tokio::test] + async fn noforce_skips_rebuild_when_package_is_locked() { + let dir = assert_fs::TempDir::new().unwrap(); + let config = luatest_config(&dir); + let tree = config.user_tree(LuaVersion::Lua51).unwrap(); + + let package = + build_luatest(&config, &tree, EntryType::Entrypoint, BuildBehaviour::Force).await; + let bin = tree.bin().join("luatest"); + assert!(bin.is_file()); + + // Simulate a committed install by writing the package into the tree lockfile. + { + let mut lockfile = tree.lockfile().unwrap().write_guard(); + lockfile.add_entrypoint(&package); + } + + // Simulate a partially broken install. + std::fs::remove_file(&bin).unwrap(); + + let skipped = + build_luatest(&config, &tree, EntryType::Entrypoint, BuildBehaviour::NoForce).await; + assert_eq!(skipped.id(), package.id()); + assert!( + !bin.is_file(), + "NoForce must not rebuild a package that is already in the lockfile" + ); + + build_luatest(&config, &tree, EntryType::Entrypoint, BuildBehaviour::Force).await; + assert!(bin.is_file(), "Force must rebuild the package"); + } } diff --git a/lux-lib/src/luarocks/install_binary_rock.rs b/lux-lib/src/luarocks/install_binary_rock.rs index 62532ca27..33281d44b 100644 --- a/lux-lib/src/luarocks/install_binary_rock.rs +++ b/lux-lib/src/luarocks/install_binary_rock.rs @@ -297,6 +297,13 @@ mod tests { .install() .await .unwrap(); + // The `source` integrity of a binary rock is the hash of the packed rock bytes. + assert_eq!( + local_package.hashes().source, + "sha256-zDmnjE2b8GxX+6j5vTCWUEBxe0aFXFVhC4JOrnAgQ2g=" + .parse() + .unwrap() + ); let foo_bar_module = tree .layout_for(&local_package) .src diff --git a/lux-lib/src/manifest/metadata.rs b/lux-lib/src/manifest/metadata.rs index da5c10665..e2d62328c 100644 --- a/lux-lib/src/manifest/metadata.rs +++ b/lux-lib/src/manifest/metadata.rs @@ -223,4 +223,46 @@ mod tests { }; assert!(metadata.latest_match(&package_req, &filter).is_none()); } + + /// Lux prefers binary rocks over rockspecs over `.src.rock` archives + /// (`RemotePackageType` ordering). `Discover` relies on this to pick a source. + #[tokio::test] + pub async fn latest_match_prefers_binary_then_rockspec_then_src() { + // `arch = "all"` is treated as a binary rock on every platform. + let manifest = r#" + commands = {} + modules = {} + repository = { + ['testpkg'] = { + ['1.0-1'] = { + { arch = "rockspec" }, { arch = "src" }, { arch = "all" } + } + } + } + "# + .to_string(); + let metadata = ManifestMetadata::new(&manifest).unwrap(); + let package_req: PackageReq = "testpkg == 1.0-1".parse().unwrap(); + + let (_, rock_type) = metadata + .latest_match(&package_req, &Default::default()) + .unwrap(); + assert_eq!(rock_type, RemotePackageType::Binary); + + let no_binary = RemotePackageTypeFilterSpec { + rockspec: true, + src: true, + binary: false, + }; + let (_, rock_type) = metadata.latest_match(&package_req, &no_binary).unwrap(); + assert_eq!(rock_type, RemotePackageType::Rockspec); + + let src_only = RemotePackageTypeFilterSpec { + rockspec: false, + src: true, + binary: false, + }; + let (_, rock_type) = metadata.latest_match(&package_req, &src_only).unwrap(); + assert_eq!(rock_type, RemotePackageType::Src); + } } diff --git a/lux-lib/tests/install.rs b/lux-lib/tests/install.rs index 26ccb7a1f..a0c2eea88 100644 --- a/lux-lib/tests/install.rs +++ b/lux-lib/tests/install.rs @@ -8,8 +8,8 @@ use lux_lib::{ lua_rockspec::RockSourceSpec, lua_version::LuaVersion, operations::{Exec, Install, PackageInstallSpec}, - package::PackageReq, - tree::EntryType, + package::{PackageName, PackageReq}, + tree::{EntryType, InstallTree}, }; use std::path::PathBuf; use walkdir::WalkDir; @@ -131,3 +131,49 @@ async fn test_install(install_spec: PackageInstallSpec) { .unwrap(); assert!(!installed.is_empty()); } + +/// Guards the resolved dependency graph recorded in the install tree lockfile: +/// entrypoints, dependency edges, and dependency-only classification. +#[flaky_test(tokio, times = 5)] +async fn install_records_dependency_edges_in_tree_lockfile() { + let dir = TempDir::new().unwrap(); + let lua_version = detect_installed_lua_version().or(Some(LuaVersion::Lua51)); + + let config = ConfigBuilder::new() + .unwrap() + .user_tree(Some(dir.to_path_buf())) + .lua_version(lua_version) + .build() + .unwrap(); + + let tree = config + .user_tree(LuaVersion::from(&config).unwrap().clone()) + .unwrap(); + + // `luassert` depends on `say`. + let install_spec = + PackageInstallSpec::new("luassert@1.9.0-1".parse().unwrap(), EntryType::Entrypoint).build(); + Install::new(&config) + .package(install_spec) + .tree(tree.clone()) + .install() + .await + .unwrap(); + + let lockfile = tree.lockfile().unwrap(); + let luassert = lockfile + .entrypoint(&PackageName::new("luassert".into())) + .expect("luassert must be recorded as an entrypoint"); + assert_eq!(luassert.version(), &"1.9.0-1".parse().unwrap()); + + let say = luassert + .dependencies() + .into_iter() + .filter_map(|id| lockfile.get(id)) + .find(|pkg| pkg.name().to_string() == "say") + .expect("luassert's lockfile entry must depend on say"); + assert!( + !lockfile.is_entrypoint(&say.id()), + "transitive dependencies must be recorded as dependency-only" + ); +} From af656ffddb2968ed878cdeea7c82aee8a7c52b5b Mon Sep 17 00:00:00 2001 From: vhyrro Date: Tue, 22 Sep 2026 20:28:52 +0200 Subject: [PATCH 02/39] feat!: pipeline implementation --- lux-cli/src/build.rs | 12 +- lux-cli/src/dist/flat_archive.rs | 8 +- lux-cli/src/doc.rs | 8 +- lux-cli/src/list.rs | 4 +- lux-cli/src/outdated.rs | 4 +- lux-cli/src/sync.rs | 21 +- lux-cli/src/uninstall.rs | 4 +- lux-lib/src/build/backend.rs | 4 +- lux-lib/src/build/luarocks.rs | 4 +- lux-lib/src/build/mod.rs | 304 +++---- lux-lib/src/lockfile/mod.rs | 268 ++++--- lux-lib/src/luarocks/install_binary_rock.rs | 8 +- lux-lib/src/operations/build_workspace.rs | 194 +---- lux-lib/src/operations/dist_bin.rs | 8 +- lux-lib/src/operations/gen_luarc.rs | 6 +- lux-lib/src/operations/install/mod.rs | 744 +----------------- .../src/operations/install_dependencies.rs | 19 +- lux-lib/src/operations/install_project.rs | 4 +- lux-lib/src/operations/install_rockspec.rs | 4 +- lux-lib/src/operations/mod.rs | 1 + lux-lib/src/operations/pack.rs | 4 +- lux-lib/src/operations/pin.rs | 8 +- lux-lib/src/operations/pipeline/build.rs | 216 +++++ lux-lib/src/operations/pipeline/discover.rs | 114 +++ .../pipeline/download_sources_and_hash.rs | 272 +++++++ .../operations/pipeline/install_packages.rs | 291 +++++++ lux-lib/src/operations/pipeline/mod.rs | 348 ++++++++ lux-lib/src/operations/pipeline/resolve.rs | 384 +++++++++ lux-lib/src/operations/resolve.rs | 20 +- lux-lib/src/operations/sync.rs | 363 +++++---- lux-lib/src/operations/test.rs | 26 +- lux-lib/src/operations/uninstall.rs | 16 +- lux-lib/src/operations/update.rs | 42 +- lux-lib/src/operations/vendor.rs | 18 +- lux-lib/src/remote_package_db/mod.rs | 14 +- lux-lib/src/tree/dist.rs | 12 +- lux-lib/src/tree/layout.rs | 22 +- lux-lib/src/tree/list.rs | 4 +- lux-lib/src/tree/mod.rs | 40 +- lux-lua/src/loader.rs | 8 +- lux-lua/src/lua_impls.rs | 88 +-- lux-lua/src/operations.rs | 28 +- 42 files changed, 2407 insertions(+), 1560 deletions(-) create mode 100644 lux-lib/src/operations/pipeline/build.rs create mode 100644 lux-lib/src/operations/pipeline/discover.rs create mode 100644 lux-lib/src/operations/pipeline/download_sources_and_hash.rs create mode 100644 lux-lib/src/operations/pipeline/install_packages.rs create mode 100644 lux-lib/src/operations/pipeline/mod.rs create mode 100644 lux-lib/src/operations/pipeline/resolve.rs diff --git a/lux-cli/src/build.rs b/lux-cli/src/build.rs index 8d15533b8..7ee4597ba 100644 --- a/lux-cli/src/build.rs +++ b/lux-cli/src/build.rs @@ -1,8 +1,8 @@ use clap::Args; use lux_lib::{ config::Config, - lockfile::LocalPackage, - operations::{self}, + lockfile::LockedPackage, + operations::pipeline::Pipeline, package::PackageName, workspace::Workspace, }; @@ -24,13 +24,13 @@ pub struct Build { } /// Returns `Some` if the `only_deps` arg is set to `false`. -pub async fn build(data: Build, config: Config) -> Result> { +pub async fn build(data: Build, config: Config) -> Result> { let workspace = Workspace::current_or_err()?; - let result = operations::BuildWorkspace::new(&workspace, &config) + let result = Pipeline::new(&config, &workspace) .maybe_package(data.package) .no_lock(data.no_lock) - .only_deps(data.only_deps) - .build() + .build_projects(!data.only_deps) + .run() .await?; Ok(result) } diff --git a/lux-cli/src/dist/flat_archive.rs b/lux-cli/src/dist/flat_archive.rs index 04c1d6cae..914352294 100644 --- a/lux-cli/src/dist/flat_archive.rs +++ b/lux-cli/src/dist/flat_archive.rs @@ -7,7 +7,7 @@ use clap::Args; use lux_lib::{ build::{Build, BuildBehaviour}, config::{Config, ConfigBuilder}, - lockfile::LocalPackage, + lockfile::LockedPackage, lua_installation::LuaInstallation, lua_rockspec::RemoteLuaRockspec, lua_version::LuaVersion, @@ -132,7 +132,7 @@ async fn install_project( package: Option<&PackageName>, staging_dir: &TempDir, config: &Config, -) -> Result<(LocalPackage, PathBuf)> { +) -> Result<(LockedPackage, PathBuf)> { let workspace = Workspace::current_or_err()?; let project = match package { Some(package) => workspace.select_member(package)?, @@ -155,7 +155,7 @@ async fn install_package( package: &PackageReq, staging_dir: &TempDir, config: &Config, -) -> Result<(LocalPackage, PathBuf)> { +) -> Result<(LockedPackage, PathBuf)> { let lua_version = LuaVersion::from(config)?.clone(); let tree = FlatDistTree::new(staging_dir.path().to_path_buf(), lua_version, config)?; let packages = Install::new(config) @@ -178,7 +178,7 @@ async fn install_rockspec( rockspec_path: &Path, staging_dir: &TempDir, config: &Config, -) -> Result<(LocalPackage, PathBuf)> { +) -> Result<(LockedPackage, PathBuf)> { let content = tokio::fs::read_to_string(&rockspec_path) .await .into_diagnostic()?; diff --git a/lux-cli/src/doc.rs b/lux-cli/src/doc.rs index 9666f8053..5e6d883c0 100644 --- a/lux-cli/src/doc.rs +++ b/lux-cli/src/doc.rs @@ -3,7 +3,7 @@ use inquire::{Confirm, Select}; use itertools::Itertools; use lux_lib::{ config::Config, - lockfile::LocalPackage, + lockfile::LockedPackage, lua_rockspec::RemoteLuaRockspec, lua_version::LuaVersion, package::PackageReq, @@ -50,7 +50,7 @@ pub async fn doc(args: Doc, config: Config) -> Result<()> { } } -async fn open_homepage(pkg: LocalPackage, tree: &Tree) -> Result<()> { +async fn open_homepage(pkg: LockedPackage, tree: &Tree) -> Result<()> { let homepage = match get_homepage(&pkg, tree)? { Some(homepage) => Ok(homepage), None => Err(miette!( @@ -62,7 +62,7 @@ async fn open_homepage(pkg: LocalPackage, tree: &Tree) -> Result<()> { Ok(()) } -fn get_homepage(pkg: &LocalPackage, tree: &Tree) -> Result> { +fn get_homepage(pkg: &LockedPackage, tree: &Tree) -> Result> { let rockspec_content = std::fs::read_to_string(tree.layout_for(pkg).rockspec_path()).into_diagnostic()?; let rockspec = RemoteLuaRockspec::new(&rockspec_content)?; @@ -88,7 +88,7 @@ fn open_doc_file(path: &Path) -> Result<()> { Ok(()) } -async fn open_local_docs(pkg: LocalPackage, tree: &Tree, config: &Config) -> Result<()> { +async fn open_local_docs(pkg: LockedPackage, tree: &Tree, config: &Config) -> Result<()> { let doc_dir = tree.layout_for(&pkg).doc; let files: Vec = WalkDir::new(&doc_dir) .into_iter() diff --git a/lux-cli/src/list.rs b/lux-cli/src/list.rs index 3e21e4d17..36532915d 100644 --- a/lux-cli/src/list.rs +++ b/lux-cli/src/list.rs @@ -4,7 +4,7 @@ use clap::Args; use itertools::Itertools as _; use lux_lib::{ config::Config, - lockfile::{LocalPackageId, PinnedState}, + lockfile::{LockedPackageId, PinnedState}, lua_version::LuaVersion, tree::InstallTree, }; @@ -33,7 +33,7 @@ pub fn list_installed(list_data: ListCmd, config: Config) -> Result<()> { let lockfile = tree.lockfile()?; let mut available_rocks = tree.list()?; if list_data.orphans { - let reachable: HashSet = lockfile + let reachable: HashSet = lockfile .reachable_rocks() .into_iter() .map(|package| package.id()) diff --git a/lux-cli/src/outdated.rs b/lux-cli/src/outdated.rs index cd848a991..943b18fa9 100644 --- a/lux-cli/src/outdated.rs +++ b/lux-cli/src/outdated.rs @@ -3,7 +3,7 @@ use std::collections::HashMap; use clap::Args; use itertools::Itertools; use lux_lib::{ - config::Config, lockfile::LocalPackage, lua_version::LuaVersion, package::PackageVersion, + config::Config, lockfile::LockedPackage, lua_version::LuaVersion, package::PackageVersion, remote_package_db::RemotePackageDB, workspace::Workspace, }; @@ -49,7 +49,7 @@ pub async fn outdated(outdated_data: Outdated, config: Config) -> Result<()> { .map(|mb_version| mb_version.map(|version| (rock, version))) }) .filter_map_ok(|mb_tuple| mb_tuple) - .try_collect::<_, Vec<(&LocalPackage, PackageVersion)>, _>()?; + .try_collect::<_, Vec<(&LockedPackage, PackageVersion)>, _>()?; let rock_list = rock_list .iter() diff --git a/lux-cli/src/sync.rs b/lux-cli/src/sync.rs index 9e439fb88..44e812907 100644 --- a/lux-cli/src/sync.rs +++ b/lux-cli/src/sync.rs @@ -1,5 +1,5 @@ use clap::Args; -use lux_lib::{config::Config, operations::Sync, workspace::Workspace}; +use lux_lib::{config::Config, operations::pipeline::Pipeline, workspace::Workspace}; use miette::Result; @@ -12,25 +12,16 @@ pub struct SyncProject { /// Sync the current project's installed packages with its lux.toml. pub async fn sync(args: SyncProject, config: Config) -> Result<()> { + // FIXME(vhyrro): reimplement + let _ = args.no_integrity_check; let workspace = Workspace::current_or_err()?; - let report = Sync::new(&workspace, &config) - .validate_integrity(!args.no_integrity_check) + Pipeline::new(&config, &workspace) .test(true) - .sync() + .run() .await?; - if report.added().is_empty() && report.removed().is_empty() { - println!("Already in sync."); - return Ok(()); - } - - for pkg in report.added() { - println!("+ {} {}", pkg.name(), pkg.version()); - } - for pkg in report.removed() { - println!("- {} {}", pkg.name(), pkg.version()); - } + // FIXME(vhyrro): Readd report publishing Ok(()) } diff --git a/lux-cli/src/uninstall.rs b/lux-cli/src/uninstall.rs index ba10bf535..9f8e83349 100644 --- a/lux-cli/src/uninstall.rs +++ b/lux-cli/src/uninstall.rs @@ -4,7 +4,7 @@ use itertools::Itertools; use lux_lib::{ build::BuildBehaviour, config::Config, - lockfile::LocalPackageId, + lockfile::LockedPackageId, lua_version::LuaVersion, operations::{self, PackageInstallSpec}, package::PackageReq, @@ -83,7 +83,7 @@ multiple packages satisfying your version requirements were found: )); } - let (dependencies, entrypoints): (Vec, Vec) = packages + let (dependencies, entrypoints): (Vec, Vec) = packages .iter() .cloned() .partition(|pkg_id| lockfile.is_dependency(pkg_id)); diff --git a/lux-lib/src/build/backend.rs b/lux-lib/src/build/backend.rs index f889d1c21..20070e1c1 100644 --- a/lux-lib/src/build/backend.rs +++ b/lux-lib/src/build/backend.rs @@ -7,14 +7,14 @@ use std::{ use bon::Builder; use crate::{ - build::external_dependency::ExternalDependencyInfo, config::Config, lockfile::LocalPackage, + build::external_dependency::ExternalDependencyInfo, config::Config, lockfile::LockedPackage, lua_installation::LuaInstallation, lua_rockspec::DeploySpec, tree::InstallTree, }; #[derive(Builder)] #[builder(start_fn(name = "new"))] pub(crate) struct RunBuildArgs<'a, T: InstallTree> { - pub(crate) package: &'a LocalPackage, + pub(crate) package: &'a LockedPackage, pub(crate) no_install: bool, pub(crate) lua: &'a LuaInstallation, pub(crate) external_dependencies: &'a HashMap, diff --git a/lux-lib/src/build/luarocks.rs b/lux-lib/src/build/luarocks.rs index 0fb665d70..2b899717d 100644 --- a/lux-lib/src/build/luarocks.rs +++ b/lux-lib/src/build/luarocks.rs @@ -10,7 +10,7 @@ use std::path::Path; use crate::{ config::Config, - lockfile::LocalPackage, + lockfile::LockedPackage, luarocks::luarocks_installation::{ExecLuaRocksError, LuaRocksError, LuaRocksInstallation}, }; @@ -83,7 +83,7 @@ async fn install( rockspec: &R, luarocks_tree: &Path, tree: &T, - package: &LocalPackage, + package: &LockedPackage, config: &Config, ) -> Result { let layout = tree.layout_for(package); diff --git a/lux-lib/src/build/mod.rs b/lux-lib/src/build/mod.rs index fa14fd0c5..0215f00b4 100644 --- a/lux-lib/src/build/mod.rs +++ b/lux-lib/src/build/mod.rs @@ -3,13 +3,12 @@ use crate::fs; use crate::lockfile::{LockfileError, OptState, RemotePackageSourceUrl}; use crate::lua_installation::LuaInstallationError; use crate::lua_rockspec::LuaVersionError; -use crate::operations::{RemotePackageSourceMetadata, UnpackError}; +use crate::operations::UnpackError; use crate::rockspec::{LuaVersionCompatibility, Rockspec}; use crate::tree::{self, EntryType, InstallTree, TreeError}; use bytes::Bytes; use std::collections::HashMap; use std::fs::DirEntry; -use std::io::Cursor; use std::path::PathBuf; use std::{io, path::Path}; use tracing::Instrument; @@ -17,11 +16,10 @@ use tracing::Instrument; use crate::{ config::Config, hash::HasIntegrity, - lockfile::{LocalPackage, LocalPackageHashes, LockConstraint, PinnedState}, + lockfile::{LockConstraint, LockedPackage, PinnedState}, lua_installation::LuaInstallation, lua_rockspec::BuildBackendSpec, - operations::{self, FetchSrcError}, - package::PackageSpec, + operations::{pipeline::download_sources_and_hash::fetch_and_hash_source, FetchSrcError}, remote_package_source::RemotePackageSource, }; use bon::Builder; @@ -106,7 +104,7 @@ impl BuildBuilder<'_, where State: build_builder::State + build_builder::IsComplete, { - pub async fn build(self) -> Result { + pub async fn build(self) -> Result { let build = self._build(); let span = tracing::info_span!( "Building", @@ -250,7 +248,7 @@ async fn run_build( async fn install( rockspec: &R, tree: &T, - package: &LocalPackage, + package: &LockedPackage, lua: &LuaInstallation, build_dir: &Path, entry_type: &EntryType, @@ -320,8 +318,102 @@ async fn install( Ok(()) } +// TODO(vhyrro): break apart deployment into a separate step (once we implement transactions) +#[allow(clippy::too_many_arguments)] +pub(crate) async fn deploy( + rockspec: &R, + tree: &T, + mut package: LockedPackage, + lua: &LuaInstallation, + source_root: &Path, + entry_type: EntryType, + config: &Config, + behaviour: BuildBehaviour, +) -> Result { + if behaviour == BuildBehaviour::NoForce { + if let Some(existing) = tree.lockfile()?.get(&package.id()) { + return Ok(existing.clone()); + } + } + + // FIXME(vhyrro): Maybe make prepare/finalize a struct with Drop behaviour? If it makes sense only. + tree.prepare(&package)?; + let layout = tree.layout_for(&package); + + let rock_source = rockspec.source().current_platform(); + let build_dir = resolve_source_dir( + source_root, + rock_source.unpack_dir.as_deref(), + &rockspec.build().current_platform().copy_directories, + )?; + + Patch::new(&build_dir, &rockspec.build().current_platform().patches).apply()?; + + let external_dependencies = rockspec + .external_dependencies() + .current_platform() + .iter() + .map(|(name, dep)| { + ExternalDependencyInfo::probe(name, dep, config.external_deps()) + .map(|info| (name.clone(), info)) + }) + .try_collect::<_, HashMap<_, _>, _>()?; + + let output = run_build( + rockspec, + RunBuildArgs::new() + .package(&package) + .no_install(false) + .lua(lua) + .external_dependencies(&external_dependencies) + .deploy(rockspec.deploy().current_platform()) + .config(config) + .tree(tree) + .build_dir(&build_dir) + .build(), + ) + .await?; + + // FIXME(vhyrro): This shouldn't require mutation. + package.spec.binaries.extend(output.binaries); + + install( + rockspec, + tree, + &package, + lua, + &build_dir, + &entry_type, + config, + ) + .await?; + + for directory in rockspec + .build() + .current_platform() + .copy_directories + .iter() + .filter(|dir| { + dir.file_name() + .is_some_and(|name| name != "doc" && name != "docs") + }) + { + recursive_copy_dir(&build_dir.join(directory), &layout.etc.join(directory)).await?; + } + + recursive_copy_doc_dir(&layout.doc, &build_dir).await?; + + if let Ok(rockspec_str) = rockspec.to_lua_remote_rockspec_string() { + fs::sync::write(layout.rockspec_path(), rockspec_str)?; + } + + tree.finalize(&package, entry_type)?; + + Ok(package) +} + #[tracing::instrument(level = "trace", skip_all)] -async fn do_build(build: Build<'_, R, T>) -> Result +async fn do_build(build: Build<'_, R, T>) -> Result where R: Rockspec + HasIntegrity, T: InstallTree + Sync, @@ -331,133 +423,29 @@ where rockspec.validate_lua_version(&lua.version)?; - let tree = build.tree; - - let temp_dir = fs::tempfile::tempdir()?; - - let source_metadata = match build.source_spec { - Some(RemotePackageSourceSpec::SrcRock(SrcRockSource { bytes, source_url })) => { - let hash = bytes.hash().await?; - let cursor = Cursor::new(bytes); - operations::unpack_src_rock(cursor, temp_dir.path().to_path_buf()) - .await - .map_err(BuildError::UnpackSrcRock)?; - RemotePackageSourceMetadata { hash, source_url } - } - Some(RemotePackageSourceSpec::RockSpec(source_url)) => { - operations::FetchSrc::new(temp_dir.path(), rockspec, build.config) - .maybe_source_url(source_url) - .fetch_internal() - .await? - } - None => { - operations::FetchSrc::new(temp_dir.path(), rockspec, build.config) - .fetch_internal() - .await? - } - }; - - let hashes = LocalPackageHashes { - rockspec: rockspec.hash().await?, - source: source_metadata.hash.clone(), - }; - - let mut package = LocalPackage::from( - &PackageSpec::new(rockspec.package().clone(), rockspec.version().clone()), + // NOTE: these futures are large; boxing them keeps `do_build`'s frame small. + let (package, temp_dir) = Box::pin(fetch_and_hash_source( + rockspec, + build.source_spec, + build.source, build.constraint, - rockspec.binaries(), - build - .source - .map(Result::Ok) - .unwrap_or_else(|| { - rockspec - .to_lua_remote_rockspec_string() - .map(RemotePackageSource::RockspecContent) - }) - .unwrap_or(RemotePackageSource::Local), - Some(source_metadata.source_url.clone()), - hashes, - ); - package.spec.pinned = build.pin; - package.spec.opt = build.opt; - - match tree.lockfile()?.get(&package.id()) { - Some(package) if build.behaviour == BuildBehaviour::NoForce => Ok(package.clone()), - _ => { - tree.prepare(&package)?; - let layout = tree.layout_for(&package); - - let rock_source = rockspec.source().current_platform(); - let build_dir = resolve_source_dir( - temp_dir.path(), - rock_source.unpack_dir.as_deref(), - &rockspec.build().current_platform().copy_directories, - )?; - - Patch::new(&build_dir, &rockspec.build().current_platform().patches).apply()?; - - let external_dependencies = rockspec - .external_dependencies() - .current_platform() - .iter() - .map(|(name, dep)| { - ExternalDependencyInfo::probe(name, dep, build.config.external_deps()) - .map(|info| (name.clone(), info)) - }) - .try_collect::<_, HashMap<_, _>, _>()?; - - let output = run_build( - rockspec, - RunBuildArgs::new() - .package(&package) - .no_install(false) - .lua(lua) - .external_dependencies(&external_dependencies) - .deploy(rockspec.deploy().current_platform()) - .config(build.config) - .tree(tree) - .build_dir(&build_dir) - .build(), - ) - .await?; - - package.spec.binaries.extend(output.binaries); - - install( - rockspec, - tree, - &package, - lua, - &build_dir, - &build.entry_type, - build.config, - ) - .await?; - - for directory in rockspec - .build() - .current_platform() - .copy_directories - .iter() - .filter(|dir| { - dir.file_name() - .is_some_and(|name| name != "doc" && name != "docs") - }) - { - recursive_copy_dir(&build_dir.join(directory), &layout.etc.join(directory)).await?; - } - - recursive_copy_doc_dir(&layout.doc, &build_dir).await?; - - if let Ok(rockspec_str) = rockspec.to_lua_remote_rockspec_string() { - fs::sync::write(layout.rockspec_path(), rockspec_str)?; - } - - tree.finalize(&package, build.entry_type)?; - - Ok(package) - } - } + build.pin, + build.opt, + build.config, + )) + .await?; + + Box::pin(deploy( + rockspec, + build.tree, + package, + lua, + temp_dir.path(), + build.entry_type, + build.config, + build.behaviour, + )) + .await } fn is_source_or_etc_dir(dir: &DirEntry, copy_dirs: &[PathBuf]) -> bool { @@ -538,9 +526,11 @@ mod tests { use crate::{ config::ConfigBuilder, + lockfile::LockedPackageHashes, lua_installation::{detect_installed_lua_version, LuaInstallation}, lua_version::LuaVersion, operations::{unpack_rockspec, DownloadedPackedRockBytes}, + package::PackageSpec, project::Project, rockspec::RockBinaries, tree::Tree, @@ -567,13 +557,13 @@ mod tests { let lua = LuaInstallation::new(lua_version, &config).await.unwrap(); let project = Project::from_exact(&project_root).unwrap().unwrap(); let rockspec = project.toml().into_remote(None).unwrap(); - let package = LocalPackage::from( + let package = LockedPackage::from( &PackageSpec::new(rockspec.package().clone(), rockspec.version().clone()), LockConstraint::Unconstrained, RockBinaries::default(), RemotePackageSource::Test, None, - LocalPackageHashes { + LockedPackageHashes { rockspec: "sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=" .parse() .unwrap(), @@ -630,7 +620,8 @@ mod tests { const LUATEST_SRC_ROCK: &str = "resources/test/luatest-0.2-1.src.rock"; /// `sha256` of the checked-in `.src.rock` archive, i.e. what `Materialize` must /// record as the `source` integrity. - const LUATEST_SRC_ROCK_SOURCE_HASH: &str = "sha256-2jS0XOq0iIVhsZJ3BVqXSlKsx2vsqCAaaYyqcBEb7RI="; + const LUATEST_SRC_ROCK_SOURCE_HASH: &str = + "sha256-2jS0XOq0iIVhsZJ3BVqXSlKsx2vsqCAaaYyqcBEb7RI="; /// `sha256` of the rockspec text embedded in the `.src.rock`, i.e. the `rockspec` integrity. const LUATEST_ROCKSPEC_HASH: &str = "sha256-NljJ20A+VadUyhhBjrRnojeQjSqRpQy7FWcgjUt2Fdc="; @@ -657,14 +648,16 @@ mod tests { tree: &Tree, entry_type: EntryType, behaviour: BuildBehaviour, - ) -> LocalPackage { + ) -> LockedPackage { let bytes = luatest_src_rock_bytes(); let rock = DownloadedPackedRockBytes { name: "luatest".into(), version: "0.2-1".parse().unwrap(), bytes: bytes.clone(), file_name: "luatest-0.2-1.src.rock".into(), - url: "https://example.org/luatest-0.2-1.src.rock".parse().unwrap(), + url: "https://example.org/luatest-0.2-1.src.rock" + .parse() + .unwrap(), }; let rockspec = unpack_rockspec(&rock).await.unwrap(); let lua = LuaInstallation::new_from_config(config).await.unwrap(); @@ -678,7 +671,9 @@ mod tests { .source_spec(RemotePackageSourceSpec::SrcRock(SrcRockSource { bytes, source_url: RemotePackageSourceUrl::Url { - url: "https://example.org/luatest-0.2-1.src.rock".parse().unwrap(), + url: "https://example.org/luatest-0.2-1.src.rock" + .parse() + .unwrap(), }, })) .build() @@ -686,7 +681,7 @@ mod tests { .unwrap() } - /// The materialized `LocalPackage` must carry the exact source and rockspec + /// The materialized `LockedPackage` must carry the exact source and rockspec /// integrities, derived from the fetched bytes. This is the contract that /// `lx generate-lockfile` and the manifest cache depend on. #[tokio::test] @@ -728,10 +723,20 @@ mod tests { let config2 = luatest_config(&dir2); let tree2 = config2.user_tree(LuaVersion::Lua51).unwrap(); - let package1 = - build_luatest(&config1, &tree1, EntryType::Entrypoint, BuildBehaviour::Force).await; - let package2 = - build_luatest(&config2, &tree2, EntryType::Entrypoint, BuildBehaviour::Force).await; + let package1 = build_luatest( + &config1, + &tree1, + EntryType::Entrypoint, + BuildBehaviour::Force, + ) + .await; + let package2 = build_luatest( + &config2, + &tree2, + EntryType::Entrypoint, + BuildBehaviour::Force, + ) + .await; assert_eq!(package1, package2); assert_eq!(package1.hashes(), package2.hashes()); @@ -759,8 +764,13 @@ mod tests { // Simulate a partially broken install. std::fs::remove_file(&bin).unwrap(); - let skipped = - build_luatest(&config, &tree, EntryType::Entrypoint, BuildBehaviour::NoForce).await; + let skipped = build_luatest( + &config, + &tree, + EntryType::Entrypoint, + BuildBehaviour::NoForce, + ) + .await; assert_eq!(skipped.id(), package.id()); assert!( !bin.is_file(), diff --git a/lux-lib/src/lockfile/mod.rs b/lux-lib/src/lockfile/mod.rs index f71bef614..7622d7aee 100644 --- a/lux-lib/src/lockfile/mod.rs +++ b/lux-lib/src/lockfile/mod.rs @@ -161,13 +161,13 @@ impl<'de> Deserialize<'de> for OptState { } #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub(crate) struct LocalPackageSpec { +pub(crate) struct LockedPackageSpec { pub name: PackageName, pub version: PackageVersion, pub pinned: PinnedState, pub opt: OptState, - pub dependencies: Vec, - pub build_dependencies: Vec, + pub dependencies: Vec, + pub build_dependencies: Vec, // TODO: Deserialize this directly into a `LuaPackageReq` pub constraint: Option, pub binaries: RockBinaries, @@ -180,9 +180,9 @@ pub(crate) struct LocalPackageSpec { /// - opt state /// - lock constraint #[derive(Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize, Clone)] -pub struct LocalPackageId(String); +pub struct LockedPackageId(String); -impl LocalPackageId { +impl LockedPackageId { pub fn new( name: &PackageName, version: &PackageVersion, @@ -223,20 +223,20 @@ impl LocalPackageId { } } -impl Display for LocalPackageId { +impl Display for LockedPackageId { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { self.0.fmt(f) } } -impl LocalPackageSpec { +impl LockedPackageSpec { #[allow(clippy::too_many_arguments)] pub(crate) fn new( name: &PackageName, version: &PackageVersion, constraint: LockConstraint, - dependencies: Vec, - build_dependencies: Vec, + dependencies: Vec, + build_dependencies: Vec, pinned: &PinnedState, opt: &OptState, binaries: RockBinaries, @@ -256,8 +256,8 @@ impl LocalPackageSpec { } } - pub fn id(&self) -> LocalPackageId { - LocalPackageId::new( + pub fn id(&self) -> LockedPackageId { + LockedPackageId::new( self.name(), self.version(), self.pinned, @@ -290,11 +290,11 @@ impl LocalPackageSpec { self.opt } - pub fn dependencies(&self) -> Vec<&LocalPackageId> { + pub fn dependencies(&self) -> Vec<&LockedPackageId> { self.dependencies.iter().collect() } - pub fn build_dependencies(&self) -> Vec<&LocalPackageId> { + pub fn build_dependencies(&self) -> Vec<&LockedPackageId> { self.build_dependencies.iter().collect() } @@ -364,14 +364,14 @@ impl Display for RemotePackageSourceUrl { /// A locally installed rock #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct LocalPackage { - pub(crate) spec: LocalPackageSpec, +pub struct LockedPackage { + pub(crate) spec: LockedPackageSpec, pub(crate) source: RemotePackageSource, pub(crate) source_url: Option, - hashes: LocalPackageHashes, + pub(crate) hashes: LockedPackageHashes, } -impl LocalPackage { +impl LockedPackage { pub fn into_package_spec(self) -> PackageSpec { PackageSpec::new(self.spec.name, self.spec.version) } @@ -382,7 +382,7 @@ impl LocalPackage { } #[derive(Debug, Serialize, Deserialize, Clone)] -struct LocalPackageIntermediate { +struct LockedPackageIntermediate { name: PackageName, version: PackageVersion, #[serde(default, skip_serializing_if = "PinnedState::is_default")] @@ -390,9 +390,9 @@ struct LocalPackageIntermediate { #[serde(default, skip_serializing_if = "OptState::is_default")] opt: OptState, #[serde(default, skip_serializing_if = "Vec::is_empty")] - dependencies: Vec, + dependencies: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] - build_dependencies: Vec, + build_dependencies: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] constraint: Option, #[serde(default, skip_serializing_if = "RockBinaries::is_default")] @@ -400,16 +400,16 @@ struct LocalPackageIntermediate { source: RemotePackageSource, #[serde(default, skip_serializing_if = "Option::is_none")] source_url: Option, - hashes: LocalPackageHashes, + hashes: LockedPackageHashes, } -impl TryFrom for LocalPackage { +impl TryFrom for LockedPackage { type Error = LockConstraintParseError; - fn try_from(value: LocalPackageIntermediate) -> Result { + fn try_from(value: LockedPackageIntermediate) -> Result { let constraint = LockConstraint::try_from(&value.constraint)?; Ok(Self { - spec: LocalPackageSpec::new( + spec: LockedPackageSpec::new( &value.name, &value.version, constraint, @@ -426,8 +426,8 @@ impl TryFrom for LocalPackage { } } -impl From<&LocalPackage> for LocalPackageIntermediate { - fn from(value: &LocalPackage) -> Self { +impl From<&LockedPackage> for LockedPackageIntermediate { + fn from(value: &LockedPackage) -> Self { Self { name: value.spec.name.clone(), version: value.spec.version.clone(), @@ -444,36 +444,36 @@ impl From<&LocalPackage> for LocalPackageIntermediate { } } -impl<'de> Deserialize<'de> for LocalPackage { +impl<'de> Deserialize<'de> for LockedPackage { fn deserialize(deserializer: D) -> Result where D: serde::Deserializer<'de>, { - LocalPackage::try_from(LocalPackageIntermediate::deserialize(deserializer)?) + LockedPackage::try_from(LockedPackageIntermediate::deserialize(deserializer)?) .map_err(de::Error::custom) } } -impl Serialize for LocalPackage { +impl Serialize for LockedPackage { fn serialize(&self, serializer: S) -> Result where S: serde::Serializer, { - LocalPackageIntermediate::from(self).serialize(serializer) + LockedPackageIntermediate::from(self).serialize(serializer) } } -impl LocalPackage { +impl LockedPackage { pub(crate) fn from( package: &PackageSpec, constraint: LockConstraint, binaries: RockBinaries, source: RemotePackageSource, source_url: Option, - hashes: LocalPackageHashes, + hashes: LockedPackageHashes, ) -> Self { Self { - spec: LocalPackageSpec::new( + spec: LockedPackageSpec::new( package.name(), package.version(), constraint, @@ -489,7 +489,7 @@ impl LocalPackage { } } - pub fn id(&self) -> LocalPackageId { + pub fn id(&self) -> LockedPackageId { self.spec.id() } @@ -513,11 +513,11 @@ impl LocalPackage { &self.source } - pub fn dependencies(&self) -> Vec<&LocalPackageId> { + pub fn dependencies(&self) -> Vec<&LockedPackageId> { self.spec.dependencies() } - pub fn build_dependencies(&self) -> Vec<&LocalPackageId> { + pub fn build_dependencies(&self) -> Vec<&LockedPackageId> { self.spec.build_dependencies() } @@ -525,7 +525,7 @@ impl LocalPackage { self.spec.constraint() } - pub fn hashes(&self) -> &LocalPackageHashes { + pub fn hashes(&self) -> &LockedPackageHashes { &self.hashes } @@ -539,12 +539,12 @@ impl LocalPackage { } #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Hash)] -pub struct LocalPackageHashes { +pub struct LockedPackageHashes { pub rockspec: Integrity, pub source: Integrity, } -impl Ord for LocalPackageHashes { +impl Ord for LockedPackageHashes { fn cmp(&self, other: &Self) -> std::cmp::Ordering { let a = (self.rockspec.to_hex().1, self.source.to_hex().1); let b = (other.rockspec.to_hex().1, other.source.to_hex().1); @@ -552,7 +552,7 @@ impl Ord for LocalPackageHashes { } } -impl PartialOrd for LocalPackageHashes { +impl PartialOrd for LockedPackageHashes { fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } @@ -638,17 +638,25 @@ impl LockfilePermissions for ReadOnly {} impl LockfilePermissions for ReadWrite {} #[derive(Clone, Debug, Serialize, Deserialize, Default)] -pub(crate) struct LocalPackageLock { +pub(crate) struct LockedPackageLock { // NOTE: We cannot directly serialize to a `Sha256` object as they don't implement serde traits. // NOTE: We want to retain ordering of rocks when de/serializing. - rocks: BTreeMap, + rocks: BTreeMap, /// The entrypoints of the tree. Enforces one entrypoint per package name. - entrypoints: BTreeMap, + entrypoints: BTreeMap, } -impl LocalPackageLock { - fn get(&self, id: &LocalPackageId) -> Option<&LocalPackage> { +impl LockedPackageLock { + pub(crate) fn insert(&mut self, package: LockedPackage, entrypoint: bool) { + if entrypoint { + self.entrypoints + .insert(package.name().clone(), package.id()); + } + self.rocks.insert(package.id(), package); + } + + fn get(&self, id: &LockedPackageId) -> Option<&LockedPackage> { self.rocks.get(id) } @@ -656,27 +664,27 @@ impl LocalPackageLock { self.rocks.is_empty() } - pub(crate) fn rocks(&self) -> &BTreeMap { + pub(crate) fn rocks(&self) -> &BTreeMap { &self.rocks } - fn is_entrypoint(&self, package: &LocalPackageId) -> bool { + fn is_entrypoint(&self, package: &LockedPackageId) -> bool { self.entrypoints.values().any(|id| id == package) } /// Returns the entrypoint with the given package name, if any. - pub(crate) fn entrypoint(&self, name: &PackageName) -> Option<&LocalPackage> { + pub(crate) fn entrypoint(&self, name: &PackageName) -> Option<&LockedPackage> { self.entrypoints.get(name).and_then(|id| self.get(id)) } - fn is_dependency(&self, package: &LocalPackageId) -> bool { + fn is_dependency(&self, package: &LockedPackageId) -> bool { self.rocks .values() .flat_map(|rock| rock.dependencies()) .any(|dep_id| dep_id == package) } - fn list(&self) -> HashMap> { + fn list(&self) -> HashMap> { self.rocks() .values() .cloned() @@ -684,11 +692,11 @@ impl LocalPackageLock { .into_group_map() } - fn remove(&mut self, target: &LocalPackage) { + fn remove(&mut self, target: &LockedPackage) { self.remove_by_id(&target.id()) } - fn remove_by_id(&mut self, target: &LocalPackageId) { + fn remove_by_id(&mut self, target: &LockedPackageId) { self.rocks.remove(target); self.entrypoints.retain(|_, id| id != target); } @@ -697,7 +705,7 @@ impl LocalPackageLock { &self, req: &PackageReq, filter: Option, - ) -> Option { + ) -> Option { self.list() .get(req.name()) .map(|packages| { @@ -721,7 +729,7 @@ impl LocalPackageLock { .cloned() } - fn has_rock_with_equal_constraint(&self, req: &LuaDependencySpec) -> Option { + fn has_rock_with_equal_constraint(&self, req: &LuaDependencySpec) -> Option { self.list() .get(req.name()) .map(|packages| { @@ -744,12 +752,12 @@ impl LocalPackageLock { packages: &[LuaDependencySpec], strategy: &SyncStrategy<'_>, ) -> PackageSyncSpec { - let pkg_dir_exists = |pkg: &LocalPackage| match strategy { + let pkg_dir_exists = |pkg: &LockedPackage| match strategy { SyncStrategy::LockfileOnly => true, SyncStrategy::EnsureInstalled(tree) => tree.layout_for(pkg).root.is_dir(), }; - let entrypoints_to_keep: HashSet = self + let entrypoints_to_keep: HashSet = self .entrypoints .values() .filter_map(|id| self.get(id)) @@ -763,7 +771,7 @@ impl LocalPackageLock { .cloned() .collect(); - let packages_to_keep: HashSet<&LocalPackage> = entrypoints_to_keep + let packages_to_keep: HashSet<&LockedPackage> = entrypoints_to_keep .iter() .flat_map(|local_pkg| self.get_all_dependencies(&local_pkg.id())) .collect(); @@ -789,7 +797,7 @@ impl LocalPackageLock { } /// Return all dependencies of a package, including itself - fn get_all_dependencies(&self, id: &LocalPackageId) -> HashSet<&LocalPackage> { + fn get_all_dependencies(&self, id: &LockedPackageId) -> HashSet<&LockedPackage> { let mut packages = HashSet::new(); if let Some(local_pkg) = self.get(id) { packages.insert(local_pkg); @@ -805,7 +813,7 @@ impl LocalPackageLock { /// The set of all packages reachable from the entrypoints via the /// dependency graph. - fn reachable(&self) -> HashSet<&LocalPackage> { + fn reachable(&self) -> HashSet<&LockedPackage> { self.entrypoints .values() .flat_map(|id| self.get_all_dependencies(id)) @@ -823,11 +831,11 @@ pub struct Lockfile { // TODO: Serialize this directly into a `Version` version: String, #[serde(flatten)] - lock: LocalPackageLock, + lock: LockedPackageLock, } #[derive(EnumIter, Debug, PartialEq, Eq)] -pub enum LocalPackageLockType { +pub enum LockedPackageLockType { Regular, Test, Build, @@ -841,12 +849,12 @@ pub struct WorkspaceLockfile { #[serde(skip)] _marker: PhantomData

, version: String, - #[serde(default, skip_serializing_if = "LocalPackageLock::is_empty")] - dependencies: LocalPackageLock, - #[serde(default, skip_serializing_if = "LocalPackageLock::is_empty")] - test_dependencies: LocalPackageLock, - #[serde(default, skip_serializing_if = "LocalPackageLock::is_empty")] - build_dependencies: LocalPackageLock, + #[serde(default, skip_serializing_if = "LockedPackageLock::is_empty")] + dependencies: LockedPackageLock, + #[serde(default, skip_serializing_if = "LockedPackageLock::is_empty")] + test_dependencies: LockedPackageLock, + #[serde(default, skip_serializing_if = "LockedPackageLock::is_empty")] + build_dependencies: LockedPackageLock, } #[derive(Error, Debug, Diagnostic)] @@ -940,7 +948,7 @@ pub struct FlushLockfileError { #[derive(Debug, Default)] pub(crate) struct PackageSyncSpec { pub to_add: Vec, - pub to_remove: Vec, + pub to_remove: Vec, } /// Controls how `package_sync_spec` determines whether a package exists. @@ -957,30 +965,30 @@ impl Lockfile

{ &self.version } - pub fn rocks(&self) -> &BTreeMap { + pub fn rocks(&self) -> &BTreeMap { self.lock.rocks() } - pub fn is_dependency(&self, package: &LocalPackageId) -> bool { + pub fn is_dependency(&self, package: &LockedPackageId) -> bool { self.lock.is_dependency(package) } - pub fn is_entrypoint(&self, package: &LocalPackageId) -> bool { + pub fn is_entrypoint(&self, package: &LockedPackageId) -> bool { self.lock.is_entrypoint(package) } /// Returns the entrypoint with the given package name, if any. - pub fn entrypoint(&self, name: &PackageName) -> Option<&LocalPackage> { + pub fn entrypoint(&self, name: &PackageName) -> Option<&LockedPackage> { self.lock.entrypoint(name) } /// Returns all rocks in the lockfile that are reachable from an entrypoint /// via the dependency graph. - pub fn reachable_rocks(&self) -> Vec { + pub fn reachable_rocks(&self) -> Vec { self.lock.reachable().into_iter().cloned().collect() } - pub fn entry_type(&self, package: &LocalPackageId) -> EntryType { + pub fn entry_type(&self, package: &LockedPackageId) -> EntryType { if self.lock.is_entrypoint(package) { EntryType::Entrypoint } else { @@ -988,11 +996,11 @@ impl Lockfile

{ } } - pub(crate) fn local_pkg_lock(&self) -> &LocalPackageLock { + pub(crate) fn local_pkg_lock(&self) -> &LockedPackageLock { &self.lock } - pub fn get(&self, id: &LocalPackageId) -> Option<&LocalPackage> { + pub fn get(&self, id: &LockedPackageId) -> Option<&LockedPackage> { self.lock.get(id) } @@ -1001,11 +1009,11 @@ impl Lockfile

{ /// # Safety /// /// Ensure that the package is present in the lockfile before calling this function. - pub unsafe fn get_unchecked(&self, id: &LocalPackageId) -> &LocalPackage { + pub unsafe fn get_unchecked(&self, id: &LockedPackageId) -> &LockedPackage { self.lock.get(id).unwrap_unchecked() } - pub(crate) fn list(&self) -> HashMap> { + pub(crate) fn list(&self) -> HashMap> { self.lock.list() } @@ -1013,12 +1021,12 @@ impl Lockfile

{ &self, req: &PackageReq, filter: Option, - ) -> Option { + ) -> Option { self.lock.has_rock(req, filter) } /// Find all rocks that match the requirement - pub(crate) fn find_rocks(&self, req: &PackageReq) -> Vec { + pub(crate) fn find_rocks(&self, req: &PackageReq) -> Vec { match self.list().get(req.name()) { Some(packages) => packages .iter() @@ -1033,7 +1041,7 @@ impl Lockfile

{ /// Validate the integrity of an installed package with the entry in this lockfile. pub(crate) fn validate_integrity( &self, - expected_package: &LocalPackage, + expected_package: &LockedPackage, ) -> Result<(), LockfileIntegrityError> { // NOTE: We can't query by ID, because when installing from a lockfile (e.g. during sync), // the constraint is always `==`. @@ -1095,67 +1103,67 @@ impl Lockfile

{ impl WorkspaceLockfile

{ pub(crate) fn rocks( &self, - deps: &LocalPackageLockType, - ) -> &BTreeMap { + deps: &LockedPackageLockType, + ) -> &BTreeMap { match deps { - LocalPackageLockType::Regular => self.dependencies.rocks(), - LocalPackageLockType::Test => self.test_dependencies.rocks(), - LocalPackageLockType::Build => self.build_dependencies.rocks(), + LockedPackageLockType::Regular => self.dependencies.rocks(), + LockedPackageLockType::Test => self.test_dependencies.rocks(), + LockedPackageLockType::Build => self.build_dependencies.rocks(), } } pub(crate) fn get( &self, - id: &LocalPackageId, - deps: &LocalPackageLockType, - ) -> Option<&LocalPackage> { + id: &LockedPackageId, + deps: &LockedPackageLockType, + ) -> Option<&LockedPackage> { match deps { - LocalPackageLockType::Regular => self.dependencies.get(id), - LocalPackageLockType::Test => self.test_dependencies.get(id), - LocalPackageLockType::Build => self.build_dependencies.get(id), + LockedPackageLockType::Regular => self.dependencies.get(id), + LockedPackageLockType::Test => self.test_dependencies.get(id), + LockedPackageLockType::Build => self.build_dependencies.get(id), } } pub(crate) fn is_entrypoint( &self, - package: &LocalPackageId, - deps: &LocalPackageLockType, + package: &LockedPackageId, + deps: &LockedPackageLockType, ) -> bool { match deps { - LocalPackageLockType::Regular => self.dependencies.is_entrypoint(package), - LocalPackageLockType::Test => self.test_dependencies.is_entrypoint(package), - LocalPackageLockType::Build => self.build_dependencies.is_entrypoint(package), + LockedPackageLockType::Regular => self.dependencies.is_entrypoint(package), + LockedPackageLockType::Test => self.test_dependencies.is_entrypoint(package), + LockedPackageLockType::Build => self.build_dependencies.is_entrypoint(package), } } pub(crate) fn package_sync_spec( &self, packages: &[LuaDependencySpec], - deps: &LocalPackageLockType, + deps: &LockedPackageLockType, strategy: &SyncStrategy<'_>, ) -> PackageSyncSpec { match deps { - LocalPackageLockType::Regular => { + LockedPackageLockType::Regular => { self.dependencies.package_sync_spec(packages, strategy) } - LocalPackageLockType::Test => { + LockedPackageLockType::Test => { self.test_dependencies.package_sync_spec(packages, strategy) } - LocalPackageLockType::Build => self + LockedPackageLockType::Build => self .build_dependencies .package_sync_spec(packages, strategy), } } - pub(crate) fn local_pkg_lock(&self, deps: &LocalPackageLockType) -> &LocalPackageLock { + pub(crate) fn local_pkg_lock(&self, deps: &LockedPackageLockType) -> &LockedPackageLock { match deps { - LocalPackageLockType::Regular => &self.dependencies, - LocalPackageLockType::Test => &self.test_dependencies, - LocalPackageLockType::Build => &self.build_dependencies, + LockedPackageLockType::Regular => &self.dependencies, + LockedPackageLockType::Test => &self.test_dependencies, + LockedPackageLockType::Build => &self.build_dependencies, } } - pub(crate) fn local_pkg_locks(&self) -> Vec { + pub(crate) fn local_pkg_locks(&self) -> Vec { vec![ self.dependencies.clone(), self.test_dependencies.clone(), @@ -1183,7 +1191,7 @@ impl Lockfile { filepath: filepath.clone(), _marker: PhantomData, version: LOCKFILE_VERSION_STR.into(), - lock: LocalPackageLock::default(), + lock: LockedPackageLock::default(), }; let json_str = serde_json::to_string(&empty_lockfile).map_err(LockfileError::WriteJson)?; @@ -1287,9 +1295,9 @@ impl WorkspaceLockfile { filepath: filepath.clone(), _marker: PhantomData, version: LOCKFILE_VERSION_STR.into(), - dependencies: LocalPackageLock::default(), - test_dependencies: LocalPackageLock::default(), - build_dependencies: LocalPackageLock::default(), + dependencies: LockedPackageLock::default(), + test_dependencies: LockedPackageLock::default(), + build_dependencies: LockedPackageLock::default(), }; let json_str = serde_json::to_string(&empty_lockfile).map_err(LockfileError::WriteJson)?; @@ -1346,12 +1354,12 @@ remove the `lux.lock` and run `lx sync` to regenerate it.", } impl Lockfile { - pub(crate) fn add_entrypoint(&mut self, rock: &LocalPackage) { + pub(crate) fn add_entrypoint(&mut self, rock: &LockedPackage) { self.add(rock); self.lock.entrypoints.insert(rock.name().clone(), rock.id()); } - fn add(&mut self, rock: &LocalPackage) { + pub(crate) fn add(&mut self, rock: &LockedPackage) { // Since rocks entries are mutable, we only add the dependency if it // has not already been added. self.lock @@ -1361,7 +1369,7 @@ impl Lockfile { } /// Add a dependency for a package. - pub(crate) fn add_dependency(&mut self, target: &LocalPackage, dependency: &LocalPackage) { + pub(crate) fn add_dependency(&mut self, target: &LockedPackage, dependency: &LockedPackage) { self.lock .rocks .entry(target.id()) @@ -1385,8 +1393,8 @@ impl Lockfile { /// Add a build dependency for a package. pub(crate) fn add_build_dependency( &mut self, - target: &LocalPackage, - dependency: &LocalPackage, + target: &LockedPackage, + dependency: &LockedPackage, ) { self.lock .rocks @@ -1407,15 +1415,15 @@ impl Lockfile { }); } - pub(crate) fn remove(&mut self, target: &LocalPackage) { + pub(crate) fn remove(&mut self, target: &LockedPackage) { self.lock.remove(target) } - pub(crate) fn remove_by_id(&mut self, target: &LocalPackageId) { + pub(crate) fn remove_by_id(&mut self, target: &LockedPackageId) { self.lock.remove_by_id(target) } - pub(crate) fn sync(&mut self, lock: &LocalPackageLock) { + pub(crate) fn sync(&mut self, lock: &LockedPackageLock) { self.lock = lock.clone(); } @@ -1423,23 +1431,23 @@ impl Lockfile { } impl WorkspaceLockfile { - pub(crate) fn remove(&mut self, target: &LocalPackage, deps: &LocalPackageLockType) { + pub(crate) fn remove(&mut self, target: &LockedPackage, deps: &LockedPackageLockType) { match deps { - LocalPackageLockType::Regular => self.dependencies.remove(target), - LocalPackageLockType::Test => self.test_dependencies.remove(target), - LocalPackageLockType::Build => self.build_dependencies.remove(target), + LockedPackageLockType::Regular => self.dependencies.remove(target), + LockedPackageLockType::Test => self.test_dependencies.remove(target), + LockedPackageLockType::Build => self.build_dependencies.remove(target), } } - pub(crate) fn sync(&mut self, lock: &LocalPackageLock, deps: &LocalPackageLockType) { + pub(crate) fn sync(&mut self, lock: &LockedPackageLock, deps: &LockedPackageLockType) { match deps { - LocalPackageLockType::Regular => { + LockedPackageLockType::Regular => { self.dependencies = lock.clone(); } - LocalPackageLockType::Test => { + LockedPackageLockType::Test => { self.test_dependencies = lock.clone(); } - LocalPackageLockType::Build => { + LockedPackageLockType::Build => { self.build_dependencies = lock.clone(); } } @@ -1660,7 +1668,7 @@ fn migrate_entrypoints( Ok(()) } -fn integrity_err_not_found(package: &LocalPackage) -> LockfileIntegrityError { +fn integrity_err_not_found(package: &LockedPackage) -> LockfileIntegrityError { LockfileIntegrityError::PackageNotFound( package.name().clone(), Box::new(package.version().clone()), @@ -1727,7 +1735,7 @@ mod tests { ) .unwrap(); - let mock_hashes = LocalPackageHashes { + let mock_hashes = LockedPackageHashes { rockspec: "sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=" .parse() .unwrap(), @@ -1745,7 +1753,7 @@ mod tests { let mut lockfile = tree.lockfile().unwrap().write_guard(); let test_package = PackageSpec::parse("test1".to_string(), "0.1.0".to_string()).unwrap(); - let test_local_package = LocalPackage::from( + let test_local_package = LockedPackage::from( &test_package, crate::lockfile::LockConstraint::Unconstrained, RockBinaries::default(), @@ -1757,7 +1765,7 @@ mod tests { let test_dep_package = PackageSpec::parse("test2".to_string(), "0.1.0".to_string()).unwrap(); - let mut test_local_dep_package = LocalPackage::from( + let mut test_local_dep_package = LockedPackage::from( &test_dep_package, crate::lockfile::LockConstraint::Constrained(">= 1.0.0".parse().unwrap()), RockBinaries::default(), diff --git a/lux-lib/src/luarocks/install_binary_rock.rs b/lux-lib/src/luarocks/install_binary_rock.rs index 33281d44b..21ddcc87e 100644 --- a/lux-lib/src/luarocks/install_binary_rock.rs +++ b/lux-lib/src/luarocks/install_binary_rock.rs @@ -13,7 +13,7 @@ use crate::{ config::Config, hash::HasIntegrity, lockfile::{ - LocalPackage, LocalPackageHashes, LockConstraint, LockfileError, OptState, PinnedState, + LockConstraint, LockedPackage, LockedPackageHashes, LockfileError, OptState, PinnedState, }, lua_rockspec::{LuaVersionError, RemoteLuaRockspec}, luarocks::rock_manifest::RockManifest, @@ -121,7 +121,7 @@ where } #[tracing::instrument(name = "Installing binary rock", skip_all)] - pub(crate) async fn install(self) -> Result { + pub(crate) async fn install(self) -> Result { let rockspec = self.rockspec; for (name, dep) in rockspec.external_dependencies().current_platform() { let _ = ExternalDependencyInfo::probe(name, dep, self.config.external_deps())?; @@ -129,7 +129,7 @@ where rockspec.validate_lua_version_from_config(self.config)?; - let hashes = LocalPackageHashes { + let hashes = LockedPackageHashes { rockspec: rockspec.hash().await?, source: self.rock_bytes.hash().await?, }; @@ -139,7 +139,7 @@ where } _ => None, }; - let mut package = LocalPackage::from( + let mut package = LockedPackage::from( &PackageSpec::new(rockspec.package().clone(), rockspec.version().clone()), self.constraint, rockspec.binaries(), diff --git a/lux-lib/src/operations/build_workspace.rs b/lux-lib/src/operations/build_workspace.rs index a5cc04889..49fa5340d 100644 --- a/lux-lib/src/operations/build_workspace.rs +++ b/lux-lib/src/operations/build_workspace.rs @@ -1,71 +1,19 @@ use crate::{ - build::{Build, BuildBehaviour, BuildError}, - config::Config, - lockfile::LocalPackage, - lua_installation::{LuaInstallation, LuaInstallationError}, - luarocks::luarocks_installation::{LuaRocksError, LuaRocksInstallError, LuaRocksInstallation}, - operations::{install_dependencies::prepare_dependencies_for_build, InstallDependencies}, - package::PackageName, - project::{project_toml::LocalProjectTomlValidationError, Project}, - tree::{self, InstallTree, TreeError}, - workspace::{Workspace, WorkspaceError, WorkspaceTreeError}, + config::Config, lockfile::LockedPackage, operations::pipeline::Pipeline, package::PackageName, + workspace::Workspace, }; use bon::Builder; -use itertools::Itertools; use miette::Diagnostic; use thiserror::Error; use tracing::{info_span, Instrument}; -use super::{InstallError, Sync, SyncError}; - #[derive(Debug, Error, Diagnostic)] #[non_exhaustive] pub enum BuildWorkspaceError { + // FIXME(vhyrro): don't use dyn here #[error(transparent)] #[diagnostic(transparent)] - LocalProjectTomlValidation(#[from] LocalProjectTomlValidationError), - #[error(transparent)] - #[diagnostic(transparent)] - Workspace(#[from] WorkspaceError), - #[error(transparent)] - #[diagnostic(transparent)] - WorkspaceTree(#[from] WorkspaceTreeError), - #[error(transparent)] - #[diagnostic(transparent)] - LuaInstallation(#[from] LuaInstallationError), - #[error(transparent)] - #[diagnostic(transparent)] - Tree(#[from] TreeError), - #[error(transparent)] - #[diagnostic(transparent)] - LuaRocks(#[from] LuaRocksError), - #[error(transparent)] - #[diagnostic(transparent)] - LuaRocksInstall(#[from] Box), - #[error("error installind dependencies")] - #[diagnostic(forward(0))] - InstallDependencies(#[source] Box), - #[error("error installind build dependencies")] - #[diagnostic(forward(0))] - InstallBuildDependencies(#[source] Box), - #[error("syncing dependencies with the workspace lockfile failed")] - #[diagnostic(forward(0))] - SyncDependencies(#[source] Box), - #[error("error building the workspace")] - #[diagnostic(forward(0))] - Build(#[from] Box), -} - -impl From for BuildWorkspaceError { - fn from(source: LuaRocksInstallError) -> Self { - Self::LuaRocksInstall(Box::new(source)) - } -} - -impl From for BuildWorkspaceError { - fn from(source: BuildError) -> Self { - Self::Build(Box::new(source)) - } + Pipeline(Box), } #[derive(Builder)] @@ -90,133 +38,27 @@ pub struct BuildWorkspace<'a> { impl BuildWorkspaceBuilder<'_, State> { - pub async fn build(self) -> Result, BuildWorkspaceError> { + pub async fn build(self) -> Result, BuildWorkspaceError> { let build = self._build(); let span = match &build.package { Some(package) => info_span!("Building workspace", package = package.to_string()), None => info_span!("Building workspace"), }; - do_build(build).instrument(span).await - } -} - -async fn do_build(args: BuildWorkspace<'_>) -> Result, BuildWorkspaceError> { - let config = args.config; - let workspace = args.workspace; - let workspace_tree = workspace.tree(config)?; - let build_tree = workspace.build_tree(config)?; - let lua = LuaInstallation::new_from_config(config).await?; - if !args.no_lock { - Sync::new(workspace, config) - .sync() - .await - .map_err(|err| BuildWorkspaceError::SyncDependencies(Box::new(err)))?; - } else { - let luarocks = LuaRocksInstallation::new(config, build_tree.clone())?; - let mut dependencies_to_install = Vec::new(); - let mut build_dependencies_to_install = Vec::new(); - if let Some(package) = &args.package { - let project = workspace.select_member(package)?; - let project_toml = project.toml().into_local()?; - prepare_dependencies_for_build( - &project_toml, - &workspace_tree, - &mut dependencies_to_install, - &mut build_dependencies_to_install, - tree::EntryType::Entrypoint, - ); - } else { - for project in workspace.members() { - let project_toml = project.toml().into_local()?; - prepare_dependencies_for_build( - &project_toml, - &workspace_tree, - &mut dependencies_to_install, - &mut build_dependencies_to_install, - tree::EntryType::Entrypoint, - ); - } - } - - let tree = workspace.tree(config)?; - - InstallDependencies::new() - .dependencies(dependencies_to_install.into_iter().unique().collect_vec()) - .build_dependencies( - build_dependencies_to_install - .into_iter() - .unique() - .collect_vec(), + async move { + // NOTE: `Pipeline::run`'s future is large, so we box it to avoid inflating this frame. + let result = Box::pin( + Pipeline::new(build.config, build.workspace) + .maybe_package(build.package) + .no_lock(build.no_lock) + .build_projects(!build.only_deps) + .run(), ) - .tree(&tree) - .lua(&lua) - .luarocks(&luarocks) - .config(config) - .build() - .await - .map_err(|err| BuildWorkspaceError::InstallBuildDependencies(Box::new(err)))?; - } - - let mut packages = Vec::new(); - if !args.only_deps { - if let Some(package) = &args.package { - let project = workspace.select_member(package)?; - let pkg = build_project(project, workspace, &lua, config).await?; - packages.push(pkg); - } else { - for project in workspace.members() { - let pkg = build_project(project, workspace, &lua, config).await?; - packages.push(pkg); - } + .await; + result.map_err(|err| BuildWorkspaceError::Pipeline(err.into())) } + .instrument(span) + .await } - Ok(packages) -} - -async fn build_project( - project: &Project, - workspace: &Workspace, - lua: &LuaInstallation, - config: &Config, -) -> Result { - let workspace_tree = workspace.tree(config)?; - let project_toml = project.toml().into_local()?; - - let package = Build::new() - .rockspec(&project_toml) - .lua(lua) - .tree(&workspace_tree) - .entry_type(tree::EntryType::Entrypoint) - .config(config) - .behaviour(BuildBehaviour::Force) - .build() - .await?; - - let lockfile = workspace_tree.lockfile()?; - let dependencies = lockfile - .rocks() - .iter() - .filter_map(|(pkg_id, value)| { - if lockfile.is_entrypoint(pkg_id) { - Some(value) - } else { - None - } - }) - .cloned() - .collect_vec(); - - let build_lockfile = workspace.build_tree(config)?.lockfile()?; - - let mut lockfile = lockfile.write_guard(); - lockfile.add_entrypoint(&package); - for dep in dependencies { - lockfile.add_dependency(&package, &dep); - } - for dep in build_lockfile.rocks().values() { - lockfile.add_build_dependency(&package, dep); - } - Ok(package) } #[cfg(test)] @@ -225,7 +67,7 @@ mod tests { use crate::{ config::ConfigBuilder, fs, lua_installation::detect_installed_lua_version, - lua_version::LuaVersion, + lua_version::LuaVersion, tree::InstallTree, }; use assert_fs::prelude::PathCopy; use std::path::PathBuf; diff --git a/lux-lib/src/operations/dist_bin.rs b/lux-lib/src/operations/dist_bin.rs index db11ab4ee..51355b4cd 100644 --- a/lux-lib/src/operations/dist_bin.rs +++ b/lux-lib/src/operations/dist_bin.rs @@ -549,15 +549,15 @@ mod tests { #[cfg(target_os = "linux")] use crate::{ fs, - lockfile::{LocalPackage, LocalPackageHashes, LockConstraint}, + lockfile::{LockConstraint, LockedPackage, LockedPackageHashes}, package::PackageSpec, remote_package_source::RemotePackageSource, rockspec::RockBinaries, }; #[cfg(target_os = "linux")] - fn mk_dummy_package(spec: PackageSpec) -> LocalPackage { - let hashes = LocalPackageHashes { + fn mk_dummy_package(spec: PackageSpec) -> LockedPackage { + let hashes = LockedPackageHashes { rockspec: "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" .parse() .unwrap(), @@ -565,7 +565,7 @@ mod tests { .parse() .unwrap(), }; - LocalPackage::from( + LockedPackage::from( &spec, LockConstraint::Unconstrained, RockBinaries::default(), diff --git a/lux-lib/src/operations/gen_luarc.rs b/lux-lib/src/operations/gen_luarc.rs index 8c359442a..afc1393cc 100644 --- a/lux-lib/src/operations/gen_luarc.rs +++ b/lux-lib/src/operations/gen_luarc.rs @@ -1,6 +1,6 @@ use crate::config::Config; use crate::fs; -use crate::lockfile::LocalPackageLockType; +use crate::lockfile::LockedPackageLockType; use crate::tree::InstallTree; use crate::workspace::Workspace; use crate::workspace::WorkspaceError; @@ -84,7 +84,7 @@ async fn do_generate_luarc(args: GenLuaRc<'_>) -> Result<(), GenLuaRcError> { let dependency_tree = workspace.tree(config)?; let dependency_dirs = lockfile - .local_pkg_lock(&LocalPackageLockType::Regular) + .local_pkg_lock(&LockedPackageLockType::Regular) .rocks() .values() .map(|dependency| dependency_tree.layout_for(dependency).src) @@ -93,7 +93,7 @@ async fn do_generate_luarc(args: GenLuaRc<'_>) -> Result<(), GenLuaRcError> { let test_dependency_tree = workspace.test_tree(config)?; let test_dependency_dirs = lockfile - .local_pkg_lock(&LocalPackageLockType::Test) + .local_pkg_lock(&LockedPackageLockType::Test) .rocks() .values() .map(|dependency| test_dependency_tree.layout_for(dependency).src) diff --git a/lux-lib/src/operations/install/mod.rs b/lux-lib/src/operations/install/mod.rs index f98a132ab..76f9a930c 100644 --- a/lux-lib/src/operations/install/mod.rs +++ b/lux-lib/src/operations/install/mod.rs @@ -1,52 +1,24 @@ -use std::{ - collections::{HashMap, HashSet}, - io, - sync::Arc, -}; - use crate::{ - build::{Build, BuildBehaviour, BuildError, RemotePackageSourceSpec, SrcRockSource}, config::Config, - lockfile::{ - FlushLockfileError, LocalPackage, LocalPackageId, LockConstraint, Lockfile, OptState, - PinnedState, ReadOnly, ReadWrite, - }, - lua_installation::{LuaInstallation, LuaInstallationError}, - lua_rockspec::BuildBackendSpec, - lua_version::LuaVersionUnset, - luarocks::{ - install_binary_rock::{BinaryRockInstall, InstallBinaryRockError}, - luarocks_installation::{LuaRocksError, LuaRocksInstallError, LuaRocksInstallation}, - }, - operations::resolve::{ - build_dependencies_to_install, PackageInstallData, Resolve, ResolveDependenciesError, - }, - package::{PackageName, PackageNameList, PackageReq}, - remote_package_db::{RemotePackageDB, RemotePackageDBError}, - rockspec::Rockspec, - tree::{self, InstallTree, Tree, TreeError}, + lockfile::LockedPackage, + remote_package_db::RemotePackageDB, + tree::{InstallTree, Tree}, workspace::{Workspace, WorkspaceTreeError}, }; pub use crate::operations::install::spec::PackageInstallSpec; -use super::{DownloadedRockspec, RemoteRockDownload}; use bon::Builder; -use bytes::Bytes; -use futures::stream::FuturesUnordered; -use futures::StreamExt; -use itertools::Itertools; use miette::Diagnostic; use thiserror::Error; -use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; -use tokio::task::{JoinError, JoinHandle}; use tracing::Instrument; + +use super::pipeline::install_packages::{InstallPackages, InstallPackagesError}; pub mod spec; /// A rocks package installer, providing fine-grained control /// over how packages should be installed. -/// Can install multiple packages in parallel. #[derive(Builder)] #[builder(start_fn = new, finish_fn(name = _build, vis = ""))] pub struct Install<'a, T> @@ -112,7 +84,7 @@ where T: InstallTree + Clone + Send + Sync + 'static, { /// Install the packages. - pub async fn install(self) -> Result, InstallError> { + pub async fn install(self) -> Result, InstallError> { let install_built = self._build(); if install_built.packages.is_empty() { return Ok(Vec::default()); @@ -128,717 +100,55 @@ where } } -type InstallWorkerOutput = Result<(LocalPackageId, (LocalPackage, tree::EntryType)), InstallError>; - #[derive(Error, Debug, Diagnostic)] pub enum InstallError { - #[error("unable to resolve dependencies")] - #[diagnostic(forward(0))] - ResolveDependencies(#[from] Box), #[error(transparent)] #[diagnostic(transparent)] - LuaVersionUnset(#[from] LuaVersionUnset), + LuaVersionUnset(#[from] crate::lua_version::LuaVersionUnset), #[error(transparent)] #[diagnostic(transparent)] - LuaInstallation(#[from] LuaInstallationError), + LuaInstallation(#[from] crate::lua_installation::LuaInstallationError), #[error(transparent)] #[diagnostic(transparent)] - FlushLockfile(#[from] FlushLockfileError), + FlushLockfile(#[from] crate::lockfile::FlushLockfileError), #[error(transparent)] #[diagnostic(transparent)] - Tree(#[from] TreeError), + Tree(#[from] crate::tree::TreeError), #[error(transparent)] #[diagnostic(transparent)] WorkspaceTree(#[from] WorkspaceTreeError), #[error("error instantiating LuaRocks compatibility layer")] #[diagnostic(forward(0))] - LuaRocks(#[from] LuaRocksError), + LuaRocks(#[from] crate::luarocks::luarocks_installation::LuaRocksError), #[error("error installing LuaRocks compatibility layer")] #[diagnostic(forward(0))] - LuaRocksInstall(#[from] Box), - #[error("failed to build {0}")] - Build(PackageName, #[source] Box), - #[error("failed to install build depencency {0}")] - BuildDependency(PackageName, #[source] Box), - #[error("error initialising remote package DB")] - #[diagnostic(forward(0))] - RemotePackageDB(#[from] RemotePackageDBError), - #[error("failed to install pre-built rock {0}")] - InstallBinaryRock(PackageName, #[source] Box), - #[error("cannot install duplicate entrypoints:\n{0}")] - DuplicateEntrypoints(PackageNameList), - #[error("cannot install conflicting entrypoints:\n{0}")] - #[diagnostic(help( - "only a single entrypoint per package is allowed.\nretry with `--force` to overwrite the existing entrypoint" - ))] - ConflictingEntrypoints(String), - #[error("install worker panicked")] - #[diagnostic(help( - r#"this is a bug in Lux, please report it, ideally with `RUST_BACKTRACE=1`. -retrying with fewer parallel jobs (`--max-jobs`) may avoid the panic in the meantime"# - ))] - Join(#[from] tokio::task::JoinError), -} - -impl From for InstallError { - fn from(source: ResolveDependenciesError) -> Self { - Self::ResolveDependencies(Box::new(source)) - } + LuaRocksInstall(#[from] Box), + #[error(transparent)] + #[diagnostic(transparent)] + Install(#[from] InstallPackagesError), } -impl From for InstallError { - fn from(source: LuaRocksInstallError) -> Self { +impl From for InstallError { + fn from( + source: crate::luarocks::luarocks_installation::LuaRocksInstallError, + ) -> Self { Self::LuaRocksInstall(Box::new(source)) } } -async fn install_impl(install: Install<'_, T>) -> Result, InstallError> +async fn install_impl(install: Install<'_, T>) -> Result, InstallError> where T: InstallTree + Clone + Send + Sync + 'static, { - let package_db = match install.package_db { - Some(db) => db, - None => RemotePackageDB::from_config(install.config).await?, - }; - - let duplicate_entrypoints = install - .packages - .iter() - .filter(|pkg| pkg.entry_type == tree::EntryType::Entrypoint) - .map(|pkg| pkg.package.name()) - .duplicates() - .cloned() - .collect_vec(); - - if !duplicate_entrypoints.is_empty() { - return Err(InstallError::DuplicateEntrypoints(PackageNameList::new( - duplicate_entrypoints, - ))); - } - let packages = install.packages; - let package_db = Arc::new(package_db); - let config = install.config; - let tree = &install.tree; - - let (dep_tx, mut dep_rx) = tokio::sync::mpsc::unbounded_channel(); - let (build_dep_tx, build_dep_rx) = tokio::sync::mpsc::unbounded_channel(); - let (build_dep_install_done_tx, mut build_dep_install_done_rx) = - tokio::sync::mpsc::unbounded_channel::<(LocalPackage, PackageInstallData)>(); - - let lockfile = tree.lockfile()?; - let build_lockfile = tree.build_tree(config)?.lockfile()?; - - let entrypoint_specs = packages - .iter() - .filter(|spec| spec.entry_type == tree::EntryType::Entrypoint) - .collect_vec(); - - // Entrypoints already installed that conflict with another entrypoint. This enumerates both - // packages that are planned to be removed (`--force`), as well as those which are - // unintentionally causing conflicts. - let conflicting_entrypoints: HashMap = entrypoint_specs - .iter() - .filter_map(|spec| lockfile.entrypoint(spec.package.name()).cloned()) - .map(|package| (package.name().clone(), package)) - .collect(); - - let unforced_conflicts = entrypoint_specs - .iter() - .filter(|spec| spec.build_behaviour != BuildBehaviour::Force) - .filter_map(|spec| conflicting_entrypoints.get(spec.package.name())) - .map(|existing| format!("{}@{}", existing.name(), existing.version())) - .collect_vec(); - - if !unforced_conflicts.is_empty() { - return Err(InstallError::ConflictingEntrypoints( - unforced_conflicts.join("\n"), - )); - } - - // Forced overwrites: remove the conflicting entrypoints from the tree. - // NOTE: non-transactional. If an error occurs, this removes the conflicting package without - // installing the substitute. Fix when transactions are implemented. - let conflicting_entrypoints = conflicting_entrypoints.into_values().collect_vec(); - for package in &conflicting_entrypoints { - tree.cleanup(package, tree::EntryType::Entrypoint)?; - } - - let lua = Arc::new(LuaInstallation::new_from_config(config).await?); - - let mut resolve_worker = spawn_resolve_worker( - config, - packages, - package_db, - lockfile.clone(), - build_lockfile.clone(), - dep_tx, - build_dep_tx, - ); - let mut build_deps_worker = spawn_build_deps_worker( - config, - tree, - lua.clone(), - build_dep_rx, - build_dep_install_done_tx, - ); - - let mut all_packages: HashMap = HashMap::new(); - let mut scheduled_packages: HashSet = HashSet::new(); - let mut installed_packages: HashMap = - HashMap::new(); - let mut installed_build_deps: HashMap = - HashMap::new(); - let mut ongoing_installs: FuturesUnordered< - tracing::instrument::Instrumented>, - > = FuturesUnordered::new(); - let mut resolve_done = false; - let mut build_deps_done = false; - let mut dep_rx_drained = false; - let mut build_dep_rx_drained = false; - let mut install_loop_result: Result<(), InstallError> = Ok(()); - let max_jobs = config.max_jobs(); - - 'install: loop { - if resolve_done && build_deps_done && dep_rx_drained && build_dep_rx_drained { - break; - } - tokio::select! { - resolve_result = &mut resolve_worker, if !resolve_done => { - if let Err(err) = worker_result(resolve_result) { - install_loop_result = Err(*err); - break 'install; - } - resolve_done = true; - } - build_deps_result = &mut build_deps_worker, if !build_deps_done => { - if let Err(err) = worker_result(build_deps_result) { - install_loop_result = Err(*err); - break 'install; - } - build_deps_done = true; - } - build_dep = build_dep_install_done_rx.recv(), if !build_dep_rx_drained => { - if let Some((build_dep, install_data)) = build_dep { - installed_build_deps.insert(build_dep.spec.id(), (build_dep, install_data)); - } else { - build_dep_rx_drained = true; - } - } - dep = dep_rx.recv(), if !dep_rx_drained => { - if let Some(dep) = dep { - all_packages.insert(dep.spec.id(), dep); - } else { - dep_rx_drained = true; - } - } - } - - for (package_id, package_install_data) in ready_to_install( - &all_packages, - &scheduled_packages, - &build_lockfile, - &installed_build_deps - .values() - .map(|(pkg, _)| pkg.name().clone()) - .collect(), - ) { - if max_jobs > 0 && ongoing_installs.len() >= max_jobs { - if let Err(err) = - wait_for_next_install(&mut ongoing_installs, &mut installed_packages).await - { - install_loop_result = Err(err); - break 'install; - } - } - scheduled_packages.insert(package_id); - ongoing_installs.push(spawn_install_worker( - package_install_data, - &lua, - tree, - config, - )); - } + if packages.is_empty() { + return Ok(Vec::new()); } - match install_loop_result { - Ok(_) => { - while wait_for_next_install(&mut ongoing_installs, &mut installed_packages).await? {} - - lockfile.map_then_flush(|lockfile| { - for package in &conflicting_entrypoints { - lockfile.remove_by_id(&package.id()); - } - for (package_id, (package, is_entrypoint)) in installed_packages.iter().unique() { - lockfile.add_dependencies( - package_id, - package, - *is_entrypoint, - &all_packages, - &installed_packages, - )?; - lockfile.add_build_dependencies( - package_id, - package, - &all_packages, - &installed_build_deps, - )?; - } - Ok::<_, io::Error>(()) - })?; - - Ok(installed_packages - .into_values() - .map(|(pkg, _)| pkg) - .collect_vec()) - } - Err(err) => { - resolve_worker.into_inner().abort(); - build_deps_worker.into_inner().abort(); - for install in ongoing_installs { - install.into_inner().abort(); - } - Err(err) - } - } -} - -fn spawn_resolve_worker( - config: &Config, - packages: Vec, - package_db: Arc, - lockfile: Lockfile, - build_lockfile: Lockfile, - dep_tx: UnboundedSender, - build_dep_tx: UnboundedSender, -) -> tracing::instrument::Instrumented>> { - tokio::spawn({ - let config = config.clone(); - let lockfile = Arc::new(lockfile); - let build_lockfile = Arc::new(build_lockfile); - async move { - Resolve::new() - .dependencies_tx(dep_tx) - .build_dependencies_tx(build_dep_tx) - .packages(packages) - .package_db(package_db) - .lockfile(lockfile) - .build_lockfile(build_lockfile) - .config(&config) - .get_all_dependencies() - .await?; - Ok::<(), InstallError>(()) - } - }) - .instrument(tracing::trace_span!("resolve_worker")) -} - -fn spawn_build_deps_worker( - config: &Config, - tree: &T, - lua: Arc, - mut build_dep_rx: UnboundedReceiver, - build_dep_install_done_tx: UnboundedSender<(LocalPackage, PackageInstallData)>, -) -> tracing::instrument::Instrumented>> -where - T: InstallTree + Clone + Send + Sync + 'static, -{ - tokio::spawn({ - let config = config.clone(); - let tree = tree.clone(); - let lua = lua.clone(); - async move { - while let Some(build_dep_spec) = build_dep_rx.recv().await { - let rockspec = build_dep_spec.downloaded_rock.rockspec(); - let package = rockspec.package().clone(); - let span = tracing::info_span!( - "Installing build dependency", - package = package.to_string(), - version = rockspec.version().to_string() - ); - let pkg = async { - let build_tree = tree.build_tree(&config)?; - let mut build_lockfile = build_tree.lockfile()?.write_guard(); - let pkg = Build::new() - .rockspec(rockspec) - .lua(&lua) - .tree(&build_tree) - .entry_type(tree::EntryType::Entrypoint) - .config(&config) - .constraint(build_dep_spec.spec.constraint()) - .behaviour(build_dep_spec.build_behaviour) - .build() - .await - .map_err(|err| InstallError::BuildDependency(package, Box::new(err)))?; - build_lockfile.add_entrypoint(&pkg); - Ok::<_, InstallError>(pkg) - } - .instrument(span) - .await?; - let _ = build_dep_install_done_tx.send((pkg, build_dep_spec)); - } - Ok::<(), InstallError>(()) - } - }) - .instrument(tracing::trace_span!("build_deps_worker")) -} - -fn ready_to_install( - all_packages: &HashMap, - scheduled: &HashSet, - build_lockfile: &Lockfile, - installed_build_deps: &HashSet, -) -> Vec<(LocalPackageId, PackageInstallData)> { - all_packages - .iter() - .filter(|(id, data)| { - !scheduled.contains(*id) - && match &data.downloaded_rock { - RemoteRockDownload::BinaryRock { .. } => true, - _ => build_dependencies_ready( - data.downloaded_rock.rockspec(), - data.build_behaviour, - build_lockfile, - installed_build_deps, - ), - } - }) - .map(|(id, data)| (id.clone(), data.clone())) - .collect() -} - -fn spawn_install_worker( - data: PackageInstallData, - lua: &Arc, - tree: &T, - config: &Config, -) -> tracing::instrument::Instrumented> -where - T: InstallTree + Clone + Send + Sync + 'static, -{ - let config = config.clone(); - let tree = tree.clone(); - let lua = lua.clone(); - let entry_type = data.entry_type; - tokio::spawn(async move { - let pkg = install_package(data, &lua, &tree, &config).await?; - Ok::<_, InstallError>((pkg.id(), (pkg, entry_type))) - }) - .instrument(tracing::trace_span!("install_worker")) -} - -#[tracing::instrument(level = "trace", skip_all)] -async fn install_package( - data: PackageInstallData, - lua: &Arc, - tree: &T, - config: &Config, -) -> Result -where - T: InstallTree + Sync, -{ - match data.downloaded_rock { - RemoteRockDownload::RockspecOnly { rockspec_download } => { - install_rockspec( - rockspec_download, - None, - data.spec.constraint(), - data.build_behaviour, - data.pin, - data.opt, - data.entry_type, - lua, - tree, - config, - ) - .await - } - RemoteRockDownload::BinaryRock { - rockspec_download, - packed_rock, - } => { - install_binary_rock( - rockspec_download, - packed_rock, - data.spec.constraint(), - data.build_behaviour, - data.pin, - data.opt, - data.entry_type, - config, - tree, - ) - .await - } - RemoteRockDownload::SrcRock { - rockspec_download, - src_rock, - source_url, - } => { - let src_rock_source = SrcRockSource { - bytes: src_rock, - source_url, - }; - install_rockspec( - rockspec_download, - Some(src_rock_source), - data.spec.constraint(), - data.build_behaviour, - data.pin, - data.opt, - data.entry_type, - lua, - tree, - config, - ) - .await - } - } -} - -fn worker_result( - result: Result, JoinError>, -) -> Result<(), Box> { - match result { - Ok(Ok(())) => Ok(()), - Ok(Err(err)) => Err(err.into()), - Err(join) => Err(InstallError::from(join).into()), - } -} - -async fn wait_for_next_install( - ongoing_installs: &mut FuturesUnordered< - tracing::instrument::Instrumented>, - >, - installed_packages: &mut HashMap, -) -> Result { - if let Some(result) = ongoing_installs.next().await { - match result { - Ok(Ok((id, installed))) => { - installed_packages.insert(id, installed); - Ok(true) - } - Ok(Err(err)) => Err(err), - Err(join) => Err(InstallError::from(join)), - } - } else { - Ok(false) - } -} -trait LockfileExt { - fn add_dependencies( - self, - id: &LocalPackageId, - pkg: &LocalPackage, - entry_type: tree::EntryType, - all_packages: &HashMap, - installed_packages: &HashMap, - ) -> io::Result<()>; - - fn add_build_dependencies( - self, - id: &LocalPackageId, - pkg: &LocalPackage, - all_packages: &HashMap, - build_dependencies: &HashMap, - ) -> io::Result<()>; -} - -impl LockfileExt for &mut Lockfile { - fn add_dependencies( - self, - id: &LocalPackageId, - pkg: &LocalPackage, - entry_type: tree::EntryType, - all_packages: &HashMap, - installed_packages: &HashMap, - ) -> io::Result<()> { - if entry_type == tree::EntryType::Entrypoint { - self.add_entrypoint(pkg); - } - - for dependency_id in all_packages - .get(id) - .map(|pkg| pkg.spec.dependencies()) - .unwrap_or_default() - .into_iter() - { - self.add_dependency( - pkg, - installed_packages - .get(dependency_id) - .map(|(pkg, _)| pkg) - .ok_or(io::Error::other( - r#" -error writing dependencies to the lockfile. -A required dependency was not installed correctly. -This is likely because an install thread panicked and was interrupted unexpectedly. - -[THIS IS A BUG!] -"#, - ))?, - ); - } - Ok(()) - } - - fn add_build_dependencies( - self, - id: &LocalPackageId, - pkg: &LocalPackage, - all_packages: &HashMap, - build_dependencies: &HashMap, - ) -> io::Result<()> { - for dependency_id in all_packages - .get(id) - .map(|pkg| pkg.spec.build_dependencies()) - .unwrap_or_default() - .into_iter() - { - self.add_build_dependency( - pkg, - build_dependencies - .get(dependency_id) - .map(|(pkg, _)| pkg) - .ok_or(io::Error::other( - r#" -error writing build dependencies to the lockfile. -A required build dependency was not installed correctly. -This is likely because an install thread panicked and was interrupted unexpectedly. - -[THIS IS A BUG!] -"#, - ))?, - ); - } - Ok(()) - } -} - -/// Whether all build dependencies of the given rockspec have been installed -/// into the build tree, so that the package can start building. -/// -/// A build dependency is considered ready when it has been freshly installed -/// into the build tree, or when it was already present in the build lockfile -/// and satisfies the dependency constraint. -fn build_dependencies_ready( - rockspec: &impl Rockspec, - behaviour: BuildBehaviour, - build_lockfile: &Lockfile, - installed: &HashSet, -) -> bool { - let build_deps = rockspec.build_dependencies().current_platform(); - build_dependencies_to_install(rockspec).iter().all(|name| { - installed.contains(name) - || (behaviour != BuildBehaviour::Force - && build_lockfile - .has_rock( - &build_deps - .iter() - .find(|dep| dep.name() == name) - .map(|dep| dep.package_req().clone()) - .unwrap_or_else(|| PackageReq::from(name.clone())), - None, - ) - .is_some()) - }) -} - -#[allow(clippy::too_many_arguments)] -#[tracing::instrument( - name = "Installing", - level = "info", - skip_all, - fields( - package = rockspec_download.rockspec.package().to_string(), - version = rockspec_download.rockspec.version().to_string(), - ), -)] -async fn install_rockspec( - rockspec_download: DownloadedRockspec, - src_rock_source: Option, - constraint: LockConstraint, - behaviour: BuildBehaviour, - pin: PinnedState, - opt: OptState, - entry_type: tree::EntryType, - lua: &LuaInstallation, - tree: &T, - config: &Config, -) -> Result -where - T: InstallTree + Sync, -{ - let package = rockspec_download.rockspec.package().clone(); - let rockspec = rockspec_download.rockspec; - let source = rockspec_download.source; - - if let Some(BuildBackendSpec::LuaRock(_)) = &rockspec.build().current_platform().build_backend { - let luarocks_tree = tree.build_tree(config)?; - let luarocks = LuaRocksInstallation::new(config, luarocks_tree)?; - luarocks.ensure_installed(lua).await?; - } - - let source_spec = match src_rock_source { - Some(src_rock_source) => RemotePackageSourceSpec::SrcRock(src_rock_source), - None => RemotePackageSourceSpec::RockSpec(rockspec_download.source_url), - }; - - let pkg = Build::new() - .rockspec(&rockspec) - .lua(lua) - .tree(tree) - .entry_type(entry_type) - .config(config) - .pin(pin) - .opt(opt) - .constraint(constraint) - .behaviour(behaviour) - .source(source) - .source_spec(source_spec) - .build() + InstallPackages::new(install.config, install.tree) + .packages(packages) + .maybe_package_db(install.package_db) + .install() .await - .map_err(|err| InstallError::Build(package, Box::new(err)))?; - Ok(pkg) -} - -#[allow(clippy::too_many_arguments)] -#[tracing::instrument( - name = "Installing (pre-built)", - level = "info", - skip_all, - fields( - package = rockspec_download.rockspec.package().to_string(), - version = rockspec_download.rockspec.version().to_string(), - ), -)] -async fn install_binary_rock( - rockspec_download: DownloadedRockspec, - packed_rock: Bytes, - constraint: LockConstraint, - behaviour: BuildBehaviour, - pin: PinnedState, - opt: OptState, - entry_type: tree::EntryType, - config: &Config, - tree: &impl InstallTree, -) -> Result { - let rockspec = rockspec_download.rockspec; - let package = rockspec.package().clone(); - let pkg = BinaryRockInstall::new( - &rockspec, - rockspec_download.source, - packed_rock, - entry_type, - config, - tree, - ) - .pin(pin) - .opt(opt) - .constraint(constraint) - .behaviour(behaviour) - .install() - .await - .map_err(|err| InstallError::InstallBinaryRock(package, Box::new(err)))?; - Ok(pkg) + .map_err(InstallError::from) } diff --git a/lux-lib/src/operations/install_dependencies.rs b/lux-lib/src/operations/install_dependencies.rs index 16e9332a5..6c6d290f5 100644 --- a/lux-lib/src/operations/install_dependencies.rs +++ b/lux-lib/src/operations/install_dependencies.rs @@ -3,10 +3,10 @@ use itertools::Itertools; use crate::{ config::Config, - lockfile::LocalPackage, + lockfile::LockedPackage, lua_installation::LuaInstallation, luarocks::luarocks_installation::LuaRocksInstallation, - operations::{Install, InstallError}, + operations::{InstallError, pipeline::install_packages::InstallPackages}, project::project_toml::LocalProjectToml, rockspec::Rockspec, tree::{self, InstallTree}, @@ -36,7 +36,7 @@ impl< { /// Installs the configured dependencies and build dependencies into [`Self::tree`], /// returning the installed regular dependencies. - pub(crate) async fn build(self) -> Result, InstallError> { + pub(crate) async fn build(self) -> Result, InstallError> { let args = self._build(); let config = args.config; let dependencies = args.dependencies; @@ -47,18 +47,17 @@ impl< let luarocks = args.luarocks; if !build_dependencies.is_empty() { luarocks.ensure_installed(lua).await?; - Install::new(config) + InstallPackages::new(config, build_tree.clone()) .packages(build_dependencies.into_iter().unique().collect_vec()) - .tree(build_tree.clone()) .install() - .await?; + .await + .map_err(InstallError::from)?; } - // for some reason, cargo can't infer the type - let dependencies = Install::new(config) + let dependencies = InstallPackages::new(config, (*tree).clone()) .packages(dependencies.into_iter().unique().collect_vec()) - .tree(tree.clone()) .install() - .await?; + .await + .map_err(InstallError::from)?; Ok(dependencies) } } diff --git a/lux-lib/src/operations/install_project.rs b/lux-lib/src/operations/install_project.rs index fa2bf851e..4fcb0472f 100644 --- a/lux-lib/src/operations/install_project.rs +++ b/lux-lib/src/operations/install_project.rs @@ -1,7 +1,7 @@ use crate::{ build::{Build, BuildBehaviour, BuildError}, config::Config, - lockfile::LocalPackage, + lockfile::LockedPackage, lua_installation::{LuaInstallation, LuaInstallationError}, luarocks::luarocks_installation::{LuaRocksError, LuaRocksInstallError, LuaRocksInstallation}, operations::{install_dependencies::prepare_dependencies_for_build, InstallDependencies}, @@ -80,7 +80,7 @@ impl< > InstallProjectBuilder<'_, T, State> { /// Returns `Some` if the `only_deps` option is set to `false`. - pub async fn build(self) -> Result { + pub async fn build(self) -> Result { let args = self._build(); let config = args.config; let project = args.project; diff --git a/lux-lib/src/operations/install_rockspec.rs b/lux-lib/src/operations/install_rockspec.rs index ae74ad96d..dc1f6fd0f 100644 --- a/lux-lib/src/operations/install_rockspec.rs +++ b/lux-lib/src/operations/install_rockspec.rs @@ -5,7 +5,7 @@ use thiserror::Error; use crate::{ build::{self, BuildBehaviour, BuildError}, config::Config, - lockfile::{LocalPackage, OptState, PinnedState}, + lockfile::{LockedPackage, OptState, PinnedState}, lua_installation::{LuaInstallation, LuaInstallationError}, lua_rockspec::{BuildBackendSpec, LuaVersionError, RemoteLuaRockspec}, luarocks::luarocks_installation::{LuaRocksError, LuaRocksInstallError, LuaRocksInstallation}, @@ -73,7 +73,7 @@ impl< State: install_rockspec_builder::State + install_rockspec_builder::IsComplete, > InstallRockspecBuilder<'_, T, State> { - pub async fn install(self) -> Result { + pub async fn install(self) -> Result { let args = self._build(); let rockspec = args.rockspec; let pin = args.pin; diff --git a/lux-lib/src/operations/mod.rs b/lux-lib/src/operations/mod.rs index 580481149..9d63be8e3 100644 --- a/lux-lib/src/operations/mod.rs +++ b/lux-lib/src/operations/mod.rs @@ -14,6 +14,7 @@ mod install_project; mod install_rockspec; mod pack; mod pin; +pub mod pipeline; mod resolve; mod run; mod run_lua; diff --git a/lux-lib/src/operations/pack.rs b/lux-lib/src/operations/pack.rs index 23dfee32c..a84e3c6a8 100644 --- a/lux-lib/src/operations/pack.rs +++ b/lux-lib/src/operations/pack.rs @@ -1,7 +1,7 @@ use crate::build::utils; use crate::build::utils::c_dylib_extension; use crate::fs; -use crate::lockfile::LocalPackage; +use crate::lockfile::LockedPackage; use crate::luarocks; use crate::luarocks::rock_manifest::DirOrFileEntry; use crate::luarocks::rock_manifest::RockManifest; @@ -42,7 +42,7 @@ pub struct Pack { #[builder(start_fn)] tree: Tree, #[builder(start_fn)] - package: LocalPackage, + package: LockedPackage, } impl PackBuilder diff --git a/lux-lib/src/operations/pin.rs b/lux-lib/src/operations/pin.rs index af755579e..497115937 100644 --- a/lux-lib/src/operations/pin.rs +++ b/lux-lib/src/operations/pin.rs @@ -2,7 +2,7 @@ use std::io; use crate::{ fs, - lockfile::{FlushLockfileError, LocalPackageId, PinnedState}, + lockfile::{FlushLockfileError, LockedPackageId, PinnedState}, package::PackageSpec, tree::{InstallTree, Tree, TreeError}, }; @@ -11,13 +11,13 @@ use itertools::Itertools; use miette::Diagnostic; use thiserror::Error; -// TODO(vhyrro): Differentiate pinned LocalPackages at the type level? +// TODO(vhyrro): Differentiate pinned LockedPackages at the type level? #[derive(Error, Debug, Diagnostic)] pub enum PinError { #[error("package with ID '{0}' not found in the lockfile")] #[diagnostic(help("this is probably a bug"))] - PackageNotFound(LocalPackageId), + PackageNotFound(LockedPackageId), #[error("rock {rock} is already {}pinned!", if *.pin_state == PinnedState::Unpinned { "un" } else { "" })] PinStateUnchanged { pin_state: PinnedState, @@ -46,7 +46,7 @@ pub enum PinError { } pub fn set_pinned_state( - package_id: &LocalPackageId, + package_id: &LockedPackageId, tree: &Tree, pin: PinnedState, ) -> Result<(), PinError> { diff --git a/lux-lib/src/operations/pipeline/build.rs b/lux-lib/src/operations/pipeline/build.rs new file mode 100644 index 000000000..9006114a1 --- /dev/null +++ b/lux-lib/src/operations/pipeline/build.rs @@ -0,0 +1,216 @@ +use std::collections::{HashMap, HashSet}; + +use bon::Builder; +use miette::Diagnostic; +use thiserror::Error; + +use crate::{ + build::{deploy, BuildBehaviour}, + config::Config, + lockfile::{LockedPackage, LockedPackageId}, + lua_installation::{LuaInstallation, LuaInstallationError}, + lua_rockspec::BuildBackendSpec, + luarocks::{ + install_binary_rock::{BinaryRockInstall, InstallBinaryRockError}, + luarocks_installation::{LuaRocksError, LuaRocksInstallError, LuaRocksInstallation}, + }, + rockspec::Rockspec, + tree::{InstallTree, TreeError}, +}; + +use super::download_sources_and_hash::{PackageSource, DownloadedPackage}; + +#[derive(Error, Debug, Diagnostic)] +#[non_exhaustive] +pub(crate) enum BuildError { + #[error("failed to build '{0}'")] + Build(String, #[source] Box), + #[error(transparent)] + #[diagnostic(transparent)] + Tree(#[from] TreeError), + #[error("failed to install pre-built rock '{0}'")] + InstallBinaryRock(String, #[source] Box), + #[error(transparent)] + #[diagnostic(transparent)] + LuaInstallation(#[from] LuaInstallationError), + #[error("error instantiating LuaRocks compatibility layer")] + #[diagnostic(forward(0))] + LuaRocks(#[from] LuaRocksError), + #[error("error installing LuaRocks compatibility layer")] + #[diagnostic(forward(0))] + LuaRocksInstall(#[from] Box), +} + +impl From for BuildError { + fn from(source: LuaRocksInstallError) -> Self { + Self::LuaRocksInstall(Box::new(source)) + } +} + +#[derive(Builder)] +#[builder(start_fn = new, finish_fn(name = _build, vis = ""))] +pub(crate) struct Build<'a, T: InstallTree + Clone + Send + Sync> { + #[builder(start_fn)] + pub(crate) config: &'a Config, + #[builder(start_fn)] + pub(crate) tree: &'a T, + #[builder(field)] + pub(crate) packages: Vec, + pub(crate) behaviour: Option, +} + +impl BuildBuilder<'_, T, State> +where + T: InstallTree + Clone + Send + Sync, + State: build_builder::State, +{ + pub(crate) fn packages(mut self, packages: Vec) -> Self { + self.packages = packages; + self + } + + pub(crate) fn package(mut self, package: DownloadedPackage) -> Self { + self.packages.push(package); + self + } +} + +impl BuildBuilder<'_, T, State> +where + T: InstallTree + Clone + Send + Sync, + State: build_builder::State + build_builder::IsComplete, +{ + // INVESTIGATE(vhyrro): Is there a benefit of having a `LocalPackage` type which + // contains a LockedPackage as well as Layout information and installation path? + // LockedPackages are emitted by DownloadSourcesAndHash, and they represent packages ready for + // the lockfile but not installed to the tree yet. Either we create an intermediate type + // or a `LocalPackage` type. + pub(crate) async fn build(self) -> Result, BuildError> { + let args = self._build(); + let behaviour = args.behaviour.unwrap_or_default(); + let lua = LuaInstallation::new_from_config(args.config).await?; + + let needs_luarocks = args.packages.iter().any(|package| { + matches!( + package.rockspec.build().current_platform().build_backend, + Some(BuildBackendSpec::LuaRock(_)) + ) + }); + if needs_luarocks { + LuaRocksInstallation::new(args.config, args.tree.build_tree(args.config)?)? + .ensure_installed(&lua) + .await?; + } + + let mut installed = Vec::new(); + + for downloaded in order_by_build_dependencies(args.packages) { + let DownloadedPackage { + package, + rockspec, + entry_type, + artifact, + } = downloaded; + + let is_entrypoint = entry_type.is_entrypoint(); + let constraint = package.spec.constraint(); + let pin = package.spec.pinned(); + let opt = package.spec.opt(); + + let pkg = match artifact { + PackageSource::SourceTree(dir) => deploy( + &rockspec, + args.tree, + package, + &lua, + dir.path(), + entry_type, + args.config, + behaviour, + ) + .await + .map_err(|err| BuildError::Build(rockspec.package().to_string(), Box::new(err)))?, + PackageSource::PackedRock(bytes) => BinaryRockInstall::new( + &rockspec, + package.source().clone(), + bytes, + entry_type, + args.config, + args.tree, + ) + .pin(pin) + .opt(opt) + .constraint(constraint) + .behaviour(behaviour) + .install() + .await + .map_err(|err| { + BuildError::InstallBinaryRock(rockspec.package().to_string(), Box::new(err)) + })?, + }; + + // Record the installed package in the tree lockfile so that its paths are visible to + // the packages built after it (e.g. LuaRocks build backends), as well as future builds. + let mut lockfile = args.tree.lockfile()?.write_guard(); + if is_entrypoint { + lockfile.add_entrypoint(&pkg); + } else { + lockfile.add(&pkg); + } + drop(lockfile); + + installed.push(pkg); + } + + Ok(installed) + } +} + +fn order_by_build_dependencies(packages: Vec) -> Vec { + let mut by_id: HashMap = packages + .into_iter() + .map(|package| (package.package.id(), package)) + .collect(); + + let mut ordered = Vec::new(); + let mut visited = HashSet::new(); + + let ids: Vec = by_id.keys().cloned().collect(); + for id in ids { + visit(id, &mut by_id, &mut visited, &mut ordered); + } + + ordered +} + +fn visit( + id: LockedPackageId, + by_id: &mut HashMap, + visited: &mut HashSet, + ordered: &mut Vec, +) { + if !visited.insert(id.clone()) { + return; + } + + let build_dependencies: Vec = by_id + .get(&id) + .map(|package| { + package + .package + .spec + .build_dependencies() + .into_iter() + .cloned() + .collect() + }) + .unwrap_or_default(); + + for dependency in build_dependencies { + visit(dependency, by_id, visited, ordered); + } + + if let Some(package) = by_id.remove(&id) { + ordered.push(package); + } +} diff --git a/lux-lib/src/operations/pipeline/discover.rs b/lux-lib/src/operations/pipeline/discover.rs new file mode 100644 index 000000000..aa282c51c --- /dev/null +++ b/lux-lib/src/operations/pipeline/discover.rs @@ -0,0 +1,114 @@ +use std::sync::Arc; + +use bon::Builder; +use miette::Diagnostic; +use thiserror::Error; + +use crate::{ + config::Config, + lua_rockspec::RemoteLuaRockspec, + operations::{self, Download, FetchVendored, FetchVendoredError, RemoteRockDownload}, + package::{PackageReq, PackageSpec, RemotePackage}, + remote_package_db::{RemotePackageDB, SearchError}, +}; + +use super::download_sources_and_hash::PackageSource; + +pub(crate) struct FoundPackage { + pub(crate) package: RemotePackage, + pub(crate) rockspec: RemoteLuaRockspec, + pub(crate) artifact: Option, +} + +pub(crate) enum FoundPackageType { + Package(FoundPackage), + Closure { + root: PackageSpec, + nodes: Vec, + }, +} + +impl FoundPackageType { + pub(crate) fn root(&self) -> &PackageSpec { + match self { + Self::Package(artifact) => &artifact.package.package, + Self::Closure { root, .. } => root, + } + } +} + +#[derive(Error, Debug, Diagnostic)] +#[non_exhaustive] +pub(crate) enum DiscoverError { + #[error(transparent)] + #[diagnostic(transparent)] + Search(#[from] SearchError), + #[error("failed to acquire metadata for '{0}'")] + #[diagnostic(forward(1))] + Download(PackageReq, Box), + #[error("failed to fetch vendored package for '{0}'")] + #[diagnostic(forward(1))] + FetchVendored(PackageReq, Box), +} + +pub(crate) trait FindPackageFromProvider: Clone + Send + Sync + 'static { + fn find( + &self, + req: &PackageReq, + ) -> impl std::future::Future> + Send; +} + +#[derive(Builder, Clone)] +#[builder(start_fn = new)] +pub(crate) struct FindPackageFromLuarocks { + #[builder(start_fn)] + pub(crate) package_db: Arc, + #[builder(start_fn)] + pub(crate) config: Arc, +} + +impl FindPackageFromProvider for FindPackageFromLuarocks { + async fn find(&self, req: &PackageReq) -> Result { + let mut package = self.package_db.find(req, None)?; + // TODO(vhyrro): Don't force us to call download here, extrapolate stuff from + // `operations` into this pipeline instead. + let download = if let Some(vendor_dir) = self.config.vendor_dir() { + FetchVendored::new() + .vendor_dir(vendor_dir) + .package(req) + .package_db(&self.package_db) + .fetch_vendored_rock() + .await + .map_err(|err| DiscoverError::FetchVendored(req.clone(), Box::new(err)))? + } else { + Download::new(req, &self.config) + .package_db(&self.package_db) + .download_remote_rock() + .await + .map_err(|err| DiscoverError::Download(req.clone(), Box::new(err)))? + }; + + let rockspec = download.rockspec().clone(); + + let artifact = match &download { + RemoteRockDownload::SrcRock { + src_rock, + source_url, + .. + } => { + package.source_url = Some(source_url.clone()); + Some(PackageSource::PackedRock(src_rock.clone())) + } + RemoteRockDownload::BinaryRock { packed_rock, .. } => { + Some(PackageSource::PackedRock(packed_rock.clone())) + } + RemoteRockDownload::RockspecOnly { .. } => None, + }; + + Ok(FoundPackageType::Package(FoundPackage { + package, + rockspec, + artifact, + })) + } +} diff --git a/lux-lib/src/operations/pipeline/download_sources_and_hash.rs b/lux-lib/src/operations/pipeline/download_sources_and_hash.rs new file mode 100644 index 000000000..cdd2b5399 --- /dev/null +++ b/lux-lib/src/operations/pipeline/download_sources_and_hash.rs @@ -0,0 +1,272 @@ +use std::collections::HashMap; +use std::io; +use std::io::Cursor; + +use bon::Builder; +use bytes::Bytes; +use miette::Diagnostic; +use tempfile::TempDir; +use thiserror::Error; + +use crate::{ + build::{BuildError, RemotePackageSourceSpec, SrcRockSource}, + config::Config, + fs::{self, FsError}, + hash::HasIntegrity, + lockfile::{ + LockConstraint, LockedPackage, LockedPackageHashes, LockedPackageId, OptState, PinnedState, + RemotePackageSourceUrl, + }, + lua_rockspec::RemoteLuaRockspec, + operations::{unpack_src_rock, FetchSrc, RemotePackageSourceMetadata}, + package::PackageSpec, + remote_package_source::RemotePackageSource, + rockspec::Rockspec, + tree::EntryType, +}; + +use super::resolve::{ResolvedArtifacts, ResolvedPackage}; + +pub(crate) enum PackageSource { + SourceTree(TempDir), + PackedRock(Bytes), +} + +pub(crate) struct DownloadedPackage { + pub(crate) package: LockedPackage, + pub(crate) rockspec: RemoteLuaRockspec, + pub(crate) entry_type: EntryType, + pub(crate) artifact: PackageSource, +} + +#[derive(Default)] +pub(crate) struct DownloadSourcesAndHashArtifacts { + pub(crate) regular: HashMap, + pub(crate) build: HashMap, + pub(crate) test: HashMap, +} + +#[derive(Error, Debug, Diagnostic)] +#[non_exhaustive] +pub(crate) enum DownloadSourcesAndHashError { + #[error("failed to fetch source for '{0}'")] + Fetch(String), + #[error(transparent)] + #[diagnostic(transparent)] + Fs(#[from] FsError), + #[error(transparent)] + #[diagnostic(transparent)] + Build(#[from] BuildError), + #[error("failed to hash the source for '{0}'")] + Hash(String, #[source] io::Error), + #[error("missing source url for '{0}'")] + MissingSourceUrl(String), +} + +#[derive(Builder)] +#[builder(start_fn = new, finish_fn(name = _build, vis = ""))] +pub(crate) struct DownloadSourcesAndHash<'a> { + #[builder(start_fn)] + pub(crate) config: &'a Config, + pub(crate) resolved: ResolvedArtifacts, +} + +impl DownloadSourcesAndHashBuilder<'_, State> +where + State: download_sources_and_hash_builder::State + download_sources_and_hash_builder::IsComplete, +{ + pub(crate) async fn download_sources_and_hash( + self, + ) -> Result { + let args = self._build(); + let mut out = DownloadSourcesAndHashArtifacts::default(); + + for (id, package) in args.resolved.regular { + out.regular + .insert(id, download_sources_and_hash(args.config, package).await?); + } + for (id, package) in args.resolved.build { + out.build + .insert(id, download_sources_and_hash(args.config, package).await?); + } + for (id, package) in args.resolved.test { + out.test + .insert(id, download_sources_and_hash(args.config, package).await?); + } + + Ok(out) + } +} + +async fn download_sources_and_hash( + config: &Config, + resolved: ResolvedPackage, +) -> Result { + let ResolvedPackage { + spec, + rockspec, + source, + source_url, + entry_type, + artifact, + } = resolved; + + let constraint = spec.constraint(); + let pin = spec.pinned(); + let opt = spec.opt(); + let name = spec.name().to_string(); + + let (package, artifact) = match artifact { + Some(PackageSource::PackedRock(bytes)) => match &source { + RemotePackageSource::LuarocksBinaryRock(url) => { + let binary_url = url.clone(); + let rockspec_hash = rockspec + .hash() + .await + .map_err(|err| DownloadSourcesAndHashError::Hash(name.clone(), err))?; + let source_hash = bytes + .hash() + .await + .map_err(|err| DownloadSourcesAndHashError::Hash(name.clone(), err))?; + let hashes = LockedPackageHashes { + rockspec: rockspec_hash, + source: source_hash, + }; + let package = LockedPackage { + spec, + source, + source_url: Some(RemotePackageSourceUrl::Url { url: binary_url }), + hashes, + }; + (package, PackageSource::PackedRock(bytes)) + } + _ => { + let source_url = source_url + .clone() + .ok_or_else(|| DownloadSourcesAndHashError::MissingSourceUrl(name.clone()))?; + let (mut package, dir) = fetch_and_hash_source( + &rockspec, + Some(RemotePackageSourceSpec::SrcRock(SrcRockSource { + bytes, + source_url, + })), + Some(source), + constraint, + pin, + opt, + config, + ) + .await?; + package.spec = spec; + (package, PackageSource::SourceTree(dir)) + } + }, + Some(PackageSource::SourceTree(dir)) => { + let rockspec_hash = rockspec + .hash() + .await + .map_err(|err| DownloadSourcesAndHashError::Hash(name.clone(), err))?; + let source_hash = dir + .path() + .hash() + .await + .map_err(|err| DownloadSourcesAndHashError::Hash(name.clone(), err))?; + let hashes = LockedPackageHashes { + rockspec: rockspec_hash, + source: source_hash, + }; + let package = LockedPackage { + spec, + source, + source_url, + hashes, + }; + (package, PackageSource::SourceTree(dir)) + } + None => { + let (mut package, dir) = fetch_and_hash_source( + &rockspec, + Some(RemotePackageSourceSpec::RockSpec(source_url)), + Some(source), + constraint, + pin, + opt, + config, + ) + .await?; + package.spec = spec; + (package, PackageSource::SourceTree(dir)) + } + }; + + Ok(DownloadedPackage { + package, + rockspec, + entry_type, + artifact, + }) +} + +pub(crate) async fn fetch_and_hash_source( + rockspec: &R, + source_spec: Option, + source: Option, + constraint: LockConstraint, + pin: PinnedState, + opt: OptState, + config: &Config, +) -> Result<(LockedPackage, TempDir), BuildError> { + let temp_dir = fs::tempfile::tempdir()?; + + let source_metadata = match source_spec { + Some(RemotePackageSourceSpec::SrcRock(SrcRockSource { bytes, source_url })) => { + // FIXME(vhyrro): This shouldn't be in this function, or the function should be named differently. + // "fetching" has nothing to do with unpacking. + let hash = bytes.hash().await?; + let cursor = Cursor::new(bytes); + unpack_src_rock(cursor, temp_dir.path().to_path_buf()) + .await + .map_err(BuildError::UnpackSrcRock)?; + RemotePackageSourceMetadata { hash, source_url } + } + Some(RemotePackageSourceSpec::RockSpec(source_url)) => { + FetchSrc::new(temp_dir.path(), rockspec, config) + .maybe_source_url(source_url) + .fetch_internal() + .await? + } + None => { + FetchSrc::new(temp_dir.path(), rockspec, config) + .fetch_internal() + .await? + } + }; + + let hashes = LockedPackageHashes { + rockspec: rockspec.hash().await?, + source: source_metadata.hash.clone(), + }; + + let mut package = LockedPackage::from( + &PackageSpec::new(rockspec.package().clone(), rockspec.version().clone()), + constraint, + rockspec.binaries(), + source + .map(Result::Ok) + .unwrap_or_else(|| { + rockspec + .to_lua_remote_rockspec_string() + .map(RemotePackageSource::RockspecContent) + }) + .unwrap_or(RemotePackageSource::Local), + Some(source_metadata.source_url.clone()), + hashes, + ); + // FIXME(vhyrro): We should reconsider our constructors if we have to set + // these variants ourselves. Either put these in a different place, or resolve + // these in an earlier step. + package.spec.pinned = pin; + package.spec.opt = opt; + + Ok((package, temp_dir)) +} diff --git a/lux-lib/src/operations/pipeline/install_packages.rs b/lux-lib/src/operations/pipeline/install_packages.rs new file mode 100644 index 000000000..9d88fcf8f --- /dev/null +++ b/lux-lib/src/operations/pipeline/install_packages.rs @@ -0,0 +1,291 @@ +use std::{collections::HashMap, io, sync::Arc}; + +use bon::Builder; +use itertools::Itertools; +use miette::Diagnostic; +use thiserror::Error; + +use crate::{ + build::BuildBehaviour, + config::Config, + lockfile::{FlushLockfileError, LockedPackage, LockedPackageId, Lockfile, ReadWrite}, + lua_installation::LuaInstallationError, + luarocks::luarocks_installation::{LuaRocksError, LuaRocksInstallError}, + package::{PackageName, PackageNameList}, + remote_package_db::{RemotePackageDB, RemotePackageDBError}, + tree::{self, InstallTree, TreeError}, + workspace::WorkspaceTreeError, +}; + +use super::{ + build::Build as PipelineBuild, + discover::FindPackageFromLuarocks, + download_sources_and_hash::DownloadSourcesAndHash, + resolve::ResolvePackageDependencies, +}; + +use crate::operations::PackageInstallSpec; + +/// Installs a set of [`PackageInstallSpec`]s into an install tree. +/// +/// This is the composable core of [`Install`](crate::operations::Install): it resolves the +/// dependency graph, downloads and hashes sources, builds the packages (build dependencies +/// first) and records the result in the tree's lockfile. +#[derive(Builder)] +#[builder(start_fn = new, finish_fn(name = _build, vis = ""))] +pub struct InstallPackages<'a, T> +where + T: InstallTree + Clone + Send + Sync, +{ + #[builder(start_fn)] + config: &'a Config, + #[builder(start_fn)] + tree: T, + packages: Vec, + package_db: Option, +} + +impl InstallPackagesBuilder<'_, T, State> +where + State: install_packages_builder::State + install_packages_builder::IsComplete, + T: InstallTree + Clone + Send + Sync + 'static, +{ + pub async fn install(self) -> Result, InstallPackagesError> { + let args = self._build(); + if args.packages.is_empty() { + return Ok(Vec::new()); + } + install_packages(args).await + } +} + +async fn install_packages( + install: InstallPackages<'_, T>, +) -> Result, InstallPackagesError> +where + T: InstallTree + Clone + Send + Sync + 'static, +{ + let package_db = match install.package_db { + Some(db) => db, + None => RemotePackageDB::from_config(install.config).await?, + }; + + let duplicate_entrypoints = install + .packages + .iter() + .filter(|pkg| pkg.entry_type == tree::EntryType::Entrypoint) + .map(|pkg| pkg.package.name()) + .duplicates() + .cloned() + .collect_vec(); + + if !duplicate_entrypoints.is_empty() { + return Err(InstallPackagesError::DuplicateEntrypoints( + PackageNameList::new(duplicate_entrypoints), + )); + } + + let packages = install.packages; + let config = install.config; + let tree = &install.tree; + + let lockfile = tree.lockfile()?; + let build_tree = tree.build_tree(config)?; + + let entrypoint_specs = packages + .iter() + .filter(|spec| spec.entry_type == tree::EntryType::Entrypoint) + .collect_vec(); + + // Entrypoints already installed that conflict with another entrypoint. This enumerates both + // packages that are planned to be removed (`--force`), as well as those which are + // unintentionally causing conflicts. + let conflicting_entrypoints: HashMap = entrypoint_specs + .iter() + .filter_map(|spec| lockfile.entrypoint(spec.package.name()).cloned()) + .map(|package| (package.name().clone(), package)) + .collect(); + + let unforced_conflicts = entrypoint_specs + .iter() + .filter(|spec| spec.build_behaviour != BuildBehaviour::Force) + .filter_map(|spec| conflicting_entrypoints.get(spec.package.name())) + .map(|existing| format!("{}@{}", existing.name(), existing.version())) + .collect_vec(); + + if !unforced_conflicts.is_empty() { + return Err(InstallPackagesError::ConflictingEntrypoints( + unforced_conflicts.join("\n"), + )); + } + + // Forced overwrites: remove the conflicting entrypoints from the tree. + // NOTE: non-transactional. If an error occurs, this removes the conflicting package without + // installing the substitute. Fix when transactions are implemented. + let conflicting_entrypoints = conflicting_entrypoints.into_values().collect_vec(); + for package in &conflicting_entrypoints { + tree.cleanup(package, tree::EntryType::Entrypoint)?; + } + + let discover = FindPackageFromLuarocks::new(Arc::new(package_db), Arc::new(config.clone())).build(); + let resolved = ResolvePackageDependencies::new(&discover, config) + .packages(packages) + .resolve() + .await + .map_err(|err| InstallPackagesError::Pipeline(Box::new(err)))?; + let artifacts = DownloadSourcesAndHash::new(config) + .resolved(resolved) + .download_sources_and_hash() + .await + .map_err(|err| InstallPackagesError::Pipeline(Box::new(err)))?; + + let build_packages = artifacts.build.into_values().collect_vec(); + let regular_packages = artifacts.regular.into_values().collect_vec(); + let regular_entry_types: HashMap = regular_packages + .iter() + .map(|pkg| (pkg.package.spec.id(), pkg.entry_type)) + .collect(); + + // Build dependencies first, then the packages themselves. + let built_build_deps = PipelineBuild::new(config, &build_tree) + .packages(build_packages) + .build() + .await + .map_err(|err| InstallPackagesError::Pipeline(Box::new(err)))?; + let built = PipelineBuild::new(config, tree) + .packages(regular_packages) + .build() + .await + .map_err(|err| InstallPackagesError::Pipeline(Box::new(err)))?; + + let installed_packages: HashMap = + built.iter().map(|pkg| (pkg.spec.id(), pkg.clone())).collect(); + let installed_build_deps: HashMap = built_build_deps + .iter() + .map(|pkg| (pkg.spec.id(), pkg.clone())) + .collect(); + + lockfile.map_then_flush(|lockfile| { + for package in &conflicting_entrypoints { + lockfile.remove_by_id(&package.id()); + } + for package in &built { + let entry_type = regular_entry_types[&package.spec.id()]; + lockfile.add_dependencies(package, entry_type, &installed_packages)?; + lockfile.add_build_dependencies(package, &installed_build_deps)?; + } + Ok::<_, io::Error>(()) + })?; + + Ok(built) +} + +#[derive(Error, Debug, Diagnostic)] +pub enum InstallPackagesError { + #[error(transparent)] + #[diagnostic(transparent)] + LuaVersionUnset(#[from] crate::lua_version::LuaVersionUnset), + #[error(transparent)] + #[diagnostic(transparent)] + LuaInstallation(#[from] LuaInstallationError), + #[error(transparent)] + #[diagnostic(transparent)] + FlushLockfile(#[from] FlushLockfileError), + #[error(transparent)] + #[diagnostic(transparent)] + Tree(#[from] TreeError), + #[error(transparent)] + #[diagnostic(transparent)] + WorkspaceTree(#[from] WorkspaceTreeError), + #[error("error instantiating LuaRocks compatibility layer")] + #[diagnostic(forward(0))] + LuaRocks(#[from] LuaRocksError), + #[error("error installing LuaRocks compatibility layer")] + #[diagnostic(forward(0))] + LuaRocksInstall(#[from] Box), + #[error("error initialising remote package DB")] + #[diagnostic(forward(0))] + RemotePackageDB(#[from] RemotePackageDBError), + #[error("cannot install duplicate entrypoints:\n{0}")] + DuplicateEntrypoints(PackageNameList), + #[error("cannot install conflicting entrypoints:\n{0}")] + #[diagnostic(help( + "only a single entrypoint per package is allowed.\nretry with `--force` to overwrite the existing entrypoint" + ))] + ConflictingEntrypoints(String), + #[error("failed to install packages")] + #[diagnostic(forward(0))] + Pipeline(Box), +} + +impl From for InstallPackagesError { + fn from(source: LuaRocksInstallError) -> Self { + Self::LuaRocksInstall(Box::new(source)) + } +} + +trait LockfileExt { + fn add_dependencies( + self, + pkg: &LockedPackage, + entry_type: tree::EntryType, + installed_packages: &HashMap, + ) -> io::Result<()>; + + fn add_build_dependencies( + self, + pkg: &LockedPackage, + build_dependencies: &HashMap, + ) -> io::Result<()>; +} + +impl LockfileExt for &mut Lockfile { + fn add_dependencies( + self, + pkg: &LockedPackage, + entry_type: tree::EntryType, + installed_packages: &HashMap, + ) -> io::Result<()> { + if entry_type == tree::EntryType::Entrypoint { + self.add_entrypoint(pkg); + } + + for dependency_id in pkg.spec.dependencies() { + self.add_dependency( + pkg, + installed_packages.get(dependency_id).ok_or(io::Error::other( + r#" +error writing dependencies to the lockfile. +A required dependency was not installed correctly. +This is likely a bug in Lux. + +[THIS IS A BUG!] +"#, + ))?, + ); + } + Ok(()) + } + + fn add_build_dependencies( + self, + pkg: &LockedPackage, + build_dependencies: &HashMap, + ) -> io::Result<()> { + for dependency_id in pkg.spec.build_dependencies() { + self.add_build_dependency( + pkg, + build_dependencies.get(dependency_id).ok_or(io::Error::other( + r#" +error writing build dependencies to the lockfile. +A required build dependency was not installed correctly. +This is likely a bug in Lux. + +[THIS IS A BUG!] +"#, + ))?, + ); + } + Ok(()) + } +} diff --git a/lux-lib/src/operations/pipeline/mod.rs b/lux-lib/src/operations/pipeline/mod.rs new file mode 100644 index 000000000..4744547f2 --- /dev/null +++ b/lux-lib/src/operations/pipeline/mod.rs @@ -0,0 +1,348 @@ +#![allow(dead_code)] + +pub(crate) mod build; +pub(crate) mod discover; +pub(crate) mod download_sources_and_hash; +pub mod install_packages; +pub(crate) mod resolve; + +use std::collections::HashMap; +use std::sync::Arc; + +use bon::Builder; +use miette::Diagnostic; +use thiserror::Error; + +use crate::{ + build::BuildBehaviour, config::Config, lockfile::{ + LockedPackage, LockedPackageId, LockedPackageLock, LockedPackageLockType, ReadWrite, + WorkspaceLockfile, + }, lua_installation::{LuaInstallation, LuaInstallationError}, operations::{GenLuaRc, GenLuaRcError, PackageInstallSpec, pipeline::download_sources_and_hash::{DownloadSourcesAndHashArtifacts, DownloadedPackage}}, package::PackageName, project::{project_toml::LocalProjectTomlValidationError, Project}, remote_package_db::{RemotePackageDB, RemotePackageDBError}, rockspec::Rockspec, tree::{EntryType, InstallTree, TreeError}, workspace::{Workspace, WorkspaceError, WorkspaceTreeError}, +}; + +use self::{ + build::Build, discover::FindPackageFromLuarocks, download_sources_and_hash::DownloadSourcesAndHash, + resolve::ResolvePackageDependencies, +}; + +#[derive(Builder)] +#[builder(start_fn = new, finish_fn(name = _build, vis = ""))] +pub struct Pipeline<'a> { + #[builder(start_fn)] + pub(crate) config: &'a Config, + #[builder(start_fn)] + pub(crate) workspace: &'a Workspace, + #[builder(field)] + pub(crate) packages: Vec, + pub(crate) test: Option, + /// Build the workspace's own project(s) after installing dependencies. + pub(crate) build_projects: Option, + /// Only build this workspace member. + pub(crate) package: Option, + /// Ignore the project's lockfile and don't create one. + pub(crate) no_lock: Option, +} + +impl PipelineBuilder<'_, State> +where + State: pipeline_builder::State, +{ + pub fn packages(mut self, packages: Vec) -> Self { + self.packages = packages; + self + } +} + +impl PipelineBuilder<'_, State> +where + State: pipeline_builder::State + pipeline_builder::IsComplete, +{ + pub async fn run(self) -> miette::Result> { + let args = self._build(); + let config = args.config; + let workspace = args.workspace; + let no_lock = args.no_lock.unwrap_or(false); + + let package_db = RemotePackageDB::from_config(config).await?; + let discover = FindPackageFromLuarocks::new(Arc::new(package_db), Arc::new(config.clone())).build(); + + let mut regular = gather_dependencies(workspace, DependencyKind::Regular)?; + regular.extend(args.packages); + let build = gather_dependencies(workspace, DependencyKind::Build)?; + let test = if args.test.unwrap_or(false) { + gather_dependencies(workspace, DependencyKind::Test)? + } else { + Vec::new() + }; + + let resolved = ResolvePackageDependencies::new(&discover, config) + .packages(regular) + .build_packages(build) + .test_packages(test) + .resolve() + .await?; + + let artifacts = DownloadSourcesAndHash::new(config) + .resolved(resolved) + .download_sources_and_hash() + .await?; + + if !no_lock { + let mut lockfile = workspace.lockfile()?.write_guard(); + EmitLockfile::new(&mut lockfile) + .artifacts(&artifacts) + .emit()?; + } + + let regular_tree = workspace.tree(config)?; + let build_tree = workspace.build_tree(config)?; + let test_tree = workspace.test_tree(config)?; + + // TODO(vhyrro): Make parallel + // make sure to build build dependencies first + Build::new(config, &build_tree) + .packages(artifacts.build.into_values().collect()) + .build() + .await?; + Build::new(config, ®ular_tree) + .packages(artifacts.regular.into_values().collect()) + .build() + .await?; + Build::new(config, &test_tree) + .packages(artifacts.test.into_values().collect()) + .build() + .await?; + + let mut built = Vec::new(); + if args.build_projects.unwrap_or(false) { + let lua = LuaInstallation::new_from_config(config).await?; + match &args.package { + Some(package) => { + let project = workspace.select_member(package)?; + built.push(build_project(project, workspace, &lua, config).await?); + } + None => { + for project in workspace.members() { + built.push(build_project(project, workspace, &lua, config).await?); + } + } + } + } + + if !no_lock { + GenLuaRc::new() + .config(config) + .workspace(workspace) + .generate_luarc() + .await?; + } + + Ok(built) + } +} + +#[derive(Clone, Copy)] +enum DependencyKind { + Regular, + Build, + Test, +} + +fn gather_dependencies( + workspace: &Workspace, + kind: DependencyKind, +) -> Result, PipelineError> { + let mut packages = Vec::new(); + for project in workspace.members() { + let toml = project.toml().into_local()?; + let dependencies = match kind { + DependencyKind::Regular => toml.dependencies().current_platform().clone(), + DependencyKind::Build => toml.build_dependencies().current_platform().clone(), + DependencyKind::Test => toml.test_dependencies().current_platform().clone(), + }; + for dependency in dependencies { + // From the perspective of a project, dependencies are entrypoints. + packages.push( + PackageInstallSpec::new(dependency.package_req().clone(), EntryType::Entrypoint) + .maybe_source(dependency.source().clone()) + .build(), + ); + } + } + Ok(packages) +} + +async fn build_project( + project: &Project, + workspace: &Workspace, + lua: &LuaInstallation, + config: &Config, +) -> Result { + let workspace_tree = workspace.tree(config)?; + let project_toml = project.toml().into_local()?; + + let package = crate::build::Build::new() + .rockspec(&project_toml) + .lua(lua) + .tree(&workspace_tree) + .entry_type(EntryType::Entrypoint) + .config(config) + .behaviour(BuildBehaviour::Force) + .build() + .await?; + + let lockfile = workspace_tree.lockfile()?; + let dependencies = lockfile + .rocks() + .iter() + .filter_map(|(id, package)| lockfile.is_entrypoint(id).then_some(package)) + .cloned() + .collect::>(); + let build_lockfile = workspace.build_tree(config)?.lockfile()?; + + let mut lockfile = lockfile.write_guard(); + lockfile.add_entrypoint(&package); + for dependency in dependencies { + lockfile.add_dependency(&package, &dependency); + } + for dependency in build_lockfile.rocks().values() { + lockfile.add_build_dependency(&package, dependency); + } + + Ok(package) +} + +#[derive(Error, Debug, Diagnostic)] +#[non_exhaustive] +pub(crate) enum PipelineError { + #[error(transparent)] + #[diagnostic(transparent)] + Resolve(#[from] resolve::ResolveError), + #[error(transparent)] + #[diagnostic(transparent)] + DownloadSourcesAndHash(#[from] download_sources_and_hash::DownloadSourcesAndHashError), + #[error(transparent)] + #[diagnostic(transparent)] + Build(#[from] build::BuildError), + #[error("failed to build the project")] + #[diagnostic(forward(0))] + ProjectBuild(#[source] Box), + #[error(transparent)] + #[diagnostic(transparent)] + LuaInstallation(#[from] LuaInstallationError), + #[error(transparent)] + #[diagnostic(transparent)] + Tree(#[from] TreeError), + #[error(transparent)] + #[diagnostic(transparent)] + Emit(#[from] EmitError), + #[error(transparent)] + #[diagnostic(transparent)] + GenLuaRc(#[from] GenLuaRcError), + #[error(transparent)] + #[diagnostic(transparent)] + Workspace(#[from] WorkspaceError), + #[error(transparent)] + #[diagnostic(transparent)] + WorkspaceTree(#[from] WorkspaceTreeError), + #[error(transparent)] + #[diagnostic(transparent)] + RemotePackageDB(#[from] RemotePackageDBError), + #[error(transparent)] + #[diagnostic(transparent)] + Project(#[from] LocalProjectTomlValidationError), +} + +// FIXME(vhyrro): Use the natural `#[from]` instead of manual implementation +impl From for PipelineError { + fn from(source: crate::build::BuildError) -> Self { + Self::ProjectBuild(Box::new(source)) + } +} + +#[derive(Builder)] +#[builder(start_fn = new, finish_fn(name = _build, vis = ""))] +pub(crate) struct EmitLockfile<'a> { + #[builder(start_fn)] + pub(crate) lockfile: &'a mut WorkspaceLockfile, + pub(crate) artifacts: &'a DownloadSourcesAndHashArtifacts, +} + +impl EmitLockfileBuilder<'_, State> +where + State: emit_lockfile_builder::State + emit_lockfile_builder::IsComplete, +{ + pub(crate) fn emit(self) -> Result<(), EmitError> { + let args = self._build(); + + args.lockfile.sync( + &lock_from(&args.artifacts.regular), + &LockedPackageLockType::Regular, + ); + args.lockfile.sync( + &lock_from(&args.artifacts.build), + &LockedPackageLockType::Build, + ); + args.lockfile.sync( + &lock_from(&args.artifacts.test), + &LockedPackageLockType::Test, + ); + + Ok(()) + } +} + +fn lock_from( + packages: &HashMap, +) -> LockedPackageLock { + // FIXME(vhyrro): Create a constructor here instead of mut overrides. + let mut lock = LockedPackageLock::default(); + for package in packages.values() { + lock.insert(package.package.clone(), package.entry_type.is_entrypoint()); + } + lock +} + +#[derive(Error, Debug, Diagnostic)] +#[non_exhaustive] +pub(crate) enum EmitError { + #[error("failed to serialize the lockfile")] + Serialize, +} + +#[cfg(all(test, feature = "impure_tests"))] +mod tests { + use std::path::PathBuf; + + use assert_fs::prelude::PathCopy; + + use crate::{config::ConfigBuilder, lua_version::LuaVersion, workspace::Workspace}; + + use super::*; + + #[tokio::test] + async fn pipeline_installs_workspace_dependencies() { + let sample = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("resources/test/sample-projects/dependencies/"); + let temp = assert_fs::TempDir::new().unwrap(); + temp.copy_from(sample, &["**"]).unwrap(); + + let workspace = Workspace::from_exact(temp.path()).unwrap().unwrap(); + let config = ConfigBuilder::new() + .unwrap() + .lua_version(Some(LuaVersion::Lua51)) + .build() + .unwrap(); + + Pipeline::new(&config, &workspace).run().await.unwrap(); + + let lockfile = workspace.lockfile().unwrap(); + let rocks = lockfile.rocks(&LockedPackageLockType::Regular); + assert!(rocks + .values() + .any(|pkg| pkg.name().to_string() == "lua-cjson")); + assert!(rocks + .values() + .any(|pkg| pkg.name().to_string() == "plenary.nvim")); + } +} diff --git a/lux-lib/src/operations/pipeline/resolve.rs b/lux-lib/src/operations/pipeline/resolve.rs new file mode 100644 index 000000000..e83a7098a --- /dev/null +++ b/lux-lib/src/operations/pipeline/resolve.rs @@ -0,0 +1,384 @@ +use std::cell::RefCell; +use std::collections::HashMap; +use std::rc::Rc; + +use async_recursion::async_recursion; +use bon::Builder; +use miette::Diagnostic; +use thiserror::Error; + +use crate::{ + config::Config, lockfile::{LockedPackage, LockedPackageHashes, LockedPackageId, LockedPackageSpec, RemotePackageSourceUrl}, lua_rockspec::RemoteLuaRockspec, operations::{PackageInstallSpec, RemoteRockDownload, resolve::build_dependencies_to_install}, package::{PackageName, PackageReq, RemotePackage}, remote_package_source::RemotePackageSource, rockspec::Rockspec, tree::EntryType, +}; + +use super::{ + discover::{FindPackageFromProvider, DiscoverError, FoundPackage, FoundPackageType}, + download_sources_and_hash::PackageSource, +}; + +pub(crate) struct ResolvedPackage { + pub(crate) spec: LockedPackageSpec, + pub(crate) rockspec: RemoteLuaRockspec, + pub(crate) source: RemotePackageSource, + pub(crate) source_url: Option, + pub(crate) entry_type: EntryType, + pub(crate) artifact: Option, +} + +impl ResolvedPackage { + pub(crate) fn id(&self) -> LockedPackageId { + self.spec.id() + } + + pub(crate) fn with_hashes(self, hashes: LockedPackageHashes) -> LockedPackage { + LockedPackage { + spec: self.spec, + source: self.source, + source_url: self.source_url, + hashes, + } + } +} + +#[derive(Default)] +pub(crate) struct ResolvedArtifacts { + pub(crate) regular: HashMap, + pub(crate) build: HashMap, + pub(crate) test: HashMap, +} + +#[derive(Error, Debug, Diagnostic)] +#[non_exhaustive] +pub(crate) enum ResolveError { + #[error(transparent)] + #[diagnostic(transparent)] + Discover(#[from] DiscoverError), + #[error("cyclic dependency detected:\n{0}")] + CyclicDependency(String), + #[error("discovery returned a precomputed closure, which this resolver does not support")] + UnsupportedClosure, + #[error(transparent)] + Join(#[from] tokio::task::JoinError), +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum ResolveStateType { + Regular, + Build, + Test, +} + +#[derive(Default)] +struct ResolveState { + regular: HashMap, + build: HashMap, + test: HashMap, +} + +impl ResolveState { + fn select(&mut self, state_type: ResolveStateType) -> &mut HashMap { + match state_type { + ResolveStateType::Regular => &mut self.regular, + ResolveStateType::Build => &mut self.build, + ResolveStateType::Test => &mut self.test, + } + } + + fn contains(&self, state_type: ResolveStateType, id: &LockedPackageId) -> bool { + match state_type { + ResolveStateType::Regular => self.regular.contains_key(id), + ResolveStateType::Build => self.build.contains_key(id), + ResolveStateType::Test => self.test.contains_key(id), + } + } +} + +#[derive(Builder)] +#[builder(start_fn = new, finish_fn(name = _build, vis = ""))] +pub(crate) struct ResolvePackageDependencies<'a, D: FindPackageFromProvider> { + #[builder(start_fn)] + pub(crate) discover: &'a D, + #[builder(start_fn)] + pub(crate) config: &'a Config, + #[builder(field)] + pub(crate) packages: Vec, + #[builder(field)] + pub(crate) build_packages: Vec, + #[builder(field)] + pub(crate) test_packages: Vec, +} + +impl ResolvePackageDependenciesBuilder<'_, D, State> +where + D: FindPackageFromProvider, + State: resolve_package_dependencies_builder::State, +{ + pub(crate) fn packages(mut self, packages: Vec) -> Self { + self.packages = packages; + self + } + + pub(crate) fn package(mut self, package: PackageInstallSpec) -> Self { + self.packages.push(package); + self + } + + pub(crate) fn build_packages(mut self, packages: Vec) -> Self { + self.build_packages = packages; + self + } + + pub(crate) fn test_packages(mut self, packages: Vec) -> Self { + self.test_packages = packages; + self + } + + pub(crate) fn test_package(mut self, package: PackageInstallSpec) -> Self { + self.test_packages.push(package); + self + } +} + +impl ResolvePackageDependenciesBuilder<'_, D, State> +where + D: FindPackageFromProvider, + State: resolve_package_dependencies_builder::State + resolve_package_dependencies_builder::IsComplete, +{ + pub(crate) async fn resolve(self) -> Result { + let args = self._build(); + // TODO(vhyrro): Rewrite to be parallel and no RefCell + let state = Rc::new(RefCell::new(ResolveState::default())); + + for package in args.packages { + resolve_spec( + package, + ResolveStateType::Regular, + Vec::new(), + state.clone(), + args.discover, + args.config, + ) + .await?; + } + + for package in args.build_packages { + resolve_spec( + package, + ResolveStateType::Build, + Vec::new(), + state.clone(), + args.discover, + args.config, + ) + .await?; + } + + for package in args.test_packages { + resolve_spec( + package, + ResolveStateType::Test, + Vec::new(), + state.clone(), + args.discover, + args.config, + ) + .await?; + } + + let mut state = state.borrow_mut(); + Ok(ResolvedArtifacts { + regular: std::mem::take(&mut state.regular), + build: std::mem::take(&mut state.build), + test: std::mem::take(&mut state.test), + }) + } +} + +#[async_recursion(?Send)] +async fn resolve_spec( + spec: PackageInstallSpec, + section: ResolveStateType, + parents: Vec, + state: Rc>, + discover: &D, + config: &Config, +) -> Result { + let FoundPackage { + package, + rockspec, + artifact, + } = find_package_from_provider(&spec, discover, config).await?; + + let package_spec = package.package.clone(); + if parents.contains(package_spec.name()) { + let chain = parents + .iter() + .map(|name| name.to_string()) + .chain(std::iter::once(package_spec.name().to_string())) + .collect::>() + .join(" -> "); + return Err(ResolveError::CyclicDependency(chain)); + } + + let constraint = spec + .constraint + .clone() + .unwrap_or_else(|| spec.package.version_req().clone().into()); + let id = LockedPackageId::new( + package_spec.name(), + package_spec.version(), + spec.pin, + spec.opt, + constraint.clone(), + ); + + if state.borrow().contains(section, &id) { + return Ok(id); + } + + let mut child_parents = parents; + child_parents.push(package_spec.name().clone()); + + let is_binary = matches!(package.source, RemotePackageSource::LuarocksBinaryRock(_)); + + let build_dependencies = if is_binary { + Vec::new() + } else { + let mut ids = Vec::new(); + for dependency in build_dependency_specs(&rockspec, &spec) { + ids.push( + resolve_spec( + dependency, + ResolveStateType::Build, + child_parents.clone(), + state.clone(), + discover, + config, + ) + .await?, + ); + } + ids + }; + + let mut dependencies = Vec::new(); + for dependency in rockspec.dependencies().current_platform() { + let dependency_spec = + PackageInstallSpec::new(dependency.package_req().clone(), EntryType::DependencyOnly) + .build_behaviour(spec.build_behaviour) + .pin(spec.pin) + .opt(spec.opt) + .maybe_source(dependency.source().clone()) + .build(); + dependencies.push( + resolve_spec( + dependency_spec, + section, + child_parents.clone(), + state.clone(), + discover, + config, + ) + .await?, + ); + } + + let locked_spec = LockedPackageSpec::new( + package_spec.name(), + package_spec.version(), + constraint, + dependencies, + build_dependencies, + &spec.pin, + &spec.opt, + rockspec.binaries(), + ); + + let resolved = ResolvedPackage { + spec: locked_spec, + rockspec, + source: package.source, + source_url: package.source_url, + entry_type: spec.entry_type, + artifact, + }; + + state + .borrow_mut() + .select(section) + .insert(id.clone(), resolved); + + Ok(id) +} + +async fn find_package_from_provider( + spec: &PackageInstallSpec, + discover: &D, + _config: &Config, +) -> Result { + if let Some(source) = &spec.source { + let download = RemoteRockDownload::from_package_req_and_source_spec( + spec.package.clone(), + source.clone(), + ) + .map_err(|err| DiscoverError::Download(spec.package.clone(), Box::new(err)))?; + let rockspec = download.rockspec().clone(); + let package = RemotePackage::new( + spec.package.clone().try_into().map_err(|err| { + DiscoverError::Download( + spec.package.clone(), + Box::new(crate::operations::SearchAndDownloadError::from(err)), + ) + })?, + download.rockspec_download().source.clone(), + download.rockspec_download().source_url.clone(), + ); + return Ok(FoundPackage { + package, + rockspec, + artifact: None, + }); + } + + // Run discovery on a separate task so that the deep download/HTTP future stack is not nested + // inside this recursive resolver's call chain. + let discover = discover.clone(); + let package = spec.package.clone(); + match tokio::spawn(async move { discover.find(&package).await }).await?? { + FoundPackageType::Package(package) => Ok(package), + FoundPackageType::Closure { .. } => Err(ResolveError::UnsupportedClosure), + } +} + +fn build_dependency_specs( + rockspec: &RemoteLuaRockspec, + parent: &PackageInstallSpec, +) -> Vec { + build_dependencies_to_install(rockspec) + .into_iter() + .filter_map(|name| { + rockspec + .build_dependencies() + .current_platform() + .iter() + .find(|dep| dep.name() == &name) + .map(|dep| { + PackageInstallSpec::new(dep.package_req().clone(), EntryType::Entrypoint) + .build_behaviour(parent.build_behaviour) + .pin(parent.pin) + .opt(parent.opt) + .maybe_source(dep.source().clone()) + .build() + }) + .or_else(|| { + Some( + PackageInstallSpec::new(PackageReq::from(name), EntryType::Entrypoint) + .build_behaviour(parent.build_behaviour) + .pin(parent.pin) + .opt(parent.opt) + .build(), + ) + }) + }) + .collect() +} diff --git a/lux-lib/src/operations/resolve.rs b/lux-lib/src/operations/resolve.rs index 3bcf3c5bb..4f3e22a77 100644 --- a/lux-lib/src/operations/resolve.rs +++ b/lux-lib/src/operations/resolve.rs @@ -13,7 +13,7 @@ use crate::{ build::BuildBehaviour, config::Config, lockfile::{ - LocalPackageId, LocalPackageSpec, Lockfile, LockfilePermissions, OptState, PinnedState, + LockedPackageId, LockedPackageSpec, Lockfile, LockfilePermissions, }, lua_rockspec::BuildBackendSpec, operations::{FetchVendored, FetchVendoredError}, @@ -56,12 +56,8 @@ impl Display for DependencyCycle { #[derive(Clone, Debug)] pub(crate) struct PackageInstallData { - pub build_behaviour: BuildBehaviour, - pub pin: PinnedState, - pub opt: OptState, pub downloaded_rock: RemoteRockDownload, - pub spec: LocalPackageSpec, - pub entry_type: tree::EntryType, + pub spec: LockedPackageSpec, } #[derive(Builder)] @@ -87,7 +83,7 @@ where { pub(crate) async fn get_all_dependencies( self, - ) -> Result, ResolveDependenciesError> { + ) -> Result, ResolveDependenciesError> { let args = self._build(); do_get_all_dependencies(args).await } @@ -132,7 +128,7 @@ pub(crate) fn luarocks_build_backend_name(rockspec: &R) -> Option

( args: Resolve<'a, P>, -) -> Result, ResolveDependenciesError> +) -> Result, ResolveDependenciesError> where 'a: 'async_recursion, P: LockfilePermissions + Send + Sync + 'static, @@ -168,9 +164,9 @@ where build_behaviour, pin, opt, - entry_type, constraint, source, + .. }| { let config = config.clone(); let dependencies_tx = dependencies_tx.clone(); @@ -344,7 +340,7 @@ where .await?; let rockspec = downloaded_rock.rockspec(); - let local_spec = LocalPackageSpec::new( + let local_spec = LockedPackageSpec::new( rockspec.package(), rockspec.version(), constraint, @@ -356,12 +352,8 @@ where ); let install_spec = PackageInstallData { - build_behaviour, - pin, - opt, spec: local_spec.clone(), downloaded_rock, - entry_type, }; dependencies_tx.send(install_spec).map_err(|err| { diff --git a/lux-lib/src/operations/sync.rs b/lux-lib/src/operations/sync.rs index bfc89c9d4..89a40d690 100644 --- a/lux-lib/src/operations/sync.rs +++ b/lux-lib/src/operations/sync.rs @@ -6,14 +6,15 @@ use crate::{ config::Config, fs, lockfile::{ - FlushLockfileError, LocalPackage, LocalPackageLockType, LockfileIntegrityError, - SyncStrategy, + FlushLockfileError, LockedPackage, LockedPackageLockType, Lockfile, + LockfileIntegrityError, PackageSyncSpec, ReadOnly, ReadWrite, SyncStrategy, + WorkspaceLockfile, }, luarocks::luarocks_installation::LUAROCKS_VERSION, operations::{self, GenLuaRcError}, package::{PackageName, PackageReq}, project::{project_toml::LocalProjectTomlValidationError, ProjectError}, - rockspec::Rockspec, + rockspec::{lua_dependency::LuaDependencySpec, Rockspec}, tree::{self, InstallTree, TreeError}, workspace::{Workspace, WorkspaceError, WorkspaceTreeError}, }; @@ -62,11 +63,11 @@ where pub async fn sync(self) -> Result { let mut args = self._build(); let test_report = if args.test.unwrap_or(false) { - Some(do_sync(&args, &LocalPackageLockType::Test).await?) + Some(do_sync(&args, &LockedPackageLockType::Test).await?) } else { None }; - let mut report = do_sync(&args, &LocalPackageLockType::Regular).await?; + let mut report = do_sync(&args, &LockedPackageLockType::Regular).await?; // `do_sync` removes dependencies that aren't listed in the workspace lockfile // or in `args.extra_packages`. @@ -95,7 +96,7 @@ where .collect_vec(); args.extra_packages.extend(transitive_build_dependencies); - let build_report = do_sync(&args, &LocalPackageLockType::Build).await?; + let build_report = do_sync(&args, &LockedPackageLockType::Build).await?; operations::GenLuaRc::new() .config(args.config) @@ -114,15 +115,15 @@ where #[derive(Debug)] pub struct SyncReport { - pub(crate) added: Vec, - pub(crate) removed: Vec, + pub(crate) added: Vec, + pub(crate) removed: Vec, } impl SyncReport { - pub fn added(&self) -> &[LocalPackage] { + pub fn added(&self) -> &[LockedPackage] { &self.added } - pub fn removed(&self) -> &[LocalPackage] { + pub fn removed(&self) -> &[LockedPackage] { &self.removed } @@ -181,93 +182,23 @@ impl From for SyncError { #[tracing::instrument(name = "Syncing dependencies", skip_all)] async fn do_sync( args: &Sync<'_>, - lock_type: &LocalPackageLockType, + lock_type: &LockedPackageLockType, ) -> Result { // NOTE(vhyrro): tools like cc and pkg-config leak cargo:rerun-if-env-changed // stdout calls, therefore gag all standard output during sync. let _stdout_gag = gag::Gag::stdout(); let tree = match lock_type { - LocalPackageLockType::Regular => args.workspace.tree(args.config)?, - LocalPackageLockType::Test => args.workspace.test_tree(args.config)?, - LocalPackageLockType::Build => args.workspace.build_tree(args.config)?, + LockedPackageLockType::Regular => args.workspace.tree(args.config)?, + LockedPackageLockType::Test => args.workspace.test_tree(args.config)?, + LockedPackageLockType::Build => args.workspace.build_tree(args.config)?, }; fs::sync::create_dir_all(tree.root())?; let mut workspace_lockfile = args.workspace.lockfile()?.write_guard(); let dest_lockfile = tree.lockfile()?; - let mut packages = Vec::new(); - for project in args.workspace.members() { - match lock_type { - LocalPackageLockType::Regular => packages.extend( - project - .toml() - .into_local()? - .dependencies() - .current_platform() - .clone(), - ), - LocalPackageLockType::Build => packages.extend( - project - .toml() - .into_local()? - .build_dependencies() - .current_platform() - .clone(), - ), - LocalPackageLockType::Test => packages.extend( - project - .toml() - .into_local()? - .test_dependencies() - .current_platform() - .clone(), - ), - } - } - - let mut extra_packages = args.extra_packages.iter().cloned().collect_vec(); - if lock_type == &LocalPackageLockType::Build { - for project in args.workspace.members() { - let toml = project.toml().into_local()?; - if let Some(backend) = operations::resolve::luarocks_build_backend_name(&toml) { - extra_packages.push(backend.into()); - if cfg!(target_family = "unix") { - let luarocks = unsafe { - PackageReq::new_unchecked("luarocks".into(), Some(LUAROCKS_VERSION.into())) - }; - extra_packages.push(luarocks); - } - } - } - } else if lock_type == &LocalPackageLockType::Test { - for project in args.workspace.members() { - let toml = project.toml().into_local()?; - for test_dep in toml - .test() - .current_platform() - .test_dependencies(project) - .iter() - .filter(|test_dep| { - !toml - .test_dependencies() - .current_platform() - .iter() - .any(|dep| dep.name() == test_dep.name()) - }) - .cloned() - { - extra_packages.push(test_dep); - } - } - } - - let packages = packages - .into_iter() - .chain(extra_packages.into_iter().unique().map_into()) - .collect_vec(); - + let packages = SyncPackages::new(args, lock_type).gather()?; let strategy = if args.fast.unwrap_or(false) { SyncStrategy::LockfileOnly } else { @@ -280,12 +211,138 @@ async fn do_sync( .iter() .for_each(|pkg| workspace_lockfile.remove(pkg, lock_type)); - let mut to_add: Vec<(tree::EntryType, LocalPackage)> = Vec::new(); + let package_db = workspace_lockfile.local_pkg_locks().into(); + + let (to_add, mut report) = reconcile_locks(&*workspace_lockfile, &dest_lockfile, lock_type); + let packages_to_install = install_specs_to_force(&to_add); + report + .added + .extend(to_add.iter().map(|(_, pkg)| pkg).cloned()); + + Install::new(args.config) + .package_db(package_db) + .packages(packages_to_install) + .tree(tree.clone()) + .install() + .await?; + + // Read the destination lockfile after installing + let install_tree_lockfile = tree.lockfile()?; + + validate_integrity(args, &to_add, &install_tree_lockfile)?; + + let packages_to_remove = report.removed.iter().map(|pkg| pkg.id()).collect_vec(); + + Uninstall::new() + .config(args.config) + .packages(packages_to_remove) + .tree(tree.clone()) + .remove() + .await?; + + install_tree_lockfile.map_then_flush(|lockfile| { + lockfile.sync(workspace_lockfile.local_pkg_lock(lock_type)); + Ok::<_, io::Error>(()) + })?; + + install_missing_packages(args, &tree, &mut workspace_lockfile, lock_type, &package_sync_spec, &mut report) + .await?; + + Ok(report) +} + +/// Collects the packages which must be synchronised for a given lockfile type: +/// workspace member dependencies plus any extra build backends or test dependencies. +struct SyncPackages<'a, 'b> { + args: &'a Sync<'b>, + lock_type: &'a LockedPackageLockType, +} + +impl<'a, 'b> SyncPackages<'a, 'b> { + fn new(args: &'a Sync<'b>, lock_type: &'a LockedPackageLockType) -> Self { + Self { args, lock_type } + } + + fn gather(self) -> Result, SyncError> { + let mut packages = Vec::new(); + for project in self.args.workspace.members() { + let toml = project.toml().into_local()?; + match self.lock_type { + LockedPackageLockType::Regular => { + packages.extend(toml.dependencies().current_platform().clone()) + } + LockedPackageLockType::Build => { + packages.extend(toml.build_dependencies().current_platform().clone()) + } + LockedPackageLockType::Test => { + packages.extend(toml.test_dependencies().current_platform().clone()) + } + } + } + + let mut extra_packages = self.args.extra_packages.iter().cloned().collect_vec(); + match self.lock_type { + LockedPackageLockType::Build => { + for project in self.args.workspace.members() { + let toml = project.toml().into_local()?; + if let Some(backend) = operations::resolve::luarocks_build_backend_name(&toml) { + extra_packages.push(backend.into()); + if cfg!(target_family = "unix") { + let luarocks = unsafe { + PackageReq::new_unchecked( + "luarocks".into(), + Some(LUAROCKS_VERSION.into()), + ) + }; + extra_packages.push(luarocks); + } + } + } + } + LockedPackageLockType::Test => { + for project in self.args.workspace.members() { + let toml = project.toml().into_local()?; + for test_dep in toml + .test() + .current_platform() + .test_dependencies(project) + .iter() + .filter(|test_dep| { + !toml + .test_dependencies() + .current_platform() + .iter() + .any(|dep| dep.name() == test_dep.name()) + }) + .cloned() + { + extra_packages.push(test_dep); + } + } + } + LockedPackageLockType::Regular => {} + } + + Ok(packages + .into_iter() + .chain(extra_packages.into_iter().unique().map_into()) + .collect()) + } +} +/// Determines which packages are present in one lockfile but not the other, producing the +/// packages to add (with their entry type) and the report of packages that must be removed. +fn reconcile_locks( + workspace_lockfile: &WorkspaceLockfile, + dest_lockfile: &Lockfile, + lock_type: &LockedPackageLockType, +) -> (Vec<(tree::EntryType, LockedPackage)>, SyncReport) { + let mut to_add: Vec<(tree::EntryType, LockedPackage)> = Vec::new(); let mut report = SyncReport { added: Vec::new(), removed: Vec::new(), }; + for (id, local_package) in workspace_lockfile.rocks(lock_type) { if dest_lockfile.get(id).is_none() { let entry_type = if workspace_lockfile.is_entrypoint(&local_package.id(), lock_type) { @@ -302,7 +359,11 @@ async fn do_sync( } } - let packages_to_install = to_add + (to_add, report) +} + +fn install_specs_to_force(to_add: &[(tree::EntryType, LockedPackage)]) -> Vec { + to_add .iter() .map(|(entry_type, pkg)| { PackageInstallSpec::new(pkg.clone().into_package_req(), *entry_type) @@ -313,85 +374,79 @@ async fn do_sync( .build() }) .unique() - .collect_vec(); - report - .added - .extend(to_add.iter().map(|(_, pkg)| pkg).cloned()); - - let package_db = workspace_lockfile.local_pkg_locks().into(); - - Install::new(args.config) - .package_db(package_db) - .packages(packages_to_install) - .tree(tree.clone()) - .install() - .await?; + .collect() +} - // Read the destination lockfile after installing - let install_tree_lockfile = tree.lockfile()?; +fn validate_integrity( + args: &Sync<'_>, + to_add: &[(tree::EntryType, LockedPackage)], + install_tree_lockfile: &Lockfile, +) -> Result<(), SyncError> { + if !args.validate_integrity.unwrap_or(true) { + return Ok(()); + } + for (_, package) in to_add { + install_tree_lockfile + .validate_integrity(package) + .map_err(|source| SyncError::Integrity { + package: package.name().clone(), + source, + })?; + } + Ok(()) +} - if args.validate_integrity.unwrap_or(true) { - for (_, package) in &to_add { - install_tree_lockfile - .validate_integrity(package) - .map_err(|source| SyncError::Integrity { - package: package.name().clone(), - source, - })?; - } +/// Installs packages that were newly added to the workspace lockfile but are not yet present in +/// the install tree, using the default package database. +async fn install_missing_packages( + args: &Sync<'_>, + tree: &T, + workspace_lockfile: &mut WorkspaceLockfile, + lock_type: &LockedPackageLockType, + package_sync_spec: &PackageSyncSpec, + report: &mut SyncReport, +) -> Result<(), SyncError> +where + T: InstallTree + Clone + Send + std::marker::Sync + 'static, +{ + if package_sync_spec.to_add.is_empty() { + return Ok(()); } - let packages_to_remove = report.removed.iter().map(|pkg| pkg.id()).collect_vec(); + let missing_packages = package_sync_spec + .to_add + .iter() + .map(|dep| { + PackageInstallSpec::new(dep.package_req().clone(), tree::EntryType::Entrypoint) + .build_behaviour(BuildBehaviour::Force) + .pin(*dep.pin()) + .opt(*dep.opt()) + .maybe_source(dep.source.clone()) + .build() + }) + .unique() + .collect(); - Uninstall::new() - .config(args.config) - .packages(packages_to_remove) + let added = Install::new(args.config) + .packages(missing_packages) .tree(tree.clone()) - .remove() + .install() .await?; - install_tree_lockfile.map_then_flush(|lockfile| { - lockfile.sync(workspace_lockfile.local_pkg_lock(lock_type)); - Ok::<_, io::Error>(()) - })?; + report.added.extend(added); - if !package_sync_spec.to_add.is_empty() { - // Install missing packages using the default package_db. - let missing_packages = package_sync_spec - .to_add - .into_iter() - .map(|dep| { - PackageInstallSpec::new(dep.package_req().clone(), tree::EntryType::Entrypoint) - .build_behaviour(BuildBehaviour::Force) - .pin(*dep.pin()) - .opt(*dep.opt()) - .maybe_source(dep.source.clone()) - .build() - }) - .unique() - .collect(); - - let added = Install::new(args.config) - .packages(missing_packages) - .tree(tree.clone()) - .install() - .await?; - - report.added.extend(added); - - // Sync the newly added packages back to the workspace lockfile - let dest_lockfile = tree.lockfile()?; - workspace_lockfile.sync(dest_lockfile.local_pkg_lock(), lock_type); - } + // Sync the newly added packages back to the workspace lockfile + let dest_lockfile = tree.lockfile()?; + workspace_lockfile.sync(dest_lockfile.local_pkg_lock(), lock_type); - Ok(report) + Ok(()) } #[cfg(all(test, feature = "impure_tests"))] mod tests { use super::Sync; use crate::{ - config::ConfigBuilder, lockfile::LocalPackageLockType, package::PackageReq, + config::ConfigBuilder, lockfile::LockedPackageLockType, package::PackageReq, workspace::Workspace, }; use assert_fs::{prelude::PathCopy, TempDir}; @@ -416,7 +471,7 @@ mod tests { let lockfile_after_sync = workspace.lockfile().unwrap(); assert!(!lockfile_after_sync - .rocks(&LocalPackageLockType::Regular) + .rocks(&LockedPackageLockType::Regular) .is_empty()); } @@ -448,7 +503,7 @@ mod tests { } let lockfile_after_sync = workspace.lockfile().unwrap(); assert!(!lockfile_after_sync - .rocks(&LocalPackageLockType::Regular) + .rocks(&LockedPackageLockType::Regular) .is_empty()); } @@ -481,7 +536,7 @@ mod tests { } let lockfile_after_sync = workspace.lockfile().unwrap(); assert!(!lockfile_after_sync - .rocks(&LocalPackageLockType::Regular) + .rocks(&LockedPackageLockType::Regular) .is_empty()); } @@ -509,7 +564,7 @@ mod tests { let lockfile_after_sync = workspace.lockfile().unwrap(); assert!(!lockfile_after_sync - .rocks(&LocalPackageLockType::Regular) + .rocks(&LockedPackageLockType::Regular) .is_empty()); } } diff --git a/lux-lib/src/operations/test.rs b/lux-lib/src/operations/test.rs index f3fe0b28b..5013f0cb9 100644 --- a/lux-lib/src/operations/test.rs +++ b/lux-lib/src/operations/test.rs @@ -158,10 +158,17 @@ async fn run_tests(test: Test<'_>) -> Result<(), RunTestsError> { if let Some(package) = test.package { let project = workspace.select_member(&package)?; - run_project_tests(&workspace, project, no_lock, &test.args, &test.env, config).await + // NOTE: `run_project_tests`' future is large, so we box it to avoid inflating this frame. + Box::pin(run_project_tests( + &workspace, project, no_lock, &test.args, &test.env, config, + )) + .await } else { for project in workspace.members() { - run_project_tests(&workspace, project, no_lock, &test.args, &test.env, config).await?; + Box::pin(run_project_tests( + &workspace, project, no_lock, &test.args, &test.env, config, + )) + .await?; } Ok(()) } @@ -186,12 +193,15 @@ async fn run_project_tests( Sync::new(workspace, &test_config).test(true).sync().await?; } - BuildWorkspace::new(workspace, &test_config) - .package(project.toml().package().clone()) - .no_lock(no_lock) - .only_deps(false) - .build() - .await?; + // NOTE: `BuildWorkspace::build`'s future is large, so we box it to avoid inflating this frame. + Box::pin( + BuildWorkspace::new(workspace, &test_config) + .package(project.toml().package().clone()) + .no_lock(no_lock) + .only_deps(false) + .build(), + ) + .await?; let lua_version = project.lua_version(&test_config)?; let project_tree = workspace.lua_version_tree(lua_version, &test_config)?; diff --git a/lux-lib/src/operations/uninstall.rs b/lux-lib/src/operations/uninstall.rs index 0920c9ce1..8a9275633 100644 --- a/lux-lib/src/operations/uninstall.rs +++ b/lux-lib/src/operations/uninstall.rs @@ -1,7 +1,7 @@ use std::io; use crate::fs; -use crate::lockfile::{FlushLockfileError, LocalPackage, LocalPackageId}; +use crate::lockfile::{FlushLockfileError, LockedPackage, LockedPackageId}; use crate::lua_version::{LuaVersion, LuaVersionUnset}; use crate::tree::{EntryType, InstallTree, TreeError}; use crate::{config::Config, tree::Tree}; @@ -31,7 +31,7 @@ pub enum RemoveError { #[builder(start_fn = new, finish_fn(name = _build, vis = ""))] pub struct Uninstall<'a> { #[builder(field)] - packages: Vec, + packages: Vec, config: &'a Config, tree: Option, } @@ -43,7 +43,7 @@ where /// Add packages to remove. pub fn packages(self, packages: I) -> Self where - I: IntoIterator, + I: IntoIterator, { Self { packages: self.packages.into_iter().chain(packages).collect_vec(), @@ -52,7 +52,7 @@ where } /// Add a package to the set of packages to remove. - pub fn package(self, package: LocalPackageId) -> Self { + pub fn package(self, package: LockedPackageId) -> Self { self.packages(std::iter::once(package)) } } @@ -62,7 +62,7 @@ where State: uninstall_builder::State + uninstall_builder::IsComplete, { /// Remove the packages. - pub async fn remove(self) -> Result, RemoveError> { + pub async fn remove(self) -> Result, RemoveError> { let args = self._build(); let tree = args.tree.unwrap_or( args.config @@ -74,10 +74,10 @@ where // TODO: Remove dependencies recursively too! async fn remove( - package_ids: Vec, + package_ids: Vec, tree: Tree, config: &Config, -) -> Result, RemoveError> { +) -> Result, RemoveError> { let lockfile = tree.lockfile()?; let packages = package_ids @@ -121,7 +121,7 @@ async fn remove( ), )] async fn remove_package( - package: LocalPackage, + package: LockedPackage, tree: Tree, entry_type: EntryType, ) -> Result<(), RemoveError> { diff --git a/lux-lib/src/operations/update.rs b/lux-lib/src/operations/update.rs index 276b0a118..ddf29a45b 100644 --- a/lux-lib/src/operations/update.rs +++ b/lux-lib/src/operations/update.rs @@ -3,7 +3,7 @@ use std::io; use crate::{ config::Config, lockfile::{ - LocalPackage, LocalPackageLockType, Lockfile, PinnedState, ReadOnly, ReadWrite, + LockedPackage, LockedPackageLockType, Lockfile, PinnedState, ReadOnly, ReadWrite, WorkspaceLockfile, }, lua_version::{LuaVersion, LuaVersionUnset}, @@ -18,7 +18,10 @@ use itertools::Itertools; use miette::Diagnostic; use thiserror::Error; -use super::{Install, InstallError, PackageInstallSpec, RemoveError, SyncError, Uninstall}; +use super::{ + InstallError, PackageInstallSpec, RemoveError, SyncError, Uninstall, + pipeline::install_packages::InstallPackages, +}; #[derive(Error, Debug, Diagnostic)] pub enum UpdateError { @@ -115,7 +118,7 @@ impl UpdateBuilder<'_, State> { #[tracing::instrument(name = "Updating packages", skip_all)] /// Returns the packages that were installed or removed - pub async fn update(self) -> Result, UpdateError> + pub async fn update(self) -> Result, UpdateError> where State: update_builder::IsComplete, { @@ -153,7 +156,7 @@ async fn update_workspace( workspace: Workspace, args: Update<'_>, package_db: RemotePackageDB, -) -> Result, UpdateError> { +) -> Result, UpdateError> { let mut project_lockfile = workspace.lockfile()?.write_guard(); let tree = workspace.tree(args.config)?; @@ -166,7 +169,7 @@ async fn update_workspace( let updated_dependencies = update_dependency_tree( tree, &mut project_lockfile, - LocalPackageLockType::Regular, + LockedPackageLockType::Regular, package_db.clone(), args.config, &args.packages, @@ -178,7 +181,7 @@ async fn update_workspace( let updated_test_dependencies = update_dependency_tree( test_tree, &mut project_lockfile, - LocalPackageLockType::Test, + LockedPackageLockType::Test, package_db.clone(), args.config, &args.test_dependencies, @@ -190,7 +193,7 @@ async fn update_workspace( let updated_build_dependencies = update_dependency_tree( build_tree, &mut project_lockfile, - LocalPackageLockType::Build, + LockedPackageLockType::Build, package_db.clone(), args.config, &args.build_dependencies, @@ -209,11 +212,11 @@ async fn update_workspace( async fn update_dependency_tree( tree: Tree, project_lockfile: &mut WorkspaceLockfile, - lock_type: LocalPackageLockType, + lock_type: LockedPackageLockType, package_db: RemotePackageDB, config: &Config, packages: &Option>, -) -> Result, UpdateError> { +) -> Result, UpdateError> { let lockfile = tree.lockfile()?; let dependencies = updatable_packages(&lockfile) .into_iter() @@ -228,7 +231,7 @@ async fn update_dependency_tree( } fn is_included( - (pkg, _): &(LocalPackage, PackageReq), + (pkg, _): &(LockedPackage, PackageReq), package_reqs: &Option>, ) -> bool { package_reqs.is_none() @@ -242,7 +245,7 @@ fn is_included( async fn update_install_tree( args: Update<'_>, package_db: RemotePackageDB, -) -> Result, UpdateError> { +) -> Result, UpdateError> { let tree = args .config .user_tree(LuaVersion::from(args.config)?.clone())?; @@ -255,12 +258,12 @@ async fn update_install_tree( } async fn update( - packages: Vec<(LocalPackage, PackageReq)>, + packages: Vec<(LockedPackage, PackageReq)>, package_db: RemotePackageDB, tree: Tree, lockfile: &Lockfile, config: &Config, -) -> Result, UpdateError> { +) -> Result, UpdateError> { let updatable = packages .clone() .into_iter() @@ -284,22 +287,23 @@ async fn update( .packages(updatable.iter().map(|(package, _)| package.id())) .remove() .await?; - let updated_packages = Install::new(config) + let updated_packages = InstallPackages::new(config, tree) .packages( updatable .iter() .map(|updatable| mk_install_spec(updatable, lockfile)) .collect(), ) - .tree(tree) - .package_db(package_db) + .maybe_package_db(Some(package_db)) .install() - .await?; + .await + .map_err(InstallError::from) + .map_err(UpdateError::from)?; Ok(updated_packages) } } -fn updatable_packages(lockfile: &Lockfile) -> Vec<(LocalPackage, PackageReq)> { +fn updatable_packages(lockfile: &Lockfile) -> Vec<(LockedPackage, PackageReq)> { lockfile .rocks() .values() @@ -322,7 +326,7 @@ fn updatable_packages(lockfile: &Lockfile) -> Vec<(LocalPackage, Packa } fn mk_install_spec( - (package, req): &(LocalPackage, PackageReq), + (package, req): &(LockedPackage, PackageReq), lockfile: &Lockfile, ) -> PackageInstallSpec { let entry_type = if lockfile.is_entrypoint(&package.id()) { diff --git a/lux-lib/src/operations/vendor.rs b/lux-lib/src/operations/vendor.rs index c5d9363c1..17510580b 100644 --- a/lux-lib/src/operations/vendor.rs +++ b/lux-lib/src/operations/vendor.rs @@ -20,7 +20,7 @@ use crate::{ build::{resolve_source_dir, RemotePackageSourceSpec, SrcRockSource}, config::Config, fs, - lockfile::{LocalPackageLockType, ReadOnly, RemotePackageSourceUrl}, + lockfile::{LockedPackageLockType, ReadOnly, RemotePackageSourceUrl}, lua_rockspec::{BuildBackendSpec, RemoteLuaRockspec}, operations::{ self, @@ -133,7 +133,7 @@ async fn do_vendor_dependencies(args: Vendor<'_>) -> Result<(), VendorError> { let config = args.config; let mut all_packages = Vec::new(); - for lock_type in LocalPackageLockType::iter() { + for lock_type in LockedPackageLockType::iter() { let (package_db, install_specs) = mk_resolve_args(lock_type, no_lock, &target, config).await?; @@ -188,7 +188,7 @@ async fn do_vendor_dependencies(args: Vendor<'_>) -> Result<(), VendorError> { } async fn mk_resolve_args( - lock_type: LocalPackageLockType, + lock_type: LockedPackageLockType, no_lock: bool, target: &VendorTarget, config: &Config, @@ -207,7 +207,7 @@ async fn mk_resolve_args( for project in workspace.members() { let toml = project.toml().into_local()?; push_dependencies(&lock_type, &toml, &mut install_specs)?; - if lock_type == LocalPackageLockType::Test { + if lock_type == LockedPackageLockType::Test { for test_spec_dependency in toml .test() .current_platform() @@ -232,31 +232,31 @@ async fn mk_resolve_args( } fn push_dependencies( - lock_type: &LocalPackageLockType, + lock_type: &LockedPackageLockType, rockspec: &R, install_specs: &mut Vec, ) -> Result<(), LocalProjectTomlValidationError> { let mut dependencies: Vec = match lock_type { - LocalPackageLockType::Regular => rockspec + LockedPackageLockType::Regular => rockspec .dependencies() .current_platform() .iter() .map(|dep| dep.package_req().clone()) .collect_vec(), - LocalPackageLockType::Test => rockspec + LockedPackageLockType::Test => rockspec .test_dependencies() .current_platform() .iter() .map(|dep| dep.package_req().clone()) .collect_vec(), - LocalPackageLockType::Build => rockspec + LockedPackageLockType::Build => rockspec .build_dependencies() .current_platform() .iter() .map(|dep| dep.package_req().clone()) .collect_vec(), }; - if *lock_type == LocalPackageLockType::Build { + if *lock_type == LockedPackageLockType::Build { if let Some(backend) = luarocks_build_backend_name(rockspec) { dependencies.insert(0, backend.into()); } diff --git a/lux-lib/src/remote_package_db/mod.rs b/lux-lib/src/remote_package_db/mod.rs index bc45a6fbf..c365b0da7 100644 --- a/lux-lib/src/remote_package_db/mod.rs +++ b/lux-lib/src/remote_package_db/mod.rs @@ -1,6 +1,6 @@ use crate::{ config::{Config, ConfigError}, - lockfile::{LocalPackageLock, LockfileIntegrityError}, + lockfile::{LockedPackageLock, LockfileIntegrityError}, manifest::{Manifest, ManifestError}, package::{ PackageName, PackageReq, PackageSpec, PackageVersion, RemotePackage, @@ -19,7 +19,7 @@ pub struct RemotePackageDB(Impl); #[derive(Clone, Debug)] enum Impl { LuarocksManifests(Vec), - LocalPackageLocks(Vec), + LockedPackageLocks(Vec), } #[derive(Error, Debug, Diagnostic)] @@ -90,7 +90,7 @@ impl RemotePackageDB { None => Err(SearchError::RockNotFound(package_req.clone())), } } - Impl::LocalPackageLocks(locks) => { + Impl::LockedPackageLocks(locks) => { match locks .iter() .filter_map(|lock| lock.has_rock(package_req, filter.clone())) @@ -138,7 +138,7 @@ impl RemotePackageDB { }) }) .collect(), - Impl::LocalPackageLocks(locks) => locks + Impl::LockedPackageLocks(locks) => locks .iter() .flat_map(|lock| lock.rocks().values()) .filter_map(|package| { @@ -180,8 +180,8 @@ impl From for RemotePackageDB { } } -impl From> for RemotePackageDB { - fn from(locks: Vec) -> Self { - Self(Impl::LocalPackageLocks(locks)) +impl From> for RemotePackageDB { + fn from(locks: Vec) -> Self { + Self(Impl::LockedPackageLocks(locks)) } } diff --git a/lux-lib/src/tree/dist.rs b/lux-lib/src/tree/dist.rs index c193b2e36..39419bc1c 100644 --- a/lux-lib/src/tree/dist.rs +++ b/lux-lib/src/tree/dist.rs @@ -4,7 +4,7 @@ use super::{EntryType, InstallTree, RockLayout, Tree, TreeError}; use crate::{ config::Config, fs, - lockfile::{LocalPackage, Lockfile, ReadOnly}, + lockfile::{LockedPackage, Lockfile, ReadOnly}, lua_version::LuaVersion, package::{PackageName, PackageVersion}, }; @@ -42,7 +42,7 @@ impl FlatDistTree { Ok(Self(tree)) } - fn guard_no_conflicting_package(&self, package: &LocalPackage) -> Result<(), io::Error> { + fn guard_no_conflicting_package(&self, package: &LockedPackage) -> Result<(), io::Error> { let lockfile = self.lockfile().map_err(io::Error::other)?; match lockfile.has_rock(&package.clone().into_package_req(), None) { Some(existing_package) => { @@ -91,7 +91,7 @@ impl InstallTree for FlatDistTree { self.0.root() } - fn layout_for(&self, _package: &LocalPackage) -> RockLayout { + fn layout_for(&self, _package: &LockedPackage) -> RockLayout { RockLayout::new(self.0.root(), self.0.bin()) } @@ -103,7 +103,7 @@ impl InstallTree for FlatDistTree { self.0.unwrapped_bin() } - fn prepare(&self, package: &LocalPackage) -> Result<(), TreeError> { + fn prepare(&self, package: &LockedPackage) -> Result<(), TreeError> { self.guard_no_conflicting_package(package) .map_err(TreeError::Io)?; let layout = self.layout_for(package); @@ -113,7 +113,7 @@ impl InstallTree for FlatDistTree { Ok(()) } - fn cleanup(&self, _package: &LocalPackage, _entry_type: EntryType) -> Result<(), TreeError> { + fn cleanup(&self, _package: &LockedPackage, _entry_type: EntryType) -> Result<(), TreeError> { unreachable!("dist trees do not undergo cleanup for packages") } @@ -133,7 +133,7 @@ impl InstallTree for FlatDistTree { self.0.test_tree(config) } - fn list(&self) -> Result>, TreeError> { + fn list(&self) -> Result>, TreeError> { self.0.list() } diff --git a/lux-lib/src/tree/layout.rs b/lux-lib/src/tree/layout.rs index 5e83bcfb9..0b5fbffcc 100644 --- a/lux-lib/src/tree/layout.rs +++ b/lux-lib/src/tree/layout.rs @@ -1,6 +1,6 @@ use crate::{ fs::{self, FsError}, - lockfile::{LocalPackage, OptState}, + lockfile::{LockedPackage, OptState}, tree::{InstallTree, Tree}, }; use std::path::{Path, PathBuf}; @@ -8,9 +8,9 @@ use std::path::{Path, PathBuf}; /// A custom layout for entrypoint packages. /// Implementations arrange symlinks so external tools can find files at the locations they expect. pub trait CustomRockLayout: std::fmt::Debug + Send + Sync { - fn make_symlinks(&self, tree: &Tree, package: &LocalPackage) -> fs::Result<()>; + fn make_symlinks(&self, tree: &Tree, package: &LockedPackage) -> fs::Result<()>; - fn remove_symlinks(&self, tree: &Tree, package: &LocalPackage) -> fs::Result<()>; + fn remove_symlinks(&self, tree: &Tree, package: &LockedPackage) -> fs::Result<()>; } /// A [`CustomRockLayout`] for Neovim plugins. Packages are exposed under `/site/pack/lux/{start,opt}/` @@ -18,7 +18,7 @@ pub trait CustomRockLayout: std::fmt::Debug + Send + Sync { pub struct NvimLayout; impl NvimLayout { - fn target_for(tree: &Tree, package: &LocalPackage) -> PathBuf { + fn target_for(tree: &Tree, package: &LockedPackage) -> PathBuf { let subdir = match package.spec.opt { OptState::Required => "start", OptState::Optional => "opt", @@ -31,7 +31,7 @@ impl NvimLayout { } impl CustomRockLayout for NvimLayout { - fn make_symlinks(&self, tree: &Tree, package: &LocalPackage) -> fs::Result<()> { + fn make_symlinks(&self, tree: &Tree, package: &LockedPackage) -> fs::Result<()> { let custom_dir = Self::target_for(tree, package); fs::sync::create_dir_all(&custom_dir)?; @@ -49,7 +49,7 @@ impl CustomRockLayout for NvimLayout { Ok(()) } - fn remove_symlinks(&self, tree: &Tree, package: &LocalPackage) -> fs::Result<()> { + fn remove_symlinks(&self, tree: &Tree, package: &LockedPackage) -> fs::Result<()> { let target = Self::target_for(tree, package); if target.is_dir() { // SAFETY: does not follow symlinks, only removes them @@ -93,7 +93,7 @@ mod tests { use crate::{ config::ConfigBuilder, - lockfile::{LocalPackage, LocalPackageHashes, LockConstraint}, + lockfile::{LockConstraint, LockedPackage, LockedPackageHashes}, lua_version::LuaVersion, package::PackageSpec, remote_package_source::RemotePackageSource, @@ -101,8 +101,8 @@ mod tests { tree::{EntryType, InstallTree, NvimLayout}, }; - fn mock_hashes() -> LocalPackageHashes { - LocalPackageHashes { + fn mock_hashes() -> LockedPackageHashes { + LockedPackageHashes { rockspec: "sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=" .parse() .unwrap(), @@ -128,8 +128,8 @@ mod tests { (temp, tree_path, tree) } - fn sample_package() -> LocalPackage { - LocalPackage::from( + fn sample_package() -> LockedPackage { + LockedPackage::from( &PackageSpec::parse("neorg".into(), "8.0.0-1".into()).unwrap(), LockConstraint::Unconstrained, RockBinaries::default(), diff --git a/lux-lib/src/tree/list.rs b/lux-lib/src/tree/list.rs index 68f0ed70d..da0790ea0 100644 --- a/lux-lib/src/tree/list.rs +++ b/lux-lib/src/tree/list.rs @@ -1,4 +1,4 @@ -use crate::{lockfile::LocalPackage, tree::InstallTree}; +use crate::{lockfile::LockedPackage, tree::InstallTree}; use super::{Tree, TreeError}; @@ -8,7 +8,7 @@ use super::{Tree, TreeError}; // Cloning isn't destructive, but it's sure expensive. impl Tree { - pub fn as_rock_list(&self) -> Result, TreeError> { + pub fn as_rock_list(&self) -> Result, TreeError> { let rock_list = self.list()?; Ok(rock_list.values().flatten().cloned().collect()) diff --git a/lux-lib/src/tree/mod.rs b/lux-lib/src/tree/mod.rs index 9cb42df6b..c10fd31ae 100644 --- a/lux-lib/src/tree/mod.rs +++ b/lux-lib/src/tree/mod.rs @@ -2,7 +2,7 @@ use crate::{ build::utils::format_path, config::Config, fs, - lockfile::{LocalPackage, LocalPackageId, Lockfile, LockfileError, ReadOnly}, + lockfile::{LockedPackage, LockedPackageId, Lockfile, LockfileError, ReadOnly}, lua_version::LuaVersion, package::{PackageName, PackageReq}, variables::{GetVariableError, HasVariables}, @@ -42,7 +42,7 @@ pub trait InstallTree { /// Where unwrapped package binaries are installed fn unwrapped_bin(&self) -> PathBuf; /// The standard install layout for a package. - fn layout_for(&self, package: &LocalPackage) -> RockLayout; + fn layout_for(&self, package: &LockedPackage) -> RockLayout; /// Create a [`Lockfile`] for this tree. fn lockfile(&self) -> Result, TreeError>; /// Get this tree's lockfile path. @@ -52,22 +52,22 @@ pub trait InstallTree { /// The tree in which to install test dependencies. fn test_tree(&self, config: &Config) -> Result; /// List the packages that are installed in this tree. - fn list(&self) -> Result>, TreeError>; + fn list(&self) -> Result>, TreeError>; /// Find installed rocks that match the given [`PackageReq`]. fn match_rocks(&self, req: &PackageReq) -> Result; /// Create the standard directories (src, lib, etc.) for a package. - fn prepare(&self, package: &LocalPackage) -> Result<(), TreeError>; + fn prepare(&self, package: &LockedPackage) -> Result<(), TreeError>; /// Apply the custom layout once a package has been fully installed. /// /// For entrypoints, this creates the symlinks for the custom layout, if one is configured. /// Must be called after the package's files have been written to the standard layout. - fn finalize(&self, _package: &LocalPackage, _entry_type: EntryType) -> Result<(), TreeError> { + fn finalize(&self, _package: &LockedPackage, _entry_type: EntryType) -> Result<(), TreeError> { Ok(()) } /// Remove the install layout directories. /// /// For entrypoints, this also applies the custom layout, if one is configured. - fn cleanup(&self, package: &LocalPackage, entry_type: EntryType) -> Result<(), TreeError>; + fn cleanup(&self, package: &LockedPackage, entry_type: EntryType) -> Result<(), TreeError>; } /// The standard install layout for a rock. @@ -128,7 +128,7 @@ impl RockLayout { } /// A Lux install tree that supports multiple versions of the same dependency, -/// with packages addressed by their [`LocalPackageId`] +/// with packages addressed by their [`LockedPackageId`] #[derive(Clone, Debug)] pub struct Tree { /// The Lua version of the tree. @@ -214,7 +214,7 @@ impl Tree { pub fn match_rocks_and(&self, req: &PackageReq, filter: F) -> Result where - F: Fn(&LocalPackage) -> bool, + F: Fn(&LockedPackage) -> bool, { match self.list()?.get(req.name()) { Some(packages) => { @@ -252,7 +252,7 @@ impl InstallTree for Tree { self.root_parent.join(self.version.to_string()) } - fn prepare(&self, package: &LocalPackage) -> Result<(), TreeError> { + fn prepare(&self, package: &LockedPackage) -> Result<(), TreeError> { let layout = self.layout_for(package); fs::sync::create_dir_all(&layout.root)?; fs::sync::create_dir_all(&layout.lib)?; @@ -262,7 +262,7 @@ impl InstallTree for Tree { Ok(()) } - fn finalize(&self, package: &LocalPackage, entry_type: EntryType) -> Result<(), TreeError> { + fn finalize(&self, package: &LockedPackage, entry_type: EntryType) -> Result<(), TreeError> { if entry_type.is_entrypoint() { if let Some(custom_layout) = &self.entrypoint_layout { custom_layout.make_symlinks(self, package)?; @@ -272,7 +272,7 @@ impl InstallTree for Tree { Ok(()) } - fn cleanup(&self, package: &LocalPackage, entry_type: EntryType) -> Result<(), TreeError> { + fn cleanup(&self, package: &LockedPackage, entry_type: EntryType) -> Result<(), TreeError> { if entry_type.is_entrypoint() { if let Some(layout) = &self.entrypoint_layout { layout.remove_symlinks(self, package)?; @@ -308,7 +308,7 @@ impl InstallTree for Tree { self.root().join(LOCKFILE_NAME) } - fn layout_for(&self, package: &LocalPackage) -> RockLayout { + fn layout_for(&self, package: &LockedPackage) -> RockLayout { RockLayout::new( self.root().join(format!( "{}-{}@{}", @@ -352,7 +352,7 @@ impl InstallTree for Tree { ) } - fn list(&self) -> Result>, TreeError> { + fn list(&self) -> Result>, TreeError> { Ok(self.lockfile()?.list()) } @@ -386,8 +386,8 @@ impl EntryType { #[derive(Clone, Debug)] pub enum RockMatches { NotFound(PackageReq), - Single(LocalPackageId), - Many(NonEmpty), + Single(LockedPackageId), + Many(NonEmpty), } // Loosely mimic the Option functions. @@ -407,7 +407,7 @@ mod tests { use crate::{ config::ConfigBuilder, - lockfile::{LocalPackage, LocalPackageHashes, LockConstraint}, + lockfile::{LockConstraint, LockedPackage, LockedPackageHashes}, lua_version::LuaVersion, package::{PackageName, PackageSpec, PackageVersion}, remote_package_source::RemotePackageSource, @@ -432,7 +432,7 @@ mod tests { .unwrap(); let tree = config.user_tree(LuaVersion::Lua51).unwrap(); - let mock_hashes = LocalPackageHashes { + let mock_hashes = LockedPackageHashes { rockspec: "sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=" .parse() .unwrap(), @@ -441,7 +441,7 @@ mod tests { .unwrap(), }; - let package = LocalPackage::from( + let package = LockedPackage::from( &PackageSpec::parse("neorg".into(), "8.0.0-1".into()).unwrap(), LockConstraint::Unconstrained, RockBinaries::default(), @@ -518,7 +518,7 @@ mod tests { .unwrap(); let tree = config.user_tree(LuaVersion::Lua51).unwrap(); - let mock_hashes = LocalPackageHashes { + let mock_hashes = LockedPackageHashes { rockspec: "sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=" .parse() .unwrap(), @@ -527,7 +527,7 @@ mod tests { .unwrap(), }; - let package = LocalPackage::from( + let package = LockedPackage::from( &PackageSpec::parse("neorg".into(), "8.0.0-1-1".into()).unwrap(), LockConstraint::Unconstrained, RockBinaries::default(), diff --git a/lux-lua/src/loader.rs b/lux-lua/src/loader.rs index 977853a67..e01ef1d9b 100644 --- a/lux-lua/src/loader.rs +++ b/lux-lua/src/loader.rs @@ -5,7 +5,7 @@ use std::{ rc::Rc, }; -use lux_lib::lockfile::{LocalPackage, LocalPackageId, Lockfile, ReadOnly}; +use lux_lib::lockfile::{LockedPackage, LockedPackageId, Lockfile, ReadOnly}; use mlua::prelude::*; use path_absolutize::Absolutize; @@ -47,8 +47,8 @@ fn current_file(lua: &Lua) -> Option { fn rock_layout( tree_root: &Path, - package_id: &LocalPackageId, - package: &LocalPackage, + package_id: &LockedPackageId, + package: &LockedPackage, ) -> (PathBuf, PathBuf) { let rock_path = tree_root.join(format!( "{}-{}@{}", @@ -218,7 +218,7 @@ fn load_from_workspace_tree( // with the lux tree and malform the package hash. In this case, this // should never cause any security-related problems anyway, as we'll // crash right after this function returns None. - let owner_id = unsafe { LocalPackageId::from_unchecked(module_hash.to_string()) }; + let owner_id = unsafe { LockedPackageId::from_unchecked(module_hash.to_string()) }; let Some(package) = lockfile.get(&owner_id) else { return Ok(None); }; diff --git a/lux-lua/src/lua_impls.rs b/lux-lua/src/lua_impls.rs index 5405f2180..ef13ceef3 100644 --- a/lux-lua/src/lua_impls.rs +++ b/lux-lua/src/lua_impls.rs @@ -21,8 +21,8 @@ use lux_lib::{ config::{Config, ConfigBuilder}, git::{GitRef, GitSource}, lockfile::{ - LocalPackage, LocalPackageHashes, LocalPackageId, LockConstraint, Lockfile, LockfileGuard, - OptState, PinnedState, ReadOnly, ReadWrite, + LockConstraint, LockedPackage, LockedPackageHashes, LockedPackageId, Lockfile, + LockfileGuard, OptState, PinnedState, ReadOnly, ReadWrite, }, lua_rockspec::{ BuildBackendSpec, BuildSpec, BuiltinBuildSpec, BustedTestSpec, CMakeBuildSpec, @@ -264,23 +264,23 @@ impl IntoLua for OptStateLua { } #[derive(Debug, Clone)] -pub struct LocalPackageIdLua(pub LocalPackageId); +pub struct LockedPackageIdLua(pub LockedPackageId); -impl Typed for LocalPackageIdLua { +impl Typed for LockedPackageIdLua { fn ty() -> Type { Type::string() } } -impl FromLua for LocalPackageIdLua { +impl FromLua for LockedPackageIdLua { fn from_lua(value: LuaValue, lua: &Lua) -> LuaResult { - Ok(LocalPackageIdLua(unsafe { - LocalPackageId::from_unchecked(String::from_lua(value, lua)?) + Ok(LockedPackageIdLua(unsafe { + LockedPackageId::from_unchecked(String::from_lua(value, lua)?) })) } } -impl IntoLua for LocalPackageIdLua { +impl IntoLua for LockedPackageIdLua { fn into_lua(self, lua: &Lua) -> LuaResult { self.0.into_string().into_lua(lua) } @@ -565,10 +565,10 @@ impl IntoLua for RockMatchesLua { table.set("is_found", lua.create_function(move |_, ()| Ok(is_found))?)?; match self.0 { RockMatches::NotFound(req) => table.set("not_found", PackageReqLua(req))?, - RockMatches::Single(id) => table.set("single", LocalPackageIdLua(id))?, + RockMatches::Single(id) => table.set("single", LockedPackageIdLua(id))?, RockMatches::Many(ids) => table.set( "many", - ids.into_iter().map(LocalPackageIdLua).collect::>(), + ids.into_iter().map(LockedPackageIdLua).collect::>(), )?, } Ok(LuaValue::Table(table)) @@ -712,7 +712,7 @@ impl IntoLua for SyncReportLua { .added() .iter() .cloned() - .map(LocalPackageLua) + .map(LockedPackageLua) .collect::>(), )?; table.set( @@ -721,7 +721,7 @@ impl IntoLua for SyncReportLua { .removed() .iter() .cloned() - .map(LocalPackageLua) + .map(LockedPackageLua) .collect::>(), )?; Ok(LuaValue::Table(table)) @@ -863,17 +863,17 @@ impl mlua::UserData for PackageReqLua { } #[derive(Debug, Clone)] -pub struct LocalPackageHashesLua(pub LocalPackageHashes); +pub struct LockedPackageHashesLua(pub LockedPackageHashes); -impl Typed for LocalPackageHashesLua { +impl Typed for LockedPackageHashesLua { fn ty() -> Type { - Type::named("LocalPackageHashes") + Type::named("LockedPackageHashes") } } -impl_from_lua_userdata!(LocalPackageHashesLua); +impl_from_lua_userdata!(LockedPackageHashesLua); -impl TypedUserData for LocalPackageHashesLua { +impl TypedUserData for LockedPackageHashesLua { fn add_methods>(methods: &mut M) { methods.param("rockspec", "sha256sum of the rockspec"); methods.add_method("rockspec", |_, this, ()| Ok(this.0.rockspec.to_hex().1)); @@ -885,7 +885,7 @@ impl TypedUserData for LocalPackageHashesLua { } } -impl mlua::UserData for LocalPackageHashesLua { +impl mlua::UserData for LockedPackageHashesLua { fn add_fields>(fields: &mut F) { let mut wrapper = mlua_extras::typed::WrappedBuilder::new(fields); ::add_fields(&mut wrapper); @@ -898,30 +898,30 @@ impl mlua::UserData for LocalPackageHashesLua { } #[derive(Debug, Clone)] -pub struct LocalPackageLua(pub LocalPackage); +pub struct LockedPackageLua(pub LockedPackage); -impl Typed for LocalPackageLua { +impl Typed for LockedPackageLua { fn ty() -> Type { - Type::named("LocalPackage") + Type::named("LockedPackage") } } -impl FromLua for LocalPackageLua { +impl FromLua for LockedPackageLua { fn from_lua(value: LuaValue, _lua: &Lua) -> LuaResult { match value { - LuaValue::UserData(ud) => Ok(ud.borrow::()?.clone()), + LuaValue::UserData(ud) => Ok(ud.borrow::()?.clone()), v => Err(LuaError::FromLuaConversionError { from: v.type_name(), - to: "LocalPackageLua".to_string(), + to: "LockedPackageLua".to_string(), message: None, }), } } } -impl TypedUserData for LocalPackageLua { +impl TypedUserData for LockedPackageLua { fn add_methods>(methods: &mut M) { - methods.add_method("id", |_, this, ()| Ok(LocalPackageIdLua(this.0.id()))); + methods.add_method("id", |_, this, ()| Ok(LockedPackageIdLua(this.0.id()))); methods.add_method("name", |_, this, ()| { Ok(PackageNameLua(this.0.name().clone())) }); @@ -934,14 +934,14 @@ impl TypedUserData for LocalPackageLua { .0 .dependencies() .into_iter() - .map(|id| LocalPackageIdLua(id.clone())) + .map(|id| LockedPackageIdLua(id.clone())) .collect::>()) }); methods.add_method("constraint", |_, this, ()| { Ok(LockConstraintLua(this.0.constraint())) }); methods.add_method("hashes", |_, this, ()| { - Ok(LocalPackageHashesLua(this.0.hashes().clone())) + Ok(LockedPackageHashesLua(this.0.hashes().clone())) }); methods.add_method("to_package", |_, this, ()| { Ok(PackageSpecLua(this.0.to_package())) @@ -955,7 +955,7 @@ impl TypedUserData for LocalPackageLua { } } -impl mlua::UserData for LocalPackageLua { +impl mlua::UserData for LockedPackageLua { fn add_fields>(fields: &mut F) { let mut wrapper = mlua_extras::typed::WrappedBuilder::new(fields); ::add_fields(&mut wrapper); @@ -1064,7 +1064,7 @@ impl TypedUserData for TreeLua { }); methods.document("The root directory of a package in this tree"); methods.param("package", ""); - methods.add_method("root_for", |_, this, package: LocalPackageLua| { + methods.add_method("root_for", |_, this, package: LockedPackageLua| { Ok(this .0 .layout_for(&package.0) @@ -1078,7 +1078,7 @@ impl TypedUserData for TreeLua { }); methods.document("Get the `RockLayout` for an installed package."); methods.param("package", ""); - methods.add_method("rock_layout", |_, this, package: LocalPackageLua| { + methods.add_method("rock_layout", |_, this, package: LockedPackageLua| { Ok(RockLayoutLua(this.0.layout_for(&package.0))) }); methods.document("Find installed rocks that match the given `PackageReq`"); @@ -2834,15 +2834,15 @@ impl TypedUserData for LockfileReadOnlyLua { .map(|(id, rock)| { ( id.clone().into_string().clone(), - LocalPackageLua(rock.clone()), + LockedPackageLua(rock.clone()), ) }) .collect::>()) }); methods.param("id", ""); - methods.add_method("get", |_, this, id: LocalPackageIdLua| { - Ok(this.0.get(&id.0).cloned().map(LocalPackageLua)) + methods.add_method("get", |_, this, id: LockedPackageIdLua| { + Ok(this.0.get(&id.0).cloned().map(LockedPackageLua)) }); methods.document( @@ -2891,15 +2891,15 @@ impl TypedUserData for LockfileGuardLua { .map(|(id, rock)| { ( id.clone().into_string().clone(), - LocalPackageLua(rock.clone()), + LockedPackageLua(rock.clone()), ) }) .collect::>()) }); methods.param("id", ""); - methods.add_method("get", |_, this, id: LocalPackageIdLua| { - Ok(this.0.get(&id.0).cloned().map(LocalPackageLua)) + methods.add_method("get", |_, this, id: LockedPackageIdLua| { + Ok(this.0.get(&id.0).cloned().map(LockedPackageLua)) }); } fn add_documentation>(docs: &mut F) { @@ -2939,7 +2939,7 @@ impl TypedUserData for LockfileReadWriteLua { .map(|(id, rock)| { ( id.clone().into_string().clone(), - LocalPackageLua(rock.clone()), + LockedPackageLua(rock.clone()), ) }) .collect::>()) @@ -2949,9 +2949,9 @@ impl TypedUserData for LockfileReadWriteLua { methods.add_method("get", |_, this, id: String| { Ok(this .0 - .get(unsafe { &LocalPackageId::from_unchecked(id) }) + .get(unsafe { &LockedPackageId::from_unchecked(id) }) .cloned() - .map(LocalPackageLua)) + .map(LockedPackageLua)) }); } fn add_documentation>(docs: &mut F) { @@ -3219,8 +3219,8 @@ mod definitions_registry { use super::{ BuildSpecLua, BustedTestSpecLua, CMakeBuildSpecLua, CommandBuildSpecLua, CommandTestSpecLua, ConfigBuilderLua, ConfigLua, DownloadedRockspecLua, GitSourceLua, - InstallSpecLua, LocalLuaRockspecLua, LocalPackageHashesLua, LocalPackageLua, - LocalProjectTomlLua, LockfileGuardLua, LockfileReadOnlyLua, LockfileReadWriteLua, + InstallSpecLua, LocalLuaRockspecLua, LocalProjectTomlLua, LockedPackageHashesLua, + LockedPackageLua, LockfileGuardLua, LockfileReadOnlyLua, LockfileReadWriteLua, LuaDependencySpecLua, LuaScriptTestSpecLua, MakeBuildSpecLua, ModulePathsLua, NvimLayoutLua, PackageReqLua, PackageSpecLua, PartialLuaRockspecLua, PartialProjectTomlLua, PlatformSupportLua, ProjectLua, RemoteLuaRockspecLua, RemotePackageDBLua, @@ -3246,8 +3246,8 @@ mod definitions_registry { submit_definitions! { "PackageSpec" => PackageSpecLua, "PackageReq" => PackageReqLua, - "LocalPackageHashes" => LocalPackageHashesLua, - "LocalPackage" => LocalPackageLua, + "LockedPackageHashes" => LockedPackageHashesLua, + "LockedPackage" => LockedPackageLua, "RockLayout" => RockLayoutLua, "Tree" => TreeLua, "NvimLayout" => NvimLayoutLua, diff --git a/lux-lua/src/operations.rs b/lux-lua/src/operations.rs index 16817ce95..af9e09bae 100644 --- a/lux-lua/src/operations.rs +++ b/lux-lua/src/operations.rs @@ -6,7 +6,7 @@ use std::{collections::HashMap, path::PathBuf}; use itertools::Itertools; use lux_lib::tree::InstallTree; use lux_lib::{ - lockfile::LocalPackageId, + lockfile::LockedPackageId, lua::lua_runtime, operations::{ set_pinned_state, BuildWorkspace, DistProjectBin, Download, Install, PackageInstallSpec, @@ -22,9 +22,9 @@ use mlua_extras::typed::{Type, Typed, TypedDataMethods, TypedUserData}; use path_slash::PathBufExt; use crate::lua_impls::{ - self, ConfigLua, DependencyTypeLua, DownloadedRockspecLua, LocalPackageIdLua, LocalPackageLua, - PackageInstallSpecLua, PackageNameLua, PackageReqLua, PinnedStateLua, ProjectLua, - SyncReportLua, TreeLua, WorkspaceLua, + self, ConfigLua, DependencyTypeLua, DownloadedRockspecLua, LockedPackageIdLua, + LockedPackageLua, PackageInstallSpecLua, PackageNameLua, PackageReqLua, PinnedStateLua, + ProjectLua, SyncReportLua, TreeLua, WorkspaceLua, }; #[derive(Clone)] @@ -67,7 +67,7 @@ impl TypedUserData for OperationsModule { .install() .await .into_lua_err() - .map(|pkgs| pkgs.into_iter().map(LocalPackageLua).collect::>()) + .map(|pkgs| pkgs.into_iter().map(LockedPackageLua).collect::>()) }, ); @@ -77,7 +77,7 @@ impl TypedUserData for OperationsModule { methods.param("config", "Lux config"); methods.add_async_function( "uninstall", - |_, (packages, tree, config): (Vec, Option, ConfigLua)| async move { + |_, (packages, tree, config): (Vec, Option, ConfigLua)| async move { let _runtime = lua_runtime().enter(); let ids = packages.into_iter().map(|p| p.0); Uninstall::new() @@ -87,7 +87,7 @@ impl TypedUserData for OperationsModule { .remove() .await .into_lua_err() - .map(|ids| ids.into_iter().map(LocalPackageIdLua).collect::>()) + .map(|ids| ids.into_iter().map(LockedPackageIdLua).collect::>()) }, ); @@ -118,7 +118,7 @@ impl TypedUserData for OperationsModule { .update() .await .into_lua_err() - .map(|pkgs| pkgs.into_iter().map(LocalPackageLua).collect::>()) + .map(|pkgs| pkgs.into_iter().map(LockedPackageLua).collect::>()) }, ); @@ -163,7 +163,7 @@ impl TypedUserData for OperationsModule { .install() .await .into_lua_err() - .map(|pkgs| pkgs.into_iter().map(LocalPackageLua).collect::>()) + .map(|pkgs| pkgs.into_iter().map(LockedPackageLua).collect::>()) }, ); @@ -187,7 +187,7 @@ impl TypedUserData for OperationsModule { let _runtime = lua_runtime().enter(); let deps = lua_impls::map_dependency_type_names(deps.0); let tree = workspace.0.tree(&config.0).into_lua_err()?; - let ids_to_remove: Vec = deps_to_ids(&deps, &tree); + let ids_to_remove: Vec = deps_to_ids(&deps, &tree); if ids_to_remove.is_empty() { return Ok(Vec::new()); @@ -207,7 +207,7 @@ impl TypedUserData for OperationsModule { .remove() .await .into_lua_err() - .map(|ids| ids.into_iter().map(LocalPackageIdLua).collect::>()) + .map(|ids| ids.into_iter().map(LockedPackageIdLua).collect::>()) }, ); @@ -261,7 +261,7 @@ impl TypedUserData for OperationsModule { .build() .await .into_lua_err() - .map(|packages: Vec<_>| packages.into_iter().map(LocalPackageLua).collect_vec()) + .map(|packages: Vec<_>| packages.into_iter().map(LockedPackageLua).collect_vec()) }, ); @@ -312,7 +312,7 @@ impl TypedUserData for OperationsModule { methods.param("pin_state", "The pinned state to set"); methods.add_function( "pin", - |_, (package_id, tree, pin_state): (LocalPackageIdLua, TreeLua, PinnedStateLua)| { + |_, (package_id, tree, pin_state): (LockedPackageIdLua, TreeLua, PinnedStateLua)| { set_pinned_state(&package_id.0, &tree.0, pin_state.0).into_lua_err() }, ); @@ -378,7 +378,7 @@ fn deps_to_specs(deps: &DependencyType) -> Vec { .collect() } -fn deps_to_ids(deps: &DependencyType, tree: &Tree) -> Vec { +fn deps_to_ids(deps: &DependencyType, tree: &Tree) -> Vec { let names = match deps { DependencyType::Regular(d) | DependencyType::Build(d) | DependencyType::Test(d) => d, DependencyType::External(_) => &Vec::default(), From 9aed4c27a5789b958ba534f420d212344f49b9b7 Mon Sep 17 00:00:00 2001 From: vhyrro Date: Thu, 24 Sep 2026 13:49:37 +0200 Subject: [PATCH 03/39] temp: more tests --- lux-lib/src/operations/vendor.rs | 260 +++++++++++++++++++++++++++++++ lux-lib/tests/vendor.rs | 53 +++++-- 2 files changed, 303 insertions(+), 10 deletions(-) diff --git a/lux-lib/src/operations/vendor.rs b/lux-lib/src/operations/vendor.rs index 17510580b..55c7427ec 100644 --- a/lux-lib/src/operations/vendor.rs +++ b/lux-lib/src/operations/vendor.rs @@ -565,3 +565,263 @@ async fn cargo_vendor( }) } } + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use bytes::Bytes; + + use super::*; + use crate::{ + config::ConfigBuilder, + lockfile::{LockConstraint, LockedPackageSpec, OptState, PinnedState, RemotePackageSourceUrl}, + lua_rockspec::RockSourceSpec, + operations::{unpack_rockspec, DownloadedPackedRockBytes, DownloadedRockspec}, + package::{PackageName, PackageVersion}, + remote_package_source::RemotePackageSource, + rockspec::RockBinaries, + }; + use assert_fs::prelude::PathCopy; + + use crate::operations::RemoteRockDownload; + + fn test_config() -> Config { + ConfigBuilder::new() + .unwrap() + .lua_version(Some(crate::lua_version::LuaVersion::Lua51)) + .build() + .unwrap() + } + + fn fixture_bytes(name: &str) -> Bytes { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("resources/test") + .join(name); + Bytes::from(std::fs::read(path).unwrap()) + } + + /// Builds a `PackageInstallData` for a source-tree-only package from a generated rockspec. + fn make_install_data( + rockspec: RemoteLuaRockspec, + source_url: Option, + source: RemotePackageSource, + ) -> PackageInstallData { + let spec = LockedPackageSpec::new( + rockspec.package(), + rockspec.version(), + LockConstraint::Unconstrained, + Vec::new(), + Vec::new(), + &PinnedState::Unpinned, + &OptState::Required, + RockBinaries::default(), + ); + let rockspec_download = DownloadedRockspec { + rockspec, + source, + source_url, + }; + PackageInstallData { + downloaded_rock: RemoteRockDownload::RockspecOnly { rockspec_download }, + spec, + } + } + + /// Vendors a `RockspecOnly`(url) dependency and asserts the rockspec and source + /// tree are laid out as expected. + #[tokio::test] + async fn vendor_rockspec_only_url_writes_rockspec_and_source_tree() { + let config = test_config(); + let vendor_dir = assert_fs::TempDir::new().unwrap(); + + // A local `File` source so no network is needed. `FetchSrc` copies the + // directory into the vendored source path. + let src_dir = assert_fs::TempDir::new().unwrap(); + std::fs::write(src_dir.path().join("hello.lua"), "return 'hello'").unwrap(); + let rockspec = RemoteLuaRockspec::from_package_and_source_spec( + "rockspec-only-url@1.0.0".parse().unwrap(), + RockSourceSpec::File(src_dir.path().to_path_buf()), + ); + + let install_data = make_install_data( + rockspec, + Some(RemotePackageSourceUrl::File { + path: src_dir.path().to_path_buf(), + }), + RemotePackageSource::RockspecContent(String::new()), + ); + + vendor_sources( + Arc::new(vendor_dir.to_path_buf()), + config, + vec![install_data], + ) + .await + .unwrap(); + + let rockspec_file = vendor_dir.path().join("rockspec-only-url-1.0.0-1.rockspec"); + assert!(rockspec_file.is_file(), "rockspec not vendored"); + let rockspec_content = std::fs::read_to_string(&rockspec_file).unwrap(); + assert!( + rockspec_content.contains("rockspec-only-url"), + "vendored rockspec is not for rockspec-only-url:\n{rockspec_content}" + ); + + let source_path = vendor_dir.path().join("rockspec-only-url@1.0.0-1"); + assert!(source_path.is_dir(), "source tree not vendored"); + assert!( + source_path.join("hello.lua").is_file(), + "source tree contents not vendored" + ); + } + + /// Vendors a `BinaryRock` dependency: both the packed `.rock` and its + /// extracted rockspec must be written. + #[tokio::test] + async fn vendor_binary_rock_writes_rock_and_rockspec() { + let vendor_dir = assert_fs::TempDir::new().unwrap(); + + let bytes = fixture_bytes("toml-edit-0.6.0-1.linux-x86_64.rock"); + let rock = DownloadedPackedRockBytes { + name: "toml-edit".into(), + version: "0.6.0-1".parse().unwrap(), + bytes: bytes.clone(), + file_name: "toml-edit-0.6.0-1.linux-x86_64.rock".into(), + url: "https://example.org/toml-edit-0.6.0-1.linux-x86_64.rock" + .parse() + .unwrap(), + }; + let rockspec = unpack_rockspec(&rock).await.unwrap(); + + vendor_binary_rock( + vendor_dir.path(), + DownloadedRockspec { + rockspec, + source: RemotePackageSource::LuarocksBinaryRock( + "https://example.org/".parse().unwrap(), + ), + source_url: None, + }, + bytes, + ) + .await + .unwrap(); + + assert!( + vendor_dir.path().join("toml-edit@0.6.0-1.rock").is_file(), + "packed rock not vendored" + ); + let rockspec_file = vendor_dir.path().join("toml-edit-0.6.0-1.rockspec"); + assert!(rockspec_file.is_file(), "rockspec not vendored"); + let rockspec_content = std::fs::read_to_string(&rockspec_file).unwrap(); + assert!( + rockspec_content.contains("package = "), + "vendored rockspec is not valid:\n{rockspec_content}" + ); + assert!( + rockspec_content.contains("toml-edit"), + "vendored rockspec is not for toml-edit:\n{rockspec_content}" + ); + } + + /// Vendors a `SrcRock` dependency. The raw `.src.rock` bytes must be written + /// verbatim to `@`. + #[tokio::test] + async fn vendor_src_rock_writes_bytes_verbatim() { + let config = test_config(); + let vendor_dir = assert_fs::TempDir::new().unwrap(); + + let bytes = fixture_bytes("luatest-0.2-1.src.rock"); + let rock = DownloadedPackedRockBytes { + name: "luatest".into(), + version: "0.2-1".parse().unwrap(), + bytes: bytes.clone(), + file_name: "luatest-0.2-1.src.rock".into(), + url: "https://example.org/luatest-0.2-1.src.rock" + .parse() + .unwrap(), + }; + let rockspec = unpack_rockspec(&rock).await.unwrap(); + + let src_rock_source = SrcRockSource { + bytes: bytes.clone(), + source_url: RemotePackageSourceUrl::Url { + url: "https://example.org/luatest-0.2-1.src.rock".parse().unwrap(), + }, + }; + + vendor_sources( + Arc::new(vendor_dir.to_path_buf()), + config, + vec![PackageInstallData { + downloaded_rock: RemoteRockDownload::SrcRock { + rockspec_download: DownloadedRockspec { + rockspec, + source: RemotePackageSource::LuarocksSrcRock( + "https://example.org/".parse().unwrap(), + ), + source_url: Some(src_rock_source.source_url.clone()), + }, + src_rock: bytes.clone(), + source_url: src_rock_source.source_url.clone(), + }, + spec: LockedPackageSpec::new( + &PackageName::new("luatest".to_string()), + &"0.2-1".parse::().unwrap(), + LockConstraint::Unconstrained, + Vec::new(), + Vec::new(), + &PinnedState::Unpinned, + &OptState::Required, + RockBinaries::default(), + ), + }], + ) + .await + .unwrap(); + + assert!( + vendor_dir.path().join("luatest-0.2-1.rockspec").is_file(), + "rockspec not vendored" + ); + let source_path = vendor_dir.path().join("luatest@0.2-1"); + assert!(source_path.is_file(), ".src.rock not vendored"); + assert_eq!( + std::fs::read(&source_path).unwrap(), + bytes.to_vec(), + "vendored .src.rock bytes differ from source" + ); + } + + /// `no_delete` must be respected: if the vendor dir exists and `no_delete` is + /// set, pre-existing contents are preserved. + #[tokio::test] + async fn vendor_workspace_no_delete_preserves_existing_contents() { + let sample_project_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("resources/test/sample-projects/busted-with-lockfile/"); + let temp_dir = assert_fs::TempDir::new().unwrap(); + temp_dir + .copy_from(&sample_project_dir, &["**"]) + .expect("failed to copy sample project"); + let workspace = Workspace::from_exact(temp_dir.path()).unwrap().unwrap(); + let config = test_config(); + let vendor_dir = assert_fs::TempDir::new().unwrap(); + let marker = vendor_dir.path().join("EXISTING_FILE"); + std::fs::write(&marker, "keep me").unwrap(); + + Vendor::new() + .target(VendorTarget::Workspace(workspace)) + .vendor_dir(vendor_dir.to_path_buf()) + .no_delete(true) + .config(&config) + .vendor_dependencies() + .await + .unwrap(); + + assert!( + marker.is_file(), + "no_delete should have preserved existing contents" + ); + } +} diff --git a/lux-lib/tests/vendor.rs b/lux-lib/tests/vendor.rs index ff9e2b4af..81ce161b1 100644 --- a/lux-lib/tests/vendor.rs +++ b/lux-lib/tests/vendor.rs @@ -20,6 +20,22 @@ async fn vendor_dependencies() { let temp_dir = assert_fs::TempDir::new().unwrap(); temp_dir.copy_from(sample_project_dir, &["**"]).unwrap(); let workspace = Workspace::from_exact(temp_dir.path()).unwrap().unwrap(); + + // Every rock in the sample project's committed lockfile must be vendored. + // (These are the `test_dependencies` of `busted-with-lockfile`.) + let expected_packages = [ + "luassert@1.9.0-1", + "lua_cliargs@3.0.2-1", + "penlight@1.15.0-1", + "lua-term@0.8-1", + "dkjson@2.11-1", + "mediator_lua@1.1.2-0", + "luafilesystem@1.9.0-1", + "busted@2.3.0-1", + "luasystem@0.7.1-1", + "say@1.4.1-3", + ]; + let config = ConfigBuilder::new().unwrap().build().unwrap(); let vendor_dir = assert_fs::TempDir::new().unwrap(); @@ -31,16 +47,33 @@ async fn vendor_dependencies() { .await .unwrap(); - let busted_rockspec = vendor_dir.child("busted-2.3.0-1.rockspec"); - busted_rockspec.assert(predicate::path::is_file()); + for package in expected_packages { + let (name, version) = package.split_once('@').unwrap(); + + // The rockspec must be vendored and parse back into a valid rockspec. + let rockspec_file = vendor_dir + .to_path_buf() + .join(format!("{name}-{version}.rockspec")); + assert!(rockspec_file.is_file(), "missing rockspec for {package}"); + let rockspec_content = std::fs::read_to_string(&rockspec_file).unwrap(); + lux_lib::lua_rockspec::RemoteLuaRockspec::new(&rockspec_content) + .unwrap_or_else(|err| panic!("vendored rockspec for {package} failed to parse: {err}")); + + // A source must be vendored alongside the rockspec (either a directory + // for `rockspec` sources or a file for `binary rock` sources). + let source_path = vendor_dir.to_path_buf().join(package); + assert!( + source_path.is_dir() || source_path.is_file(), + "missing vendored source for {package}" + ); + } + + // The sample lockfile's dependencies are all `rockspec` sources, so the + // vendored sources must be non-empty directories containing their contents. let busted_dir = vendor_dir.child("busted@2.3.0-1"); busted_dir.assert(predicate::path::is_dir()); - let luasystem_rockspec = vendor_dir.child("luasystem-0.7.1-1.rockspec"); - luasystem_rockspec.assert(predicate::path::is_file()); - let luasystem_dir = vendor_dir.child("luasystem@0.7.1-1"); - luasystem_dir.assert(predicate::path::is_dir()); - let say_rockspec = vendor_dir.child("say-1.4.1-3.rockspec"); - say_rockspec.assert(predicate::path::is_file()); - let say_dir = vendor_dir.child("say@1.4.1-3"); - say_dir.assert(predicate::path::is_dir()); + assert!( + std::fs::read_dir(busted_dir.path()).unwrap().next().is_some(), + "vendored source tree for busted is empty" + ); } From 10459ace76226fc84284ea5ea3eac428ce4d64e7 Mon Sep 17 00:00:00 2001 From: vhyrro Date: Thu, 24 Sep 2026 14:16:52 +0200 Subject: [PATCH 04/39] refactor!: reimplement vendor logic --- lux-cli/src/build.rs | 8 +- lux-cli/src/sync.rs | 16 +- lux-lib/src/fs/tokio.rs | 26 + lux-lib/src/operations/build_workspace.rs | 85 ++- lux-lib/src/operations/gen_luarc.rs | 2 +- lux-lib/src/operations/mod.rs | 1 - lux-lib/src/operations/pipeline/build.rs | 5 - .../src/operations/pipeline/build_project.rs | 96 +++ lux-lib/src/operations/pipeline/discover.rs | 15 +- .../pipeline/download_sources_and_hash.rs | 2 - .../src/operations/pipeline/emit_lockfile.rs | 52 ++ .../operations/pipeline/install_workspace.rs | 222 +++++++ lux-lib/src/operations/pipeline/mod.rs | 348 +---------- lux-lib/src/operations/pipeline/resolve.rs | 71 ++- lux-lib/src/operations/resolve.rs | 377 ------------ lux-lib/src/operations/sync.rs | 4 +- lux-lib/src/operations/update.rs | 6 +- lux-lib/src/operations/vendor.rs | 547 ++++++++---------- lux-lib/src/reqwest.rs | 13 - 19 files changed, 763 insertions(+), 1133 deletions(-) create mode 100644 lux-lib/src/operations/pipeline/build_project.rs create mode 100644 lux-lib/src/operations/pipeline/emit_lockfile.rs create mode 100644 lux-lib/src/operations/pipeline/install_workspace.rs delete mode 100644 lux-lib/src/operations/resolve.rs diff --git a/lux-cli/src/build.rs b/lux-cli/src/build.rs index 7ee4597ba..798bdf32b 100644 --- a/lux-cli/src/build.rs +++ b/lux-cli/src/build.rs @@ -2,7 +2,7 @@ use clap::Args; use lux_lib::{ config::Config, lockfile::LockedPackage, - operations::pipeline::Pipeline, + operations::BuildWorkspace, package::PackageName, workspace::Workspace, }; @@ -26,11 +26,11 @@ pub struct Build { /// Returns `Some` if the `only_deps` arg is set to `false`. pub async fn build(data: Build, config: Config) -> Result> { let workspace = Workspace::current_or_err()?; - let result = Pipeline::new(&config, &workspace) + let result = BuildWorkspace::new(&workspace, &config) .maybe_package(data.package) .no_lock(data.no_lock) - .build_projects(!data.only_deps) - .run() + .only_deps(data.only_deps) + .build() .await?; Ok(result) } diff --git a/lux-cli/src/sync.rs b/lux-cli/src/sync.rs index 44e812907..c96ae2bbd 100644 --- a/lux-cli/src/sync.rs +++ b/lux-cli/src/sync.rs @@ -1,5 +1,9 @@ use clap::Args; -use lux_lib::{config::Config, operations::pipeline::Pipeline, workspace::Workspace}; +use lux_lib::{ + config::Config, + operations::{GenLuaRc, pipeline::install_workspace::InstallWorkspaceDependencies}, + workspace::Workspace, +}; use miette::Result; @@ -16,9 +20,15 @@ pub async fn sync(args: SyncProject, config: Config) -> Result<()> { let _ = args.no_integrity_check; let workspace = Workspace::current_or_err()?; - Pipeline::new(&config, &workspace) + InstallWorkspaceDependencies::new(&config, &workspace) .test(true) - .run() + .install() + .await?; + + GenLuaRc::new() + .config(&config) + .workspace(&workspace) + .generate_luarc() .await?; // FIXME(vhyrro): Readd report publishing diff --git a/lux-lib/src/fs/tokio.rs b/lux-lib/src/fs/tokio.rs index b8da8510e..cafa1d3fd 100644 --- a/lux-lib/src/fs/tokio.rs +++ b/lux-lib/src/fs/tokio.rs @@ -60,6 +60,32 @@ pub(crate) async fn create_dir_all(path: impl AsRef) -> Result<(), FsError }) } +/// Recursively copy the contents of a directory into `dest`. +pub(crate) async fn copy_dir_all(src: &Path, dest: &Path) -> Result<(), FsError> { + Box::pin(async move { + create_dir_all(dest).await?; + let mut entries = read_dir(src).await?; + while let Some(entry) = entries + .next_entry() + .await + .map_err(|source| FsError::ReadDir { + path: src.to_path_buf(), + source, + })? + { + let entry_path = entry.path(); + let target = dest.join(entry.file_name()); + if entry_path.is_dir() { + copy_dir_all(&entry_path, &target).await?; + } else { + copy(&entry_path, &target).await?; + } + } + Ok(()) + }) + .await +} + /// Wrapped [`fs::remove_file`]. pub(crate) async fn remove_file(path: impl AsRef) -> Result<(), FsError> { let path = path.as_ref(); diff --git a/lux-lib/src/operations/build_workspace.rs b/lux-lib/src/operations/build_workspace.rs index 49fa5340d..ae00c5338 100644 --- a/lux-lib/src/operations/build_workspace.rs +++ b/lux-lib/src/operations/build_workspace.rs @@ -1,19 +1,46 @@ use crate::{ - config::Config, lockfile::LockedPackage, operations::pipeline::Pipeline, package::PackageName, - workspace::Workspace, + config::Config, + lockfile::LockedPackage, + lua_installation::{LuaInstallation, LuaInstallationError}, + operations::{ + GenLuaRc, GenLuaRcError, + pipeline::{ + build_project::{BuildProject, BuildProjectError}, + install_workspace::{InstallWorkspaceDependencies, InstallWorkspaceDependenciesError}, + }, + }, + package::PackageName, + workspace::{Workspace, WorkspaceError}, }; use bon::Builder; -use miette::Diagnostic; use thiserror::Error; use tracing::{info_span, Instrument}; -#[derive(Debug, Error, Diagnostic)] +#[derive(Debug, Error, miette::Diagnostic)] #[non_exhaustive] pub enum BuildWorkspaceError { // FIXME(vhyrro): don't use dyn here #[error(transparent)] #[diagnostic(transparent)] - Pipeline(Box), + InstallWorkspaceDependencies(#[from] InstallWorkspaceDependenciesError), + #[error("failed to build the workspace")] + #[diagnostic(forward(0))] + BuildProject(Box), + #[error(transparent)] + #[diagnostic(transparent)] + LuaInstallation(#[from] LuaInstallationError), + #[error(transparent)] + #[diagnostic(transparent)] + GenLuaRc(#[from] GenLuaRcError), + #[error(transparent)] + #[diagnostic(transparent)] + Workspace(#[from] WorkspaceError), +} + +impl From for BuildWorkspaceError { + fn from(source: BuildProjectError) -> Self { + Self::BuildProject(Box::new(source)) + } } #[derive(Builder)] @@ -45,16 +72,44 @@ impl info_span!("Building workspace"), }; async move { - // NOTE: `Pipeline::run`'s future is large, so we box it to avoid inflating this frame. - let result = Box::pin( - Pipeline::new(build.config, build.workspace) - .maybe_package(build.package) - .no_lock(build.no_lock) - .build_projects(!build.only_deps) - .run(), - ) - .await; - result.map_err(|err| BuildWorkspaceError::Pipeline(err.into())) + InstallWorkspaceDependencies::new(build.config, build.workspace) + .no_lock(build.no_lock) + .install() + .await?; + + let mut built = Vec::new(); + if !build.only_deps { + let lua = LuaInstallation::new_from_config(build.config).await?; + match &build.package { + Some(package) => { + let project = build.workspace.select_member(package)?; + built.push( + BuildProject::new(project, build.workspace, build.config, &lua) + .build() + .await?, + ); + } + None => { + for project in build.workspace.members() { + built.push( + BuildProject::new(project, build.workspace, build.config, &lua) + .build() + .await?, + ); + } + } + } + } + + if !build.no_lock { + GenLuaRc::new() + .config(build.config) + .workspace(build.workspace) + .generate_luarc() + .await?; + } + + Ok(built) } .instrument(span) .await diff --git a/lux-lib/src/operations/gen_luarc.rs b/lux-lib/src/operations/gen_luarc.rs index afc1393cc..3dea41d26 100644 --- a/lux-lib/src/operations/gen_luarc.rs +++ b/lux-lib/src/operations/gen_luarc.rs @@ -35,7 +35,7 @@ pub enum GenLuaRcError { #[derive(Builder)] #[builder(start_fn = new, finish_fn(name = _build, vis = ""))] -pub(crate) struct GenLuaRc<'a> { +pub struct GenLuaRc<'a> { config: &'a Config, workspace: &'a Workspace, } diff --git a/lux-lib/src/operations/mod.rs b/lux-lib/src/operations/mod.rs index 9d63be8e3..dfe4e4ab1 100644 --- a/lux-lib/src/operations/mod.rs +++ b/lux-lib/src/operations/mod.rs @@ -15,7 +15,6 @@ mod install_rockspec; mod pack; mod pin; pub mod pipeline; -mod resolve; mod run; mod run_lua; mod sync; diff --git a/lux-lib/src/operations/pipeline/build.rs b/lux-lib/src/operations/pipeline/build.rs index 9006114a1..cc439b980 100644 --- a/lux-lib/src/operations/pipeline/build.rs +++ b/lux-lib/src/operations/pipeline/build.rs @@ -68,11 +68,6 @@ where self.packages = packages; self } - - pub(crate) fn package(mut self, package: DownloadedPackage) -> Self { - self.packages.push(package); - self - } } impl BuildBuilder<'_, T, State> diff --git a/lux-lib/src/operations/pipeline/build_project.rs b/lux-lib/src/operations/pipeline/build_project.rs new file mode 100644 index 000000000..29baf8844 --- /dev/null +++ b/lux-lib/src/operations/pipeline/build_project.rs @@ -0,0 +1,96 @@ +use bon::Builder; +use miette::Diagnostic; +use thiserror::Error; + +use crate::{ + build::BuildBehaviour, + config::Config, + lockfile::LockedPackage, + lua_installation::{LuaInstallation, LuaInstallationError}, + project::project_toml::LocalProjectTomlValidationError, + tree::{EntryType, InstallTree, TreeError}, + workspace::{Workspace, WorkspaceTreeError}, +}; + +/// Builds a single workspace member into the workspace's install tree, recording +/// it as an entrypoint in the tree's lockfile. +#[derive(Builder)] +#[builder(start_fn = new, finish_fn(name = _build, vis = ""))] +pub(crate) struct BuildProject<'a> { + #[builder(start_fn)] + project: &'a crate::project::Project, + #[builder(start_fn)] + workspace: &'a Workspace, + #[builder(start_fn)] + config: &'a Config, + #[builder(start_fn)] + lua: &'a LuaInstallation, +} + +impl BuildProjectBuilder<'_, State> +where + State: build_project_builder::State + build_project_builder::IsComplete, +{ + pub(crate) async fn build(self) -> Result { + let args = self._build(); + + let workspace_tree = args.workspace.tree(args.config)?; + let project_toml = args.project.toml().into_local()?; + + let package = crate::build::Build::new() + .rockspec(&project_toml) + .lua(args.lua) + .tree(&workspace_tree) + .entry_type(EntryType::Entrypoint) + .config(args.config) + .behaviour(BuildBehaviour::Force) + .build() + .await?; + + let lockfile = workspace_tree.lockfile()?; + let dependencies = lockfile + .rocks() + .iter() + .filter_map(|(id, package)| lockfile.is_entrypoint(id).then_some(package)) + .cloned() + .collect::>(); + let build_lockfile = args.workspace.build_tree(args.config)?.lockfile()?; + + let mut lockfile = lockfile.write_guard(); + lockfile.add_entrypoint(&package); + for dependency in dependencies { + lockfile.add_dependency(&package, &dependency); + } + for dependency in build_lockfile.rocks().values() { + lockfile.add_build_dependency(&package, dependency); + } + + Ok(package) + } +} + +#[derive(Error, Debug, Diagnostic)] +#[non_exhaustive] +pub(crate) enum BuildProjectError { + #[error("failed to build the project")] + #[diagnostic(forward(0))] + Build(#[source] Box), + #[error(transparent)] + #[diagnostic(transparent)] + LuaInstallation(#[from] LuaInstallationError), + #[error(transparent)] + #[diagnostic(transparent)] + Tree(#[from] TreeError), + #[error(transparent)] + #[diagnostic(transparent)] + WorkspaceTree(#[from] WorkspaceTreeError), + #[error(transparent)] + #[diagnostic(transparent)] + Project(#[from] LocalProjectTomlValidationError), +} + +impl From for BuildProjectError { + fn from(source: crate::build::BuildError) -> Self { + Self::Build(Box::new(source)) + } +} diff --git a/lux-lib/src/operations/pipeline/discover.rs b/lux-lib/src/operations/pipeline/discover.rs index aa282c51c..a47bd2656 100644 --- a/lux-lib/src/operations/pipeline/discover.rs +++ b/lux-lib/src/operations/pipeline/discover.rs @@ -8,7 +8,7 @@ use crate::{ config::Config, lua_rockspec::RemoteLuaRockspec, operations::{self, Download, FetchVendored, FetchVendoredError, RemoteRockDownload}, - package::{PackageReq, PackageSpec, RemotePackage}, + package::{PackageReq, RemotePackage}, remote_package_db::{RemotePackageDB, SearchError}, }; @@ -22,19 +22,6 @@ pub(crate) struct FoundPackage { pub(crate) enum FoundPackageType { Package(FoundPackage), - Closure { - root: PackageSpec, - nodes: Vec, - }, -} - -impl FoundPackageType { - pub(crate) fn root(&self) -> &PackageSpec { - match self { - Self::Package(artifact) => &artifact.package.package, - Self::Closure { root, .. } => root, - } - } } #[derive(Error, Debug, Diagnostic)] diff --git a/lux-lib/src/operations/pipeline/download_sources_and_hash.rs b/lux-lib/src/operations/pipeline/download_sources_and_hash.rs index cdd2b5399..145835fb5 100644 --- a/lux-lib/src/operations/pipeline/download_sources_and_hash.rs +++ b/lux-lib/src/operations/pipeline/download_sources_and_hash.rs @@ -49,8 +49,6 @@ pub(crate) struct DownloadSourcesAndHashArtifacts { #[derive(Error, Debug, Diagnostic)] #[non_exhaustive] pub(crate) enum DownloadSourcesAndHashError { - #[error("failed to fetch source for '{0}'")] - Fetch(String), #[error(transparent)] #[diagnostic(transparent)] Fs(#[from] FsError), diff --git a/lux-lib/src/operations/pipeline/emit_lockfile.rs b/lux-lib/src/operations/pipeline/emit_lockfile.rs new file mode 100644 index 000000000..219e396fa --- /dev/null +++ b/lux-lib/src/operations/pipeline/emit_lockfile.rs @@ -0,0 +1,52 @@ +use std::collections::HashMap; + +use bon::Builder; + +use crate::lockfile::{ + LockedPackageId, LockedPackageLock, LockedPackageLockType, ReadWrite, WorkspaceLockfile, +}; + +use super::download_sources_and_hash::{DownloadSourcesAndHashArtifacts, DownloadedPackage}; + +/// Writes the resolved [`DownloadSourcesAndHashArtifacts`] to a workspace lockfile. +#[derive(Builder)] +#[builder(start_fn = new, finish_fn(name = _build, vis = ""))] +pub(crate) struct EmitLockfile<'a> { + #[builder(start_fn)] + pub(crate) lockfile: &'a mut WorkspaceLockfile, + pub(crate) artifacts: &'a DownloadSourcesAndHashArtifacts, +} + +impl EmitLockfileBuilder<'_, State> +where + State: emit_lockfile_builder::State + emit_lockfile_builder::IsComplete, +{ + pub(crate) fn emit(self) { + let args = self._build(); + + args.lockfile.sync( + &lock_from(&args.artifacts.regular), + &LockedPackageLockType::Regular, + ); + args.lockfile.sync( + &lock_from(&args.artifacts.build), + &LockedPackageLockType::Build, + ); + args.lockfile.sync( + &lock_from(&args.artifacts.test), + &LockedPackageLockType::Test, + ); + } +} + +fn lock_from(packages: &HashMap) -> LockedPackageLock { + // FIXME(vhyrro): Create a constructor here instead of mut overrides. + let mut lock = LockedPackageLock::default(); + for package in packages.values() { + lock.insert( + package.package.clone(), + package.entry_type.is_entrypoint(), + ); + } + lock +} diff --git a/lux-lib/src/operations/pipeline/install_workspace.rs b/lux-lib/src/operations/pipeline/install_workspace.rs new file mode 100644 index 000000000..65001e6b7 --- /dev/null +++ b/lux-lib/src/operations/pipeline/install_workspace.rs @@ -0,0 +1,222 @@ +use std::sync::Arc; + +use bon::Builder; +use miette::Diagnostic; +use thiserror::Error; + +use crate::{ + config::Config, + lockfile::LockedPackage, + operations::PackageInstallSpec, + project::project_toml::LocalProjectTomlValidationError, + remote_package_db::{RemotePackageDB, RemotePackageDBError}, + rockspec::Rockspec, + tree::EntryType, + workspace::{Workspace, WorkspaceError, WorkspaceTreeError}, +}; + +use super::{ + build::Build, + discover::FindPackageFromLuarocks, + download_sources_and_hash::DownloadSourcesAndHash, + emit_lockfile::EmitLockfile, + resolve::ResolvePackageDependencies, +}; + +/// Installs all of a workspace's dependencies into its regular, build and test trees. +/// +/// This resolves the workspace's dependency graph, downloads and hashes the sources, +/// writes the workspace lockfile, and builds the packages into the appropriate trees. +#[derive(Builder)] +#[builder(start_fn = new, finish_fn(name = _build, vis = ""))] +pub struct InstallWorkspaceDependencies<'a> { + #[builder(start_fn)] + config: &'a Config, + #[builder(start_fn)] + workspace: &'a Workspace, + /// Additional packages to install alongside the workspace's own dependencies. + #[builder(field)] + packages: Vec, + /// Also install the workspace's test dependencies. + test: Option, + /// Ignore the project's lockfile and don't create one. + no_lock: Option, +} + +impl InstallWorkspaceDependenciesBuilder<'_, State> +where + State: install_workspace_dependencies_builder::State, +{ + pub fn packages(mut self, packages: Vec) -> Self { + self.packages = packages; + self + } +} + +impl InstallWorkspaceDependenciesBuilder<'_, State> +where + State: install_workspace_dependencies_builder::State + + install_workspace_dependencies_builder::IsComplete, +{ + pub async fn install( + self, + ) -> Result, InstallWorkspaceDependenciesError> { + let args = self._build(); + let config = args.config; + let workspace = args.workspace; + let no_lock = args.no_lock.unwrap_or(false); + + let package_db = RemotePackageDB::from_config(config).await?; + let discover = + FindPackageFromLuarocks::new(Arc::new(package_db), Arc::new(config.clone())).build(); + + let mut regular = gather_dependencies(workspace, DependencyKind::Regular)?; + regular.extend(args.packages); + let build = gather_dependencies(workspace, DependencyKind::Build)?; + let test = if args.test.unwrap_or(false) { + gather_dependencies(workspace, DependencyKind::Test)? + } else { + Vec::new() + }; + + let resolved = ResolvePackageDependencies::new(&discover, config) + .packages(regular) + .build_packages(build) + .test_packages(test) + .resolve() + .await + .map_err(|err| InstallWorkspaceDependenciesError::Pipeline(Box::new(err)))?; + + let artifacts = DownloadSourcesAndHash::new(config) + .resolved(resolved) + .download_sources_and_hash() + .await + .map_err(|err| InstallWorkspaceDependenciesError::Pipeline(Box::new(err)))?; + + if !no_lock { + let mut lockfile = workspace.lockfile()?.write_guard(); + EmitLockfile::new(&mut lockfile) + .artifacts(&artifacts) + .emit(); + } + + let regular_tree = workspace.tree(config)?; + let build_tree = workspace.build_tree(config)?; + let test_tree = workspace.test_tree(config)?; + + // TODO(vhyrro): Make parallel + // make sure to build build dependencies first + Build::new(config, &build_tree) + .packages(artifacts.build.into_values().collect()) + .build() + .await + .map_err(|err| InstallWorkspaceDependenciesError::Pipeline(Box::new(err)))?; + let built = Build::new(config, ®ular_tree) + .packages(artifacts.regular.into_values().collect()) + .build() + .await + .map_err(|err| InstallWorkspaceDependenciesError::Pipeline(Box::new(err)))?; + Build::new(config, &test_tree) + .packages(artifacts.test.into_values().collect()) + .build() + .await + .map_err(|err| InstallWorkspaceDependenciesError::Pipeline(Box::new(err)))?; + + Ok(built) + } +} + +/// The errors that may occur while installing a workspace's dependencies. +#[derive(Error, Debug, Diagnostic)] +#[non_exhaustive] +pub enum InstallWorkspaceDependenciesError { + #[error(transparent)] + #[diagnostic(transparent)] + Workspace(#[from] WorkspaceError), + #[error(transparent)] + #[diagnostic(transparent)] + WorkspaceTree(#[from] WorkspaceTreeError), + #[error(transparent)] + #[diagnostic(transparent)] + RemotePackageDB(#[from] RemotePackageDBError), + #[error(transparent)] + #[diagnostic(transparent)] + Project(#[from] LocalProjectTomlValidationError), + #[error("failed to install workspace dependencies")] + #[diagnostic(forward(0))] + Pipeline(Box), +} + +#[derive(Clone, Copy)] +enum DependencyKind { + Regular, + Build, + Test, +} + +fn gather_dependencies( + workspace: &Workspace, + kind: DependencyKind, +) -> Result, InstallWorkspaceDependenciesError> { + let mut packages = Vec::new(); + for project in workspace.members() { + let toml = project.toml().into_local()?; + let dependencies = match kind { + DependencyKind::Regular => toml.dependencies().current_platform().clone(), + DependencyKind::Build => toml.build_dependencies().current_platform().clone(), + DependencyKind::Test => toml.test_dependencies().current_platform().clone(), + }; + for dependency in dependencies { + // From the perspective of a project, dependencies are entrypoints. + packages.push( + PackageInstallSpec::new(dependency.package_req().clone(), EntryType::Entrypoint) + .maybe_source(dependency.source().clone()) + .build(), + ); + } + } + Ok(packages) +} + +#[cfg(all(test, feature = "impure_tests"))] +mod tests { + use std::path::PathBuf; + + use assert_fs::prelude::PathCopy; + + use crate::{ + config::ConfigBuilder, lockfile::LockedPackageLockType, lua_version::LuaVersion, + workspace::Workspace, + }; + + use super::InstallWorkspaceDependencies; + + #[tokio::test] + async fn installs_workspace_dependencies() { + let sample = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("resources/test/sample-projects/dependencies/"); + let temp = assert_fs::TempDir::new().unwrap(); + temp.copy_from(sample, &["**"]).unwrap(); + + let workspace = Workspace::from_exact(temp.path()).unwrap().unwrap(); + let config = ConfigBuilder::new() + .unwrap() + .lua_version(Some(LuaVersion::Lua51)) + .build() + .unwrap(); + + InstallWorkspaceDependencies::new(&config, &workspace) + .install() + .await + .unwrap(); + + let lockfile = workspace.lockfile().unwrap(); + let rocks = lockfile.rocks(&LockedPackageLockType::Regular); + assert!(rocks + .values() + .any(|pkg| pkg.name().to_string() == "lua-cjson")); + assert!(rocks + .values() + .any(|pkg| pkg.name().to_string() == "plenary.nvim")); + } +} diff --git a/lux-lib/src/operations/pipeline/mod.rs b/lux-lib/src/operations/pipeline/mod.rs index 4744547f2..56ff61572 100644 --- a/lux-lib/src/operations/pipeline/mod.rs +++ b/lux-lib/src/operations/pipeline/mod.rs @@ -1,348 +1,8 @@ -#![allow(dead_code)] - pub(crate) mod build; +pub(crate) mod build_project; pub(crate) mod discover; pub(crate) mod download_sources_and_hash; -pub mod install_packages; +pub(crate) mod emit_lockfile; pub(crate) mod resolve; - -use std::collections::HashMap; -use std::sync::Arc; - -use bon::Builder; -use miette::Diagnostic; -use thiserror::Error; - -use crate::{ - build::BuildBehaviour, config::Config, lockfile::{ - LockedPackage, LockedPackageId, LockedPackageLock, LockedPackageLockType, ReadWrite, - WorkspaceLockfile, - }, lua_installation::{LuaInstallation, LuaInstallationError}, operations::{GenLuaRc, GenLuaRcError, PackageInstallSpec, pipeline::download_sources_and_hash::{DownloadSourcesAndHashArtifacts, DownloadedPackage}}, package::PackageName, project::{project_toml::LocalProjectTomlValidationError, Project}, remote_package_db::{RemotePackageDB, RemotePackageDBError}, rockspec::Rockspec, tree::{EntryType, InstallTree, TreeError}, workspace::{Workspace, WorkspaceError, WorkspaceTreeError}, -}; - -use self::{ - build::Build, discover::FindPackageFromLuarocks, download_sources_and_hash::DownloadSourcesAndHash, - resolve::ResolvePackageDependencies, -}; - -#[derive(Builder)] -#[builder(start_fn = new, finish_fn(name = _build, vis = ""))] -pub struct Pipeline<'a> { - #[builder(start_fn)] - pub(crate) config: &'a Config, - #[builder(start_fn)] - pub(crate) workspace: &'a Workspace, - #[builder(field)] - pub(crate) packages: Vec, - pub(crate) test: Option, - /// Build the workspace's own project(s) after installing dependencies. - pub(crate) build_projects: Option, - /// Only build this workspace member. - pub(crate) package: Option, - /// Ignore the project's lockfile and don't create one. - pub(crate) no_lock: Option, -} - -impl PipelineBuilder<'_, State> -where - State: pipeline_builder::State, -{ - pub fn packages(mut self, packages: Vec) -> Self { - self.packages = packages; - self - } -} - -impl PipelineBuilder<'_, State> -where - State: pipeline_builder::State + pipeline_builder::IsComplete, -{ - pub async fn run(self) -> miette::Result> { - let args = self._build(); - let config = args.config; - let workspace = args.workspace; - let no_lock = args.no_lock.unwrap_or(false); - - let package_db = RemotePackageDB::from_config(config).await?; - let discover = FindPackageFromLuarocks::new(Arc::new(package_db), Arc::new(config.clone())).build(); - - let mut regular = gather_dependencies(workspace, DependencyKind::Regular)?; - regular.extend(args.packages); - let build = gather_dependencies(workspace, DependencyKind::Build)?; - let test = if args.test.unwrap_or(false) { - gather_dependencies(workspace, DependencyKind::Test)? - } else { - Vec::new() - }; - - let resolved = ResolvePackageDependencies::new(&discover, config) - .packages(regular) - .build_packages(build) - .test_packages(test) - .resolve() - .await?; - - let artifacts = DownloadSourcesAndHash::new(config) - .resolved(resolved) - .download_sources_and_hash() - .await?; - - if !no_lock { - let mut lockfile = workspace.lockfile()?.write_guard(); - EmitLockfile::new(&mut lockfile) - .artifacts(&artifacts) - .emit()?; - } - - let regular_tree = workspace.tree(config)?; - let build_tree = workspace.build_tree(config)?; - let test_tree = workspace.test_tree(config)?; - - // TODO(vhyrro): Make parallel - // make sure to build build dependencies first - Build::new(config, &build_tree) - .packages(artifacts.build.into_values().collect()) - .build() - .await?; - Build::new(config, ®ular_tree) - .packages(artifacts.regular.into_values().collect()) - .build() - .await?; - Build::new(config, &test_tree) - .packages(artifacts.test.into_values().collect()) - .build() - .await?; - - let mut built = Vec::new(); - if args.build_projects.unwrap_or(false) { - let lua = LuaInstallation::new_from_config(config).await?; - match &args.package { - Some(package) => { - let project = workspace.select_member(package)?; - built.push(build_project(project, workspace, &lua, config).await?); - } - None => { - for project in workspace.members() { - built.push(build_project(project, workspace, &lua, config).await?); - } - } - } - } - - if !no_lock { - GenLuaRc::new() - .config(config) - .workspace(workspace) - .generate_luarc() - .await?; - } - - Ok(built) - } -} - -#[derive(Clone, Copy)] -enum DependencyKind { - Regular, - Build, - Test, -} - -fn gather_dependencies( - workspace: &Workspace, - kind: DependencyKind, -) -> Result, PipelineError> { - let mut packages = Vec::new(); - for project in workspace.members() { - let toml = project.toml().into_local()?; - let dependencies = match kind { - DependencyKind::Regular => toml.dependencies().current_platform().clone(), - DependencyKind::Build => toml.build_dependencies().current_platform().clone(), - DependencyKind::Test => toml.test_dependencies().current_platform().clone(), - }; - for dependency in dependencies { - // From the perspective of a project, dependencies are entrypoints. - packages.push( - PackageInstallSpec::new(dependency.package_req().clone(), EntryType::Entrypoint) - .maybe_source(dependency.source().clone()) - .build(), - ); - } - } - Ok(packages) -} - -async fn build_project( - project: &Project, - workspace: &Workspace, - lua: &LuaInstallation, - config: &Config, -) -> Result { - let workspace_tree = workspace.tree(config)?; - let project_toml = project.toml().into_local()?; - - let package = crate::build::Build::new() - .rockspec(&project_toml) - .lua(lua) - .tree(&workspace_tree) - .entry_type(EntryType::Entrypoint) - .config(config) - .behaviour(BuildBehaviour::Force) - .build() - .await?; - - let lockfile = workspace_tree.lockfile()?; - let dependencies = lockfile - .rocks() - .iter() - .filter_map(|(id, package)| lockfile.is_entrypoint(id).then_some(package)) - .cloned() - .collect::>(); - let build_lockfile = workspace.build_tree(config)?.lockfile()?; - - let mut lockfile = lockfile.write_guard(); - lockfile.add_entrypoint(&package); - for dependency in dependencies { - lockfile.add_dependency(&package, &dependency); - } - for dependency in build_lockfile.rocks().values() { - lockfile.add_build_dependency(&package, dependency); - } - - Ok(package) -} - -#[derive(Error, Debug, Diagnostic)] -#[non_exhaustive] -pub(crate) enum PipelineError { - #[error(transparent)] - #[diagnostic(transparent)] - Resolve(#[from] resolve::ResolveError), - #[error(transparent)] - #[diagnostic(transparent)] - DownloadSourcesAndHash(#[from] download_sources_and_hash::DownloadSourcesAndHashError), - #[error(transparent)] - #[diagnostic(transparent)] - Build(#[from] build::BuildError), - #[error("failed to build the project")] - #[diagnostic(forward(0))] - ProjectBuild(#[source] Box), - #[error(transparent)] - #[diagnostic(transparent)] - LuaInstallation(#[from] LuaInstallationError), - #[error(transparent)] - #[diagnostic(transparent)] - Tree(#[from] TreeError), - #[error(transparent)] - #[diagnostic(transparent)] - Emit(#[from] EmitError), - #[error(transparent)] - #[diagnostic(transparent)] - GenLuaRc(#[from] GenLuaRcError), - #[error(transparent)] - #[diagnostic(transparent)] - Workspace(#[from] WorkspaceError), - #[error(transparent)] - #[diagnostic(transparent)] - WorkspaceTree(#[from] WorkspaceTreeError), - #[error(transparent)] - #[diagnostic(transparent)] - RemotePackageDB(#[from] RemotePackageDBError), - #[error(transparent)] - #[diagnostic(transparent)] - Project(#[from] LocalProjectTomlValidationError), -} - -// FIXME(vhyrro): Use the natural `#[from]` instead of manual implementation -impl From for PipelineError { - fn from(source: crate::build::BuildError) -> Self { - Self::ProjectBuild(Box::new(source)) - } -} - -#[derive(Builder)] -#[builder(start_fn = new, finish_fn(name = _build, vis = ""))] -pub(crate) struct EmitLockfile<'a> { - #[builder(start_fn)] - pub(crate) lockfile: &'a mut WorkspaceLockfile, - pub(crate) artifacts: &'a DownloadSourcesAndHashArtifacts, -} - -impl EmitLockfileBuilder<'_, State> -where - State: emit_lockfile_builder::State + emit_lockfile_builder::IsComplete, -{ - pub(crate) fn emit(self) -> Result<(), EmitError> { - let args = self._build(); - - args.lockfile.sync( - &lock_from(&args.artifacts.regular), - &LockedPackageLockType::Regular, - ); - args.lockfile.sync( - &lock_from(&args.artifacts.build), - &LockedPackageLockType::Build, - ); - args.lockfile.sync( - &lock_from(&args.artifacts.test), - &LockedPackageLockType::Test, - ); - - Ok(()) - } -} - -fn lock_from( - packages: &HashMap, -) -> LockedPackageLock { - // FIXME(vhyrro): Create a constructor here instead of mut overrides. - let mut lock = LockedPackageLock::default(); - for package in packages.values() { - lock.insert(package.package.clone(), package.entry_type.is_entrypoint()); - } - lock -} - -#[derive(Error, Debug, Diagnostic)] -#[non_exhaustive] -pub(crate) enum EmitError { - #[error("failed to serialize the lockfile")] - Serialize, -} - -#[cfg(all(test, feature = "impure_tests"))] -mod tests { - use std::path::PathBuf; - - use assert_fs::prelude::PathCopy; - - use crate::{config::ConfigBuilder, lua_version::LuaVersion, workspace::Workspace}; - - use super::*; - - #[tokio::test] - async fn pipeline_installs_workspace_dependencies() { - let sample = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("resources/test/sample-projects/dependencies/"); - let temp = assert_fs::TempDir::new().unwrap(); - temp.copy_from(sample, &["**"]).unwrap(); - - let workspace = Workspace::from_exact(temp.path()).unwrap().unwrap(); - let config = ConfigBuilder::new() - .unwrap() - .lua_version(Some(LuaVersion::Lua51)) - .build() - .unwrap(); - - Pipeline::new(&config, &workspace).run().await.unwrap(); - - let lockfile = workspace.lockfile().unwrap(); - let rocks = lockfile.rocks(&LockedPackageLockType::Regular); - assert!(rocks - .values() - .any(|pkg| pkg.name().to_string() == "lua-cjson")); - assert!(rocks - .values() - .any(|pkg| pkg.name().to_string() == "plenary.nvim")); - } -} +pub mod install_packages; +pub mod install_workspace; diff --git a/lux-lib/src/operations/pipeline/resolve.rs b/lux-lib/src/operations/pipeline/resolve.rs index e83a7098a..bba3a8153 100644 --- a/lux-lib/src/operations/pipeline/resolve.rs +++ b/lux-lib/src/operations/pipeline/resolve.rs @@ -4,11 +4,12 @@ use std::rc::Rc; use async_recursion::async_recursion; use bon::Builder; +use itertools::Itertools; use miette::Diagnostic; use thiserror::Error; use crate::{ - config::Config, lockfile::{LockedPackage, LockedPackageHashes, LockedPackageId, LockedPackageSpec, RemotePackageSourceUrl}, lua_rockspec::RemoteLuaRockspec, operations::{PackageInstallSpec, RemoteRockDownload, resolve::build_dependencies_to_install}, package::{PackageName, PackageReq, RemotePackage}, remote_package_source::RemotePackageSource, rockspec::Rockspec, tree::EntryType, + config::Config, lockfile::{LockedPackageId, LockedPackageSpec, RemotePackageSourceUrl}, lua_rockspec::{BuildBackendSpec, RemoteLuaRockspec}, operations::{PackageInstallSpec, RemoteRockDownload}, package::{PackageName, PackageReq, RemotePackage}, remote_package_source::RemotePackageSource, rockspec::Rockspec, tree::EntryType, }; use super::{ @@ -16,6 +17,40 @@ use super::{ download_sources_and_hash::PackageSource, }; +/// The build dependencies of a rockspec that still need to be installed, with the +/// LuaRocks build backend (if any) first. +pub(crate) fn build_dependencies_to_install(rockspec: &R) -> Vec { + let mut names = rockspec + .build_dependencies() + .current_platform() + .iter() + .filter(|dep| { + !matches!( + dep.name().to_string().as_str(), + "luarocks-build-rust-mlua" + | "luarocks-build-rust-binary" + | "luarocks-build-treesitter-parser" + ) + }) + .map(|dep| dep.name().clone()) + .collect_vec(); + + if let Some(backend) = luarocks_build_backend_name(rockspec) { + names.insert(0, backend); + } + names +} + +/// The name of the luarocks build backend rock required to build this rockspec (if any). +pub(crate) fn luarocks_build_backend_name(rockspec: &R) -> Option { + match &rockspec.build().current_platform().build_backend { + Some(BuildBackendSpec::LuaRock(backend)) => { + Some(PackageName::new(format!("luarocks-build-{backend}"))) + } + _ => None, + } +} + pub(crate) struct ResolvedPackage { pub(crate) spec: LockedPackageSpec, pub(crate) rockspec: RemoteLuaRockspec, @@ -25,21 +60,6 @@ pub(crate) struct ResolvedPackage { pub(crate) artifact: Option, } -impl ResolvedPackage { - pub(crate) fn id(&self) -> LockedPackageId { - self.spec.id() - } - - pub(crate) fn with_hashes(self, hashes: LockedPackageHashes) -> LockedPackage { - LockedPackage { - spec: self.spec, - source: self.source, - source_url: self.source_url, - hashes, - } - } -} - #[derive(Default)] pub(crate) struct ResolvedArtifacts { pub(crate) regular: HashMap, @@ -55,8 +75,6 @@ pub(crate) enum ResolveError { Discover(#[from] DiscoverError), #[error("cyclic dependency detected:\n{0}")] CyclicDependency(String), - #[error("discovery returned a precomputed closure, which this resolver does not support")] - UnsupportedClosure, #[error(transparent)] Join(#[from] tokio::task::JoinError), } @@ -118,11 +136,6 @@ where self } - pub(crate) fn package(mut self, package: PackageInstallSpec) -> Self { - self.packages.push(package); - self - } - pub(crate) fn build_packages(mut self, packages: Vec) -> Self { self.build_packages = packages; self @@ -132,11 +145,6 @@ where self.test_packages = packages; self } - - pub(crate) fn test_package(mut self, package: PackageInstallSpec) -> Self { - self.test_packages.push(package); - self - } } impl ResolvePackageDependenciesBuilder<'_, D, State> @@ -344,10 +352,9 @@ async fn find_package_from_provider( // inside this recursive resolver's call chain. let discover = discover.clone(); let package = spec.package.clone(); - match tokio::spawn(async move { discover.find(&package).await }).await?? { - FoundPackageType::Package(package) => Ok(package), - FoundPackageType::Closure { .. } => Err(ResolveError::UnsupportedClosure), - } + let FoundPackageType::Package(package) = + tokio::spawn(async move { discover.find(&package).await }).await??; + Ok(package) } fn build_dependency_specs( diff --git a/lux-lib/src/operations/resolve.rs b/lux-lib/src/operations/resolve.rs deleted file mode 100644 index 4f3e22a77..000000000 --- a/lux-lib/src/operations/resolve.rs +++ /dev/null @@ -1,377 +0,0 @@ -use std::{fmt::Display, sync::Arc}; - -use async_recursion::async_recursion; -use bon::Builder; -use futures::StreamExt; -use itertools::Itertools; -use miette::Diagnostic; -use thiserror::Error; -use tokio::sync::mpsc::UnboundedSender; -use tracing::Instrument; - -use crate::{ - build::BuildBehaviour, - config::Config, - lockfile::{ - LockedPackageId, LockedPackageSpec, Lockfile, LockfilePermissions, - }, - lua_rockspec::BuildBackendSpec, - operations::{FetchVendored, FetchVendoredError}, - package::{PackageName, PackageReq}, - remote_package_db::RemotePackageDB, - rockspec::Rockspec, - tree, -}; - -use super::{Download, PackageInstallSpec, RemoteRockDownload, SearchAndDownloadError}; - -#[derive(Error, Debug, Diagnostic)] -#[non_exhaustive] -pub enum ResolveDependenciesError { - #[error(transparent)] - #[diagnostic(transparent)] - SearchAndDownload(#[from] Box), - #[error("cyclic dependency detected:\n{0}")] - CyclicDependency(DependencyCycle), - #[error("error processing resolved dependency:\n{0}")] - ChannelSend(String), - #[error("error fetching vendored dependency '{0}'")] - FetchVendored(PackageReq, #[source] Box), -} - -impl From for ResolveDependenciesError { - fn from(source: SearchAndDownloadError) -> Self { - Self::SearchAndDownload(Box::new(source)) - } -} - -#[derive(Debug)] -pub struct DependencyCycle(Vec); - -impl Display for DependencyCycle { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - self.0.iter().join(" -> ").fmt(f) - } -} - -#[derive(Clone, Debug)] -pub(crate) struct PackageInstallData { - pub downloaded_rock: RemoteRockDownload, - pub spec: LockedPackageSpec, -} - -#[derive(Builder)] -#[builder(start_fn = new, finish_fn(name = _build, vis = ""))] -pub(crate) struct Resolve<'a, P> -where - P: LockfilePermissions + Send + Sync + 'static, -{ - dependencies_tx: UnboundedSender, - build_dependencies_tx: UnboundedSender, - packages: Vec, - package_db: Arc, - parent_packages: Option>>, - lockfile: Option>>, - build_lockfile: Option>>, - config: &'a Config, -} - -impl ResolveBuilder<'_, P, State> -where - P: LockfilePermissions + Send + Sync + 'static, - State: resolve_builder::State + resolve_builder::IsComplete, -{ - pub(crate) async fn get_all_dependencies( - self, - ) -> Result, ResolveDependenciesError> { - let args = self._build(); - do_get_all_dependencies(args).await - } -} - -/// The names of the build dependencies to install, including the -/// build backend rock (if any), excluding the luarocks build backends -/// that Lux implements natively. -pub(crate) fn build_dependencies_to_install(rockspec: &R) -> Vec { - let mut names = rockspec - .build_dependencies() - .current_platform() - .iter() - .filter(|dep| { - !matches!( - dep.name().to_string().as_str(), - "luarocks-build-rust-mlua" - | "luarocks-build-rust-binary" - | "luarocks-build-treesitter-parser" - ) - }) - .map(|dep| dep.name().clone()) - .collect_vec(); - - if let Some(backend) = luarocks_build_backend_name(rockspec) { - names.insert(0, backend); - } - names -} - -/// The name of the luarocks build backend rock required to build this rockspec (if any). -pub(crate) fn luarocks_build_backend_name(rockspec: &R) -> Option { - match &rockspec.build().current_platform().build_backend { - Some(BuildBackendSpec::LuaRock(backend)) => { - Some(PackageName::new(format!("luarocks-build-{backend}"))) - } - _ => None, - } -} - -#[tracing::instrument(name = "Resolving dependencies", skip_all)] -#[async_recursion] -async fn do_get_all_dependencies<'a, P>( - args: Resolve<'a, P>, -) -> Result, ResolveDependenciesError> -where - 'a: 'async_recursion, - P: LockfilePermissions + Send + Sync + 'static, -{ - let dependencies_tx = args.dependencies_tx; - let build_dependencies_tx = args.build_dependencies_tx; - let packages = args.packages; - let parent_packages = args.parent_packages.unwrap_or_default(); - let package_db = args.package_db; - let lockfile = args.lockfile; - let build_lockfile = args.build_lockfile; - let config = args.config; - futures::stream::iter( - packages - .into_iter() - // If there is a lockfile, exclude packages that are already installed - .filter( - |PackageInstallSpec { - package, - build_behaviour, - .. - }| { - *build_behaviour == BuildBehaviour::Force - || lockfile - .as_ref() - .is_none_or(|lockfile| lockfile.has_rock(package, None).is_none()) - }, - ) - .map( - // NOTE: we propagate build_behaviour, pin and opt to all dependencies - |PackageInstallSpec { - package, - build_behaviour, - pin, - opt, - constraint, - source, - .. - }| { - let config = config.clone(); - let dependencies_tx = dependencies_tx.clone(); - let build_dependencies_tx = build_dependencies_tx.clone(); - let parent_packages = Arc::clone(&parent_packages); - let package_db = Arc::clone(&package_db); - let lockfile = match &lockfile { - Some(lockfile) => Some(Arc::clone(lockfile)), - None => None, - }; - let build_lockfile = match &build_lockfile { - Some(lockfile) => Some(Arc::clone(lockfile)), - None => None, - }; - - tokio::spawn( - async move { - let downloaded_rock = if let Some(source) = source { - RemoteRockDownload::from_package_req_and_source_spec( - package.clone(), - source, - )? - } else if let Some(vendor_dir) = config.vendor_dir() { - FetchVendored::new() - .vendor_dir(vendor_dir) - .package(&package) - .package_db(&package_db) - .fetch_vendored_rock() - .await - .map_err(|err| { - ResolveDependenciesError::FetchVendored( - package.clone(), - Box::new(err), - ) - })? - } else { - Download::new(&package, &config) - .package_db(&package_db) - .download_remote_rock() - .await? - }; - - let constraint = - constraint.unwrap_or(package.version_req().clone().into()); - - let rockspec = downloaded_rock.rockspec(); - - if parent_packages.contains(rockspec.package()) { - return Err(ResolveDependenciesError::CyclicDependency( - DependencyCycle( - parent_packages - .iter() - .cloned() - .chain(std::iter::once(rockspec.package().clone())) - .collect_vec(), - ), - )); - } - - // NOTE: We don't need to install build dependencies to install binary rocks. - let build_dependencies = if !matches!( - downloaded_rock, - RemoteRockDownload::BinaryRock { .. } - ) { - let packages = build_dependencies_to_install(rockspec) - .into_iter() - .filter_map(|name| { - rockspec - .build_dependencies() - .current_platform() - .iter() - .find(|dep| dep.name() == &name) - .map(|dep| { - // We always install build dependencies as entrypoints - // with regard to the build tree - let entry_type = tree::EntryType::Entrypoint; - PackageInstallSpec::new( - dep.package_req().clone(), - entry_type, - ) - .build_behaviour(build_behaviour) - .pin(pin) - .opt(opt) - .maybe_source(dep.source().clone()) - .build() - }) - // The build backend rock (e.g. `luarocks-build-foo`) - // is not declared as a build dependency in the rockspec. - .or_else(|| { - PackageInstallSpec::new( - PackageReq::from(name), - tree::EntryType::Entrypoint, - ) - .build_behaviour(build_behaviour) - .pin(pin) - .opt(opt) - .build() - .into() - }) - }) - .collect_vec(); - - // NOTE: We treat transitive regular dependencies of build dependencies - // as build dependencies - Resolve::new() - .dependencies_tx(build_dependencies_tx.clone()) - .build_dependencies_tx(build_dependencies_tx.clone()) - .packages(packages) - .parent_packages(Arc::new( - parent_packages - .iter() - .cloned() - .chain(std::iter::once(rockspec.package().clone())) - .collect_vec(), - )) - .package_db(package_db.clone()) - .maybe_lockfile(build_lockfile.clone()) - .maybe_build_lockfile(build_lockfile.clone()) - .config(&config) - .get_all_dependencies() - .await? - } else { - Vec::new() - }; - - let dependencies = rockspec - .dependencies() - .current_platform() - .iter() - .map(|dep| { - // If we're forcing a rebuild, retain the `EntryType` - // of existing dependencies - let entry_type = if build_behaviour == BuildBehaviour::Force - && lockfile.as_ref().is_some_and(|lockfile| { - let installed_rock = - lockfile.has_rock(dep.package_req(), None); - installed_rock.is_some_and(|installed_rock| { - lockfile.is_entrypoint(&installed_rock.id()) - }) - }) { - tree::EntryType::Entrypoint - } else { - tree::EntryType::DependencyOnly - }; - - PackageInstallSpec::new(dep.package_req().clone(), entry_type) - .build_behaviour(build_behaviour) - .pin(pin) - .opt(opt) - .maybe_source(dep.source().clone()) - .build() - }) - .collect_vec(); - - let dependencies = Resolve::new() - .dependencies_tx(dependencies_tx.clone()) - .build_dependencies_tx(build_dependencies_tx) - .packages(dependencies) - .parent_packages(Arc::new( - parent_packages - .iter() - .cloned() - .chain(std::iter::once(rockspec.package().clone())) - .collect_vec(), - )) - .package_db(package_db) - .maybe_lockfile(lockfile) - .maybe_build_lockfile(build_lockfile) - .config(&config) - .get_all_dependencies() - .await?; - - let rockspec = downloaded_rock.rockspec(); - let local_spec = LockedPackageSpec::new( - rockspec.package(), - rockspec.version(), - constraint, - dependencies, - build_dependencies, - &pin, - &opt, - rockspec.binaries(), - ); - - let install_spec = PackageInstallData { - spec: local_spec.clone(), - downloaded_rock, - }; - - dependencies_tx.send(install_spec).map_err(|err| { - ResolveDependenciesError::ChannelSend(err.to_string()) - })?; - - Ok::<_, ResolveDependenciesError>(local_spec.id()) - } - .instrument(tracing::trace_span!("resolve_worker")), - ) - }, - ), - ) - .buffered(config.max_jobs()) - .collect::>() - .instrument(tracing::trace_span!("resolve_collector")) - .await - .into_iter() - .flatten() - .try_collect() -} diff --git a/lux-lib/src/operations/sync.rs b/lux-lib/src/operations/sync.rs index 89a40d690..0f50fb73a 100644 --- a/lux-lib/src/operations/sync.rs +++ b/lux-lib/src/operations/sync.rs @@ -11,7 +11,7 @@ use crate::{ WorkspaceLockfile, }, luarocks::luarocks_installation::LUAROCKS_VERSION, - operations::{self, GenLuaRcError}, + operations::{self, pipeline::resolve::luarocks_build_backend_name, GenLuaRcError}, package::{PackageName, PackageReq}, project::{project_toml::LocalProjectTomlValidationError, ProjectError}, rockspec::{lua_dependency::LuaDependencySpec, Rockspec}, @@ -285,7 +285,7 @@ impl<'a, 'b> SyncPackages<'a, 'b> { LockedPackageLockType::Build => { for project in self.args.workspace.members() { let toml = project.toml().into_local()?; - if let Some(backend) = operations::resolve::luarocks_build_backend_name(&toml) { + if let Some(backend) = luarocks_build_backend_name(&toml) { extra_packages.push(backend.into()); if cfg!(target_family = "unix") { let luarocks = unsafe { diff --git a/lux-lib/src/operations/update.rs b/lux-lib/src/operations/update.rs index ddf29a45b..69cc17be8 100644 --- a/lux-lib/src/operations/update.rs +++ b/lux-lib/src/operations/update.rs @@ -116,8 +116,6 @@ impl UpdateBuilder<'_, State> { impl UpdateBuilder<'_, State> { #[tracing::instrument(name = "Updating packages", skip_all)] - - /// Returns the packages that were installed or removed pub async fn update(self) -> Result, UpdateError> where State: update_builder::IsComplete, @@ -146,8 +144,8 @@ impl UpdateBuilder<'_, State> { }; match workspace { - Some(workspace) => update_workspace(workspace, args, package_db).await, - None => update_install_tree(args, package_db).await, + Some(workspace) => Box::pin(update_workspace(workspace, args, package_db)).await, + None => Box::pin(update_install_tree(args, package_db)).await, } } } diff --git a/lux-lib/src/operations/vendor.rs b/lux-lib/src/operations/vendor.rs index 55c7427ec..2b584a187 100644 --- a/lux-lib/src/operations/vendor.rs +++ b/lux-lib/src/operations/vendor.rs @@ -6,33 +6,36 @@ use std::{ }; use bon::Builder; -use bytes::Bytes; use futures::StreamExt; +use futures::TryStreamExt; use itertools::Itertools; use miette::Diagnostic; use path_slash::PathExt; use strum::IntoEnumIterator; use thiserror::Error; use tokio::io::AsyncWriteExt; -use tracing::Instrument; use crate::{ - build::{resolve_source_dir, RemotePackageSourceSpec, SrcRockSource}, + build::resolve_source_dir, config::Config, fs, - lockfile::{LockedPackageLockType, ReadOnly, RemotePackageSourceUrl}, + lockfile::LockedPackageLockType, lua_rockspec::{BuildBackendSpec, RemoteLuaRockspec}, operations::{ self, - resolve::{ - luarocks_build_backend_name, PackageInstallData, Resolve, ResolveDependenciesError, + pipeline::{ + discover::FindPackageFromLuarocks, + download_sources_and_hash::{ + DownloadSourcesAndHash, DownloadSourcesAndHashArtifacts, DownloadedPackage, + PackageSource, + }, + resolve::{luarocks_build_backend_name, ResolvePackageDependencies}, }, - DownloadedRockspec, FetchSrcError, PackageInstallSpec, UnpackError, + PackageInstallSpec, UnpackError, }, package::{PackageReq, PackageSpec}, project::project_toml::LocalProjectTomlValidationError, remote_package_db::{RemotePackageDB, RemotePackageDBError}, - reqwest::RequestError, rockspec::Rockspec, tree::EntryType, workspace::{Workspace, WorkspaceError}, @@ -81,7 +84,8 @@ pub enum VendorError { RemotePackageDB(#[from] RemotePackageDBError), #[error("failed to resolve dependencies")] #[diagnostic(forward(0))] - ResolveDependencies(#[from] Box), + // FIXME(vhyrro): Don't use dyn here + Pipeline(Box), #[error(transparent)] #[diagnostic(transparent)] Fs(#[from] fs::FsError), @@ -92,10 +96,7 @@ pub enum VendorError { Unpack(#[from] UnpackError), #[error("failed to fetch rock source")] #[diagnostic(forward(0))] - FetchSrc(#[from] FetchSrcError), - #[error("failed to download rock source")] - #[diagnostic(forward(0))] - Request(#[from] RequestError), + FetchSrc(#[from] crate::operations::FetchSrcError), #[error("failed to run `cargo vendor`")] #[diagnostic(help("ensure cargo is installed"))] CargoVendor { source: io::Error }, @@ -108,12 +109,6 @@ pub enum VendorError { }, } -impl From for VendorError { - fn from(source: ResolveDependenciesError) -> Self { - Self::ResolveDependencies(Box::new(source)) - } -} - impl VendorBuilder<'_, State> where State: vendor_builder::State + vendor_builder::IsComplete, @@ -125,70 +120,148 @@ where const CARGO_VENDOR_SUBDIR: &str = "cargo"; +/// The set of packages to vendor, resolved and materialized into sources. +struct VendoredPackages { + packages: Vec, +} + +impl VendoredPackages { + /// Resolves and materializes every dependency of the target, across all lock types. + async fn new( + target: &VendorTarget, + no_lock: bool, + config: &Config, + ) -> Result { + let mut packages = Vec::new(); + for lock_type in LockedPackageLockType::iter() { + let (package_db, install_specs) = + gather_install_specs(&lock_type, no_lock, target, config).await?; + packages.extend( + ResolveAndDownload::new(config, package_db, install_specs) + .run() + .await?, + ); + } + + // The lockfile may contain the same package (name@version) multiple times, + // with different constraints. + let packages = packages + .into_iter() + .unique_by(|pkg| (pkg.package.spec.name().clone(), pkg.package.spec.version().clone())) + .collect(); + + Ok(Self { packages }) + } + + /// Vendors the sources of all packages into `vendor_dir`. + async fn vendor_sources( + &self, + vendor_dir: &Path, + config: &Config, + ) -> Result<(), VendorError> { + futures::stream::iter( + self.packages + .iter() + .map(|package| vendor_package_sources(vendor_dir, package)), + ) + .buffered(config.max_jobs()) + .try_collect() + .await + } + + /// Cargo-based build backends need their Cargo dependencies vendored too. + fn cargo_dependencies(&self) -> Vec<(PackageSpec, Option, Vec)> { + self.packages + .iter() + .filter_map(|package| { + let rockspec = &package.rockspec; + match rockspec.build().current_platform().build_backend { + Some(BuildBackendSpec::RustMlua(_) | BuildBackendSpec::RustBinary(_)) => Some(( + package.package.spec.to_package(), + rockspec.source().current_platform().unpack_dir.clone(), + rockspec.build().current_platform().copy_directories.clone(), + )), + _ => None, + } + }) + .collect() + } +} + +/// Resolves a set of packages and downloads (and hashes) their sources. +struct ResolveAndDownload<'a> { + config: &'a Config, + package_db: RemotePackageDB, + install_specs: Vec, +} + +impl<'a> ResolveAndDownload<'a> { + fn new( + config: &'a Config, + package_db: RemotePackageDB, + install_specs: Vec, + ) -> Self { + Self { + config, + package_db, + install_specs, + } + } + + async fn run(self) -> Result, VendorError> { + let discover = FindPackageFromLuarocks::new( + Arc::new(self.package_db), + Arc::new(self.config.clone()), + ) + .build(); + let resolved = ResolvePackageDependencies::new(&discover, self.config) + .packages(self.install_specs) + .resolve() + .await + .map_err(|err| VendorError::Pipeline(Box::new(err)))?; + let artifacts = DownloadSourcesAndHash::new(self.config) + .resolved(resolved) + .download_sources_and_hash() + .await + .map_err(|err| VendorError::Pipeline(Box::new(err)))?; + let DownloadSourcesAndHashArtifacts { + regular, + build, + test, + } = artifacts; + Ok(regular + .into_values() + .chain(build.into_values()) + .chain(test.into_values()) + .collect()) + } +} + async fn do_vendor_dependencies(args: Vendor<'_>) -> Result<(), VendorError> { let vendor_dir = args.vendor_dir; let no_delete = args.no_delete.unwrap_or(false); let no_lock = args.no_lock.unwrap_or(false); let target = args.target; let config = args.config; - let mut all_packages = Vec::new(); - - for lock_type in LockedPackageLockType::iter() { - let (package_db, install_specs) = - mk_resolve_args(lock_type, no_lock, &target, config).await?; - - let (dep_tx, mut dep_rx) = tokio::sync::mpsc::unbounded_channel(); - Resolve::<'_, ReadOnly>::new() - .dependencies_tx(dep_tx.clone()) - .build_dependencies_tx(dep_tx) - .packages(install_specs) - .package_db(Arc::new(package_db)) - .config(config) - .get_all_dependencies() - .await?; - - while let Some(dep) = dep_rx.recv().await { - all_packages.push(dep); - } - } - // The lockfile may contain the same package (name@version) multiple times, - // with different constraints. - let all_packages = all_packages - .into_iter() - .unique_by(|pkg| (pkg.spec.name().clone(), pkg.spec.version().clone())) - .collect_vec(); - - let cargo_deps: Vec<(PackageSpec, Option, Vec)> = all_packages - .iter() - .filter_map(|pkg| { - let rockspec = pkg.downloaded_rock.rockspec(); - match rockspec.build().current_platform().build_backend { - Some(BuildBackendSpec::RustMlua(_) | BuildBackendSpec::RustBinary(_)) => Some(( - pkg.spec.to_package(), - rockspec.source().current_platform().unpack_dir.clone(), - rockspec.build().current_platform().copy_directories.clone(), - )), - _ => None, - } - }) - .collect(); + let vendored = VendoredPackages::new(&target, no_lock, config).await?; + + let cargo_deps = vendored.cargo_dependencies(); if !no_delete && vendor_dir.exists() { fs::tokio::remove_dir_all(&vendor_dir).await?; } let vendor_dir = Arc::new(vendor_dir); - vendor_sources(vendor_dir.clone(), config.clone(), all_packages).await?; + vendored.vendor_sources(&vendor_dir, config).await?; vendor_target_cargo_deps(&vendor_dir, &target, config).await?; for (dep, unpack_dir, copy_dirs) in cargo_deps { vendor_package_cargo_deps(&vendor_dir, &dep, &unpack_dir, ©_dirs, config).await?; } - Ok(()) -} + Ok(())} -async fn mk_resolve_args( - lock_type: LockedPackageLockType, +async fn gather_install_specs( + lock_type: &LockedPackageLockType, no_lock: bool, target: &VendorTarget, config: &Config, @@ -206,8 +279,8 @@ async fn mk_resolve_args( let mut install_specs = Vec::new(); for project in workspace.members() { let toml = project.toml().into_local()?; - push_dependencies(&lock_type, &toml, &mut install_specs)?; - if lock_type == LockedPackageLockType::Test { + push_dependencies(lock_type, &toml, &mut install_specs)?; + if *lock_type == LockedPackageLockType::Test { for test_spec_dependency in toml .test() .current_platform() @@ -225,7 +298,7 @@ async fn mk_resolve_args( VendorTarget::Rockspec(remote_lua_rockspec) => { let package_db = RemotePackageDB::from_config(config).await?; let mut install_specs = Vec::new(); - push_dependencies(&lock_type, remote_lua_rockspec, &mut install_specs)?; + push_dependencies(lock_type, remote_lua_rockspec, &mut install_specs)?; Ok((package_db, install_specs)) } } @@ -271,161 +344,53 @@ fn push_dependencies( Ok(()) } -async fn vendor_sources( - vendor_dir: Arc, - config: Config, - packages: Vec, -) -> Result<(), VendorError> { - futures::stream::iter(packages.into_iter().map(|dep| { - let vendor_dir = Arc::clone(&vendor_dir); - let config = config.clone(); - tokio::spawn( - async move { - match dep.downloaded_rock { - crate::operations::RemoteRockDownload::RockspecOnly { rockspec_download } => { - vendor_rockspec_sources(&vendor_dir, rockspec_download, None, &config) - .await? - } - crate::operations::RemoteRockDownload::BinaryRock { - rockspec_download, - packed_rock, - } => vendor_binary_rock(&vendor_dir, rockspec_download, packed_rock).await?, - crate::operations::RemoteRockDownload::SrcRock { - rockspec_download, - src_rock, - source_url, - } => { - let src_rock_source = SrcRockSource { - bytes: src_rock, - source_url, - }; - vendor_rockspec_sources( - &vendor_dir, - rockspec_download, - Some(src_rock_source), - &config, - ) - .await? - } - }; - Ok::<_, VendorError>(()) - } - .instrument(tracing::trace_span!("vendor_worker")), - ) - })) - .buffered(config.max_jobs()) - .collect::>() - .instrument(tracing::trace_span!("vendor_collector")) - .await - .into_iter() - .flatten() - .try_collect() -} - +/// Vendors the materialized source and rockspec of a single package. #[tracing::instrument( name = "Vendoring source", level = "info", skip_all, fields( - package = rockspec_download.rockspec.package().to_string(), - version = rockspec_download.rockspec.version().to_string(), + package = package.package.spec.name().to_string(), + version = package.package.spec.version().to_string(), ), )] -async fn vendor_rockspec_sources( +async fn vendor_package_sources( vendor_dir: &Path, - rockspec_download: DownloadedRockspec, - src_rock_source: Option, - config: &Config, + package: &DownloadedPackage, ) -> Result<(), VendorError> { - let rockspec = rockspec_download.rockspec; - let package = rockspec.package(); + let rockspec = &package.rockspec; + let name = rockspec.package(); let version = rockspec.version(); - let package_version_str = format!("{}@{}", package, version); - - let source_spec = match src_rock_source { - Some(src_rock_source) => RemotePackageSourceSpec::SrcRock(src_rock_source), - None => RemotePackageSourceSpec::RockSpec(rockspec_download.source_url), - }; - - let source_path = vendor_dir.join(&package_version_str); fs::tokio::create_dir_all(vendor_dir).await?; let rockspec_lua_content = rockspec .to_lua_remote_rockspec_string() .map_err(|err| VendorError::LuaRockSpec(err.to_string()))?; - - let rockspec_file_name = format!("{}-{}.rockspec", package, version); - let rockspec_path = vendor_dir.join(rockspec_file_name); + let rockspec_path = vendor_dir.join(format!("{}-{}.rockspec", name, version)); fs::tokio::write(&rockspec_path, rockspec_lua_content).await?; - match source_spec { - RemotePackageSourceSpec::SrcRock(SrcRockSource { - bytes, - source_url: _, - }) => { - fs::tokio::write(&source_path, &bytes).await?; + let source_path = vendor_dir.join(format!("{}@{}", name, version)); + match &package.artifact { + // A fully materialized source tree. + PackageSource::SourceTree(dir) => { + fs::tokio::remove_dir_all(&source_path).await.ok(); + fs::tokio::copy_dir_all(dir.path(), &source_path).await?; + } + // A pre-built binary rock. + PackageSource::PackedRock(bytes) => { + let rock_path = vendor_dir.join(format!("{}@{}.rock", name, version)); + let mut file = fs::tokio::create(&rock_path).await?; + file.write_all(bytes).await.map_err(|source| fs::FsError::Write { + path: rock_path, + source, + })?; } - RemotePackageSourceSpec::RockSpec(source_url) => match source_url { - Some(RemotePackageSourceUrl::Url { url }) => { - let bytes = crate::reqwest::download_bytes(config, &url).await?; - fs::tokio::write(&source_path, &bytes).await?; - } - _ => { - fs::tokio::create_dir_all(&source_path).await?; - operations::FetchSrc::new(&source_path, &rockspec, config) - .maybe_source_url(source_url) - .fetch_internal() - .await?; - } - }, } Ok(()) } -#[tracing::instrument( - name = "Vendoring pre-built binary", - level = "info", - skip_all, - fields( - package = rockspec_download.rockspec.package().to_string(), - version = rockspec_download.rockspec.version().to_string(), - ), -)] -async fn vendor_binary_rock( - vendor_dir: &Path, - rockspec_download: DownloadedRockspec, - packed_rock: Bytes, -) -> Result<(), VendorError> { - let rockspec = rockspec_download.rockspec; - let package = rockspec.package(); - let version = rockspec.version(); - - let file_name = format!("{}@{}.rock", package, version); - - fs::tokio::create_dir_all(&vendor_dir).await?; - - let dest_file = vendor_dir.join(&file_name); - let mut file = fs::tokio::create(&dest_file).await?; - file.write_all(&packed_rock) - .await - .map_err(|source| fs::FsError::Write { - path: dest_file.to_path_buf(), - source, - })?; - - let rockspec_lua_content = rockspec - .to_lua_remote_rockspec_string() - .map_err(|err| VendorError::LuaRockSpec(err.to_string()))?; - - let rockspec_file_name = format!("{}-{}.rockspec", package, version); - let rockspec_path = vendor_dir.join(rockspec_file_name); - fs::tokio::write(&rockspec_path, rockspec_lua_content).await?; - - Ok(()) -} - #[tracing::instrument(name = "Vendoring cargo dependencies", level = "info", skip_all)] async fn vendor_target_cargo_deps( vendor_dir: &Path, @@ -571,21 +536,18 @@ mod tests { use std::path::PathBuf; use bytes::Bytes; + use tempfile::TempDir; use super::*; use crate::{ config::ConfigBuilder, - lockfile::{LockConstraint, LockedPackageSpec, OptState, PinnedState, RemotePackageSourceUrl}, - lua_rockspec::RockSourceSpec, - operations::{unpack_rockspec, DownloadedPackedRockBytes, DownloadedRockspec}, - package::{PackageName, PackageVersion}, + lockfile::{LockConstraint, LockedPackage, LockedPackageSpec, OptState, PinnedState}, + operations::unpack_rockspec, remote_package_source::RemotePackageSource, rockspec::RockBinaries, }; use assert_fs::prelude::PathCopy; - use crate::operations::RemoteRockDownload; - fn test_config() -> Config { ConfigBuilder::new() .unwrap() @@ -601,12 +563,12 @@ mod tests { Bytes::from(std::fs::read(path).unwrap()) } - /// Builds a `PackageInstallData` for a source-tree-only package from a generated rockspec. - fn make_install_data( + /// Builds a `DownloadedPackage` for a materialized source tree. + fn make_downloaded_package( rockspec: RemoteLuaRockspec, - source_url: Option, source: RemotePackageSource, - ) -> PackageInstallData { + artifact: PackageSource, + ) -> DownloadedPackage { let spec = LockedPackageSpec::new( rockspec.package(), rockspec.version(), @@ -617,48 +579,49 @@ mod tests { &OptState::Required, RockBinaries::default(), ); - let rockspec_download = DownloadedRockspec { - rockspec, + let package = LockedPackage { + spec, source, - source_url, + source_url: None, + hashes: crate::lockfile::LockedPackageHashes { + rockspec: "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + .parse() + .unwrap(), + source: "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + .parse() + .unwrap(), + }, }; - PackageInstallData { - downloaded_rock: RemoteRockDownload::RockspecOnly { rockspec_download }, - spec, + DownloadedPackage { + package, + rockspec, + entry_type: EntryType::Entrypoint, + artifact, } } - /// Vendors a `RockspecOnly`(url) dependency and asserts the rockspec and source - /// tree are laid out as expected. + /// Vendors a package with a materialized source tree: the rockspec and the + /// source tree directory must both be written. #[tokio::test] - async fn vendor_rockspec_only_url_writes_rockspec_and_source_tree() { - let config = test_config(); + async fn vendor_source_tree_writes_rockspec_and_source_dir() { let vendor_dir = assert_fs::TempDir::new().unwrap(); - // A local `File` source so no network is needed. `FetchSrc` copies the - // directory into the vendored source path. - let src_dir = assert_fs::TempDir::new().unwrap(); + // A source tree that has already been materialized by the pipeline. + let src_dir = TempDir::new().unwrap(); std::fs::write(src_dir.path().join("hello.lua"), "return 'hello'").unwrap(); let rockspec = RemoteLuaRockspec::from_package_and_source_spec( "rockspec-only-url@1.0.0".parse().unwrap(), - RockSourceSpec::File(src_dir.path().to_path_buf()), + crate::lua_rockspec::RockSourceSpec::File(src_dir.path().to_path_buf()), ); - - let install_data = make_install_data( + let package = make_downloaded_package( rockspec, - Some(RemotePackageSourceUrl::File { - path: src_dir.path().to_path_buf(), - }), RemotePackageSource::RockspecContent(String::new()), + PackageSource::SourceTree(src_dir), ); - vendor_sources( - Arc::new(vendor_dir.to_path_buf()), - config, - vec![install_data], - ) - .await - .unwrap(); + vendor_package_sources(vendor_dir.path(), &package) + .await + .unwrap(); let rockspec_file = vendor_dir.path().join("rockspec-only-url-1.0.0-1.rockspec"); assert!(rockspec_file.is_file(), "rockspec not vendored"); @@ -676,14 +639,13 @@ mod tests { ); } - /// Vendors a `BinaryRock` dependency: both the packed `.rock` and its - /// extracted rockspec must be written. + /// Vendors a binary rock: both the packed `.rock` and its rockspec must be written. #[tokio::test] async fn vendor_binary_rock_writes_rock_and_rockspec() { let vendor_dir = assert_fs::TempDir::new().unwrap(); let bytes = fixture_bytes("toml-edit-0.6.0-1.linux-x86_64.rock"); - let rock = DownloadedPackedRockBytes { + let rock = crate::operations::DownloadedPackedRockBytes { name: "toml-edit".into(), version: "0.6.0-1".parse().unwrap(), bytes: bytes.clone(), @@ -693,20 +655,15 @@ mod tests { .unwrap(), }; let rockspec = unpack_rockspec(&rock).await.unwrap(); + let package = make_downloaded_package( + rockspec, + RemotePackageSource::LuarocksBinaryRock("https://example.org/".parse().unwrap()), + PackageSource::PackedRock(bytes), + ); - vendor_binary_rock( - vendor_dir.path(), - DownloadedRockspec { - rockspec, - source: RemotePackageSource::LuarocksBinaryRock( - "https://example.org/".parse().unwrap(), - ), - source_url: None, - }, - bytes, - ) - .await - .unwrap(); + vendor_package_sources(vendor_dir.path(), &package) + .await + .unwrap(); assert!( vendor_dir.path().join("toml-edit@0.6.0-1.rock").is_file(), @@ -716,82 +673,40 @@ mod tests { assert!(rockspec_file.is_file(), "rockspec not vendored"); let rockspec_content = std::fs::read_to_string(&rockspec_file).unwrap(); assert!( - rockspec_content.contains("package = "), + rockspec_content.contains("package = ") && rockspec_content.contains("toml-edit"), "vendored rockspec is not valid:\n{rockspec_content}" ); - assert!( - rockspec_content.contains("toml-edit"), - "vendored rockspec is not for toml-edit:\n{rockspec_content}" - ); } - /// Vendors a `SrcRock` dependency. The raw `.src.rock` bytes must be written - /// verbatim to `@`. + /// A pre-existing vendored source directory must be replaced, not merged. #[tokio::test] - async fn vendor_src_rock_writes_bytes_verbatim() { - let config = test_config(); + async fn vendor_source_tree_replaces_stale_source_dir() { let vendor_dir = assert_fs::TempDir::new().unwrap(); + let stale_dir = vendor_dir.path().join("rockspec-only-url@1.0.0-1"); + std::fs::create_dir_all(&stale_dir).unwrap(); + std::fs::write(stale_dir.join("stale.lua"), "return 'stale'").unwrap(); - let bytes = fixture_bytes("luatest-0.2-1.src.rock"); - let rock = DownloadedPackedRockBytes { - name: "luatest".into(), - version: "0.2-1".parse().unwrap(), - bytes: bytes.clone(), - file_name: "luatest-0.2-1.src.rock".into(), - url: "https://example.org/luatest-0.2-1.src.rock" - .parse() - .unwrap(), - }; - let rockspec = unpack_rockspec(&rock).await.unwrap(); - - let src_rock_source = SrcRockSource { - bytes: bytes.clone(), - source_url: RemotePackageSourceUrl::Url { - url: "https://example.org/luatest-0.2-1.src.rock".parse().unwrap(), - }, - }; + let src_dir = TempDir::new().unwrap(); + std::fs::write(src_dir.path().join("hello.lua"), "return 'hello'").unwrap(); + let rockspec = RemoteLuaRockspec::from_package_and_source_spec( + "rockspec-only-url@1.0.0".parse().unwrap(), + crate::lua_rockspec::RockSourceSpec::File(src_dir.path().to_path_buf()), + ); + let package = make_downloaded_package( + rockspec, + RemotePackageSource::RockspecContent(String::new()), + PackageSource::SourceTree(src_dir), + ); - vendor_sources( - Arc::new(vendor_dir.to_path_buf()), - config, - vec![PackageInstallData { - downloaded_rock: RemoteRockDownload::SrcRock { - rockspec_download: DownloadedRockspec { - rockspec, - source: RemotePackageSource::LuarocksSrcRock( - "https://example.org/".parse().unwrap(), - ), - source_url: Some(src_rock_source.source_url.clone()), - }, - src_rock: bytes.clone(), - source_url: src_rock_source.source_url.clone(), - }, - spec: LockedPackageSpec::new( - &PackageName::new("luatest".to_string()), - &"0.2-1".parse::().unwrap(), - LockConstraint::Unconstrained, - Vec::new(), - Vec::new(), - &PinnedState::Unpinned, - &OptState::Required, - RockBinaries::default(), - ), - }], - ) - .await - .unwrap(); + vendor_package_sources(vendor_dir.path(), &package) + .await + .unwrap(); assert!( - vendor_dir.path().join("luatest-0.2-1.rockspec").is_file(), - "rockspec not vendored" - ); - let source_path = vendor_dir.path().join("luatest@0.2-1"); - assert!(source_path.is_file(), ".src.rock not vendored"); - assert_eq!( - std::fs::read(&source_path).unwrap(), - bytes.to_vec(), - "vendored .src.rock bytes differ from source" + !stale_dir.join("stale.lua").exists(), + "stale vendored source was not replaced" ); + assert!(stale_dir.join("hello.lua").is_file()); } /// `no_delete` must be respected: if the vendor dir exists and `no_delete` is diff --git a/lux-lib/src/reqwest.rs b/lux-lib/src/reqwest.rs index 0c04792d4..1349a149c 100644 --- a/lux-lib/src/reqwest.rs +++ b/lux-lib/src/reqwest.rs @@ -1,6 +1,5 @@ use std::sync::OnceLock; -use bytes::Bytes; use miette::Diagnostic; use reqwest::{header::AUTHORIZATION, Client, RequestBuilder}; use thiserror::Error; @@ -110,18 +109,6 @@ pub(crate) fn http_client(config: &Config) -> Result<&Client, reqwest::Error> { client(false, config) } -pub(crate) async fn download_bytes(config: &Config, url: &Url) -> Result { - let bytes = http_client(config)? - .get(url.clone()) - .apply_access_token(config, url) - .send() - .await? - .error_for_status()? - .bytes() - .await?; - Ok(bytes) -} - mod private { pub trait Sealed {} impl Sealed for reqwest::RequestBuilder {} From 32edf77c9e628765b1dcd95a2614e7ec88c7cfba Mon Sep 17 00:00:00 2001 From: vhyrro Date: Sat, 3 Oct 2026 14:22:31 +0200 Subject: [PATCH 05/39] fix: collapse artifact types into single `Artifact` --- lux-lib/src/lockfile/mod.rs | 2 +- .../pipeline/download_sources_and_hash.rs | 44 +++--- .../src/operations/pipeline/emit_lockfile.rs | 20 ++- lux-lib/src/operations/pipeline/mod.rs | 51 ++++++- lux-lib/src/operations/pipeline/resolve.rs | 126 ++++++------------ 5 files changed, 121 insertions(+), 122 deletions(-) diff --git a/lux-lib/src/lockfile/mod.rs b/lux-lib/src/lockfile/mod.rs index 7622d7aee..9808984a0 100644 --- a/lux-lib/src/lockfile/mod.rs +++ b/lux-lib/src/lockfile/mod.rs @@ -834,7 +834,7 @@ pub struct Lockfile { lock: LockedPackageLock, } -#[derive(EnumIter, Debug, PartialEq, Eq)] +#[derive(EnumIter, Debug, Clone, Copy, PartialEq, Eq)] pub enum LockedPackageLockType { Regular, Test, diff --git a/lux-lib/src/operations/pipeline/download_sources_and_hash.rs b/lux-lib/src/operations/pipeline/download_sources_and_hash.rs index 145835fb5..2a95b933e 100644 --- a/lux-lib/src/operations/pipeline/download_sources_and_hash.rs +++ b/lux-lib/src/operations/pipeline/download_sources_and_hash.rs @@ -4,6 +4,7 @@ use std::io::Cursor; use bon::Builder; use bytes::Bytes; +use futures::{StreamExt, TryStreamExt}; use miette::Diagnostic; use tempfile::TempDir; use thiserror::Error; @@ -25,7 +26,10 @@ use crate::{ tree::EntryType, }; -use super::resolve::{ResolvedArtifacts, ResolvedPackage}; +use super::{ + resolve::{ResolvedArtifacts, ResolvedPackage}, + Artifacts, +}; pub(crate) enum PackageSource { SourceTree(TempDir), @@ -39,12 +43,8 @@ pub(crate) struct DownloadedPackage { pub(crate) artifact: PackageSource, } -#[derive(Default)] -pub(crate) struct DownloadSourcesAndHashArtifacts { - pub(crate) regular: HashMap, - pub(crate) build: HashMap, - pub(crate) test: HashMap, -} +pub(crate) type DownloadSourcesAndHashArtifacts = + Artifacts>; #[derive(Error, Debug, Diagnostic)] #[non_exhaustive] @@ -77,22 +77,22 @@ where self, ) -> Result { let args = self._build(); - let mut out = DownloadSourcesAndHashArtifacts::default(); + let config = args.config; - for (id, package) in args.resolved.regular { - out.regular - .insert(id, download_sources_and_hash(args.config, package).await?); - } - for (id, package) in args.resolved.build { - out.build - .insert(id, download_sources_and_hash(args.config, package).await?); - } - for (id, package) in args.resolved.test { - out.test - .insert(id, download_sources_and_hash(args.config, package).await?); - } - - Ok(out) + futures::stream::iter(args.resolved) + .then(|(section, packages)| async move { + let packages = futures::stream::iter(packages) + .then(|(id, package)| async move { + download_sources_and_hash(config, package) + .await + .map(|package| (id, package)) + }) + .try_collect::>() + .await?; + Ok((section, packages)) + }) + .try_collect() + .await } } diff --git a/lux-lib/src/operations/pipeline/emit_lockfile.rs b/lux-lib/src/operations/pipeline/emit_lockfile.rs index 219e396fa..361c4f418 100644 --- a/lux-lib/src/operations/pipeline/emit_lockfile.rs +++ b/lux-lib/src/operations/pipeline/emit_lockfile.rs @@ -24,18 +24,14 @@ where pub(crate) fn emit(self) { let args = self._build(); - args.lockfile.sync( - &lock_from(&args.artifacts.regular), - &LockedPackageLockType::Regular, - ); - args.lockfile.sync( - &lock_from(&args.artifacts.build), - &LockedPackageLockType::Build, - ); - args.lockfile.sync( - &lock_from(&args.artifacts.test), - &LockedPackageLockType::Test, - ); + for section in [ + LockedPackageLockType::Regular, + LockedPackageLockType::Build, + LockedPackageLockType::Test, + ] { + args.lockfile + .sync(&lock_from(args.artifacts.get(section)), §ion); + } } } diff --git a/lux-lib/src/operations/pipeline/mod.rs b/lux-lib/src/operations/pipeline/mod.rs index 56ff61572..8011a6fc3 100644 --- a/lux-lib/src/operations/pipeline/mod.rs +++ b/lux-lib/src/operations/pipeline/mod.rs @@ -1,8 +1,57 @@ +use crate::lockfile::LockedPackageLockType; + pub(crate) mod build; pub(crate) mod build_project; pub(crate) mod discover; pub(crate) mod download_sources_and_hash; pub(crate) mod emit_lockfile; -pub(crate) mod resolve; pub mod install_packages; pub mod install_workspace; +pub(crate) mod resolve; + +/// Packages grouped by the lockfile section they belong to. +#[derive(Default)] +pub(crate) struct Artifacts { + pub(crate) regular: T, + pub(crate) build: T, + pub(crate) test: T, +} + +impl Artifacts { + pub(crate) fn get(&self, section: LockedPackageLockType) -> &T { + match section { + LockedPackageLockType::Regular => &self.regular, + LockedPackageLockType::Build => &self.build, + LockedPackageLockType::Test => &self.test, + } + } + + pub(crate) fn get_mut(&mut self, section: LockedPackageLockType) -> &mut T { + match section { + LockedPackageLockType::Regular => &mut self.regular, + LockedPackageLockType::Build => &mut self.build, + LockedPackageLockType::Test => &mut self.test, + } + } +} + +impl Extend<(LockedPackageLockType, T)> for Artifacts { + fn extend>(&mut self, iter: I) { + iter.into_iter() + .for_each(|(section, value)| *self.get_mut(section) = value); + } +} + +impl IntoIterator for Artifacts { + type Item = (LockedPackageLockType, T); + type IntoIter = std::array::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + [ + (LockedPackageLockType::Regular, self.regular), + (LockedPackageLockType::Build, self.build), + (LockedPackageLockType::Test, self.test), + ] + .into_iter() + } +} diff --git a/lux-lib/src/operations/pipeline/resolve.rs b/lux-lib/src/operations/pipeline/resolve.rs index bba3a8153..87258465f 100644 --- a/lux-lib/src/operations/pipeline/resolve.rs +++ b/lux-lib/src/operations/pipeline/resolve.rs @@ -4,17 +4,19 @@ use std::rc::Rc; use async_recursion::async_recursion; use bon::Builder; +use futures::{StreamExt, TryStreamExt}; use itertools::Itertools; use miette::Diagnostic; use thiserror::Error; use crate::{ - config::Config, lockfile::{LockedPackageId, LockedPackageSpec, RemotePackageSourceUrl}, lua_rockspec::{BuildBackendSpec, RemoteLuaRockspec}, operations::{PackageInstallSpec, RemoteRockDownload}, package::{PackageName, PackageReq, RemotePackage}, remote_package_source::RemotePackageSource, rockspec::Rockspec, tree::EntryType, + config::Config, lockfile::{LockedPackageId, LockedPackageLockType, LockedPackageSpec, RemotePackageSourceUrl}, lua_rockspec::{BuildBackendSpec, RemoteLuaRockspec}, operations::{PackageInstallSpec, RemoteRockDownload}, package::{PackageName, PackageReq, RemotePackage}, remote_package_source::RemotePackageSource, rockspec::Rockspec, tree::EntryType, }; use super::{ discover::{FindPackageFromProvider, DiscoverError, FoundPackage, FoundPackageType}, download_sources_and_hash::PackageSource, + Artifacts, }; /// The build dependencies of a rockspec that still need to be installed, with the @@ -60,12 +62,7 @@ pub(crate) struct ResolvedPackage { pub(crate) artifact: Option, } -#[derive(Default)] -pub(crate) struct ResolvedArtifacts { - pub(crate) regular: HashMap, - pub(crate) build: HashMap, - pub(crate) test: HashMap, -} +pub(crate) type ResolvedArtifacts = Artifacts>; #[derive(Error, Debug, Diagnostic)] #[non_exhaustive] @@ -79,38 +76,6 @@ pub(crate) enum ResolveError { Join(#[from] tokio::task::JoinError), } -#[derive(Clone, Copy, PartialEq, Eq)] -enum ResolveStateType { - Regular, - Build, - Test, -} - -#[derive(Default)] -struct ResolveState { - regular: HashMap, - build: HashMap, - test: HashMap, -} - -impl ResolveState { - fn select(&mut self, state_type: ResolveStateType) -> &mut HashMap { - match state_type { - ResolveStateType::Regular => &mut self.regular, - ResolveStateType::Build => &mut self.build, - ResolveStateType::Test => &mut self.test, - } - } - - fn contains(&self, state_type: ResolveStateType, id: &LockedPackageId) -> bool { - match state_type { - ResolveStateType::Regular => self.regular.contains_key(id), - ResolveStateType::Build => self.build.contains_key(id), - ResolveStateType::Test => self.test.contains_key(id), - } - } -} - #[derive(Builder)] #[builder(start_fn = new, finish_fn(name = _build, vis = ""))] pub(crate) struct ResolvePackageDependencies<'a, D: FindPackageFromProvider> { @@ -155,59 +120,48 @@ where pub(crate) async fn resolve(self) -> Result { let args = self._build(); // TODO(vhyrro): Rewrite to be parallel and no RefCell - let state = Rc::new(RefCell::new(ResolveState::default())); - - for package in args.packages { - resolve_spec( - package, - ResolveStateType::Regular, - Vec::new(), - state.clone(), - args.discover, - args.config, - ) - .await?; - } - - for package in args.build_packages { - resolve_spec( - package, - ResolveStateType::Build, - Vec::new(), - state.clone(), - args.discover, - args.config, - ) + let state = Rc::new(RefCell::new(ResolvedArtifacts::default())); + let (discover, config) = (args.discover, args.config); + + let packages = Artifacts { + regular: args.packages, + build: args.build_packages, + test: args.test_packages, + }; + + futures::stream::iter(packages) + .then(|(section, packages)| { + let state = state.clone(); + async move { + futures::stream::iter(packages) + .then(|package| { + resolve_spec( + package, + section, + Vec::new(), + state.clone(), + discover, + config, + ) + }) + .try_for_each(|_| async { Ok(()) }) + .await + } + }) + .try_for_each(|_| async { Ok(()) }) .await?; - } - for package in args.test_packages { - resolve_spec( - package, - ResolveStateType::Test, - Vec::new(), - state.clone(), - args.discover, - args.config, - ) - .await?; - } - - let mut state = state.borrow_mut(); - Ok(ResolvedArtifacts { - regular: std::mem::take(&mut state.regular), - build: std::mem::take(&mut state.build), - test: std::mem::take(&mut state.test), - }) + let resolved = std::mem::take(&mut *state.borrow_mut()); + Ok(resolved) } } #[async_recursion(?Send)] async fn resolve_spec( spec: PackageInstallSpec, - section: ResolveStateType, + section: LockedPackageLockType, parents: Vec, - state: Rc>, + state: Rc>, discover: &D, config: &Config, ) -> Result { @@ -240,7 +194,7 @@ async fn resolve_spec( constraint.clone(), ); - if state.borrow().contains(section, &id) { + if state.borrow().get(section).contains_key(&id) { return Ok(id); } @@ -257,7 +211,7 @@ async fn resolve_spec( ids.push( resolve_spec( dependency, - ResolveStateType::Build, + LockedPackageLockType::Build, child_parents.clone(), state.clone(), discover, @@ -313,7 +267,7 @@ async fn resolve_spec( state .borrow_mut() - .select(section) + .get_mut(section) .insert(id.clone(), resolved); Ok(id) From daca2c430dd205cc8622a3a7fff0b1176db06bba Mon Sep 17 00:00:00 2001 From: vhyrro Date: Sat, 3 Oct 2026 14:47:44 +0200 Subject: [PATCH 06/39] fix: `EmitLockfile` -> `LockfileHandle` --- lux-lib/src/operations/pipeline/build.rs | 4 +- .../pipeline/download_sources_and_hash.rs | 9 ++- .../src/operations/pipeline/emit_lockfile.rs | 53 ++++++-------- .../operations/pipeline/install_packages.rs | 43 ++++++++---- .../operations/pipeline/install_workspace.rs | 56 ++++++++------- lux-lib/src/operations/pipeline/mod.rs | 54 ++++++++++---- lux-lib/src/operations/pipeline/resolve.rs | 63 ++++++++++------- lux-lib/src/operations/vendor.rs | 70 +++++++++---------- 8 files changed, 200 insertions(+), 152 deletions(-) diff --git a/lux-lib/src/operations/pipeline/build.rs b/lux-lib/src/operations/pipeline/build.rs index cc439b980..b714d26ac 100644 --- a/lux-lib/src/operations/pipeline/build.rs +++ b/lux-lib/src/operations/pipeline/build.rs @@ -5,7 +5,7 @@ use miette::Diagnostic; use thiserror::Error; use crate::{ - build::{deploy, BuildBehaviour}, + build::{BuildBehaviour, deploy}, config::Config, lockfile::{LockedPackage, LockedPackageId}, lua_installation::{LuaInstallation, LuaInstallationError}, @@ -18,7 +18,7 @@ use crate::{ tree::{InstallTree, TreeError}, }; -use super::download_sources_and_hash::{PackageSource, DownloadedPackage}; +use super::download_sources_and_hash::{DownloadedPackage, PackageSource}; #[derive(Error, Debug, Diagnostic)] #[non_exhaustive] diff --git a/lux-lib/src/operations/pipeline/download_sources_and_hash.rs b/lux-lib/src/operations/pipeline/download_sources_and_hash.rs index 2a95b933e..7b6afd749 100644 --- a/lux-lib/src/operations/pipeline/download_sources_and_hash.rs +++ b/lux-lib/src/operations/pipeline/download_sources_and_hash.rs @@ -19,7 +19,7 @@ use crate::{ RemotePackageSourceUrl, }, lua_rockspec::RemoteLuaRockspec, - operations::{unpack_src_rock, FetchSrc, RemotePackageSourceMetadata}, + operations::{FetchSrc, RemotePackageSourceMetadata, unpack_src_rock}, package::PackageSpec, remote_package_source::RemotePackageSource, rockspec::Rockspec, @@ -27,8 +27,8 @@ use crate::{ }; use super::{ - resolve::{ResolvedArtifacts, ResolvedPackage}, Artifacts, + resolve::{ResolvedArtifacts, ResolvedPackage}, }; pub(crate) enum PackageSource { @@ -81,6 +81,9 @@ where futures::stream::iter(args.resolved) .then(|(section, packages)| async move { + let Some(packages) = packages else { + return Ok((section, None)); + }; let packages = futures::stream::iter(packages) .then(|(id, package)| async move { download_sources_and_hash(config, package) @@ -89,7 +92,7 @@ where }) .try_collect::>() .await?; - Ok((section, packages)) + Ok((section, Some(packages))) }) .try_collect() .await diff --git a/lux-lib/src/operations/pipeline/emit_lockfile.rs b/lux-lib/src/operations/pipeline/emit_lockfile.rs index 361c4f418..04959c331 100644 --- a/lux-lib/src/operations/pipeline/emit_lockfile.rs +++ b/lux-lib/src/operations/pipeline/emit_lockfile.rs @@ -1,37 +1,31 @@ use std::collections::HashMap; -use bon::Builder; +use crate::lockfile::{LockedPackageId, LockedPackageLock, ReadWrite, WorkspaceLockfile}; -use crate::lockfile::{ - LockedPackageId, LockedPackageLock, LockedPackageLockType, ReadWrite, WorkspaceLockfile, +use super::{ + Artifacts, + download_sources_and_hash::{DownloadSourcesAndHashArtifacts, DownloadedPackage}, }; -use super::download_sources_and_hash::{DownloadSourcesAndHashArtifacts, DownloadedPackage}; +/// The lockfile entries resolved by the pipeline, grouped by section. +pub(crate) struct LockfileHandle(Artifacts); -/// Writes the resolved [`DownloadSourcesAndHashArtifacts`] to a workspace lockfile. -#[derive(Builder)] -#[builder(start_fn = new, finish_fn(name = _build, vis = ""))] -pub(crate) struct EmitLockfile<'a> { - #[builder(start_fn)] - pub(crate) lockfile: &'a mut WorkspaceLockfile, - pub(crate) artifacts: &'a DownloadSourcesAndHashArtifacts, -} - -impl EmitLockfileBuilder<'_, State> -where - State: emit_lockfile_builder::State + emit_lockfile_builder::IsComplete, -{ - pub(crate) fn emit(self) { - let args = self._build(); +impl LockfileHandle { + pub(crate) fn from_artifacts(artifacts: &DownloadSourcesAndHashArtifacts) -> Self { + Self( + artifacts + .iter() + .map(|(section, packages)| (section, packages.map(lock_from))) + .collect(), + ) + } - for section in [ - LockedPackageLockType::Regular, - LockedPackageLockType::Build, - LockedPackageLockType::Test, - ] { - args.lockfile - .sync(&lock_from(args.artifacts.get(section)), §ion); - } + pub(crate) fn commit(&self, lockfile: &mut WorkspaceLockfile) { + self.0.iter().for_each(|(section, lock)| { + if let Some(lock) = lock { + lockfile.sync(lock, §ion); + } + }); } } @@ -39,10 +33,7 @@ fn lock_from(packages: &HashMap) -> LockedPa // FIXME(vhyrro): Create a constructor here instead of mut overrides. let mut lock = LockedPackageLock::default(); for package in packages.values() { - lock.insert( - package.package.clone(), - package.entry_type.is_entrypoint(), - ); + lock.insert(package.package.clone(), package.entry_type.is_entrypoint()); } lock } diff --git a/lux-lib/src/operations/pipeline/install_packages.rs b/lux-lib/src/operations/pipeline/install_packages.rs index 9d88fcf8f..34fdd0242 100644 --- a/lux-lib/src/operations/pipeline/install_packages.rs +++ b/lux-lib/src/operations/pipeline/install_packages.rs @@ -18,10 +18,8 @@ use crate::{ }; use super::{ - build::Build as PipelineBuild, - discover::FindPackageFromLuarocks, - download_sources_and_hash::DownloadSourcesAndHash, - resolve::ResolvePackageDependencies, + build::Build as PipelineBuild, discover::FindPackageFromLuarocks, + download_sources_and_hash::DownloadSourcesAndHash, resolve::ResolvePackageDependencies, }; use crate::operations::PackageInstallSpec; @@ -127,7 +125,8 @@ where tree.cleanup(package, tree::EntryType::Entrypoint)?; } - let discover = FindPackageFromLuarocks::new(Arc::new(package_db), Arc::new(config.clone())).build(); + let discover = + FindPackageFromLuarocks::new(Arc::new(package_db), Arc::new(config.clone())).build(); let resolved = ResolvePackageDependencies::new(&discover, config) .packages(packages) .resolve() @@ -139,8 +138,16 @@ where .await .map_err(|err| InstallPackagesError::Pipeline(Box::new(err)))?; - let build_packages = artifacts.build.into_values().collect_vec(); - let regular_packages = artifacts.regular.into_values().collect_vec(); + let build_packages = artifacts + .build + .unwrap_or_default() + .into_values() + .collect_vec(); + let regular_packages = artifacts + .regular + .unwrap_or_default() + .into_values() + .collect_vec(); let regular_entry_types: HashMap = regular_packages .iter() .map(|pkg| (pkg.package.spec.id(), pkg.entry_type)) @@ -158,8 +165,10 @@ where .await .map_err(|err| InstallPackagesError::Pipeline(Box::new(err)))?; - let installed_packages: HashMap = - built.iter().map(|pkg| (pkg.spec.id(), pkg.clone())).collect(); + let installed_packages: HashMap = built + .iter() + .map(|pkg| (pkg.spec.id(), pkg.clone())) + .collect(); let installed_build_deps: HashMap = built_build_deps .iter() .map(|pkg| (pkg.spec.id(), pkg.clone())) @@ -253,15 +262,17 @@ impl LockfileExt for &mut Lockfile { for dependency_id in pkg.spec.dependencies() { self.add_dependency( pkg, - installed_packages.get(dependency_id).ok_or(io::Error::other( - r#" + installed_packages + .get(dependency_id) + .ok_or(io::Error::other( + r#" error writing dependencies to the lockfile. A required dependency was not installed correctly. This is likely a bug in Lux. [THIS IS A BUG!] "#, - ))?, + ))?, ); } Ok(()) @@ -275,15 +286,17 @@ This is likely a bug in Lux. for dependency_id in pkg.spec.build_dependencies() { self.add_build_dependency( pkg, - build_dependencies.get(dependency_id).ok_or(io::Error::other( - r#" + build_dependencies + .get(dependency_id) + .ok_or(io::Error::other( + r#" error writing build dependencies to the lockfile. A required build dependency was not installed correctly. This is likely a bug in Lux. [THIS IS A BUG!] "#, - ))?, + ))?, ); } Ok(()) diff --git a/lux-lib/src/operations/pipeline/install_workspace.rs b/lux-lib/src/operations/pipeline/install_workspace.rs index 65001e6b7..7e293201f 100644 --- a/lux-lib/src/operations/pipeline/install_workspace.rs +++ b/lux-lib/src/operations/pipeline/install_workspace.rs @@ -16,10 +16,8 @@ use crate::{ }; use super::{ - build::Build, - discover::FindPackageFromLuarocks, - download_sources_and_hash::DownloadSourcesAndHash, - emit_lockfile::EmitLockfile, + build::Build, discover::FindPackageFromLuarocks, + download_sources_and_hash::DownloadSourcesAndHash, emit_lockfile::LockfileHandle, resolve::ResolvePackageDependencies, }; @@ -58,9 +56,7 @@ where State: install_workspace_dependencies_builder::State + install_workspace_dependencies_builder::IsComplete, { - pub async fn install( - self, - ) -> Result, InstallWorkspaceDependenciesError> { + pub async fn install(self) -> Result, InstallWorkspaceDependenciesError> { let args = self._build(); let config = args.config; let workspace = args.workspace; @@ -73,16 +69,14 @@ where let mut regular = gather_dependencies(workspace, DependencyKind::Regular)?; regular.extend(args.packages); let build = gather_dependencies(workspace, DependencyKind::Build)?; - let test = if args.test.unwrap_or(false) { - gather_dependencies(workspace, DependencyKind::Test)? - } else { - Vec::new() - }; - let resolved = ResolvePackageDependencies::new(&discover, config) + let mut resolve = ResolvePackageDependencies::new(&discover, config) .packages(regular) - .build_packages(build) - .test_packages(test) + .build_packages(build); + if args.test.unwrap_or(false) { + resolve = resolve.test_packages(gather_dependencies(workspace, DependencyKind::Test)?); + } + let resolved = resolve .resolve() .await .map_err(|err| InstallWorkspaceDependenciesError::Pipeline(Box::new(err)))?; @@ -95,9 +89,7 @@ where if !no_lock { let mut lockfile = workspace.lockfile()?.write_guard(); - EmitLockfile::new(&mut lockfile) - .artifacts(&artifacts) - .emit(); + LockfileHandle::from_artifacts(&artifacts).commit(&mut lockfile); } let regular_tree = workspace.tree(config)?; @@ -107,17 +99,23 @@ where // TODO(vhyrro): Make parallel // make sure to build build dependencies first Build::new(config, &build_tree) - .packages(artifacts.build.into_values().collect()) + .packages(artifacts.build.unwrap_or_default().into_values().collect()) .build() .await .map_err(|err| InstallWorkspaceDependenciesError::Pipeline(Box::new(err)))?; let built = Build::new(config, ®ular_tree) - .packages(artifacts.regular.into_values().collect()) + .packages( + artifacts + .regular + .unwrap_or_default() + .into_values() + .collect(), + ) .build() .await .map_err(|err| InstallWorkspaceDependenciesError::Pipeline(Box::new(err)))?; Build::new(config, &test_tree) - .packages(artifacts.test.into_values().collect()) + .packages(artifacts.test.unwrap_or_default().into_values().collect()) .build() .await .map_err(|err| InstallWorkspaceDependenciesError::Pipeline(Box::new(err)))?; @@ -212,11 +210,15 @@ mod tests { let lockfile = workspace.lockfile().unwrap(); let rocks = lockfile.rocks(&LockedPackageLockType::Regular); - assert!(rocks - .values() - .any(|pkg| pkg.name().to_string() == "lua-cjson")); - assert!(rocks - .values() - .any(|pkg| pkg.name().to_string() == "plenary.nvim")); + assert!( + rocks + .values() + .any(|pkg| pkg.name().to_string() == "lua-cjson") + ); + assert!( + rocks + .values() + .any(|pkg| pkg.name().to_string() == "plenary.nvim") + ); } } diff --git a/lux-lib/src/operations/pipeline/mod.rs b/lux-lib/src/operations/pipeline/mod.rs index 8011a6fc3..8b15060c3 100644 --- a/lux-lib/src/operations/pipeline/mod.rs +++ b/lux-lib/src/operations/pipeline/mod.rs @@ -10,40 +10,70 @@ pub mod install_workspace; pub(crate) mod resolve; /// Packages grouped by the lockfile section they belong to. -#[derive(Default)] +/// +/// A `None` section was never requested and must be left untouched (e.g. not +/// written to a lockfile), whereas `Some(empty)` means the section was resolved +/// and is legitimately empty. pub(crate) struct Artifacts { - pub(crate) regular: T, - pub(crate) build: T, - pub(crate) test: T, + pub(crate) regular: Option, + pub(crate) build: Option, + pub(crate) test: Option, +} + +impl Default for Artifacts { + fn default() -> Self { + Self { + regular: None, + build: None, + test: None, + } + } } impl Artifacts { - pub(crate) fn get(&self, section: LockedPackageLockType) -> &T { + pub(crate) fn get(&self, section: LockedPackageLockType) -> Option<&T> { match section { - LockedPackageLockType::Regular => &self.regular, - LockedPackageLockType::Build => &self.build, - LockedPackageLockType::Test => &self.test, + LockedPackageLockType::Regular => self.regular.as_ref(), + LockedPackageLockType::Build => self.build.as_ref(), + LockedPackageLockType::Test => self.test.as_ref(), } } - pub(crate) fn get_mut(&mut self, section: LockedPackageLockType) -> &mut T { + pub(crate) fn get_mut(&mut self, section: LockedPackageLockType) -> &mut Option { match section { LockedPackageLockType::Regular => &mut self.regular, LockedPackageLockType::Build => &mut self.build, LockedPackageLockType::Test => &mut self.test, } } + + pub(crate) fn iter(&self) -> impl Iterator)> { + [ + (LockedPackageLockType::Regular, self.regular.as_ref()), + (LockedPackageLockType::Build, self.build.as_ref()), + (LockedPackageLockType::Test, self.test.as_ref()), + ] + .into_iter() + } } -impl Extend<(LockedPackageLockType, T)> for Artifacts { - fn extend>(&mut self, iter: I) { +impl Extend<(LockedPackageLockType, Option)> for Artifacts { + fn extend)>>(&mut self, iter: I) { iter.into_iter() .for_each(|(section, value)| *self.get_mut(section) = value); } } +impl FromIterator<(LockedPackageLockType, Option)> for Artifacts { + fn from_iter)>>(iter: I) -> Self { + let mut artifacts = Artifacts::default(); + artifacts.extend(iter); + artifacts + } +} + impl IntoIterator for Artifacts { - type Item = (LockedPackageLockType, T); + type Item = (LockedPackageLockType, Option); type IntoIter = std::array::IntoIter; fn into_iter(self) -> Self::IntoIter { diff --git a/lux-lib/src/operations/pipeline/resolve.rs b/lux-lib/src/operations/pipeline/resolve.rs index 87258465f..594891ff8 100644 --- a/lux-lib/src/operations/pipeline/resolve.rs +++ b/lux-lib/src/operations/pipeline/resolve.rs @@ -10,13 +10,20 @@ use miette::Diagnostic; use thiserror::Error; use crate::{ - config::Config, lockfile::{LockedPackageId, LockedPackageLockType, LockedPackageSpec, RemotePackageSourceUrl}, lua_rockspec::{BuildBackendSpec, RemoteLuaRockspec}, operations::{PackageInstallSpec, RemoteRockDownload}, package::{PackageName, PackageReq, RemotePackage}, remote_package_source::RemotePackageSource, rockspec::Rockspec, tree::EntryType, + config::Config, + lockfile::{LockedPackageId, LockedPackageLockType, LockedPackageSpec, RemotePackageSourceUrl}, + lua_rockspec::{BuildBackendSpec, RemoteLuaRockspec}, + operations::{PackageInstallSpec, RemoteRockDownload}, + package::{PackageName, PackageReq, RemotePackage}, + remote_package_source::RemotePackageSource, + rockspec::Rockspec, + tree::EntryType, }; use super::{ - discover::{FindPackageFromProvider, DiscoverError, FoundPackage, FoundPackageType}, - download_sources_and_hash::PackageSource, Artifacts, + discover::{DiscoverError, FindPackageFromProvider, FoundPackage, FoundPackageType}, + download_sources_and_hash::PackageSource, }; /// The build dependencies of a rockspec that still need to be installed, with the @@ -84,11 +91,7 @@ pub(crate) struct ResolvePackageDependencies<'a, D: FindPackageFromProvider> { #[builder(start_fn)] pub(crate) config: &'a Config, #[builder(field)] - pub(crate) packages: Vec, - #[builder(field)] - pub(crate) build_packages: Vec, - #[builder(field)] - pub(crate) test_packages: Vec, + pub(crate) packages: Artifacts>, } impl ResolvePackageDependenciesBuilder<'_, D, State> @@ -96,26 +99,29 @@ where D: FindPackageFromProvider, State: resolve_package_dependencies_builder::State, { - pub(crate) fn packages(mut self, packages: Vec) -> Self { - self.packages = packages; + fn set(mut self, section: LockedPackageLockType, packages: Vec) -> Self { + *self.packages.get_mut(section) = Some(packages); self } - pub(crate) fn build_packages(mut self, packages: Vec) -> Self { - self.build_packages = packages; - self + pub(crate) fn packages(self, packages: Vec) -> Self { + self.set(LockedPackageLockType::Regular, packages) } - pub(crate) fn test_packages(mut self, packages: Vec) -> Self { - self.test_packages = packages; - self + pub(crate) fn build_packages(self, packages: Vec) -> Self { + self.set(LockedPackageLockType::Build, packages) + } + + pub(crate) fn test_packages(self, packages: Vec) -> Self { + self.set(LockedPackageLockType::Test, packages) } } impl ResolvePackageDependenciesBuilder<'_, D, State> where D: FindPackageFromProvider, - State: resolve_package_dependencies_builder::State + resolve_package_dependencies_builder::IsComplete, + State: resolve_package_dependencies_builder::State + + resolve_package_dependencies_builder::IsComplete, { pub(crate) async fn resolve(self) -> Result { let args = self._build(); @@ -123,16 +129,18 @@ where let state = Rc::new(RefCell::new(ResolvedArtifacts::default())); let (discover, config) = (args.discover, args.config); - let packages = Artifacts { - regular: args.packages, - build: args.build_packages, - test: args.test_packages, - }; - - futures::stream::iter(packages) + futures::stream::iter(args.packages) .then(|(section, packages)| { let state = state.clone(); async move { + let Some(packages) = packages else { + return Ok(()); + }; + // Mark the section as requested even if it resolves to nothing. + state + .borrow_mut() + .get_mut(section) + .get_or_insert_with(HashMap::new); futures::stream::iter(packages) .then(|package| { resolve_spec( @@ -194,7 +202,11 @@ async fn resolve_spec( constraint.clone(), ); - if state.borrow().get(section).contains_key(&id) { + if state + .borrow() + .get(section) + .is_some_and(|packages| packages.contains_key(&id)) + { return Ok(id); } @@ -268,6 +280,7 @@ async fn resolve_spec( state .borrow_mut() .get_mut(section) + .get_or_insert_with(HashMap::new) .insert(id.clone(), resolved); Ok(id) diff --git a/lux-lib/src/operations/vendor.rs b/lux-lib/src/operations/vendor.rs index 2b584a187..9448af1a7 100644 --- a/lux-lib/src/operations/vendor.rs +++ b/lux-lib/src/operations/vendor.rs @@ -1,4 +1,5 @@ use std::{ + collections::HashMap, io::{self, Cursor}, path::{Path, PathBuf}, process::ExitStatus, @@ -22,16 +23,12 @@ use crate::{ lockfile::LockedPackageLockType, lua_rockspec::{BuildBackendSpec, RemoteLuaRockspec}, operations::{ - self, + self, PackageInstallSpec, UnpackError, pipeline::{ discover::FindPackageFromLuarocks, - download_sources_and_hash::{ - DownloadSourcesAndHash, DownloadSourcesAndHashArtifacts, DownloadedPackage, - PackageSource, - }, - resolve::{luarocks_build_backend_name, ResolvePackageDependencies}, + download_sources_and_hash::{DownloadSourcesAndHash, DownloadedPackage, PackageSource}, + resolve::{ResolvePackageDependencies, luarocks_build_backend_name}, }, - PackageInstallSpec, UnpackError, }, package::{PackageReq, PackageSpec}, project::project_toml::LocalProjectTomlValidationError, @@ -147,18 +144,19 @@ impl VendoredPackages { // with different constraints. let packages = packages .into_iter() - .unique_by(|pkg| (pkg.package.spec.name().clone(), pkg.package.spec.version().clone())) + .unique_by(|pkg| { + ( + pkg.package.spec.name().clone(), + pkg.package.spec.version().clone(), + ) + }) .collect(); Ok(Self { packages }) } /// Vendors the sources of all packages into `vendor_dir`. - async fn vendor_sources( - &self, - vendor_dir: &Path, - config: &Config, - ) -> Result<(), VendorError> { + async fn vendor_sources(&self, vendor_dir: &Path, config: &Config) -> Result<(), VendorError> { futures::stream::iter( self.packages .iter() @@ -176,11 +174,13 @@ impl VendoredPackages { .filter_map(|package| { let rockspec = &package.rockspec; match rockspec.build().current_platform().build_backend { - Some(BuildBackendSpec::RustMlua(_) | BuildBackendSpec::RustBinary(_)) => Some(( - package.package.spec.to_package(), - rockspec.source().current_platform().unpack_dir.clone(), - rockspec.build().current_platform().copy_directories.clone(), - )), + Some(BuildBackendSpec::RustMlua(_) | BuildBackendSpec::RustBinary(_)) => { + Some(( + package.package.spec.to_package(), + rockspec.source().current_platform().unpack_dir.clone(), + rockspec.build().current_platform().copy_directories.clone(), + )) + } _ => None, } }) @@ -209,11 +209,9 @@ impl<'a> ResolveAndDownload<'a> { } async fn run(self) -> Result, VendorError> { - let discover = FindPackageFromLuarocks::new( - Arc::new(self.package_db), - Arc::new(self.config.clone()), - ) - .build(); + let discover = + FindPackageFromLuarocks::new(Arc::new(self.package_db), Arc::new(self.config.clone())) + .build(); let resolved = ResolvePackageDependencies::new(&discover, self.config) .packages(self.install_specs) .resolve() @@ -224,15 +222,10 @@ impl<'a> ResolveAndDownload<'a> { .download_sources_and_hash() .await .map_err(|err| VendorError::Pipeline(Box::new(err)))?; - let DownloadSourcesAndHashArtifacts { - regular, - build, - test, - } = artifacts; - Ok(regular - .into_values() - .chain(build.into_values()) - .chain(test.into_values()) + Ok(artifacts + .into_iter() + .flat_map(|(_, packages)| packages) + .flat_map(HashMap::into_values) .collect()) } } @@ -258,7 +251,8 @@ async fn do_vendor_dependencies(args: Vendor<'_>) -> Result<(), VendorError> { for (dep, unpack_dir, copy_dirs) in cargo_deps { vendor_package_cargo_deps(&vendor_dir, &dep, &unpack_dir, ©_dirs, config).await?; } - Ok(())} + Ok(()) +} async fn gather_install_specs( lock_type: &LockedPackageLockType, @@ -381,10 +375,12 @@ async fn vendor_package_sources( PackageSource::PackedRock(bytes) => { let rock_path = vendor_dir.join(format!("{}@{}.rock", name, version)); let mut file = fs::tokio::create(&rock_path).await?; - file.write_all(bytes).await.map_err(|source| fs::FsError::Write { - path: rock_path, - source, - })?; + file.write_all(bytes) + .await + .map_err(|source| fs::FsError::Write { + path: rock_path, + source, + })?; } } From a8358bd37fa9feecef0dc6ab4074495ddb1cde3b Mon Sep 17 00:00:00 2001 From: vhyrro Date: Sat, 3 Oct 2026 15:02:21 +0200 Subject: [PATCH 07/39] refactor: simplify `InstallWorkspaceDependencies` --- lux-lib/src/operations/install/mod.rs | 9 ++- .../src/operations/install_dependencies.rs | 8 +-- .../src/operations/pipeline/emit_lockfile.rs | 3 +- .../operations/pipeline/install_packages.rs | 58 ++++++++++++++--- .../operations/pipeline/install_workspace.rs | 64 +++---------------- lux-lib/src/operations/pipeline/mod.rs | 2 +- lux-lib/src/operations/update.rs | 2 +- 7 files changed, 70 insertions(+), 76 deletions(-) diff --git a/lux-lib/src/operations/install/mod.rs b/lux-lib/src/operations/install/mod.rs index 76f9a930c..05f0f74a2 100644 --- a/lux-lib/src/operations/install/mod.rs +++ b/lux-lib/src/operations/install/mod.rs @@ -129,9 +129,7 @@ pub enum InstallError { } impl From for InstallError { - fn from( - source: crate::luarocks::luarocks_installation::LuaRocksInstallError, - ) -> Self { + fn from(source: crate::luarocks::luarocks_installation::LuaRocksInstallError) -> Self { Self::LuaRocksInstall(Box::new(source)) } } @@ -145,10 +143,11 @@ where return Ok(Vec::new()); } - InstallPackages::new(install.config, install.tree) + let (packages, _lockfile) = InstallPackages::new(install.config, install.tree) .packages(packages) .maybe_package_db(install.package_db) .install() .await - .map_err(InstallError::from) + .map_err(InstallError::from)?; + Ok(packages) } diff --git a/lux-lib/src/operations/install_dependencies.rs b/lux-lib/src/operations/install_dependencies.rs index 6c6d290f5..2aa2975d2 100644 --- a/lux-lib/src/operations/install_dependencies.rs +++ b/lux-lib/src/operations/install_dependencies.rs @@ -30,9 +30,9 @@ where } impl< - T: InstallTree + Sync + Send + Clone + 'static, - State: install_dependencies_builder::State + install_dependencies_builder::IsComplete, - > InstallDependenciesBuilder<'_, T, State> + T: InstallTree + Sync + Send + Clone + 'static, + State: install_dependencies_builder::State + install_dependencies_builder::IsComplete, +> InstallDependenciesBuilder<'_, T, State> { /// Installs the configured dependencies and build dependencies into [`Self::tree`], /// returning the installed regular dependencies. @@ -53,7 +53,7 @@ impl< .await .map_err(InstallError::from)?; } - let dependencies = InstallPackages::new(config, (*tree).clone()) + let (dependencies, _lockfile) = InstallPackages::new(config, (*tree).clone()) .packages(dependencies.into_iter().unique().collect_vec()) .install() .await diff --git a/lux-lib/src/operations/pipeline/emit_lockfile.rs b/lux-lib/src/operations/pipeline/emit_lockfile.rs index 04959c331..e434b67ef 100644 --- a/lux-lib/src/operations/pipeline/emit_lockfile.rs +++ b/lux-lib/src/operations/pipeline/emit_lockfile.rs @@ -8,7 +8,8 @@ use super::{ }; /// The lockfile entries resolved by the pipeline, grouped by section. -pub(crate) struct LockfileHandle(Artifacts); +#[derive(Default)] +pub struct LockfileHandle(Artifacts); impl LockfileHandle { pub(crate) fn from_artifacts(artifacts: &DownloadSourcesAndHashArtifacts) -> Self { diff --git a/lux-lib/src/operations/pipeline/install_packages.rs b/lux-lib/src/operations/pipeline/install_packages.rs index 34fdd0242..bb9ef1589 100644 --- a/lux-lib/src/operations/pipeline/install_packages.rs +++ b/lux-lib/src/operations/pipeline/install_packages.rs @@ -19,7 +19,8 @@ use crate::{ use super::{ build::Build as PipelineBuild, discover::FindPackageFromLuarocks, - download_sources_and_hash::DownloadSourcesAndHash, resolve::ResolvePackageDependencies, + download_sources_and_hash::DownloadSourcesAndHash, emit_lockfile::LockfileHandle, + resolve::ResolvePackageDependencies, }; use crate::operations::PackageInstallSpec; @@ -28,7 +29,9 @@ use crate::operations::PackageInstallSpec; /// /// This is the composable core of [`Install`](crate::operations::Install): it resolves the /// dependency graph, downloads and hashes sources, builds the packages (build dependencies -/// first) and records the result in the tree's lockfile. +/// first) into the regular, build and test trees, and records the result in the trees' +/// lockfiles. It also returns a [`LockfileHandle`] so callers (e.g. a workspace) can commit +/// the resolved sections to their own lockfile. #[derive(Builder)] #[builder(start_fn = new, finish_fn(name = _build, vis = ""))] pub struct InstallPackages<'a, T> @@ -39,19 +42,44 @@ where config: &'a Config, #[builder(start_fn)] tree: T, + #[builder(field)] + build_packages: Vec, + #[builder(field)] + test_packages: Option>, packages: Vec, package_db: Option, } +impl<'a, T, State> InstallPackagesBuilder<'a, T, State> +where + T: InstallTree + Clone + Send + Sync, + State: install_packages_builder::State, +{ + pub fn build_packages(mut self, packages: Vec) -> Self { + self.build_packages = packages; + self + } + + pub fn test_packages(mut self, packages: Vec) -> Self { + self.test_packages = Some(packages); + self + } +} + impl InstallPackagesBuilder<'_, T, State> where State: install_packages_builder::State + install_packages_builder::IsComplete, T: InstallTree + Clone + Send + Sync + 'static, { - pub async fn install(self) -> Result, InstallPackagesError> { + pub async fn install( + self, + ) -> Result<(Vec, LockfileHandle), InstallPackagesError> { let args = self._build(); - if args.packages.is_empty() { - return Ok(Vec::new()); + if args.packages.is_empty() + && args.build_packages.is_empty() + && args.test_packages.is_none() + { + return Ok((Vec::new(), LockfileHandle::default())); } install_packages(args).await } @@ -59,7 +87,7 @@ where async fn install_packages( install: InstallPackages<'_, T>, -) -> Result, InstallPackagesError> +) -> Result<(Vec, LockfileHandle), InstallPackagesError> where T: InstallTree + Clone + Send + Sync + 'static, { @@ -127,8 +155,13 @@ where let discover = FindPackageFromLuarocks::new(Arc::new(package_db), Arc::new(config.clone())).build(); - let resolved = ResolvePackageDependencies::new(&discover, config) + let mut resolve = ResolvePackageDependencies::new(&discover, config) .packages(packages) + .build_packages(install.build_packages); + if let Some(test_packages) = install.test_packages { + resolve = resolve.test_packages(test_packages); + } + let resolved = resolve .resolve() .await .map_err(|err| InstallPackagesError::Pipeline(Box::new(err)))?; @@ -137,6 +170,7 @@ where .download_sources_and_hash() .await .map_err(|err| InstallPackagesError::Pipeline(Box::new(err)))?; + let handle = LockfileHandle::from_artifacts(&artifacts); let build_packages = artifacts .build @@ -164,6 +198,14 @@ where .build() .await .map_err(|err| InstallPackagesError::Pipeline(Box::new(err)))?; + if let Some(test_packages) = artifacts.test { + let test_tree = tree.test_tree(config)?; + PipelineBuild::new(config, &test_tree) + .packages(test_packages.into_values().collect_vec()) + .build() + .await + .map_err(|err| InstallPackagesError::Pipeline(Box::new(err)))?; + } let installed_packages: HashMap = built .iter() @@ -186,7 +228,7 @@ where Ok::<_, io::Error>(()) })?; - Ok(built) + Ok((built, handle)) } #[derive(Error, Debug, Diagnostic)] diff --git a/lux-lib/src/operations/pipeline/install_workspace.rs b/lux-lib/src/operations/pipeline/install_workspace.rs index 7e293201f..927401ac6 100644 --- a/lux-lib/src/operations/pipeline/install_workspace.rs +++ b/lux-lib/src/operations/pipeline/install_workspace.rs @@ -1,5 +1,3 @@ -use std::sync::Arc; - use bon::Builder; use miette::Diagnostic; use thiserror::Error; @@ -9,17 +7,12 @@ use crate::{ lockfile::LockedPackage, operations::PackageInstallSpec, project::project_toml::LocalProjectTomlValidationError, - remote_package_db::{RemotePackageDB, RemotePackageDBError}, rockspec::Rockspec, tree::EntryType, workspace::{Workspace, WorkspaceError, WorkspaceTreeError}, }; -use super::{ - build::Build, discover::FindPackageFromLuarocks, - download_sources_and_hash::DownloadSourcesAndHash, emit_lockfile::LockfileHandle, - resolve::ResolvePackageDependencies, -}; +use super::install_packages::InstallPackages; /// Installs all of a workspace's dependencies into its regular, build and test trees. /// @@ -60,66 +53,28 @@ where let args = self._build(); let config = args.config; let workspace = args.workspace; - let no_lock = args.no_lock.unwrap_or(false); - - let package_db = RemotePackageDB::from_config(config).await?; - let discover = - FindPackageFromLuarocks::new(Arc::new(package_db), Arc::new(config.clone())).build(); let mut regular = gather_dependencies(workspace, DependencyKind::Regular)?; regular.extend(args.packages); let build = gather_dependencies(workspace, DependencyKind::Build)?; - let mut resolve = ResolvePackageDependencies::new(&discover, config) + let mut install = InstallPackages::new(config, workspace.tree(config)?) .packages(regular) .build_packages(build); if args.test.unwrap_or(false) { - resolve = resolve.test_packages(gather_dependencies(workspace, DependencyKind::Test)?); + install = install.test_packages(gather_dependencies(workspace, DependencyKind::Test)?); } - let resolved = resolve - .resolve() - .await - .map_err(|err| InstallWorkspaceDependenciesError::Pipeline(Box::new(err)))?; - let artifacts = DownloadSourcesAndHash::new(config) - .resolved(resolved) - .download_sources_and_hash() + let (built, lockfile) = install + .install() .await .map_err(|err| InstallWorkspaceDependenciesError::Pipeline(Box::new(err)))?; - if !no_lock { - let mut lockfile = workspace.lockfile()?.write_guard(); - LockfileHandle::from_artifacts(&artifacts).commit(&mut lockfile); + // Commit the workspace lockfile only once installation has succeeded. + if !args.no_lock.unwrap_or(false) { + lockfile.commit(&mut workspace.lockfile()?.write_guard()); } - let regular_tree = workspace.tree(config)?; - let build_tree = workspace.build_tree(config)?; - let test_tree = workspace.test_tree(config)?; - - // TODO(vhyrro): Make parallel - // make sure to build build dependencies first - Build::new(config, &build_tree) - .packages(artifacts.build.unwrap_or_default().into_values().collect()) - .build() - .await - .map_err(|err| InstallWorkspaceDependenciesError::Pipeline(Box::new(err)))?; - let built = Build::new(config, ®ular_tree) - .packages( - artifacts - .regular - .unwrap_or_default() - .into_values() - .collect(), - ) - .build() - .await - .map_err(|err| InstallWorkspaceDependenciesError::Pipeline(Box::new(err)))?; - Build::new(config, &test_tree) - .packages(artifacts.test.unwrap_or_default().into_values().collect()) - .build() - .await - .map_err(|err| InstallWorkspaceDependenciesError::Pipeline(Box::new(err)))?; - Ok(built) } } @@ -136,9 +91,6 @@ pub enum InstallWorkspaceDependenciesError { WorkspaceTree(#[from] WorkspaceTreeError), #[error(transparent)] #[diagnostic(transparent)] - RemotePackageDB(#[from] RemotePackageDBError), - #[error(transparent)] - #[diagnostic(transparent)] Project(#[from] LocalProjectTomlValidationError), #[error("failed to install workspace dependencies")] #[diagnostic(forward(0))] diff --git a/lux-lib/src/operations/pipeline/mod.rs b/lux-lib/src/operations/pipeline/mod.rs index 8b15060c3..170bb67f3 100644 --- a/lux-lib/src/operations/pipeline/mod.rs +++ b/lux-lib/src/operations/pipeline/mod.rs @@ -4,7 +4,7 @@ pub(crate) mod build; pub(crate) mod build_project; pub(crate) mod discover; pub(crate) mod download_sources_and_hash; -pub(crate) mod emit_lockfile; +pub mod emit_lockfile; pub mod install_packages; pub mod install_workspace; pub(crate) mod resolve; diff --git a/lux-lib/src/operations/update.rs b/lux-lib/src/operations/update.rs index 69cc17be8..6fedece13 100644 --- a/lux-lib/src/operations/update.rs +++ b/lux-lib/src/operations/update.rs @@ -285,7 +285,7 @@ async fn update( .packages(updatable.iter().map(|(package, _)| package.id())) .remove() .await?; - let updated_packages = InstallPackages::new(config, tree) + let (updated_packages, _lockfile) = InstallPackages::new(config, tree) .packages( updatable .iter() From ddb0d8ba53347f7dee3cf3f8809d1d5c8a99e3a9 Mon Sep 17 00:00:00 2001 From: vhyrro Date: Sat, 3 Oct 2026 15:29:40 +0200 Subject: [PATCH 08/39] fix(BuildProject): only read project dependencies not all workspace dependencies --- .../src/operations/pipeline/build_project.rs | 24 +++++++++++------- lux-lib/tests/build_workspace.rs | 25 ++++++++++++++++++- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/lux-lib/src/operations/pipeline/build_project.rs b/lux-lib/src/operations/pipeline/build_project.rs index 29baf8844..b68794a6b 100644 --- a/lux-lib/src/operations/pipeline/build_project.rs +++ b/lux-lib/src/operations/pipeline/build_project.rs @@ -7,7 +7,9 @@ use crate::{ config::Config, lockfile::LockedPackage, lua_installation::{LuaInstallation, LuaInstallationError}, + package::PackageName, project::project_toml::LocalProjectTomlValidationError, + rockspec::Rockspec, tree::{EntryType, InstallTree, TreeError}, workspace::{Workspace, WorkspaceTreeError}, }; @@ -48,21 +50,23 @@ where .await?; let lockfile = workspace_tree.lockfile()?; - let dependencies = lockfile - .rocks() - .iter() - .filter_map(|(id, package)| lockfile.is_entrypoint(id).then_some(package)) - .cloned() - .collect::>(); let build_lockfile = args.workspace.build_tree(args.config)?.lockfile()?; let mut lockfile = lockfile.write_guard(); lockfile.add_entrypoint(&package); - for dependency in dependencies { + for dependency in project_toml.dependencies().current_platform() { + let dependency = lockfile + .entrypoint(dependency.name()) + .ok_or_else(|| BuildProjectError::MissingDependency(dependency.name().clone()))? + .clone(); lockfile.add_dependency(&package, &dependency); } - for dependency in build_lockfile.rocks().values() { - lockfile.add_build_dependency(&package, dependency); + for dependency in project_toml.build_dependencies().current_platform() { + let dependency = build_lockfile + .entrypoint(dependency.name()) + .ok_or_else(|| BuildProjectError::MissingDependency(dependency.name().clone()))? + .clone(); + lockfile.add_build_dependency(&package, &dependency); } Ok(package) @@ -87,6 +91,8 @@ pub(crate) enum BuildProjectError { #[error(transparent)] #[diagnostic(transparent)] Project(#[from] LocalProjectTomlValidationError), + #[error("dependency `{0}` was not found in the lockfile")] + MissingDependency(PackageName), } impl From for BuildProjectError { diff --git a/lux-lib/tests/build_workspace.rs b/lux-lib/tests/build_workspace.rs index 0306175f1..2ac76ffa4 100644 --- a/lux-lib/tests/build_workspace.rs +++ b/lux-lib/tests/build_workspace.rs @@ -1,10 +1,12 @@ use std::path::PathBuf; -use assert_fs::prelude::PathCopy; use assert_fs::TempDir; +use assert_fs::prelude::PathCopy; use flaky_test::flaky_test; use lux_lib::lua_version::LuaVersion; use lux_lib::operations::BuildWorkspace; +use lux_lib::package::PackageName; +use lux_lib::tree::InstallTree; use lux_lib::workspace::Workspace; use lux_lib::{config::ConfigBuilder, lua_installation::detect_installed_lua_version}; @@ -29,4 +31,25 @@ async fn test_build_multi_workspace_local_dependencies() { .build() .await .unwrap(); + + let lockfile = workspace.tree(&config).unwrap().lockfile().unwrap(); + + let foo = lockfile + .entrypoint(&PackageName::new("foo".into())) + .expect("foo must be recorded as an entrypoint"); + let bar = lockfile + .entrypoint(&PackageName::new("bar".into())) + .expect("bar must be recorded as an entrypoint"); + + assert!( + foo.dependencies().is_empty(), + "foo declares no dependencies and must not be linked to its sibling member bar" + ); + assert!( + bar.dependencies() + .into_iter() + .filter_map(|id| lockfile.get(id)) + .any(|pkg| pkg.name() == foo.name()), + "bar's path dependency on foo must be recorded as a tree edge" + ); } From 1fe913f6387e0edd1fbc3fc815da156aa7027f12 Mon Sep 17 00:00:00 2001 From: vhyrro Date: Sat, 3 Oct 2026 15:45:46 +0200 Subject: [PATCH 09/39] fix: remove `dyn` errors --- lux-lib/src/build/luarocks.rs | 7 ++-- lux-lib/src/operations/build_workspace.rs | 15 ++------ lux-lib/src/operations/pipeline/build.rs | 2 +- .../src/operations/pipeline/build_project.rs | 2 +- lux-lib/src/operations/pipeline/discover.rs | 2 +- .../pipeline/download_sources_and_hash.rs | 10 ++++- .../operations/pipeline/install_packages.rs | 37 ++++++++++--------- .../operations/pipeline/install_workspace.rs | 19 ++++++---- lux-lib/src/operations/pipeline/mod.rs | 10 ++--- lux-lib/src/operations/pipeline/resolve.rs | 2 +- lux-lib/src/operations/vendor.rs | 23 +++++++----- 11 files changed, 67 insertions(+), 62 deletions(-) diff --git a/lux-lib/src/build/luarocks.rs b/lux-lib/src/build/luarocks.rs index 2b899717d..192f94b0a 100644 --- a/lux-lib/src/build/luarocks.rs +++ b/lux-lib/src/build/luarocks.rs @@ -34,9 +34,8 @@ pub enum LuarocksBuildError { #[error(transparent)] #[diagnostic(transparent)] Tree(#[from] TreeError), - #[error(transparent)] - #[diagnostic(transparent)] - Rockspec(Box), + #[error("failed to serialize rockspec: {0}")] + Rockspec(String), #[error("error installing luarocks compatibility layer")] #[diagnostic(forward(0))] LuaVersion(#[from] LuaVersionError), @@ -68,7 +67,7 @@ pub(crate) async fn build( &rockspec_file, rockspec .to_lua_remote_rockspec_string() - .map_err(|err| LuarocksBuildError::Rockspec(Box::new(err)))?, + .map_err(|err| LuarocksBuildError::Rockspec(err.to_string()))?, ) .await?; let luarocks = LuaRocksInstallation::new(config, tree.build_tree(config)?)?; diff --git a/lux-lib/src/operations/build_workspace.rs b/lux-lib/src/operations/build_workspace.rs index ae00c5338..f318376f9 100644 --- a/lux-lib/src/operations/build_workspace.rs +++ b/lux-lib/src/operations/build_workspace.rs @@ -14,18 +14,17 @@ use crate::{ }; use bon::Builder; use thiserror::Error; -use tracing::{info_span, Instrument}; +use tracing::{Instrument, info_span}; #[derive(Debug, Error, miette::Diagnostic)] #[non_exhaustive] pub enum BuildWorkspaceError { - // FIXME(vhyrro): don't use dyn here #[error(transparent)] #[diagnostic(transparent)] InstallWorkspaceDependencies(#[from] InstallWorkspaceDependenciesError), - #[error("failed to build the workspace")] - #[diagnostic(forward(0))] - BuildProject(Box), + #[error(transparent)] + #[diagnostic(transparent)] + BuildProject(#[from] BuildProjectError), #[error(transparent)] #[diagnostic(transparent)] LuaInstallation(#[from] LuaInstallationError), @@ -37,12 +36,6 @@ pub enum BuildWorkspaceError { Workspace(#[from] WorkspaceError), } -impl From for BuildWorkspaceError { - fn from(source: BuildProjectError) -> Self { - Self::BuildProject(Box::new(source)) - } -} - #[derive(Builder)] #[builder(start_fn = new, finish_fn(name = _build, vis = ""))] pub struct BuildWorkspace<'a> { diff --git a/lux-lib/src/operations/pipeline/build.rs b/lux-lib/src/operations/pipeline/build.rs index b714d26ac..c9900771e 100644 --- a/lux-lib/src/operations/pipeline/build.rs +++ b/lux-lib/src/operations/pipeline/build.rs @@ -22,7 +22,7 @@ use super::download_sources_and_hash::{DownloadedPackage, PackageSource}; #[derive(Error, Debug, Diagnostic)] #[non_exhaustive] -pub(crate) enum BuildError { +pub enum BuildError { #[error("failed to build '{0}'")] Build(String, #[source] Box), #[error(transparent)] diff --git a/lux-lib/src/operations/pipeline/build_project.rs b/lux-lib/src/operations/pipeline/build_project.rs index b68794a6b..b3aa62ae3 100644 --- a/lux-lib/src/operations/pipeline/build_project.rs +++ b/lux-lib/src/operations/pipeline/build_project.rs @@ -75,7 +75,7 @@ where #[derive(Error, Debug, Diagnostic)] #[non_exhaustive] -pub(crate) enum BuildProjectError { +pub enum BuildProjectError { #[error("failed to build the project")] #[diagnostic(forward(0))] Build(#[source] Box), diff --git a/lux-lib/src/operations/pipeline/discover.rs b/lux-lib/src/operations/pipeline/discover.rs index a47bd2656..c18838fd1 100644 --- a/lux-lib/src/operations/pipeline/discover.rs +++ b/lux-lib/src/operations/pipeline/discover.rs @@ -26,7 +26,7 @@ pub(crate) enum FoundPackageType { #[derive(Error, Debug, Diagnostic)] #[non_exhaustive] -pub(crate) enum DiscoverError { +pub enum DiscoverError { #[error(transparent)] #[diagnostic(transparent)] Search(#[from] SearchError), diff --git a/lux-lib/src/operations/pipeline/download_sources_and_hash.rs b/lux-lib/src/operations/pipeline/download_sources_and_hash.rs index 7b6afd749..4f8463b35 100644 --- a/lux-lib/src/operations/pipeline/download_sources_and_hash.rs +++ b/lux-lib/src/operations/pipeline/download_sources_and_hash.rs @@ -48,19 +48,25 @@ pub(crate) type DownloadSourcesAndHashArtifacts = #[derive(Error, Debug, Diagnostic)] #[non_exhaustive] -pub(crate) enum DownloadSourcesAndHashError { +pub enum DownloadSourcesAndHashError { #[error(transparent)] #[diagnostic(transparent)] Fs(#[from] FsError), #[error(transparent)] #[diagnostic(transparent)] - Build(#[from] BuildError), + Build(#[from] Box), #[error("failed to hash the source for '{0}'")] Hash(String, #[source] io::Error), #[error("missing source url for '{0}'")] MissingSourceUrl(String), } +impl From for DownloadSourcesAndHashError { + fn from(source: BuildError) -> Self { + Self::Build(Box::new(source)) + } +} + #[derive(Builder)] #[builder(start_fn = new, finish_fn(name = _build, vis = ""))] pub(crate) struct DownloadSourcesAndHash<'a> { diff --git a/lux-lib/src/operations/pipeline/install_packages.rs b/lux-lib/src/operations/pipeline/install_packages.rs index bb9ef1589..5b691cd14 100644 --- a/lux-lib/src/operations/pipeline/install_packages.rs +++ b/lux-lib/src/operations/pipeline/install_packages.rs @@ -18,9 +18,11 @@ use crate::{ }; use super::{ - build::Build as PipelineBuild, discover::FindPackageFromLuarocks, - download_sources_and_hash::DownloadSourcesAndHash, emit_lockfile::LockfileHandle, - resolve::ResolvePackageDependencies, + build::{Build as PipelineBuild, BuildError as PipelineBuildError}, + discover::FindPackageFromLuarocks, + download_sources_and_hash::{DownloadSourcesAndHash, DownloadSourcesAndHashError}, + emit_lockfile::LockfileHandle, + resolve::{ResolveError, ResolvePackageDependencies}, }; use crate::operations::PackageInstallSpec; @@ -161,15 +163,11 @@ where if let Some(test_packages) = install.test_packages { resolve = resolve.test_packages(test_packages); } - let resolved = resolve - .resolve() - .await - .map_err(|err| InstallPackagesError::Pipeline(Box::new(err)))?; + let resolved = resolve.resolve().await?; let artifacts = DownloadSourcesAndHash::new(config) .resolved(resolved) .download_sources_and_hash() - .await - .map_err(|err| InstallPackagesError::Pipeline(Box::new(err)))?; + .await?; let handle = LockfileHandle::from_artifacts(&artifacts); let build_packages = artifacts @@ -191,20 +189,17 @@ where let built_build_deps = PipelineBuild::new(config, &build_tree) .packages(build_packages) .build() - .await - .map_err(|err| InstallPackagesError::Pipeline(Box::new(err)))?; + .await?; let built = PipelineBuild::new(config, tree) .packages(regular_packages) .build() - .await - .map_err(|err| InstallPackagesError::Pipeline(Box::new(err)))?; + .await?; if let Some(test_packages) = artifacts.test { let test_tree = tree.test_tree(config)?; PipelineBuild::new(config, &test_tree) .packages(test_packages.into_values().collect_vec()) .build() - .await - .map_err(|err| InstallPackagesError::Pipeline(Box::new(err)))?; + .await?; } let installed_packages: HashMap = built @@ -264,9 +259,15 @@ pub enum InstallPackagesError { "only a single entrypoint per package is allowed.\nretry with `--force` to overwrite the existing entrypoint" ))] ConflictingEntrypoints(String), - #[error("failed to install packages")] - #[diagnostic(forward(0))] - Pipeline(Box), + #[error(transparent)] + #[diagnostic(transparent)] + Resolve(#[from] ResolveError), + #[error(transparent)] + #[diagnostic(transparent)] + Download(#[from] DownloadSourcesAndHashError), + #[error(transparent)] + #[diagnostic(transparent)] + Build(#[from] PipelineBuildError), } impl From for InstallPackagesError { diff --git a/lux-lib/src/operations/pipeline/install_workspace.rs b/lux-lib/src/operations/pipeline/install_workspace.rs index 927401ac6..3eca0fc54 100644 --- a/lux-lib/src/operations/pipeline/install_workspace.rs +++ b/lux-lib/src/operations/pipeline/install_workspace.rs @@ -12,7 +12,7 @@ use crate::{ workspace::{Workspace, WorkspaceError, WorkspaceTreeError}, }; -use super::install_packages::InstallPackages; +use super::install_packages::{InstallPackages, InstallPackagesError}; /// Installs all of a workspace's dependencies into its regular, build and test trees. /// @@ -65,10 +65,7 @@ where install = install.test_packages(gather_dependencies(workspace, DependencyKind::Test)?); } - let (built, lockfile) = install - .install() - .await - .map_err(|err| InstallWorkspaceDependenciesError::Pipeline(Box::new(err)))?; + let (built, lockfile) = install.install().await?; // Commit the workspace lockfile only once installation has succeeded. if !args.no_lock.unwrap_or(false) { @@ -92,9 +89,15 @@ pub enum InstallWorkspaceDependenciesError { #[error(transparent)] #[diagnostic(transparent)] Project(#[from] LocalProjectTomlValidationError), - #[error("failed to install workspace dependencies")] - #[diagnostic(forward(0))] - Pipeline(Box), + #[error(transparent)] + #[diagnostic(transparent)] + InstallPackages(#[from] Box), +} + +impl From for InstallWorkspaceDependenciesError { + fn from(source: InstallPackagesError) -> Self { + Self::InstallPackages(Box::new(source)) + } } #[derive(Clone, Copy)] diff --git a/lux-lib/src/operations/pipeline/mod.rs b/lux-lib/src/operations/pipeline/mod.rs index 170bb67f3..1ed02ed70 100644 --- a/lux-lib/src/operations/pipeline/mod.rs +++ b/lux-lib/src/operations/pipeline/mod.rs @@ -1,13 +1,13 @@ use crate::lockfile::LockedPackageLockType; -pub(crate) mod build; -pub(crate) mod build_project; -pub(crate) mod discover; -pub(crate) mod download_sources_and_hash; +pub mod build; +pub mod build_project; +pub mod discover; +pub mod download_sources_and_hash; pub mod emit_lockfile; pub mod install_packages; pub mod install_workspace; -pub(crate) mod resolve; +pub mod resolve; /// Packages grouped by the lockfile section they belong to. /// diff --git a/lux-lib/src/operations/pipeline/resolve.rs b/lux-lib/src/operations/pipeline/resolve.rs index 594891ff8..e47bf99e8 100644 --- a/lux-lib/src/operations/pipeline/resolve.rs +++ b/lux-lib/src/operations/pipeline/resolve.rs @@ -73,7 +73,7 @@ pub(crate) type ResolvedArtifacts = Artifacts), + #[error(transparent)] + #[diagnostic(transparent)] + Resolve(#[from] ResolveError), + #[error(transparent)] + #[diagnostic(transparent)] + Download(#[from] DownloadSourcesAndHashError), #[error(transparent)] #[diagnostic(transparent)] Fs(#[from] fs::FsError), @@ -215,13 +220,11 @@ impl<'a> ResolveAndDownload<'a> { let resolved = ResolvePackageDependencies::new(&discover, self.config) .packages(self.install_specs) .resolve() - .await - .map_err(|err| VendorError::Pipeline(Box::new(err)))?; + .await?; let artifacts = DownloadSourcesAndHash::new(self.config) .resolved(resolved) .download_sources_and_hash() - .await - .map_err(|err| VendorError::Pipeline(Box::new(err)))?; + .await?; Ok(artifacts .into_iter() .flat_map(|(_, packages)| packages) From 88cdb4518cac7e4803637bd30799f5b2f144d5ca Mon Sep 17 00:00:00 2001 From: vhyrro Date: Sat, 3 Oct 2026 16:03:08 +0200 Subject: [PATCH 10/39] refactor!: break apart `LockedPackage::binaries()` into the lockfile --- lux-cli/src/doc.rs | 6 +- lux-lib/resources/test/lux.lock | 4 - .../busted-with-lockfile/lux.lock | 11 - .../lockfile-missing-deps/lux.lock | 4 - .../resources/test/sample-tree/5.1/lux.lock | 4 - lux-lib/src/build/backend.rs | 7 +- lux-lib/src/build/cmake.rs | 2 +- lux-lib/src/build/command.rs | 10 +- lux-lib/src/build/external_dependency.rs | 2 +- lux-lib/src/build/luarocks.rs | 4 +- lux-lib/src/build/make.rs | 4 +- lux-lib/src/build/mod.rs | 72 +- lux-lib/src/build/rust_binary.rs | 2 +- lux-lib/src/build/source.rs | 2 +- lux-lib/src/build/utils.rs | 10 +- lux-lib/src/lockfile/mod.rs | 669 ++++++++---------- .../lux_lib__lockfile__tests__add_rocks.snap | 5 - ..._lib__lockfile__tests__parse_lockfile.snap | 5 - lux-lib/src/luarocks/install_binary_rock.rs | 42 +- lux-lib/src/operations/build_workspace.rs | 2 +- lux-lib/src/operations/dist_bin.rs | 36 +- lux-lib/src/operations/gen_luarc.rs | 10 +- lux-lib/src/operations/pack.rs | 21 +- lux-lib/src/operations/pin.rs | 14 +- .../pipeline/download_sources_and_hash.rs | 48 +- .../operations/pipeline/install_packages.rs | 10 +- lux-lib/src/operations/pipeline/resolve.rs | 1 - lux-lib/src/operations/sync.rs | 78 +- lux-lib/src/operations/vendor.rs | 12 +- lux-lib/src/path.rs | 2 +- lux-lib/src/remote_package_db/mod.rs | 6 +- lux-lib/src/rockspec/mod.rs | 6 - lux-lib/src/tree/dist.rs | 8 +- lux-lib/src/tree/layout.rs | 26 +- lux-lib/src/tree/mod.rs | 36 +- lux-lib/src/which/mod.rs | 2 +- lux-lib/tests/build.rs | 14 +- 37 files changed, 546 insertions(+), 651 deletions(-) diff --git a/lux-cli/src/doc.rs b/lux-cli/src/doc.rs index 5e6d883c0..69988ff93 100644 --- a/lux-cli/src/doc.rs +++ b/lux-cli/src/doc.rs @@ -10,7 +10,7 @@ use lux_lib::{ rockspec::Rockspec, tree::{InstallTree, RockMatches, Tree}, }; -use miette::{miette, Context, IntoDiagnostic, Result}; +use miette::{Context, IntoDiagnostic, Result, miette}; use std::io::Read; use std::path::Path; use url::Url; @@ -64,7 +64,7 @@ async fn open_homepage(pkg: LockedPackage, tree: &Tree) -> Result<()> { fn get_homepage(pkg: &LockedPackage, tree: &Tree) -> Result> { let rockspec_content = - std::fs::read_to_string(tree.layout_for(pkg).rockspec_path()).into_diagnostic()?; + std::fs::read_to_string(tree.layout_for(pkg.spec()).rockspec_path()).into_diagnostic()?; let rockspec = RemoteLuaRockspec::new(&rockspec_content)?; Ok(rockspec.description().homepage.clone()) } @@ -89,7 +89,7 @@ fn open_doc_file(path: &Path) -> Result<()> { } async fn open_local_docs(pkg: LockedPackage, tree: &Tree, config: &Config) -> Result<()> { - let doc_dir = tree.layout_for(&pkg).doc; + let doc_dir = tree.layout_for(pkg.spec()).doc; let files: Vec = WalkDir::new(&doc_dir) .into_iter() .filter_map_ok(|file| { diff --git a/lux-lib/resources/test/lux.lock b/lux-lib/resources/test/lux.lock index c8a9283c3..261910019 100644 --- a/lux-lib/resources/test/lux.lock +++ b/lux-lib/resources/test/lux.lock @@ -109,10 +109,6 @@ "name": "lua-cjson", "version": "2.1.0-1", "constraint": "==2.1.0", - "binaries": [ - "json2lua", - "lua2json" - ], "source": "luarocks_rockspec+https://luarocks.org/", "source_url": { "type": "url", diff --git a/lux-lib/resources/test/sample-projects/busted-with-lockfile/lux.lock b/lux-lib/resources/test/sample-projects/busted-with-lockfile/lux.lock index a433b5905..1596a2272 100644 --- a/lux-lib/resources/test/sample-projects/busted-with-lockfile/lux.lock +++ b/lux-lib/resources/test/sample-projects/busted-with-lockfile/lux.lock @@ -98,13 +98,6 @@ "name": "lua_cliargs", "version": "3.0.2-1", "constraint": ">=3.0", - "binaries": [ - "watch-tests.sh", - "release", - "lint", - "docs", - "coverage" - ], "source": "luarocks_rockspec+https://luarocks.org/", "source_url": { "type": "git", @@ -158,10 +151,6 @@ "2bd1c8e4fea66a88aeb65c1c8a3d0efdf188d681d2d47f163b976a771545aac9", "01a3c364614bddff7370223a5a9c4580f8e62d144384148444c518ec5367a59b" ], - "binaries": [ - "busted", - "busted" - ], "source": "luarocks_rockspec+https://luarocks.org/", "source_url": { "type": "git", diff --git a/lux-lib/resources/test/sample-projects/lockfile-missing-deps/lux.lock b/lux-lib/resources/test/sample-projects/lockfile-missing-deps/lux.lock index 098155682..efafe79a2 100644 --- a/lux-lib/resources/test/sample-projects/lockfile-missing-deps/lux.lock +++ b/lux-lib/resources/test/sample-projects/lockfile-missing-deps/lux.lock @@ -178,10 +178,6 @@ "e7c4c9fcd3cb6fe33c1c0a2ab8cdbbe8ee81d11334277d1f5631c80317770769": { "name": "lua-cjson", "version": "2.1.0-1", - "binaries": [ - "lua2json", - "json2lua" - ], "source": "luarocks_rockspec+https://luarocks.org/", "source_url": { "type": "url", diff --git a/lux-lib/resources/test/sample-tree/5.1/lux.lock b/lux-lib/resources/test/sample-tree/5.1/lux.lock index 9449cac51..8d515dbf6 100644 --- a/lux-lib/resources/test/sample-tree/5.1/lux.lock +++ b/lux-lib/resources/test/sample-tree/5.1/lux.lock @@ -108,10 +108,6 @@ "name": "lua-cjson", "version": "2.1.0-1", "constraint": "==2.1.0", - "binaries": [ - "json2lua", - "lua2json" - ], "source": "luarocks_rockspec+https://luarocks.org/", "source_url": { "type": "url", diff --git a/lux-lib/src/build/backend.rs b/lux-lib/src/build/backend.rs index 20070e1c1..23eba0fc6 100644 --- a/lux-lib/src/build/backend.rs +++ b/lux-lib/src/build/backend.rs @@ -7,14 +7,15 @@ use std::{ use bon::Builder; use crate::{ - build::external_dependency::ExternalDependencyInfo, config::Config, lockfile::LockedPackage, - lua_installation::LuaInstallation, lua_rockspec::DeploySpec, tree::InstallTree, + build::external_dependency::ExternalDependencyInfo, config::Config, + lockfile::LockedPackageSpec, lua_installation::LuaInstallation, lua_rockspec::DeploySpec, + tree::InstallTree, }; #[derive(Builder)] #[builder(start_fn(name = "new"))] pub(crate) struct RunBuildArgs<'a, T: InstallTree> { - pub(crate) package: &'a LockedPackage, + pub(crate) package: &'a LockedPackageSpec, pub(crate) no_install: bool, pub(crate) lua: &'a LuaInstallation, pub(crate) external_dependencies: &'a HashMap, diff --git a/lux-lib/src/build/cmake.rs b/lux-lib/src/build/cmake.rs index 6c82e5bfe..21e1712c5 100644 --- a/lux-lib/src/build/cmake.rs +++ b/lux-lib/src/build/cmake.rs @@ -6,7 +6,7 @@ use std::{ }; use thiserror::Error; use tokio::process::Command; -use tracing::{info_span, Instrument}; +use tracing::{Instrument, info_span}; use crate::{ build::{ diff --git a/lux-lib/src/build/command.rs b/lux-lib/src/build/command.rs index 5bfef0f71..79ae505eb 100644 --- a/lux-lib/src/build/command.rs +++ b/lux-lib/src/build/command.rs @@ -7,7 +7,7 @@ use std::{ }; use thiserror::Error; -use tracing::{info_span, Instrument}; +use tracing::{Instrument, info_span}; use which::which; use crate::{ @@ -50,7 +50,9 @@ pub enum CommandError { ShellNotFoundError(#[from] which::Error), #[error("error executing command:\n{command}\n\nerror: {err}")] Io { err: io::Error, command: String }, - #[error("failed to execute command:\n{command}\n\nstatus: {status}\nstdout: {stdout}\nstderr: {stderr}")] + #[error( + "failed to execute command:\n{command}\n\nstatus: {status}\nstdout: {stdout}\nstderr: {stderr}" + )] CommandFailure { command: String, status: ExitStatus, @@ -151,7 +153,7 @@ async fn run_command( return Err(CommandError::Io { err, command: substituted_cmd, - }) + }); } Ok(child) => match child .wait_with_output() @@ -174,7 +176,7 @@ async fn run_command( return Err(CommandError::Io { err, command: substituted_cmd, - }) + }); } }, } diff --git a/lux-lib/src/build/external_dependency.rs b/lux-lib/src/build/external_dependency.rs index 32e16d1d9..4929fe580 100644 --- a/lux-lib/src/build/external_dependency.rs +++ b/lux-lib/src/build/external_dependency.rs @@ -422,7 +422,7 @@ fn is_lib_name(file_name: &str, prefix: &str) -> bool { #[cfg(test)] mod tests { use super::*; - use assert_fs::{prelude::*, TempDir}; + use assert_fs::{TempDir, prelude::*}; #[tokio::test] async fn test_detect_zlib_pkg_config_header() { diff --git a/lux-lib/src/build/luarocks.rs b/lux-lib/src/build/luarocks.rs index 192f94b0a..dcd6fdd6a 100644 --- a/lux-lib/src/build/luarocks.rs +++ b/lux-lib/src/build/luarocks.rs @@ -10,7 +10,7 @@ use std::path::Path; use crate::{ config::Config, - lockfile::LockedPackage, + lockfile::LockedPackageSpec, luarocks::luarocks_installation::{ExecLuaRocksError, LuaRocksError, LuaRocksInstallation}, }; @@ -82,7 +82,7 @@ async fn install( rockspec: &R, luarocks_tree: &Path, tree: &T, - package: &LockedPackage, + package: &LockedPackageSpec, config: &Config, ) -> Result { let layout = tree.layout_for(package); diff --git a/lux-lib/src/build/make.rs b/lux-lib/src/build/make.rs index 82b26c248..ea95426a9 100644 --- a/lux-lib/src/build/make.rs +++ b/lux-lib/src/build/make.rs @@ -8,7 +8,7 @@ use std::{ }; use thiserror::Error; -use tracing::{info_span, Instrument}; +use tracing::{Instrument, info_span}; use crate::{ build::{ @@ -173,7 +173,7 @@ impl BuildBackend for MakeBuildSpec { status: output.status, stdout: String::from_utf8_lossy(&output.stdout).into(), stderr: String::from_utf8_lossy(&output.stderr).into(), - }) + }); } Err(err) => return Err(MakeError::Io(err)), } diff --git a/lux-lib/src/build/mod.rs b/lux-lib/src/build/mod.rs index 0215f00b4..a68ef0598 100644 --- a/lux-lib/src/build/mod.rs +++ b/lux-lib/src/build/mod.rs @@ -16,10 +16,10 @@ use tracing::Instrument; use crate::{ config::Config, hash::HasIntegrity, - lockfile::{LockConstraint, LockedPackage, PinnedState}, + lockfile::{LockConstraint, LockedPackage, LockedPackageSpec, PinnedState}, lua_installation::LuaInstallation, lua_rockspec::BuildBackendSpec, - operations::{pipeline::download_sources_and_hash::fetch_and_hash_source, FetchSrcError}, + operations::{FetchSrcError, pipeline::download_sources_and_hash::fetch_and_hash_source}, remote_package_source::RemotePackageSource, }; use bon::Builder; @@ -40,7 +40,7 @@ use source::SourceBuildError; use ssri::Integrity; use thiserror::Error; use treesitter_parser::TreesitterBuildError; -use utils::{recursive_copy_dir, CompileCFilesError, InstallBinaryError}; +use utils::{CompileCFilesError, InstallBinaryError, recursive_copy_dir}; mod builtin; mod cmake; @@ -248,7 +248,7 @@ async fn run_build( async fn install( rockspec: &R, tree: &T, - package: &LockedPackage, + package: &LockedPackageSpec, lua: &LuaInstallation, build_dir: &Path, entry_type: &EntryType, @@ -323,7 +323,7 @@ async fn install( pub(crate) async fn deploy( rockspec: &R, tree: &T, - mut package: LockedPackage, + package: LockedPackage, lua: &LuaInstallation, source_root: &Path, entry_type: EntryType, @@ -337,8 +337,8 @@ pub(crate) async fn deploy( } // FIXME(vhyrro): Maybe make prepare/finalize a struct with Drop behaviour? If it makes sense only. - tree.prepare(&package)?; - let layout = tree.layout_for(&package); + tree.prepare(&package.spec)?; + let layout = tree.layout_for(&package.spec); let rock_source = rockspec.source().current_platform(); let build_dir = resolve_source_dir( @@ -362,7 +362,7 @@ pub(crate) async fn deploy( let output = run_build( rockspec, RunBuildArgs::new() - .package(&package) + .package(&package.spec) .no_install(false) .lua(lua) .external_dependencies(&external_dependencies) @@ -374,13 +374,14 @@ pub(crate) async fn deploy( ) .await?; - // FIXME(vhyrro): This shouldn't require mutation. - package.spec.binaries.extend(output.binaries); + let mut binaries = rockspec.binaries(); + binaries.extend(output.binaries); + tree.lockfile()?.write_guard().set_binaries(&package, binaries); install( rockspec, tree, - &package, + &package.spec, lua, &build_dir, &entry_type, @@ -424,13 +425,20 @@ where rockspec.validate_lua_version(&lua.version)?; // NOTE: these futures are large; boxing them keeps `do_build`'s frame small. + let spec = LockedPackageSpec::new( + rockspec.package(), + rockspec.version(), + build.constraint, + Vec::new(), + Vec::new(), + &build.pin, + &build.opt, + ); let (package, temp_dir) = Box::pin(fetch_and_hash_source( rockspec, + spec, build.source_spec, build.source, - build.constraint, - build.pin, - build.opt, build.config, )) .await?; @@ -527,12 +535,11 @@ mod tests { use crate::{ config::ConfigBuilder, lockfile::LockedPackageHashes, - lua_installation::{detect_installed_lua_version, LuaInstallation}, + lua_installation::{LuaInstallation, detect_installed_lua_version}, lua_version::LuaVersion, - operations::{unpack_rockspec, DownloadedPackedRockBytes}, + operations::{DownloadedPackedRockBytes, unpack_rockspec}, package::PackageSpec, project::Project, - rockspec::RockBinaries, tree::Tree, }; @@ -560,7 +567,6 @@ mod tests { let package = LockedPackage::from( &PackageSpec::new(rockspec.package().clone(), rockspec.version().clone()), LockConstraint::Unconstrained, - RockBinaries::default(), RemotePackageSource::Test, None, LockedPackageHashes { @@ -572,12 +578,12 @@ mod tests { .unwrap(), }, ); - tree.prepare(&package).unwrap(); - let src_dir = tree.layout_for(&package).src; + tree.prepare(&package.spec).unwrap(); + let src_dir = tree.layout_for(&package.spec).src; run_build( &rockspec, RunBuildArgs::new() - .package(&package) + .package(&package.spec) .no_install(false) .lua(&lua) .external_dependencies(&HashMap::default()) @@ -593,21 +599,27 @@ mod tests { assert!(foo_dir.is_dir()); let foo_init = foo_dir.join("init.lua"); assert!(foo_init.is_file()); - assert!(std::fs::read_to_string(&foo_init) - .unwrap() - .contains("return true")); + assert!( + std::fs::read_to_string(&foo_init) + .unwrap() + .contains("return true") + ); let foo_bar_dir = foo_dir.join("bar"); assert!(foo_bar_dir.is_dir()); let foo_bar_init = foo_bar_dir.join("init.lua"); assert!(foo_bar_init.is_file()); - assert!(std::fs::read_to_string(&foo_bar_init) - .unwrap() - .contains("return true")); + assert!( + std::fs::read_to_string(&foo_bar_init) + .unwrap() + .contains("return true") + ); let foo_bar_baz = foo_bar_dir.join("baz.lua"); assert!(foo_bar_baz.is_file()); - assert!(std::fs::read_to_string(&foo_bar_baz) - .unwrap() - .contains("return true")); + assert!( + std::fs::read_to_string(&foo_bar_baz) + .unwrap() + .contains("return true") + ); let bin_file = tree_dir .child(lua_version.to_string()) .child("bin") diff --git a/lux-lib/src/build/rust_binary.rs b/lux-lib/src/build/rust_binary.rs index 22acda561..8c2dd5b80 100644 --- a/lux-lib/src/build/rust_binary.rs +++ b/lux-lib/src/build/rust_binary.rs @@ -1,6 +1,6 @@ use crate::build::backend::{BuildBackend, BuildInfo, RunBuildArgs}; use crate::build::utils::{self, InstallBinaryError}; -use crate::config::{build, Config}; +use crate::config::{Config, build}; use crate::fs; use crate::lua_rockspec::RustBinaryBuildSpec; use crate::tree::InstallTree; diff --git a/lux-lib/src/build/source.rs b/lux-lib/src/build/source.rs index 9bdf8a358..f08150f09 100644 --- a/lux-lib/src/build/source.rs +++ b/lux-lib/src/build/source.rs @@ -5,8 +5,8 @@ use crate::{ build::backend::{BuildBackend, BuildInfo, RunBuildArgs}, lua_rockspec::{BuildBackendSpec, BuildSpec, LocalLuaRockspec, LuaRockspecError}, project::{ + PROJECT_TOML, ProjectRoot, TomlDeError, project_toml::{LocalProjectTomlValidationError, PartialProjectToml}, - ProjectRoot, TomlDeError, PROJECT_TOML, }, rockspec::Rockspec, tree::InstallTree, diff --git a/lux-lib/src/build/utils.rs b/lux-lib/src/build/utils.rs index 0ad4a2e0c..3f72b033a 100644 --- a/lux-lib/src/build/utils.rs +++ b/lux-lib/src/build/utils.rs @@ -1015,9 +1015,11 @@ mod tests { #[cfg(unix)] set_executable_permissions(&script_path).await.unwrap(); - assert!(Command::new(script_path.to_string_lossy().to_string()) - .status() - .await - .is_ok_and(|status| status.success())); + assert!( + Command::new(script_path.to_string_lossy().to_string()) + .status() + .await + .is_ok_and(|status| status.success()) + ); } } diff --git a/lux-lib/src/lockfile/mod.rs b/lux-lib/src/lockfile/mod.rs index 9808984a0..d06a25c4a 100644 --- a/lux-lib/src/lockfile/mod.rs +++ b/lux-lib/src/lockfile/mod.rs @@ -14,7 +14,7 @@ use std::{ use itertools::Itertools; use miette::Diagnostic; -use serde::{de, Deserialize, Serialize, Serializer}; +use serde::{Deserialize, Serialize, Serializer, de}; use sha2::{Digest, Sha256}; use ssri::Integrity; use strum_macros::EnumIter; @@ -27,8 +27,8 @@ use crate::package::{ PackageVersionReqError, RemotePackageTypeFilterSpec, }; use crate::remote_package_source::RemotePackageSource; -use crate::rockspec::lua_dependency::LuaDependencySpec; use crate::rockspec::RockBinaries; +use crate::rockspec::lua_dependency::LuaDependencySpec; use crate::tree::{EntryType, InstallTree, Tree}; /// Bump this whenever an incompatible change is made to the lockfile format. @@ -56,11 +56,7 @@ impl Display for PinnedState { impl From for PinnedState { fn from(value: bool) -> Self { - if value { - Self::Pinned - } else { - Self::Unpinned - } + if value { Self::Pinned } else { Self::Unpinned } } } @@ -161,16 +157,15 @@ impl<'de> Deserialize<'de> for OptState { } #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub(crate) struct LockedPackageSpec { - pub name: PackageName, - pub version: PackageVersion, - pub pinned: PinnedState, - pub opt: OptState, - pub dependencies: Vec, - pub build_dependencies: Vec, +pub struct LockedPackageSpec { + name: PackageName, + version: PackageVersion, + pinned: PinnedState, + opt: OptState, + dependencies: Vec, + build_dependencies: Vec, // TODO: Deserialize this directly into a `LuaPackageReq` - pub constraint: Option, - pub binaries: RockBinaries, + constraint: Option, } /// ID of a local package, a hash that is comprised of: @@ -239,7 +234,6 @@ impl LockedPackageSpec { build_dependencies: Vec, pinned: &PinnedState, opt: &OptState, - binaries: RockBinaries, ) -> Self { Self { name: name.clone(), @@ -252,7 +246,6 @@ impl LockedPackageSpec { LockConstraint::Unconstrained => None, LockConstraint::Constrained(version_req) => Some(version_req.to_string()), }, - binaries, } } @@ -298,10 +291,6 @@ impl LockedPackageSpec { self.build_dependencies.iter().collect() } - pub fn binaries(&self) -> Vec<&PathBuf> { - self.binaries.iter().collect() - } - pub fn to_package(&self) -> PackageSpec { PackageSpec::new(self.name.clone(), self.version.clone()) } @@ -362,7 +351,12 @@ impl Display for RemotePackageSourceUrl { // TODO(vhyrro): Move to `package/local.rs` -/// A locally installed rock + +/// A package whose source has been downloaded and hashed, but which has not +/// yet been installed into a tree. +/// +/// This is the value produced by the download stage. Installation adds the +/// binaries discovered during the build, producing a [`LockedPackage`]. #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct LockedPackage { pub(crate) spec: LockedPackageSpec, @@ -372,12 +366,116 @@ pub struct LockedPackage { } impl LockedPackage { + /// Returns a copy of this package with a different pin state. + /// + /// Pin state is part of the package id, so the result has a different + /// identity and must be re-inserted under its new id. + pub(crate) fn rekey_pinned(mut self, pinned: PinnedState) -> Self { + self.spec.pinned = pinned; + self + } + + #[cfg(test)] + pub(crate) fn from( + package: &PackageSpec, + constraint: LockConstraint, + source: RemotePackageSource, + source_url: Option, + hashes: LockedPackageHashes, + ) -> Self { + Self::new( + LockedPackageSpec::new( + package.name(), + package.version(), + constraint, + Vec::default(), + Vec::default(), + &PinnedState::Unpinned, + &OptState::Required, + ), + source, + source_url, + hashes, + ) + } + + + pub(crate) fn new( + spec: LockedPackageSpec, + source: RemotePackageSource, + source_url: Option, + hashes: LockedPackageHashes, + ) -> Self { + Self { + spec, + source, + source_url, + hashes, + } + } + + pub fn spec(&self) -> &LockedPackageSpec { + &self.spec + } + pub fn into_package_spec(self) -> PackageSpec { - PackageSpec::new(self.spec.name, self.spec.version) + self.spec.to_package() } pub fn as_package_spec(&self) -> PackageSpec { - PackageSpec::new(self.spec.name.clone(), self.spec.version.clone()) + self.spec.to_package() + } + + pub fn id(&self) -> LockedPackageId { + self.spec.id() + } + + pub fn name(&self) -> &PackageName { + self.spec.name() + } + + pub fn version(&self) -> &PackageVersion { + self.spec.version() + } + + pub fn pinned(&self) -> PinnedState { + self.spec.pinned() + } + + pub fn opt(&self) -> OptState { + self.spec.opt() + } + + pub fn dependencies(&self) -> Vec<&LockedPackageId> { + self.spec.dependencies() + } + + pub fn build_dependencies(&self) -> Vec<&LockedPackageId> { + self.spec.build_dependencies() + } + + pub fn constraint(&self) -> LockConstraint { + self.spec.constraint() + } + + pub(crate) fn source(&self) -> &RemotePackageSource { + &self.source + } + + pub(crate) fn source_url(&self) -> Option<&RemotePackageSourceUrl> { + self.source_url.as_ref() + } + + pub fn hashes(&self) -> &LockedPackageHashes { + &self.hashes + } + + pub fn to_package(&self) -> PackageSpec { + self.spec.to_package() + } + + pub fn into_package_req(self) -> PackageReq { + self.spec.into_package_req() } } @@ -395,8 +493,6 @@ struct LockedPackageIntermediate { build_dependencies: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] constraint: Option, - #[serde(default, skip_serializing_if = "RockBinaries::is_default")] - binaries: RockBinaries, source: RemotePackageSource, #[serde(default, skip_serializing_if = "Option::is_none")] source_url: Option, @@ -417,7 +513,6 @@ impl TryFrom for LockedPackage { value.build_dependencies, &value.pinned, &value.opt, - value.binaries, ), source: value.source, source_url: value.source_url, @@ -436,7 +531,6 @@ impl From<&LockedPackage> for LockedPackageIntermediate { dependencies: value.spec.dependencies.clone(), build_dependencies: value.spec.build_dependencies.clone(), constraint: value.spec.constraint.clone(), - binaries: value.spec.binaries.clone(), source: value.source.clone(), source_url: value.source_url.clone(), hashes: value.hashes.clone(), @@ -463,81 +557,6 @@ impl Serialize for LockedPackage { } } -impl LockedPackage { - pub(crate) fn from( - package: &PackageSpec, - constraint: LockConstraint, - binaries: RockBinaries, - source: RemotePackageSource, - source_url: Option, - hashes: LockedPackageHashes, - ) -> Self { - Self { - spec: LockedPackageSpec::new( - package.name(), - package.version(), - constraint, - Vec::default(), - Vec::default(), - &PinnedState::Unpinned, - &OptState::Required, - binaries, - ), - source, - source_url, - hashes, - } - } - - pub fn id(&self) -> LockedPackageId { - self.spec.id() - } - - pub fn name(&self) -> &PackageName { - self.spec.name() - } - - pub fn version(&self) -> &PackageVersion { - self.spec.version() - } - - pub fn pinned(&self) -> PinnedState { - self.spec.pinned() - } - - pub fn opt(&self) -> OptState { - self.spec.opt() - } - - pub(crate) fn source(&self) -> &RemotePackageSource { - &self.source - } - - pub fn dependencies(&self) -> Vec<&LockedPackageId> { - self.spec.dependencies() - } - - pub fn build_dependencies(&self) -> Vec<&LockedPackageId> { - self.spec.build_dependencies() - } - - pub fn constraint(&self) -> LockConstraint { - self.spec.constraint() - } - - pub fn hashes(&self) -> &LockedPackageHashes { - &self.hashes - } - - pub fn to_package(&self) -> PackageSpec { - self.spec.to_package() - } - - pub fn into_package_req(self) -> PackageReq { - self.spec.into_package_req() - } -} - #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Hash)] pub struct LockedPackageHashes { pub rockspec: Integrity, @@ -712,7 +731,7 @@ impl LockedPackageLock { packages .iter() .filter(|package| match &filter { - Some(filter_spec) => match package.source { + Some(filter_spec) => match &package.source { RemotePackageSource::LuarocksRockspec(_) => filter_spec.rockspec, RemotePackageSource::LuarocksSrcRock(_) => filter_spec.src, RemotePackageSource::LuarocksBinaryRock(_) => filter_spec.binary, @@ -754,7 +773,7 @@ impl LockedPackageLock { ) -> PackageSyncSpec { let pkg_dir_exists = |pkg: &LockedPackage| match strategy { SyncStrategy::LockfileOnly => true, - SyncStrategy::EnsureInstalled(tree) => tree.layout_for(pkg).root.is_dir(), + SyncStrategy::EnsureInstalled(tree) => tree.layout_for(pkg.spec()).root.is_dir(), }; let entrypoints_to_keep: HashSet = self @@ -830,6 +849,10 @@ pub struct Lockfile { _marker: PhantomData

, // TODO: Serialize this directly into a `Version` version: String, + /// Binaries installed into the tree for each package, discovered during + /// the build. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + binaries: BTreeMap, #[serde(flatten)] lock: LockedPackageLock, } @@ -890,9 +913,6 @@ pub enum LockfileError { expected: String, help: String, }, - #[error("cannot migrate lockfile '{}'", lockfile.display())] - #[diagnostic(code(lux_lib::lockfile::conflicting_entrypoints), help("{help}"))] - ConflictingEntrypoints { lockfile: PathBuf, help: String }, } #[derive(Error, Debug, Diagnostic)] @@ -931,7 +951,9 @@ check the source URL, then rerun the command with `lx --no-lock` to update the h expected: Integrity, got: Integrity, }, - #[error("package {0} version {1} with pinned state {2} and constraint {3} not found in the lockfile.")] + #[error( + "package {0} version {1} with pinned state {2} and constraint {3} not found in the lockfile." + )] PackageNotFound(PackageName, Box, PinnedState, String), } @@ -996,6 +1018,11 @@ impl Lockfile

{ } } + /// The binaries installed into the tree for `id`, if recorded. + pub(crate) fn binaries(&self, id: &LockedPackageId) -> Option<&RockBinaries> { + self.binaries.get(id) + } + pub(crate) fn local_pkg_lock(&self) -> &LockedPackageLock { &self.lock } @@ -1191,6 +1218,7 @@ impl Lockfile { filepath: filepath.clone(), _marker: PhantomData, version: LOCKFILE_VERSION_STR.into(), + binaries: BTreeMap::new(), lock: LockedPackageLock::default(), }; let json_str = @@ -1205,7 +1233,7 @@ impl Lockfile { return Err(LockfileError::Fs(fs::FsError::FileOpen { path: filepath.to_path_buf(), source, - })) + })); } } @@ -1222,7 +1250,7 @@ impl Lockfile { remove the tree at '{}' and reinstall all packages it contained.", tree_dir.display() ); - let mut lockfile: Lockfile = load_lockfile(&content, &filepath, &help)?; + let mut lockfile: Lockfile = parse_lockfile(&content, &filepath, &help)?; lockfile.filepath = filepath; Ok(lockfile) } @@ -1233,6 +1261,7 @@ remove the tree at '{}' and reinstall all packages it contained.", _marker: PhantomData, filepath: self.filepath, version: self.version, + binaries: self.binaries, lock: self.lock, } } @@ -1311,7 +1340,7 @@ impl WorkspaceLockfile { return Err(LockfileError::Fs(fs::FsError::FileOpen { path: filepath.to_path_buf(), source, - })) + })); } } @@ -1322,7 +1351,7 @@ impl WorkspaceLockfile { #[tracing::instrument(level = "trace")] pub fn load(filepath: PathBuf) -> Result, LockfileError> { let content = fs::sync::read_to_string(&filepath)?; - let mut lockfile: WorkspaceLockfile = load_lockfile( + let mut lockfile: WorkspaceLockfile = parse_lockfile( &content, &filepath, "this lockfile was created by an incompatible version of Lux. @@ -1415,11 +1444,18 @@ impl Lockfile { }); } + /// Record the binaries installed for `package` into the tree lockfile. + pub(crate) fn set_binaries(&mut self, package: &LockedPackage, binaries: RockBinaries) { + self.binaries.insert(package.id(), binaries); + } + pub(crate) fn remove(&mut self, target: &LockedPackage) { + self.binaries.remove(&target.id()); self.lock.remove(target) } pub(crate) fn remove_by_id(&mut self, target: &LockedPackageId) { + self.binaries.remove(target); self.lock.remove_by_id(target) } @@ -1539,133 +1575,36 @@ impl Drop for ProjectLockfileGuard { } } -fn load_lockfile(content: &str, lockfile: &Path, help: &str) -> Result +fn parse_lockfile(content: &str, lockfile: &Path, help: &str) -> Result where T: serde::de::DeserializeOwned, { - let mut value = parse_lockfile_json(content)?; - if lockfile_version(&value, lockfile)?.major == 1 { - migrate_v1_lockfile(&mut value, lockfile)?; - } - deserialize_lockfile(value, lockfile, help) -} + let value: serde_json::Value = + serde_json::from_str(content).map_err(LockfileError::ParseJson)?; -fn parse_lockfile_json( - content: &str, -) -> Result, LockfileError> { - serde_json::from_str(content).map_err(LockfileError::ParseJson) -} - -fn lockfile_version( - value: &serde_json::Map, - lockfile: &Path, -) -> Result { let version = value .get("version") .and_then(|version| version.as_str()) - .ok_or(LockfileError::MissingVersion { + .ok_or_else(|| LockfileError::MissingVersion { lockfile: lockfile.to_path_buf(), })?; - semver::Version::parse(version).map_err(|source| LockfileError::InvalidVersion { - version: version.to_string(), - source, - }) -} -fn deserialize_lockfile( - value: serde_json::Map, - lockfile: &Path, - help: &str, -) -> Result -where - T: serde::de::DeserializeOwned, -{ - let found = lockfile_version(&value, lockfile)?; + let found = + semver::Version::parse(version).map_err(|source| LockfileError::InvalidVersion { + version: version.to_string(), + source, + })?; + if found.major != LOCKFILE_VERSION { return Err(LockfileError::IncompatibleVersion { lockfile: lockfile.to_path_buf(), - found: found.to_string(), + found: version.into(), expected: LOCKFILE_VERSION_STR.into(), help: help.into(), }); } - serde_json::from_value(serde_json::Value::Object(value)).map_err(LockfileError::ParseJson) -} - -fn migrate_v1_lockfile( - value: &mut serde_json::Map, - lockfile: &Path, -) -> Result<(), LockfileError> { - let tree_dir = lockfile.parent().unwrap_or(lockfile); - let found = lockfile_version(value, lockfile)?; - if value.contains_key("entrypoint_layout") { - return Err(LockfileError::IncompatibleVersion { - lockfile: lockfile.to_path_buf(), - found: found.to_string(), - expected: LOCKFILE_VERSION_STR.into(), - help: format!( - "this install tree uses a custom entrypoint layout that cannot be migrated automatically. -remove the tree at '{}' and reinstall all packages it contained.", - tree_dir.display() - ), - }); - } - let conflict_help = format!( - "this lockfile contains entrypoints that cannot be represented in the current format. -remove the tree at '{}' and reinstall all packages it contained.", - tree_dir.display() - ); - - if value.contains_key("entrypoints") { - migrate_entrypoints(value, lockfile, &conflict_help)?; - } - for section in ["dependencies", "test_dependencies", "build_dependencies"] { - if let Some(section) = value - .get_mut(section) - .and_then(|section| section.as_object_mut()) - { - migrate_entrypoints(section, lockfile, &conflict_help)?; - } - } - value.insert( - "version".into(), - serde_json::Value::String(LOCKFILE_VERSION_STR.into()), - ); - Ok(()) -} - -fn migrate_entrypoints( - section: &mut serde_json::Map, - lockfile: &Path, - help: &str, -) -> Result<(), LockfileError> { - let conflicting_entrypoints = || LockfileError::ConflictingEntrypoints { - lockfile: lockfile.to_path_buf(), - help: help.to_string(), - }; - - let rocks = section.get("rocks").and_then(|rocks| rocks.as_object()); - let mut entrypoints = serde_json::Map::new(); - if let Some(old_entrypoints) = section.get("entrypoints") { - for id in old_entrypoints - .as_array() - .ok_or_else(&conflicting_entrypoints)? - { - let id = id.as_str().ok_or_else(&conflicting_entrypoints)?; - let name = rocks - .and_then(|rocks| rocks.get(id)) - .and_then(|package| package.get("name")) - .and_then(|name| name.as_str()) - .ok_or_else(&conflicting_entrypoints)?; - if entrypoints.contains_key(name) { - return Err(conflicting_entrypoints()); - } - entrypoints.insert(name.to_string(), serde_json::Value::String(id.to_string())); - } - } - section.insert("entrypoints".into(), serde_json::Value::Object(entrypoints)); - Ok(()) + serde_json::from_value(value).map_err(LockfileError::ParseJson) } fn integrity_err_not_found(package: &LockedPackage) -> LockfileIntegrityError { @@ -1756,7 +1695,6 @@ mod tests { let test_local_package = LockedPackage::from( &test_package, crate::lockfile::LockConstraint::Unconstrained, - RockBinaries::default(), RemotePackageSource::Test, None, mock_hashes.clone(), @@ -1768,7 +1706,6 @@ mod tests { let mut test_local_dep_package = LockedPackage::from( &test_dep_package, crate::lockfile::LockConstraint::Constrained(">= 1.0.0".parse().unwrap()), - RockBinaries::default(), RemotePackageSource::Test, None, mock_hashes.clone(), @@ -1821,33 +1758,43 @@ mod tests { assert_eq!(sync_spec.to_add.len(), 1); // Should keep dependencies of neorg 8.8.1-1 - assert!(!sync_spec - .to_remove - .iter() - .any(|pkg| pkg.name().to_string() == "nvim-nio" - && pkg.constraint() - == LockConstraint::Constrained(">=1.7.0, <1.8.0".parse().unwrap()))); - assert!(!sync_spec - .to_remove - .iter() - .any(|pkg| pkg.name().to_string() == "lua-utils.nvim" - && pkg.constraint() == LockConstraint::Constrained("=1.0.2".parse().unwrap()))); - assert!(!sync_spec - .to_remove - .iter() - .any(|pkg| pkg.name().to_string() == "plenary.nvim" - && pkg.constraint() == LockConstraint::Constrained("=0.1.4".parse().unwrap()))); - assert!(!sync_spec - .to_remove - .iter() - .any(|pkg| pkg.name().to_string() == "nui.nvim" - && pkg.constraint() == LockConstraint::Constrained("=0.3.0".parse().unwrap()))); - assert!(!sync_spec - .to_remove - .iter() - .any(|pkg| pkg.name().to_string() == "pathlib.nvim" - && pkg.constraint() - == LockConstraint::Constrained(">=2.2.0, <2.3.0".parse().unwrap()))); + assert!( + !sync_spec + .to_remove + .iter() + .any(|pkg| pkg.name().to_string() == "nvim-nio" + && pkg.constraint() + == LockConstraint::Constrained(">=1.7.0, <1.8.0".parse().unwrap())) + ); + assert!( + !sync_spec + .to_remove + .iter() + .any(|pkg| pkg.name().to_string() == "lua-utils.nvim" + && pkg.constraint() == LockConstraint::Constrained("=1.0.2".parse().unwrap())) + ); + assert!( + !sync_spec + .to_remove + .iter() + .any(|pkg| pkg.name().to_string() == "plenary.nvim" + && pkg.constraint() == LockConstraint::Constrained("=0.1.4".parse().unwrap())) + ); + assert!( + !sync_spec + .to_remove + .iter() + .any(|pkg| pkg.name().to_string() == "nui.nvim" + && pkg.constraint() == LockConstraint::Constrained("=0.3.0".parse().unwrap())) + ); + assert!( + !sync_spec + .to_remove + .iter() + .any(|pkg| pkg.name().to_string() == "pathlib.nvim" + && pkg.constraint() + == LockConstraint::Constrained(">=2.2.0, <2.3.0".parse().unwrap())) + ); } #[test] @@ -1867,38 +1814,50 @@ mod tests { // Should remove: // - neorg // - dependencies unique to neorg - assert!(sync_spec - .to_remove - .iter() - .any(|pkg| pkg.name().to_string() == "neorg" - && pkg.version() == &"8.8.1-1".parse().unwrap())); - assert!(sync_spec - .to_remove - .iter() - .any(|pkg| pkg.name().to_string() == "nvim-nio" - && pkg.constraint() - == LockConstraint::Constrained(">=1.7.0, <1.8.0".parse().unwrap()))); - assert!(sync_spec - .to_remove - .iter() - .any(|pkg| pkg.name().to_string() == "lua-utils.nvim" - && pkg.constraint() == LockConstraint::Constrained("=1.0.2".parse().unwrap()))); - assert!(sync_spec - .to_remove - .iter() - .any(|pkg| pkg.name().to_string() == "plenary.nvim" - && pkg.constraint() == LockConstraint::Constrained("=0.1.4".parse().unwrap()))); - assert!(sync_spec - .to_remove - .iter() - .any(|pkg| pkg.name().to_string() == "nui.nvim" - && pkg.constraint() == LockConstraint::Constrained("=0.3.0".parse().unwrap()))); - assert!(sync_spec - .to_remove - .iter() - .any(|pkg| pkg.name().to_string() == "pathlib.nvim" - && pkg.constraint() - == LockConstraint::Constrained(">=2.2.0, <2.3.0".parse().unwrap()))); + assert!( + sync_spec + .to_remove + .iter() + .any(|pkg| pkg.name().to_string() == "neorg" + && pkg.version() == &"8.8.1-1".parse().unwrap()) + ); + assert!( + sync_spec + .to_remove + .iter() + .any(|pkg| pkg.name().to_string() == "nvim-nio" + && pkg.constraint() + == LockConstraint::Constrained(">=1.7.0, <1.8.0".parse().unwrap())) + ); + assert!( + sync_spec + .to_remove + .iter() + .any(|pkg| pkg.name().to_string() == "lua-utils.nvim" + && pkg.constraint() == LockConstraint::Constrained("=1.0.2".parse().unwrap())) + ); + assert!( + sync_spec + .to_remove + .iter() + .any(|pkg| pkg.name().to_string() == "plenary.nvim" + && pkg.constraint() == LockConstraint::Constrained("=0.1.4".parse().unwrap())) + ); + assert!( + sync_spec + .to_remove + .iter() + .any(|pkg| pkg.name().to_string() == "nui.nvim" + && pkg.constraint() == LockConstraint::Constrained("=0.3.0".parse().unwrap())) + ); + assert!( + sync_spec + .to_remove + .iter() + .any(|pkg| pkg.name().to_string() == "pathlib.nvim" + && pkg.constraint() + == LockConstraint::Constrained(">=2.2.0, <2.3.0".parse().unwrap())) + ); } #[test] @@ -1923,15 +1882,19 @@ mod tests { .package_sync_spec(&packages, &SyncStrategy::LockfileOnly); let expected: PackageVersionReq = ">=2.0.0".parse().unwrap(); - assert!(sync_spec - .to_add - .iter() - .any(|req| req.name().to_string() == "nvim-nio" && req.version_req() == &expected)); + assert!( + sync_spec + .to_add + .iter() + .any(|req| req.name().to_string() == "nvim-nio" && req.version_req() == &expected) + ); - assert!(sync_spec - .to_remove - .iter() - .any(|pkg| pkg.name().to_string() == "nvim-nio")); + assert!( + sync_spec + .to_remove + .iter() + .any(|pkg| pkg.name().to_string() == "nvim-nio") + ); } #[test] @@ -1964,103 +1927,43 @@ mod tests { .lock .package_sync_spec(&packages, &SyncStrategy::EnsureInstalled(&tree)); - assert!(!sync_spec - .to_add - .iter() - .any(|req| req.name().to_string() == "neorg")); + assert!( + !sync_spec + .to_add + .iter() + .any(|req| req.name().to_string() == "neorg") + ); - assert!(sync_spec - .to_add - .iter() - .any(|req| req.name().to_string() == "lua-cjson")); + assert!( + sync_spec + .to_add + .iter() + .any(|req| req.name().to_string() == "lua-cjson") + ); - assert!(sync_spec - .to_add - .iter() - .any(|req| req.name().to_string() == "nonexistent")); + assert!( + sync_spec + .to_add + .iter() + .any(|req| req.name().to_string() == "nonexistent") + ); } #[test] fn verify_lockfile_version() { - let deserialize = |version: &str| { - let content = format!(r#"{{"version": "{version}"}}"#); - parse_lockfile_json(&content).and_then(|value| { - deserialize_lockfile::>(value, Path::new("lux.lock"), "") - }) - }; - - assert!(matches!( - deserialize("3.0.0"), - Err(LockfileError::IncompatibleVersion { .. }) - )); - assert!(matches!( - deserialize("not-a-version"), - Err(LockfileError::InvalidVersion { .. }) - )); - } - - #[test] - fn migrate_v1_tree_lockfile() { - let sample = - PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("resources/test/lockfile-v1/tree.lock"); - - let temp = assert_fs::TempDir::new().unwrap(); - let lockfile_path = temp.path().join("lux.lock"); - std::fs::copy(&sample, &lockfile_path).unwrap(); - - let lockfile = Lockfile::load(lockfile_path.clone()).unwrap(); - - let on_disk: serde_json::Value = - serde_json::from_str(&std::fs::read_to_string(&lockfile_path).unwrap()).unwrap(); - assert_eq!(on_disk["version"], "1.0.0"); - assert!(on_disk["entrypoints"].is_array()); - - assert_eq!(lockfile.version, "2.0.0"); - assert_eq!(lockfile.lock.entrypoints.len(), 3); - } - - #[test] - fn refuse_migration_for_custom_layout() { - let lockfile = r#"{"version":"1.0.0","entrypoint_layout":{"root":"site/pack/lux"}}"#; - assert!(matches!( - load_lockfile::>(lockfile, Path::new("lux.lock"), ""), - Err(LockfileError::IncompatibleVersion { .. }) - )); - } - - #[test] - fn refuse_migration_for_conflicting_entrypoints() { - let lockfile = r#"{ - "version": "1.0.0", - "rocks": { - "a": { "name": "foo" }, - "b": { "name": "foo" } - }, - "entrypoints": ["a", "b"] - }"#; - assert!(matches!( - load_lockfile::>(lockfile, Path::new("lux.lock"), ""), - Err(LockfileError::ConflictingEntrypoints { .. }) - )); - } - - #[test] - fn migrate_v1_workspace_lockfile() { - let sample = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("resources/test/lockfile-v1/workspace.lock"); - - let temp = assert_fs::TempDir::new().unwrap(); - let lockfile_path = temp.path().join("lux.lock"); - std::fs::copy(&sample, &lockfile_path).unwrap(); - - let lockfile = WorkspaceLockfile::load(lockfile_path.clone()).unwrap(); - - let on_disk: serde_json::Value = - serde_json::from_str(&std::fs::read_to_string(&lockfile_path).unwrap()).unwrap(); - assert_eq!(on_disk["version"], "1.0.0"); - assert!(on_disk["dependencies"]["entrypoints"].is_array()); + for version in ["1.0.0", "9999.0.0", "not-a-version"] { + let lockfile = format!( + r#" + {{ + "version": {version} + }} + "# + ); - assert_eq!(lockfile.version, "2.0.0"); - assert_eq!(lockfile.dependencies.entrypoints.len(), 3); + assert!( + super::parse_lockfile::>(&lockfile, Path::new("lux.lock"), "") + .is_err() + ); + } } } diff --git a/lux-lib/src/lockfile/snapshots/lux_lib__lockfile__tests__add_rocks.snap b/lux-lib/src/lockfile/snapshots/lux_lib__lockfile__tests__add_rocks.snap index 720fa4fc7..a112ab549 100644 --- a/lux-lib/src/lockfile/snapshots/lux_lib__lockfile__tests__add_rocks.snap +++ b/lux-lib/src/lockfile/snapshots/lux_lib__lockfile__tests__add_rocks.snap @@ -1,6 +1,5 @@ --- source: lux-lib/src/lockfile/mod.rs -assertion_line: 1494 expression: lockfile --- { @@ -136,10 +135,6 @@ expression: lockfile "name": "lua-cjson", "version": "2.1.0-1", "constraint": "==2.1.0", - "binaries": [ - "json2lua", - "lua2json" - ], "source": "luarocks_rockspec+https://luarocks.org/", "source_url": { "type": "url", diff --git a/lux-lib/src/lockfile/snapshots/lux_lib__lockfile__tests__parse_lockfile.snap b/lux-lib/src/lockfile/snapshots/lux_lib__lockfile__tests__parse_lockfile.snap index 88b3e09af..729d525a0 100644 --- a/lux-lib/src/lockfile/snapshots/lux_lib__lockfile__tests__parse_lockfile.snap +++ b/lux-lib/src/lockfile/snapshots/lux_lib__lockfile__tests__parse_lockfile.snap @@ -1,6 +1,5 @@ --- source: lux-lib/src/lockfile/mod.rs -assertion_line: 1441 expression: lockfile --- { @@ -113,10 +112,6 @@ expression: lockfile "name": "lua-cjson", "version": "2.1.0-1", "constraint": "==2.1.0", - "binaries": [ - "json2lua", - "lua2json" - ], "source": "luarocks_rockspec+https://luarocks.org/", "source_url": { "type": "url", diff --git a/lux-lib/src/luarocks/install_binary_rock.rs b/lux-lib/src/luarocks/install_binary_rock.rs index 21ddcc87e..a6cd88500 100644 --- a/lux-lib/src/luarocks/install_binary_rock.rs +++ b/lux-lib/src/luarocks/install_binary_rock.rs @@ -6,18 +6,18 @@ use std::{ use crate::{ build::{ + BuildBehaviour, external_dependency::{ExternalDependencyError, ExternalDependencyInfo}, utils::recursive_copy_dir, - BuildBehaviour, }, config::Config, hash::HasIntegrity, lockfile::{ - LockConstraint, LockedPackage, LockedPackageHashes, LockfileError, OptState, PinnedState, + LockConstraint, LockedPackage, LockedPackageHashes, LockedPackageSpec, LockfileError, + OptState, PinnedState, }, lua_rockspec::{LuaVersionError, RemoteLuaRockspec}, luarocks::rock_manifest::RockManifest, - package::PackageSpec, remote_package_source::RemotePackageSource, rockspec::Rockspec, tree::{self, InstallTree, TreeError}, @@ -51,14 +51,14 @@ pub enum InstallBinaryRockError { LuaVersionError(#[from] LuaVersionError), #[error("failed to unpack packed rock")] Zip(#[from] zip::result::ZipError), - #[error("rock_manifest not found. Cannot install rock files that were packed using LuaRocks version 1")] + #[error( + "rock_manifest not found. Cannot install rock files that were packed using LuaRocks version 1" + )] RockManifestNotFound, #[error(transparent)] #[diagnostic(transparent)] RockManifestError(#[from] RockManifestError), - #[error( - "the entry {0} listed in the `rock_manifest` is neither a file nor a directory: {1:?}" - )] + #[error("the entry {0} listed in the `rock_manifest` is neither a file nor a directory: {1:?}")] NotAFileOrDirectory(String, Box), } @@ -139,16 +139,16 @@ where } _ => None, }; - let mut package = LockedPackage::from( - &PackageSpec::new(rockspec.package().clone(), rockspec.version().clone()), + let spec = LockedPackageSpec::new( + rockspec.package(), + rockspec.version(), self.constraint, - rockspec.binaries(), - self.source, - source_url, - hashes, + Vec::new(), + Vec::new(), + &self.pin, + &self.opt, ); - package.spec.pinned = self.pin; - package.spec.opt = self.opt; + let package = LockedPackage::new(spec, self.source, source_url, hashes); match self.tree.lockfile()?.get(&package.id()) { Some(package) if self.behaviour == BuildBehaviour::NoForce => Ok(package.clone()), _ => { @@ -166,8 +166,8 @@ where return Err(InstallBinaryRockError::RockManifestNotFound); } let rock_manifest_content = fs::tokio::read_to_string(rock_manifest_file).await?; - self.tree.prepare(&package)?; - let layout = self.tree.layout_for(&package); + self.tree.prepare(&package.spec)?; + let layout = self.tree.layout_for(&package.spec); let rock_manifest = RockManifest::new(&rock_manifest_content)?; install_manifest_entries( &rock_manifest.lib.entries, @@ -209,6 +209,10 @@ where fs::tokio::remove_file(&rockspec_path).await?; } self.tree.finalize(&package, self.entry_type)?; + self.tree + .lockfile()? + .write_guard() + .set_binaries(&package, rockspec.binaries()); Ok(package) } } @@ -249,7 +253,7 @@ mod tests { use crate::{ config::ConfigBuilder, - operations::{unpack_rockspec, DownloadedPackedRockBytes}, + operations::{DownloadedPackedRockBytes, unpack_rockspec}, }; use super::*; @@ -305,7 +309,7 @@ mod tests { .unwrap() ); let foo_bar_module = tree - .layout_for(&local_package) + .layout_for(&local_package.spec) .src .join("foo") .join("bar.lua"); diff --git a/lux-lib/src/operations/build_workspace.rs b/lux-lib/src/operations/build_workspace.rs index f318376f9..4df33a441 100644 --- a/lux-lib/src/operations/build_workspace.rs +++ b/lux-lib/src/operations/build_workspace.rs @@ -192,7 +192,7 @@ mod tests { .unwrap(); let package = package.first().unwrap(); let tree = workspace.tree(&config).unwrap(); - let src_dir = tree.layout_for(package).src; + let src_dir = tree.layout_for(&package.spec).src; assert!(src_dir.join("init.lua").is_file()); } } diff --git a/lux-lib/src/operations/dist_bin.rs b/lux-lib/src/operations/dist_bin.rs index 51355b4cd..5593fc267 100644 --- a/lux-lib/src/operations/dist_bin.rs +++ b/lux-lib/src/operations/dist_bin.rs @@ -16,7 +16,7 @@ use crate::{ lua_installation::{LuaInstallation, LuaInstallationError}, lua_rockspec::LuaModule, operations::{InstallProject, InstallProjectError}, - project::{project_toml::LocalProjectTomlValidationError, Project}, + project::{Project, project_toml::LocalProjectTomlValidationError}, rockspec::Rockspec, tree::{InstallTree, TreeError}, }; @@ -154,7 +154,7 @@ where std::env::current_dir()?.join(output) }; - let lib_root = args.tree.layout_for(&package).lib; + let lib_root = args.tree.layout_for(&package.spec).lib; let c_src = generate_c_source(&entrypoint_module, &files, &lib_root).await?; let work_dir = fs::tempfile::tempdir()?; @@ -173,7 +173,7 @@ fn collect_installed_files(tree: &impl InstallTree) -> Result) -> Result<(), GenLuaRcError> { .local_pkg_lock(&LockedPackageLockType::Regular) .rocks() .values() - .map(|dependency| dependency_tree.layout_for(dependency).src) + .map(|dependency| dependency_tree.layout_for(&dependency.spec).src) .filter(|dir| dir.is_dir()) .filter_map(|dependency_dir| diff_paths(dependency_dir, workspace.root())); @@ -96,7 +96,11 @@ async fn do_generate_luarc(args: GenLuaRc<'_>) -> Result<(), GenLuaRcError> { .local_pkg_lock(&LockedPackageLockType::Test) .rocks() .values() - .map(|dependency| test_dependency_tree.layout_for(dependency).src) + .map(|dependency| { + test_dependency_tree + .layout_for(&dependency.spec) + .src + }) .filter(|dir| dir.is_dir()) .filter_map(|test_dependency_dir| diff_paths(test_dependency_dir, workspace.root())); diff --git a/lux-lib/src/operations/pack.rs b/lux-lib/src/operations/pack.rs index a84e3c6a8..1ad00420a 100644 --- a/lux-lib/src/operations/pack.rs +++ b/lux-lib/src/operations/pack.rs @@ -13,6 +13,7 @@ use crate::luarocks::rock_manifest::RockManifestLua; use crate::luarocks::rock_manifest::RockManifestRoot; use crate::tree::InstallTree; use crate::tree::Tree; +use crate::tree::TreeError; use bon::Builder; use clean_path::Clean; use itertools::Itertools; @@ -27,8 +28,8 @@ use std::path::Path; use std::path::PathBuf; use thiserror::Error; use walkdir::WalkDir; -use zip::write::SimpleFileOptions; use zip::ZipWriter; +use zip::write::SimpleFileOptions; #[cfg(unix)] use std::os::unix::fs::PermissionsExt; @@ -63,6 +64,9 @@ pub enum PackError { #[diagnostic(transparent)] Fs(#[from] fs::FsError), Walkdir(#[from] walkdir::Error), + #[error(transparent)] + #[diagnostic(transparent)] + Tree(#[from] TreeError), #[error("expected a `package.rockspec` in the package root.")] MissingRockspec, } @@ -70,7 +74,7 @@ pub enum PackError { async fn do_pack(args: Pack) -> Result { let package = args.package; let tree = args.tree; - let layout = tree.layout_for(&package); + let layout = tree.layout_for(&package.spec); let suffix = if is_binary_rock(&layout.lib) { format!("{}.rock", luarocks::current_platform_luarocks_identifier()) } else { @@ -109,7 +113,12 @@ async fn do_pack(args: Pack) -> Result { fs::tokio::copy(layout.rockspec_path(), &renamed_rockspec_entry).await?; let root_entries = add_rock_entries(&mut zip, temp_root_dir.path(), "".into())?; let mut bin_entries = HashMap::new(); - for relative_binary_path in package.spec.binaries() { + let lockfile = tree.lockfile()?; + for relative_binary_path in lockfile + .binaries(&package.id()) + .into_iter() + .flat_map(|binaries| binaries.iter()) + { if let Some(binary_name) = relative_binary_path.clean().file_name() { let binary_path = tree.bin().join(binary_name); if binary_path.is_file() { @@ -171,11 +180,7 @@ fn add_rock_entries( if source_dir.is_dir() { for file in WalkDir::new(source_dir).into_iter().filter_map_ok(|entry| { let file = entry.into_path(); - if file.is_file() { - Some(file) - } else { - None - } + if file.is_file() { Some(file) } else { None } }) { let file = file?; let (relative_path, digest) = add_rock_entry(zip, file, source_dir, &zip_dir)?; diff --git a/lux-lib/src/operations/pin.rs b/lux-lib/src/operations/pin.rs index 497115937..78540023d 100644 --- a/lux-lib/src/operations/pin.rs +++ b/lux-lib/src/operations/pin.rs @@ -38,7 +38,9 @@ pub enum PinError { #[diagnostic(help("make sure Lux has write access to the install directory"))] MoveItemsFailure(#[from] fs_extra::error::Error), #[error("cannot change pin state of {rock}, because it is not an entrypoint")] - #[diagnostic(help("Lux does not allow pinning dependencies, as doing so could break the version requirement in a future update."))] + #[diagnostic(help( + "Lux does not allow pinning dependencies, as doing so could break the version requirement in a future update." + ))] NotAnEntrypoint { rock: PackageSpec }, #[error(transparent)] #[diagnostic(transparent)] @@ -51,7 +53,7 @@ pub fn set_pinned_state( pin: PinnedState, ) -> Result<(), PinError> { let lockfile = tree.lockfile()?; - let mut package = lockfile + let package = lockfile .get(package_id) .ok_or_else(|| PinError::PackageNotFound(package_id.clone()))? .clone(); @@ -70,13 +72,13 @@ pub fn set_pinned_state( } let old_package = package.clone(); - let layout = tree.layout_for(&package); + let layout = tree.layout_for(&package.spec); let items = fs::sync::read_dir(&layout.root)? .filter_map(Result::ok) .map(|dir| dir.path()) .collect_vec(); - package.spec.pinned = pin; + let package = package.rekey_pinned(pin); if lockfile.get(&package.id()).is_some() { return Err(PinError::PinStateConflict { @@ -90,8 +92,12 @@ pub fn set_pinned_state( fs_extra::move_items(&items, layout.root, &CopyOptions::new())?; lockfile.map_then_flush(|lockfile| { + let binaries = lockfile.binaries(&old_package.id()).cloned(); lockfile.remove(&old_package); lockfile.add_entrypoint(&package); + if let Some(binaries) = binaries { + lockfile.set_binaries(&package, binaries); + } Ok::<_, io::Error>(()) })?; diff --git a/lux-lib/src/operations/pipeline/download_sources_and_hash.rs b/lux-lib/src/operations/pipeline/download_sources_and_hash.rs index 4f8463b35..3e8c518ea 100644 --- a/lux-lib/src/operations/pipeline/download_sources_and_hash.rs +++ b/lux-lib/src/operations/pipeline/download_sources_and_hash.rs @@ -15,12 +15,11 @@ use crate::{ fs::{self, FsError}, hash::HasIntegrity, lockfile::{ - LockConstraint, LockedPackage, LockedPackageHashes, LockedPackageId, OptState, PinnedState, + LockedPackage, LockedPackageHashes, LockedPackageId, LockedPackageSpec, RemotePackageSourceUrl, }, lua_rockspec::RemoteLuaRockspec, operations::{FetchSrc, RemotePackageSourceMetadata, unpack_src_rock}, - package::PackageSpec, remote_package_source::RemotePackageSource, rockspec::Rockspec, tree::EntryType, @@ -118,9 +117,6 @@ async fn download_sources_and_hash( artifact, } = resolved; - let constraint = spec.constraint(); - let pin = spec.pinned(); - let opt = spec.opt(); let name = spec.name().to_string(); let (package, artifact) = match artifact { @@ -139,32 +135,29 @@ async fn download_sources_and_hash( rockspec: rockspec_hash, source: source_hash, }; - let package = LockedPackage { + let package = LockedPackage::new( spec, source, - source_url: Some(RemotePackageSourceUrl::Url { url: binary_url }), + Some(RemotePackageSourceUrl::Url { url: binary_url }), hashes, - }; + ); (package, PackageSource::PackedRock(bytes)) } _ => { let source_url = source_url .clone() .ok_or_else(|| DownloadSourcesAndHashError::MissingSourceUrl(name.clone()))?; - let (mut package, dir) = fetch_and_hash_source( + let (package, dir) = fetch_and_hash_source( &rockspec, + spec, Some(RemotePackageSourceSpec::SrcRock(SrcRockSource { bytes, source_url, })), Some(source), - constraint, - pin, - opt, config, ) .await?; - package.spec = spec; (package, PackageSource::SourceTree(dir)) } }, @@ -182,26 +175,18 @@ async fn download_sources_and_hash( rockspec: rockspec_hash, source: source_hash, }; - let package = LockedPackage { - spec, - source, - source_url, - hashes, - }; + let package = LockedPackage::new(spec, source, source_url, hashes); (package, PackageSource::SourceTree(dir)) } None => { - let (mut package, dir) = fetch_and_hash_source( + let (package, dir) = fetch_and_hash_source( &rockspec, + spec, Some(RemotePackageSourceSpec::RockSpec(source_url)), Some(source), - constraint, - pin, - opt, config, ) .await?; - package.spec = spec; (package, PackageSource::SourceTree(dir)) } }; @@ -216,11 +201,9 @@ async fn download_sources_and_hash( pub(crate) async fn fetch_and_hash_source( rockspec: &R, + spec: LockedPackageSpec, source_spec: Option, source: Option, - constraint: LockConstraint, - pin: PinnedState, - opt: OptState, config: &Config, ) -> Result<(LockedPackage, TempDir), BuildError> { let temp_dir = fs::tempfile::tempdir()?; @@ -254,10 +237,8 @@ pub(crate) async fn fetch_and_hash_source( source: source_metadata.hash.clone(), }; - let mut package = LockedPackage::from( - &PackageSpec::new(rockspec.package().clone(), rockspec.version().clone()), - constraint, - rockspec.binaries(), + let package = LockedPackage::new( + spec, source .map(Result::Ok) .unwrap_or_else(|| { @@ -269,11 +250,6 @@ pub(crate) async fn fetch_and_hash_source( Some(source_metadata.source_url.clone()), hashes, ); - // FIXME(vhyrro): We should reconsider our constructors if we have to set - // these variants ourselves. Either put these in a different place, or resolve - // these in an earlier step. - package.spec.pinned = pin; - package.spec.opt = opt; Ok((package, temp_dir)) } diff --git a/lux-lib/src/operations/pipeline/install_packages.rs b/lux-lib/src/operations/pipeline/install_packages.rs index 5b691cd14..fe6a273f6 100644 --- a/lux-lib/src/operations/pipeline/install_packages.rs +++ b/lux-lib/src/operations/pipeline/install_packages.rs @@ -204,11 +204,11 @@ where let installed_packages: HashMap = built .iter() - .map(|pkg| (pkg.spec.id(), pkg.clone())) + .map(|pkg| (pkg.spec().id(), pkg.clone())) .collect(); let installed_build_deps: HashMap = built_build_deps .iter() - .map(|pkg| (pkg.spec.id(), pkg.clone())) + .map(|pkg| (pkg.spec().id(), pkg.clone())) .collect(); lockfile.map_then_flush(|lockfile| { @@ -216,7 +216,7 @@ where lockfile.remove_by_id(&package.id()); } for package in &built { - let entry_type = regular_entry_types[&package.spec.id()]; + let entry_type = regular_entry_types[&package.spec().id()]; lockfile.add_dependencies(package, entry_type, &installed_packages)?; lockfile.add_build_dependencies(package, &installed_build_deps)?; } @@ -302,7 +302,7 @@ impl LockfileExt for &mut Lockfile { self.add_entrypoint(pkg); } - for dependency_id in pkg.spec.dependencies() { + for dependency_id in pkg.spec().dependencies() { self.add_dependency( pkg, installed_packages @@ -326,7 +326,7 @@ This is likely a bug in Lux. pkg: &LockedPackage, build_dependencies: &HashMap, ) -> io::Result<()> { - for dependency_id in pkg.spec.build_dependencies() { + for dependency_id in pkg.spec().build_dependencies() { self.add_build_dependency( pkg, build_dependencies diff --git a/lux-lib/src/operations/pipeline/resolve.rs b/lux-lib/src/operations/pipeline/resolve.rs index e47bf99e8..d3e62bc18 100644 --- a/lux-lib/src/operations/pipeline/resolve.rs +++ b/lux-lib/src/operations/pipeline/resolve.rs @@ -265,7 +265,6 @@ async fn resolve_spec( build_dependencies, &spec.pin, &spec.opt, - rockspec.binaries(), ); let resolved = ResolvedPackage { diff --git a/lux-lib/src/operations/sync.rs b/lux-lib/src/operations/sync.rs index 0f50fb73a..b39a0dc3b 100644 --- a/lux-lib/src/operations/sync.rs +++ b/lux-lib/src/operations/sync.rs @@ -6,15 +6,14 @@ use crate::{ config::Config, fs, lockfile::{ - FlushLockfileError, LockedPackage, LockedPackageLockType, Lockfile, - LockfileIntegrityError, PackageSyncSpec, ReadOnly, ReadWrite, SyncStrategy, - WorkspaceLockfile, + FlushLockfileError, LockedPackage, LockedPackageLockType, Lockfile, LockfileIntegrityError, + PackageSyncSpec, ReadOnly, ReadWrite, SyncStrategy, WorkspaceLockfile, }, luarocks::luarocks_installation::LUAROCKS_VERSION, - operations::{self, pipeline::resolve::luarocks_build_backend_name, GenLuaRcError}, + operations::{self, GenLuaRcError, pipeline::resolve::luarocks_build_backend_name}, package::{PackageName, PackageReq}, - project::{project_toml::LocalProjectTomlValidationError, ProjectError}, - rockspec::{lua_dependency::LuaDependencySpec, Rockspec}, + project::{ProjectError, project_toml::LocalProjectTomlValidationError}, + rockspec::{Rockspec, lua_dependency::LuaDependencySpec}, tree::{self, InstallTree, TreeError}, workspace::{Workspace, WorkspaceError, WorkspaceTreeError}, }; @@ -92,7 +91,7 @@ where .flat_map(|rock| rock.build_dependencies()) .filter_map(|dep_id| test_build_lockfile.get(dep_id)), ) - .map(|pkg| pkg.spec.as_package_req()) + .map(|pkg| pkg.spec().as_package_req()) .collect_vec(); args.extra_packages.extend(transitive_build_dependencies); @@ -245,8 +244,15 @@ async fn do_sync( Ok::<_, io::Error>(()) })?; - install_missing_packages(args, &tree, &mut workspace_lockfile, lock_type, &package_sync_spec, &mut report) - .await?; + install_missing_packages( + args, + &tree, + &mut workspace_lockfile, + lock_type, + &package_sync_spec, + &mut report, + ) + .await?; Ok(report) } @@ -449,7 +455,7 @@ mod tests { config::ConfigBuilder, lockfile::LockedPackageLockType, package::PackageReq, workspace::Workspace, }; - use assert_fs::{prelude::PathCopy, TempDir}; + use assert_fs::{TempDir, prelude::PathCopy}; use flaky_test::flaky_test; use std::path::PathBuf; @@ -470,9 +476,11 @@ mod tests { assert!(!report.added.is_empty()); let lockfile_after_sync = workspace.lockfile().unwrap(); - assert!(!lockfile_after_sync - .rocks(&LockedPackageLockType::Regular) - .is_empty()); + assert!( + !lockfile_after_sync + .rocks(&LockedPackageLockType::Regular) + .is_empty() + ); } #[flaky_test(tokio, times = 5)] @@ -496,15 +504,19 @@ mod tests { .unwrap(); assert!(report.removed.is_empty()); assert!(!report.added.is_empty()); - assert!(report - .added - .iter() - .any(|pkg| pkg.name().to_string() == "toml-edit")); + assert!( + report + .added + .iter() + .any(|pkg| pkg.name().to_string() == "toml-edit") + ); } let lockfile_after_sync = workspace.lockfile().unwrap(); - assert!(!lockfile_after_sync - .rocks(&LockedPackageLockType::Regular) - .is_empty()); + assert!( + !lockfile_after_sync + .rocks(&LockedPackageLockType::Regular) + .is_empty() + ); } #[flaky_test(tokio, times = 5)] @@ -529,15 +541,19 @@ mod tests { .unwrap(); assert!(report.removed.is_empty()); assert!(!report.added.is_empty()); - assert!(report - .added - .iter() - .any(|pkg| pkg.name().to_string() == "toml-edit")); + assert!( + report + .added + .iter() + .any(|pkg| pkg.name().to_string() == "toml-edit") + ); } let lockfile_after_sync = workspace.lockfile().unwrap(); - assert!(!lockfile_after_sync - .rocks(&LockedPackageLockType::Regular) - .is_empty()); + assert!( + !lockfile_after_sync + .rocks(&LockedPackageLockType::Regular) + .is_empty() + ); } #[flaky_test(tokio, times = 5)] @@ -563,8 +579,10 @@ mod tests { assert!(report.added.is_empty()); let lockfile_after_sync = workspace.lockfile().unwrap(); - assert!(!lockfile_after_sync - .rocks(&LockedPackageLockType::Regular) - .is_empty()); + assert!( + !lockfile_after_sync + .rocks(&LockedPackageLockType::Regular) + .is_empty() + ); } } diff --git a/lux-lib/src/operations/vendor.rs b/lux-lib/src/operations/vendor.rs index eb4ece08c..64004cc20 100644 --- a/lux-lib/src/operations/vendor.rs +++ b/lux-lib/src/operations/vendor.rs @@ -540,10 +540,9 @@ mod tests { use super::*; use crate::{ config::ConfigBuilder, - lockfile::{LockConstraint, LockedPackage, LockedPackageSpec, OptState, PinnedState}, + lockfile::{LockedPackage, LockConstraint, LockedPackageSpec, OptState, PinnedState}, operations::unpack_rockspec, remote_package_source::RemotePackageSource, - rockspec::RockBinaries, }; use assert_fs::prelude::PathCopy; @@ -576,13 +575,12 @@ mod tests { Vec::new(), &PinnedState::Unpinned, &OptState::Required, - RockBinaries::default(), ); - let package = LockedPackage { + let package = LockedPackage::new( spec, source, - source_url: None, - hashes: crate::lockfile::LockedPackageHashes { + None, + crate::lockfile::LockedPackageHashes { rockspec: "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" .parse() .unwrap(), @@ -590,7 +588,7 @@ mod tests { .parse() .unwrap(), }, - }; + ); DownloadedPackage { package, rockspec, diff --git a/lux-lib/src/path.rs b/lux-lib/src/path.rs index d5670f441..d41cb108a 100644 --- a/lux-lib/src/path.rs +++ b/lux-lib/src/path.rs @@ -45,7 +45,7 @@ impl Paths { pub fn new(tree: &impl InstallTree) -> Result { let mut paths = Self::default(tree); for package in tree.list()?.values().flatten() { - let layout = tree.layout_for(package); + let layout = tree.layout_for(&package.spec); paths.src.0.push(layout.src.join("?.lua")); paths.src.0.push(layout.src.join("?").join("init.lua")); paths diff --git a/lux-lib/src/remote_package_db/mod.rs b/lux-lib/src/remote_package_db/mod.rs index c365b0da7..d88ce4025 100644 --- a/lux-lib/src/remote_package_db/mod.rs +++ b/lux-lib/src/remote_package_db/mod.rs @@ -96,9 +96,9 @@ impl RemotePackageDB { .filter_map(|lock| lock.has_rock(package_req, filter.clone())) .map(|local_package| { RemotePackage::new( - PackageSpec::new(local_package.spec.name, local_package.spec.version), - local_package.source, - local_package.source_url, + local_package.to_package(), + local_package.source().clone(), + local_package.source_url().cloned(), ) }) .next() diff --git a/lux-lib/src/rockspec/mod.rs b/lux-lib/src/rockspec/mod.rs index 30bc4dc04..c967c67e6 100644 --- a/lux-lib/src/rockspec/mod.rs +++ b/lux-lib/src/rockspec/mod.rs @@ -136,12 +136,6 @@ impl LuaVersionCompatibility for T { #[derive(Debug, Default, Serialize, Deserialize, Clone, PartialOrd, Ord, Hash, PartialEq, Eq)] pub struct RockBinaries(Vec); -impl RockBinaries { - pub(crate) fn is_default(&self) -> bool { - &Self::default() == self - } -} - impl Deref for RockBinaries { type Target = Vec; diff --git a/lux-lib/src/tree/dist.rs b/lux-lib/src/tree/dist.rs index 39419bc1c..8c2a34089 100644 --- a/lux-lib/src/tree/dist.rs +++ b/lux-lib/src/tree/dist.rs @@ -4,7 +4,7 @@ use super::{EntryType, InstallTree, RockLayout, Tree, TreeError}; use crate::{ config::Config, fs, - lockfile::{LockedPackage, Lockfile, ReadOnly}, + lockfile::{LockedPackage, LockedPackageSpec, Lockfile, ReadOnly}, lua_version::LuaVersion, package::{PackageName, PackageVersion}, }; @@ -42,7 +42,7 @@ impl FlatDistTree { Ok(Self(tree)) } - fn guard_no_conflicting_package(&self, package: &LockedPackage) -> Result<(), io::Error> { + fn guard_no_conflicting_package(&self, package: &LockedPackageSpec) -> Result<(), io::Error> { let lockfile = self.lockfile().map_err(io::Error::other)?; match lockfile.has_rock(&package.clone().into_package_req(), None) { Some(existing_package) => { @@ -91,7 +91,7 @@ impl InstallTree for FlatDistTree { self.0.root() } - fn layout_for(&self, _package: &LockedPackage) -> RockLayout { + fn layout_for(&self, _package: &LockedPackageSpec) -> RockLayout { RockLayout::new(self.0.root(), self.0.bin()) } @@ -103,7 +103,7 @@ impl InstallTree for FlatDistTree { self.0.unwrapped_bin() } - fn prepare(&self, package: &LockedPackage) -> Result<(), TreeError> { + fn prepare(&self, package: &LockedPackageSpec) -> Result<(), TreeError> { self.guard_no_conflicting_package(package) .map_err(TreeError::Io)?; let layout = self.layout_for(package); diff --git a/lux-lib/src/tree/layout.rs b/lux-lib/src/tree/layout.rs index 0b5fbffcc..025caba13 100644 --- a/lux-lib/src/tree/layout.rs +++ b/lux-lib/src/tree/layout.rs @@ -1,6 +1,6 @@ use crate::{ fs::{self, FsError}, - lockfile::{LockedPackage, OptState}, + lockfile::{LockedPackageSpec, OptState}, tree::{InstallTree, Tree}, }; use std::path::{Path, PathBuf}; @@ -8,9 +8,9 @@ use std::path::{Path, PathBuf}; /// A custom layout for entrypoint packages. /// Implementations arrange symlinks so external tools can find files at the locations they expect. pub trait CustomRockLayout: std::fmt::Debug + Send + Sync { - fn make_symlinks(&self, tree: &Tree, package: &LockedPackage) -> fs::Result<()>; + fn make_symlinks(&self, tree: &Tree, package: &LockedPackageSpec) -> fs::Result<()>; - fn remove_symlinks(&self, tree: &Tree, package: &LockedPackage) -> fs::Result<()>; + fn remove_symlinks(&self, tree: &Tree, package: &LockedPackageSpec) -> fs::Result<()>; } /// A [`CustomRockLayout`] for Neovim plugins. Packages are exposed under `/site/pack/lux/{start,opt}/` @@ -18,8 +18,8 @@ pub trait CustomRockLayout: std::fmt::Debug + Send + Sync { pub struct NvimLayout; impl NvimLayout { - fn target_for(tree: &Tree, package: &LockedPackage) -> PathBuf { - let subdir = match package.spec.opt { + fn target_for(tree: &Tree, package: &LockedPackageSpec) -> PathBuf { + let subdir = match package.opt() { OptState::Required => "start", OptState::Optional => "opt", }; @@ -31,7 +31,7 @@ impl NvimLayout { } impl CustomRockLayout for NvimLayout { - fn make_symlinks(&self, tree: &Tree, package: &LockedPackage) -> fs::Result<()> { + fn make_symlinks(&self, tree: &Tree, package: &LockedPackageSpec) -> fs::Result<()> { let custom_dir = Self::target_for(tree, package); fs::sync::create_dir_all(&custom_dir)?; @@ -49,7 +49,7 @@ impl CustomRockLayout for NvimLayout { Ok(()) } - fn remove_symlinks(&self, tree: &Tree, package: &LockedPackage) -> fs::Result<()> { + fn remove_symlinks(&self, tree: &Tree, package: &LockedPackageSpec) -> fs::Result<()> { let target = Self::target_for(tree, package); if target.is_dir() { // SAFETY: does not follow symlinks, only removes them @@ -97,7 +97,6 @@ mod tests { lua_version::LuaVersion, package::PackageSpec, remote_package_source::RemotePackageSource, - rockspec::RockBinaries, tree::{EntryType, InstallTree, NvimLayout}, }; @@ -132,7 +131,6 @@ mod tests { LockedPackage::from( &PackageSpec::parse("neorg".into(), "8.0.0-1".into()).unwrap(), LockConstraint::Unconstrained, - RockBinaries::default(), RemotePackageSource::Test, None, mock_hashes(), @@ -144,7 +142,7 @@ mod tests { let (_temp, tree_path, tree) = sample_tree(); let package = sample_package(); - tree.prepare(&package).unwrap(); + tree.prepare(&package.spec).unwrap(); tree.finalize(&package, EntryType::Entrypoint).unwrap(); let custom_dir = tree_path @@ -161,9 +159,9 @@ mod tests { let (_temp, tree_path, tree) = sample_tree(); let package = sample_package(); - tree.prepare(&package).unwrap(); + tree.prepare(&package.spec).unwrap(); - let etc = tree.layout_for(&package).etc; + let etc = tree.layout_for(&package.spec).etc; std::fs::create_dir_all(etc.join("plugin")).unwrap(); std::fs::write(etc.join("plugin/foo.vim"), "lua _G.foo = 1\n").unwrap(); @@ -180,7 +178,7 @@ mod tests { let (_temp, tree_path, tree) = sample_tree(); let package = sample_package(); - tree.prepare(&package).unwrap(); + tree.prepare(&package.spec).unwrap(); tree.finalize(&package, EntryType::DependencyOnly).unwrap(); let custom_dir = tree_path @@ -194,7 +192,7 @@ mod tests { let (_temp, tree_path, tree) = sample_tree(); let package = sample_package(); - tree.prepare(&package).unwrap(); + tree.prepare(&package.spec).unwrap(); tree.finalize(&package, EntryType::Entrypoint).unwrap(); let custom_dir = tree_path .join("5.1/site/pack/lux/start") diff --git a/lux-lib/src/tree/mod.rs b/lux-lib/src/tree/mod.rs index c10fd31ae..194e74a36 100644 --- a/lux-lib/src/tree/mod.rs +++ b/lux-lib/src/tree/mod.rs @@ -2,7 +2,9 @@ use crate::{ build::utils::format_path, config::Config, fs, - lockfile::{LockedPackage, LockedPackageId, Lockfile, LockfileError, ReadOnly}, + lockfile::{ + LockedPackage, LockedPackageId, LockedPackageSpec, Lockfile, LockfileError, ReadOnly, + }, lua_version::LuaVersion, package::{PackageName, PackageReq}, variables::{GetVariableError, HasVariables}, @@ -42,7 +44,7 @@ pub trait InstallTree { /// Where unwrapped package binaries are installed fn unwrapped_bin(&self) -> PathBuf; /// The standard install layout for a package. - fn layout_for(&self, package: &LockedPackage) -> RockLayout; + fn layout_for(&self, package: &LockedPackageSpec) -> RockLayout; /// Create a [`Lockfile`] for this tree. fn lockfile(&self) -> Result, TreeError>; /// Get this tree's lockfile path. @@ -56,7 +58,7 @@ pub trait InstallTree { /// Find installed rocks that match the given [`PackageReq`]. fn match_rocks(&self, req: &PackageReq) -> Result; /// Create the standard directories (src, lib, etc.) for a package. - fn prepare(&self, package: &LockedPackage) -> Result<(), TreeError>; + fn prepare(&self, package: &LockedPackageSpec) -> Result<(), TreeError>; /// Apply the custom layout once a package has been fully installed. /// /// For entrypoints, this creates the symlinks for the custom layout, if one is configured. @@ -252,7 +254,7 @@ impl InstallTree for Tree { self.root_parent.join(self.version.to_string()) } - fn prepare(&self, package: &LockedPackage) -> Result<(), TreeError> { + fn prepare(&self, package: &LockedPackageSpec) -> Result<(), TreeError> { let layout = self.layout_for(package); fs::sync::create_dir_all(&layout.root)?; fs::sync::create_dir_all(&layout.lib)?; @@ -265,7 +267,7 @@ impl InstallTree for Tree { fn finalize(&self, package: &LockedPackage, entry_type: EntryType) -> Result<(), TreeError> { if entry_type.is_entrypoint() { if let Some(custom_layout) = &self.entrypoint_layout { - custom_layout.make_symlinks(self, package)?; + custom_layout.make_symlinks(self, &package.spec)?; } } @@ -275,15 +277,20 @@ impl InstallTree for Tree { fn cleanup(&self, package: &LockedPackage, entry_type: EntryType) -> Result<(), TreeError> { if entry_type.is_entrypoint() { if let Some(layout) = &self.entrypoint_layout { - layout.remove_symlinks(self, package)?; + layout.remove_symlinks(self, &package.spec)?; } } - let layout = self.layout_for(package); + let layout = self.layout_for(&package.spec); fs::sync::remove_dir_all(&layout.etc)?; fs::sync::remove_dir_all(&layout.root)?; - for relative_binary_path in package.spec.binaries() { + let lockfile = self.lockfile()?; + for relative_binary_path in lockfile + .binaries(&package.id()) + .into_iter() + .flat_map(|binaries| binaries.iter()) + { if let Some(binary_file_name) = relative_binary_path.file_name() { let binary_path = self.bin().join(binary_file_name); if binary_path.is_file() { @@ -308,7 +315,7 @@ impl InstallTree for Tree { self.root().join(LOCKFILE_NAME) } - fn layout_for(&self, package: &LockedPackage) -> RockLayout { + fn layout_for(&self, package: &LockedPackageSpec) -> RockLayout { RockLayout::new( self.root().join(format!( "{}-{}@{}", @@ -411,7 +418,6 @@ mod tests { lua_version::LuaVersion, package::{PackageName, PackageSpec, PackageVersion}, remote_package_source::RemotePackageSource, - rockspec::RockBinaries, tree::{InstallTree, RockLayout}, variables, }; @@ -444,17 +450,16 @@ mod tests { let package = LockedPackage::from( &PackageSpec::parse("neorg".into(), "8.0.0-1".into()).unwrap(), LockConstraint::Unconstrained, - RockBinaries::default(), RemotePackageSource::Test, None, mock_hashes.clone(), ); let id = package.id(); - tree.prepare(&package).unwrap(); + tree.prepare(&package.spec).unwrap(); assert_eq!( - tree.layout_for(&package), + tree.layout_for(&package.spec), RockLayout { bin: tree_path.join("5.1/bin"), root: tree_path.join(format!("5.1/{id}-neorg@8.0.0-1")), @@ -492,7 +497,7 @@ mod tests { name, package .into_iter() - .map(|package| package.spec.version) + .map(|package| package.version().clone()) .sorted() .collect_vec(), ) @@ -530,12 +535,11 @@ mod tests { let package = LockedPackage::from( &PackageSpec::parse("neorg".into(), "8.0.0-1-1".into()).unwrap(), LockConstraint::Unconstrained, - RockBinaries::default(), RemotePackageSource::Test, None, mock_hashes.clone(), ); - let layout = tree.layout_for(&package); + let layout = tree.layout_for(&package.spec); let build_variables = vec![ "$(PREFIX)", "$(LIBDIR)", diff --git a/lux-lib/src/which/mod.rs b/lux-lib/src/which/mod.rs index 0d5466901..871082de1 100644 --- a/lux-lib/src/which/mod.rs +++ b/lux-lib/src/which/mod.rs @@ -86,7 +86,7 @@ fn do_search(which: Which<'_>) -> Result { local_packages .into_iter() .filter_map(|pkg| { - let layout = tree.layout_for(&pkg); + let layout = tree.layout_for(&pkg.spec); let lib_path = layout.lib.join(which.module.to_lib_path()); if lib_path.is_file() { return Some(lib_path); diff --git a/lux-lib/tests/build.rs b/lux-lib/tests/build.rs index 9f170e62a..bfb48806f 100644 --- a/lux-lib/tests/build.rs +++ b/lux-lib/tests/build.rs @@ -1,7 +1,7 @@ use std::path::PathBuf; -use assert_fs::prelude::PathCopy; use assert_fs::TempDir; +use assert_fs::prelude::PathCopy; use flaky_test::flaky_test; use lux_lib::lua_version::LuaVersion; use lux_lib::rockspec::Rockspec; @@ -10,7 +10,7 @@ use lux_lib::workspace::Workspace; use lux_lib::{ build::{Build, BuildBehaviour::Force}, config::ConfigBuilder, - lua_installation::{detect_installed_lua_version, LuaInstallation}, + lua_installation::{LuaInstallation, detect_installed_lua_version}, lua_rockspec::RemoteLuaRockspec, tree, }; @@ -193,7 +193,7 @@ async fn treesitter_parser_build() { .await .unwrap(); - let etc = tree.layout_for(&package).etc; + let etc = tree.layout_for(package.spec()).etc; let folds_query = etc.join("queries").join("rust").join("folds.scm"); assert!(folds_query.is_file()); @@ -239,7 +239,7 @@ async fn treesitter_parser_build_source_queries() { .await .unwrap(); - let etc = tree.layout_for(&package).etc; + let etc = tree.layout_for(package.spec()).etc; let highlights_query = etc.join("queries").join("tmux").join("highlights.scm"); assert!(highlights_query.is_file()); @@ -297,7 +297,7 @@ async fn test_build_local_project_no_source() { .await .unwrap(); - let layout = tree.layout_for(&package); + let layout = tree.layout_for(package.spec()); let conf_file = layout.conf.join("foo").join("bar.toml"); assert!(conf_file.is_file()); @@ -338,7 +338,7 @@ async fn test_build_local_project_only_src() { .await .unwrap(); - let src = tree.layout_for(&pkg).src; + let src = tree.layout_for(pkg.spec()).src; assert!(src.is_dir()); assert!(src.join("main.lua").is_file()); assert!(src.join("foo.lua").is_file()); @@ -475,7 +475,7 @@ async fn test_multiline_command_build() { .await .unwrap(); - let success_dir = tree.layout_for(&package).src.join("success"); + let success_dir = tree.layout_for(package.spec()).src.join("success"); assert!(success_dir.is_dir()); } From 50c3aafe62079b710aaedfb80d77a2441a460e58 Mon Sep 17 00:00:00 2001 From: vhyrro Date: Sat, 3 Oct 2026 19:06:29 +0200 Subject: [PATCH 11/39] feat!: parallelize resolve and build --- lux-lib/src/lockfile/mod.rs | 2 +- .../pipeline/download_sources_and_hash.rs | 49 +- lux-lib/src/operations/pipeline/mod.rs | 8 - lux-lib/src/operations/pipeline/resolve.rs | 604 +++++++++++++----- 4 files changed, 479 insertions(+), 184 deletions(-) diff --git a/lux-lib/src/lockfile/mod.rs b/lux-lib/src/lockfile/mod.rs index d06a25c4a..a3fa5a7c4 100644 --- a/lux-lib/src/lockfile/mod.rs +++ b/lux-lib/src/lockfile/mod.rs @@ -857,7 +857,7 @@ pub struct Lockfile { lock: LockedPackageLock, } -#[derive(EnumIter, Debug, Clone, Copy, PartialEq, Eq)] +#[derive(EnumIter, Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum LockedPackageLockType { Regular, Test, diff --git a/lux-lib/src/operations/pipeline/download_sources_and_hash.rs b/lux-lib/src/operations/pipeline/download_sources_and_hash.rs index 3e8c518ea..2c2a51255 100644 --- a/lux-lib/src/operations/pipeline/download_sources_and_hash.rs +++ b/lux-lib/src/operations/pipeline/download_sources_and_hash.rs @@ -84,22 +84,41 @@ where let args = self._build(); let config = args.config; - futures::stream::iter(args.resolved) - .then(|(section, packages)| async move { - let Some(packages) = packages else { - return Ok((section, None)); - }; - let packages = futures::stream::iter(packages) - .then(|(id, package)| async move { - download_sources_and_hash(config, package) - .await - .map(|package| (id, package)) - }) - .try_collect::>() - .await?; - Ok((section, Some(packages))) + // Preserve which sections were requested (even if empty), then run every + // download as a single bounded-concurrency stream and fold the results + // back into their sections. `buffered` keeps at most `max_jobs` + // downloads in flight; hashing is already `spawn_blocking`, so it runs + // across the blocking thread pool. + let artifacts: DownloadSourcesAndHashArtifacts = args + .resolved + .iter() + .map(|(section, packages)| (section, packages.map(|_| HashMap::new()))) + .collect(); + + let jobs = args.resolved.into_iter().flat_map(|(section, packages)| { + packages + .into_iter() + .flatten() + .map(move |(id, package)| (section, id, package)) + }); + + futures::stream::iter(jobs) + .map(|(section, id, package)| async move { + download_sources_and_hash(config, package) + .await + .map(|package| (section, id, package)) }) - .try_collect() + .buffered(config.max_jobs()) + .try_fold( + artifacts, + |mut artifacts, (section, id, package)| async move { + artifacts + .get_mut(section) + .get_or_insert_with(HashMap::new) + .insert(id, package); + Ok(artifacts) + }, + ) .await } } diff --git a/lux-lib/src/operations/pipeline/mod.rs b/lux-lib/src/operations/pipeline/mod.rs index 1ed02ed70..91735daef 100644 --- a/lux-lib/src/operations/pipeline/mod.rs +++ b/lux-lib/src/operations/pipeline/mod.rs @@ -31,14 +31,6 @@ impl Default for Artifacts { } impl Artifacts { - pub(crate) fn get(&self, section: LockedPackageLockType) -> Option<&T> { - match section { - LockedPackageLockType::Regular => self.regular.as_ref(), - LockedPackageLockType::Build => self.build.as_ref(), - LockedPackageLockType::Test => self.test.as_ref(), - } - } - pub(crate) fn get_mut(&mut self, section: LockedPackageLockType) -> &mut Option { match section { LockedPackageLockType::Regular => &mut self.regular, diff --git a/lux-lib/src/operations/pipeline/resolve.rs b/lux-lib/src/operations/pipeline/resolve.rs index d3e62bc18..d7099cd62 100644 --- a/lux-lib/src/operations/pipeline/resolve.rs +++ b/lux-lib/src/operations/pipeline/resolve.rs @@ -1,17 +1,17 @@ -use std::cell::RefCell; -use std::collections::HashMap; -use std::rc::Rc; +use std::collections::{HashMap, HashSet, VecDeque}; -use async_recursion::async_recursion; use bon::Builder; -use futures::{StreamExt, TryStreamExt}; use itertools::Itertools; use miette::Diagnostic; use thiserror::Error; +use tokio::task::JoinSet; use crate::{ config::Config, - lockfile::{LockedPackageId, LockedPackageLockType, LockedPackageSpec, RemotePackageSourceUrl}, + lockfile::{ + LockedPackageId, LockedPackageLockType, LockedPackageSpec, OptState, PinnedState, + RemotePackageSourceUrl, + }, lua_rockspec::{BuildBackendSpec, RemoteLuaRockspec}, operations::{PackageInstallSpec, RemoteRockDownload}, package::{PackageName, PackageReq, RemotePackage}, @@ -125,177 +125,329 @@ where { pub(crate) async fn resolve(self) -> Result { let args = self._build(); - // TODO(vhyrro): Rewrite to be parallel and no RefCell - let state = Rc::new(RefCell::new(ResolvedArtifacts::default())); - let (discover, config) = (args.discover, args.config); - - futures::stream::iter(args.packages) - .then(|(section, packages)| { - let state = state.clone(); - async move { - let Some(packages) = packages else { - return Ok(()); - }; - // Mark the section as requested even if it resolves to nothing. - state - .borrow_mut() - .get_mut(section) - .get_or_insert_with(HashMap::new); - futures::stream::iter(packages) - .then(|package| { - resolve_spec( - package, - section, - Vec::new(), - state.clone(), - discover, - config, - ) - }) - .try_for_each(|_| async { Ok(()) }) - .await - } - }) - .try_for_each(|_| async { Ok(()) }) - .await?; - - let resolved = std::mem::take(&mut *state.borrow_mut()); - Ok(resolved) + Resolver::new(args.discover, args.config) + .run(args.packages) + .await } } -#[async_recursion(?Send)] -async fn resolve_spec( - spec: PackageInstallSpec, +/// Identifies a single discovery request. Two requests with the same key resolve to the +/// same package, so the diamond problem is solved by only discovering each key once. +#[derive(Clone, PartialEq, Eq, Hash)] +struct RequestKey { section: LockedPackageLockType, - parents: Vec, - state: Rc>, - discover: &D, - config: &Config, -) -> Result { - let FoundPackage { - package, - rockspec, - artifact, - } = find_package_from_provider(&spec, discover, config).await?; - - let package_spec = package.package.clone(); - if parents.contains(package_spec.name()) { - let chain = parents - .iter() - .map(|name| name.to_string()) - .chain(std::iter::once(package_spec.name().to_string())) - .collect::>() - .join(" -> "); - return Err(ResolveError::CyclicDependency(chain)); + package: PackageReq, + pin: PinnedState, + opt: OptState, +} + +impl RequestKey { + fn from_install_spec(section: LockedPackageLockType, spec: &PackageInstallSpec) -> Self { + Self { + section, + package: spec.package.clone(), + pin: spec.pin, + opt: spec.opt, + } } +} + +/// A discovered node in the dependency graph. +struct Node { + install_spec: PackageInstallSpec, + id: LockedPackageId, + found: FoundPackage, + /// `(is_build_dependency, child)` in the order the dependencies were declared. + children: Vec<(bool, RequestKey)>, +} + +/// Owns the whole resolution graph while it is being discovered. Discovery futures run on +/// the runtime (bounded by `Config::max_jobs`), but all shared state stays here, owned by a +/// single task — no `Rc`, `RefCell`, or locks. +struct Resolver<'a, D: FindPackageFromProvider> { + discover: &'a D, + max_inflight: usize, + joinset: JoinSet<(RequestKey, PackageInstallSpec, Result)>, + pending: VecDeque<(RequestKey, PackageInstallSpec)>, + seen: HashSet, + nodes: HashMap, + resolved: ResolvedArtifacts, +} - let constraint = spec - .constraint - .clone() - .unwrap_or_else(|| spec.package.version_req().clone().into()); - let id = LockedPackageId::new( - package_spec.name(), - package_spec.version(), - spec.pin, - spec.opt, - constraint.clone(), - ); - - if state - .borrow() - .get(section) - .is_some_and(|packages| packages.contains_key(&id)) - { - return Ok(id); +impl<'a, D: FindPackageFromProvider> Resolver<'a, D> { + fn new(discover: &'a D, config: &Config) -> Self { + Self::with_max_inflight(discover, config.max_jobs()) } - let mut child_parents = parents; - child_parents.push(package_spec.name().clone()); + /// Builds a resolver with an explicit in-flight bound. Used by tests, which do not + /// construct a full [`Config`]. + fn with_max_inflight(discover: &'a D, max_inflight: usize) -> Self { + Self { + discover, + max_inflight, + joinset: JoinSet::new(), + pending: VecDeque::new(), + seen: HashSet::new(), + nodes: HashMap::new(), + resolved: ResolvedArtifacts::default(), + } + } - let is_binary = matches!(package.source, RemotePackageSource::LuarocksBinaryRock(_)); + async fn run( + mut self, + packages: Artifacts>, + ) -> Result { + for (section, specs) in packages { + let Some(specs) = specs else { + continue; + }; + // Mark the section as requested even if it resolves to nothing. + self.resolved.get_mut(section).get_or_insert_with(HashMap::new); + for spec in specs { + let key = RequestKey::from_install_spec(section, &spec); + self.enqueue(key, spec); + } + } - let build_dependencies = if is_binary { - Vec::new() - } else { - let mut ids = Vec::new(); - for dependency in build_dependency_specs(&rockspec, &spec) { - ids.push( - resolve_spec( - dependency, - LockedPackageLockType::Build, - child_parents.clone(), - state.clone(), - discover, - config, - ) - .await?, - ); + self.spawn_ready(); + while let Some(joined) = self.joinset.join_next().await { + let (key, spec, result) = joined.map_err(ResolveError::Join)?; + self.on_discovered(key, spec, result?); + self.spawn_ready(); } - ids - }; - let mut dependencies = Vec::new(); - for dependency in rockspec.dependencies().current_platform() { - let dependency_spec = - PackageInstallSpec::new(dependency.package_req().clone(), EntryType::DependencyOnly) - .build_behaviour(spec.build_behaviour) - .pin(spec.pin) - .opt(spec.opt) - .maybe_source(dependency.source().clone()) - .build(); - dependencies.push( - resolve_spec( - dependency_spec, - section, - child_parents.clone(), - state.clone(), - discover, - config, - ) - .await?, + detect_cycles(&self.nodes)?; + Ok(self.assemble()) + } + + /// Registers a request for discovery, unless it has already been seen. Seeing the same + /// request twice is exactly the diamond problem, so it is only discovered once. + fn enqueue(&mut self, key: RequestKey, spec: PackageInstallSpec) { + if self.seen.insert(key.clone()) { + self.pending.push_back((key, spec)); + } + } + + /// Spawns queued discoveries up to the configured concurrency limit. + fn spawn_ready(&mut self) { + while self.joinset.len() < self.max_inflight { + let Some((key, spec)) = self.pending.pop_front() else { + break; + }; + let discover = D::clone(self.discover); + let task_spec = spec.clone(); + self.joinset.spawn(async move { + let result = find_package_from_provider(task_spec, discover).await; + (key, spec, result) + }); + } + } + + /// Records a discovered package and queues any children it introduces. + fn on_discovered(&mut self, key: RequestKey, spec: PackageInstallSpec, found: FoundPackage) { + let section = key.section; + let is_binary = matches!( + found.package.source, + RemotePackageSource::LuarocksBinaryRock(_) + ); + let constraint = spec + .constraint + .clone() + .unwrap_or_else(|| spec.package.version_req().clone().into()); + let id = LockedPackageId::new( + found.package.package.name(), + found.package.package.version(), + spec.pin, + spec.opt, + constraint, + ); + + let mut child_specs: Vec<(bool, RequestKey, PackageInstallSpec)> = Vec::new(); + if !is_binary { + for child_spec in build_dependency_specs(&found.rockspec, &spec) { + push_child(&mut child_specs, true, section, child_spec); + } + } + for dependency in found.rockspec.dependencies().current_platform() { + let child_spec = + PackageInstallSpec::new(dependency.package_req().clone(), EntryType::DependencyOnly) + .build_behaviour(spec.build_behaviour) + .pin(spec.pin) + .opt(spec.opt) + .maybe_source(dependency.source().clone()) + .build(); + push_child(&mut child_specs, false, section, child_spec); + } + + let children = child_specs + .iter() + .map(|(is_build, child_key, _)| (*is_build, child_key.clone())) + .collect(); + self.resolved.get_mut(section).get_or_insert_with(HashMap::new); + self.nodes.insert( + key, + Node { + install_spec: spec, + id, + found, + children, + }, ); + + for (_, child_key, child_spec) in child_specs { + self.enqueue(child_key, child_spec); + } + } + + /// Computes the final specs and assembles the resolved packages now that discovery is + /// complete and every child id is known. + fn assemble(self) -> ResolvedArtifacts { + let Resolver { + nodes, + mut resolved, + .. + } = self; + + let ids: HashMap = nodes + .iter() + .map(|(key, node)| (key.clone(), node.id.clone())) + .collect(); + + for (key, node) in nodes { + let Node { + install_spec, + id, + found, + children, + } = node; + let dependencies = children + .iter() + .filter(|(is_build, _)| !is_build) + .map(|(_, child)| ids[child].clone()) + .collect_vec(); + let build_dependencies = children + .iter() + .filter(|(is_build, _)| *is_build) + .map(|(_, child)| ids[child].clone()) + .collect_vec(); + let constraint = install_spec + .constraint + .clone() + .unwrap_or_else(|| install_spec.package.version_req().clone().into()); + + let spec = LockedPackageSpec::new( + found.package.package.name(), + found.package.package.version(), + constraint, + dependencies, + build_dependencies, + &install_spec.pin, + &install_spec.opt, + ); + let package = ResolvedPackage { + spec, + rockspec: found.rockspec, + source: found.package.source, + source_url: found.package.source_url, + entry_type: install_spec.entry_type, + artifact: found.artifact, + }; + + resolved + .get_mut(key.section) + .get_or_insert_with(HashMap::new) + .entry(id) + .or_insert(package); + } + + resolved } +} - let locked_spec = LockedPackageSpec::new( - package_spec.name(), - package_spec.version(), - constraint, - dependencies, - build_dependencies, - &spec.pin, - &spec.opt, - ); - - let resolved = ResolvedPackage { - spec: locked_spec, - rockspec, - source: package.source, - source_url: package.source_url, - entry_type: spec.entry_type, - artifact, +/// Adds a child request unless an equivalent request is already present. +fn push_child( + children: &mut Vec<(bool, RequestKey, PackageInstallSpec)>, + is_build: bool, + section: LockedPackageLockType, + spec: PackageInstallSpec, +) { + let child_section = if is_build { + LockedPackageLockType::Build + } else { + section }; + let child_key = RequestKey::from_install_spec(child_section, &spec); + if !children.iter().any(|(_, existing, _)| existing == &child_key) { + children.push((is_build, child_key, spec)); + } +} - state - .borrow_mut() - .get_mut(section) - .get_or_insert_with(HashMap::new) - .insert(id.clone(), resolved); +/// Depth-first search with white/grey/black colouring over the discovered graph. A grey +/// back-edge means a cycle, which we render as a `a -> b -> a` chain for the error message. +fn detect_cycles(nodes: &HashMap) -> Result<(), ResolveError> { + #[derive(Clone, Copy, PartialEq, Eq)] + enum Color { + White, + Grey, + Black, + } + + let mut colors: HashMap<&RequestKey, Color> = HashMap::with_capacity(nodes.len()); - Ok(id) + for start in nodes.keys() { + if colors.get(start).copied().unwrap_or(Color::White) != Color::White { + continue; + } + colors.insert(start, Color::Grey); + let mut stack: Vec<(&RequestKey, usize)> = vec![(start, 0)]; + let mut path: Vec<&RequestKey> = vec![start]; + + while let Some(&(node, index)) = stack.last() { + let children = &nodes[node].children; + if index < children.len() { + if let Some(top) = stack.last_mut() { + top.1 += 1; + } + let child = &children[index].1; + match colors.get(child).copied().unwrap_or(Color::White) { + Color::White => { + colors.insert(child, Color::Grey); + path.push(child); + stack.push((child, 0)); + } + Color::Grey => { + let Some(position) = path.iter().position(|key| *key == child) else { + continue; + }; + let chain = path[position..] + .iter() + .map(|key| key.package.name().to_string()) + .chain(std::iter::once(child.package.name().to_string())) + .collect_vec() + .join(" -> "); + return Err(ResolveError::CyclicDependency(chain)); + } + Color::Black => {} + } + } else { + colors.insert(node, Color::Black); + stack.pop(); + path.pop(); + } + } + } + + Ok(()) } async fn find_package_from_provider( - spec: &PackageInstallSpec, - discover: &D, - _config: &Config, + spec: PackageInstallSpec, + discover: D, ) -> Result { if let Some(source) = &spec.source { - let download = RemoteRockDownload::from_package_req_and_source_spec( - spec.package.clone(), - source.clone(), - ) - .map_err(|err| DiscoverError::Download(spec.package.clone(), Box::new(err)))?; + let download = + RemoteRockDownload::from_package_req_and_source_spec(spec.package.clone(), source.clone()) + .map_err(|err| DiscoverError::Download(spec.package.clone(), Box::new(err)))?; let rockspec = download.rockspec().clone(); let package = RemotePackage::new( spec.package.clone().try_into().map_err(|err| { @@ -314,12 +466,7 @@ async fn find_package_from_provider( }); } - // Run discovery on a separate task so that the deep download/HTTP future stack is not nested - // inside this recursive resolver's call chain. - let discover = discover.clone(); - let package = spec.package.clone(); - let FoundPackageType::Package(package) = - tokio::spawn(async move { discover.find(&package).await }).await??; + let FoundPackageType::Package(package) = discover.find(&spec.package).await?; Ok(package) } @@ -355,3 +502,140 @@ fn build_dependency_specs( }) .collect() } + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::future::Future; + use std::sync::{Arc, Mutex}; + + use super::*; + use crate::package::PackageSpec; + use crate::remote_package_db::SearchError; + + #[derive(Clone)] + struct MockDiscover { + rockspecs: HashMap, + calls: Arc>>, + } + + impl MockDiscover { + fn new(rockspecs: &[String]) -> Self { + let mut map = HashMap::new(); + for content in rockspecs { + let rockspec = match RemoteLuaRockspec::new(content) { + Ok(rockspec) => rockspec, + Err(err) => panic!("invalid test rockspec: {err}"), + }; + map.insert(rockspec.package().to_string(), (content.clone(), rockspec)); + } + Self { + rockspecs: map, + calls: Arc::new(Mutex::new(HashMap::new())), + } + } + + fn calls(&self, name: &str) -> usize { + let calls = self.calls.lock().unwrap_or_else(|err| err.into_inner()); + calls.get(name).copied().unwrap_or(0) + } + } + + impl FindPackageFromProvider for MockDiscover { + fn find( + &self, + req: &PackageReq, + ) -> impl Future> + Send { + let name = req.name().to_string(); + { + let mut calls = self.calls.lock().unwrap_or_else(|err| err.into_inner()); + *calls.entry(name.clone()).or_insert(0) += 1; + } + let result = self.rockspecs.get(&name).map_or_else( + || Err(DiscoverError::Search(SearchError::RockNotFound(req.clone()))), + |(content, rockspec)| { + let package = RemotePackage::new( + PackageSpec::new(req.name().clone(), rockspec.version().clone()), + RemotePackageSource::RockspecContent(content.clone()), + None, + ); + Ok(FoundPackageType::Package(FoundPackage { + package, + rockspec: rockspec.clone(), + artifact: None, + })) + }, + ); + async move { result } + } + } + + fn rockspec(package: &str, dependencies: &[&str]) -> String { + let dependencies = if dependencies.is_empty() { + String::new() + } else { + let list = dependencies + .iter() + .map(|dependency| format!("'{dependency}'")) + .collect::>() + .join(", "); + format!("\ndependencies = {{ {list} }}") + }; + format!( + "rockspec_format = '1.0'\npackage = '{package}'\nversion = '1.0.0-1'\nsource = {{ url = 'https://example.com/{package}.zip' }}{dependencies}" + ) + } + + fn install_spec(name: &str) -> PackageInstallSpec { + let package = match name.parse::() { + Ok(package) => package, + Err(err) => panic!("invalid package req: {err}"), + }; + PackageInstallSpec::new(package, EntryType::Entrypoint).build() + } + + fn roots(spec: PackageInstallSpec) -> Artifacts> { + Artifacts { + regular: Some(vec![spec]), + build: None, + test: None, + } + } + + #[tokio::test] + async fn diamond_is_discovered_once_per_package() { + let discover = MockDiscover::new(&[ + rockspec("a", &["b", "c"]), + rockspec("b", &["d"]), + rockspec("c", &["d"]), + rockspec("d", &[]), + ]); + + let resolved = match Resolver::with_max_inflight(&discover, 8) + .run(roots(install_spec("a"))) + .await + { + Ok(resolved) => resolved, + Err(err) => panic!("resolution failed: {err}"), + }; + + assert_eq!(resolved.regular.as_ref().map(|rocks| rocks.len()), Some(4)); + for name in ["a", "b", "c", "d"] { + assert_eq!(discover.calls(name), 1, "{name} should be discovered once"); + } + } + + #[tokio::test] + async fn cyclic_dependencies_are_rejected() { + let discover = MockDiscover::new(&[ + rockspec("a", &["b"]), + rockspec("b", &["a"]), + ]); + + let result = Resolver::with_max_inflight(&discover, 8) + .run(roots(install_spec("a"))) + .await; + + assert!(matches!(result, Err(ResolveError::CyclicDependency(_)))); + } +} From 69113bdfc0a956363050221e96e0e16f3628f02c Mon Sep 17 00:00:00 2001 From: vhyrro Date: Sun, 4 Oct 2026 10:58:41 +0200 Subject: [PATCH 12/39] feat: paralellize build step --- lux-lib/src/config/mod.rs | 4 +++- lux-lib/src/operations/fetch.rs | 30 +++++++++++++++++++++--------- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/lux-lib/src/config/mod.rs b/lux-lib/src/config/mod.rs index ba2904607..81e7b4aa4 100644 --- a/lux-lib/src/config/mod.rs +++ b/lux-lib/src/config/mod.rs @@ -821,7 +821,9 @@ impl ConfigBuilder { no_progress: self.no_progress.unwrap_or(false), no_prompt: self.no_prompt.unwrap_or(false), timeout: self.timeout.unwrap_or_else(|| Duration::from_secs(30)), - max_jobs: match self.max_jobs.unwrap_or(usize::MAX) { + max_jobs: match self.max_jobs.unwrap_or_else(|| { + std::thread::available_parallelism().map_or(1, |available| available.get()) + }) { 0 => usize::MAX, max_jobs => max_jobs, }, diff --git a/lux-lib/src/operations/fetch.rs b/lux-lib/src/operations/fetch.rs index 330759f9c..ad5fdb497 100644 --- a/lux-lib/src/operations/fetch.rs +++ b/lux-lib/src/operations/fetch.rs @@ -117,6 +117,9 @@ pub enum FetchSrcError { #[error("failed to clone rock source")] #[diagnostic(help("check your network connection and verify the git URL is correct."))] GitClone(#[from] git2::Error), + #[error("git operation failed")] + #[diagnostic(help("this is a bug; please report it."))] + GitJoin(#[from] tokio::task::JoinError), #[error("failed to parse git URL")] #[diagnostic(forward(0))] GitUrlParse(#[from] RemoteGitUrlParseError), @@ -257,8 +260,15 @@ async fn fetch_src_impl( let url = git.url.to_string(); tracing::debug!(message = format!("Cloning {url}").as_str()); - let (checkout_ref, has_submodules) = { - let mut auth = if config.no_prompt() { + let clone_url = url.clone(); + let clone_dir = dest_dir.to_path_buf(); + let host = git.url.host().to_string(); + let no_prompt = config.no_prompt(); + let access_token = config.access_token(&host); + let git_ref = git.git_ref.clone(); + + let (checkout_ref, has_submodules) = tokio::task::spawn_blocking(move || { + let mut auth = if no_prompt { GitAuthenticator::default() .try_password_prompt(0) .prompt_ssh_key_password(false) @@ -266,9 +276,9 @@ async fn fetch_src_impl( } else { GitAuthenticator::default() }; - if let Some(access_token) = config.access_token(git.url.host()) { + if let Some(access_token) = access_token { auth = auth.add_plaintext_credentials( - git.url.host(), + &host, access_token.username(), unsafe { access_token.password() }, ); @@ -279,7 +289,7 @@ async fn fetch_src_impl( let mut fetch_options = FetchOptions::new(); fetch_options.update_fetchhead(false); fetch_options.remote_callbacks(callbacks); - let checkout_ref = match &git.git_ref { + let checkout_ref = match &git_ref { Some(GitRef::Tag(tag)) => Some(tag.as_str()), Some(GitRef::Branch(branch)) => Some(branch.as_str()), None => None, @@ -289,11 +299,11 @@ async fn fetch_src_impl( }; let mut repo_builder = RepoBuilder::new(); repo_builder.fetch_options(fetch_options); - let repo = repo_builder.clone(&url, dest_dir)?; + let repo = repo_builder.clone(&clone_url, &clone_dir)?; let checkout_ref = match checkout_ref { Some(checkout_ref) => { - let ref_name = if let Some(GitRef::Branch(branch)) = &git.git_ref { + let ref_name = if let Some(GitRef::Branch(branch)) = &git_ref { format!("origin/{branch}") } else { checkout_ref.to_string() @@ -311,8 +321,10 @@ async fn fetch_src_impl( let submodule_paths = init_submodules(&repo, &auth, &git_config)?; - (checkout_ref, !submodule_paths.is_empty()) - }; + Ok::<_, FetchSrcError>((checkout_ref, !submodule_paths.is_empty())) + }) + .await??; + // The .git directory is not deterministic remove_dir_all(dest_dir.join(".git")).map_err(FetchSrcError::CleanGitDir)?; let hash = dest_dir.hash().await.map_err(FetchSrcError::Hash)?; From f440da76c234716e36f5168a85fc89845d1250f9 Mon Sep 17 00:00:00 2001 From: vhyrro Date: Sun, 4 Oct 2026 17:09:49 +0200 Subject: [PATCH 13/39] fix(vendor): deduplicate code in vendor logic --- lux-lib/src/operations/vendor.rs | 249 ++++++++++++++----------------- 1 file changed, 111 insertions(+), 138 deletions(-) diff --git a/lux-lib/src/operations/vendor.rs b/lux-lib/src/operations/vendor.rs index 64004cc20..d41debc34 100644 --- a/lux-lib/src/operations/vendor.rs +++ b/lux-lib/src/operations/vendor.rs @@ -25,6 +25,7 @@ use crate::{ operations::{ self, PackageInstallSpec, UnpackError, pipeline::{ + Artifacts, discover::FindPackageFromLuarocks, download_sources_and_hash::{ DownloadSourcesAndHash, DownloadSourcesAndHashError, DownloadedPackage, @@ -122,115 +123,86 @@ where const CARGO_VENDOR_SUBDIR: &str = "cargo"; -/// The set of packages to vendor, resolved and materialized into sources. -struct VendoredPackages { - packages: Vec, -} - -impl VendoredPackages { - /// Resolves and materializes every dependency of the target, across all lock types. - async fn new( - target: &VendorTarget, - no_lock: bool, - config: &Config, - ) -> Result { - let mut packages = Vec::new(); - for lock_type in LockedPackageLockType::iter() { - let (package_db, install_specs) = - gather_install_specs(&lock_type, no_lock, target, config).await?; - packages.extend( - ResolveAndDownload::new(config, package_db, install_specs) - .run() - .await?, - ); - } - - // The lockfile may contain the same package (name@version) multiple times, - // with different constraints. - let packages = packages - .into_iter() - .unique_by(|pkg| { - ( - pkg.package.spec.name().clone(), - pkg.package.spec.version().clone(), - ) - }) - .collect(); - - Ok(Self { packages }) +/// Resolves the requested packages and downloads (and hashes) their sources. +async fn resolve_and_download( + config: &Config, + package_db: RemotePackageDB, + install_specs: Artifacts>, +) -> Result, VendorError> { + let discover = + FindPackageFromLuarocks::new(Arc::new(package_db), Arc::new(config.clone())).build(); + + let Artifacts { + regular, + build, + test, + } = install_specs; + let mut resolve = + ResolvePackageDependencies::new(&discover, config).packages(regular.unwrap_or_default()); + if let Some(build) = build { + resolve = resolve.build_packages(build); } - - /// Vendors the sources of all packages into `vendor_dir`. - async fn vendor_sources(&self, vendor_dir: &Path, config: &Config) -> Result<(), VendorError> { - futures::stream::iter( - self.packages - .iter() - .map(|package| vendor_package_sources(vendor_dir, package)), - ) - .buffered(config.max_jobs()) - .try_collect() - .await + if let Some(test) = test { + resolve = resolve.test_packages(test); } + let resolved = resolve.resolve().await?; - /// Cargo-based build backends need their Cargo dependencies vendored too. - fn cargo_dependencies(&self) -> Vec<(PackageSpec, Option, Vec)> { - self.packages - .iter() - .filter_map(|package| { - let rockspec = &package.rockspec; - match rockspec.build().current_platform().build_backend { - Some(BuildBackendSpec::RustMlua(_) | BuildBackendSpec::RustBinary(_)) => { - Some(( - package.package.spec.to_package(), - rockspec.source().current_platform().unpack_dir.clone(), - rockspec.build().current_platform().copy_directories.clone(), - )) - } - _ => None, - } - }) - .collect() - } -} + let artifacts = DownloadSourcesAndHash::new(config) + .resolved(resolved) + .download_sources_and_hash() + .await?; -/// Resolves a set of packages and downloads (and hashes) their sources. -struct ResolveAndDownload<'a> { - config: &'a Config, - package_db: RemotePackageDB, - install_specs: Vec, + // The lockfile may contain the same package (name@version) multiple times, + // with different constraints. + Ok(artifacts + .into_iter() + .flat_map(|(_, packages)| packages) + .flat_map(HashMap::into_values) + .unique_by(|pkg| { + ( + pkg.package.spec.name().clone(), + pkg.package.spec.version().clone(), + ) + }) + .collect()) } -impl<'a> ResolveAndDownload<'a> { - fn new( - config: &'a Config, - package_db: RemotePackageDB, - install_specs: Vec, - ) -> Self { - Self { - config, - package_db, - install_specs, - } - } +/// Vendors the sources of all packages into `vendor_dir`. +async fn vendor_sources( + vendor_dir: &Path, + packages: &[DownloadedPackage], + config: &Config, +) -> Result<(), VendorError> { + futures::stream::iter( + packages + .iter() + .map(|package| vendor_package_sources(vendor_dir, package)), + ) + .buffered(config.max_jobs()) + .try_collect() + .await +} - async fn run(self) -> Result, VendorError> { - let discover = - FindPackageFromLuarocks::new(Arc::new(self.package_db), Arc::new(self.config.clone())) - .build(); - let resolved = ResolvePackageDependencies::new(&discover, self.config) - .packages(self.install_specs) - .resolve() - .await?; - let artifacts = DownloadSourcesAndHash::new(self.config) - .resolved(resolved) - .download_sources_and_hash() - .await?; - Ok(artifacts - .into_iter() - .flat_map(|(_, packages)| packages) - .flat_map(HashMap::into_values) - .collect()) - } +/// Cargo-based build backends need their Cargo dependencies vendored too. +fn cargo_dependencies( + packages: &[DownloadedPackage], +) -> Vec<(PackageSpec, Option, Vec)> { + packages + .iter() + .filter_map(|package| { + let rockspec = &package.rockspec; + match rockspec.build().current_platform().build_backend { + Some(BuildBackendSpec::RustMlua(_) | BuildBackendSpec::RustBinary(_)) => { + Some(( + package.package.spec.to_package(), + rockspec.source().current_platform().unpack_dir.clone(), + rockspec.build().current_platform().copy_directories.clone(), + )) + } + _ => None, + } + }) + .collect() } async fn do_vendor_dependencies(args: Vendor<'_>) -> Result<(), VendorError> { @@ -240,16 +212,17 @@ async fn do_vendor_dependencies(args: Vendor<'_>) -> Result<(), VendorError> { let target = args.target; let config = args.config; - let vendored = VendoredPackages::new(&target, no_lock, config).await?; + let (package_db, install_specs) = gather_install_specs(no_lock, &target, config).await?; + let packages = resolve_and_download(config, package_db, install_specs).await?; - let cargo_deps = vendored.cargo_dependencies(); + let cargo_deps = cargo_dependencies(&packages); if !no_delete && vendor_dir.exists() { fs::tokio::remove_dir_all(&vendor_dir).await?; } let vendor_dir = Arc::new(vendor_dir); - vendored.vendor_sources(&vendor_dir, config).await?; + vendor_sources(&vendor_dir, &packages, config).await?; vendor_target_cargo_deps(&vendor_dir, &target, config).await?; for (dep, unpack_dir, copy_dirs) in cargo_deps { vendor_package_cargo_deps(&vendor_dir, &dep, &unpack_dir, ©_dirs, config).await?; @@ -258,47 +231,47 @@ async fn do_vendor_dependencies(args: Vendor<'_>) -> Result<(), VendorError> { } async fn gather_install_specs( - lock_type: &LockedPackageLockType, no_lock: bool, target: &VendorTarget, config: &Config, -) -> Result<(RemotePackageDB, Vec), VendorError> { - match &target { - VendorTarget::Workspace(workspace) => { - // Resolve against the project's lockfile if present, otherwise fall - // back to the remote package DB (e.g. for a project that has not - // yet generated a lockfile). - let lockfile = workspace.try_lockfile()?; - let package_db = match lockfile { - Some(lockfile) if !no_lock => lockfile.local_pkg_locks().into(), - _ => RemotePackageDB::from_config(config).await?, - }; - let mut install_specs = Vec::new(); - for project in workspace.members() { - let toml = project.toml().into_local()?; - push_dependencies(lock_type, &toml, &mut install_specs)?; - if *lock_type == LockedPackageLockType::Test { - for test_spec_dependency in toml - .test() - .current_platform() - .test_dependencies(project) - .iter() - .cloned() - .map(|dep| PackageInstallSpec::new(dep, EntryType::Entrypoint).build()) - { - install_specs.push(test_spec_dependency); +) -> Result<(RemotePackageDB, Artifacts>), VendorError> { + // Resolve against the project's lockfile if present, otherwise fall back to + // the remote package DB (e.g. for a project that has not yet generated a lockfile). + let package_db = match target { + VendorTarget::Workspace(workspace) => match workspace.try_lockfile()? { + Some(lockfile) if !no_lock => lockfile.local_pkg_locks().into(), + _ => RemotePackageDB::from_config(config).await?, + }, + VendorTarget::Rockspec(_) => RemotePackageDB::from_config(config).await?, + }; + + let mut install_specs: Artifacts> = Artifacts::default(); + for lock_type in LockedPackageLockType::iter() { + let specs = install_specs.get_mut(lock_type).get_or_insert_with(Vec::new); + match target { + VendorTarget::Workspace(workspace) => { + for project in workspace.members() { + let toml = project.toml().into_local()?; + push_dependencies(&lock_type, &toml, specs)?; + if lock_type == LockedPackageLockType::Test { + specs.extend( + toml.test() + .current_platform() + .test_dependencies(project) + .iter() + .cloned() + .map(|dep| PackageInstallSpec::new(dep, EntryType::Entrypoint).build()), + ); } } } - Ok((package_db, install_specs)) - } - VendorTarget::Rockspec(remote_lua_rockspec) => { - let package_db = RemotePackageDB::from_config(config).await?; - let mut install_specs = Vec::new(); - push_dependencies(lock_type, remote_lua_rockspec, &mut install_specs)?; - Ok((package_db, install_specs)) + VendorTarget::Rockspec(rockspec) => { + push_dependencies(&lock_type, rockspec, specs)?; + } } } + + Ok((package_db, install_specs)) } fn push_dependencies( From 3a2b7cf3fbd3bb6ca5025c7cea0af3efcff59445 Mon Sep 17 00:00:00 2001 From: vhyrro Date: Sun, 4 Oct 2026 17:15:14 +0200 Subject: [PATCH 14/39] feat(lockfile): use atomic writes to prevent corruption during multithreading --- lux-cli/src/build.rs | 5 +- lux-cli/src/doc.rs | 2 +- lux-cli/src/sync.rs | 2 +- lux-lib/src/build/cmake.rs | 2 +- lux-lib/src/build/command.rs | 2 +- lux-lib/src/build/external_dependency.rs | 2 +- lux-lib/src/build/make.rs | 2 +- lux-lib/src/build/mod.rs | 36 ++- lux-lib/src/build/rust_binary.rs | 2 +- lux-lib/src/build/source.rs | 2 +- lux-lib/src/build/utils.rs | 10 +- lux-lib/src/fs/sync.rs | 34 +++ lux-lib/src/lockfile/mod.rs | 216 ++++++++---------- lux-lib/src/luarocks/install_binary_rock.rs | 8 +- lux-lib/src/operations/build_workspace.rs | 4 +- lux-lib/src/operations/dist_bin.rs | 22 +- lux-lib/src/operations/fetch.rs | 8 +- lux-lib/src/operations/gen_luarc.rs | 8 +- .../src/operations/install_dependencies.rs | 8 +- lux-lib/src/operations/pack.rs | 8 +- lux-lib/src/operations/pipeline/build.rs | 2 +- .../pipeline/download_sources_and_hash.rs | 4 +- .../src/operations/pipeline/emit_lockfile.rs | 2 +- .../operations/pipeline/install_workspace.rs | 16 +- lux-lib/src/operations/pipeline/resolve.rs | 56 +++-- lux-lib/src/operations/sync.rs | 60 ++--- lux-lib/src/operations/update.rs | 4 +- lux-lib/src/operations/vendor.rs | 29 +-- lux-lib/tests/build.rs | 4 +- lux-lib/tests/build_workspace.rs | 2 +- lux-lib/tests/vendor.rs | 5 +- 31 files changed, 283 insertions(+), 284 deletions(-) diff --git a/lux-cli/src/build.rs b/lux-cli/src/build.rs index 798bdf32b..3aec2ecf4 100644 --- a/lux-cli/src/build.rs +++ b/lux-cli/src/build.rs @@ -1,9 +1,6 @@ use clap::Args; use lux_lib::{ - config::Config, - lockfile::LockedPackage, - operations::BuildWorkspace, - package::PackageName, + config::Config, lockfile::LockedPackage, operations::BuildWorkspace, package::PackageName, workspace::Workspace, }; use miette::Result; diff --git a/lux-cli/src/doc.rs b/lux-cli/src/doc.rs index 69988ff93..aae94230b 100644 --- a/lux-cli/src/doc.rs +++ b/lux-cli/src/doc.rs @@ -10,7 +10,7 @@ use lux_lib::{ rockspec::Rockspec, tree::{InstallTree, RockMatches, Tree}, }; -use miette::{Context, IntoDiagnostic, Result, miette}; +use miette::{miette, Context, IntoDiagnostic, Result}; use std::io::Read; use std::path::Path; use url::Url; diff --git a/lux-cli/src/sync.rs b/lux-cli/src/sync.rs index c96ae2bbd..490d738c5 100644 --- a/lux-cli/src/sync.rs +++ b/lux-cli/src/sync.rs @@ -1,7 +1,7 @@ use clap::Args; use lux_lib::{ config::Config, - operations::{GenLuaRc, pipeline::install_workspace::InstallWorkspaceDependencies}, + operations::{pipeline::install_workspace::InstallWorkspaceDependencies, GenLuaRc}, workspace::Workspace, }; diff --git a/lux-lib/src/build/cmake.rs b/lux-lib/src/build/cmake.rs index 21e1712c5..6c82e5bfe 100644 --- a/lux-lib/src/build/cmake.rs +++ b/lux-lib/src/build/cmake.rs @@ -6,7 +6,7 @@ use std::{ }; use thiserror::Error; use tokio::process::Command; -use tracing::{Instrument, info_span}; +use tracing::{info_span, Instrument}; use crate::{ build::{ diff --git a/lux-lib/src/build/command.rs b/lux-lib/src/build/command.rs index 79ae505eb..d7f73e6be 100644 --- a/lux-lib/src/build/command.rs +++ b/lux-lib/src/build/command.rs @@ -7,7 +7,7 @@ use std::{ }; use thiserror::Error; -use tracing::{Instrument, info_span}; +use tracing::{info_span, Instrument}; use which::which; use crate::{ diff --git a/lux-lib/src/build/external_dependency.rs b/lux-lib/src/build/external_dependency.rs index 4929fe580..32e16d1d9 100644 --- a/lux-lib/src/build/external_dependency.rs +++ b/lux-lib/src/build/external_dependency.rs @@ -422,7 +422,7 @@ fn is_lib_name(file_name: &str, prefix: &str) -> bool { #[cfg(test)] mod tests { use super::*; - use assert_fs::{TempDir, prelude::*}; + use assert_fs::{prelude::*, TempDir}; #[tokio::test] async fn test_detect_zlib_pkg_config_header() { diff --git a/lux-lib/src/build/make.rs b/lux-lib/src/build/make.rs index ea95426a9..6f136cd0c 100644 --- a/lux-lib/src/build/make.rs +++ b/lux-lib/src/build/make.rs @@ -8,7 +8,7 @@ use std::{ }; use thiserror::Error; -use tracing::{Instrument, info_span}; +use tracing::{info_span, Instrument}; use crate::{ build::{ diff --git a/lux-lib/src/build/mod.rs b/lux-lib/src/build/mod.rs index a68ef0598..a002bf080 100644 --- a/lux-lib/src/build/mod.rs +++ b/lux-lib/src/build/mod.rs @@ -19,7 +19,7 @@ use crate::{ lockfile::{LockConstraint, LockedPackage, LockedPackageSpec, PinnedState}, lua_installation::LuaInstallation, lua_rockspec::BuildBackendSpec, - operations::{FetchSrcError, pipeline::download_sources_and_hash::fetch_and_hash_source}, + operations::{pipeline::download_sources_and_hash::fetch_and_hash_source, FetchSrcError}, remote_package_source::RemotePackageSource, }; use bon::Builder; @@ -40,7 +40,7 @@ use source::SourceBuildError; use ssri::Integrity; use thiserror::Error; use treesitter_parser::TreesitterBuildError; -use utils::{CompileCFilesError, InstallBinaryError, recursive_copy_dir}; +use utils::{recursive_copy_dir, CompileCFilesError, InstallBinaryError}; mod builtin; mod cmake; @@ -376,7 +376,9 @@ pub(crate) async fn deploy( let mut binaries = rockspec.binaries(); binaries.extend(output.binaries); - tree.lockfile()?.write_guard().set_binaries(&package, binaries); + tree.lockfile()? + .write_guard() + .set_binaries(&package, binaries); install( rockspec, @@ -535,9 +537,9 @@ mod tests { use crate::{ config::ConfigBuilder, lockfile::LockedPackageHashes, - lua_installation::{LuaInstallation, detect_installed_lua_version}, + lua_installation::{detect_installed_lua_version, LuaInstallation}, lua_version::LuaVersion, - operations::{DownloadedPackedRockBytes, unpack_rockspec}, + operations::{unpack_rockspec, DownloadedPackedRockBytes}, package::PackageSpec, project::Project, tree::Tree, @@ -599,27 +601,21 @@ mod tests { assert!(foo_dir.is_dir()); let foo_init = foo_dir.join("init.lua"); assert!(foo_init.is_file()); - assert!( - std::fs::read_to_string(&foo_init) - .unwrap() - .contains("return true") - ); + assert!(std::fs::read_to_string(&foo_init) + .unwrap() + .contains("return true")); let foo_bar_dir = foo_dir.join("bar"); assert!(foo_bar_dir.is_dir()); let foo_bar_init = foo_bar_dir.join("init.lua"); assert!(foo_bar_init.is_file()); - assert!( - std::fs::read_to_string(&foo_bar_init) - .unwrap() - .contains("return true") - ); + assert!(std::fs::read_to_string(&foo_bar_init) + .unwrap() + .contains("return true")); let foo_bar_baz = foo_bar_dir.join("baz.lua"); assert!(foo_bar_baz.is_file()); - assert!( - std::fs::read_to_string(&foo_bar_baz) - .unwrap() - .contains("return true") - ); + assert!(std::fs::read_to_string(&foo_bar_baz) + .unwrap() + .contains("return true")); let bin_file = tree_dir .child(lua_version.to_string()) .child("bin") diff --git a/lux-lib/src/build/rust_binary.rs b/lux-lib/src/build/rust_binary.rs index 8c2dd5b80..22acda561 100644 --- a/lux-lib/src/build/rust_binary.rs +++ b/lux-lib/src/build/rust_binary.rs @@ -1,6 +1,6 @@ use crate::build::backend::{BuildBackend, BuildInfo, RunBuildArgs}; use crate::build::utils::{self, InstallBinaryError}; -use crate::config::{Config, build}; +use crate::config::{build, Config}; use crate::fs; use crate::lua_rockspec::RustBinaryBuildSpec; use crate::tree::InstallTree; diff --git a/lux-lib/src/build/source.rs b/lux-lib/src/build/source.rs index f08150f09..9bdf8a358 100644 --- a/lux-lib/src/build/source.rs +++ b/lux-lib/src/build/source.rs @@ -5,8 +5,8 @@ use crate::{ build::backend::{BuildBackend, BuildInfo, RunBuildArgs}, lua_rockspec::{BuildBackendSpec, BuildSpec, LocalLuaRockspec, LuaRockspecError}, project::{ - PROJECT_TOML, ProjectRoot, TomlDeError, project_toml::{LocalProjectTomlValidationError, PartialProjectToml}, + ProjectRoot, TomlDeError, PROJECT_TOML, }, rockspec::Rockspec, tree::InstallTree, diff --git a/lux-lib/src/build/utils.rs b/lux-lib/src/build/utils.rs index 3f72b033a..0ad4a2e0c 100644 --- a/lux-lib/src/build/utils.rs +++ b/lux-lib/src/build/utils.rs @@ -1015,11 +1015,9 @@ mod tests { #[cfg(unix)] set_executable_permissions(&script_path).await.unwrap(); - assert!( - Command::new(script_path.to_string_lossy().to_string()) - .status() - .await - .is_ok_and(|status| status.success()) - ); + assert!(Command::new(script_path.to_string_lossy().to_string()) + .status() + .await + .is_ok_and(|status| status.success())); } } diff --git a/lux-lib/src/fs/sync.rs b/lux-lib/src/fs/sync.rs index b6d17db79..6ee75c339 100644 --- a/lux-lib/src/fs/sync.rs +++ b/lux-lib/src/fs/sync.rs @@ -23,6 +23,40 @@ pub(crate) fn write(path: impl AsRef, contents: impl AsRef<[u8]>) -> Resul }) } +pub(crate) fn write_atomic( + path: impl AsRef, + contents: impl AsRef<[u8]>, +) -> Result<(), FsError> { + use std::io::Write; + + let path = path.as_ref(); + let parent = match path.parent() { + Some(parent) if !parent.as_os_str().is_empty() => parent, + _ => Path::new("."), + }; + + let mut file = + tempfile::NamedTempFile::new_in(parent).map_err(|source| FsError::FileCreate { + path: path.to_path_buf(), + source, + })?; + file.write_all(contents.as_ref()) + .map_err(|source| FsError::Write { + path: path.to_path_buf(), + source, + })?; + file.as_file().sync_all().map_err(|source| FsError::Write { + path: path.to_path_buf(), + source, + })?; + file.persist(path) + .map_err(|err| FsError::Write { + path: path.to_path_buf(), + source: err.error, + }) + .map(|_| ()) +} + /// Wrapped [`fs::copy`]. pub(crate) fn copy(from: impl AsRef, to: impl AsRef) -> Result { let from = from.as_ref(); diff --git a/lux-lib/src/lockfile/mod.rs b/lux-lib/src/lockfile/mod.rs index a3fa5a7c4..2ce6687c2 100644 --- a/lux-lib/src/lockfile/mod.rs +++ b/lux-lib/src/lockfile/mod.rs @@ -14,7 +14,7 @@ use std::{ use itertools::Itertools; use miette::Diagnostic; -use serde::{Deserialize, Serialize, Serializer, de}; +use serde::{de, Deserialize, Serialize, Serializer}; use sha2::{Digest, Sha256}; use ssri::Integrity; use strum_macros::EnumIter; @@ -27,8 +27,8 @@ use crate::package::{ PackageVersionReqError, RemotePackageTypeFilterSpec, }; use crate::remote_package_source::RemotePackageSource; -use crate::rockspec::RockBinaries; use crate::rockspec::lua_dependency::LuaDependencySpec; +use crate::rockspec::RockBinaries; use crate::tree::{EntryType, InstallTree, Tree}; /// Bump this whenever an incompatible change is made to the lockfile format. @@ -56,7 +56,11 @@ impl Display for PinnedState { impl From for PinnedState { fn from(value: bool) -> Self { - if value { Self::Pinned } else { Self::Unpinned } + if value { + Self::Pinned + } else { + Self::Unpinned + } } } @@ -351,7 +355,6 @@ impl Display for RemotePackageSourceUrl { // TODO(vhyrro): Move to `package/local.rs` - /// A package whose source has been downloaded and hashed, but which has not /// yet been installed into a tree. /// @@ -399,7 +402,6 @@ impl LockedPackage { ) } - pub(crate) fn new( spec: LockedPackageSpec, source: RemotePackageSource, @@ -1120,7 +1122,7 @@ impl Lockfile

{ source: io::Error::other(err), })?; - fs::sync::write(&self.filepath, content).map_err(|err| FlushLockfileError { + fs::sync::write_atomic(&self.filepath, content).map_err(|err| FlushLockfileError { filepath: self.filepath.to_path_buf(), source: io::Error::other(err), }) @@ -1201,7 +1203,7 @@ impl WorkspaceLockfile

{ fn flush(&self) -> io::Result<()> { let content = serde_json::to_string_pretty(&self)?; - fs::sync::write(&self.filepath, content).map_err(io::Error::other)?; + fs::sync::write_atomic(&self.filepath, content).map_err(io::Error::other)?; Ok(()) } @@ -1758,43 +1760,33 @@ mod tests { assert_eq!(sync_spec.to_add.len(), 1); // Should keep dependencies of neorg 8.8.1-1 - assert!( - !sync_spec - .to_remove - .iter() - .any(|pkg| pkg.name().to_string() == "nvim-nio" - && pkg.constraint() - == LockConstraint::Constrained(">=1.7.0, <1.8.0".parse().unwrap())) - ); - assert!( - !sync_spec - .to_remove - .iter() - .any(|pkg| pkg.name().to_string() == "lua-utils.nvim" - && pkg.constraint() == LockConstraint::Constrained("=1.0.2".parse().unwrap())) - ); - assert!( - !sync_spec - .to_remove - .iter() - .any(|pkg| pkg.name().to_string() == "plenary.nvim" - && pkg.constraint() == LockConstraint::Constrained("=0.1.4".parse().unwrap())) - ); - assert!( - !sync_spec - .to_remove - .iter() - .any(|pkg| pkg.name().to_string() == "nui.nvim" - && pkg.constraint() == LockConstraint::Constrained("=0.3.0".parse().unwrap())) - ); - assert!( - !sync_spec - .to_remove - .iter() - .any(|pkg| pkg.name().to_string() == "pathlib.nvim" - && pkg.constraint() - == LockConstraint::Constrained(">=2.2.0, <2.3.0".parse().unwrap())) - ); + assert!(!sync_spec + .to_remove + .iter() + .any(|pkg| pkg.name().to_string() == "nvim-nio" + && pkg.constraint() + == LockConstraint::Constrained(">=1.7.0, <1.8.0".parse().unwrap()))); + assert!(!sync_spec + .to_remove + .iter() + .any(|pkg| pkg.name().to_string() == "lua-utils.nvim" + && pkg.constraint() == LockConstraint::Constrained("=1.0.2".parse().unwrap()))); + assert!(!sync_spec + .to_remove + .iter() + .any(|pkg| pkg.name().to_string() == "plenary.nvim" + && pkg.constraint() == LockConstraint::Constrained("=0.1.4".parse().unwrap()))); + assert!(!sync_spec + .to_remove + .iter() + .any(|pkg| pkg.name().to_string() == "nui.nvim" + && pkg.constraint() == LockConstraint::Constrained("=0.3.0".parse().unwrap()))); + assert!(!sync_spec + .to_remove + .iter() + .any(|pkg| pkg.name().to_string() == "pathlib.nvim" + && pkg.constraint() + == LockConstraint::Constrained(">=2.2.0, <2.3.0".parse().unwrap()))); } #[test] @@ -1814,50 +1806,38 @@ mod tests { // Should remove: // - neorg // - dependencies unique to neorg - assert!( - sync_spec - .to_remove - .iter() - .any(|pkg| pkg.name().to_string() == "neorg" - && pkg.version() == &"8.8.1-1".parse().unwrap()) - ); - assert!( - sync_spec - .to_remove - .iter() - .any(|pkg| pkg.name().to_string() == "nvim-nio" - && pkg.constraint() - == LockConstraint::Constrained(">=1.7.0, <1.8.0".parse().unwrap())) - ); - assert!( - sync_spec - .to_remove - .iter() - .any(|pkg| pkg.name().to_string() == "lua-utils.nvim" - && pkg.constraint() == LockConstraint::Constrained("=1.0.2".parse().unwrap())) - ); - assert!( - sync_spec - .to_remove - .iter() - .any(|pkg| pkg.name().to_string() == "plenary.nvim" - && pkg.constraint() == LockConstraint::Constrained("=0.1.4".parse().unwrap())) - ); - assert!( - sync_spec - .to_remove - .iter() - .any(|pkg| pkg.name().to_string() == "nui.nvim" - && pkg.constraint() == LockConstraint::Constrained("=0.3.0".parse().unwrap())) - ); - assert!( - sync_spec - .to_remove - .iter() - .any(|pkg| pkg.name().to_string() == "pathlib.nvim" - && pkg.constraint() - == LockConstraint::Constrained(">=2.2.0, <2.3.0".parse().unwrap())) - ); + assert!(sync_spec + .to_remove + .iter() + .any(|pkg| pkg.name().to_string() == "neorg" + && pkg.version() == &"8.8.1-1".parse().unwrap())); + assert!(sync_spec + .to_remove + .iter() + .any(|pkg| pkg.name().to_string() == "nvim-nio" + && pkg.constraint() + == LockConstraint::Constrained(">=1.7.0, <1.8.0".parse().unwrap()))); + assert!(sync_spec + .to_remove + .iter() + .any(|pkg| pkg.name().to_string() == "lua-utils.nvim" + && pkg.constraint() == LockConstraint::Constrained("=1.0.2".parse().unwrap()))); + assert!(sync_spec + .to_remove + .iter() + .any(|pkg| pkg.name().to_string() == "plenary.nvim" + && pkg.constraint() == LockConstraint::Constrained("=0.1.4".parse().unwrap()))); + assert!(sync_spec + .to_remove + .iter() + .any(|pkg| pkg.name().to_string() == "nui.nvim" + && pkg.constraint() == LockConstraint::Constrained("=0.3.0".parse().unwrap()))); + assert!(sync_spec + .to_remove + .iter() + .any(|pkg| pkg.name().to_string() == "pathlib.nvim" + && pkg.constraint() + == LockConstraint::Constrained(">=2.2.0, <2.3.0".parse().unwrap()))); } #[test] @@ -1882,19 +1862,15 @@ mod tests { .package_sync_spec(&packages, &SyncStrategy::LockfileOnly); let expected: PackageVersionReq = ">=2.0.0".parse().unwrap(); - assert!( - sync_spec - .to_add - .iter() - .any(|req| req.name().to_string() == "nvim-nio" && req.version_req() == &expected) - ); + assert!(sync_spec + .to_add + .iter() + .any(|req| req.name().to_string() == "nvim-nio" && req.version_req() == &expected)); - assert!( - sync_spec - .to_remove - .iter() - .any(|pkg| pkg.name().to_string() == "nvim-nio") - ); + assert!(sync_spec + .to_remove + .iter() + .any(|pkg| pkg.name().to_string() == "nvim-nio")); } #[test] @@ -1927,26 +1903,20 @@ mod tests { .lock .package_sync_spec(&packages, &SyncStrategy::EnsureInstalled(&tree)); - assert!( - !sync_spec - .to_add - .iter() - .any(|req| req.name().to_string() == "neorg") - ); + assert!(!sync_spec + .to_add + .iter() + .any(|req| req.name().to_string() == "neorg")); - assert!( - sync_spec - .to_add - .iter() - .any(|req| req.name().to_string() == "lua-cjson") - ); + assert!(sync_spec + .to_add + .iter() + .any(|req| req.name().to_string() == "lua-cjson")); - assert!( - sync_spec - .to_add - .iter() - .any(|req| req.name().to_string() == "nonexistent") - ); + assert!(sync_spec + .to_add + .iter() + .any(|req| req.name().to_string() == "nonexistent")); } #[test] @@ -1960,10 +1930,12 @@ mod tests { "# ); - assert!( - super::parse_lockfile::>(&lockfile, Path::new("lux.lock"), "") - .is_err() - ); + assert!(super::parse_lockfile::>( + &lockfile, + Path::new("lux.lock"), + "" + ) + .is_err()); } } } diff --git a/lux-lib/src/luarocks/install_binary_rock.rs b/lux-lib/src/luarocks/install_binary_rock.rs index a6cd88500..977adc203 100644 --- a/lux-lib/src/luarocks/install_binary_rock.rs +++ b/lux-lib/src/luarocks/install_binary_rock.rs @@ -6,9 +6,9 @@ use std::{ use crate::{ build::{ - BuildBehaviour, external_dependency::{ExternalDependencyError, ExternalDependencyInfo}, utils::recursive_copy_dir, + BuildBehaviour, }, config::Config, hash::HasIntegrity, @@ -58,7 +58,9 @@ pub enum InstallBinaryRockError { #[error(transparent)] #[diagnostic(transparent)] RockManifestError(#[from] RockManifestError), - #[error("the entry {0} listed in the `rock_manifest` is neither a file nor a directory: {1:?}")] + #[error( + "the entry {0} listed in the `rock_manifest` is neither a file nor a directory: {1:?}" + )] NotAFileOrDirectory(String, Box), } @@ -253,7 +255,7 @@ mod tests { use crate::{ config::ConfigBuilder, - operations::{DownloadedPackedRockBytes, unpack_rockspec}, + operations::{unpack_rockspec, DownloadedPackedRockBytes}, }; use super::*; diff --git a/lux-lib/src/operations/build_workspace.rs b/lux-lib/src/operations/build_workspace.rs index 4df33a441..839f6c728 100644 --- a/lux-lib/src/operations/build_workspace.rs +++ b/lux-lib/src/operations/build_workspace.rs @@ -3,18 +3,18 @@ use crate::{ lockfile::LockedPackage, lua_installation::{LuaInstallation, LuaInstallationError}, operations::{ - GenLuaRc, GenLuaRcError, pipeline::{ build_project::{BuildProject, BuildProjectError}, install_workspace::{InstallWorkspaceDependencies, InstallWorkspaceDependenciesError}, }, + GenLuaRc, GenLuaRcError, }, package::PackageName, workspace::{Workspace, WorkspaceError}, }; use bon::Builder; use thiserror::Error; -use tracing::{Instrument, info_span}; +use tracing::{info_span, Instrument}; #[derive(Debug, Error, miette::Diagnostic)] #[non_exhaustive] diff --git a/lux-lib/src/operations/dist_bin.rs b/lux-lib/src/operations/dist_bin.rs index 5593fc267..d4d1b4e67 100644 --- a/lux-lib/src/operations/dist_bin.rs +++ b/lux-lib/src/operations/dist_bin.rs @@ -16,7 +16,7 @@ use crate::{ lua_installation::{LuaInstallation, LuaInstallationError}, lua_rockspec::LuaModule, operations::{InstallProject, InstallProjectError}, - project::{Project, project_toml::LocalProjectTomlValidationError}, + project::{project_toml::LocalProjectTomlValidationError, Project}, rockspec::Rockspec, tree::{InstallTree, TreeError}, }; @@ -539,10 +539,10 @@ local unpack = unpack or table.unpack mod tests { use super::*; - use assert_fs::TempDir; use assert_fs::fixture::PathCopy; #[cfg(target_os = "linux")] use assert_fs::prelude::{PathChild, PathCreateDir}; + use assert_fs::TempDir; use crate::lua_installation::detect_installed_lua_version; use crate::{config::ConfigBuilder, lua_version::LuaVersion, tree::FlatDistTree}; @@ -627,18 +627,14 @@ mod tests { let files = collect_installed_files(&tree).unwrap(); assert_eq!(files.src.len(), 2); - assert!( - files - .src - .iter() - .all(|(_, p)| p.extension().is_some_and(|e| e == "lua")) - ); + assert!(files + .src + .iter() + .all(|(_, p)| p.extension().is_some_and(|e| e == "lua"))); assert_eq!(files.lib.len(), 1); - assert!( - files.lib[0] - .extension() - .is_some_and(|e| e == c_dylib_extension()) - ); + assert!(files.lib[0] + .extension() + .is_some_and(|e| e == c_dylib_extension())); } #[tokio::test] diff --git a/lux-lib/src/operations/fetch.rs b/lux-lib/src/operations/fetch.rs index ad5fdb497..06a501f01 100644 --- a/lux-lib/src/operations/fetch.rs +++ b/lux-lib/src/operations/fetch.rs @@ -277,11 +277,9 @@ async fn fetch_src_impl( GitAuthenticator::default() }; if let Some(access_token) = access_token { - auth = auth.add_plaintext_credentials( - &host, - access_token.username(), - unsafe { access_token.password() }, - ); + auth = auth.add_plaintext_credentials(&host, access_token.username(), unsafe { + access_token.password() + }); } let git_config = git2::Config::open_default()?; let mut callbacks = RemoteCallbacks::new(); diff --git a/lux-lib/src/operations/gen_luarc.rs b/lux-lib/src/operations/gen_luarc.rs index 5a8ea57e4..9a852b066 100644 --- a/lux-lib/src/operations/gen_luarc.rs +++ b/lux-lib/src/operations/gen_luarc.rs @@ -2,10 +2,10 @@ use crate::config::Config; use crate::fs; use crate::lockfile::LockedPackageLockType; use crate::tree::InstallTree; -use crate::workspace::LUX_DIR_NAME; use crate::workspace::Workspace; use crate::workspace::WorkspaceError; use crate::workspace::WorkspaceTreeError; +use crate::workspace::LUX_DIR_NAME; use bon::Builder; use itertools::Itertools; use miette::Diagnostic; @@ -96,11 +96,7 @@ async fn do_generate_luarc(args: GenLuaRc<'_>) -> Result<(), GenLuaRcError> { .local_pkg_lock(&LockedPackageLockType::Test) .rocks() .values() - .map(|dependency| { - test_dependency_tree - .layout_for(&dependency.spec) - .src - }) + .map(|dependency| test_dependency_tree.layout_for(&dependency.spec).src) .filter(|dir| dir.is_dir()) .filter_map(|test_dependency_dir| diff_paths(test_dependency_dir, workspace.root())); diff --git a/lux-lib/src/operations/install_dependencies.rs b/lux-lib/src/operations/install_dependencies.rs index 2aa2975d2..0126cd38c 100644 --- a/lux-lib/src/operations/install_dependencies.rs +++ b/lux-lib/src/operations/install_dependencies.rs @@ -6,7 +6,7 @@ use crate::{ lockfile::LockedPackage, lua_installation::LuaInstallation, luarocks::luarocks_installation::LuaRocksInstallation, - operations::{InstallError, pipeline::install_packages::InstallPackages}, + operations::{pipeline::install_packages::InstallPackages, InstallError}, project::project_toml::LocalProjectToml, rockspec::Rockspec, tree::{self, InstallTree}, @@ -30,9 +30,9 @@ where } impl< - T: InstallTree + Sync + Send + Clone + 'static, - State: install_dependencies_builder::State + install_dependencies_builder::IsComplete, -> InstallDependenciesBuilder<'_, T, State> + T: InstallTree + Sync + Send + Clone + 'static, + State: install_dependencies_builder::State + install_dependencies_builder::IsComplete, + > InstallDependenciesBuilder<'_, T, State> { /// Installs the configured dependencies and build dependencies into [`Self::tree`], /// returning the installed regular dependencies. diff --git a/lux-lib/src/operations/pack.rs b/lux-lib/src/operations/pack.rs index 1ad00420a..a227c109c 100644 --- a/lux-lib/src/operations/pack.rs +++ b/lux-lib/src/operations/pack.rs @@ -28,8 +28,8 @@ use std::path::Path; use std::path::PathBuf; use thiserror::Error; use walkdir::WalkDir; -use zip::ZipWriter; use zip::write::SimpleFileOptions; +use zip::ZipWriter; #[cfg(unix)] use std::os::unix::fs::PermissionsExt; @@ -180,7 +180,11 @@ fn add_rock_entries( if source_dir.is_dir() { for file in WalkDir::new(source_dir).into_iter().filter_map_ok(|entry| { let file = entry.into_path(); - if file.is_file() { Some(file) } else { None } + if file.is_file() { + Some(file) + } else { + None + } }) { let file = file?; let (relative_path, digest) = add_rock_entry(zip, file, source_dir, &zip_dir)?; diff --git a/lux-lib/src/operations/pipeline/build.rs b/lux-lib/src/operations/pipeline/build.rs index c9900771e..ec1b5cbb6 100644 --- a/lux-lib/src/operations/pipeline/build.rs +++ b/lux-lib/src/operations/pipeline/build.rs @@ -5,7 +5,7 @@ use miette::Diagnostic; use thiserror::Error; use crate::{ - build::{BuildBehaviour, deploy}, + build::{deploy, BuildBehaviour}, config::Config, lockfile::{LockedPackage, LockedPackageId}, lua_installation::{LuaInstallation, LuaInstallationError}, diff --git a/lux-lib/src/operations/pipeline/download_sources_and_hash.rs b/lux-lib/src/operations/pipeline/download_sources_and_hash.rs index 2c2a51255..641136227 100644 --- a/lux-lib/src/operations/pipeline/download_sources_and_hash.rs +++ b/lux-lib/src/operations/pipeline/download_sources_and_hash.rs @@ -19,15 +19,15 @@ use crate::{ RemotePackageSourceUrl, }, lua_rockspec::RemoteLuaRockspec, - operations::{FetchSrc, RemotePackageSourceMetadata, unpack_src_rock}, + operations::{unpack_src_rock, FetchSrc, RemotePackageSourceMetadata}, remote_package_source::RemotePackageSource, rockspec::Rockspec, tree::EntryType, }; use super::{ - Artifacts, resolve::{ResolvedArtifacts, ResolvedPackage}, + Artifacts, }; pub(crate) enum PackageSource { diff --git a/lux-lib/src/operations/pipeline/emit_lockfile.rs b/lux-lib/src/operations/pipeline/emit_lockfile.rs index e434b67ef..ab7a47f13 100644 --- a/lux-lib/src/operations/pipeline/emit_lockfile.rs +++ b/lux-lib/src/operations/pipeline/emit_lockfile.rs @@ -3,8 +3,8 @@ use std::collections::HashMap; use crate::lockfile::{LockedPackageId, LockedPackageLock, ReadWrite, WorkspaceLockfile}; use super::{ - Artifacts, download_sources_and_hash::{DownloadSourcesAndHashArtifacts, DownloadedPackage}, + Artifacts, }; /// The lockfile entries resolved by the pipeline, grouped by section. diff --git a/lux-lib/src/operations/pipeline/install_workspace.rs b/lux-lib/src/operations/pipeline/install_workspace.rs index 3eca0fc54..33e79fac7 100644 --- a/lux-lib/src/operations/pipeline/install_workspace.rs +++ b/lux-lib/src/operations/pipeline/install_workspace.rs @@ -165,15 +165,11 @@ mod tests { let lockfile = workspace.lockfile().unwrap(); let rocks = lockfile.rocks(&LockedPackageLockType::Regular); - assert!( - rocks - .values() - .any(|pkg| pkg.name().to_string() == "lua-cjson") - ); - assert!( - rocks - .values() - .any(|pkg| pkg.name().to_string() == "plenary.nvim") - ); + assert!(rocks + .values() + .any(|pkg| pkg.name().to_string() == "lua-cjson")); + assert!(rocks + .values() + .any(|pkg| pkg.name().to_string() == "plenary.nvim")); } } diff --git a/lux-lib/src/operations/pipeline/resolve.rs b/lux-lib/src/operations/pipeline/resolve.rs index d7099cd62..fdb5bfeca 100644 --- a/lux-lib/src/operations/pipeline/resolve.rs +++ b/lux-lib/src/operations/pipeline/resolve.rs @@ -21,9 +21,9 @@ use crate::{ }; use super::{ - Artifacts, discover::{DiscoverError, FindPackageFromProvider, FoundPackage, FoundPackageType}, download_sources_and_hash::PackageSource, + Artifacts, }; /// The build dependencies of a rockspec that still need to be installed, with the @@ -167,7 +167,11 @@ struct Node { struct Resolver<'a, D: FindPackageFromProvider> { discover: &'a D, max_inflight: usize, - joinset: JoinSet<(RequestKey, PackageInstallSpec, Result)>, + joinset: JoinSet<( + RequestKey, + PackageInstallSpec, + Result, + )>, pending: VecDeque<(RequestKey, PackageInstallSpec)>, seen: HashSet, nodes: HashMap, @@ -202,7 +206,9 @@ impl<'a, D: FindPackageFromProvider> Resolver<'a, D> { continue; }; // Mark the section as requested even if it resolves to nothing. - self.resolved.get_mut(section).get_or_insert_with(HashMap::new); + self.resolved + .get_mut(section) + .get_or_insert_with(HashMap::new); for spec in specs { let key = RequestKey::from_install_spec(section, &spec); self.enqueue(key, spec); @@ -269,13 +275,15 @@ impl<'a, D: FindPackageFromProvider> Resolver<'a, D> { } } for dependency in found.rockspec.dependencies().current_platform() { - let child_spec = - PackageInstallSpec::new(dependency.package_req().clone(), EntryType::DependencyOnly) - .build_behaviour(spec.build_behaviour) - .pin(spec.pin) - .opt(spec.opt) - .maybe_source(dependency.source().clone()) - .build(); + let child_spec = PackageInstallSpec::new( + dependency.package_req().clone(), + EntryType::DependencyOnly, + ) + .build_behaviour(spec.build_behaviour) + .pin(spec.pin) + .opt(spec.opt) + .maybe_source(dependency.source().clone()) + .build(); push_child(&mut child_specs, false, section, child_spec); } @@ -283,7 +291,9 @@ impl<'a, D: FindPackageFromProvider> Resolver<'a, D> { .iter() .map(|(is_build, child_key, _)| (*is_build, child_key.clone())) .collect(); - self.resolved.get_mut(section).get_or_insert_with(HashMap::new); + self.resolved + .get_mut(section) + .get_or_insert_with(HashMap::new); self.nodes.insert( key, Node { @@ -377,7 +387,10 @@ fn push_child( section }; let child_key = RequestKey::from_install_spec(child_section, &spec); - if !children.iter().any(|(_, existing, _)| existing == &child_key) { + if !children + .iter() + .any(|(_, existing, _)| existing == &child_key) + { children.push((is_build, child_key, spec)); } } @@ -445,9 +458,11 @@ async fn find_package_from_provider( discover: D, ) -> Result { if let Some(source) = &spec.source { - let download = - RemoteRockDownload::from_package_req_and_source_spec(spec.package.clone(), source.clone()) - .map_err(|err| DiscoverError::Download(spec.package.clone(), Box::new(err)))?; + let download = RemoteRockDownload::from_package_req_and_source_spec( + spec.package.clone(), + source.clone(), + ) + .map_err(|err| DiscoverError::Download(spec.package.clone(), Box::new(err)))?; let rockspec = download.rockspec().clone(); let package = RemotePackage::new( spec.package.clone().try_into().map_err(|err| { @@ -552,7 +567,11 @@ mod tests { *calls.entry(name.clone()).or_insert(0) += 1; } let result = self.rockspecs.get(&name).map_or_else( - || Err(DiscoverError::Search(SearchError::RockNotFound(req.clone()))), + || { + Err(DiscoverError::Search(SearchError::RockNotFound( + req.clone(), + ))) + }, |(content, rockspec)| { let package = RemotePackage::new( PackageSpec::new(req.name().clone(), rockspec.version().clone()), @@ -627,10 +646,7 @@ mod tests { #[tokio::test] async fn cyclic_dependencies_are_rejected() { - let discover = MockDiscover::new(&[ - rockspec("a", &["b"]), - rockspec("b", &["a"]), - ]); + let discover = MockDiscover::new(&[rockspec("a", &["b"]), rockspec("b", &["a"])]); let result = Resolver::with_max_inflight(&discover, 8) .run(roots(install_spec("a"))) diff --git a/lux-lib/src/operations/sync.rs b/lux-lib/src/operations/sync.rs index b39a0dc3b..3969e911c 100644 --- a/lux-lib/src/operations/sync.rs +++ b/lux-lib/src/operations/sync.rs @@ -10,10 +10,10 @@ use crate::{ PackageSyncSpec, ReadOnly, ReadWrite, SyncStrategy, WorkspaceLockfile, }, luarocks::luarocks_installation::LUAROCKS_VERSION, - operations::{self, GenLuaRcError, pipeline::resolve::luarocks_build_backend_name}, + operations::{self, pipeline::resolve::luarocks_build_backend_name, GenLuaRcError}, package::{PackageName, PackageReq}, - project::{ProjectError, project_toml::LocalProjectTomlValidationError}, - rockspec::{Rockspec, lua_dependency::LuaDependencySpec}, + project::{project_toml::LocalProjectTomlValidationError, ProjectError}, + rockspec::{lua_dependency::LuaDependencySpec, Rockspec}, tree::{self, InstallTree, TreeError}, workspace::{Workspace, WorkspaceError, WorkspaceTreeError}, }; @@ -455,7 +455,7 @@ mod tests { config::ConfigBuilder, lockfile::LockedPackageLockType, package::PackageReq, workspace::Workspace, }; - use assert_fs::{TempDir, prelude::PathCopy}; + use assert_fs::{prelude::PathCopy, TempDir}; use flaky_test::flaky_test; use std::path::PathBuf; @@ -476,11 +476,9 @@ mod tests { assert!(!report.added.is_empty()); let lockfile_after_sync = workspace.lockfile().unwrap(); - assert!( - !lockfile_after_sync - .rocks(&LockedPackageLockType::Regular) - .is_empty() - ); + assert!(!lockfile_after_sync + .rocks(&LockedPackageLockType::Regular) + .is_empty()); } #[flaky_test(tokio, times = 5)] @@ -504,19 +502,15 @@ mod tests { .unwrap(); assert!(report.removed.is_empty()); assert!(!report.added.is_empty()); - assert!( - report - .added - .iter() - .any(|pkg| pkg.name().to_string() == "toml-edit") - ); + assert!(report + .added + .iter() + .any(|pkg| pkg.name().to_string() == "toml-edit")); } let lockfile_after_sync = workspace.lockfile().unwrap(); - assert!( - !lockfile_after_sync - .rocks(&LockedPackageLockType::Regular) - .is_empty() - ); + assert!(!lockfile_after_sync + .rocks(&LockedPackageLockType::Regular) + .is_empty()); } #[flaky_test(tokio, times = 5)] @@ -541,19 +535,15 @@ mod tests { .unwrap(); assert!(report.removed.is_empty()); assert!(!report.added.is_empty()); - assert!( - report - .added - .iter() - .any(|pkg| pkg.name().to_string() == "toml-edit") - ); + assert!(report + .added + .iter() + .any(|pkg| pkg.name().to_string() == "toml-edit")); } let lockfile_after_sync = workspace.lockfile().unwrap(); - assert!( - !lockfile_after_sync - .rocks(&LockedPackageLockType::Regular) - .is_empty() - ); + assert!(!lockfile_after_sync + .rocks(&LockedPackageLockType::Regular) + .is_empty()); } #[flaky_test(tokio, times = 5)] @@ -579,10 +569,8 @@ mod tests { assert!(report.added.is_empty()); let lockfile_after_sync = workspace.lockfile().unwrap(); - assert!( - !lockfile_after_sync - .rocks(&LockedPackageLockType::Regular) - .is_empty() - ); + assert!(!lockfile_after_sync + .rocks(&LockedPackageLockType::Regular) + .is_empty()); } } diff --git a/lux-lib/src/operations/update.rs b/lux-lib/src/operations/update.rs index 6fedece13..1924e24b9 100644 --- a/lux-lib/src/operations/update.rs +++ b/lux-lib/src/operations/update.rs @@ -19,8 +19,8 @@ use miette::Diagnostic; use thiserror::Error; use super::{ - InstallError, PackageInstallSpec, RemoveError, SyncError, Uninstall, - pipeline::install_packages::InstallPackages, + pipeline::install_packages::InstallPackages, InstallError, PackageInstallSpec, RemoveError, + SyncError, Uninstall, }; #[derive(Error, Debug, Diagnostic)] diff --git a/lux-lib/src/operations/vendor.rs b/lux-lib/src/operations/vendor.rs index d41debc34..a49a21a34 100644 --- a/lux-lib/src/operations/vendor.rs +++ b/lux-lib/src/operations/vendor.rs @@ -23,16 +23,17 @@ use crate::{ lockfile::LockedPackageLockType, lua_rockspec::{BuildBackendSpec, RemoteLuaRockspec}, operations::{ - self, PackageInstallSpec, UnpackError, + self, pipeline::{ - Artifacts, discover::FindPackageFromLuarocks, download_sources_and_hash::{ DownloadSourcesAndHash, DownloadSourcesAndHashError, DownloadedPackage, PackageSource, }, - resolve::{ResolveError, ResolvePackageDependencies, luarocks_build_backend_name}, + resolve::{luarocks_build_backend_name, ResolveError, ResolvePackageDependencies}, + Artifacts, }, + PackageInstallSpec, UnpackError, }, package::{PackageReq, PackageSpec}, project::project_toml::LocalProjectTomlValidationError, @@ -192,13 +193,11 @@ fn cargo_dependencies( .filter_map(|package| { let rockspec = &package.rockspec; match rockspec.build().current_platform().build_backend { - Some(BuildBackendSpec::RustMlua(_) | BuildBackendSpec::RustBinary(_)) => { - Some(( - package.package.spec.to_package(), - rockspec.source().current_platform().unpack_dir.clone(), - rockspec.build().current_platform().copy_directories.clone(), - )) - } + Some(BuildBackendSpec::RustMlua(_) | BuildBackendSpec::RustBinary(_)) => Some(( + package.package.spec.to_package(), + rockspec.source().current_platform().unpack_dir.clone(), + rockspec.build().current_platform().copy_directories.clone(), + )), _ => None, } }) @@ -247,7 +246,9 @@ async fn gather_install_specs( let mut install_specs: Artifacts> = Artifacts::default(); for lock_type in LockedPackageLockType::iter() { - let specs = install_specs.get_mut(lock_type).get_or_insert_with(Vec::new); + let specs = install_specs + .get_mut(lock_type) + .get_or_insert_with(Vec::new); match target { VendorTarget::Workspace(workspace) => { for project in workspace.members() { @@ -260,7 +261,9 @@ async fn gather_install_specs( .test_dependencies(project) .iter() .cloned() - .map(|dep| PackageInstallSpec::new(dep, EntryType::Entrypoint).build()), + .map(|dep| { + PackageInstallSpec::new(dep, EntryType::Entrypoint).build() + }), ); } } @@ -513,7 +516,7 @@ mod tests { use super::*; use crate::{ config::ConfigBuilder, - lockfile::{LockedPackage, LockConstraint, LockedPackageSpec, OptState, PinnedState}, + lockfile::{LockConstraint, LockedPackage, LockedPackageSpec, OptState, PinnedState}, operations::unpack_rockspec, remote_package_source::RemotePackageSource, }; diff --git a/lux-lib/tests/build.rs b/lux-lib/tests/build.rs index bfb48806f..f3cd0ce52 100644 --- a/lux-lib/tests/build.rs +++ b/lux-lib/tests/build.rs @@ -1,7 +1,7 @@ use std::path::PathBuf; -use assert_fs::TempDir; use assert_fs::prelude::PathCopy; +use assert_fs::TempDir; use flaky_test::flaky_test; use lux_lib::lua_version::LuaVersion; use lux_lib::rockspec::Rockspec; @@ -10,7 +10,7 @@ use lux_lib::workspace::Workspace; use lux_lib::{ build::{Build, BuildBehaviour::Force}, config::ConfigBuilder, - lua_installation::{LuaInstallation, detect_installed_lua_version}, + lua_installation::{detect_installed_lua_version, LuaInstallation}, lua_rockspec::RemoteLuaRockspec, tree, }; diff --git a/lux-lib/tests/build_workspace.rs b/lux-lib/tests/build_workspace.rs index 2ac76ffa4..6fde00b82 100644 --- a/lux-lib/tests/build_workspace.rs +++ b/lux-lib/tests/build_workspace.rs @@ -1,7 +1,7 @@ use std::path::PathBuf; -use assert_fs::TempDir; use assert_fs::prelude::PathCopy; +use assert_fs::TempDir; use flaky_test::flaky_test; use lux_lib::lua_version::LuaVersion; use lux_lib::operations::BuildWorkspace; diff --git a/lux-lib/tests/vendor.rs b/lux-lib/tests/vendor.rs index 81ce161b1..cdbfc5647 100644 --- a/lux-lib/tests/vendor.rs +++ b/lux-lib/tests/vendor.rs @@ -73,7 +73,10 @@ async fn vendor_dependencies() { let busted_dir = vendor_dir.child("busted@2.3.0-1"); busted_dir.assert(predicate::path::is_dir()); assert!( - std::fs::read_dir(busted_dir.path()).unwrap().next().is_some(), + std::fs::read_dir(busted_dir.path()) + .unwrap() + .next() + .is_some(), "vendored source tree for busted is empty" ); } From b1713d1cd18e526a28f3ca8909078701bfbb69bf Mon Sep 17 00:00:00 2001 From: vhyrro Date: Sun, 4 Oct 2026 18:20:41 +0200 Subject: [PATCH 15/39] fix(sync): use `Sync` in `SyncWorkspace` instead of `InstallWorkspaceDependencies` --- lux-cli/src/sync.rs | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/lux-cli/src/sync.rs b/lux-cli/src/sync.rs index 490d738c5..9e439fb88 100644 --- a/lux-cli/src/sync.rs +++ b/lux-cli/src/sync.rs @@ -1,9 +1,5 @@ use clap::Args; -use lux_lib::{ - config::Config, - operations::{pipeline::install_workspace::InstallWorkspaceDependencies, GenLuaRc}, - workspace::Workspace, -}; +use lux_lib::{config::Config, operations::Sync, workspace::Workspace}; use miette::Result; @@ -16,22 +12,25 @@ pub struct SyncProject { /// Sync the current project's installed packages with its lux.toml. pub async fn sync(args: SyncProject, config: Config) -> Result<()> { - // FIXME(vhyrro): reimplement - let _ = args.no_integrity_check; let workspace = Workspace::current_or_err()?; - InstallWorkspaceDependencies::new(&config, &workspace) + let report = Sync::new(&workspace, &config) + .validate_integrity(!args.no_integrity_check) .test(true) - .install() + .sync() .await?; - GenLuaRc::new() - .config(&config) - .workspace(&workspace) - .generate_luarc() - .await?; + if report.added().is_empty() && report.removed().is_empty() { + println!("Already in sync."); + return Ok(()); + } - // FIXME(vhyrro): Readd report publishing + for pkg in report.added() { + println!("+ {} {}", pkg.name(), pkg.version()); + } + for pkg in report.removed() { + println!("- {} {}", pkg.name(), pkg.version()); + } Ok(()) } From 823c47dbe2ae386e32d346a905ad476c5c565f0f Mon Sep 17 00:00:00 2001 From: vhyrro Date: Sun, 4 Oct 2026 18:34:13 +0200 Subject: [PATCH 16/39] feat!: unify search and resolution into RemotePackageDB --- lux-lib/src/operations/pipeline/discover.rs | 77 +---- .../pipeline/download_sources_and_hash.rs | 7 +- .../operations/pipeline/install_packages.rs | 7 +- lux-lib/src/operations/pipeline/resolve.rs | 195 +++++------- lux-lib/src/operations/vendor.rs | 10 +- lux-lib/src/remote_package_db/mod.rs | 283 +++++++++++++----- 6 files changed, 297 insertions(+), 282 deletions(-) diff --git a/lux-lib/src/operations/pipeline/discover.rs b/lux-lib/src/operations/pipeline/discover.rs index c18838fd1..692b802cc 100644 --- a/lux-lib/src/operations/pipeline/discover.rs +++ b/lux-lib/src/operations/pipeline/discover.rs @@ -1,29 +1,22 @@ -use std::sync::Arc; - -use bon::Builder; use miette::Diagnostic; use thiserror::Error; use crate::{ - config::Config, lua_rockspec::RemoteLuaRockspec, - operations::{self, Download, FetchVendored, FetchVendoredError, RemoteRockDownload}, + operations, package::{PackageReq, RemotePackage}, - remote_package_db::{RemotePackageDB, SearchError}, + remote_package_db::SearchError, }; use super::download_sources_and_hash::PackageSource; +#[derive(Clone, Debug)] pub(crate) struct FoundPackage { pub(crate) package: RemotePackage, pub(crate) rockspec: RemoteLuaRockspec, pub(crate) artifact: Option, } -pub(crate) enum FoundPackageType { - Package(FoundPackage), -} - #[derive(Error, Debug, Diagnostic)] #[non_exhaustive] pub enum DiscoverError { @@ -35,67 +28,5 @@ pub enum DiscoverError { Download(PackageReq, Box), #[error("failed to fetch vendored package for '{0}'")] #[diagnostic(forward(1))] - FetchVendored(PackageReq, Box), -} - -pub(crate) trait FindPackageFromProvider: Clone + Send + Sync + 'static { - fn find( - &self, - req: &PackageReq, - ) -> impl std::future::Future> + Send; -} - -#[derive(Builder, Clone)] -#[builder(start_fn = new)] -pub(crate) struct FindPackageFromLuarocks { - #[builder(start_fn)] - pub(crate) package_db: Arc, - #[builder(start_fn)] - pub(crate) config: Arc, -} - -impl FindPackageFromProvider for FindPackageFromLuarocks { - async fn find(&self, req: &PackageReq) -> Result { - let mut package = self.package_db.find(req, None)?; - // TODO(vhyrro): Don't force us to call download here, extrapolate stuff from - // `operations` into this pipeline instead. - let download = if let Some(vendor_dir) = self.config.vendor_dir() { - FetchVendored::new() - .vendor_dir(vendor_dir) - .package(req) - .package_db(&self.package_db) - .fetch_vendored_rock() - .await - .map_err(|err| DiscoverError::FetchVendored(req.clone(), Box::new(err)))? - } else { - Download::new(req, &self.config) - .package_db(&self.package_db) - .download_remote_rock() - .await - .map_err(|err| DiscoverError::Download(req.clone(), Box::new(err)))? - }; - - let rockspec = download.rockspec().clone(); - - let artifact = match &download { - RemoteRockDownload::SrcRock { - src_rock, - source_url, - .. - } => { - package.source_url = Some(source_url.clone()); - Some(PackageSource::PackedRock(src_rock.clone())) - } - RemoteRockDownload::BinaryRock { packed_rock, .. } => { - Some(PackageSource::PackedRock(packed_rock.clone())) - } - RemoteRockDownload::RockspecOnly { .. } => None, - }; - - Ok(FoundPackageType::Package(FoundPackage { - package, - rockspec, - artifact, - })) - } + FetchVendored(PackageReq, Box), } diff --git a/lux-lib/src/operations/pipeline/download_sources_and_hash.rs b/lux-lib/src/operations/pipeline/download_sources_and_hash.rs index 641136227..1f1fcc000 100644 --- a/lux-lib/src/operations/pipeline/download_sources_and_hash.rs +++ b/lux-lib/src/operations/pipeline/download_sources_and_hash.rs @@ -30,8 +30,9 @@ use super::{ Artifacts, }; +#[derive(Clone, Debug)] pub(crate) enum PackageSource { - SourceTree(TempDir), + SourceTree(std::sync::Arc), PackedRock(Bytes), } @@ -177,7 +178,7 @@ async fn download_sources_and_hash( config, ) .await?; - (package, PackageSource::SourceTree(dir)) + (package, PackageSource::SourceTree(std::sync::Arc::new(dir))) } }, Some(PackageSource::SourceTree(dir)) => { @@ -206,7 +207,7 @@ async fn download_sources_and_hash( config, ) .await?; - (package, PackageSource::SourceTree(dir)) + (package, PackageSource::SourceTree(std::sync::Arc::new(dir))) } }; diff --git a/lux-lib/src/operations/pipeline/install_packages.rs b/lux-lib/src/operations/pipeline/install_packages.rs index fe6a273f6..979fb1ef2 100644 --- a/lux-lib/src/operations/pipeline/install_packages.rs +++ b/lux-lib/src/operations/pipeline/install_packages.rs @@ -1,4 +1,4 @@ -use std::{collections::HashMap, io, sync::Arc}; +use std::{collections::HashMap, io}; use bon::Builder; use itertools::Itertools; @@ -19,7 +19,6 @@ use crate::{ use super::{ build::{Build as PipelineBuild, BuildError as PipelineBuildError}, - discover::FindPackageFromLuarocks, download_sources_and_hash::{DownloadSourcesAndHash, DownloadSourcesAndHashError}, emit_lockfile::LockfileHandle, resolve::{ResolveError, ResolvePackageDependencies}, @@ -155,9 +154,7 @@ where tree.cleanup(package, tree::EntryType::Entrypoint)?; } - let discover = - FindPackageFromLuarocks::new(Arc::new(package_db), Arc::new(config.clone())).build(); - let mut resolve = ResolvePackageDependencies::new(&discover, config) + let mut resolve = ResolvePackageDependencies::new(package_db, config) .packages(packages) .build_packages(install.build_packages); if let Some(test_packages) = install.test_packages { diff --git a/lux-lib/src/operations/pipeline/resolve.rs b/lux-lib/src/operations/pipeline/resolve.rs index fdb5bfeca..afc88a9e1 100644 --- a/lux-lib/src/operations/pipeline/resolve.rs +++ b/lux-lib/src/operations/pipeline/resolve.rs @@ -1,4 +1,5 @@ use std::collections::{HashMap, HashSet, VecDeque}; +use std::sync::Arc; use bon::Builder; use itertools::Itertools; @@ -13,15 +14,16 @@ use crate::{ RemotePackageSourceUrl, }, lua_rockspec::{BuildBackendSpec, RemoteLuaRockspec}, - operations::{PackageInstallSpec, RemoteRockDownload}, - package::{PackageName, PackageReq, RemotePackage}, + operations::PackageInstallSpec, + package::{PackageName, PackageReq}, + remote_package_db::RemotePackageDB, remote_package_source::RemotePackageSource, rockspec::Rockspec, tree::EntryType, }; use super::{ - discover::{DiscoverError, FindPackageFromProvider, FoundPackage, FoundPackageType}, + discover::{DiscoverError, FoundPackage}, download_sources_and_hash::PackageSource, Artifacts, }; @@ -85,18 +87,17 @@ pub enum ResolveError { #[derive(Builder)] #[builder(start_fn = new, finish_fn(name = _build, vis = ""))] -pub(crate) struct ResolvePackageDependencies<'a, D: FindPackageFromProvider> { +pub(crate) struct ResolvePackageDependencies<'a> { #[builder(start_fn)] - pub(crate) discover: &'a D, + pub(crate) package_db: RemotePackageDB, #[builder(start_fn)] pub(crate) config: &'a Config, #[builder(field)] pub(crate) packages: Artifacts>, } -impl ResolvePackageDependenciesBuilder<'_, D, State> +impl ResolvePackageDependenciesBuilder<'_, State> where - D: FindPackageFromProvider, State: resolve_package_dependencies_builder::State, { fn set(mut self, section: LockedPackageLockType, packages: Vec) -> Self { @@ -117,15 +118,14 @@ where } } -impl ResolvePackageDependenciesBuilder<'_, D, State> +impl ResolvePackageDependenciesBuilder<'_, State> where - D: FindPackageFromProvider, State: resolve_package_dependencies_builder::State + resolve_package_dependencies_builder::IsComplete, { pub(crate) async fn resolve(self) -> Result { let args = self._build(); - Resolver::new(args.discover, args.config) + Resolver::new(args.package_db, args.config) .run(args.packages) .await } @@ -164,8 +164,9 @@ struct Node { /// Owns the whole resolution graph while it is being discovered. Discovery futures run on /// the runtime (bounded by `Config::max_jobs`), but all shared state stays here, owned by a /// single task — no `Rc`, `RefCell`, or locks. -struct Resolver<'a, D: FindPackageFromProvider> { - discover: &'a D, +struct Resolver { + package_db: RemotePackageDB, + config: Arc, max_inflight: usize, joinset: JoinSet<( RequestKey, @@ -178,16 +179,17 @@ struct Resolver<'a, D: FindPackageFromProvider> { resolved: ResolvedArtifacts, } -impl<'a, D: FindPackageFromProvider> Resolver<'a, D> { - fn new(discover: &'a D, config: &Config) -> Self { - Self::with_max_inflight(discover, config.max_jobs()) +impl Resolver { + fn new(package_db: RemotePackageDB, config: &Config) -> Self { + Self::with_max_inflight(package_db, Arc::new(config.clone()), config.max_jobs()) } /// Builds a resolver with an explicit in-flight bound. Used by tests, which do not /// construct a full [`Config`]. - fn with_max_inflight(discover: &'a D, max_inflight: usize) -> Self { + fn with_max_inflight(package_db: RemotePackageDB, config: Arc, max_inflight: usize) -> Self { Self { - discover, + package_db, + config, max_inflight, joinset: JoinSet::new(), pending: VecDeque::new(), @@ -240,10 +242,14 @@ impl<'a, D: FindPackageFromProvider> Resolver<'a, D> { let Some((key, spec)) = self.pending.pop_front() else { break; }; - let discover = D::clone(self.discover); + let package_db = self.package_db.clone(); + let config = Arc::clone(&self.config); let task_spec = spec.clone(); self.joinset.spawn(async move { - let result = find_package_from_provider(task_spec, discover).await; + let result = package_db + .resolve(&task_spec, &config) + .await + .map_err(ResolveError::Discover); (key, spec, result) }); } @@ -453,37 +459,6 @@ fn detect_cycles(nodes: &HashMap) -> Result<(), ResolveError> Ok(()) } -async fn find_package_from_provider( - spec: PackageInstallSpec, - discover: D, -) -> Result { - if let Some(source) = &spec.source { - let download = RemoteRockDownload::from_package_req_and_source_spec( - spec.package.clone(), - source.clone(), - ) - .map_err(|err| DiscoverError::Download(spec.package.clone(), Box::new(err)))?; - let rockspec = download.rockspec().clone(); - let package = RemotePackage::new( - spec.package.clone().try_into().map_err(|err| { - DiscoverError::Download( - spec.package.clone(), - Box::new(crate::operations::SearchAndDownloadError::from(err)), - ) - })?, - download.rockspec_download().source.clone(), - download.rockspec_download().source_url.clone(), - ); - return Ok(FoundPackage { - package, - rockspec, - artifact: None, - }); - } - - let FoundPackageType::Package(package) = discover.find(&spec.package).await?; - Ok(package) -} fn build_dependency_specs( rockspec: &RemoteLuaRockspec, @@ -520,74 +495,11 @@ fn build_dependency_specs( #[cfg(test)] mod tests { - use std::collections::HashMap; - use std::future::Future; - use std::sync::{Arc, Mutex}; + use std::sync::Arc; use super::*; - use crate::package::PackageSpec; - use crate::remote_package_db::SearchError; - - #[derive(Clone)] - struct MockDiscover { - rockspecs: HashMap, - calls: Arc>>, - } - - impl MockDiscover { - fn new(rockspecs: &[String]) -> Self { - let mut map = HashMap::new(); - for content in rockspecs { - let rockspec = match RemoteLuaRockspec::new(content) { - Ok(rockspec) => rockspec, - Err(err) => panic!("invalid test rockspec: {err}"), - }; - map.insert(rockspec.package().to_string(), (content.clone(), rockspec)); - } - Self { - rockspecs: map, - calls: Arc::new(Mutex::new(HashMap::new())), - } - } - - fn calls(&self, name: &str) -> usize { - let calls = self.calls.lock().unwrap_or_else(|err| err.into_inner()); - calls.get(name).copied().unwrap_or(0) - } - } - - impl FindPackageFromProvider for MockDiscover { - fn find( - &self, - req: &PackageReq, - ) -> impl Future> + Send { - let name = req.name().to_string(); - { - let mut calls = self.calls.lock().unwrap_or_else(|err| err.into_inner()); - *calls.entry(name.clone()).or_insert(0) += 1; - } - let result = self.rockspecs.get(&name).map_or_else( - || { - Err(DiscoverError::Search(SearchError::RockNotFound( - req.clone(), - ))) - }, - |(content, rockspec)| { - let package = RemotePackage::new( - PackageSpec::new(req.name().clone(), rockspec.version().clone()), - RemotePackageSource::RockspecContent(content.clone()), - None, - ); - Ok(FoundPackageType::Package(FoundPackage { - package, - rockspec: rockspec.clone(), - artifact: None, - })) - }, - ); - async move { result } - } - } + use crate::config::ConfigBuilder; + use crate::package::{PackageSpec, RemotePackage}; fn rockspec(package: &str, dependencies: &[&str]) -> String { let dependencies = if dependencies.is_empty() { @@ -605,6 +517,32 @@ mod tests { ) } + fn local_packages(rockspecs: &[String]) -> RemotePackageDB { + RemotePackageDB::from_local( + rockspecs + .iter() + .map(|content| { + let rockspec = match RemoteLuaRockspec::new(content) { + Ok(rockspec) => rockspec, + Err(err) => panic!("invalid test rockspec: {err}"), + }; + FoundPackage { + package: RemotePackage::new( + PackageSpec::new( + rockspec.package().clone(), + rockspec.version().clone(), + ), + RemotePackageSource::RockspecContent(content.clone()), + None, + ), + rockspec, + artifact: None, + } + }) + .collect(), + ) + } + fn install_spec(name: &str) -> PackageInstallSpec { let package = match name.parse::() { Ok(package) => package, @@ -621,16 +559,28 @@ mod tests { } } + fn test_config() -> Arc { + let builder = match ConfigBuilder::new() { + Ok(builder) => builder, + Err(err) => panic!("invalid test config: {err}"), + }; + let config = match builder.build() { + Ok(config) => config, + Err(err) => panic!("invalid test config: {err}"), + }; + Arc::new(config) + } + #[tokio::test] - async fn diamond_is_discovered_once_per_package() { - let discover = MockDiscover::new(&[ + async fn diamond_is_resolved_once() { + let packages = local_packages(&[ rockspec("a", &["b", "c"]), rockspec("b", &["d"]), rockspec("c", &["d"]), rockspec("d", &[]), ]); - let resolved = match Resolver::with_max_inflight(&discover, 8) + let resolved = match Resolver::with_max_inflight(packages, test_config(), 8) .run(roots(install_spec("a"))) .await { @@ -639,16 +589,13 @@ mod tests { }; assert_eq!(resolved.regular.as_ref().map(|rocks| rocks.len()), Some(4)); - for name in ["a", "b", "c", "d"] { - assert_eq!(discover.calls(name), 1, "{name} should be discovered once"); - } } #[tokio::test] async fn cyclic_dependencies_are_rejected() { - let discover = MockDiscover::new(&[rockspec("a", &["b"]), rockspec("b", &["a"])]); + let packages = local_packages(&[rockspec("a", &["b"]), rockspec("b", &["a"])]); - let result = Resolver::with_max_inflight(&discover, 8) + let result = Resolver::with_max_inflight(packages, test_config(), 8) .run(roots(install_spec("a"))) .await; diff --git a/lux-lib/src/operations/vendor.rs b/lux-lib/src/operations/vendor.rs index a49a21a34..6512744d9 100644 --- a/lux-lib/src/operations/vendor.rs +++ b/lux-lib/src/operations/vendor.rs @@ -25,7 +25,6 @@ use crate::{ operations::{ self, pipeline::{ - discover::FindPackageFromLuarocks, download_sources_and_hash::{ DownloadSourcesAndHash, DownloadSourcesAndHashError, DownloadedPackage, PackageSource, @@ -130,16 +129,13 @@ async fn resolve_and_download( package_db: RemotePackageDB, install_specs: Artifacts>, ) -> Result, VendorError> { - let discover = - FindPackageFromLuarocks::new(Arc::new(package_db), Arc::new(config.clone())).build(); - let Artifacts { regular, build, test, } = install_specs; let mut resolve = - ResolvePackageDependencies::new(&discover, config).packages(regular.unwrap_or_default()); + ResolvePackageDependencies::new(package_db, config).packages(regular.unwrap_or_default()); if let Some(build) = build { resolve = resolve.build_packages(build); } @@ -589,7 +585,7 @@ mod tests { let package = make_downloaded_package( rockspec, RemotePackageSource::RockspecContent(String::new()), - PackageSource::SourceTree(src_dir), + PackageSource::SourceTree(std::sync::Arc::new(src_dir)), ); vendor_package_sources(vendor_dir.path(), &package) @@ -668,7 +664,7 @@ mod tests { let package = make_downloaded_package( rockspec, RemotePackageSource::RockspecContent(String::new()), - PackageSource::SourceTree(src_dir), + PackageSource::SourceTree(std::sync::Arc::new(src_dir)), ); vendor_package_sources(vendor_dir.path(), &package) diff --git a/lux-lib/src/remote_package_db/mod.rs b/lux-lib/src/remote_package_db/mod.rs index d88ce4025..63907d0e2 100644 --- a/lux-lib/src/remote_package_db/mod.rs +++ b/lux-lib/src/remote_package_db/mod.rs @@ -2,6 +2,13 @@ use crate::{ config::{Config, ConfigError}, lockfile::{LockedPackageLock, LockfileIntegrityError}, manifest::{Manifest, ManifestError}, + operations::{ + Download, FetchVendored, PackageInstallSpec, RemoteRockDownload, + pipeline::{ + discover::{DiscoverError, FoundPackage}, + download_sources_and_hash::PackageSource, + }, + }, package::{ PackageName, PackageReq, PackageSpec, PackageVersion, RemotePackage, RemotePackageTypeFilterSpec, @@ -14,12 +21,13 @@ use thiserror::Error; /// Package database, used to look up remote rocks #[derive(Clone, Debug)] -pub struct RemotePackageDB(Impl); +pub struct RemotePackageDB(Vec); #[derive(Clone, Debug)] -enum Impl { +pub(crate) enum RemoteSource { LuarocksManifests(Vec), LockedPackageLocks(Vec), + Local(Vec), } #[derive(Error, Debug, Diagnostic)] @@ -65,7 +73,44 @@ impl RemotePackageDB { manifests.push(manifest); } manifests.push(Manifest::from_config(config.server().clone(), config).await?); - Ok(Self(Impl::LuarocksManifests(manifests))) + Ok(Self(vec![RemoteSource::LuarocksManifests(manifests)])) + } + + /// Builds a package database backed solely by local packages. + pub(crate) fn from_local(packages: Vec) -> Self { + Self(vec![RemoteSource::Local(packages)]) + } + + /// Prepends local packages to the lookup order. + pub(crate) fn add_local(&mut self, packages: Vec) { + self.0.insert(0, RemoteSource::Local(packages)); + } + + fn find_in_source( + source: &RemoteSource, + package_req: &PackageReq, + filter: Option, + ) -> Result, SearchError> { + match source { + RemoteSource::LuarocksManifests(manifests) => Ok(manifests + .iter() + .find_map(|manifest| manifest.find(package_req, filter.clone()))), + RemoteSource::LockedPackageLocks(locks) => Ok(locks + .iter() + .filter_map(|lock| lock.has_rock(package_req, filter.clone())) + .map(|local_package| { + RemotePackage::new( + local_package.to_package(), + local_package.source().clone(), + local_package.source_url().cloned(), + ) + }) + .next()), + RemoteSource::Local(packages) => Ok(packages + .iter() + .find(|package| package.package.package.name() == package_req.name()) + .map(|package| package.package.clone())), + } } /// Find a remote package that matches the requirement, returning the latest match. @@ -80,79 +125,177 @@ impl RemotePackageDB { package_req: &PackageReq, filter: Option, ) -> Result { - match &self.0 { - Impl::LuarocksManifests(manifests) => { - match manifests - .iter() - .find_map(|manifest| manifest.find(package_req, filter.clone())) - { - Some(package) => Ok(package), - None => Err(SearchError::RockNotFound(package_req.clone())), - } + for source in &self.0 { + if let Some(package) = Self::find_in_source(source, package_req, filter.clone())? { + return Ok(package); } - Impl::LockedPackageLocks(locks) => { - match locks - .iter() - .filter_map(|lock| lock.has_rock(package_req, filter.clone())) - .map(|local_package| { - RemotePackage::new( - local_package.to_package(), - local_package.source().clone(), - local_package.source_url().cloned(), - ) - }) - .next() - { - Some(package) => Ok(package), - None => Err(SearchError::RockNotFoundInLockfile(package_req.clone())), + } + if self + .0 + .iter() + .any(|source| matches!(source, RemoteSource::LuarocksManifests(_))) + { + Err(SearchError::RockNotFound(package_req.clone())) + } else if self + .0 + .iter() + .any(|source| matches!(source, RemoteSource::LockedPackageLocks(_))) + { + Err(SearchError::RockNotFoundInLockfile(package_req.clone())) + } else { + Err(SearchError::RockNotFound(package_req.clone())) + } + } + + /// Resolves a package install spec into a fully discovered package. + pub(crate) async fn resolve( + &self, + spec: &PackageInstallSpec, + config: &Config, + ) -> Result { + if let Some(source) = &spec.source { + let download = + RemoteRockDownload::from_package_req_and_source_spec(spec.package.clone(), source.clone()) + .map_err(|err| DiscoverError::Download(spec.package.clone(), Box::new(err)))?; + let rockspec = download.rockspec().clone(); + let package = RemotePackage::new( + spec.package.clone().try_into().map_err(|err| { + DiscoverError::Download( + spec.package.clone(), + Box::new(crate::operations::SearchAndDownloadError::from(err)), + ) + })?, + download.rockspec_download().source.clone(), + download.rockspec_download().source_url.clone(), + ); + return Ok(FoundPackage { + package, + rockspec, + artifact: None, + }); + } + + for source in &self.0 { + match source { + RemoteSource::Local(packages) => { + if let Some(package) = packages + .iter() + .find(|package| package.package.package.name() == spec.package.name()) + { + return Ok(package.clone()); + } + } + RemoteSource::LuarocksManifests(_) | RemoteSource::LockedPackageLocks(_) => { + let mut package = self.find(&spec.package, None)?; + let download = if let Some(vendor_dir) = config.vendor_dir() { + FetchVendored::new() + .vendor_dir(vendor_dir) + .package(&spec.package) + .package_db(self) + .fetch_vendored_rock() + .await + .map_err(|err| { + DiscoverError::FetchVendored(spec.package.clone(), Box::new(err)) + })? + } else { + Download::new(&spec.package, config) + .package_db(self) + .download_remote_rock() + .await + .map_err(|err| { + DiscoverError::Download(spec.package.clone(), Box::new(err)) + })? + }; + let rockspec = download.rockspec().clone(); + let artifact = match &download { + RemoteRockDownload::SrcRock { + src_rock, + source_url, + .. + } => { + package.source_url = Some(source_url.clone()); + Some(PackageSource::PackedRock(src_rock.clone())) + } + RemoteRockDownload::BinaryRock { packed_rock, .. } => { + Some(PackageSource::PackedRock(packed_rock.clone())) + } + RemoteRockDownload::RockspecOnly { .. } => None, + }; + return Ok(FoundPackage { + package, + rockspec, + artifact, + }); } } } + + Err(DiscoverError::Search(SearchError::RockNotFound( + spec.package.clone(), + ))) } /// Search for all packages that match the requirement. pub fn search(&self, package_req: &PackageReq) -> Vec<(&PackageName, Vec<&PackageVersion>)> { - match &self.0 { - Impl::LuarocksManifests(manifests) => manifests - .iter() - .flat_map(|manifest| { - manifest - .metadata() - .repository - .iter() - .filter_map(|(name, elements)| { - if name.to_string().contains(&package_req.name().to_string()) { - Some(( - name, - elements - .keys() - .filter(|version| { - package_req.version_req().matches(version) - }) - .sorted_by(|a, b| Ord::cmp(b, a)) - .collect_vec(), - )) - } else { - None - } - }) - }) - .collect(), - Impl::LockedPackageLocks(locks) => locks - .iter() - .flat_map(|lock| lock.rocks().values()) - .filter_map(|package| { - // NOTE: This doesn't group packages by name, but we don't care for now, - // as we shouldn't need to use this function with a lockfile. - let name = package.name(); - if name.to_string().contains(&package_req.name().to_string()) { - Some((name, vec![package.version()])) - } else { - None - } - }) - .collect_vec(), - } + self.0 + .iter() + .flat_map(|source| match source { + RemoteSource::LuarocksManifests(manifests) => manifests + .iter() + .flat_map(|manifest| { + manifest + .metadata() + .repository + .iter() + .filter_map(|(name, elements)| { + if name.to_string().contains(&package_req.name().to_string()) { + Some(( + name, + elements + .keys() + .filter(|version| { + package_req.version_req().matches(version) + }) + .sorted_by(|a, b| Ord::cmp(b, a)) + .collect_vec(), + )) + } else { + None + } + }) + }) + .collect_vec(), + RemoteSource::LockedPackageLocks(locks) => locks + .iter() + .flat_map(|lock| lock.rocks().values()) + .filter_map(|package| { + // NOTE: This doesn't group packages by name, but we don't care for now, + // as we shouldn't need to use this function with a lockfile. + let name = package.name(); + if name.to_string().contains(&package_req.name().to_string()) { + Some((name, vec![package.version()])) + } else { + None + } + }) + .collect_vec(), + RemoteSource::Local(packages) => packages + .iter() + .filter_map(|package| { + let name = package.package.package.name(); + if name.to_string().contains(&package_req.name().to_string()) + && package_req + .version_req() + .matches(package.package.package.version()) + { + Some((name, vec![package.package.package.version()])) + } else { + None + } + }) + .collect_vec(), + }) + .collect() } /// Find the latest version for a package by name. @@ -176,12 +319,12 @@ impl RemotePackageDB { impl From for RemotePackageDB { fn from(manifest: Manifest) -> Self { - Self(Impl::LuarocksManifests(vec![manifest])) + Self(vec![RemoteSource::LuarocksManifests(vec![manifest])]) } } impl From> for RemotePackageDB { fn from(locks: Vec) -> Self { - Self(Impl::LockedPackageLocks(locks)) + Self(vec![RemoteSource::LockedPackageLocks(locks)]) } } From 4f60b76ef85a7b51d424d6d59d15b8872a189edb Mon Sep 17 00:00:00 2001 From: vhyrro Date: Sun, 4 Oct 2026 19:17:13 +0200 Subject: [PATCH 17/39] feat!: unify install_rockspec logic with new pipeline --- lux-lib/src/operations/install/mod.rs | 3 + lux-lib/src/operations/install_rockspec.rs | 119 +++++------------- lux-lib/src/operations/pipeline/discover.rs | 48 ++++++- .../operations/pipeline/install_packages.rs | 4 + lux-lib/src/operations/pipeline/resolve.rs | 5 +- lux-lib/src/remote_package_db/mod.rs | 10 +- 6 files changed, 88 insertions(+), 101 deletions(-) diff --git a/lux-lib/src/operations/install/mod.rs b/lux-lib/src/operations/install/mod.rs index 05f0f74a2..7206e765c 100644 --- a/lux-lib/src/operations/install/mod.rs +++ b/lux-lib/src/operations/install/mod.rs @@ -1,4 +1,5 @@ use crate::{ + build::BuildBehaviour, config::Config, lockfile::LockedPackage, remote_package_db::RemotePackageDB, @@ -32,6 +33,7 @@ where #[builder(setters(name = "_tree", vis = ""))] tree: T, package_db: Option, + behaviour: Option, } impl<'a, State> InstallBuilder<'a, Tree, State> @@ -146,6 +148,7 @@ where let (packages, _lockfile) = InstallPackages::new(install.config, install.tree) .packages(packages) .maybe_package_db(install.package_db) + .maybe_behaviour(install.behaviour) .install() .await .map_err(InstallError::from)?; diff --git a/lux-lib/src/operations/install_rockspec.rs b/lux-lib/src/operations/install_rockspec.rs index dc1f6fd0f..d791ec550 100644 --- a/lux-lib/src/operations/install_rockspec.rs +++ b/lux-lib/src/operations/install_rockspec.rs @@ -3,13 +3,15 @@ use miette::Diagnostic; use thiserror::Error; use crate::{ - build::{self, BuildBehaviour, BuildError}, + build::{BuildBehaviour, BuildError}, config::Config, lockfile::{LockedPackage, OptState, PinnedState}, - lua_installation::{LuaInstallation, LuaInstallationError}, - lua_rockspec::{BuildBackendSpec, LuaVersionError, RemoteLuaRockspec}, - luarocks::luarocks_installation::{LuaRocksError, LuaRocksInstallError, LuaRocksInstallation}, - operations::{Install, InstallError, PackageInstallSpec}, + lua_installation::LuaInstallationError, + lua_rockspec::{LuaVersionError, RemoteLuaRockspec}, + luarocks::luarocks_installation::{LuaRocksError, LuaRocksInstallError}, + operations::{Install, InstallError, PackageInstallSpec, pipeline::discover::FoundPackage}, + package::{PackageName, PackageReq}, + remote_package_db::{RemotePackageDB, RemotePackageDBError}, rockspec::{LuaVersionCompatibility, Rockspec}, tree::{self, InstallTree, TreeError}, }; @@ -23,6 +25,8 @@ pub enum InstallRockspecError { #[diagnostic(transparent)] LuaVersion(#[from] LuaVersionError), #[diagnostic(transparent)] + RemotePackageDB(#[from] RemotePackageDBError), + #[diagnostic(transparent)] Tree(#[from] TreeError), #[diagnostic(transparent)] Install(#[from] Box), @@ -32,6 +36,8 @@ pub enum InstallRockspecError { LuaRocksInstall(#[from] Box), #[diagnostic(transparent)] Build(#[from] Box), + #[error("package '{0}' was not installed")] + PackageNotInstalled(PackageName), } impl From for InstallRockspecError { @@ -80,99 +86,30 @@ impl< let config = args.config; let tree = args.tree; - let lua_version = rockspec.lua_version_matches(config)?; - let lua = LuaInstallation::new(&lua_version, config).await?; - - // Ensure all dependencies and build dependencies are installed first - - let build_dependencies = rockspec.build_dependencies().current_platform(); - - let build_dependencies_to_install = build_dependencies - .iter() - .filter(|dep| { - // Exclude luarocks build backends that we have implemented in lux - !matches!( - dep.name().to_string().as_str(), - "luarocks-build-rust-mlua" - | "luarocks-build-rust-binary" - | "luarocks-build-treesitter-parser" - ) - }) - .filter(|dep| { - tree.match_rocks(dep.package_req()) - .is_ok_and(|rock_match| rock_match.is_found()) - }) - .map(|dep| { - PackageInstallSpec::new(dep.package_req().clone(), tree::EntryType::Entrypoint) - .build_behaviour(BuildBehaviour::NoForce) - .pin(pin) - .opt(OptState::Required) - .maybe_source(dep.source().clone()) - .build() - }) - .collect(); - - Install::new(config) - .packages(build_dependencies_to_install) - .tree(tree.build_tree(config)?) - .install() - .await?; + rockspec.lua_version_matches(config)?; - let dependencies = rockspec.dependencies().current_platform(); + let name = rockspec.package().clone(); + let root = FoundPackage::from_rockspec(rockspec); + let package_db = RemotePackageDB::from_config(config) + .await? + .with_local(vec![root]); - let mut dependencies_to_install = Vec::new(); - for dep in dependencies { - let rock_match = tree.match_rocks(dep.package_req())?; - if !rock_match.is_found() { - let dep = PackageInstallSpec::new( - dep.package_req().clone(), - tree::EntryType::DependencyOnly, - ) - .build_behaviour(BuildBehaviour::NoForce) + let install_spec = + PackageInstallSpec::new(PackageReq::from(name.clone()), tree::EntryType::Entrypoint) + .build_behaviour(BuildBehaviour::Force) .pin(pin) .opt(OptState::Required) - .maybe_source(dep.source().clone()) .build(); - dependencies_to_install.push(dep); - } - } - let dependencies = Install::new(config) - .packages(dependencies_to_install) + Install::new(config) + .package_db(package_db) + .package(install_spec) + .behaviour(BuildBehaviour::Force) .tree(tree.clone()) .install() - .await?; - - if let Some(BuildBackendSpec::LuaRock(_)) = - &rockspec.build().current_platform().build_backend - { - let build_tree = tree.build_tree(config)?; - let luarocks = LuaRocksInstallation::new(config, build_tree)?; - luarocks.ensure_installed(&lua).await?; - } - - let package = build::Build::new() - .rockspec(&rockspec) - .tree(tree) - .lua(&lua) - .entry_type(tree::EntryType::Entrypoint) - .config(config) - .pin(pin) - .behaviour(BuildBehaviour::Force) - .build() - .await?; - - let lockfile = tree.lockfile()?; - let build_lockfile = tree.build_tree(config)?.lockfile()?; - - let mut lockfile = lockfile.write_guard(); - lockfile.add_entrypoint(&package); - for dep in dependencies { - lockfile.add_dependency(&package, &dep); - } - for dep in build_lockfile.rocks().values() { - lockfile.add_build_dependency(&package, dep); - } - Ok(package) + .await? + .into_iter() + .find(|package| package.name() == &name) + .ok_or(InstallRockspecError::PackageNotInstalled(name)) } } diff --git a/lux-lib/src/operations/pipeline/discover.rs b/lux-lib/src/operations/pipeline/discover.rs index 692b802cc..30d2ccfe3 100644 --- a/lux-lib/src/operations/pipeline/discover.rs +++ b/lux-lib/src/operations/pipeline/discover.rs @@ -4,8 +4,10 @@ use thiserror::Error; use crate::{ lua_rockspec::RemoteLuaRockspec, operations, - package::{PackageReq, RemotePackage}, + package::{PackageReq, PackageSpec, RemotePackage}, + remote_package_source::RemotePackageSource, remote_package_db::SearchError, + rockspec::Rockspec, }; use super::download_sources_and_hash::PackageSource; @@ -17,6 +19,27 @@ pub(crate) struct FoundPackage { pub(crate) artifact: Option, } +impl FoundPackage { + /// Builds a discovered package from an already-parsed rockspec, so it can + /// be resolved as a root without contacting a remote package database. + pub(crate) fn from_rockspec(rockspec: RemoteLuaRockspec) -> Self { + let source = rockspec + .to_lua_remote_rockspec_string() + .map(RemotePackageSource::RockspecContent) + .unwrap_or(RemotePackageSource::Local); + let package = RemotePackage::new( + PackageSpec::new(rockspec.package().clone(), rockspec.version().clone()), + source, + None, + ); + Self { + package, + rockspec, + artifact: None, + } + } +} + #[derive(Error, Debug, Diagnostic)] #[non_exhaustive] pub enum DiscoverError { @@ -30,3 +53,26 @@ pub enum DiscoverError { #[diagnostic(forward(1))] FetchVendored(PackageReq, Box), } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn from_rockspec_builds_local_found_package() { + let content = "rockspec_format = '1.0'\npackage = 'foo'\nversion = '1.0.0-1'\nsource = { url = 'https://example.com/foo.zip' }"; + let Ok(rockspec) = RemoteLuaRockspec::new(content) else { + panic!("invalid test rockspec"); + }; + + let found = FoundPackage::from_rockspec(rockspec); + + assert_eq!(found.package.package.name().to_string(), "foo"); + assert_eq!(found.package.package.version().to_string(), "1.0.0-1"); + assert!(matches!( + found.package.source, + RemotePackageSource::RockspecContent(_) + )); + assert!(found.artifact.is_none()); + } +} diff --git a/lux-lib/src/operations/pipeline/install_packages.rs b/lux-lib/src/operations/pipeline/install_packages.rs index 979fb1ef2..0de7154ac 100644 --- a/lux-lib/src/operations/pipeline/install_packages.rs +++ b/lux-lib/src/operations/pipeline/install_packages.rs @@ -49,6 +49,7 @@ where test_packages: Option>, packages: Vec, package_db: Option, + behaviour: Option, } impl<'a, T, State> InstallPackagesBuilder<'a, T, State> @@ -185,16 +186,19 @@ where // Build dependencies first, then the packages themselves. let built_build_deps = PipelineBuild::new(config, &build_tree) .packages(build_packages) + .maybe_behaviour(install.behaviour) .build() .await?; let built = PipelineBuild::new(config, tree) .packages(regular_packages) + .maybe_behaviour(install.behaviour) .build() .await?; if let Some(test_packages) = artifacts.test { let test_tree = tree.test_tree(config)?; PipelineBuild::new(config, &test_tree) .packages(test_packages.into_values().collect_vec()) + .maybe_behaviour(install.behaviour) .build() .await?; } diff --git a/lux-lib/src/operations/pipeline/resolve.rs b/lux-lib/src/operations/pipeline/resolve.rs index afc88a9e1..ce2e8b0ab 100644 --- a/lux-lib/src/operations/pipeline/resolve.rs +++ b/lux-lib/src/operations/pipeline/resolve.rs @@ -500,6 +500,7 @@ mod tests { use super::*; use crate::config::ConfigBuilder; use crate::package::{PackageSpec, RemotePackage}; + use crate::remote_package_db::RemoteSource; fn rockspec(package: &str, dependencies: &[&str]) -> String { let dependencies = if dependencies.is_empty() { @@ -518,7 +519,7 @@ mod tests { } fn local_packages(rockspecs: &[String]) -> RemotePackageDB { - RemotePackageDB::from_local( + RemotePackageDB(vec![RemoteSource::Local( rockspecs .iter() .map(|content| { @@ -540,7 +541,7 @@ mod tests { } }) .collect(), - ) + )]) } fn install_spec(name: &str) -> PackageInstallSpec { diff --git a/lux-lib/src/remote_package_db/mod.rs b/lux-lib/src/remote_package_db/mod.rs index 63907d0e2..d18680e42 100644 --- a/lux-lib/src/remote_package_db/mod.rs +++ b/lux-lib/src/remote_package_db/mod.rs @@ -21,7 +21,7 @@ use thiserror::Error; /// Package database, used to look up remote rocks #[derive(Clone, Debug)] -pub struct RemotePackageDB(Vec); +pub struct RemotePackageDB(pub(crate) Vec); #[derive(Clone, Debug)] pub(crate) enum RemoteSource { @@ -76,14 +76,10 @@ impl RemotePackageDB { Ok(Self(vec![RemoteSource::LuarocksManifests(manifests)])) } - /// Builds a package database backed solely by local packages. - pub(crate) fn from_local(packages: Vec) -> Self { - Self(vec![RemoteSource::Local(packages)]) - } - /// Prepends local packages to the lookup order. - pub(crate) fn add_local(&mut self, packages: Vec) { + pub(crate) fn with_local(mut self, packages: Vec) -> Self { self.0.insert(0, RemoteSource::Local(packages)); + self } fn find_in_source( From a384735f01ecdb9e8f7f6315244d7ef32d76e7fd Mon Sep 17 00:00:00 2001 From: vhyrro Date: Sun, 4 Oct 2026 19:44:43 +0200 Subject: [PATCH 18/39] refactor: make PackageSource take a PathBuf --- lux-lib/src/operations/pipeline/build.rs | 5 +++-- .../pipeline/download_sources_and_hash.rs | 22 +++++++++++-------- lux-lib/src/operations/vendor.rs | 15 +++++++++---- 3 files changed, 27 insertions(+), 15 deletions(-) diff --git a/lux-lib/src/operations/pipeline/build.rs b/lux-lib/src/operations/pipeline/build.rs index ec1b5cbb6..14ddbf182 100644 --- a/lux-lib/src/operations/pipeline/build.rs +++ b/lux-lib/src/operations/pipeline/build.rs @@ -105,6 +105,7 @@ where rockspec, entry_type, artifact, + temp_dir: _temp_dir, } = downloaded; let is_entrypoint = entry_type.is_entrypoint(); @@ -113,12 +114,12 @@ where let opt = package.spec.opt(); let pkg = match artifact { - PackageSource::SourceTree(dir) => deploy( + PackageSource::SourceDir(path) => deploy( &rockspec, args.tree, package, &lua, - dir.path(), + &path, entry_type, args.config, behaviour, diff --git a/lux-lib/src/operations/pipeline/download_sources_and_hash.rs b/lux-lib/src/operations/pipeline/download_sources_and_hash.rs index 1f1fcc000..c763f2a97 100644 --- a/lux-lib/src/operations/pipeline/download_sources_and_hash.rs +++ b/lux-lib/src/operations/pipeline/download_sources_and_hash.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use std::io; use std::io::Cursor; +use std::path::PathBuf; use bon::Builder; use bytes::Bytes; @@ -32,7 +33,7 @@ use super::{ #[derive(Clone, Debug)] pub(crate) enum PackageSource { - SourceTree(std::sync::Arc), + SourceDir(PathBuf), PackedRock(Bytes), } @@ -41,6 +42,7 @@ pub(crate) struct DownloadedPackage { pub(crate) rockspec: RemoteLuaRockspec, pub(crate) entry_type: EntryType, pub(crate) artifact: PackageSource, + pub(crate) temp_dir: Option, } pub(crate) type DownloadSourcesAndHashArtifacts = @@ -139,7 +141,7 @@ async fn download_sources_and_hash( let name = spec.name().to_string(); - let (package, artifact) = match artifact { + let (package, artifact, temp_dir) = match artifact { Some(PackageSource::PackedRock(bytes)) => match &source { RemotePackageSource::LuarocksBinaryRock(url) => { let binary_url = url.clone(); @@ -161,7 +163,7 @@ async fn download_sources_and_hash( Some(RemotePackageSourceUrl::Url { url: binary_url }), hashes, ); - (package, PackageSource::PackedRock(bytes)) + (package, PackageSource::PackedRock(bytes), None) } _ => { let source_url = source_url @@ -178,16 +180,16 @@ async fn download_sources_and_hash( config, ) .await?; - (package, PackageSource::SourceTree(std::sync::Arc::new(dir))) + let artifact = PackageSource::SourceDir(dir.path().to_path_buf()); + (package, artifact, Some(dir)) } }, - Some(PackageSource::SourceTree(dir)) => { + Some(PackageSource::SourceDir(path)) => { let rockspec_hash = rockspec .hash() .await .map_err(|err| DownloadSourcesAndHashError::Hash(name.clone(), err))?; - let source_hash = dir - .path() + let source_hash = path .hash() .await .map_err(|err| DownloadSourcesAndHashError::Hash(name.clone(), err))?; @@ -196,7 +198,7 @@ async fn download_sources_and_hash( source: source_hash, }; let package = LockedPackage::new(spec, source, source_url, hashes); - (package, PackageSource::SourceTree(dir)) + (package, PackageSource::SourceDir(path), None) } None => { let (package, dir) = fetch_and_hash_source( @@ -207,7 +209,8 @@ async fn download_sources_and_hash( config, ) .await?; - (package, PackageSource::SourceTree(std::sync::Arc::new(dir))) + let artifact = PackageSource::SourceDir(dir.path().to_path_buf()); + (package, artifact, Some(dir)) } }; @@ -216,6 +219,7 @@ async fn download_sources_and_hash( rockspec, entry_type, artifact, + temp_dir, }) } diff --git a/lux-lib/src/operations/vendor.rs b/lux-lib/src/operations/vendor.rs index 6512744d9..eb5298b37 100644 --- a/lux-lib/src/operations/vendor.rs +++ b/lux-lib/src/operations/vendor.rs @@ -342,9 +342,9 @@ async fn vendor_package_sources( let source_path = vendor_dir.join(format!("{}@{}", name, version)); match &package.artifact { // A fully materialized source tree. - PackageSource::SourceTree(dir) => { + PackageSource::SourceDir(path) => { fs::tokio::remove_dir_all(&source_path).await.ok(); - fs::tokio::copy_dir_all(dir.path(), &source_path).await?; + fs::tokio::copy_dir_all(path, &source_path).await?; } // A pre-built binary rock. PackageSource::PackedRock(bytes) => { @@ -538,6 +538,7 @@ mod tests { rockspec: RemoteLuaRockspec, source: RemotePackageSource, artifact: PackageSource, + temp_dir: Option, ) -> DownloadedPackage { let spec = LockedPackageSpec::new( rockspec.package(), @@ -566,6 +567,7 @@ mod tests { rockspec, entry_type: EntryType::Entrypoint, artifact, + temp_dir, } } @@ -582,10 +584,12 @@ mod tests { "rockspec-only-url@1.0.0".parse().unwrap(), crate::lua_rockspec::RockSourceSpec::File(src_dir.path().to_path_buf()), ); + let artifact = PackageSource::SourceDir(src_dir.path().to_path_buf()); let package = make_downloaded_package( rockspec, RemotePackageSource::RockspecContent(String::new()), - PackageSource::SourceTree(std::sync::Arc::new(src_dir)), + artifact, + Some(src_dir), ); vendor_package_sources(vendor_dir.path(), &package) @@ -628,6 +632,7 @@ mod tests { rockspec, RemotePackageSource::LuarocksBinaryRock("https://example.org/".parse().unwrap()), PackageSource::PackedRock(bytes), + None, ); vendor_package_sources(vendor_dir.path(), &package) @@ -661,10 +666,12 @@ mod tests { "rockspec-only-url@1.0.0".parse().unwrap(), crate::lua_rockspec::RockSourceSpec::File(src_dir.path().to_path_buf()), ); + let artifact = PackageSource::SourceDir(src_dir.path().to_path_buf()); let package = make_downloaded_package( rockspec, RemotePackageSource::RockspecContent(String::new()), - PackageSource::SourceTree(std::sync::Arc::new(src_dir)), + artifact, + Some(src_dir), ); vendor_package_sources(vendor_dir.path(), &package) From 77c6feccad4227dc5c314b519700dc23bc3a7d94 Mon Sep 17 00:00:00 2001 From: vhyrro Date: Sun, 4 Oct 2026 19:58:05 +0200 Subject: [PATCH 19/39] refactor!: join `Build` to pipeline --- lux-cli/src/dist/flat_archive.rs | 10 +- lux-cli/src/pack.rs | 6 +- lux-lib/src/build/cmake.rs | 2 +- lux-lib/src/build/command.rs | 2 +- lux-lib/src/build/external_dependency.rs | 2 +- lux-lib/src/build/make.rs | 2 +- lux-lib/src/build/mod.rs | 134 ++++-------------- lux-lib/src/build/rust_binary.rs | 2 +- lux-lib/src/build/source.rs | 2 +- lux-lib/src/build/utils.rs | 10 +- lux-lib/src/luarocks/luarocks_installation.rs | 6 +- lux-lib/src/operations/install_project.rs | 15 +- lux-lib/src/operations/pipeline/build.rs | 2 +- .../src/operations/pipeline/build_local.rs | 101 +++++++++++++ .../src/operations/pipeline/build_project.rs | 2 +- lux-lib/src/operations/pipeline/discover.rs | 2 +- .../pipeline/download_sources_and_hash.rs | 4 +- .../src/operations/pipeline/emit_lockfile.rs | 2 +- .../operations/pipeline/install_workspace.rs | 16 ++- lux-lib/src/operations/pipeline/mod.rs | 1 + lux-lib/src/operations/pipeline/resolve.rs | 9 +- lux-lib/tests/build.rs | 7 +- 22 files changed, 187 insertions(+), 152 deletions(-) create mode 100644 lux-lib/src/operations/pipeline/build_local.rs diff --git a/lux-cli/src/dist/flat_archive.rs b/lux-cli/src/dist/flat_archive.rs index 914352294..61e882427 100644 --- a/lux-cli/src/dist/flat_archive.rs +++ b/lux-cli/src/dist/flat_archive.rs @@ -5,24 +5,24 @@ use std::{ use clap::Args; use lux_lib::{ - build::{Build, BuildBehaviour}, + build::BuildBehaviour, config::{Config, ConfigBuilder}, lockfile::LockedPackage, lua_installation::LuaInstallation, lua_rockspec::RemoteLuaRockspec, lua_version::LuaVersion, - operations::{Install, InstallProject, PackageInstallSpec}, + operations::{Install, InstallProject, PackageInstallSpec, pipeline::build_local::Build}, package::{PackageName, PackageReq}, tree::{self, FlatDistTree, InstallTree}, workspace::Workspace, }; -use miette::{miette, IntoDiagnostic, Result, WrapErr}; +use miette::{IntoDiagnostic, Result, WrapErr, miette}; use path_slash::PathExt; -use tempfile::{tempdir, TempDir}; +use tempfile::{TempDir, tempdir}; use tokio::fs::{self, File}; use walkdir::WalkDir; -use zip::{write::SimpleFileOptions, ZipWriter}; +use zip::{ZipWriter, write::SimpleFileOptions}; use crate::{ args::{OutputFormat, PackageOrRockspec}, diff --git a/lux-cli/src/pack.rs b/lux-cli/src/pack.rs index 1bca82a05..b0168fb78 100644 --- a/lux-cli/src/pack.rs +++ b/lux-cli/src/pack.rs @@ -4,18 +4,18 @@ use crate::{args::PackageOrRockspec, build, workspace::exists_matching_workspace use clap::Args; use itertools::Itertools; use lux_lib::{ - build::{Build, BuildBehaviour}, + build::BuildBehaviour, config::Config, lua_installation::LuaInstallation, lua_rockspec::RemoteLuaRockspec, lua_version::LuaVersion, - operations::{self, Install, PackageInstallSpec}, + operations::{self, Install, PackageInstallSpec, pipeline::build_local::Build}, package::PackageName, rockspec::Rockspec as _, tree::{self, InstallTree}, workspace::Workspace, }; -use miette::{miette, IntoDiagnostic, Result}; +use miette::{IntoDiagnostic, Result, miette}; use path_slash::PathBufExt; use tempfile::tempdir; diff --git a/lux-lib/src/build/cmake.rs b/lux-lib/src/build/cmake.rs index 6c82e5bfe..21e1712c5 100644 --- a/lux-lib/src/build/cmake.rs +++ b/lux-lib/src/build/cmake.rs @@ -6,7 +6,7 @@ use std::{ }; use thiserror::Error; use tokio::process::Command; -use tracing::{info_span, Instrument}; +use tracing::{Instrument, info_span}; use crate::{ build::{ diff --git a/lux-lib/src/build/command.rs b/lux-lib/src/build/command.rs index d7f73e6be..79ae505eb 100644 --- a/lux-lib/src/build/command.rs +++ b/lux-lib/src/build/command.rs @@ -7,7 +7,7 @@ use std::{ }; use thiserror::Error; -use tracing::{info_span, Instrument}; +use tracing::{Instrument, info_span}; use which::which; use crate::{ diff --git a/lux-lib/src/build/external_dependency.rs b/lux-lib/src/build/external_dependency.rs index 32e16d1d9..4929fe580 100644 --- a/lux-lib/src/build/external_dependency.rs +++ b/lux-lib/src/build/external_dependency.rs @@ -422,7 +422,7 @@ fn is_lib_name(file_name: &str, prefix: &str) -> bool { #[cfg(test)] mod tests { use super::*; - use assert_fs::{prelude::*, TempDir}; + use assert_fs::{TempDir, prelude::*}; #[tokio::test] async fn test_detect_zlib_pkg_config_header() { diff --git a/lux-lib/src/build/make.rs b/lux-lib/src/build/make.rs index 6f136cd0c..ea95426a9 100644 --- a/lux-lib/src/build/make.rs +++ b/lux-lib/src/build/make.rs @@ -8,7 +8,7 @@ use std::{ }; use thiserror::Error; -use tracing::{info_span, Instrument}; +use tracing::{Instrument, info_span}; use crate::{ build::{ diff --git a/lux-lib/src/build/mod.rs b/lux-lib/src/build/mod.rs index a002bf080..7cec1b127 100644 --- a/lux-lib/src/build/mod.rs +++ b/lux-lib/src/build/mod.rs @@ -1,11 +1,11 @@ use crate::build::backend::{BuildBackend, BuildInfo, RunBuildArgs}; use crate::fs; -use crate::lockfile::{LockfileError, OptState, RemotePackageSourceUrl}; +use crate::lockfile::{LockfileError, RemotePackageSourceUrl}; use crate::lua_installation::LuaInstallationError; use crate::lua_rockspec::LuaVersionError; use crate::operations::UnpackError; -use crate::rockspec::{LuaVersionCompatibility, Rockspec}; -use crate::tree::{self, EntryType, InstallTree, TreeError}; +use crate::rockspec::Rockspec; +use crate::tree::{EntryType, InstallTree, TreeError}; use bytes::Bytes; use std::collections::HashMap; use std::fs::DirEntry; @@ -16,13 +16,11 @@ use tracing::Instrument; use crate::{ config::Config, hash::HasIntegrity, - lockfile::{LockConstraint, LockedPackage, LockedPackageSpec, PinnedState}, + lockfile::{LockedPackage, LockedPackageSpec}, lua_installation::LuaInstallation, lua_rockspec::BuildBackendSpec, - operations::{pipeline::download_sources_and_hash::fetch_and_hash_source, FetchSrcError}, - remote_package_source::RemotePackageSource, + operations::FetchSrcError, }; -use bon::Builder; use builtin::BuiltinBuildError; use cmake::CMakeError; use command::CommandError; @@ -40,7 +38,7 @@ use source::SourceBuildError; use ssri::Integrity; use thiserror::Error; use treesitter_parser::TreesitterBuildError; -use utils::{recursive_copy_dir, CompileCFilesError, InstallBinaryError}; +use utils::{CompileCFilesError, InstallBinaryError, recursive_copy_dir}; mod builtin; mod cmake; @@ -58,34 +56,6 @@ pub(crate) mod utils; pub mod external_dependency; -/// A rocks package builder, providing fine-grained control -/// over how a package should be built. -#[derive(Builder)] -#[builder(start_fn = new, finish_fn(name = _build, vis = ""))] -pub struct Build<'a, R: Rockspec + HasIntegrity, T: InstallTree> { - rockspec: &'a R, - tree: &'a T, - entry_type: tree::EntryType, - config: &'a Config, - lua: &'a LuaInstallation, - - #[builder(default)] - pin: PinnedState, - #[builder(default)] - opt: OptState, - #[builder(default)] - constraint: LockConstraint, - #[builder(default)] - behaviour: BuildBehaviour, - - #[builder(setters(vis = "pub(crate)"))] - source_spec: Option, - - // TODO(vhyrro): Remove this and enforce that this is provided at a type level. - #[builder(setters(vis = "pub(crate)"))] - source: Option, -} - #[derive(Debug)] pub(crate) enum RemotePackageSourceSpec { RockSpec(Option), @@ -99,22 +69,6 @@ pub(crate) struct SrcRockSource { pub source_url: RemotePackageSourceUrl, } -// Overwrite the `build()` function to use our own instead. -impl BuildBuilder<'_, R, T, State> -where - State: build_builder::State + build_builder::IsComplete, -{ - pub async fn build(self) -> Result { - let build = self._build(); - let span = tracing::info_span!( - "Building", - package = build.rockspec.package().to_string(), - version = build.rockspec.version().to_string(), - ); - do_build(build).instrument(span).await - } -} - #[derive(Error, Debug, Diagnostic)] #[non_exhaustive] pub enum BuildError { @@ -415,49 +369,6 @@ pub(crate) async fn deploy( Ok(package) } -#[tracing::instrument(level = "trace", skip_all)] -async fn do_build(build: Build<'_, R, T>) -> Result -where - R: Rockspec + HasIntegrity, - T: InstallTree + Sync, -{ - let rockspec = build.rockspec; - let lua = build.lua; - - rockspec.validate_lua_version(&lua.version)?; - - // NOTE: these futures are large; boxing them keeps `do_build`'s frame small. - let spec = LockedPackageSpec::new( - rockspec.package(), - rockspec.version(), - build.constraint, - Vec::new(), - Vec::new(), - &build.pin, - &build.opt, - ); - let (package, temp_dir) = Box::pin(fetch_and_hash_source( - rockspec, - spec, - build.source_spec, - build.source, - build.config, - )) - .await?; - - Box::pin(deploy( - rockspec, - build.tree, - package, - lua, - temp_dir.path(), - build.entry_type, - build.config, - build.behaviour, - )) - .await -} - fn is_source_or_etc_dir(dir: &DirEntry, copy_dirs: &[PathBuf]) -> bool { let dir_name = dir.file_name().to_string_lossy().to_string(); matches!(dir_name.as_str(), "lua" | "src") @@ -536,12 +447,13 @@ mod tests { use crate::{ config::ConfigBuilder, - lockfile::LockedPackageHashes, - lua_installation::{detect_installed_lua_version, LuaInstallation}, + lockfile::{LockConstraint, LockedPackageHashes}, + lua_installation::{LuaInstallation, detect_installed_lua_version}, lua_version::LuaVersion, - operations::{unpack_rockspec, DownloadedPackedRockBytes}, + operations::{DownloadedPackedRockBytes, unpack_rockspec}, package::PackageSpec, project::Project, + remote_package_source::RemotePackageSource, tree::Tree, }; @@ -601,21 +513,27 @@ mod tests { assert!(foo_dir.is_dir()); let foo_init = foo_dir.join("init.lua"); assert!(foo_init.is_file()); - assert!(std::fs::read_to_string(&foo_init) - .unwrap() - .contains("return true")); + assert!( + std::fs::read_to_string(&foo_init) + .unwrap() + .contains("return true") + ); let foo_bar_dir = foo_dir.join("bar"); assert!(foo_bar_dir.is_dir()); let foo_bar_init = foo_bar_dir.join("init.lua"); assert!(foo_bar_init.is_file()); - assert!(std::fs::read_to_string(&foo_bar_init) - .unwrap() - .contains("return true")); + assert!( + std::fs::read_to_string(&foo_bar_init) + .unwrap() + .contains("return true") + ); let foo_bar_baz = foo_bar_dir.join("baz.lua"); assert!(foo_bar_baz.is_file()); - assert!(std::fs::read_to_string(&foo_bar_baz) - .unwrap() - .contains("return true")); + assert!( + std::fs::read_to_string(&foo_bar_baz) + .unwrap() + .contains("return true") + ); let bin_file = tree_dir .child(lua_version.to_string()) .child("bin") @@ -669,7 +587,7 @@ mod tests { }; let rockspec = unpack_rockspec(&rock).await.unwrap(); let lua = LuaInstallation::new_from_config(config).await.unwrap(); - Build::new() + crate::operations::pipeline::build_local::Build::new() .rockspec(&rockspec) .lua(&lua) .tree(tree) diff --git a/lux-lib/src/build/rust_binary.rs b/lux-lib/src/build/rust_binary.rs index 22acda561..8c2dd5b80 100644 --- a/lux-lib/src/build/rust_binary.rs +++ b/lux-lib/src/build/rust_binary.rs @@ -1,6 +1,6 @@ use crate::build::backend::{BuildBackend, BuildInfo, RunBuildArgs}; use crate::build::utils::{self, InstallBinaryError}; -use crate::config::{build, Config}; +use crate::config::{Config, build}; use crate::fs; use crate::lua_rockspec::RustBinaryBuildSpec; use crate::tree::InstallTree; diff --git a/lux-lib/src/build/source.rs b/lux-lib/src/build/source.rs index 9bdf8a358..f08150f09 100644 --- a/lux-lib/src/build/source.rs +++ b/lux-lib/src/build/source.rs @@ -5,8 +5,8 @@ use crate::{ build::backend::{BuildBackend, BuildInfo, RunBuildArgs}, lua_rockspec::{BuildBackendSpec, BuildSpec, LocalLuaRockspec, LuaRockspecError}, project::{ + PROJECT_TOML, ProjectRoot, TomlDeError, project_toml::{LocalProjectTomlValidationError, PartialProjectToml}, - ProjectRoot, TomlDeError, PROJECT_TOML, }, rockspec::Rockspec, tree::InstallTree, diff --git a/lux-lib/src/build/utils.rs b/lux-lib/src/build/utils.rs index 0ad4a2e0c..3f72b033a 100644 --- a/lux-lib/src/build/utils.rs +++ b/lux-lib/src/build/utils.rs @@ -1015,9 +1015,11 @@ mod tests { #[cfg(unix)] set_executable_permissions(&script_path).await.unwrap(); - assert!(Command::new(script_path.to_string_lossy().to_string()) - .status() - .await - .is_ok_and(|status| status.success())); + assert!( + Command::new(script_path.to_string_lossy().to_string()) + .status() + .await + .is_ok_and(|status| status.success()) + ); } } diff --git a/lux-lib/src/luarocks/luarocks_installation.rs b/lux-lib/src/luarocks/luarocks_installation.rs index e5b29f243..e47d3fff7 100644 --- a/lux-lib/src/luarocks/luarocks_installation.rs +++ b/lux-lib/src/luarocks/luarocks_installation.rs @@ -26,7 +26,7 @@ use crate::tree::{self, Tree, TreeError}; use crate::tree::{Tree, TreeError}; #[cfg(target_family = "unix")] -use crate::build::Build; +use crate::operations::pipeline::build_local::Build; #[cfg(target_family = "unix")] const LUAROCKS_EXE: &str = "luarocks"; @@ -97,7 +97,9 @@ pub enum ExecLuaRocksError { #[error(transparent)] #[diagnostic(transparent)] LuaVersionUnset(#[from] LuaVersionUnset), - #[error("could not substitute '$(LUA_LIBDIR)' and '$(LUA_INCDIR)' variables in the luarocks config template")] + #[error( + "could not substitute '$(LUA_LIBDIR)' and '$(LUA_INCDIR)' variables in the luarocks config template" + )] #[diagnostic(forward(0))] VariableSubstitutionInConfig(#[from] VariableSubstitutionError), #[error("failed to run luarocks")] diff --git a/lux-lib/src/operations/install_project.rs b/lux-lib/src/operations/install_project.rs index 4fcb0472f..f7daed752 100644 --- a/lux-lib/src/operations/install_project.rs +++ b/lux-lib/src/operations/install_project.rs @@ -1,11 +1,14 @@ use crate::{ - build::{Build, BuildBehaviour, BuildError}, + build::{BuildBehaviour, BuildError}, config::Config, lockfile::LockedPackage, lua_installation::{LuaInstallation, LuaInstallationError}, luarocks::luarocks_installation::{LuaRocksError, LuaRocksInstallError, LuaRocksInstallation}, - operations::{install_dependencies::prepare_dependencies_for_build, InstallDependencies}, - project::{project_toml::LocalProjectTomlValidationError, Project, ProjectError}, + operations::{ + InstallDependencies, install_dependencies::prepare_dependencies_for_build, + pipeline::build_local::Build, + }, + project::{Project, ProjectError, project_toml::LocalProjectTomlValidationError}, tree::{self, InstallTree, TreeError}, }; use bon::Builder; @@ -75,9 +78,9 @@ where } impl< - T: InstallTree + Sync + Send + Clone + 'static, - State: install_project_builder::State + install_project_builder::IsComplete, - > InstallProjectBuilder<'_, T, State> + T: InstallTree + Sync + Send + Clone + 'static, + State: install_project_builder::State + install_project_builder::IsComplete, +> InstallProjectBuilder<'_, T, State> { /// Returns `Some` if the `only_deps` option is set to `false`. pub async fn build(self) -> Result { diff --git a/lux-lib/src/operations/pipeline/build.rs b/lux-lib/src/operations/pipeline/build.rs index 14ddbf182..9080f1f60 100644 --- a/lux-lib/src/operations/pipeline/build.rs +++ b/lux-lib/src/operations/pipeline/build.rs @@ -5,7 +5,7 @@ use miette::Diagnostic; use thiserror::Error; use crate::{ - build::{deploy, BuildBehaviour}, + build::{BuildBehaviour, deploy}, config::Config, lockfile::{LockedPackage, LockedPackageId}, lua_installation::{LuaInstallation, LuaInstallationError}, diff --git a/lux-lib/src/operations/pipeline/build_local.rs b/lux-lib/src/operations/pipeline/build_local.rs new file mode 100644 index 000000000..bd02dd66d --- /dev/null +++ b/lux-lib/src/operations/pipeline/build_local.rs @@ -0,0 +1,101 @@ +use bon::Builder; +use tracing::Instrument; + +use crate::{ + build::{BuildBehaviour, BuildError, RemotePackageSourceSpec, deploy}, + config::Config, + hash::HasIntegrity, + lockfile::{LockConstraint, LockedPackage, LockedPackageSpec, OptState, PinnedState}, + lua_installation::LuaInstallation, + remote_package_source::RemotePackageSource, + rockspec::{LuaVersionCompatibility, Rockspec}, + tree::{EntryType, InstallTree}, +}; + +use super::download_sources_and_hash::fetch_and_hash_source; + +/// A rocks package builder, providing fine-grained control +/// over how a package should be built. +#[derive(Builder)] +#[builder(start_fn = new, finish_fn(name = _build, vis = ""))] +pub struct Build<'a, R: Rockspec + HasIntegrity, T: InstallTree> { + rockspec: &'a R, + tree: &'a T, + entry_type: EntryType, + config: &'a Config, + lua: &'a LuaInstallation, + + #[builder(default)] + pin: PinnedState, + #[builder(default)] + opt: OptState, + #[builder(default)] + constraint: LockConstraint, + #[builder(default)] + behaviour: BuildBehaviour, + + #[builder(setters(vis = "pub(crate)"))] + source_spec: Option, + + // TODO(vhyrro): Remove this and enforce that this is provided at a type level. + #[builder(setters(vis = "pub(crate)"))] + source: Option, +} + +impl BuildBuilder<'_, R, T, State> +where + State: build_builder::State + build_builder::IsComplete, +{ + pub async fn build(self) -> Result { + let build = self._build(); + let span = tracing::info_span!( + "Building", + package = build.rockspec.package().to_string(), + version = build.rockspec.version().to_string(), + ); + do_build(build).instrument(span).await + } +} + +#[tracing::instrument(level = "trace", skip_all)] +async fn do_build(build: Build<'_, R, T>) -> Result +where + R: Rockspec + HasIntegrity, + T: InstallTree + Sync, +{ + let rockspec = build.rockspec; + let lua = build.lua; + + rockspec.validate_lua_version(&lua.version)?; + + // NOTE: these futures are large; boxing them keeps `do_build`'s frame small. + let spec = LockedPackageSpec::new( + rockspec.package(), + rockspec.version(), + build.constraint, + Vec::new(), + Vec::new(), + &build.pin, + &build.opt, + ); + let (package, temp_dir) = Box::pin(fetch_and_hash_source( + rockspec, + spec, + build.source_spec, + build.source, + build.config, + )) + .await?; + + Box::pin(deploy( + rockspec, + build.tree, + package, + lua, + temp_dir.path(), + build.entry_type, + build.config, + build.behaviour, + )) + .await +} diff --git a/lux-lib/src/operations/pipeline/build_project.rs b/lux-lib/src/operations/pipeline/build_project.rs index b3aa62ae3..62ee4948a 100644 --- a/lux-lib/src/operations/pipeline/build_project.rs +++ b/lux-lib/src/operations/pipeline/build_project.rs @@ -39,7 +39,7 @@ where let workspace_tree = args.workspace.tree(args.config)?; let project_toml = args.project.toml().into_local()?; - let package = crate::build::Build::new() + let package = super::build_local::Build::new() .rockspec(&project_toml) .lua(args.lua) .tree(&workspace_tree) diff --git a/lux-lib/src/operations/pipeline/discover.rs b/lux-lib/src/operations/pipeline/discover.rs index 30d2ccfe3..7b3ad82ac 100644 --- a/lux-lib/src/operations/pipeline/discover.rs +++ b/lux-lib/src/operations/pipeline/discover.rs @@ -5,8 +5,8 @@ use crate::{ lua_rockspec::RemoteLuaRockspec, operations, package::{PackageReq, PackageSpec, RemotePackage}, - remote_package_source::RemotePackageSource, remote_package_db::SearchError, + remote_package_source::RemotePackageSource, rockspec::Rockspec, }; diff --git a/lux-lib/src/operations/pipeline/download_sources_and_hash.rs b/lux-lib/src/operations/pipeline/download_sources_and_hash.rs index c763f2a97..eb30d59dc 100644 --- a/lux-lib/src/operations/pipeline/download_sources_and_hash.rs +++ b/lux-lib/src/operations/pipeline/download_sources_and_hash.rs @@ -20,15 +20,15 @@ use crate::{ RemotePackageSourceUrl, }, lua_rockspec::RemoteLuaRockspec, - operations::{unpack_src_rock, FetchSrc, RemotePackageSourceMetadata}, + operations::{FetchSrc, RemotePackageSourceMetadata, unpack_src_rock}, remote_package_source::RemotePackageSource, rockspec::Rockspec, tree::EntryType, }; use super::{ - resolve::{ResolvedArtifacts, ResolvedPackage}, Artifacts, + resolve::{ResolvedArtifacts, ResolvedPackage}, }; #[derive(Clone, Debug)] diff --git a/lux-lib/src/operations/pipeline/emit_lockfile.rs b/lux-lib/src/operations/pipeline/emit_lockfile.rs index ab7a47f13..e434b67ef 100644 --- a/lux-lib/src/operations/pipeline/emit_lockfile.rs +++ b/lux-lib/src/operations/pipeline/emit_lockfile.rs @@ -3,8 +3,8 @@ use std::collections::HashMap; use crate::lockfile::{LockedPackageId, LockedPackageLock, ReadWrite, WorkspaceLockfile}; use super::{ - download_sources_and_hash::{DownloadSourcesAndHashArtifacts, DownloadedPackage}, Artifacts, + download_sources_and_hash::{DownloadSourcesAndHashArtifacts, DownloadedPackage}, }; /// The lockfile entries resolved by the pipeline, grouped by section. diff --git a/lux-lib/src/operations/pipeline/install_workspace.rs b/lux-lib/src/operations/pipeline/install_workspace.rs index 33e79fac7..3eca0fc54 100644 --- a/lux-lib/src/operations/pipeline/install_workspace.rs +++ b/lux-lib/src/operations/pipeline/install_workspace.rs @@ -165,11 +165,15 @@ mod tests { let lockfile = workspace.lockfile().unwrap(); let rocks = lockfile.rocks(&LockedPackageLockType::Regular); - assert!(rocks - .values() - .any(|pkg| pkg.name().to_string() == "lua-cjson")); - assert!(rocks - .values() - .any(|pkg| pkg.name().to_string() == "plenary.nvim")); + assert!( + rocks + .values() + .any(|pkg| pkg.name().to_string() == "lua-cjson") + ); + assert!( + rocks + .values() + .any(|pkg| pkg.name().to_string() == "plenary.nvim") + ); } } diff --git a/lux-lib/src/operations/pipeline/mod.rs b/lux-lib/src/operations/pipeline/mod.rs index 91735daef..c2af45eea 100644 --- a/lux-lib/src/operations/pipeline/mod.rs +++ b/lux-lib/src/operations/pipeline/mod.rs @@ -1,6 +1,7 @@ use crate::lockfile::LockedPackageLockType; pub mod build; +pub mod build_local; pub mod build_project; pub mod discover; pub mod download_sources_and_hash; diff --git a/lux-lib/src/operations/pipeline/resolve.rs b/lux-lib/src/operations/pipeline/resolve.rs index ce2e8b0ab..41dd1fb03 100644 --- a/lux-lib/src/operations/pipeline/resolve.rs +++ b/lux-lib/src/operations/pipeline/resolve.rs @@ -23,9 +23,9 @@ use crate::{ }; use super::{ + Artifacts, discover::{DiscoverError, FoundPackage}, download_sources_and_hash::PackageSource, - Artifacts, }; /// The build dependencies of a rockspec that still need to be installed, with the @@ -186,7 +186,11 @@ impl Resolver { /// Builds a resolver with an explicit in-flight bound. Used by tests, which do not /// construct a full [`Config`]. - fn with_max_inflight(package_db: RemotePackageDB, config: Arc, max_inflight: usize) -> Self { + fn with_max_inflight( + package_db: RemotePackageDB, + config: Arc, + max_inflight: usize, + ) -> Self { Self { package_db, config, @@ -459,7 +463,6 @@ fn detect_cycles(nodes: &HashMap) -> Result<(), ResolveError> Ok(()) } - fn build_dependency_specs( rockspec: &RemoteLuaRockspec, parent: &PackageInstallSpec, diff --git a/lux-lib/tests/build.rs b/lux-lib/tests/build.rs index f3cd0ce52..b02a301a2 100644 --- a/lux-lib/tests/build.rs +++ b/lux-lib/tests/build.rs @@ -1,17 +1,18 @@ use std::path::PathBuf; -use assert_fs::prelude::PathCopy; use assert_fs::TempDir; +use assert_fs::prelude::PathCopy; use flaky_test::flaky_test; use lux_lib::lua_version::LuaVersion; use lux_lib::rockspec::Rockspec; use lux_lib::tree::InstallTree; use lux_lib::workspace::Workspace; use lux_lib::{ - build::{Build, BuildBehaviour::Force}, + build::BuildBehaviour::Force, config::ConfigBuilder, - lua_installation::{detect_installed_lua_version, LuaInstallation}, + lua_installation::{LuaInstallation, detect_installed_lua_version}, lua_rockspec::RemoteLuaRockspec, + operations::pipeline::build_local::Build, tree, }; use tokio::runtime::Builder; From 34d664f2be95bf2b9cdc55d8a8d82e032b32f091 Mon Sep 17 00:00:00 2001 From: vhyrro Date: Sun, 4 Oct 2026 20:37:00 +0200 Subject: [PATCH 20/39] refactor!: route Project building through the same code as package building --- lux-cli/src/dist/flat_archive.rs | 10 +- lux-cli/src/install.rs | 2 +- lux-cli/src/pack.rs | 6 +- lux-cli/src/uninstall.rs | 2 +- lux-lib/src/build/cmake.rs | 2 +- lux-lib/src/build/command.rs | 2 +- lux-lib/src/build/external_dependency.rs | 2 +- lux-lib/src/build/make.rs | 2 +- lux-lib/src/build/mod.rs | 30 ++- lux-lib/src/build/rust_binary.rs | 2 +- lux-lib/src/build/source.rs | 2 +- lux-lib/src/build/utils.rs | 10 +- lux-lib/src/lua_rockspec/mod.rs | 44 ++++ lux-lib/src/operations/build_workspace.rs | 28 ++- lux-lib/src/operations/exec.rs | 6 +- lux-lib/src/operations/install/mod.rs | 40 +--- .../src/operations/install_dependencies.rs | 117 ----------- lux-lib/src/operations/install_project.rs | 124 ++++-------- lux-lib/src/operations/install_rockspec.rs | 5 +- lux-lib/src/operations/mod.rs | 4 +- lux-lib/src/operations/pipeline/build.rs | 15 +- .../src/operations/pipeline/build_local.rs | 2 +- .../src/operations/pipeline/build_project.rs | 102 ---------- lux-lib/src/operations/pipeline/discover.rs | 17 ++ .../pipeline/download_sources_and_hash.rs | 7 +- .../src/operations/pipeline/emit_lockfile.rs | 2 +- .../operations/pipeline/install_packages.rs | 16 +- .../operations/pipeline/install_workspace.rs | 19 +- lux-lib/src/operations/pipeline/mod.rs | 1 - lux-lib/src/operations/pipeline/resolve.rs | 4 +- lux-lib/src/operations/sync.rs | 4 +- lux-lib/src/operations/test.rs | 2 +- lux-lib/src/operations/update.rs | 2 +- lux-lib/src/operations/vendor.rs | 2 + lux-lib/src/project/mod.rs | 11 + lux-lib/src/project/project_toml.rs | 190 ++++++++++-------- lux-lib/src/remote_package_db/mod.rs | 10 +- lux-lib/tests/build.rs | 4 +- lux-lib/tests/install.rs | 6 +- 39 files changed, 328 insertions(+), 528 deletions(-) delete mode 100644 lux-lib/src/operations/install_dependencies.rs delete mode 100644 lux-lib/src/operations/pipeline/build_project.rs diff --git a/lux-cli/src/dist/flat_archive.rs b/lux-cli/src/dist/flat_archive.rs index 61e882427..aa3c7fa93 100644 --- a/lux-cli/src/dist/flat_archive.rs +++ b/lux-cli/src/dist/flat_archive.rs @@ -11,18 +11,18 @@ use lux_lib::{ lua_installation::LuaInstallation, lua_rockspec::RemoteLuaRockspec, lua_version::LuaVersion, - operations::{Install, InstallProject, PackageInstallSpec, pipeline::build_local::Build}, + operations::{pipeline::build_local::Build, Install, InstallProject, PackageInstallSpec}, package::{PackageName, PackageReq}, tree::{self, FlatDistTree, InstallTree}, workspace::Workspace, }; -use miette::{IntoDiagnostic, Result, WrapErr, miette}; +use miette::{miette, IntoDiagnostic, Result, WrapErr}; use path_slash::PathExt; -use tempfile::{TempDir, tempdir}; +use tempfile::{tempdir, TempDir}; use tokio::fs::{self, File}; use walkdir::WalkDir; -use zip::{ZipWriter, write::SimpleFileOptions}; +use zip::{write::SimpleFileOptions, ZipWriter}; use crate::{ args::{OutputFormat, PackageOrRockspec}, @@ -164,7 +164,7 @@ async fn install_package( .build_behaviour(BuildBehaviour::Force) .build(), ) - .tree(tree.clone()) + .tree(&tree) .install() .await?; let package = packages diff --git a/lux-cli/src/install.rs b/lux-cli/src/install.rs index a604bb305..d123e57b3 100644 --- a/lux-cli/src/install.rs +++ b/lux-cli/src/install.rs @@ -79,7 +79,7 @@ async fn install_remote(data: Install, config: Config) -> Result<()> { // TODO(vhyrro): If the tree doesn't exist then error out. operations::Install::new(&config) .packages(packages) - .tree(tree) + .tree(&tree) .install() .await?; diff --git a/lux-cli/src/pack.rs b/lux-cli/src/pack.rs index b0168fb78..16a8db89a 100644 --- a/lux-cli/src/pack.rs +++ b/lux-cli/src/pack.rs @@ -9,13 +9,13 @@ use lux_lib::{ lua_installation::LuaInstallation, lua_rockspec::RemoteLuaRockspec, lua_version::LuaVersion, - operations::{self, Install, PackageInstallSpec, pipeline::build_local::Build}, + operations::{self, pipeline::build_local::Build, Install, PackageInstallSpec}, package::PackageName, rockspec::Rockspec as _, tree::{self, InstallTree}, workspace::Workspace, }; -use miette::{IntoDiagnostic, Result, miette}; +use miette::{miette, IntoDiagnostic, Result}; use path_slash::PathBufExt; use tempfile::tempdir; @@ -113,7 +113,7 @@ pub async fn pack(args: Pack, config: Config) -> Result<()> { .build_behaviour(BuildBehaviour::Force) .build(), ) - .tree(tree.clone()) + .tree(&tree) .install() .await?; let package = packages diff --git a/lux-cli/src/uninstall.rs b/lux-cli/src/uninstall.rs index 9f8e83349..727ac2b72 100644 --- a/lux-cli/src/uninstall.rs +++ b/lux-cli/src/uninstall.rs @@ -153,7 +153,7 @@ Reinstall? .await?; operations::Install::new(&config) .packages(reinstall_specs) - .tree(tree) + .tree(&tree) .install() .await?; } else { diff --git a/lux-lib/src/build/cmake.rs b/lux-lib/src/build/cmake.rs index 21e1712c5..6c82e5bfe 100644 --- a/lux-lib/src/build/cmake.rs +++ b/lux-lib/src/build/cmake.rs @@ -6,7 +6,7 @@ use std::{ }; use thiserror::Error; use tokio::process::Command; -use tracing::{Instrument, info_span}; +use tracing::{info_span, Instrument}; use crate::{ build::{ diff --git a/lux-lib/src/build/command.rs b/lux-lib/src/build/command.rs index 79ae505eb..d7f73e6be 100644 --- a/lux-lib/src/build/command.rs +++ b/lux-lib/src/build/command.rs @@ -7,7 +7,7 @@ use std::{ }; use thiserror::Error; -use tracing::{Instrument, info_span}; +use tracing::{info_span, Instrument}; use which::which; use crate::{ diff --git a/lux-lib/src/build/external_dependency.rs b/lux-lib/src/build/external_dependency.rs index 4929fe580..32e16d1d9 100644 --- a/lux-lib/src/build/external_dependency.rs +++ b/lux-lib/src/build/external_dependency.rs @@ -422,7 +422,7 @@ fn is_lib_name(file_name: &str, prefix: &str) -> bool { #[cfg(test)] mod tests { use super::*; - use assert_fs::{TempDir, prelude::*}; + use assert_fs::{prelude::*, TempDir}; #[tokio::test] async fn test_detect_zlib_pkg_config_header() { diff --git a/lux-lib/src/build/make.rs b/lux-lib/src/build/make.rs index ea95426a9..6f136cd0c 100644 --- a/lux-lib/src/build/make.rs +++ b/lux-lib/src/build/make.rs @@ -8,7 +8,7 @@ use std::{ }; use thiserror::Error; -use tracing::{Instrument, info_span}; +use tracing::{info_span, Instrument}; use crate::{ build::{ diff --git a/lux-lib/src/build/mod.rs b/lux-lib/src/build/mod.rs index 7cec1b127..0329233ca 100644 --- a/lux-lib/src/build/mod.rs +++ b/lux-lib/src/build/mod.rs @@ -38,7 +38,7 @@ use source::SourceBuildError; use ssri::Integrity; use thiserror::Error; use treesitter_parser::TreesitterBuildError; -use utils::{CompileCFilesError, InstallBinaryError, recursive_copy_dir}; +use utils::{recursive_copy_dir, CompileCFilesError, InstallBinaryError}; mod builtin; mod cmake; @@ -448,9 +448,9 @@ mod tests { use crate::{ config::ConfigBuilder, lockfile::{LockConstraint, LockedPackageHashes}, - lua_installation::{LuaInstallation, detect_installed_lua_version}, + lua_installation::{detect_installed_lua_version, LuaInstallation}, lua_version::LuaVersion, - operations::{DownloadedPackedRockBytes, unpack_rockspec}, + operations::{unpack_rockspec, DownloadedPackedRockBytes}, package::PackageSpec, project::Project, remote_package_source::RemotePackageSource, @@ -513,27 +513,21 @@ mod tests { assert!(foo_dir.is_dir()); let foo_init = foo_dir.join("init.lua"); assert!(foo_init.is_file()); - assert!( - std::fs::read_to_string(&foo_init) - .unwrap() - .contains("return true") - ); + assert!(std::fs::read_to_string(&foo_init) + .unwrap() + .contains("return true")); let foo_bar_dir = foo_dir.join("bar"); assert!(foo_bar_dir.is_dir()); let foo_bar_init = foo_bar_dir.join("init.lua"); assert!(foo_bar_init.is_file()); - assert!( - std::fs::read_to_string(&foo_bar_init) - .unwrap() - .contains("return true") - ); + assert!(std::fs::read_to_string(&foo_bar_init) + .unwrap() + .contains("return true")); let foo_bar_baz = foo_bar_dir.join("baz.lua"); assert!(foo_bar_baz.is_file()); - assert!( - std::fs::read_to_string(&foo_bar_baz) - .unwrap() - .contains("return true") - ); + assert!(std::fs::read_to_string(&foo_bar_baz) + .unwrap() + .contains("return true")); let bin_file = tree_dir .child(lua_version.to_string()) .child("bin") diff --git a/lux-lib/src/build/rust_binary.rs b/lux-lib/src/build/rust_binary.rs index 8c2dd5b80..22acda561 100644 --- a/lux-lib/src/build/rust_binary.rs +++ b/lux-lib/src/build/rust_binary.rs @@ -1,6 +1,6 @@ use crate::build::backend::{BuildBackend, BuildInfo, RunBuildArgs}; use crate::build::utils::{self, InstallBinaryError}; -use crate::config::{Config, build}; +use crate::config::{build, Config}; use crate::fs; use crate::lua_rockspec::RustBinaryBuildSpec; use crate::tree::InstallTree; diff --git a/lux-lib/src/build/source.rs b/lux-lib/src/build/source.rs index f08150f09..9bdf8a358 100644 --- a/lux-lib/src/build/source.rs +++ b/lux-lib/src/build/source.rs @@ -5,8 +5,8 @@ use crate::{ build::backend::{BuildBackend, BuildInfo, RunBuildArgs}, lua_rockspec::{BuildBackendSpec, BuildSpec, LocalLuaRockspec, LuaRockspecError}, project::{ - PROJECT_TOML, ProjectRoot, TomlDeError, project_toml::{LocalProjectTomlValidationError, PartialProjectToml}, + ProjectRoot, TomlDeError, PROJECT_TOML, }, rockspec::Rockspec, tree::InstallTree, diff --git a/lux-lib/src/build/utils.rs b/lux-lib/src/build/utils.rs index 3f72b033a..0ad4a2e0c 100644 --- a/lux-lib/src/build/utils.rs +++ b/lux-lib/src/build/utils.rs @@ -1015,11 +1015,9 @@ mod tests { #[cfg(unix)] set_executable_permissions(&script_path).await.unwrap(); - assert!( - Command::new(script_path.to_string_lossy().to_string()) - .status() - .await - .is_ok_and(|status| status.success()) - ); + assert!(Command::new(script_path.to_string_lossy().to_string()) + .status() + .await + .is_ok_and(|status| status.success())); } } diff --git a/lux-lib/src/lua_rockspec/mod.rs b/lux-lib/src/lua_rockspec/mod.rs index 8896c89a2..30173d525 100644 --- a/lux-lib/src/lua_rockspec/mod.rs +++ b/lux-lib/src/lua_rockspec/mod.rs @@ -206,6 +206,50 @@ impl<'gc> HasRockspecKey<'gc> for ottavino::Table<'gc> { } impl LocalLuaRockspec { + /// Builds a [`LocalLuaRockspec`] from an already-parsed project. + /// + /// Unlike [`crate::project::project_toml::LocalProjectToml::to_lua_rockspec`], + /// this does not round-trip through a rendered rockspec string, so it + /// preserves path dependencies and cannot fail. + pub(crate) fn from_project_toml( + project: &crate::project::project_toml::LocalProjectToml, + ) -> Self { + use crate::rockspec::Rockspec; + + Self { + rockspec_format: project.format().clone(), + package: project.package().clone(), + version: project.version().clone(), + description: project.description().clone(), + supported_platforms: project.supported_platforms().clone(), + lua: project.lua().clone(), + dependencies: project.dependencies().clone(), + build_dependencies: project.build_dependencies().clone(), + external_dependencies: project.external_dependencies().clone(), + test_dependencies: project.test_dependencies().clone(), + build: project.build().clone(), + source: project.source().clone(), + test: project.test().clone(), + deploy: project.deploy().clone(), + raw_content: String::new(), + } + } + + /// Wraps this rockspec into a [`RemoteLuaRockspec`], overriding its source + /// with the given `source`. + /// + /// Used to build a project's root package through the pipeline, pointing + /// the source at the project directory itself. + pub fn into_remote_with_source( + self, + source: PerPlatform, + ) -> RemoteLuaRockspec { + RemoteLuaRockspec { + local: self, + source, + } + } + pub fn new( rockspec_content: &str, project_root: ProjectRoot, diff --git a/lux-lib/src/operations/build_workspace.rs b/lux-lib/src/operations/build_workspace.rs index 839f6c728..699e69abd 100644 --- a/lux-lib/src/operations/build_workspace.rs +++ b/lux-lib/src/operations/build_workspace.rs @@ -1,16 +1,15 @@ use crate::{ config::Config, lockfile::LockedPackage, - lua_installation::{LuaInstallation, LuaInstallationError}, + lua_installation::LuaInstallationError, operations::{ - pipeline::{ - build_project::{BuildProject, BuildProjectError}, - install_workspace::{InstallWorkspaceDependencies, InstallWorkspaceDependenciesError}, + pipeline::install_workspace::{ + InstallWorkspaceDependencies, InstallWorkspaceDependenciesError, }, - GenLuaRc, GenLuaRcError, + GenLuaRc, GenLuaRcError, InstallProject, InstallProjectError, }, package::PackageName, - workspace::{Workspace, WorkspaceError}, + workspace::{Workspace, WorkspaceError, WorkspaceTreeError}, }; use bon::Builder; use thiserror::Error; @@ -24,7 +23,10 @@ pub enum BuildWorkspaceError { InstallWorkspaceDependencies(#[from] InstallWorkspaceDependenciesError), #[error(transparent)] #[diagnostic(transparent)] - BuildProject(#[from] BuildProjectError), + InstallProject(#[from] InstallProjectError), + #[error(transparent)] + #[diagnostic(transparent)] + WorkspaceTree(#[from] WorkspaceTreeError), #[error(transparent)] #[diagnostic(transparent)] LuaInstallation(#[from] LuaInstallationError), @@ -72,12 +74,15 @@ impl { let project = build.workspace.select_member(package)?; built.push( - BuildProject::new(project, build.workspace, build.config, &lua) + InstallProject::new() + .project(project) + .config(build.config) + .tree(&workspace_tree) .build() .await?, ); @@ -85,7 +90,10 @@ impl { for project in build.workspace.members() { built.push( - BuildProject::new(project, build.workspace, build.config, &lua) + InstallProject::new() + .project(project) + .config(build.config) + .tree(&workspace_tree) .build() .await?, ); diff --git a/lux-lib/src/operations/exec.rs b/lux-lib/src/operations/exec.rs index d630976da..db39fa738 100644 --- a/lux-lib/src/operations/exec.rs +++ b/lux-lib/src/operations/exec.rs @@ -91,7 +91,7 @@ pub enum ExecError { LuaVersionError(#[from] LuaVersionError), #[error(transparent)] #[diagnostic(transparent)] - BuildProject(#[from] Box), + BuildWorkspace(#[from] Box), #[error(transparent)] #[diagnostic(transparent)] InstallCommand(#[from] Box), @@ -104,7 +104,7 @@ pub enum ExecError { impl From for ExecError { fn from(source: BuildWorkspaceError) -> Self { - Self::BuildProject(Box::new(source)) + Self::BuildWorkspace(Box::new(source)) } } @@ -228,7 +228,7 @@ async fn install_command(command: &str, config: &Config) -> Result<(), InstallCo let tree = config.user_tree(LuaVersion::from(config)?.clone())?; Install::new(config) .package(install_spec) - .tree(tree) + .tree(&tree) .install() .await?; Ok(()) diff --git a/lux-lib/src/operations/install/mod.rs b/lux-lib/src/operations/install/mod.rs index 7206e765c..58ec33936 100644 --- a/lux-lib/src/operations/install/mod.rs +++ b/lux-lib/src/operations/install/mod.rs @@ -1,10 +1,5 @@ use crate::{ - build::BuildBehaviour, - config::Config, - lockfile::LockedPackage, - remote_package_db::RemotePackageDB, - tree::{InstallTree, Tree}, - workspace::{Workspace, WorkspaceTreeError}, + config::Config, lockfile::LockedPackage, remote_package_db::RemotePackageDB, tree::InstallTree, }; pub use crate::operations::install::spec::PackageInstallSpec; @@ -24,40 +19,23 @@ pub mod spec; #[builder(start_fn = new, finish_fn(name = _build, vis = ""))] pub struct Install<'a, T> where - T: InstallTree + Clone + Send + Sync, + T: InstallTree + Send + Sync, { #[builder(start_fn)] config: &'a Config, #[builder(field)] packages: Vec, #[builder(setters(name = "_tree", vis = ""))] - tree: T, + tree: &'a T, package_db: Option, - behaviour: Option, -} - -impl<'a, State> InstallBuilder<'a, Tree, State> -where - State: install_builder::State, -{ - pub fn workspace( - self, - workspace: &'a Workspace, - ) -> Result>, WorkspaceTreeError> - where - State::Tree: install_builder::IsUnset, - { - let config = self.config; - Ok(self._tree(workspace.tree(config)?)) - } } impl<'a, T, State> InstallBuilder<'a, T, State> where State: install_builder::State, - T: InstallTree + Clone + Send + Sync, + T: InstallTree + Send + Sync, { - pub fn tree(self, tree: T) -> InstallBuilder<'a, T, install_builder::SetTree> + pub fn tree(self, tree: &'a T) -> InstallBuilder<'a, T, install_builder::SetTree> where State::Tree: install_builder::IsUnset, { @@ -83,7 +61,7 @@ where impl InstallBuilder<'_, T, State> where State: install_builder::State + install_builder::IsComplete, - T: InstallTree + Clone + Send + Sync + 'static, + T: InstallTree + Send + Sync, { /// Install the packages. pub async fn install(self) -> Result, InstallError> { @@ -116,9 +94,6 @@ pub enum InstallError { #[error(transparent)] #[diagnostic(transparent)] Tree(#[from] crate::tree::TreeError), - #[error(transparent)] - #[diagnostic(transparent)] - WorkspaceTree(#[from] WorkspaceTreeError), #[error("error instantiating LuaRocks compatibility layer")] #[diagnostic(forward(0))] LuaRocks(#[from] crate::luarocks::luarocks_installation::LuaRocksError), @@ -138,7 +113,7 @@ impl From for Inst async fn install_impl(install: Install<'_, T>) -> Result, InstallError> where - T: InstallTree + Clone + Send + Sync + 'static, + T: InstallTree + Send + Sync, { let packages = install.packages; if packages.is_empty() { @@ -148,7 +123,6 @@ where let (packages, _lockfile) = InstallPackages::new(install.config, install.tree) .packages(packages) .maybe_package_db(install.package_db) - .maybe_behaviour(install.behaviour) .install() .await .map_err(InstallError::from)?; diff --git a/lux-lib/src/operations/install_dependencies.rs b/lux-lib/src/operations/install_dependencies.rs deleted file mode 100644 index 0126cd38c..000000000 --- a/lux-lib/src/operations/install_dependencies.rs +++ /dev/null @@ -1,117 +0,0 @@ -use bon::Builder; -use itertools::Itertools; - -use crate::{ - config::Config, - lockfile::LockedPackage, - lua_installation::LuaInstallation, - luarocks::luarocks_installation::LuaRocksInstallation, - operations::{pipeline::install_packages::InstallPackages, InstallError}, - project::project_toml::LocalProjectToml, - rockspec::Rockspec, - tree::{self, InstallTree}, -}; - -use super::PackageInstallSpec; - -#[derive(Builder)] -#[builder(start_fn = new, finish_fn(name = _build, vis = ""))] -pub(crate) struct InstallDependencies<'a, T> -where - T: InstallTree, -{ - dependencies: Vec, - build_dependencies: Vec, - tree: &'a T, - - lua: &'a LuaInstallation, - luarocks: &'a LuaRocksInstallation, - config: &'a Config, -} - -impl< - T: InstallTree + Sync + Send + Clone + 'static, - State: install_dependencies_builder::State + install_dependencies_builder::IsComplete, - > InstallDependenciesBuilder<'_, T, State> -{ - /// Installs the configured dependencies and build dependencies into [`Self::tree`], - /// returning the installed regular dependencies. - pub(crate) async fn build(self) -> Result, InstallError> { - let args = self._build(); - let config = args.config; - let dependencies = args.dependencies; - let build_dependencies = args.build_dependencies; - let tree = args.tree; - let build_tree = tree.build_tree(config)?; - let lua = args.lua; - let luarocks = args.luarocks; - if !build_dependencies.is_empty() { - luarocks.ensure_installed(lua).await?; - InstallPackages::new(config, build_tree.clone()) - .packages(build_dependencies.into_iter().unique().collect_vec()) - .install() - .await - .map_err(InstallError::from)?; - } - let (dependencies, _lockfile) = InstallPackages::new(config, (*tree).clone()) - .packages(dependencies.into_iter().unique().collect_vec()) - .install() - .await - .map_err(InstallError::from)?; - Ok(dependencies) - } -} - -pub(crate) fn prepare_dependencies_for_build( - project_toml: &LocalProjectToml, - workspace_tree: &impl InstallTree, - dependencies_to_install: &mut Vec, - build_dependencies_to_install: &mut Vec, - entry_type: tree::EntryType, -) { - let dependencies = project_toml - .dependencies() - .current_platform() - .iter() - .cloned() - .collect_vec(); - - let build_dependencies = project_toml - .build_dependencies() - .current_platform() - .iter() - .cloned() - .collect_vec(); - - dependencies - .into_iter() - .filter(|dep| { - workspace_tree - .match_rocks(dep.package_req()) - .is_ok_and(|rock_match| !rock_match.is_found()) - }) - .map(|dep| { - PackageInstallSpec::new(dep.clone().into_package_req(), entry_type) - .pin(*dep.pin()) - .opt(*dep.opt()) - .maybe_source(dep.source().clone()) - .build() - }) - .for_each(|dep| dependencies_to_install.push(dep)); - - build_dependencies - .into_iter() - .filter(|dep| { - workspace_tree - .match_rocks(dep.package_req()) - .is_ok_and(|rock_match| !rock_match.is_found()) - }) - .map(|dep| { - PackageInstallSpec::new(dep.clone().into_package_req(), entry_type) - .pin(*dep.pin()) - .opt(*dep.opt()) - .maybe_source(dep.source().clone()) - .build() - }) - .for_each(|dep| build_dependencies_to_install.push(dep)); -} diff --git a/lux-lib/src/operations/install_project.rs b/lux-lib/src/operations/install_project.rs index f7daed752..db9507a98 100644 --- a/lux-lib/src/operations/install_project.rs +++ b/lux-lib/src/operations/install_project.rs @@ -1,57 +1,46 @@ +use bon::Builder; +use miette::Diagnostic; +use thiserror::Error; + use crate::{ build::{BuildBehaviour, BuildError}, config::Config, lockfile::LockedPackage, - lua_installation::{LuaInstallation, LuaInstallationError}, - luarocks::luarocks_installation::{LuaRocksError, LuaRocksInstallError, LuaRocksInstallation}, - operations::{ - InstallDependencies, install_dependencies::prepare_dependencies_for_build, - pipeline::build_local::Build, - }, - project::{Project, ProjectError, project_toml::LocalProjectTomlValidationError}, + operations::{pipeline::discover::FoundPackage, Install, InstallError, PackageInstallSpec}, + package::{PackageName, PackageReq}, + project::{IntoLocalRockspecError, Project, ProjectError}, + remote_package_db::{RemotePackageDB, RemotePackageDBError}, + rockspec::Rockspec, tree::{self, InstallTree, TreeError}, }; -use bon::Builder; -use itertools::Itertools; -use miette::Diagnostic; -use thiserror::Error; - -use super::InstallError; #[derive(Debug, Error, Diagnostic)] pub enum InstallProjectError { #[error(transparent)] #[diagnostic(transparent)] - LocalProjectTomlValidation(#[from] LocalProjectTomlValidationError), + Project(#[from] ProjectError), #[error(transparent)] #[diagnostic(transparent)] - Project(#[from] ProjectError), + LocalRockspec(#[from] IntoLocalRockspecError), #[error(transparent)] #[diagnostic(transparent)] - LuaInstallation(#[from] LuaInstallationError), + RemotePackageDB(#[from] RemotePackageDBError), #[error(transparent)] #[diagnostic(transparent)] Tree(#[from] TreeError), #[error(transparent)] #[diagnostic(transparent)] - LuaRocks(#[from] LuaRocksError), + Install(#[from] Box), #[error(transparent)] #[diagnostic(transparent)] - LuaRocksInstall(#[from] Box), - #[error("error installind dependencies")] - #[diagnostic(forward(0))] - InstallDependencies(Box), - #[error("error installind build dependencies")] - #[diagnostic(forward(0))] - InstallBuildDependencies(Box), - #[error("error building project")] - #[diagnostic(forward(0))] Build(#[from] Box), + #[error("package '{0}' was not installed")] + PackageNotInstalled(PackageName), } -impl From for InstallProjectError { - fn from(source: LuaRocksInstallError) -> Self { - Self::LuaRocksInstall(Box::new(source)) +impl From for InstallProjectError { + fn from(source: InstallError) -> Self { + Self::Install(Box::new(source)) } } @@ -78,67 +67,38 @@ where } impl< - T: InstallTree + Sync + Send + Clone + 'static, - State: install_project_builder::State + install_project_builder::IsComplete, -> InstallProjectBuilder<'_, T, State> + T: InstallTree + Sync + Send, + State: install_project_builder::State + install_project_builder::IsComplete, + > InstallProjectBuilder<'_, T, State> { - /// Returns `Some` if the `only_deps` option is set to `false`. + /// Builds the project's root package, installing its dependencies through + /// the pipeline. Returns the installed root package. pub async fn build(self) -> Result { let args = self._build(); let config = args.config; let project = args.project; let tree = args.tree; - let build_tree = tree.build_tree(config)?; - let lua = LuaInstallation::new_from_config(config).await?; - let luarocks = LuaRocksInstallation::new(config, build_tree.clone())?; - let mut dependencies_to_install = Vec::new(); - let mut build_dependencies_to_install = Vec::new(); - let project_toml = project.toml().into_local()?; - prepare_dependencies_for_build( - &project_toml, - tree, - &mut dependencies_to_install, - &mut build_dependencies_to_install, - tree::EntryType::DependencyOnly, - ); - let dependencies = InstallDependencies::new() - .dependencies(dependencies_to_install.into_iter().unique().collect_vec()) - .build_dependencies( - build_dependencies_to_install - .into_iter() - .unique() - .collect_vec(), - ) - .tree(tree) - .lua(&lua) - .luarocks(&luarocks) - .config(config) - .build() - .await - .map_err(|err| InstallProjectError::InstallBuildDependencies(Box::new(err)))?; + let rockspec = project.local_remote_rockspec()?; + let name = rockspec.package().clone(); + let root = FoundPackage::from_project_root(rockspec, project.root().to_path_buf()); + let package_db = RemotePackageDB::from_config(config) + .await? + .with_local(vec![root]); - let package = Build::new() - .rockspec(&project_toml) - .lua(&lua) - .tree(tree) - .entry_type(tree::EntryType::Entrypoint) - .config(config) - .behaviour(BuildBehaviour::Force) - .build() - .await?; - - let lockfile = tree.lockfile()?; - let mut lockfile = lockfile.write_guard(); - let build_lockfile = tree.build_tree(config)?.lockfile()?; + let install_spec = + PackageInstallSpec::new(PackageReq::from(name.clone()), tree::EntryType::Entrypoint) + .build_behaviour(BuildBehaviour::Force) + .build(); - lockfile.add_entrypoint(&package); - for dep in dependencies { - lockfile.add_dependency(&package, &dep); - } - for dep in build_lockfile.rocks().values() { - lockfile.add_build_dependency(&package, dep); - } - Ok(package) + Install::new(config) + .package_db(package_db) + .package(install_spec) + .tree(tree) + .install() + .await? + .into_iter() + .find(|package| package.name() == &name) + .ok_or(InstallProjectError::PackageNotInstalled(name)) } } diff --git a/lux-lib/src/operations/install_rockspec.rs b/lux-lib/src/operations/install_rockspec.rs index d791ec550..a387ce1d4 100644 --- a/lux-lib/src/operations/install_rockspec.rs +++ b/lux-lib/src/operations/install_rockspec.rs @@ -9,7 +9,7 @@ use crate::{ lua_installation::LuaInstallationError, lua_rockspec::{LuaVersionError, RemoteLuaRockspec}, luarocks::luarocks_installation::{LuaRocksError, LuaRocksInstallError}, - operations::{Install, InstallError, PackageInstallSpec, pipeline::discover::FoundPackage}, + operations::{pipeline::discover::FoundPackage, Install, InstallError, PackageInstallSpec}, package::{PackageName, PackageReq}, remote_package_db::{RemotePackageDB, RemotePackageDBError}, rockspec::{LuaVersionCompatibility, Rockspec}, @@ -104,8 +104,7 @@ impl< Install::new(config) .package_db(package_db) .package(install_spec) - .behaviour(BuildBehaviour::Force) - .tree(tree.clone()) + .tree(tree) .install() .await? .into_iter() diff --git a/lux-lib/src/operations/mod.rs b/lux-lib/src/operations/mod.rs index dfe4e4ab1..e3137c5b7 100644 --- a/lux-lib/src/operations/mod.rs +++ b/lux-lib/src/operations/mod.rs @@ -9,7 +9,6 @@ mod fetch; mod fetch_vendored; mod gen_luarc; pub mod install; -mod install_dependencies; mod install_project; mod install_rockspec; mod pack; @@ -31,9 +30,8 @@ pub use download::*; pub use exec::*; pub use fetch::*; pub(crate) use fetch_vendored::*; -pub use gen_luarc::*; +pub(crate) use gen_luarc::*; pub use install::*; -pub(crate) use install_dependencies::*; pub use install_project::*; pub use install_rockspec::*; pub use pack::*; diff --git a/lux-lib/src/operations/pipeline/build.rs b/lux-lib/src/operations/pipeline/build.rs index 9080f1f60..dcc539c83 100644 --- a/lux-lib/src/operations/pipeline/build.rs +++ b/lux-lib/src/operations/pipeline/build.rs @@ -5,7 +5,7 @@ use miette::Diagnostic; use thiserror::Error; use crate::{ - build::{BuildBehaviour, deploy}, + build::deploy, config::Config, lockfile::{LockedPackage, LockedPackageId}, lua_installation::{LuaInstallation, LuaInstallationError}, @@ -49,19 +49,18 @@ impl From for BuildError { #[derive(Builder)] #[builder(start_fn = new, finish_fn(name = _build, vis = ""))] -pub(crate) struct Build<'a, T: InstallTree + Clone + Send + Sync> { +pub(crate) struct Build<'a, T: InstallTree + Send + Sync> { #[builder(start_fn)] pub(crate) config: &'a Config, #[builder(start_fn)] pub(crate) tree: &'a T, #[builder(field)] pub(crate) packages: Vec, - pub(crate) behaviour: Option, } impl BuildBuilder<'_, T, State> where - T: InstallTree + Clone + Send + Sync, + T: InstallTree + Send + Sync, State: build_builder::State, { pub(crate) fn packages(mut self, packages: Vec) -> Self { @@ -72,7 +71,7 @@ where impl BuildBuilder<'_, T, State> where - T: InstallTree + Clone + Send + Sync, + T: InstallTree + Send + Sync, State: build_builder::State + build_builder::IsComplete, { // INVESTIGATE(vhyrro): Is there a benefit of having a `LocalPackage` type which @@ -82,7 +81,6 @@ where // or a `LocalPackage` type. pub(crate) async fn build(self) -> Result, BuildError> { let args = self._build(); - let behaviour = args.behaviour.unwrap_or_default(); let lua = LuaInstallation::new_from_config(args.config).await?; let needs_luarocks = args.packages.iter().any(|package| { @@ -104,6 +102,7 @@ where package, rockspec, entry_type, + build_behaviour: package_behaviour, artifact, temp_dir: _temp_dir, } = downloaded; @@ -122,7 +121,7 @@ where &path, entry_type, args.config, - behaviour, + package_behaviour, ) .await .map_err(|err| BuildError::Build(rockspec.package().to_string(), Box::new(err)))?, @@ -137,7 +136,7 @@ where .pin(pin) .opt(opt) .constraint(constraint) - .behaviour(behaviour) + .behaviour(package_behaviour) .install() .await .map_err(|err| { diff --git a/lux-lib/src/operations/pipeline/build_local.rs b/lux-lib/src/operations/pipeline/build_local.rs index bd02dd66d..33f99bbf8 100644 --- a/lux-lib/src/operations/pipeline/build_local.rs +++ b/lux-lib/src/operations/pipeline/build_local.rs @@ -2,7 +2,7 @@ use bon::Builder; use tracing::Instrument; use crate::{ - build::{BuildBehaviour, BuildError, RemotePackageSourceSpec, deploy}, + build::{deploy, BuildBehaviour, BuildError, RemotePackageSourceSpec}, config::Config, hash::HasIntegrity, lockfile::{LockConstraint, LockedPackage, LockedPackageSpec, OptState, PinnedState}, diff --git a/lux-lib/src/operations/pipeline/build_project.rs b/lux-lib/src/operations/pipeline/build_project.rs deleted file mode 100644 index 62ee4948a..000000000 --- a/lux-lib/src/operations/pipeline/build_project.rs +++ /dev/null @@ -1,102 +0,0 @@ -use bon::Builder; -use miette::Diagnostic; -use thiserror::Error; - -use crate::{ - build::BuildBehaviour, - config::Config, - lockfile::LockedPackage, - lua_installation::{LuaInstallation, LuaInstallationError}, - package::PackageName, - project::project_toml::LocalProjectTomlValidationError, - rockspec::Rockspec, - tree::{EntryType, InstallTree, TreeError}, - workspace::{Workspace, WorkspaceTreeError}, -}; - -/// Builds a single workspace member into the workspace's install tree, recording -/// it as an entrypoint in the tree's lockfile. -#[derive(Builder)] -#[builder(start_fn = new, finish_fn(name = _build, vis = ""))] -pub(crate) struct BuildProject<'a> { - #[builder(start_fn)] - project: &'a crate::project::Project, - #[builder(start_fn)] - workspace: &'a Workspace, - #[builder(start_fn)] - config: &'a Config, - #[builder(start_fn)] - lua: &'a LuaInstallation, -} - -impl BuildProjectBuilder<'_, State> -where - State: build_project_builder::State + build_project_builder::IsComplete, -{ - pub(crate) async fn build(self) -> Result { - let args = self._build(); - - let workspace_tree = args.workspace.tree(args.config)?; - let project_toml = args.project.toml().into_local()?; - - let package = super::build_local::Build::new() - .rockspec(&project_toml) - .lua(args.lua) - .tree(&workspace_tree) - .entry_type(EntryType::Entrypoint) - .config(args.config) - .behaviour(BuildBehaviour::Force) - .build() - .await?; - - let lockfile = workspace_tree.lockfile()?; - let build_lockfile = args.workspace.build_tree(args.config)?.lockfile()?; - - let mut lockfile = lockfile.write_guard(); - lockfile.add_entrypoint(&package); - for dependency in project_toml.dependencies().current_platform() { - let dependency = lockfile - .entrypoint(dependency.name()) - .ok_or_else(|| BuildProjectError::MissingDependency(dependency.name().clone()))? - .clone(); - lockfile.add_dependency(&package, &dependency); - } - for dependency in project_toml.build_dependencies().current_platform() { - let dependency = build_lockfile - .entrypoint(dependency.name()) - .ok_or_else(|| BuildProjectError::MissingDependency(dependency.name().clone()))? - .clone(); - lockfile.add_build_dependency(&package, &dependency); - } - - Ok(package) - } -} - -#[derive(Error, Debug, Diagnostic)] -#[non_exhaustive] -pub enum BuildProjectError { - #[error("failed to build the project")] - #[diagnostic(forward(0))] - Build(#[source] Box), - #[error(transparent)] - #[diagnostic(transparent)] - LuaInstallation(#[from] LuaInstallationError), - #[error(transparent)] - #[diagnostic(transparent)] - Tree(#[from] TreeError), - #[error(transparent)] - #[diagnostic(transparent)] - WorkspaceTree(#[from] WorkspaceTreeError), - #[error(transparent)] - #[diagnostic(transparent)] - Project(#[from] LocalProjectTomlValidationError), - #[error("dependency `{0}` was not found in the lockfile")] - MissingDependency(PackageName), -} - -impl From for BuildProjectError { - fn from(source: crate::build::BuildError) -> Self { - Self::Build(Box::new(source)) - } -} diff --git a/lux-lib/src/operations/pipeline/discover.rs b/lux-lib/src/operations/pipeline/discover.rs index 7b3ad82ac..38e94aa77 100644 --- a/lux-lib/src/operations/pipeline/discover.rs +++ b/lux-lib/src/operations/pipeline/discover.rs @@ -9,6 +9,7 @@ use crate::{ remote_package_source::RemotePackageSource, rockspec::Rockspec, }; +use std::path::PathBuf; use super::download_sources_and_hash::PackageSource; @@ -38,6 +39,22 @@ impl FoundPackage { artifact: None, } } + + /// Builds a discovered package from a project root rockspec, using the + /// project's own directory as the source. The source is hashed and built in + /// place, so no download happens. + pub(crate) fn from_project_root(rockspec: RemoteLuaRockspec, root_dir: PathBuf) -> Self { + let package = RemotePackage::new( + PackageSpec::new(rockspec.package().clone(), rockspec.version().clone()), + RemotePackageSource::Local, + None, + ); + Self { + package, + rockspec, + artifact: Some(PackageSource::SourceDir(root_dir)), + } + } } #[derive(Error, Debug, Diagnostic)] diff --git a/lux-lib/src/operations/pipeline/download_sources_and_hash.rs b/lux-lib/src/operations/pipeline/download_sources_and_hash.rs index eb30d59dc..21861f4b0 100644 --- a/lux-lib/src/operations/pipeline/download_sources_and_hash.rs +++ b/lux-lib/src/operations/pipeline/download_sources_and_hash.rs @@ -20,15 +20,15 @@ use crate::{ RemotePackageSourceUrl, }, lua_rockspec::RemoteLuaRockspec, - operations::{FetchSrc, RemotePackageSourceMetadata, unpack_src_rock}, + operations::{unpack_src_rock, FetchSrc, RemotePackageSourceMetadata}, remote_package_source::RemotePackageSource, rockspec::Rockspec, tree::EntryType, }; use super::{ - Artifacts, resolve::{ResolvedArtifacts, ResolvedPackage}, + Artifacts, }; #[derive(Clone, Debug)] @@ -41,6 +41,7 @@ pub(crate) struct DownloadedPackage { pub(crate) package: LockedPackage, pub(crate) rockspec: RemoteLuaRockspec, pub(crate) entry_type: EntryType, + pub(crate) build_behaviour: crate::build::BuildBehaviour, pub(crate) artifact: PackageSource, pub(crate) temp_dir: Option, } @@ -136,6 +137,7 @@ async fn download_sources_and_hash( source, source_url, entry_type, + build_behaviour, artifact, } = resolved; @@ -218,6 +220,7 @@ async fn download_sources_and_hash( package, rockspec, entry_type, + build_behaviour, artifact, temp_dir, }) diff --git a/lux-lib/src/operations/pipeline/emit_lockfile.rs b/lux-lib/src/operations/pipeline/emit_lockfile.rs index e434b67ef..ab7a47f13 100644 --- a/lux-lib/src/operations/pipeline/emit_lockfile.rs +++ b/lux-lib/src/operations/pipeline/emit_lockfile.rs @@ -3,8 +3,8 @@ use std::collections::HashMap; use crate::lockfile::{LockedPackageId, LockedPackageLock, ReadWrite, WorkspaceLockfile}; use super::{ - Artifacts, download_sources_and_hash::{DownloadSourcesAndHashArtifacts, DownloadedPackage}, + Artifacts, }; /// The lockfile entries resolved by the pipeline, grouped by section. diff --git a/lux-lib/src/operations/pipeline/install_packages.rs b/lux-lib/src/operations/pipeline/install_packages.rs index 0de7154ac..59437c08d 100644 --- a/lux-lib/src/operations/pipeline/install_packages.rs +++ b/lux-lib/src/operations/pipeline/install_packages.rs @@ -37,24 +37,23 @@ use crate::operations::PackageInstallSpec; #[builder(start_fn = new, finish_fn(name = _build, vis = ""))] pub struct InstallPackages<'a, T> where - T: InstallTree + Clone + Send + Sync, + T: InstallTree + Send + Sync, { #[builder(start_fn)] config: &'a Config, #[builder(start_fn)] - tree: T, + tree: &'a T, #[builder(field)] build_packages: Vec, #[builder(field)] test_packages: Option>, packages: Vec, package_db: Option, - behaviour: Option, } impl<'a, T, State> InstallPackagesBuilder<'a, T, State> where - T: InstallTree + Clone + Send + Sync, + T: InstallTree + Send + Sync, State: install_packages_builder::State, { pub fn build_packages(mut self, packages: Vec) -> Self { @@ -71,7 +70,7 @@ where impl InstallPackagesBuilder<'_, T, State> where State: install_packages_builder::State + install_packages_builder::IsComplete, - T: InstallTree + Clone + Send + Sync + 'static, + T: InstallTree + Send + Sync, { pub async fn install( self, @@ -91,7 +90,7 @@ async fn install_packages( install: InstallPackages<'_, T>, ) -> Result<(Vec, LockfileHandle), InstallPackagesError> where - T: InstallTree + Clone + Send + Sync + 'static, + T: InstallTree + Send + Sync, { let package_db = match install.package_db { Some(db) => db, @@ -115,7 +114,7 @@ where let packages = install.packages; let config = install.config; - let tree = &install.tree; + let tree = install.tree; let lockfile = tree.lockfile()?; let build_tree = tree.build_tree(config)?; @@ -186,19 +185,16 @@ where // Build dependencies first, then the packages themselves. let built_build_deps = PipelineBuild::new(config, &build_tree) .packages(build_packages) - .maybe_behaviour(install.behaviour) .build() .await?; let built = PipelineBuild::new(config, tree) .packages(regular_packages) - .maybe_behaviour(install.behaviour) .build() .await?; if let Some(test_packages) = artifacts.test { let test_tree = tree.test_tree(config)?; PipelineBuild::new(config, &test_tree) .packages(test_packages.into_values().collect_vec()) - .maybe_behaviour(install.behaviour) .build() .await?; } diff --git a/lux-lib/src/operations/pipeline/install_workspace.rs b/lux-lib/src/operations/pipeline/install_workspace.rs index 3eca0fc54..aca4cc90a 100644 --- a/lux-lib/src/operations/pipeline/install_workspace.rs +++ b/lux-lib/src/operations/pipeline/install_workspace.rs @@ -58,7 +58,8 @@ where regular.extend(args.packages); let build = gather_dependencies(workspace, DependencyKind::Build)?; - let mut install = InstallPackages::new(config, workspace.tree(config)?) + let tree = workspace.tree(config)?; + let mut install = InstallPackages::new(config, &tree) .packages(regular) .build_packages(build); if args.test.unwrap_or(false) { @@ -165,15 +166,11 @@ mod tests { let lockfile = workspace.lockfile().unwrap(); let rocks = lockfile.rocks(&LockedPackageLockType::Regular); - assert!( - rocks - .values() - .any(|pkg| pkg.name().to_string() == "lua-cjson") - ); - assert!( - rocks - .values() - .any(|pkg| pkg.name().to_string() == "plenary.nvim") - ); + assert!(rocks + .values() + .any(|pkg| pkg.name().to_string() == "lua-cjson")); + assert!(rocks + .values() + .any(|pkg| pkg.name().to_string() == "plenary.nvim")); } } diff --git a/lux-lib/src/operations/pipeline/mod.rs b/lux-lib/src/operations/pipeline/mod.rs index c2af45eea..212807930 100644 --- a/lux-lib/src/operations/pipeline/mod.rs +++ b/lux-lib/src/operations/pipeline/mod.rs @@ -2,7 +2,6 @@ use crate::lockfile::LockedPackageLockType; pub mod build; pub mod build_local; -pub mod build_project; pub mod discover; pub mod download_sources_and_hash; pub mod emit_lockfile; diff --git a/lux-lib/src/operations/pipeline/resolve.rs b/lux-lib/src/operations/pipeline/resolve.rs index 41dd1fb03..c0addd5f4 100644 --- a/lux-lib/src/operations/pipeline/resolve.rs +++ b/lux-lib/src/operations/pipeline/resolve.rs @@ -23,9 +23,9 @@ use crate::{ }; use super::{ - Artifacts, discover::{DiscoverError, FoundPackage}, download_sources_and_hash::PackageSource, + Artifacts, }; /// The build dependencies of a rockspec that still need to be installed, with the @@ -68,6 +68,7 @@ pub(crate) struct ResolvedPackage { pub(crate) source: RemotePackageSource, pub(crate) source_url: Option, pub(crate) entry_type: EntryType, + pub(crate) build_behaviour: crate::build::BuildBehaviour, pub(crate) artifact: Option, } @@ -370,6 +371,7 @@ impl Resolver { source: found.package.source, source_url: found.package.source_url, entry_type: install_spec.entry_type, + build_behaviour: install_spec.build_behaviour, artifact: found.artifact, }; diff --git a/lux-lib/src/operations/sync.rs b/lux-lib/src/operations/sync.rs index 3969e911c..bfa48c069 100644 --- a/lux-lib/src/operations/sync.rs +++ b/lux-lib/src/operations/sync.rs @@ -221,7 +221,7 @@ async fn do_sync( Install::new(args.config) .package_db(package_db) .packages(packages_to_install) - .tree(tree.clone()) + .tree(&tree) .install() .await?; @@ -435,7 +435,7 @@ where let added = Install::new(args.config) .packages(missing_packages) - .tree(tree.clone()) + .tree(tree) .install() .await?; diff --git a/lux-lib/src/operations/test.rs b/lux-lib/src/operations/test.rs index 5013f0cb9..9af5d81c5 100644 --- a/lux-lib/src/operations/test.rs +++ b/lux-lib/src/operations/test.rs @@ -367,7 +367,7 @@ async fn ensure_test_dependencies( Install::new(config) .packages(test_dependencies) - .tree(test_tree) + .tree(&test_tree) .install() .await?; diff --git a/lux-lib/src/operations/update.rs b/lux-lib/src/operations/update.rs index 1924e24b9..2e50aa897 100644 --- a/lux-lib/src/operations/update.rs +++ b/lux-lib/src/operations/update.rs @@ -285,7 +285,7 @@ async fn update( .packages(updatable.iter().map(|(package, _)| package.id())) .remove() .await?; - let (updated_packages, _lockfile) = InstallPackages::new(config, tree) + let (updated_packages, _lockfile) = InstallPackages::new(config, &tree) .packages( updatable .iter() diff --git a/lux-lib/src/operations/vendor.rs b/lux-lib/src/operations/vendor.rs index eb5298b37..c75328c32 100644 --- a/lux-lib/src/operations/vendor.rs +++ b/lux-lib/src/operations/vendor.rs @@ -511,6 +511,7 @@ mod tests { use super::*; use crate::{ + build::BuildBehaviour, config::ConfigBuilder, lockfile::{LockConstraint, LockedPackage, LockedPackageSpec, OptState, PinnedState}, operations::unpack_rockspec, @@ -566,6 +567,7 @@ mod tests { package, rockspec, entry_type: EntryType::Entrypoint, + build_behaviour: BuildBehaviour::default(), artifact, temp_dir, } diff --git a/lux-lib/src/project/mod.rs b/lux-lib/src/project/mod.rs index 14ee361cd..06bcaec0e 100644 --- a/lux-lib/src/project/mod.rs +++ b/lux-lib/src/project/mod.rs @@ -285,6 +285,17 @@ impl Project { .map_err(|err| IntoLocalRockspecError::RockspecError(Box::new(err))) } + /// Returns this project's rockspec as a [`RemoteLuaRockspec`] whose source + /// points at the project's own directory, so it can be built through the + /// pipeline like any other package. + pub fn local_remote_rockspec(&self) -> Result { + self.toml() + .into_local() + .map_err(|err| IntoLocalRockspecError::LocalProjectTomlValidationError(Box::new(err)))? + .to_local_rockspec() + .map_err(|err| IntoLocalRockspecError::RockspecError(Box::new(err))) + } + pub fn remote_rockspec( &self, specrev: Option, diff --git a/lux-lib/src/project/project_toml.rs b/lux-lib/src/project/project_toml.rs index 9c1905d7a..bc853b172 100644 --- a/lux-lib/src/project/project_toml.rs +++ b/lux-lib/src/project/project_toml.rs @@ -25,11 +25,11 @@ use std::{collections::HashMap, path::PathBuf}; use crate::{ config::Config, lua_rockspec::{ - BuildSpec, BuildSpecInternal, BuildSpecInternalError, DisplayAsLuaKV, ExternalDependencies, - ExternalDependencySpec, LuaVersionError, PartialLuaRockspec, PerPlatform, - PlatformIdentifier, PlatformSupport, PlatformValidationError, RemoteRockSource, - RockDescription, RockSourceError, RockspecFormat, TestSpec, TestSpecDecodeError, - TestSpecInternal, + BuildSpec, BuildSpecInternal, BuildSpecInternalError, DisplayAsLuaKV, DisplayLuaKV, + ExternalDependencies, ExternalDependencySpec, LuaVersionError, PartialLuaRockspec, + PerPlatform, PlatformIdentifier, PlatformSupport, PlatformValidationError, + RemoteRockSource, RockDescription, RockSourceError, RockspecFormat, TestSpec, + TestSpecDecodeError, TestSpecInternal, }, package::{ BuildDependencies, Dependencies, PackageName, PackageReq, PackageVersion, PackageVersionReq, @@ -709,6 +709,103 @@ impl LocalProjectToml { ProjectRoot(self.internal.lux_toml_dir.clone()), ) } + + /// Renders the rockspec for local builds, using the rockspec's own version + /// and its source pointing at the project root. + pub fn to_local_rockspec(&self) -> Result { + Ok(LocalLuaRockspec::from_project_toml(self).into_remote_with_source(self.source.clone())) + } + + fn render_rockspec( + &self, + version: &PackageVersion, + source: Option, + ) -> Result { + let starter = format!( + r#" +rockspec_format = "{}" +package = "{}" +version = "{}""#, + self.rockspec_format + .as_ref() + .unwrap_or(&RockspecFormat::default()), + self.package, + version + ); + + let mut template = Vec::new(); + + if self.description != RockDescription::default() { + template.push(self.description.display_lua()); + } + + if self.supported_platforms != PlatformSupport::default() { + template.push(self.supported_platforms.display_lua()); + } + + { + let mut dependencies = self.internal.dependencies.clone().unwrap_or_default(); + dependencies.insert( + 0, + PackageReq { + name: "lua".into(), + version_req: self.lua.clone(), + } + .into(), + ); + template.push(Dependencies(&dependencies).display_lua()); + } + + let mut build_dependencies = self + .internal + .build_dependencies + .as_ref() + .cloned() + .unwrap_or_default(); + + let build_backend_dependency = self + .internal + .build + .build_type + .as_ref() + .and_then(|build_type| build_type.luarocks_build_backend()); + + if let Some(build_backend_dependency) = build_backend_dependency { + build_dependencies.push(build_backend_dependency); + } + + if !build_dependencies.is_empty() { + template.push(BuildDependencies(&build_dependencies).display_lua()); + } + + match self.internal.external_dependencies { + Some(ref external_dependencies) if !external_dependencies.is_empty() => { + template.push(ExternalDependencies(external_dependencies).display_lua()); + } + _ => {} + } + + if let Some(source) = source { + template.push(source); + } + + template.push(self.internal.build.display_lua()); + + let unformatted_code = std::iter::once(starter) + .chain(template.into_iter().map(|kv| kv.to_string())) + .join("\n\n"); + let result = match stylua_lib::format_code( + &unformatted_code, + stylua_lib::Config::default(), + None, + stylua_lib::OutputVerification::Full, + ) { + Ok(formatted_code) => formatted_code, + Err(_) => unformatted_code, + }; + validate_generated_lua(&result)?; + Ok(result) + } } impl Rockspec for LocalProjectToml { @@ -792,92 +889,11 @@ impl Rockspec for LocalProjectToml { .internal .version_template .try_generate(lux_toml_dir, None)?; - let starter = format!( - r#" -rockspec_format = "{}" -package = "{}" -version = "{}""#, - self.rockspec_format - .as_ref() - .unwrap_or(&RockspecFormat::default()), - self.package, - version - ); - - let mut template = Vec::new(); - - if self.description != RockDescription::default() { - template.push(self.description.display_lua()); - } - - if self.supported_platforms != PlatformSupport::default() { - template.push(self.supported_platforms.display_lua()); - } - - { - let mut dependencies = self.internal.dependencies.clone().unwrap_or_default(); - dependencies.insert( - 0, - PackageReq { - name: "lua".into(), - version_req: self.lua.clone(), - } - .into(), - ); - template.push(Dependencies(&dependencies).display_lua()); - } - - let mut build_dependencies = self - .internal - .build_dependencies - .as_ref() - .cloned() - .unwrap_or_default(); - - let build_backend_dependency = self - .internal - .build - .build_type - .as_ref() - .and_then(|build_type| build_type.luarocks_build_backend()); - - if let Some(build_backend_dependency) = build_backend_dependency { - build_dependencies.push(build_backend_dependency); - } - - if !build_dependencies.is_empty() { - template.push(BuildDependencies(&build_dependencies).display_lua()); - } - - match self.internal.external_dependencies { - Some(ref external_dependencies) if !external_dependencies.is_empty() => { - template.push(ExternalDependencies(external_dependencies).display_lua()); - } - _ => {} - } - let source = self.internal .source_template .try_generate(lux_toml_dir, &self.package, &version)?; - template.push(source.display_lua()); - - template.push(self.internal.build.display_lua()); - - let unformatted_code = std::iter::once(starter) - .chain(template.into_iter().map(|kv| kv.to_string())) - .join("\n\n"); - let result = match stylua_lib::format_code( - &unformatted_code, - stylua_lib::Config::default(), - None, - stylua_lib::OutputVerification::Full, - ) { - Ok(formatted_code) => formatted_code, - Err(_) => unformatted_code, - }; - validate_generated_lua(&result)?; - Ok(result) + self.render_rockspec(&version, Some(source.display_lua())) } } diff --git a/lux-lib/src/remote_package_db/mod.rs b/lux-lib/src/remote_package_db/mod.rs index d18680e42..a76063aac 100644 --- a/lux-lib/src/remote_package_db/mod.rs +++ b/lux-lib/src/remote_package_db/mod.rs @@ -3,11 +3,11 @@ use crate::{ lockfile::{LockedPackageLock, LockfileIntegrityError}, manifest::{Manifest, ManifestError}, operations::{ - Download, FetchVendored, PackageInstallSpec, RemoteRockDownload, pipeline::{ discover::{DiscoverError, FoundPackage}, download_sources_and_hash::PackageSource, }, + Download, FetchVendored, PackageInstallSpec, RemoteRockDownload, }, package::{ PackageName, PackageReq, PackageSpec, PackageVersion, RemotePackage, @@ -150,9 +150,11 @@ impl RemotePackageDB { config: &Config, ) -> Result { if let Some(source) = &spec.source { - let download = - RemoteRockDownload::from_package_req_and_source_spec(spec.package.clone(), source.clone()) - .map_err(|err| DiscoverError::Download(spec.package.clone(), Box::new(err)))?; + let download = RemoteRockDownload::from_package_req_and_source_spec( + spec.package.clone(), + source.clone(), + ) + .map_err(|err| DiscoverError::Download(spec.package.clone(), Box::new(err)))?; let rockspec = download.rockspec().clone(); let package = RemotePackage::new( spec.package.clone().try_into().map_err(|err| { diff --git a/lux-lib/tests/build.rs b/lux-lib/tests/build.rs index b02a301a2..0c876c11e 100644 --- a/lux-lib/tests/build.rs +++ b/lux-lib/tests/build.rs @@ -1,7 +1,7 @@ use std::path::PathBuf; -use assert_fs::TempDir; use assert_fs::prelude::PathCopy; +use assert_fs::TempDir; use flaky_test::flaky_test; use lux_lib::lua_version::LuaVersion; use lux_lib::rockspec::Rockspec; @@ -10,7 +10,7 @@ use lux_lib::workspace::Workspace; use lux_lib::{ build::BuildBehaviour::Force, config::ConfigBuilder, - lua_installation::{LuaInstallation, detect_installed_lua_version}, + lua_installation::{detect_installed_lua_version, LuaInstallation}, lua_rockspec::RemoteLuaRockspec, operations::pipeline::build_local::Build, tree, diff --git a/lux-lib/tests/install.rs b/lux-lib/tests/install.rs index a0c2eea88..9199f1655 100644 --- a/lux-lib/tests/install.rs +++ b/lux-lib/tests/install.rs @@ -73,7 +73,7 @@ async fn install_and_use(package: PackageReq, module_name: &str) { .unwrap(); let installed = Install::new(&config) .package(install_spec) - .tree(tree) + .tree(&tree) .install() .await .unwrap(); @@ -125,7 +125,7 @@ async fn test_install(install_spec: PackageInstallSpec) { .unwrap(); let installed = Install::new(&config) .package(install_spec) - .tree(tree) + .tree(&tree) .install() .await .unwrap(); @@ -155,7 +155,7 @@ async fn install_records_dependency_edges_in_tree_lockfile() { PackageInstallSpec::new("luassert@1.9.0-1".parse().unwrap(), EntryType::Entrypoint).build(); Install::new(&config) .package(install_spec) - .tree(tree.clone()) + .tree(&tree) .install() .await .unwrap(); From a75aa26dfc02de2c8b0485e4a3cf92f4f7144e10 Mon Sep 17 00:00:00 2001 From: vhyrro Date: Sun, 4 Oct 2026 21:19:12 +0200 Subject: [PATCH 21/39] fix: multiple small fixes and refactors --- lux-cli/src/dist/flat_archive.rs | 11 +- lux-cli/src/install.rs | 3 +- lux-cli/src/pack.rs | 12 +- lux-cli/src/uninstall.rs | 3 +- lux-lib/src/lockfile/mod.rs | 66 ++++----- lux-lib/src/lua_rockspec/mod.rs | 8 +- lux-lib/src/luarocks/install_binary_rock.rs | 1 - lux-lib/src/operations/exec.rs | 16 ++- lux-lib/src/operations/fetch.rs | 2 +- lux-lib/src/operations/install/mod.rs | 127 ------------------ lux-lib/src/operations/install_project.rs | 18 ++- lux-lib/src/operations/install_rockspec.rs | 18 ++- lux-lib/src/operations/pin.rs | 2 +- .../src/operations/pipeline/build_local.rs | 2 +- .../pipeline/download_sources_and_hash.rs | 2 - .../src/operations/pipeline/emit_lockfile.rs | 33 +++-- .../operations/pipeline/install_packages.rs | 23 +++- lux-lib/src/operations/sync.rs | 20 +-- lux-lib/src/operations/test.rs | 12 +- lux-lib/src/operations/update.rs | 14 +- lux-lib/src/remote_package_db/mod.rs | 8 +- lux-lib/tests/install.rs | 17 ++- lux-lua/src/operations.rs | 15 +-- 23 files changed, 165 insertions(+), 268 deletions(-) diff --git a/lux-cli/src/dist/flat_archive.rs b/lux-cli/src/dist/flat_archive.rs index aa3c7fa93..2d9605c57 100644 --- a/lux-cli/src/dist/flat_archive.rs +++ b/lux-cli/src/dist/flat_archive.rs @@ -11,7 +11,10 @@ use lux_lib::{ lua_installation::LuaInstallation, lua_rockspec::RemoteLuaRockspec, lua_version::LuaVersion, - operations::{pipeline::build_local::Build, Install, InstallProject, PackageInstallSpec}, + operations::{ + pipeline::{build_local::Build, install_packages::InstallPackages}, + InstallProject, PackageInstallSpec, + }, package::{PackageName, PackageReq}, tree::{self, FlatDistTree, InstallTree}, workspace::Workspace, @@ -158,15 +161,15 @@ async fn install_package( ) -> Result<(LockedPackage, PathBuf)> { let lua_version = LuaVersion::from(config)?.clone(); let tree = FlatDistTree::new(staging_dir.path().to_path_buf(), lua_version, config)?; - let packages = Install::new(config) + let packages = InstallPackages::new(config, &tree) .package( PackageInstallSpec::new(package.clone(), tree::EntryType::Entrypoint) .build_behaviour(BuildBehaviour::Force) .build(), ) - .tree(&tree) .install() - .await?; + .await? + .0; let package = packages .into_iter() .find(|pkg| pkg.name() == package.name()) diff --git a/lux-cli/src/install.rs b/lux-cli/src/install.rs index d123e57b3..5581a8854 100644 --- a/lux-cli/src/install.rs +++ b/lux-cli/src/install.rs @@ -77,9 +77,8 @@ async fn install_remote(data: Install, config: Config) -> Result<()> { let packages = apply_build_behaviour(data.package_req, pin, data.force, &tree, &config)?; // TODO(vhyrro): If the tree doesn't exist then error out. - operations::Install::new(&config) + operations::pipeline::install_packages::InstallPackages::new(&config, &tree) .packages(packages) - .tree(&tree) .install() .await?; diff --git a/lux-cli/src/pack.rs b/lux-cli/src/pack.rs index 16a8db89a..d995f1f54 100644 --- a/lux-cli/src/pack.rs +++ b/lux-cli/src/pack.rs @@ -9,7 +9,11 @@ use lux_lib::{ lua_installation::LuaInstallation, lua_rockspec::RemoteLuaRockspec, lua_version::LuaVersion, - operations::{self, pipeline::build_local::Build, Install, PackageInstallSpec}, + operations::{ + self, + pipeline::{build_local::Build, install_packages::InstallPackages}, + PackageInstallSpec, + }, package::PackageName, rockspec::Rockspec as _, tree::{self, InstallTree}, @@ -107,15 +111,15 @@ pub async fn pack(args: Pack, config: Config) -> Result<()> { let temp_dir = tempdir().into_diagnostic()?; let temp_config = config.with_tree(temp_dir.path().to_path_buf()); let tree = temp_config.user_tree(lua_version.clone())?; - let packages = Install::new(&temp_config) + let packages = InstallPackages::new(&temp_config, &tree) .package( PackageInstallSpec::new(package_req, tree::EntryType::Entrypoint) .build_behaviour(BuildBehaviour::Force) .build(), ) - .tree(&tree) .install() - .await?; + .await? + .0; let package = packages .first() .ok_or_else(|| miette!("no packages installed"))?; diff --git a/lux-cli/src/uninstall.rs b/lux-cli/src/uninstall.rs index 727ac2b72..1aa40a9ad 100644 --- a/lux-cli/src/uninstall.rs +++ b/lux-cli/src/uninstall.rs @@ -151,9 +151,8 @@ Reinstall? .packages(dependencies) .remove() .await?; - operations::Install::new(&config) + operations::pipeline::install_packages::InstallPackages::new(&config, &tree) .packages(reinstall_specs) - .tree(&tree) .install() .await?; } else { diff --git a/lux-lib/src/lockfile/mod.rs b/lux-lib/src/lockfile/mod.rs index 2ce6687c2..cc368c670 100644 --- a/lux-lib/src/lockfile/mod.rs +++ b/lux-lib/src/lockfile/mod.rs @@ -355,11 +355,7 @@ impl Display for RemotePackageSourceUrl { // TODO(vhyrro): Move to `package/local.rs` -/// A package whose source has been downloaded and hashed, but which has not -/// yet been installed into a tree. -/// -/// This is the value produced by the download stage. Installation adds the -/// binaries discovered during the build, producing a [`LockedPackage`]. +/// A package whose source has been downloaded, hashed and which is installed in a tree. #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct LockedPackage { pub(crate) spec: LockedPackageSpec, @@ -369,11 +365,7 @@ pub struct LockedPackage { } impl LockedPackage { - /// Returns a copy of this package with a different pin state. - /// - /// Pin state is part of the package id, so the result has a different - /// identity and must be re-inserted under its new id. - pub(crate) fn rekey_pinned(mut self, pinned: PinnedState) -> Self { + pub(crate) fn repin(mut self, pinned: PinnedState) -> Self { self.spec.pinned = pinned; self } @@ -659,7 +651,7 @@ impl LockfilePermissions for ReadOnly {} impl LockfilePermissions for ReadWrite {} #[derive(Clone, Debug, Serialize, Deserialize, Default)] -pub(crate) struct LockedPackageLock { +pub(crate) struct PackageLock { // NOTE: We cannot directly serialize to a `Sha256` object as they don't implement serde traits. // NOTE: We want to retain ordering of rocks when de/serializing. rocks: BTreeMap, @@ -668,13 +660,21 @@ pub(crate) struct LockedPackageLock { entrypoints: BTreeMap, } -impl LockedPackageLock { - pub(crate) fn insert(&mut self, package: LockedPackage, entrypoint: bool) { - if entrypoint { - self.entrypoints - .insert(package.name().clone(), package.id()); +impl PackageLock { + pub(crate) fn from_packages( + packages: impl Iterator, + ) -> Self { + let mut lock = Self::default(); + + for (package, entry_type) in packages { + if matches!(entry_type, EntryType::Entrypoint) { + lock.entrypoints + .insert(package.name().clone(), package.id()); + } + lock.rocks.insert(package.id(), package); } - self.rocks.insert(package.id(), package); + + lock } fn get(&self, id: &LockedPackageId) -> Option<&LockedPackage> { @@ -856,7 +856,7 @@ pub struct Lockfile { #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] binaries: BTreeMap, #[serde(flatten)] - lock: LockedPackageLock, + lock: PackageLock, } #[derive(EnumIter, Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -874,12 +874,12 @@ pub struct WorkspaceLockfile { #[serde(skip)] _marker: PhantomData

, version: String, - #[serde(default, skip_serializing_if = "LockedPackageLock::is_empty")] - dependencies: LockedPackageLock, - #[serde(default, skip_serializing_if = "LockedPackageLock::is_empty")] - test_dependencies: LockedPackageLock, - #[serde(default, skip_serializing_if = "LockedPackageLock::is_empty")] - build_dependencies: LockedPackageLock, + #[serde(default, skip_serializing_if = "PackageLock::is_empty")] + dependencies: PackageLock, + #[serde(default, skip_serializing_if = "PackageLock::is_empty")] + test_dependencies: PackageLock, + #[serde(default, skip_serializing_if = "PackageLock::is_empty")] + build_dependencies: PackageLock, } #[derive(Error, Debug, Diagnostic)] @@ -1025,7 +1025,7 @@ impl Lockfile

{ self.binaries.get(id) } - pub(crate) fn local_pkg_lock(&self) -> &LockedPackageLock { + pub(crate) fn local_pkg_lock(&self) -> &PackageLock { &self.lock } @@ -1184,7 +1184,7 @@ impl WorkspaceLockfile

{ } } - pub(crate) fn local_pkg_lock(&self, deps: &LockedPackageLockType) -> &LockedPackageLock { + pub(crate) fn local_pkg_lock(&self, deps: &LockedPackageLockType) -> &PackageLock { match deps { LockedPackageLockType::Regular => &self.dependencies, LockedPackageLockType::Test => &self.test_dependencies, @@ -1192,7 +1192,7 @@ impl WorkspaceLockfile

{ } } - pub(crate) fn local_pkg_locks(&self) -> Vec { + pub(crate) fn local_pkg_locks(&self) -> Vec { vec![ self.dependencies.clone(), self.test_dependencies.clone(), @@ -1221,7 +1221,7 @@ impl Lockfile { _marker: PhantomData, version: LOCKFILE_VERSION_STR.into(), binaries: BTreeMap::new(), - lock: LockedPackageLock::default(), + lock: PackageLock::default(), }; let json_str = serde_json::to_string(&empty_lockfile).map_err(LockfileError::WriteJson)?; @@ -1326,9 +1326,9 @@ impl WorkspaceLockfile { filepath: filepath.clone(), _marker: PhantomData, version: LOCKFILE_VERSION_STR.into(), - dependencies: LockedPackageLock::default(), - test_dependencies: LockedPackageLock::default(), - build_dependencies: LockedPackageLock::default(), + dependencies: PackageLock::default(), + test_dependencies: PackageLock::default(), + build_dependencies: PackageLock::default(), }; let json_str = serde_json::to_string(&empty_lockfile).map_err(LockfileError::WriteJson)?; @@ -1461,7 +1461,7 @@ impl Lockfile { self.lock.remove_by_id(target) } - pub(crate) fn sync(&mut self, lock: &LockedPackageLock) { + pub(crate) fn sync(&mut self, lock: &PackageLock) { self.lock = lock.clone(); } @@ -1477,7 +1477,7 @@ impl WorkspaceLockfile { } } - pub(crate) fn sync(&mut self, lock: &LockedPackageLock, deps: &LockedPackageLockType) { + pub(crate) fn sync(&mut self, lock: &PackageLock, deps: &LockedPackageLockType) { match deps { LockedPackageLockType::Regular => { self.dependencies = lock.clone(); diff --git a/lux-lib/src/lua_rockspec/mod.rs b/lux-lib/src/lua_rockspec/mod.rs index 30173d525..89f24fc4d 100644 --- a/lux-lib/src/lua_rockspec/mod.rs +++ b/lux-lib/src/lua_rockspec/mod.rs @@ -207,10 +207,7 @@ impl<'gc> HasRockspecKey<'gc> for ottavino::Table<'gc> { impl LocalLuaRockspec { /// Builds a [`LocalLuaRockspec`] from an already-parsed project. - /// - /// Unlike [`crate::project::project_toml::LocalProjectToml::to_lua_rockspec`], - /// this does not round-trip through a rendered rockspec string, so it - /// preserves path dependencies and cannot fail. + /// FIXME(vhyrro): investigate if necessary or can be deduplicated pub(crate) fn from_project_toml( project: &crate::project::project_toml::LocalProjectToml, ) -> Self { @@ -237,9 +234,6 @@ impl LocalLuaRockspec { /// Wraps this rockspec into a [`RemoteLuaRockspec`], overriding its source /// with the given `source`. - /// - /// Used to build a project's root package through the pipeline, pointing - /// the source at the project directory itself. pub fn into_remote_with_source( self, source: PerPlatform, diff --git a/lux-lib/src/luarocks/install_binary_rock.rs b/lux-lib/src/luarocks/install_binary_rock.rs index 977adc203..5cc98cd0f 100644 --- a/lux-lib/src/luarocks/install_binary_rock.rs +++ b/lux-lib/src/luarocks/install_binary_rock.rs @@ -303,7 +303,6 @@ mod tests { .install() .await .unwrap(); - // The `source` integrity of a binary rock is the hash of the packed rock bytes. assert_eq!( local_package.hashes().source, "sha256-zDmnjE2b8GxX+6j5vTCWUEBxe0aFXFVhC4JOrnAgQ2g=" diff --git a/lux-lib/src/operations/exec.rs b/lux-lib/src/operations/exec.rs index db39fa738..8e20badfe 100644 --- a/lux-lib/src/operations/exec.rs +++ b/lux-lib/src/operations/exec.rs @@ -5,7 +5,10 @@ use crate::{ config::Config, lua_rockspec::LuaVersionError, lua_version::{LuaVersion, LuaVersionUnset}, - operations::{BuildWorkspace, BuildWorkspaceError, Install}, + operations::{ + pipeline::install_packages::{InstallPackages, InstallPackagesError}, + BuildWorkspace, BuildWorkspaceError, + }, package::{PackageReq, PackageVersionReqError}, path::{Paths, PathsError}, remote_package_db::RemotePackageDBError, @@ -18,7 +21,7 @@ use miette::Diagnostic; use thiserror::Error; use which::which; -use super::{InstallError, PackageInstallSpec}; +use super::PackageInstallSpec; /// Rocks package runner, providing fine-grained control /// over how a package should be run. @@ -117,15 +120,15 @@ impl From for ExecError { #[derive(Error, Debug, Diagnostic)] #[error(transparent)] pub enum InstallCommandError { - InstallError(#[from] Box), + InstallError(#[from] Box), PackageVersionReqError(#[from] PackageVersionReqError), RemotePackageDBError(#[from] RemotePackageDBError), Tree(#[from] TreeError), LuaVersionUnset(#[from] LuaVersionUnset), } -impl From for InstallCommandError { - fn from(source: InstallError) -> Self { +impl From for InstallCommandError { + fn from(source: InstallPackagesError) -> Self { Self::InstallError(Box::new(source)) } } @@ -226,9 +229,8 @@ async fn install_command(command: &str, config: &Config) -> Result<(), InstallCo ) .build(); let tree = config.user_tree(LuaVersion::from(config)?.clone())?; - Install::new(config) + InstallPackages::new(config, &tree) .package(install_spec) - .tree(&tree) .install() .await?; Ok(()) diff --git a/lux-lib/src/operations/fetch.rs b/lux-lib/src/operations/fetch.rs index 06a501f01..23a128e1f 100644 --- a/lux-lib/src/operations/fetch.rs +++ b/lux-lib/src/operations/fetch.rs @@ -118,7 +118,7 @@ pub enum FetchSrcError { #[diagnostic(help("check your network connection and verify the git URL is correct."))] GitClone(#[from] git2::Error), #[error("git operation failed")] - #[diagnostic(help("this is a bug; please report it."))] + #[diagnostic(help("this is a bug, please report it."))] GitJoin(#[from] tokio::task::JoinError), #[error("failed to parse git URL")] #[diagnostic(forward(0))] diff --git a/lux-lib/src/operations/install/mod.rs b/lux-lib/src/operations/install/mod.rs index 58ec33936..2a6ed0923 100644 --- a/lux-lib/src/operations/install/mod.rs +++ b/lux-lib/src/operations/install/mod.rs @@ -1,130 +1,3 @@ -use crate::{ - config::Config, lockfile::LockedPackage, remote_package_db::RemotePackageDB, tree::InstallTree, -}; - pub use crate::operations::install::spec::PackageInstallSpec; -use bon::Builder; -use miette::Diagnostic; -use thiserror::Error; - -use tracing::Instrument; - -use super::pipeline::install_packages::{InstallPackages, InstallPackagesError}; pub mod spec; - -/// A rocks package installer, providing fine-grained control -/// over how packages should be installed. -#[derive(Builder)] -#[builder(start_fn = new, finish_fn(name = _build, vis = ""))] -pub struct Install<'a, T> -where - T: InstallTree + Send + Sync, -{ - #[builder(start_fn)] - config: &'a Config, - #[builder(field)] - packages: Vec, - #[builder(setters(name = "_tree", vis = ""))] - tree: &'a T, - package_db: Option, -} - -impl<'a, T, State> InstallBuilder<'a, T, State> -where - State: install_builder::State, - T: InstallTree + Send + Sync, -{ - pub fn tree(self, tree: &'a T) -> InstallBuilder<'a, T, install_builder::SetTree> - where - State::Tree: install_builder::IsUnset, - { - self._tree(tree) - } - - pub fn packages(self, packages: Vec) -> Self { - Self { packages, ..self } - } - - pub fn package(self, package: PackageInstallSpec) -> Self { - Self { - packages: self - .packages - .into_iter() - .chain(std::iter::once(package)) - .collect(), - ..self - } - } -} - -impl InstallBuilder<'_, T, State> -where - State: install_builder::State + install_builder::IsComplete, - T: InstallTree + Send + Sync, -{ - /// Install the packages. - pub async fn install(self) -> Result, InstallError> { - let install_built = self._build(); - if install_built.packages.is_empty() { - return Ok(Vec::default()); - } - let count = install_built.packages.len(); - let span = if count > 1 { - tracing::info_span!("Installing", count) - } else { - let install_spec = &install_built.packages[0]; - tracing::info_span!("Installing", package = install_spec.package.to_string()) - }; - install_impl(install_built).instrument(span).await - } -} - -#[derive(Error, Debug, Diagnostic)] -pub enum InstallError { - #[error(transparent)] - #[diagnostic(transparent)] - LuaVersionUnset(#[from] crate::lua_version::LuaVersionUnset), - #[error(transparent)] - #[diagnostic(transparent)] - LuaInstallation(#[from] crate::lua_installation::LuaInstallationError), - #[error(transparent)] - #[diagnostic(transparent)] - FlushLockfile(#[from] crate::lockfile::FlushLockfileError), - #[error(transparent)] - #[diagnostic(transparent)] - Tree(#[from] crate::tree::TreeError), - #[error("error instantiating LuaRocks compatibility layer")] - #[diagnostic(forward(0))] - LuaRocks(#[from] crate::luarocks::luarocks_installation::LuaRocksError), - #[error("error installing LuaRocks compatibility layer")] - #[diagnostic(forward(0))] - LuaRocksInstall(#[from] Box), - #[error(transparent)] - #[diagnostic(transparent)] - Install(#[from] InstallPackagesError), -} - -impl From for InstallError { - fn from(source: crate::luarocks::luarocks_installation::LuaRocksInstallError) -> Self { - Self::LuaRocksInstall(Box::new(source)) - } -} - -async fn install_impl(install: Install<'_, T>) -> Result, InstallError> -where - T: InstallTree + Send + Sync, -{ - let packages = install.packages; - if packages.is_empty() { - return Ok(Vec::new()); - } - - let (packages, _lockfile) = InstallPackages::new(install.config, install.tree) - .packages(packages) - .maybe_package_db(install.package_db) - .install() - .await - .map_err(InstallError::from)?; - Ok(packages) -} diff --git a/lux-lib/src/operations/install_project.rs b/lux-lib/src/operations/install_project.rs index db9507a98..d04758d42 100644 --- a/lux-lib/src/operations/install_project.rs +++ b/lux-lib/src/operations/install_project.rs @@ -6,7 +6,13 @@ use crate::{ build::{BuildBehaviour, BuildError}, config::Config, lockfile::LockedPackage, - operations::{pipeline::discover::FoundPackage, Install, InstallError, PackageInstallSpec}, + operations::{ + pipeline::{ + discover::FoundPackage, + install_packages::{InstallPackages, InstallPackagesError}, + }, + PackageInstallSpec, + }, package::{PackageName, PackageReq}, project::{IntoLocalRockspecError, Project, ProjectError}, remote_package_db::{RemotePackageDB, RemotePackageDBError}, @@ -30,7 +36,7 @@ pub enum InstallProjectError { Tree(#[from] TreeError), #[error(transparent)] #[diagnostic(transparent)] - Install(#[from] Box), + Install(#[from] Box), #[error(transparent)] #[diagnostic(transparent)] Build(#[from] Box), @@ -38,8 +44,8 @@ pub enum InstallProjectError { PackageNotInstalled(PackageName), } -impl From for InstallProjectError { - fn from(source: InstallError) -> Self { +impl From for InstallProjectError { + fn from(source: InstallPackagesError) -> Self { Self::Install(Box::new(source)) } } @@ -91,12 +97,12 @@ impl< .build_behaviour(BuildBehaviour::Force) .build(); - Install::new(config) + InstallPackages::new(config, tree) .package_db(package_db) .package(install_spec) - .tree(tree) .install() .await? + .0 .into_iter() .find(|package| package.name() == &name) .ok_or(InstallProjectError::PackageNotInstalled(name)) diff --git a/lux-lib/src/operations/install_rockspec.rs b/lux-lib/src/operations/install_rockspec.rs index a387ce1d4..565e5fe6a 100644 --- a/lux-lib/src/operations/install_rockspec.rs +++ b/lux-lib/src/operations/install_rockspec.rs @@ -9,7 +9,13 @@ use crate::{ lua_installation::LuaInstallationError, lua_rockspec::{LuaVersionError, RemoteLuaRockspec}, luarocks::luarocks_installation::{LuaRocksError, LuaRocksInstallError}, - operations::{pipeline::discover::FoundPackage, Install, InstallError, PackageInstallSpec}, + operations::{ + pipeline::{ + discover::FoundPackage, + install_packages::{InstallPackages, InstallPackagesError}, + }, + PackageInstallSpec, + }, package::{PackageName, PackageReq}, remote_package_db::{RemotePackageDB, RemotePackageDBError}, rockspec::{LuaVersionCompatibility, Rockspec}, @@ -29,7 +35,7 @@ pub enum InstallRockspecError { #[diagnostic(transparent)] Tree(#[from] TreeError), #[diagnostic(transparent)] - Install(#[from] Box), + Install(#[from] Box), #[diagnostic(transparent)] LuaRocks(#[from] LuaRocksError), #[diagnostic(transparent)] @@ -40,8 +46,8 @@ pub enum InstallRockspecError { PackageNotInstalled(PackageName), } -impl From for InstallRockspecError { - fn from(source: InstallError) -> Self { +impl From for InstallRockspecError { + fn from(source: InstallPackagesError) -> Self { Self::Install(Box::new(source)) } } @@ -101,12 +107,12 @@ impl< .opt(OptState::Required) .build(); - Install::new(config) + InstallPackages::new(config, tree) .package_db(package_db) .package(install_spec) - .tree(tree) .install() .await? + .0 .into_iter() .find(|package| package.name() == &name) .ok_or(InstallRockspecError::PackageNotInstalled(name)) diff --git a/lux-lib/src/operations/pin.rs b/lux-lib/src/operations/pin.rs index 78540023d..3f6a1834d 100644 --- a/lux-lib/src/operations/pin.rs +++ b/lux-lib/src/operations/pin.rs @@ -78,7 +78,7 @@ pub fn set_pinned_state( .map(|dir| dir.path()) .collect_vec(); - let package = package.rekey_pinned(pin); + let package = package.repin(pin); if lockfile.get(&package.id()).is_some() { return Err(PinError::PinStateConflict { diff --git a/lux-lib/src/operations/pipeline/build_local.rs b/lux-lib/src/operations/pipeline/build_local.rs index 33f99bbf8..c8fd72a5e 100644 --- a/lux-lib/src/operations/pipeline/build_local.rs +++ b/lux-lib/src/operations/pipeline/build_local.rs @@ -68,7 +68,6 @@ where rockspec.validate_lua_version(&lua.version)?; - // NOTE: these futures are large; boxing them keeps `do_build`'s frame small. let spec = LockedPackageSpec::new( rockspec.package(), rockspec.version(), @@ -78,6 +77,7 @@ where &build.pin, &build.opt, ); + // Pin the futures otherwise they fill up the stack quite a bit let (package, temp_dir) = Box::pin(fetch_and_hash_source( rockspec, spec, diff --git a/lux-lib/src/operations/pipeline/download_sources_and_hash.rs b/lux-lib/src/operations/pipeline/download_sources_and_hash.rs index 21861f4b0..c04a0033a 100644 --- a/lux-lib/src/operations/pipeline/download_sources_and_hash.rs +++ b/lux-lib/src/operations/pipeline/download_sources_and_hash.rs @@ -237,8 +237,6 @@ pub(crate) async fn fetch_and_hash_source( let source_metadata = match source_spec { Some(RemotePackageSourceSpec::SrcRock(SrcRockSource { bytes, source_url })) => { - // FIXME(vhyrro): This shouldn't be in this function, or the function should be named differently. - // "fetching" has nothing to do with unpacking. let hash = bytes.hash().await?; let cursor = Cursor::new(bytes); unpack_src_rock(cursor, temp_dir.path().to_path_buf()) diff --git a/lux-lib/src/operations/pipeline/emit_lockfile.rs b/lux-lib/src/operations/pipeline/emit_lockfile.rs index ab7a47f13..5e8a608e1 100644 --- a/lux-lib/src/operations/pipeline/emit_lockfile.rs +++ b/lux-lib/src/operations/pipeline/emit_lockfile.rs @@ -1,22 +1,28 @@ -use std::collections::HashMap; +use crate::lockfile::{PackageLock, ReadWrite, WorkspaceLockfile}; -use crate::lockfile::{LockedPackageId, LockedPackageLock, ReadWrite, WorkspaceLockfile}; - -use super::{ - download_sources_and_hash::{DownloadSourcesAndHashArtifacts, DownloadedPackage}, - Artifacts, -}; +use super::{download_sources_and_hash::DownloadSourcesAndHashArtifacts, Artifacts}; /// The lockfile entries resolved by the pipeline, grouped by section. #[derive(Default)] -pub struct LockfileHandle(Artifacts); +pub struct LockfileHandle(Artifacts); impl LockfileHandle { pub(crate) fn from_artifacts(artifacts: &DownloadSourcesAndHashArtifacts) -> Self { Self( artifacts .iter() - .map(|(section, packages)| (section, packages.map(lock_from))) + .map(|(section, packages)| { + ( + section, + packages.map(|packages| { + PackageLock::from_packages( + packages + .values() + .map(|pkg| (pkg.package.clone(), pkg.entry_type)), + ) + }), + ) + }) .collect(), ) } @@ -29,12 +35,3 @@ impl LockfileHandle { }); } } - -fn lock_from(packages: &HashMap) -> LockedPackageLock { - // FIXME(vhyrro): Create a constructor here instead of mut overrides. - let mut lock = LockedPackageLock::default(); - for package in packages.values() { - lock.insert(package.package.clone(), package.entry_type.is_entrypoint()); - } - lock -} diff --git a/lux-lib/src/operations/pipeline/install_packages.rs b/lux-lib/src/operations/pipeline/install_packages.rs index 59437c08d..4ec5a4025 100644 --- a/lux-lib/src/operations/pipeline/install_packages.rs +++ b/lux-lib/src/operations/pipeline/install_packages.rs @@ -4,6 +4,7 @@ use bon::Builder; use itertools::Itertools; use miette::Diagnostic; use thiserror::Error; +use tracing::Instrument; use crate::{ build::BuildBehaviour, @@ -28,7 +29,7 @@ use crate::operations::PackageInstallSpec; /// Installs a set of [`PackageInstallSpec`]s into an install tree. /// -/// This is the composable core of [`Install`](crate::operations::Install): it resolves the +/// This is the composable core of package installation: it resolves the /// dependency graph, downloads and hashes sources, builds the packages (build dependencies /// first) into the regular, build and test trees, and records the result in the trees' /// lockfiles. It also returns a [`LockfileHandle`] so callers (e.g. a workspace) can commit @@ -47,6 +48,7 @@ where build_packages: Vec, #[builder(field)] test_packages: Option>, + #[builder(field)] packages: Vec, package_db: Option, } @@ -56,6 +58,16 @@ where T: InstallTree + Send + Sync, State: install_packages_builder::State, { + pub fn packages(mut self, packages: Vec) -> Self { + self.packages = packages; + self + } + + pub fn package(mut self, package: PackageInstallSpec) -> Self { + self.packages.push(package); + self + } + pub fn build_packages(mut self, packages: Vec) -> Self { self.build_packages = packages; self @@ -82,7 +94,14 @@ where { return Ok((Vec::new(), LockfileHandle::default())); } - install_packages(args).await + let span = match args.packages.as_slice() { + [] => tracing::info_span!("Installing"), + [install_spec] => { + tracing::info_span!("Installing", package = install_spec.package.to_string()) + } + packages => tracing::info_span!("Installing", count = packages.len()), + }; + install_packages(args).instrument(span).await } } diff --git a/lux-lib/src/operations/sync.rs b/lux-lib/src/operations/sync.rs index bfa48c069..5e09e6461 100644 --- a/lux-lib/src/operations/sync.rs +++ b/lux-lib/src/operations/sync.rs @@ -1,6 +1,9 @@ use std::io; -use super::{Install, InstallError, PackageInstallSpec, RemoveError, Uninstall}; +use super::{ + pipeline::install_packages::{InstallPackages, InstallPackagesError}, + PackageInstallSpec, RemoveError, Uninstall, +}; use crate::{ build::BuildBehaviour, config::Config, @@ -145,7 +148,7 @@ pub enum SyncError { Tree(#[from] TreeError), #[error(transparent)] #[diagnostic(transparent)] - Install(#[from] Box), + Install(#[from] Box), #[error(transparent)] #[diagnostic(transparent)] Remove(#[from] RemoveError), @@ -172,8 +175,8 @@ pub enum SyncError { GenLuaRc(#[from] GenLuaRcError), } -impl From for SyncError { - fn from(source: InstallError) -> Self { +impl From for SyncError { + fn from(source: InstallPackagesError) -> Self { Self::Install(Box::new(source)) } } @@ -218,10 +221,9 @@ async fn do_sync( .added .extend(to_add.iter().map(|(_, pkg)| pkg).cloned()); - Install::new(args.config) + InstallPackages::new(args.config, &tree) .package_db(package_db) .packages(packages_to_install) - .tree(&tree) .install() .await?; @@ -433,11 +435,11 @@ where .unique() .collect(); - let added = Install::new(args.config) + let added = InstallPackages::new(args.config, tree) .packages(missing_packages) - .tree(tree) .install() - .await?; + .await? + .0; report.added.extend(added); diff --git a/lux-lib/src/operations/test.rs b/lux-lib/src/operations/test.rs index 9af5d81c5..60c492ca4 100644 --- a/lux-lib/src/operations/test.rs +++ b/lux-lib/src/operations/test.rs @@ -1,7 +1,8 @@ use std::{io, ops::Deref, path::PathBuf, process::Command}; use super::{ - BuildWorkspace, BuildWorkspaceError, Install, InstallError, PackageInstallSpec, Sync, SyncError, + pipeline::install_packages::{InstallPackages, InstallPackagesError}, + BuildWorkspace, BuildWorkspaceError, PackageInstallSpec, Sync, SyncError, }; use crate::fs; use crate::tree::InstallTree; @@ -292,12 +293,12 @@ async fn run_project_tests( pub enum InstallTestDependenciesError { WorkspaceTree(#[from] WorkspaceTreeError), Tree(#[from] TreeError), - Install(#[from] Box), + Install(#[from] Box), PackageVersionReq(#[from] PackageVersionReqError), } -impl From for InstallTestDependenciesError { - fn from(source: InstallError) -> Self { +impl From for InstallTestDependenciesError { + fn from(source: InstallPackagesError) -> Self { Self::Install(Box::new(source)) } } @@ -365,9 +366,8 @@ async fn ensure_test_dependencies( ) .collect(); - Install::new(config) + InstallPackages::new(config, &test_tree) .packages(test_dependencies) - .tree(&test_tree) .install() .await?; diff --git a/lux-lib/src/operations/update.rs b/lux-lib/src/operations/update.rs index 2e50aa897..0a570d216 100644 --- a/lux-lib/src/operations/update.rs +++ b/lux-lib/src/operations/update.rs @@ -19,8 +19,8 @@ use miette::Diagnostic; use thiserror::Error; use super::{ - pipeline::install_packages::InstallPackages, InstallError, PackageInstallSpec, RemoveError, - SyncError, Uninstall, + pipeline::install_packages::{InstallPackages, InstallPackagesError}, + PackageInstallSpec, RemoveError, SyncError, Uninstall, }; #[derive(Error, Debug, Diagnostic)] @@ -30,7 +30,7 @@ pub enum UpdateError { RockConstraintUnsatisfied(#[from] RockConstraintUnsatisfied), #[error("failed to update rock")] #[diagnostic(forward(0))] - Install(#[from] Box), + Install(#[from] Box), #[error("failed to remove old rock")] #[diagnostic(forward(0))] Remove(#[from] RemoveError), @@ -56,8 +56,8 @@ pub enum UpdateError { Sync(#[from] Box), } -impl From for UpdateError { - fn from(source: InstallError) -> Self { +impl From for UpdateError { + fn from(source: InstallPackagesError) -> Self { Self::Install(Box::new(source)) } } @@ -294,9 +294,7 @@ async fn update( ) .maybe_package_db(Some(package_db)) .install() - .await - .map_err(InstallError::from) - .map_err(UpdateError::from)?; + .await?; Ok(updated_packages) } } diff --git a/lux-lib/src/remote_package_db/mod.rs b/lux-lib/src/remote_package_db/mod.rs index a76063aac..efb0cffa8 100644 --- a/lux-lib/src/remote_package_db/mod.rs +++ b/lux-lib/src/remote_package_db/mod.rs @@ -1,6 +1,6 @@ use crate::{ config::{Config, ConfigError}, - lockfile::{LockedPackageLock, LockfileIntegrityError}, + lockfile::{LockfileIntegrityError, PackageLock}, manifest::{Manifest, ManifestError}, operations::{ pipeline::{ @@ -26,7 +26,7 @@ pub struct RemotePackageDB(pub(crate) Vec); #[derive(Clone, Debug)] pub(crate) enum RemoteSource { LuarocksManifests(Vec), - LockedPackageLocks(Vec), + LockedPackageLocks(Vec), Local(Vec), } @@ -321,8 +321,8 @@ impl From for RemotePackageDB { } } -impl From> for RemotePackageDB { - fn from(locks: Vec) -> Self { +impl From> for RemotePackageDB { + fn from(locks: Vec) -> Self { Self(vec![RemoteSource::LockedPackageLocks(locks)]) } } diff --git a/lux-lib/tests/install.rs b/lux-lib/tests/install.rs index 9199f1655..c838e93e4 100644 --- a/lux-lib/tests/install.rs +++ b/lux-lib/tests/install.rs @@ -7,7 +7,7 @@ use lux_lib::{ lua_installation::detect_installed_lua_version, lua_rockspec::RockSourceSpec, lua_version::LuaVersion, - operations::{Exec, Install, PackageInstallSpec}, + operations::{pipeline::install_packages::InstallPackages, Exec, PackageInstallSpec}, package::{PackageName, PackageReq}, tree::{EntryType, InstallTree}, }; @@ -71,12 +71,12 @@ async fn install_and_use(package: PackageReq, module_name: &str) { let tree = config .user_tree(LuaVersion::from(&config).unwrap().clone()) .unwrap(); - let installed = Install::new(&config) + let installed = InstallPackages::new(&config, &tree) .package(install_spec) - .tree(&tree) .install() .await - .unwrap(); + .unwrap() + .0; assert!(!installed.is_empty()); Exec::new("lua", None, &config) @@ -123,12 +123,12 @@ async fn test_install(install_spec: PackageInstallSpec) { let tree = config .user_tree(LuaVersion::from(&config).unwrap().clone()) .unwrap(); - let installed = Install::new(&config) + let installed = InstallPackages::new(&config, &tree) .package(install_spec) - .tree(&tree) .install() .await - .unwrap(); + .unwrap() + .0; assert!(!installed.is_empty()); } @@ -153,9 +153,8 @@ async fn install_records_dependency_edges_in_tree_lockfile() { // `luassert` depends on `say`. let install_spec = PackageInstallSpec::new("luassert@1.9.0-1".parse().unwrap(), EntryType::Entrypoint).build(); - Install::new(&config) + InstallPackages::new(&config, &tree) .package(install_spec) - .tree(&tree) .install() .await .unwrap(); diff --git a/lux-lua/src/operations.rs b/lux-lua/src/operations.rs index af9e09bae..9b72835ab 100644 --- a/lux-lua/src/operations.rs +++ b/lux-lua/src/operations.rs @@ -9,8 +9,8 @@ use lux_lib::{ lockfile::LockedPackageId, lua::lua_runtime, operations::{ - set_pinned_state, BuildWorkspace, DistProjectBin, Download, Install, PackageInstallSpec, - Sync, Uninstall, Update, + pipeline::install_packages::InstallPackages, set_pinned_state, BuildWorkspace, + DistProjectBin, Download, PackageInstallSpec, Sync, Uninstall, Update, }, package::{PackageName, PackageReq}, remote_package_db::RemotePackageDB, @@ -61,13 +61,12 @@ impl TypedUserData for OperationsModule { |_, (packages, tree, config): (Vec, TreeLua, ConfigLua)| async move { let _runtime = lua_runtime().enter(); let specs = packages.into_iter().map(|p| p.0).collect(); - Install::new(&config.0) + InstallPackages::new(&config.0, &tree.0) .packages(specs) - .tree(tree.0) .install() .await .into_lua_err() - .map(|pkgs| pkgs.into_iter().map(LockedPackageLua).collect::>()) + .map(|(pkgs, _)| pkgs.into_iter().map(LockedPackageLua).collect::>()) }, ); @@ -157,13 +156,13 @@ impl TypedUserData for OperationsModule { .await .into_lua_err()?; - Install::new(&config.0) + let tree = workspace.0.tree(&config.0).into_lua_err()?; + InstallPackages::new(&config.0, &tree) .packages(install_specs) - .tree(workspace.0.tree(&config.0).into_lua_err()?) .install() .await .into_lua_err() - .map(|pkgs| pkgs.into_iter().map(LockedPackageLua).collect::>()) + .map(|(pkgs, _)| pkgs.into_iter().map(LockedPackageLua).collect::>()) }, ); From 629966a406702d85fda6590ff4d7b6542378b24a Mon Sep 17 00:00:00 2001 From: vhyrro Date: Mon, 5 Oct 2026 12:15:31 +0200 Subject: [PATCH 22/39] fix: more fixes, cleanup --- lux-cli/src/install.rs | 4 +- lux-lib/src/build/mod.rs | 30 +++-------- lux-lib/src/lockfile/mod.rs | 4 +- lux-lib/src/manifest/metadata.rs | 3 -- lux-lib/src/operations/pipeline/build.rs | 18 +++---- .../pipeline/download_sources_and_hash.rs | 5 -- .../src/operations/pipeline/emit_lockfile.rs | 1 + .../operations/pipeline/install_packages.rs | 15 ++---- .../operations/pipeline/install_workspace.rs | 3 -- lux-lib/src/operations/pipeline/mod.rs | 5 +- lux-lib/src/operations/pipeline/resolve.rs | 29 ++++------- lux-lib/src/operations/sync.rs | 2 +- lux-lib/src/operations/test.rs | 2 - lux-lib/src/operations/vendor.rs | 52 ------------------- lux-lib/src/project/mod.rs | 5 +- lux-lib/tests/install.rs | 45 ---------------- lux-lib/tests/vendor.rs | 5 +- 17 files changed, 38 insertions(+), 190 deletions(-) diff --git a/lux-cli/src/install.rs b/lux-cli/src/install.rs index 5581a8854..33052f3c3 100644 --- a/lux-cli/src/install.rs +++ b/lux-cli/src/install.rs @@ -4,7 +4,7 @@ use lux_lib::{ config::Config, lockfile::PinnedState, lua_version::LuaVersion, - operations, + operations::{self, pipeline::install_packages::InstallPackages}, package::{PackageName, PackageReq}, workspace::{Workspace, WorkspaceError}, }; @@ -77,7 +77,7 @@ async fn install_remote(data: Install, config: Config) -> Result<()> { let packages = apply_build_behaviour(data.package_req, pin, data.force, &tree, &config)?; // TODO(vhyrro): If the tree doesn't exist then error out. - operations::pipeline::install_packages::InstallPackages::new(&config, &tree) + InstallPackages::new(&config, &tree) .packages(packages) .install() .await?; diff --git a/lux-lib/src/build/mod.rs b/lux-lib/src/build/mod.rs index 0329233ca..9da4ed338 100644 --- a/lux-lib/src/build/mod.rs +++ b/lux-lib/src/build/mod.rs @@ -450,7 +450,7 @@ mod tests { lockfile::{LockConstraint, LockedPackageHashes}, lua_installation::{detect_installed_lua_version, LuaInstallation}, lua_version::LuaVersion, - operations::{unpack_rockspec, DownloadedPackedRockBytes}, + operations::{pipeline::build_local::Build, unpack_rockspec, DownloadedPackedRockBytes}, package::PackageSpec, project::Project, remote_package_source::RemotePackageSource, @@ -538,11 +538,8 @@ mod tests { } const LUATEST_SRC_ROCK: &str = "resources/test/luatest-0.2-1.src.rock"; - /// `sha256` of the checked-in `.src.rock` archive, i.e. what `Materialize` must - /// record as the `source` integrity. const LUATEST_SRC_ROCK_SOURCE_HASH: &str = "sha256-2jS0XOq0iIVhsZJ3BVqXSlKsx2vsqCAaaYyqcBEb7RI="; - /// `sha256` of the rockspec text embedded in the `.src.rock`, i.e. the `rockspec` integrity. const LUATEST_ROCKSPEC_HASH: &str = "sha256-NljJ20A+VadUyhhBjrRnojeQjSqRpQy7FWcgjUt2Fdc="; fn luatest_src_rock_bytes() -> Bytes { @@ -561,8 +558,6 @@ mod tests { .unwrap() } - /// Builds `luatest` from the checked-in `.src.rock` fixture, entirely offline. - /// This exercises the fetch + hash + build + deploy path end to end. async fn build_luatest( config: &Config, tree: &Tree, @@ -581,7 +576,7 @@ mod tests { }; let rockspec = unpack_rockspec(&rock).await.unwrap(); let lua = LuaInstallation::new_from_config(config).await.unwrap(); - crate::operations::pipeline::build_local::Build::new() + Build::new() .rockspec(&rockspec) .lua(&lua) .tree(tree) @@ -601,11 +596,8 @@ mod tests { .unwrap() } - /// The materialized `LockedPackage` must carry the exact source and rockspec - /// integrities, derived from the fetched bytes. This is the contract that - /// `lx generate-lockfile` and the manifest cache depend on. #[tokio::test] - async fn materialize_src_rock_produces_expected_hashes() { + async fn luatest_src_rock_produces_expected_hashes() { let dir = assert_fs::TempDir::new().unwrap(); let config = luatest_config(&dir); let tree = config.user_tree(LuaVersion::Lua51).unwrap(); @@ -621,18 +613,13 @@ mod tests { package.hashes().rockspec, LUATEST_ROCKSPEC_HASH.parse().unwrap() ); - // The source integrity is the hash of the fetched artifact. assert_eq!( package.hashes().source, luatest_src_rock_bytes().hash().await.unwrap() ); - // Entrypoints expose their binaries. assert!(tree.bin().join("luatest").is_file()); } - /// Two independent forced builds of the same rockspec must produce identical - /// lockfile entries (id, source, source_url, hashes). This guards the - /// determinism that reproducible lockfiles depend on. #[tokio::test] async fn force_build_is_deterministic() { let dir1 = assert_fs::TempDir::new().unwrap(); @@ -662,8 +649,6 @@ mod tests { assert_eq!(package1.hashes(), package2.hashes()); } - /// `NoForce` must short-circuit on the tree lockfile without touching the - /// tree, while `Force` must rebuild and redeploy. #[tokio::test] async fn noforce_skips_rebuild_when_package_is_locked() { let dir = assert_fs::TempDir::new().unwrap(); @@ -675,7 +660,7 @@ mod tests { let bin = tree.bin().join("luatest"); assert!(bin.is_file()); - // Simulate a committed install by writing the package into the tree lockfile. + // Simulate a committed install { let mut lockfile = tree.lockfile().unwrap().write_guard(); lockfile.add_entrypoint(&package); @@ -692,12 +677,9 @@ mod tests { ) .await; assert_eq!(skipped.id(), package.id()); - assert!( - !bin.is_file(), - "NoForce must not rebuild a package that is already in the lockfile" - ); + assert!(!bin.is_file()); build_luatest(&config, &tree, EntryType::Entrypoint, BuildBehaviour::Force).await; - assert!(bin.is_file(), "Force must rebuild the package"); + assert!(bin.is_file()); } } diff --git a/lux-lib/src/lockfile/mod.rs b/lux-lib/src/lockfile/mod.rs index cc368c670..36bd07c28 100644 --- a/lux-lib/src/lockfile/mod.rs +++ b/lux-lib/src/lockfile/mod.rs @@ -851,8 +851,6 @@ pub struct Lockfile { _marker: PhantomData

, // TODO: Serialize this directly into a `Version` version: String, - /// Binaries installed into the tree for each package, discovered during - /// the build. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] binaries: BTreeMap, #[serde(flatten)] @@ -1020,7 +1018,7 @@ impl Lockfile

{ } } - /// The binaries installed into the tree for `id`, if recorded. + /// The binaries installed in the tree for a given package. pub(crate) fn binaries(&self, id: &LockedPackageId) -> Option<&RockBinaries> { self.binaries.get(id) } diff --git a/lux-lib/src/manifest/metadata.rs b/lux-lib/src/manifest/metadata.rs index e2d62328c..b18751aab 100644 --- a/lux-lib/src/manifest/metadata.rs +++ b/lux-lib/src/manifest/metadata.rs @@ -224,11 +224,8 @@ mod tests { assert!(metadata.latest_match(&package_req, &filter).is_none()); } - /// Lux prefers binary rocks over rockspecs over `.src.rock` archives - /// (`RemotePackageType` ordering). `Discover` relies on this to pick a source. #[tokio::test] pub async fn latest_match_prefers_binary_then_rockspec_then_src() { - // `arch = "all"` is treated as a binary rock on every platform. let manifest = r#" commands = {} modules = {} diff --git a/lux-lib/src/operations/pipeline/build.rs b/lux-lib/src/operations/pipeline/build.rs index dcc539c83..3dfb2f57a 100644 --- a/lux-lib/src/operations/pipeline/build.rs +++ b/lux-lib/src/operations/pipeline/build.rs @@ -74,11 +74,6 @@ where T: InstallTree + Send + Sync, State: build_builder::State + build_builder::IsComplete, { - // INVESTIGATE(vhyrro): Is there a benefit of having a `LocalPackage` type which - // contains a LockedPackage as well as Layout information and installation path? - // LockedPackages are emitted by DownloadSourcesAndHash, and they represent packages ready for - // the lockfile but not installed to the tree yet. Either we create an intermediate type - // or a `LocalPackage` type. pub(crate) async fn build(self) -> Result, BuildError> { let args = self._build(); let lua = LuaInstallation::new_from_config(args.config).await?; @@ -146,13 +141,14 @@ where // Record the installed package in the tree lockfile so that its paths are visible to // the packages built after it (e.g. LuaRocks build backends), as well as future builds. - let mut lockfile = args.tree.lockfile()?.write_guard(); - if is_entrypoint { - lockfile.add_entrypoint(&pkg); - } else { - lockfile.add(&pkg); + { + let mut lockfile = args.tree.lockfile()?.write_guard(); + if is_entrypoint { + lockfile.add_entrypoint(&pkg); + } else { + lockfile.add(&pkg); + } } - drop(lockfile); installed.push(pkg); } diff --git a/lux-lib/src/operations/pipeline/download_sources_and_hash.rs b/lux-lib/src/operations/pipeline/download_sources_and_hash.rs index c04a0033a..2dc1f1fd9 100644 --- a/lux-lib/src/operations/pipeline/download_sources_and_hash.rs +++ b/lux-lib/src/operations/pipeline/download_sources_and_hash.rs @@ -88,11 +88,6 @@ where let args = self._build(); let config = args.config; - // Preserve which sections were requested (even if empty), then run every - // download as a single bounded-concurrency stream and fold the results - // back into their sections. `buffered` keeps at most `max_jobs` - // downloads in flight; hashing is already `spawn_blocking`, so it runs - // across the blocking thread pool. let artifacts: DownloadSourcesAndHashArtifacts = args .resolved .iter() diff --git a/lux-lib/src/operations/pipeline/emit_lockfile.rs b/lux-lib/src/operations/pipeline/emit_lockfile.rs index 5e8a608e1..5265c0d1e 100644 --- a/lux-lib/src/operations/pipeline/emit_lockfile.rs +++ b/lux-lib/src/operations/pipeline/emit_lockfile.rs @@ -3,6 +3,7 @@ use crate::lockfile::{PackageLock, ReadWrite, WorkspaceLockfile}; use super::{download_sources_and_hash::DownloadSourcesAndHashArtifacts, Artifacts}; /// The lockfile entries resolved by the pipeline, grouped by section. +/// Allows for transactionality when committing to the lockfile. #[derive(Default)] pub struct LockfileHandle(Artifacts); diff --git a/lux-lib/src/operations/pipeline/install_packages.rs b/lux-lib/src/operations/pipeline/install_packages.rs index 4ec5a4025..09b70e491 100644 --- a/lux-lib/src/operations/pipeline/install_packages.rs +++ b/lux-lib/src/operations/pipeline/install_packages.rs @@ -27,13 +27,10 @@ use super::{ use crate::operations::PackageInstallSpec; -/// Installs a set of [`PackageInstallSpec`]s into an install tree. +/// Installs a set of [`PackageInstallSpec`]s into a tree. /// -/// This is the composable core of package installation: it resolves the -/// dependency graph, downloads and hashes sources, builds the packages (build dependencies -/// first) into the regular, build and test trees, and records the result in the trees' -/// lockfiles. It also returns a [`LockfileHandle`] so callers (e.g. a workspace) can commit -/// the resolved sections to their own lockfile. +/// Returns a [`LockfileHandle`] so changes can be committed into +/// the correct lockfile. #[derive(Builder)] #[builder(start_fn = new, finish_fn(name = _build, vis = ""))] pub struct InstallPackages<'a, T> @@ -95,7 +92,6 @@ where return Ok((Vec::new(), LockfileHandle::default())); } let span = match args.packages.as_slice() { - [] => tracing::info_span!("Installing"), [install_spec] => { tracing::info_span!("Installing", package = install_spec.package.to_string()) } @@ -165,9 +161,8 @@ where )); } - // Forced overwrites: remove the conflicting entrypoints from the tree. - // NOTE: non-transactional. If an error occurs, this removes the conflicting package without - // installing the substitute. Fix when transactions are implemented. + // FIXME(vhyrro): non-transactional. If an error occurs this removes the conflicting package + // without installing the substitute. Implement transactions at some point. let conflicting_entrypoints = conflicting_entrypoints.into_values().collect_vec(); for package in &conflicting_entrypoints { tree.cleanup(package, tree::EntryType::Entrypoint)?; diff --git a/lux-lib/src/operations/pipeline/install_workspace.rs b/lux-lib/src/operations/pipeline/install_workspace.rs index aca4cc90a..eff223d80 100644 --- a/lux-lib/src/operations/pipeline/install_workspace.rs +++ b/lux-lib/src/operations/pipeline/install_workspace.rs @@ -15,9 +15,6 @@ use crate::{ use super::install_packages::{InstallPackages, InstallPackagesError}; /// Installs all of a workspace's dependencies into its regular, build and test trees. -/// -/// This resolves the workspace's dependency graph, downloads and hashes the sources, -/// writes the workspace lockfile, and builds the packages into the appropriate trees. #[derive(Builder)] #[builder(start_fn = new, finish_fn(name = _build, vis = ""))] pub struct InstallWorkspaceDependencies<'a> { diff --git a/lux-lib/src/operations/pipeline/mod.rs b/lux-lib/src/operations/pipeline/mod.rs index 212807930..51dc6ff19 100644 --- a/lux-lib/src/operations/pipeline/mod.rs +++ b/lux-lib/src/operations/pipeline/mod.rs @@ -10,10 +10,7 @@ pub mod install_workspace; pub mod resolve; /// Packages grouped by the lockfile section they belong to. -/// -/// A `None` section was never requested and must be left untouched (e.g. not -/// written to a lockfile), whereas `Some(empty)` means the section was resolved -/// and is legitimately empty. +/// `None` means that that section shouldn't be modified. pub(crate) struct Artifacts { pub(crate) regular: Option, pub(crate) build: Option, diff --git a/lux-lib/src/operations/pipeline/resolve.rs b/lux-lib/src/operations/pipeline/resolve.rs index c0addd5f4..214fb9b3d 100644 --- a/lux-lib/src/operations/pipeline/resolve.rs +++ b/lux-lib/src/operations/pipeline/resolve.rs @@ -158,17 +158,13 @@ struct Node { install_spec: PackageInstallSpec, id: LockedPackageId, found: FoundPackage, - /// `(is_build_dependency, child)` in the order the dependencies were declared. children: Vec<(bool, RequestKey)>, } -/// Owns the whole resolution graph while it is being discovered. Discovery futures run on -/// the runtime (bounded by `Config::max_jobs`), but all shared state stays here, owned by a -/// single task — no `Rc`, `RefCell`, or locks. struct Resolver { package_db: RemotePackageDB, config: Arc, - max_inflight: usize, + max_concurrent: usize, joinset: JoinSet<( RequestKey, PackageInstallSpec, @@ -182,20 +178,18 @@ struct Resolver { impl Resolver { fn new(package_db: RemotePackageDB, config: &Config) -> Self { - Self::with_max_inflight(package_db, Arc::new(config.clone()), config.max_jobs()) + Self::with_max_concurrent(package_db, Arc::new(config.clone()), config.max_jobs()) } - /// Builds a resolver with an explicit in-flight bound. Used by tests, which do not - /// construct a full [`Config`]. - fn with_max_inflight( + fn with_max_concurrent( package_db: RemotePackageDB, config: Arc, - max_inflight: usize, + max_concurrent: usize, ) -> Self { Self { package_db, config, - max_inflight, + max_concurrent, joinset: JoinSet::new(), pending: VecDeque::new(), seen: HashSet::new(), @@ -233,8 +227,7 @@ impl Resolver { Ok(self.assemble()) } - /// Registers a request for discovery, unless it has already been seen. Seeing the same - /// request twice is exactly the diamond problem, so it is only discovered once. + /// Registers a request for discovery, unless it has already been seen. fn enqueue(&mut self, key: RequestKey, spec: PackageInstallSpec) { if self.seen.insert(key.clone()) { self.pending.push_back((key, spec)); @@ -243,7 +236,7 @@ impl Resolver { /// Spawns queued discoveries up to the configured concurrency limit. fn spawn_ready(&mut self) { - while self.joinset.len() < self.max_inflight { + while self.joinset.len() < self.max_concurrent { let Some((key, spec)) = self.pending.pop_front() else { break; }; @@ -407,8 +400,6 @@ fn push_child( } } -/// Depth-first search with white/grey/black colouring over the discovered graph. A grey -/// back-edge means a cycle, which we render as a `a -> b -> a` chain for the error message. fn detect_cycles(nodes: &HashMap) -> Result<(), ResolveError> { #[derive(Clone, Copy, PartialEq, Eq)] enum Color { @@ -578,7 +569,7 @@ mod tests { } #[tokio::test] - async fn diamond_is_resolved_once() { + async fn diamond_problem() { let packages = local_packages(&[ rockspec("a", &["b", "c"]), rockspec("b", &["d"]), @@ -586,7 +577,7 @@ mod tests { rockspec("d", &[]), ]); - let resolved = match Resolver::with_max_inflight(packages, test_config(), 8) + let resolved = match Resolver::with_max_concurrent(packages, test_config(), 8) .run(roots(install_spec("a"))) .await { @@ -601,7 +592,7 @@ mod tests { async fn cyclic_dependencies_are_rejected() { let packages = local_packages(&[rockspec("a", &["b"]), rockspec("b", &["a"])]); - let result = Resolver::with_max_inflight(packages, test_config(), 8) + let result = Resolver::with_max_concurrent(packages, test_config(), 8) .run(roots(install_spec("a"))) .await; diff --git a/lux-lib/src/operations/sync.rs b/lux-lib/src/operations/sync.rs index 5e09e6461..927a32f7f 100644 --- a/lux-lib/src/operations/sync.rs +++ b/lux-lib/src/operations/sync.rs @@ -215,7 +215,7 @@ async fn do_sync( let package_db = workspace_lockfile.local_pkg_locks().into(); - let (to_add, mut report) = reconcile_locks(&*workspace_lockfile, &dest_lockfile, lock_type); + let (to_add, mut report) = reconcile_locks(&workspace_lockfile, &dest_lockfile, lock_type); let packages_to_install = install_specs_to_force(&to_add); report .added diff --git a/lux-lib/src/operations/test.rs b/lux-lib/src/operations/test.rs index 60c492ca4..a830c23c5 100644 --- a/lux-lib/src/operations/test.rs +++ b/lux-lib/src/operations/test.rs @@ -159,7 +159,6 @@ async fn run_tests(test: Test<'_>) -> Result<(), RunTestsError> { if let Some(package) = test.package { let project = workspace.select_member(&package)?; - // NOTE: `run_project_tests`' future is large, so we box it to avoid inflating this frame. Box::pin(run_project_tests( &workspace, project, no_lock, &test.args, &test.env, config, )) @@ -194,7 +193,6 @@ async fn run_project_tests( Sync::new(workspace, &test_config).test(true).sync().await?; } - // NOTE: `BuildWorkspace::build`'s future is large, so we box it to avoid inflating this frame. Box::pin( BuildWorkspace::new(workspace, &test_config) .package(project.toml().package().clone()) diff --git a/lux-lib/src/operations/vendor.rs b/lux-lib/src/operations/vendor.rs index c75328c32..3fb0cf85b 100644 --- a/lux-lib/src/operations/vendor.rs +++ b/lux-lib/src/operations/vendor.rs @@ -313,7 +313,6 @@ fn push_dependencies( Ok(()) } -/// Vendors the materialized source and rockspec of a single package. #[tracing::instrument( name = "Vendoring source", level = "info", @@ -341,12 +340,10 @@ async fn vendor_package_sources( let source_path = vendor_dir.join(format!("{}@{}", name, version)); match &package.artifact { - // A fully materialized source tree. PackageSource::SourceDir(path) => { fs::tokio::remove_dir_all(&source_path).await.ok(); fs::tokio::copy_dir_all(path, &source_path).await?; } - // A pre-built binary rock. PackageSource::PackedRock(bytes) => { let rock_path = vendor_dir.join(format!("{}@{}.rock", name, version)); let mut file = fs::tokio::create(&rock_path).await?; @@ -420,8 +417,6 @@ async fn vendor_package_cargo_deps( ) .await?; } else if source_dir.is_file() { - // The vendored source is an archive; extract it so `cargo vendor` can - // read its `Cargo.toml`. Keep the temp dir alive while cargo runs. let temp_dir = fs::tempfile::tempdir()?; extract_source_archive(&source_dir, unpack_dir.as_deref(), temp_dir.path()).await?; cargo_vendor( @@ -534,7 +529,6 @@ mod tests { Bytes::from(std::fs::read(path).unwrap()) } - /// Builds a `DownloadedPackage` for a materialized source tree. fn make_downloaded_package( rockspec: RemoteLuaRockspec, source: RemotePackageSource, @@ -573,13 +567,10 @@ mod tests { } } - /// Vendors a package with a materialized source tree: the rockspec and the - /// source tree directory must both be written. #[tokio::test] async fn vendor_source_tree_writes_rockspec_and_source_dir() { let vendor_dir = assert_fs::TempDir::new().unwrap(); - // A source tree that has already been materialized by the pipeline. let src_dir = TempDir::new().unwrap(); std::fs::write(src_dir.path().join("hello.lua"), "return 'hello'").unwrap(); let rockspec = RemoteLuaRockspec::from_package_and_source_spec( @@ -614,47 +605,6 @@ mod tests { ); } - /// Vendors a binary rock: both the packed `.rock` and its rockspec must be written. - #[tokio::test] - async fn vendor_binary_rock_writes_rock_and_rockspec() { - let vendor_dir = assert_fs::TempDir::new().unwrap(); - - let bytes = fixture_bytes("toml-edit-0.6.0-1.linux-x86_64.rock"); - let rock = crate::operations::DownloadedPackedRockBytes { - name: "toml-edit".into(), - version: "0.6.0-1".parse().unwrap(), - bytes: bytes.clone(), - file_name: "toml-edit-0.6.0-1.linux-x86_64.rock".into(), - url: "https://example.org/toml-edit-0.6.0-1.linux-x86_64.rock" - .parse() - .unwrap(), - }; - let rockspec = unpack_rockspec(&rock).await.unwrap(); - let package = make_downloaded_package( - rockspec, - RemotePackageSource::LuarocksBinaryRock("https://example.org/".parse().unwrap()), - PackageSource::PackedRock(bytes), - None, - ); - - vendor_package_sources(vendor_dir.path(), &package) - .await - .unwrap(); - - assert!( - vendor_dir.path().join("toml-edit@0.6.0-1.rock").is_file(), - "packed rock not vendored" - ); - let rockspec_file = vendor_dir.path().join("toml-edit-0.6.0-1.rockspec"); - assert!(rockspec_file.is_file(), "rockspec not vendored"); - let rockspec_content = std::fs::read_to_string(&rockspec_file).unwrap(); - assert!( - rockspec_content.contains("package = ") && rockspec_content.contains("toml-edit"), - "vendored rockspec is not valid:\n{rockspec_content}" - ); - } - - /// A pre-existing vendored source directory must be replaced, not merged. #[tokio::test] async fn vendor_source_tree_replaces_stale_source_dir() { let vendor_dir = assert_fs::TempDir::new().unwrap(); @@ -687,8 +637,6 @@ mod tests { assert!(stale_dir.join("hello.lua").is_file()); } - /// `no_delete` must be respected: if the vendor dir exists and `no_delete` is - /// set, pre-existing contents are preserved. #[tokio::test] async fn vendor_workspace_no_delete_preserves_existing_contents() { let sample_project_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")) diff --git a/lux-lib/src/project/mod.rs b/lux-lib/src/project/mod.rs index 06bcaec0e..4d12b41b7 100644 --- a/lux-lib/src/project/mod.rs +++ b/lux-lib/src/project/mod.rs @@ -285,9 +285,8 @@ impl Project { .map_err(|err| IntoLocalRockspecError::RockspecError(Box::new(err))) } - /// Returns this project's rockspec as a [`RemoteLuaRockspec`] whose source - /// points at the project's own directory, so it can be built through the - /// pipeline like any other package. + /// Returns this project's rockspec as a [`RemoteLuaRockspec`] whose source points at the + /// project's own directory, so it can be built like any other package. pub fn local_remote_rockspec(&self) -> Result { self.toml() .into_local() diff --git a/lux-lib/tests/install.rs b/lux-lib/tests/install.rs index c838e93e4..2d16be7fb 100644 --- a/lux-lib/tests/install.rs +++ b/lux-lib/tests/install.rs @@ -131,48 +131,3 @@ async fn test_install(install_spec: PackageInstallSpec) { .0; assert!(!installed.is_empty()); } - -/// Guards the resolved dependency graph recorded in the install tree lockfile: -/// entrypoints, dependency edges, and dependency-only classification. -#[flaky_test(tokio, times = 5)] -async fn install_records_dependency_edges_in_tree_lockfile() { - let dir = TempDir::new().unwrap(); - let lua_version = detect_installed_lua_version().or(Some(LuaVersion::Lua51)); - - let config = ConfigBuilder::new() - .unwrap() - .user_tree(Some(dir.to_path_buf())) - .lua_version(lua_version) - .build() - .unwrap(); - - let tree = config - .user_tree(LuaVersion::from(&config).unwrap().clone()) - .unwrap(); - - // `luassert` depends on `say`. - let install_spec = - PackageInstallSpec::new("luassert@1.9.0-1".parse().unwrap(), EntryType::Entrypoint).build(); - InstallPackages::new(&config, &tree) - .package(install_spec) - .install() - .await - .unwrap(); - - let lockfile = tree.lockfile().unwrap(); - let luassert = lockfile - .entrypoint(&PackageName::new("luassert".into())) - .expect("luassert must be recorded as an entrypoint"); - assert_eq!(luassert.version(), &"1.9.0-1".parse().unwrap()); - - let say = luassert - .dependencies() - .into_iter() - .filter_map(|id| lockfile.get(id)) - .find(|pkg| pkg.name().to_string() == "say") - .expect("luassert's lockfile entry must depend on say"); - assert!( - !lockfile.is_entrypoint(&say.id()), - "transitive dependencies must be recorded as dependency-only" - ); -} diff --git a/lux-lib/tests/vendor.rs b/lux-lib/tests/vendor.rs index cdbfc5647..61d66b400 100644 --- a/lux-lib/tests/vendor.rs +++ b/lux-lib/tests/vendor.rs @@ -7,6 +7,7 @@ use assert_fs::{ use flaky_test::flaky_test; use lux_lib::{ config::ConfigBuilder, + lua_rockspec::RemoteLuaRockspec, operations::{Vendor, VendorTarget}, workspace::Workspace, }; @@ -56,11 +57,9 @@ async fn vendor_dependencies() { .join(format!("{name}-{version}.rockspec")); assert!(rockspec_file.is_file(), "missing rockspec for {package}"); let rockspec_content = std::fs::read_to_string(&rockspec_file).unwrap(); - lux_lib::lua_rockspec::RemoteLuaRockspec::new(&rockspec_content) + RemoteLuaRockspec::new(&rockspec_content) .unwrap_or_else(|err| panic!("vendored rockspec for {package} failed to parse: {err}")); - // A source must be vendored alongside the rockspec (either a directory - // for `rockspec` sources or a file for `binary rock` sources). let source_path = vendor_dir.to_path_buf().join(package); assert!( source_path.is_dir() || source_path.is_file(), From ddfc43447fc379ee17a2556eb90a02ccad4be63a Mon Sep 17 00:00:00 2001 From: vhyrro Date: Mon, 5 Oct 2026 17:42:39 +0200 Subject: [PATCH 23/39] feat!: granularize BuildBehaviour --- lux-cli/src/dist/flat_archive.rs | 1 + lux-cli/src/pack.rs | 1 + lux-cli/src/uninstall.rs | 2 +- lux-lib/src/build/mod.rs | 17 +++-- lux-lib/src/luarocks/install_binary_rock.rs | 11 ++- lux-lib/src/luarocks/luarocks_installation.rs | 1 + lux-lib/src/operations/exec.rs | 2 + lux-lib/src/operations/install/spec.rs | 1 - .../src/operations/pipeline/build_local.rs | 1 - .../operations/pipeline/install_packages.rs | 75 ++++++++++--------- .../operations/pipeline/install_workspace.rs | 2 + lux-lib/src/operations/pipeline/resolve.rs | 11 ++- lux-lib/src/operations/test.rs | 4 +- lux-lib/src/operations/update.rs | 2 + lux-lib/src/operations/vendor.rs | 14 +++- lux-lib/tests/build_workspace.rs | 8 ++ lux-lib/tests/install.rs | 14 +++- lux-lua/src/lua_impls.rs | 8 +- lux-lua/src/operations.rs | 9 ++- 19 files changed, 119 insertions(+), 65 deletions(-) diff --git a/lux-cli/src/dist/flat_archive.rs b/lux-cli/src/dist/flat_archive.rs index 2d9605c57..c95dcc99e 100644 --- a/lux-cli/src/dist/flat_archive.rs +++ b/lux-cli/src/dist/flat_archive.rs @@ -206,6 +206,7 @@ async fn install_rockspec( .tree(&tree) .entry_type(tree::EntryType::Entrypoint) .config(config) + .behaviour(BuildBehaviour::Force) .build() .await?; Ok((package, tree.root())) diff --git a/lux-cli/src/pack.rs b/lux-cli/src/pack.rs index d995f1f54..e9fb0e924 100644 --- a/lux-cli/src/pack.rs +++ b/lux-cli/src/pack.rs @@ -178,6 +178,7 @@ pub async fn pack(args: Pack, config: Config) -> Result<()> { .tree(&tree) .entry_type(tree::EntryType::Entrypoint) .config(&config) + .behaviour(BuildBehaviour::Force) .build() .await?; let rock_path = operations::Pack::new(dest_dir, tree, package) diff --git a/lux-cli/src/uninstall.rs b/lux-cli/src/uninstall.rs index 1aa40a9ad..9678c805f 100644 --- a/lux-cli/src/uninstall.rs +++ b/lux-cli/src/uninstall.rs @@ -103,7 +103,7 @@ multiple packages satisfying your version requirements were found: format!( " Package {} can be removed from the entrypoints, but it is also a dependency, so it will have to be reinstalled. -Reinstall? +Overwrite? ", package_names[0] ) diff --git a/lux-lib/src/build/mod.rs b/lux-lib/src/build/mod.rs index 9da4ed338..3e5df0a61 100644 --- a/lux-lib/src/build/mod.rs +++ b/lux-lib/src/build/mod.rs @@ -161,12 +161,15 @@ impl From for BuildError { } } -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, Default)] +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] pub enum BuildBehaviour { - /// Don't force a rebuild if the package is already installed - #[default] - NoForce, - /// Force a rebuild if the package is already installed + /// If a package with the same version is already installed, do nothing. + /// Otherwise, behave like [`BuildBehaviour::Conflict`]. + Ignore, + /// Error if an entrypoint with the same name is already installed, even if + /// it is the same version. + Conflict, + /// Remove any installed entrypoint with the same name and rebuild. Force, } @@ -284,7 +287,7 @@ pub(crate) async fn deploy( config: &Config, behaviour: BuildBehaviour, ) -> Result { - if behaviour == BuildBehaviour::NoForce { + if matches!(behaviour, BuildBehaviour::Ignore | BuildBehaviour::Conflict) { if let Some(existing) = tree.lockfile()?.get(&package.id()) { return Ok(existing.clone()); } @@ -673,7 +676,7 @@ mod tests { &config, &tree, EntryType::Entrypoint, - BuildBehaviour::NoForce, + BuildBehaviour::Ignore, ) .await; assert_eq!(skipped.id(), package.id()); diff --git a/lux-lib/src/luarocks/install_binary_rock.rs b/lux-lib/src/luarocks/install_binary_rock.rs index 5cc98cd0f..b04ec0938 100644 --- a/lux-lib/src/luarocks/install_binary_rock.rs +++ b/lux-lib/src/luarocks/install_binary_rock.rs @@ -99,7 +99,7 @@ where config, tree, constraint: LockConstraint::default(), - behaviour: BuildBehaviour::default(), + behaviour: BuildBehaviour::Ignore, pin: PinnedState::default(), opt: OptState::default(), entry_type, @@ -152,7 +152,14 @@ where ); let package = LockedPackage::new(spec, self.source, source_url, hashes); match self.tree.lockfile()?.get(&package.id()) { - Some(package) if self.behaviour == BuildBehaviour::NoForce => Ok(package.clone()), + Some(package) + if matches!( + self.behaviour, + BuildBehaviour::Ignore | BuildBehaviour::Conflict + ) => + { + Ok(package.clone()) + } _ => { let unpack_dir = fs::tempfile::tempdir()?; let cursor = Cursor::new(self.rock_bytes); diff --git a/lux-lib/src/luarocks/luarocks_installation.rs b/lux-lib/src/luarocks/luarocks_installation.rs index e47d3fff7..f86c031d7 100644 --- a/lux-lib/src/luarocks/luarocks_installation.rs +++ b/lux-lib/src/luarocks/luarocks_installation.rs @@ -167,6 +167,7 @@ impl LuaRocksInstallation { .tree(&self.tree) .entry_type(tree::EntryType::Entrypoint) .config(&self.config) + .behaviour(build::BuildBehaviour::Ignore) .constraint(luarocks_req.version_req().clone().into()) .build() .await?; diff --git a/lux-lib/src/operations/exec.rs b/lux-lib/src/operations/exec.rs index 8e20badfe..1cd6cc069 100644 --- a/lux-lib/src/operations/exec.rs +++ b/lux-lib/src/operations/exec.rs @@ -2,6 +2,7 @@ use std::io; use tokio::process::Command; use crate::{ + build::BuildBehaviour, config::Config, lua_rockspec::LuaVersionError, lua_version::{LuaVersion, LuaVersionUnset}, @@ -227,6 +228,7 @@ async fn install_command(command: &str, config: &Config) -> Result<(), InstallCo PackageReq::new(command.into(), None)?, tree::EntryType::Entrypoint, ) + .build_behaviour(BuildBehaviour::Force) .build(); let tree = config.user_tree(LuaVersion::from(config)?.clone())?; InstallPackages::new(config, &tree) diff --git a/lux-lib/src/operations/install/spec.rs b/lux-lib/src/operations/install/spec.rs index 712a76709..6ba76764f 100644 --- a/lux-lib/src/operations/install/spec.rs +++ b/lux-lib/src/operations/install/spec.rs @@ -16,7 +16,6 @@ pub struct PackageInstallSpec { pub(crate) package: PackageReq, #[builder(start_fn)] pub(crate) entry_type: tree::EntryType, - #[builder(default)] pub(crate) build_behaviour: BuildBehaviour, #[builder(default)] pub(crate) pin: PinnedState, diff --git a/lux-lib/src/operations/pipeline/build_local.rs b/lux-lib/src/operations/pipeline/build_local.rs index c8fd72a5e..e5b55ef78 100644 --- a/lux-lib/src/operations/pipeline/build_local.rs +++ b/lux-lib/src/operations/pipeline/build_local.rs @@ -31,7 +31,6 @@ pub struct Build<'a, R: Rockspec + HasIntegrity, T: InstallTree> { opt: OptState, #[builder(default)] constraint: LockConstraint, - #[builder(default)] behaviour: BuildBehaviour, #[builder(setters(vis = "pub(crate)"))] diff --git a/lux-lib/src/operations/pipeline/install_packages.rs b/lux-lib/src/operations/pipeline/install_packages.rs index 09b70e491..5cfa19896 100644 --- a/lux-lib/src/operations/pipeline/install_packages.rs +++ b/lux-lib/src/operations/pipeline/install_packages.rs @@ -12,7 +12,7 @@ use crate::{ lockfile::{FlushLockfileError, LockedPackage, LockedPackageId, Lockfile, ReadWrite}, lua_installation::LuaInstallationError, luarocks::luarocks_installation::{LuaRocksError, LuaRocksInstallError}, - package::{PackageName, PackageNameList}, + package::PackageNameList, remote_package_db::{RemotePackageDB, RemotePackageDBError}, tree::{self, InstallTree, TreeError}, workspace::WorkspaceTreeError, @@ -134,40 +134,6 @@ where let lockfile = tree.lockfile()?; let build_tree = tree.build_tree(config)?; - let entrypoint_specs = packages - .iter() - .filter(|spec| spec.entry_type == tree::EntryType::Entrypoint) - .collect_vec(); - - // Entrypoints already installed that conflict with another entrypoint. This enumerates both - // packages that are planned to be removed (`--force`), as well as those which are - // unintentionally causing conflicts. - let conflicting_entrypoints: HashMap = entrypoint_specs - .iter() - .filter_map(|spec| lockfile.entrypoint(spec.package.name()).cloned()) - .map(|package| (package.name().clone(), package)) - .collect(); - - let unforced_conflicts = entrypoint_specs - .iter() - .filter(|spec| spec.build_behaviour != BuildBehaviour::Force) - .filter_map(|spec| conflicting_entrypoints.get(spec.package.name())) - .map(|existing| format!("{}@{}", existing.name(), existing.version())) - .collect_vec(); - - if !unforced_conflicts.is_empty() { - return Err(InstallPackagesError::ConflictingEntrypoints( - unforced_conflicts.join("\n"), - )); - } - - // FIXME(vhyrro): non-transactional. If an error occurs this removes the conflicting package - // without installing the substitute. Implement transactions at some point. - let conflicting_entrypoints = conflicting_entrypoints.into_values().collect_vec(); - for package in &conflicting_entrypoints { - tree.cleanup(package, tree::EntryType::Entrypoint)?; - } - let mut resolve = ResolvePackageDependencies::new(package_db, config) .packages(packages) .build_packages(install.build_packages); @@ -191,6 +157,45 @@ where .unwrap_or_default() .into_values() .collect_vec(); + + // Entrypoints that are already installed. The requested build behaviour decides how to + // react to them. + let mut conflicting_entrypoints: HashMap = HashMap::new(); + let mut unforced_conflicts = Vec::new(); + for package in regular_packages + .iter() + .filter(|package| package.entry_type == tree::EntryType::Entrypoint) + { + let Some(existing) = lockfile.entrypoint(package.package.name()) else { + continue; + }; + match package.build_behaviour { + // Forced installs replace the existing entrypoint. + BuildBehaviour::Force => { + conflicting_entrypoints.insert(existing.id(), existing.clone()); + } + // `Ignore` tolerates re-installing the exact same version. + BuildBehaviour::Ignore if existing.version() == package.package.version() => {} + // Any other existing entrypoint is a conflict. + BuildBehaviour::Ignore | BuildBehaviour::Conflict => { + unforced_conflicts.push(format!("{}@{}", existing.name(), existing.version())); + } + } + } + + if !unforced_conflicts.is_empty() { + return Err(InstallPackagesError::ConflictingEntrypoints( + unforced_conflicts.join("\n"), + )); + } + + // FIXME(vhyrro): non-transactional. If an error occurs this removes the conflicting package + // without installing the substitute. Implement transactions at some point. + let conflicting_entrypoints = conflicting_entrypoints.into_values().collect_vec(); + for package in &conflicting_entrypoints { + tree.cleanup(package, tree::EntryType::Entrypoint)?; + } + let regular_entry_types: HashMap = regular_packages .iter() .map(|pkg| (pkg.package.spec.id(), pkg.entry_type)) diff --git a/lux-lib/src/operations/pipeline/install_workspace.rs b/lux-lib/src/operations/pipeline/install_workspace.rs index eff223d80..a77a3856e 100644 --- a/lux-lib/src/operations/pipeline/install_workspace.rs +++ b/lux-lib/src/operations/pipeline/install_workspace.rs @@ -3,6 +3,7 @@ use miette::Diagnostic; use thiserror::Error; use crate::{ + build::BuildBehaviour, config::Config, lockfile::LockedPackage, operations::PackageInstallSpec, @@ -121,6 +122,7 @@ fn gather_dependencies( // From the perspective of a project, dependencies are entrypoints. packages.push( PackageInstallSpec::new(dependency.package_req().clone(), EntryType::Entrypoint) + .build_behaviour(BuildBehaviour::Ignore) .maybe_source(dependency.source().clone()) .build(), ); diff --git a/lux-lib/src/operations/pipeline/resolve.rs b/lux-lib/src/operations/pipeline/resolve.rs index 214fb9b3d..a7393fc4e 100644 --- a/lux-lib/src/operations/pipeline/resolve.rs +++ b/lux-lib/src/operations/pipeline/resolve.rs @@ -8,6 +8,7 @@ use thiserror::Error; use tokio::task::JoinSet; use crate::{ + build::BuildBehaviour, config::Config, lockfile::{ LockedPackageId, LockedPackageLockType, LockedPackageSpec, OptState, PinnedState, @@ -283,7 +284,7 @@ impl Resolver { dependency.package_req().clone(), EntryType::DependencyOnly, ) - .build_behaviour(spec.build_behaviour) + .build_behaviour(BuildBehaviour::Ignore) .pin(spec.pin) .opt(spec.opt) .maybe_source(dependency.source().clone()) @@ -470,7 +471,7 @@ fn build_dependency_specs( .find(|dep| dep.name() == &name) .map(|dep| { PackageInstallSpec::new(dep.package_req().clone(), EntryType::Entrypoint) - .build_behaviour(parent.build_behaviour) + .build_behaviour(BuildBehaviour::Ignore) .pin(parent.pin) .opt(parent.opt) .maybe_source(dep.source().clone()) @@ -479,7 +480,7 @@ fn build_dependency_specs( .or_else(|| { Some( PackageInstallSpec::new(PackageReq::from(name), EntryType::Entrypoint) - .build_behaviour(parent.build_behaviour) + .build_behaviour(BuildBehaviour::Ignore) .pin(parent.pin) .opt(parent.opt) .build(), @@ -545,7 +546,9 @@ mod tests { Ok(package) => package, Err(err) => panic!("invalid package req: {err}"), }; - PackageInstallSpec::new(package, EntryType::Entrypoint).build() + PackageInstallSpec::new(package, EntryType::Entrypoint) + .build_behaviour(BuildBehaviour::Ignore) + .build() } fn roots(spec: PackageInstallSpec) -> Artifacts> { diff --git a/lux-lib/src/operations/test.rs b/lux-lib/src/operations/test.rs index a830c23c5..1498261b5 100644 --- a/lux-lib/src/operations/test.rs +++ b/lux-lib/src/operations/test.rs @@ -326,7 +326,7 @@ async fn ensure_test_dependencies( .match_rocks(dep) .is_ok_and(|matches| matches.is_found()) { - Some(BuildBehaviour::NoForce) + Some(BuildBehaviour::Ignore) } else { Some(BuildBehaviour::Force) }; @@ -345,7 +345,7 @@ async fn ensure_test_dependencies( .match_rocks(dep.package_req()) .is_ok_and(|matches| matches.is_found()) { - Some(BuildBehaviour::NoForce) + Some(BuildBehaviour::Ignore) } else { Some(BuildBehaviour::Force) }; diff --git a/lux-lib/src/operations/update.rs b/lux-lib/src/operations/update.rs index 0a570d216..14b4cf62a 100644 --- a/lux-lib/src/operations/update.rs +++ b/lux-lib/src/operations/update.rs @@ -1,6 +1,7 @@ use std::io; use crate::{ + build::BuildBehaviour, config::Config, lockfile::{ LockedPackage, LockedPackageLockType, Lockfile, PinnedState, ReadOnly, ReadWrite, @@ -331,6 +332,7 @@ fn mk_install_spec( tree::EntryType::DependencyOnly }; PackageInstallSpec::new(req.clone(), entry_type) + .build_behaviour(BuildBehaviour::Force) .pin(PinnedState::Unpinned) .opt(package.opt()) .build() diff --git a/lux-lib/src/operations/vendor.rs b/lux-lib/src/operations/vendor.rs index 3fb0cf85b..8cfa2a2af 100644 --- a/lux-lib/src/operations/vendor.rs +++ b/lux-lib/src/operations/vendor.rs @@ -17,7 +17,7 @@ use thiserror::Error; use tokio::io::AsyncWriteExt; use crate::{ - build::resolve_source_dir, + build::{resolve_source_dir, BuildBehaviour}, config::Config, fs, lockfile::LockedPackageLockType, @@ -258,7 +258,9 @@ async fn gather_install_specs( .iter() .cloned() .map(|dep| { - PackageInstallSpec::new(dep, EntryType::Entrypoint).build() + PackageInstallSpec::new(dep, EntryType::Entrypoint) + .build_behaviour(BuildBehaviour::Ignore) + .build() }), ); } @@ -307,7 +309,11 @@ fn push_dependencies( dependencies .into_iter() .unique() - .map(|dep| PackageInstallSpec::new(dep, EntryType::Entrypoint).build()) + .map(|dep| { + PackageInstallSpec::new(dep, EntryType::Entrypoint) + .build_behaviour(BuildBehaviour::Ignore) + .build() + }) .collect_vec(), ); Ok(()) @@ -561,7 +567,7 @@ mod tests { package, rockspec, entry_type: EntryType::Entrypoint, - build_behaviour: BuildBehaviour::default(), + build_behaviour: BuildBehaviour::Ignore, artifact, temp_dir, } diff --git a/lux-lib/tests/build_workspace.rs b/lux-lib/tests/build_workspace.rs index 6fde00b82..60854aff0 100644 --- a/lux-lib/tests/build_workspace.rs +++ b/lux-lib/tests/build_workspace.rs @@ -32,6 +32,14 @@ async fn test_build_multi_workspace_local_dependencies() { .await .unwrap(); + // Building an already-built workspace must be idempotent + BuildWorkspace::new(&workspace, &config) + .no_lock(false) + .only_deps(false) + .build() + .await + .unwrap(); + let lockfile = workspace.tree(&config).unwrap().lockfile().unwrap(); let foo = lockfile diff --git a/lux-lib/tests/install.rs b/lux-lib/tests/install.rs index 2d16be7fb..323870ae7 100644 --- a/lux-lib/tests/install.rs +++ b/lux-lib/tests/install.rs @@ -2,6 +2,7 @@ use assert_fs::TempDir; use flaky_test::flaky_test; use itertools::Itertools; use lux_lib::{ + build::BuildBehaviour, config::ConfigBuilder, git::{GitRef, GitSource}, lua_installation::detect_installed_lua_version, @@ -27,6 +28,7 @@ async fn install_git_package() { .unwrap(), git_ref: Some(GitRef::Tag("v6.0.3".into())), })) + .build_behaviour(BuildBehaviour::Force) .build(); test_install(install_spec).await } @@ -40,7 +42,9 @@ async fn install_http_package() { // See https://github.com/wahern/luaossl/issues/220#issuecomment-3401472124 std::env::set_var("CFLAGS", "-Wno-error=incompatible-pointer-types"); let install_spec = - PackageInstallSpec::new("http@0.4-0".parse().unwrap(), EntryType::Entrypoint).build(); + PackageInstallSpec::new("http@0.4-0".parse().unwrap(), EntryType::Entrypoint) + .build_behaviour(BuildBehaviour::Force) + .build(); test_install(install_spec).await; std::env::set_var("CFLAGS", cflags); } @@ -57,7 +61,9 @@ async fn install_and_use_toml_edit() { #[cfg(test)] async fn install_and_use(package: PackageReq, module_name: &str) { - let install_spec = PackageInstallSpec::new(package, EntryType::Entrypoint).build(); + let install_spec = PackageInstallSpec::new(package, EntryType::Entrypoint) + .build_behaviour(BuildBehaviour::Force) + .build(); let dir = TempDir::new().unwrap(); let lua_version = detect_installed_lua_version().or(Some(LuaVersion::Lua51)); @@ -97,7 +103,9 @@ async fn no_build_artifacts_in_cwd() { .map(|entry| entry.into_path()) .collect_vec(); let install_spec = - PackageInstallSpec::new("bit32@5.3.5".parse().unwrap(), EntryType::Entrypoint).build(); + PackageInstallSpec::new("bit32@5.3.5".parse().unwrap(), EntryType::Entrypoint) + .build_behaviour(BuildBehaviour::Force) + .build(); test_install(install_spec).await; let build_artifacts = WalkDir::new(&cwd) .into_iter() diff --git a/lux-lua/src/lua_impls.rs b/lux-lua/src/lua_impls.rs index ef13ceef3..7bdc5774e 100644 --- a/lux-lua/src/lua_impls.rs +++ b/lux-lua/src/lua_impls.rs @@ -3329,14 +3329,16 @@ impl From for EntryType { #[serde(rename_all = "snake_case")] enum BuildBehaviourLua { #[default] - NoForce, + Ignore, + Conflict, Force, } impl From for BuildBehaviour { fn from(val: BuildBehaviourLua) -> Self { match val { - BuildBehaviourLua::NoForce => Self::NoForce, + BuildBehaviourLua::Ignore => Self::Ignore, + BuildBehaviourLua::Conflict => Self::Conflict, BuildBehaviourLua::Force => Self::Force, } } @@ -3346,7 +3348,7 @@ impl From for BuildBehaviour { /// ```lua /// "say >= 1.3" /// -/// { package = "say >= 1.3", entry_type = "entrypoint", pin = false, opt = false, build_behaviour = "no_force" } +/// { package = "say >= 1.3", entry_type = "entrypoint", pin = false, opt = false, build_behaviour = "ignore" } /// ``` #[derive(Deserialize)] #[serde(untagged)] diff --git a/lux-lua/src/operations.rs b/lux-lua/src/operations.rs index 9b72835ab..e77a82d20 100644 --- a/lux-lua/src/operations.rs +++ b/lux-lua/src/operations.rs @@ -6,6 +6,7 @@ use std::{collections::HashMap, path::PathBuf}; use itertools::Itertools; use lux_lib::tree::InstallTree; use lux_lib::{ + build::BuildBehaviour, lockfile::LockedPackageId, lua::lua_runtime, operations::{ @@ -373,7 +374,11 @@ fn deps_to_specs(deps: &DependencyType) -> Vec { }; reqs.iter() - .map(|r| PackageInstallSpec::new(r.clone(), EntryType::Entrypoint).build()) + .map(|r| { + PackageInstallSpec::new(r.clone(), EntryType::Entrypoint) + .build_behaviour(BuildBehaviour::Force) + .build() + }) .collect() } @@ -476,7 +481,7 @@ type = "builtin" fn test_package_install_spec_from_full_table() { let lua = Lua::new(); let value = lua - .load(r#"{ package = "busted >= 2.0", entry_type = "dependency_only", pin = true, opt = true, build_behaviour = "no_force" }"#) + .load(r#"{ package = "busted >= 2.0", entry_type = "dependency_only", pin = true, opt = true, build_behaviour = "ignore" }"#) .eval() .unwrap(); PackageInstallSpecLua::from_lua(value, &lua).unwrap(); From cf3e0ab13352585483ef308312bf7e16feafa938 Mon Sep 17 00:00:00 2001 From: vhyrro Date: Mon, 5 Oct 2026 20:43:55 +0200 Subject: [PATCH 24/39] fix: lux-lua compilation --- lux-lua/src/lua_impls.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lux-lua/src/lua_impls.rs b/lux-lua/src/lua_impls.rs index 7bdc5774e..25b0baa8b 100644 --- a/lux-lua/src/lua_impls.rs +++ b/lux-lua/src/lua_impls.rs @@ -1067,7 +1067,7 @@ impl TypedUserData for TreeLua { methods.add_method("root_for", |_, this, package: LockedPackageLua| { Ok(this .0 - .layout_for(&package.0) + .layout_for(package.0.spec()) .root .to_slash_lossy() .into_owned()) @@ -1079,7 +1079,7 @@ impl TypedUserData for TreeLua { methods.document("Get the `RockLayout` for an installed package."); methods.param("package", ""); methods.add_method("rock_layout", |_, this, package: LockedPackageLua| { - Ok(RockLayoutLua(this.0.layout_for(&package.0))) + Ok(RockLayoutLua(this.0.layout_for(package.0.spec()))) }); methods.document("Find installed rocks that match the given `PackageReq`"); methods.param("req", ""); From 878fe9d10d1f5c8e6d78bf2c51e58e062f1f1b61 Mon Sep 17 00:00:00 2001 From: vhyrro Date: Mon, 5 Oct 2026 21:32:14 +0200 Subject: [PATCH 25/39] feat!(resolve): do not reinstall already existing packages, rebuild project on source change --- lux-cli/src/build.rs | 13 +- lux-cli/src/run_lua.rs | 5 +- lux-cli/src/uninstall.rs | 2 +- lux-lib/src/hash.rs | 4 +- lux-lib/src/operations/build_workspace.rs | 6 + lux-lib/src/operations/install_project.rs | 30 ++++- .../operations/pipeline/install_packages.rs | 44 ++++++- .../operations/pipeline/install_workspace.rs | 18 ++- lux-lib/src/operations/pipeline/resolve.rs | 115 +++++++++++++++--- 9 files changed, 209 insertions(+), 28 deletions(-) diff --git a/lux-cli/src/build.rs b/lux-cli/src/build.rs index 3aec2ecf4..0e8b052e2 100644 --- a/lux-cli/src/build.rs +++ b/lux-cli/src/build.rs @@ -1,7 +1,7 @@ use clap::Args; use lux_lib::{ - config::Config, lockfile::LockedPackage, operations::BuildWorkspace, package::PackageName, - workspace::Workspace, + build::BuildBehaviour, config::Config, lockfile::LockedPackage, operations::BuildWorkspace, + package::PackageName, workspace::Workspace, }; use miette::Result; @@ -22,11 +22,20 @@ pub struct Build { /// Returns `Some` if the `only_deps` arg is set to `false`. pub async fn build(data: Build, config: Config) -> Result> { + build_with_behaviour(data, config, BuildBehaviour::Force).await +} + +pub async fn build_with_behaviour( + data: Build, + config: Config, + behaviour: BuildBehaviour, +) -> Result> { let workspace = Workspace::current_or_err()?; let result = BuildWorkspace::new(&workspace, &config) .maybe_package(data.package) .no_lock(data.no_lock) .only_deps(data.only_deps) + .behaviour(behaviour) .build() .await?; Ok(result) diff --git a/lux-cli/src/run_lua.rs b/lux-cli/src/run_lua.rs index 6d0272736..171af0375 100644 --- a/lux-cli/src/run_lua.rs +++ b/lux-cli/src/run_lua.rs @@ -17,6 +17,8 @@ use miette::{IntoDiagnostic, Result, WrapErr}; use crate::build::{self, Build}; +use lux_lib::build::BuildBehaviour; + #[derive(Args, Default)] #[clap(disable_help_flag = true)] pub struct RunLua { @@ -102,7 +104,8 @@ To exit type 'exit()' or . } if workspace.is_some() { - build::build(run_lua.build_args, config.clone()).await?; + build::build_with_behaviour(run_lua.build_args, config.clone(), BuildBehaviour::Ignore) + .await?; } let args = &run_lua.args.unwrap_or_default(); diff --git a/lux-cli/src/uninstall.rs b/lux-cli/src/uninstall.rs index 9678c805f..1aa40a9ad 100644 --- a/lux-cli/src/uninstall.rs +++ b/lux-cli/src/uninstall.rs @@ -103,7 +103,7 @@ multiple packages satisfying your version requirements were found: format!( " Package {} can be removed from the entrypoints, but it is also a dependency, so it will have to be reinstalled. -Overwrite? +Reinstall? ", package_names[0] ) diff --git a/lux-lib/src/hash.rs b/lux-lib/src/hash.rs index cb471b757..7044e23f5 100644 --- a/lux-lib/src/hash.rs +++ b/lux-lib/src/hash.rs @@ -99,7 +99,7 @@ impl FileSystem for VcsExcludingFileSystem { fn read_dir(&self, path: &Utf8Path) -> io::Result> { Ok(FileSystem::read_dir(&NativeFileSystem {}, path)? .into_iter() - .filter(|name| !is_vcs_dir(name)) + .filter(|name| !is_vcs_dir(name) && name != crate::workspace::LUX_DIR_NAME) .filter(|name| { let full = path.join(name); let is_dir = full.as_std_path().is_dir(); @@ -252,6 +252,8 @@ mod tests { write(temp.child(".git/config").path(), "nondeterministic").unwrap(); temp.child(".jj").create_dir_all().unwrap(); write(temp.child(".jj/repo").path(), "nondeterministic").unwrap(); + temp.child(".lux").create_dir_all().unwrap(); + write(temp.child(".lux/tree").path(), "nondeterministic").unwrap(); let hash_with_vcs = temp.path().to_path_buf().hash().await.unwrap(); diff --git a/lux-lib/src/operations/build_workspace.rs b/lux-lib/src/operations/build_workspace.rs index 699e69abd..cec18cb2c 100644 --- a/lux-lib/src/operations/build_workspace.rs +++ b/lux-lib/src/operations/build_workspace.rs @@ -1,4 +1,5 @@ use crate::{ + build::BuildBehaviour, config::Config, lockfile::LockedPackage, lua_installation::LuaInstallationError, @@ -55,6 +56,9 @@ pub struct BuildWorkspace<'a> { /// Build only the dependencies only_deps: bool, + + #[builder(default = BuildBehaviour::Force)] + behaviour: BuildBehaviour, } impl @@ -83,6 +87,7 @@ impl), #[error("package '{0}' was not installed")] PackageNotInstalled(PackageName), + #[error("failed to hash the project's sources")] + Hash(#[source] std::io::Error), } impl From for InstallProjectError { @@ -70,6 +73,9 @@ where config: &'a Config, tree: &'a T, + + #[builder(default = BuildBehaviour::Force)] + behaviour: BuildBehaviour, } impl< @@ -87,6 +93,28 @@ impl< let rockspec = project.local_remote_rockspec()?; let name = rockspec.package().clone(); + + let behaviour = if matches!(args.behaviour, BuildBehaviour::Ignore) { + match tree.lockfile()?.entrypoint(&name) { + Some(existing) => { + let source_hash = project + .root() + .hash() + .await + .map_err(InstallProjectError::Hash)?; + if existing.version() == rockspec.version() + && existing.hashes().source == source_hash + { + return Ok(existing.clone()); + } + BuildBehaviour::Force + } + None => args.behaviour, + } + } else { + args.behaviour + }; + let root = FoundPackage::from_project_root(rockspec, project.root().to_path_buf()); let package_db = RemotePackageDB::from_config(config) .await? @@ -94,7 +122,7 @@ impl< let install_spec = PackageInstallSpec::new(PackageReq::from(name.clone()), tree::EntryType::Entrypoint) - .build_behaviour(BuildBehaviour::Force) + .build_behaviour(behaviour) .build(); InstallPackages::new(config, tree) diff --git a/lux-lib/src/operations/pipeline/install_packages.rs b/lux-lib/src/operations/pipeline/install_packages.rs index 5cfa19896..ffd31655a 100644 --- a/lux-lib/src/operations/pipeline/install_packages.rs +++ b/lux-lib/src/operations/pipeline/install_packages.rs @@ -9,7 +9,10 @@ use tracing::Instrument; use crate::{ build::BuildBehaviour, config::Config, - lockfile::{FlushLockfileError, LockedPackage, LockedPackageId, Lockfile, ReadWrite}, + lockfile::{ + FlushLockfileError, LockedPackage, LockedPackageId, LockedPackageLockType, Lockfile, + ReadWrite, + }, lua_installation::LuaInstallationError, luarocks::luarocks_installation::{LuaRocksError, LuaRocksInstallError}, package::PackageNameList, @@ -133,8 +136,31 @@ where let lockfile = tree.lockfile()?; let build_tree = tree.build_tree(config)?; + let build_lockfile = build_tree.lockfile()?; + + let mut installed: HashMap> = HashMap::new(); + installed.insert( + LockedPackageLockType::Regular, + lockfile.rocks().values().cloned().collect_vec(), + ); + installed.insert( + LockedPackageLockType::Build, + build_lockfile.rocks().values().cloned().collect_vec(), + ); + if install.test_packages.is_some() { + installed.insert( + LockedPackageLockType::Test, + tree.test_tree(config)? + .lockfile()? + .rocks() + .values() + .cloned() + .collect_vec(), + ); + } let mut resolve = ResolvePackageDependencies::new(package_db, config) + .installed(installed) .packages(packages) .build_packages(install.build_packages); if let Some(test_packages) = install.test_packages { @@ -218,14 +244,22 @@ where .await?; } - let installed_packages: HashMap = built + let mut installed_packages: HashMap = lockfile + .rocks() .iter() - .map(|pkg| (pkg.spec().id(), pkg.clone())) + .map(|(id, package)| (id.clone(), package.clone())) .collect(); - let installed_build_deps: HashMap = built_build_deps + installed_packages.extend(built.iter().map(|pkg| (pkg.spec().id(), pkg.clone()))); + let mut installed_build_deps: HashMap = build_lockfile + .rocks() .iter() - .map(|pkg| (pkg.spec().id(), pkg.clone())) + .map(|(id, package)| (id.clone(), package.clone())) .collect(); + installed_build_deps.extend( + built_build_deps + .iter() + .map(|pkg| (pkg.spec().id(), pkg.clone())), + ); lockfile.map_then_flush(|lockfile| { for package in &conflicting_entrypoints { diff --git a/lux-lib/src/operations/pipeline/install_workspace.rs b/lux-lib/src/operations/pipeline/install_workspace.rs index a77a3856e..853da0fe3 100644 --- a/lux-lib/src/operations/pipeline/install_workspace.rs +++ b/lux-lib/src/operations/pipeline/install_workspace.rs @@ -9,7 +9,7 @@ use crate::{ operations::PackageInstallSpec, project::project_toml::LocalProjectTomlValidationError, rockspec::Rockspec, - tree::EntryType, + tree::{EntryType, InstallTree, TreeError}, workspace::{Workspace, WorkspaceError, WorkspaceTreeError}, }; @@ -52,11 +52,22 @@ where let config = args.config; let workspace = args.workspace; + let tree = workspace.tree(config)?; + let build_tree = tree.build_tree(config)?; + let mut regular = gather_dependencies(workspace, DependencyKind::Regular)?; + regular.retain(|spec| !tree.match_rocks(&spec.package).is_ok_and(|m| m.is_found())); regular.extend(args.packages); let build = gather_dependencies(workspace, DependencyKind::Build)?; + let build = build + .into_iter() + .filter(|spec| { + !build_tree + .match_rocks(&spec.package) + .is_ok_and(|m| m.is_found()) + }) + .collect(); - let tree = workspace.tree(config)?; let mut install = InstallPackages::new(config, &tree) .packages(regular) .build_packages(build); @@ -87,6 +98,9 @@ pub enum InstallWorkspaceDependenciesError { WorkspaceTree(#[from] WorkspaceTreeError), #[error(transparent)] #[diagnostic(transparent)] + Tree(#[from] TreeError), + #[error(transparent)] + #[diagnostic(transparent)] Project(#[from] LocalProjectTomlValidationError), #[error(transparent)] #[diagnostic(transparent)] diff --git a/lux-lib/src/operations/pipeline/resolve.rs b/lux-lib/src/operations/pipeline/resolve.rs index a7393fc4e..d12049ca7 100644 --- a/lux-lib/src/operations/pipeline/resolve.rs +++ b/lux-lib/src/operations/pipeline/resolve.rs @@ -11,8 +11,8 @@ use crate::{ build::BuildBehaviour, config::Config, lockfile::{ - LockedPackageId, LockedPackageLockType, LockedPackageSpec, OptState, PinnedState, - RemotePackageSourceUrl, + LockedPackage, LockedPackageId, LockedPackageLockType, LockedPackageSpec, OptState, + PinnedState, RemotePackageSourceUrl, }, lua_rockspec::{BuildBackendSpec, RemoteLuaRockspec}, operations::PackageInstallSpec, @@ -96,6 +96,8 @@ pub(crate) struct ResolvePackageDependencies<'a> { pub(crate) config: &'a Config, #[builder(field)] pub(crate) packages: Artifacts>, + #[builder(default)] + pub(crate) installed: HashMap>, } impl ResolvePackageDependenciesBuilder<'_, State> @@ -127,7 +129,7 @@ where { pub(crate) async fn resolve(self) -> Result { let args = self._build(); - Resolver::new(args.package_db, args.config) + Resolver::new(args.package_db, args.config, args.installed) .run(args.packages) .await } @@ -175,17 +177,29 @@ struct Resolver { seen: HashSet, nodes: HashMap, resolved: ResolvedArtifacts, + installed: HashMap>, + installed_ids: HashMap, } impl Resolver { - fn new(package_db: RemotePackageDB, config: &Config) -> Self { - Self::with_max_concurrent(package_db, Arc::new(config.clone()), config.max_jobs()) + fn new( + package_db: RemotePackageDB, + config: &Config, + installed: HashMap>, + ) -> Self { + Self::with_max_concurrent( + package_db, + Arc::new(config.clone()), + config.max_jobs(), + installed, + ) } fn with_max_concurrent( package_db: RemotePackageDB, config: Arc, max_concurrent: usize, + installed: HashMap>, ) -> Self { Self { package_db, @@ -196,6 +210,8 @@ impl Resolver { seen: HashSet::new(), nodes: HashMap::new(), resolved: ResolvedArtifacts::default(), + installed, + installed_ids: HashMap::new(), } } @@ -309,9 +325,37 @@ impl Resolver { }, ); - for (_, child_key, child_spec) in child_specs { - self.enqueue(child_key, child_spec); + for (is_build, child_key, child_spec) in child_specs { + let child_section = if is_build { + LockedPackageLockType::Build + } else { + section + }; + if let Some(installed_id) = self.installed_match(child_section, &child_spec) { + self.installed_ids.insert(child_key, installed_id); + } else { + self.enqueue(child_key, child_spec); + } + } + } + + fn installed_match( + &self, + section: LockedPackageLockType, + spec: &PackageInstallSpec, + ) -> Option { + if !matches!(spec.build_behaviour, BuildBehaviour::Ignore) || spec.source.is_some() { + return None; } + self.installed + .get(§ion)? + .iter() + .rev() + .find(|package| { + package.name() == spec.package.name() + && spec.package.version_req().matches(package.version()) + }) + .map(|package| package.id()) } /// Computes the final specs and assembles the resolved packages now that discovery is @@ -320,13 +364,17 @@ impl Resolver { let Resolver { nodes, mut resolved, + installed_ids, .. } = self; - let ids: HashMap = nodes + let mut ids: HashMap = nodes .iter() .map(|(key, node)| (key.clone(), node.id.clone())) .collect(); + for (key, id) in installed_ids { + ids.entry(key).or_insert(id); + } for (key, node) in nodes { let Node { @@ -426,6 +474,9 @@ fn detect_cycles(nodes: &HashMap) -> Result<(), ResolveError> top.1 += 1; } let child = &children[index].1; + if !nodes.contains_key(child) { + continue; + } match colors.get(child).copied().unwrap_or(Color::White) { Color::White => { colors.insert(child, Color::Grey); @@ -496,6 +547,7 @@ mod tests { use super::*; use crate::config::ConfigBuilder; + use crate::lockfile::{LockConstraint, LockedPackageHashes}; use crate::package::{PackageSpec, RemotePackage}; use crate::remote_package_db::RemoteSource; @@ -551,6 +603,24 @@ mod tests { .build() } + fn installed_package(name: &str) -> LockedPackage { + let spec = PackageSpec::parse(name.to_string(), "1.0.0-1".to_string()).unwrap(); + LockedPackage::from( + &spec, + LockConstraint::Unconstrained, + RemotePackageSource::Test, + None, + LockedPackageHashes { + rockspec: "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + .parse() + .unwrap(), + source: "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + .parse() + .unwrap(), + }, + ) + } + fn roots(spec: PackageInstallSpec) -> Artifacts> { Artifacts { regular: Some(vec![spec]), @@ -580,22 +650,37 @@ mod tests { rockspec("d", &[]), ]); - let resolved = match Resolver::with_max_concurrent(packages, test_config(), 8) + let resolved = + match Resolver::with_max_concurrent(packages, test_config(), 8, HashMap::new()) + .run(roots(install_spec("a"))) + .await + { + Ok(resolved) => resolved, + Err(err) => panic!("resolution failed: {err}"), + }; + + assert_eq!(resolved.regular.as_ref().map(|rocks| rocks.len()), Some(4)); + } + + #[tokio::test] + async fn installed_dependency_is_reused() { + let packages = local_packages(&[rockspec("a", &["d"])]); + let installed = + HashMap::from([(LockedPackageLockType::Regular, vec![installed_package("d")])]); + + let resolved = Resolver::with_max_concurrent(packages, test_config(), 8, installed) .run(roots(install_spec("a"))) .await - { - Ok(resolved) => resolved, - Err(err) => panic!("resolution failed: {err}"), - }; + .unwrap(); - assert_eq!(resolved.regular.as_ref().map(|rocks| rocks.len()), Some(4)); + assert_eq!(resolved.regular.as_ref().map(|rocks| rocks.len()), Some(1)); } #[tokio::test] async fn cyclic_dependencies_are_rejected() { let packages = local_packages(&[rockspec("a", &["b"]), rockspec("b", &["a"])]); - let result = Resolver::with_max_concurrent(packages, test_config(), 8) + let result = Resolver::with_max_concurrent(packages, test_config(), 8, HashMap::new()) .run(roots(install_spec("a"))) .await; From 3b1866acc658b4d06b8c7ffcf9b53bc45ce64b29 Mon Sep 17 00:00:00 2001 From: vhyrro Date: Mon, 5 Oct 2026 22:16:23 +0200 Subject: [PATCH 26/39] feat!: unify Sync operation across codebase --- lux-cli/src/build.rs | 24 +- lux-cli/src/dist/flat_archive.rs | 7 +- lux-cli/src/install.rs | 4 +- lux-cli/src/outdated.rs | 9 +- lux-cli/src/pack.rs | 8 +- lux-cli/src/pin.rs | 9 +- lux-cli/src/run.rs | 4 +- lux-cli/src/sync.rs | 12 +- lux-cli/src/uninstall.rs | 2 +- lux-cli/src/workspace.rs | 15 +- lux-lib/src/build/mod.rs | 3 +- .../pipeline => drivers}/install_packages.rs | 2 +- lux-lib/src/drivers/mod.rs | 2 + lux-lib/src/drivers/sync.rs | 512 ++++++++++++++++++ .../sync.rs => drivers/sync/reconcile.rs} | 46 +- lux-lib/src/lib.rs | 2 + lux-lib/src/lockfile/mod.rs | 23 + lux-lib/src/luarocks/install_binary_rock.rs | 4 +- lux-lib/src/luarocks/luarocks_installation.rs | 2 +- lux-lib/src/operations/build_workspace.rs | 81 +-- lux-lib/src/operations/exec.rs | 6 +- lux-lib/src/operations/install_project.rs | 22 +- lux-lib/src/operations/install_rockspec.rs | 10 +- lux-lib/src/operations/mod.rs | 4 +- .../operations/pipeline/install_workspace.rs | 189 ------- lux-lib/src/operations/test.rs | 134 +---- lux-lib/src/operations/update.rs | 16 +- lux-lib/src/operations/vendor.rs | 18 +- .../src/{operations => }/pipeline/build.rs | 0 .../{operations => }/pipeline/build_local.rs | 0 .../src/{operations => }/pipeline/discover.rs | 0 .../pipeline/download_sources_and_hash.rs | 0 .../pipeline/emit_lockfile.rs | 2 +- lux-lib/src/{operations => }/pipeline/mod.rs | 2 - .../src/{operations => }/pipeline/resolve.rs | 0 lux-lib/src/remote_package_db/mod.rs | 18 +- lux-lib/src/workspace/mod.rs | 8 + lux-lib/tests/build.rs | 2 +- lux-lib/tests/install.rs | 3 +- lux-lib/tests/sync.rs | 20 +- lux-lua/src/operations.rs | 17 +- 41 files changed, 742 insertions(+), 500 deletions(-) rename lux-lib/src/{operations/pipeline => drivers}/install_packages.rs (99%) create mode 100644 lux-lib/src/drivers/mod.rs create mode 100644 lux-lib/src/drivers/sync.rs rename lux-lib/src/{operations/sync.rs => drivers/sync/reconcile.rs} (96%) delete mode 100644 lux-lib/src/operations/pipeline/install_workspace.rs rename lux-lib/src/{operations => }/pipeline/build.rs (100%) rename lux-lib/src/{operations => }/pipeline/build_local.rs (100%) rename lux-lib/src/{operations => }/pipeline/discover.rs (100%) rename lux-lib/src/{operations => }/pipeline/download_sources_and_hash.rs (100%) rename lux-lib/src/{operations => }/pipeline/emit_lockfile.rs (96%) rename lux-lib/src/{operations => }/pipeline/mod.rs (97%) rename lux-lib/src/{operations => }/pipeline/resolve.rs (100%) diff --git a/lux-cli/src/build.rs b/lux-cli/src/build.rs index 0e8b052e2..284fa06d7 100644 --- a/lux-cli/src/build.rs +++ b/lux-cli/src/build.rs @@ -1,7 +1,11 @@ use clap::Args; use lux_lib::{ - build::BuildBehaviour, config::Config, lockfile::LockedPackage, operations::BuildWorkspace, - package::PackageName, workspace::Workspace, + build::BuildBehaviour, + config::Config, + drivers::sync::{Sync, SyncMode, TargetSet}, + lockfile::LockedPackage, + package::PackageName, + workspace::Workspace, }; use miette::Result; @@ -31,14 +35,18 @@ pub async fn build_with_behaviour( behaviour: BuildBehaviour, ) -> Result> { let workspace = Workspace::current_or_err()?; - let result = BuildWorkspace::new(&workspace, &config) - .maybe_package(data.package) - .no_lock(data.no_lock) - .only_deps(data.only_deps) + let report = Sync::new(&workspace, &config) + .mode(SyncMode::Open) + .targets(TargetSet { + test: false, + members: data.package.map(|package| vec![package]), + }) .behaviour(behaviour) - .build() + .only_deps(data.only_deps) + .no_lock(data.no_lock) + .sync() .await?; - Ok(result) + Ok(report.added().to_vec()) } #[cfg(test)] diff --git a/lux-cli/src/dist/flat_archive.rs b/lux-cli/src/dist/flat_archive.rs index c95dcc99e..71b6e9b67 100644 --- a/lux-cli/src/dist/flat_archive.rs +++ b/lux-cli/src/dist/flat_archive.rs @@ -7,15 +7,14 @@ use clap::Args; use lux_lib::{ build::BuildBehaviour, config::{Config, ConfigBuilder}, + drivers::install_packages::InstallPackages, lockfile::LockedPackage, lua_installation::LuaInstallation, lua_rockspec::RemoteLuaRockspec, lua_version::LuaVersion, - operations::{ - pipeline::{build_local::Build, install_packages::InstallPackages}, - InstallProject, PackageInstallSpec, - }, + operations::{InstallProject, PackageInstallSpec}, package::{PackageName, PackageReq}, + pipeline::build_local::Build, tree::{self, FlatDistTree, InstallTree}, workspace::Workspace, }; diff --git a/lux-cli/src/install.rs b/lux-cli/src/install.rs index 33052f3c3..235694dc6 100644 --- a/lux-cli/src/install.rs +++ b/lux-cli/src/install.rs @@ -2,9 +2,10 @@ use std::path::PathBuf; use lux_lib::{ config::Config, + drivers::install_packages::InstallPackages, lockfile::PinnedState, lua_version::LuaVersion, - operations::{self, pipeline::install_packages::InstallPackages}, + operations, package::{PackageName, PackageReq}, workspace::{Workspace, WorkspaceError}, }; @@ -62,6 +63,7 @@ async fn install_from_path( .project(project) .config(&config) .tree(&tree) + .workspace(&workspace) .build() .await?; diff --git a/lux-cli/src/outdated.rs b/lux-cli/src/outdated.rs index 943b18fa9..6f7483447 100644 --- a/lux-cli/src/outdated.rs +++ b/lux-cli/src/outdated.rs @@ -43,13 +43,14 @@ pub async fn outdated(outdated_data: Outdated, config: Config) -> Result<()> { let rock_list = tree.as_rock_list()?; let rock_list = rock_list .iter() - .map(|rock| { + .filter_map(|rock| { rock.to_package() .has_update(&package_db) - .map(|mb_version| mb_version.map(|version| (rock, version))) + .ok() + .flatten() + .map(|version| (rock, version)) }) - .filter_map_ok(|mb_tuple| mb_tuple) - .try_collect::<_, Vec<(&LockedPackage, PackageVersion)>, _>()?; + .collect::>(); let rock_list = rock_list .iter() diff --git a/lux-cli/src/pack.rs b/lux-cli/src/pack.rs index e9fb0e924..048a84432 100644 --- a/lux-cli/src/pack.rs +++ b/lux-cli/src/pack.rs @@ -6,15 +6,13 @@ use itertools::Itertools; use lux_lib::{ build::BuildBehaviour, config::Config, + drivers::install_packages::InstallPackages, lua_installation::LuaInstallation, lua_rockspec::RemoteLuaRockspec, lua_version::LuaVersion, - operations::{ - self, - pipeline::{build_local::Build, install_packages::InstallPackages}, - PackageInstallSpec, - }, + operations::{self, PackageInstallSpec}, package::PackageName, + pipeline::build_local::Build, rockspec::Rockspec as _, tree::{self, InstallTree}, workspace::Workspace, diff --git a/lux-cli/src/pin.rs b/lux-cli/src/pin.rs index cbdecfac3..c64d414c0 100644 --- a/lux-cli/src/pin.rs +++ b/lux-cli/src/pin.rs @@ -1,6 +1,7 @@ use clap::Args; use itertools::Itertools; use lux_lib::config::Config; +use lux_lib::drivers::sync::{Sync, SyncMode, TargetSet}; use lux_lib::lockfile::PinnedState; use lux_lib::lua_version::LuaVersion; use lux_lib::operations; @@ -82,8 +83,12 @@ pub async fn set_pinned_state(data: ChangePin, config: Config, pin: PinnedState) ) .await?; } - operations::Sync::new(&workspace, &config) - .test(!test_packages.is_empty()) + Sync::new(&workspace, &config) + .mode(SyncMode::Frozen) + .targets(TargetSet { + test: !test_packages.is_empty(), + members: None, + }) .sync() .await .wrap_err("syncing dependencies with the workspace lockfile failed.")?; diff --git a/lux-cli/src/run.rs b/lux-cli/src/run.rs index e3a9caed7..5b7a254ac 100644 --- a/lux-cli/src/run.rs +++ b/lux-cli/src/run.rs @@ -1,7 +1,7 @@ use std::path::PathBuf; use clap::Args; -use lux_lib::{config::Config, operations, workspace::Workspace}; +use lux_lib::{build::BuildBehaviour, config::Config, operations, workspace::Workspace}; use miette::Result; use crate::build::{self, Build}; @@ -31,7 +31,7 @@ pub async fn run(run_args: Run, config: Config) -> Result<()> { let workspace = Workspace::current_or_err()?; let package = run_args.build.package.clone(); - build::build(run_args.build, config.clone()).await?; + build::build_with_behaviour(run_args.build, config.clone(), BuildBehaviour::Ignore).await?; operations::Run::new() .workspace(&workspace) diff --git a/lux-cli/src/sync.rs b/lux-cli/src/sync.rs index 9e439fb88..e031b05c1 100644 --- a/lux-cli/src/sync.rs +++ b/lux-cli/src/sync.rs @@ -1,5 +1,9 @@ use clap::Args; -use lux_lib::{config::Config, operations::Sync, workspace::Workspace}; +use lux_lib::{ + config::Config, + drivers::sync::{Sync, SyncMode, TargetSet}, + workspace::Workspace, +}; use miette::Result; @@ -15,8 +19,12 @@ pub async fn sync(args: SyncProject, config: Config) -> Result<()> { let workspace = Workspace::current_or_err()?; let report = Sync::new(&workspace, &config) + .mode(SyncMode::Frozen) + .targets(TargetSet { + test: true, + members: None, + }) .validate_integrity(!args.no_integrity_check) - .test(true) .sync() .await?; diff --git a/lux-cli/src/uninstall.rs b/lux-cli/src/uninstall.rs index 1aa40a9ad..34d077bef 100644 --- a/lux-cli/src/uninstall.rs +++ b/lux-cli/src/uninstall.rs @@ -151,7 +151,7 @@ Reinstall? .packages(dependencies) .remove() .await?; - operations::pipeline::install_packages::InstallPackages::new(&config, &tree) + lux_lib::drivers::install_packages::InstallPackages::new(&config, &tree) .packages(reinstall_specs) .install() .await?; diff --git a/lux-cli/src/workspace.rs b/lux-cli/src/workspace.rs index f9bb588f3..1ca81d801 100644 --- a/lux-cli/src/workspace.rs +++ b/lux-cli/src/workspace.rs @@ -2,8 +2,13 @@ use std::{collections::HashSet, path::PathBuf, str::FromStr}; use itertools::Itertools; use lux_lib::{ - config::Config, git::shorthand::RemoteGitUrlShorthand, lua_version::LuaVersion, - operations::Sync, package::PackageReq, tree::Tree, workspace::Workspace, + config::Config, + drivers::sync::{Sync, SyncMode, TargetSet}, + git::shorthand::RemoteGitUrlShorthand, + lua_version::LuaVersion, + package::PackageReq, + tree::Tree, + workspace::Workspace, }; use miette::{Context, Result}; use walkdir::WalkDir; @@ -80,7 +85,11 @@ pub async fn sync_dependencies_if_locked(workspace: &Workspace, config: &Config) // NOTE: We only update the lockfile if one exists. // Otherwise, the next `lx build` will remove the packages. Sync::new(workspace, config) - .test(true) + .mode(SyncMode::Frozen) + .targets(TargetSet { + test: true, + members: None, + }) .sync() .await .wrap_err("syncing dependencies with the workspace lockfile failed.")?; diff --git a/lux-lib/src/build/mod.rs b/lux-lib/src/build/mod.rs index 3e5df0a61..69df100e2 100644 --- a/lux-lib/src/build/mod.rs +++ b/lux-lib/src/build/mod.rs @@ -453,8 +453,9 @@ mod tests { lockfile::{LockConstraint, LockedPackageHashes}, lua_installation::{detect_installed_lua_version, LuaInstallation}, lua_version::LuaVersion, - operations::{pipeline::build_local::Build, unpack_rockspec, DownloadedPackedRockBytes}, + operations::{unpack_rockspec, DownloadedPackedRockBytes}, package::PackageSpec, + pipeline::build_local::Build, project::Project, remote_package_source::RemotePackageSource, tree::Tree, diff --git a/lux-lib/src/operations/pipeline/install_packages.rs b/lux-lib/src/drivers/install_packages.rs similarity index 99% rename from lux-lib/src/operations/pipeline/install_packages.rs rename to lux-lib/src/drivers/install_packages.rs index ffd31655a..e20fda2db 100644 --- a/lux-lib/src/operations/pipeline/install_packages.rs +++ b/lux-lib/src/drivers/install_packages.rs @@ -21,7 +21,7 @@ use crate::{ workspace::WorkspaceTreeError, }; -use super::{ +use crate::pipeline::{ build::{Build as PipelineBuild, BuildError as PipelineBuildError}, download_sources_and_hash::{DownloadSourcesAndHash, DownloadSourcesAndHashError}, emit_lockfile::LockfileHandle, diff --git a/lux-lib/src/drivers/mod.rs b/lux-lib/src/drivers/mod.rs new file mode 100644 index 000000000..8b0a230b1 --- /dev/null +++ b/lux-lib/src/drivers/mod.rs @@ -0,0 +1,2 @@ +pub mod install_packages; +pub mod sync; diff --git a/lux-lib/src/drivers/sync.rs b/lux-lib/src/drivers/sync.rs new file mode 100644 index 000000000..aa83d6520 --- /dev/null +++ b/lux-lib/src/drivers/sync.rs @@ -0,0 +1,512 @@ +use bon::Builder; +use miette::Diagnostic; +use thiserror::Error; + +mod reconcile; + +use crate::{ + build::BuildBehaviour, + config::Config, + drivers::install_packages::{InstallPackages, InstallPackagesError}, + lockfile::{LockedPackage, LockedPackageLockType}, + operations::{ + GenLuaRc, GenLuaRcError, InstallProject, InstallProjectError, PackageInstallSpec, + }, + package::PackageName, + project::project_toml::LocalProjectTomlValidationError, + remote_package_db::{RemotePackageDB, RemotePackageDBError}, + rockspec::Rockspec, + tree::{EntryType, InstallTree, TreeError}, + workspace::{Workspace, WorkspaceError, WorkspaceTreeError}, +}; + +/// How a [`Sync`] reconciles the workspace's install trees with its lockfile. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SyncMode { + /// The lockfile may be rewritten. Missing dependencies are resolved from + /// manifests, preferring locked versions. + Open, + /// The lockfile is authoritative. Nothing is resolved from manifests. + Frozen, +} + +/// The set of targets a [`Sync`] should realize. +#[derive(Clone, Debug)] +pub struct TargetSet { + /// Also install test dependencies. + pub test: bool, + /// Members to build. `None` builds every workspace member, `Some(vec![])` + /// builds none. + pub members: Option>, +} + +impl Default for TargetSet { + fn default() -> Self { + Self { + test: false, + members: None, + } + } +} + +#[derive(Debug)] +pub struct SyncReport { + pub(crate) added: Vec, + pub(crate) removed: Vec, +} + +impl SyncReport { + pub fn added(&self) -> &[LockedPackage] { + &self.added + } + pub fn removed(&self) -> &[LockedPackage] { + &self.removed + } + + pub(crate) fn merge(&mut self, other: SyncReport) { + self.added.extend(other.added); + self.removed.extend(other.removed); + } +} + +#[derive(Error, Debug, Diagnostic)] +pub enum SyncError { + #[error(transparent)] + #[diagnostic(transparent)] + InstallPackages(#[from] Box), + #[error(transparent)] + #[diagnostic(transparent)] + InstallProject(#[from] InstallProjectError), + #[error(transparent)] + #[diagnostic(transparent)] + Workspace(#[from] WorkspaceError), + #[error(transparent)] + #[diagnostic(transparent)] + WorkspaceTree(#[from] WorkspaceTreeError), + #[error(transparent)] + #[diagnostic(transparent)] + Tree(#[from] TreeError), + #[error(transparent)] + #[diagnostic(transparent)] + Project(#[from] LocalProjectTomlValidationError), + #[error(transparent)] + #[diagnostic(transparent)] + RemotePackageDB(#[from] RemotePackageDBError), + #[error(transparent)] + #[diagnostic(transparent)] + GenLuaRc(#[from] GenLuaRcError), + #[error(transparent)] + #[diagnostic(transparent)] + Frozen(#[from] reconcile::SyncError), +} + +impl From for SyncError { + fn from(source: InstallPackagesError) -> Self { + Self::InstallPackages(Box::new(source)) + } +} + +/// Reconciles a workspace's install trees with its dependencies. +#[derive(Builder)] +#[builder(start_fn = new, finish_fn(name = _build, vis = ""))] +pub struct Sync<'a> { + #[builder(start_fn)] + workspace: &'a Workspace, + #[builder(start_fn)] + config: &'a Config, + + mode: SyncMode, + + #[builder(default = TargetSet::default())] + targets: TargetSet, + + #[builder(default = BuildBehaviour::Force)] + behaviour: BuildBehaviour, + + /// Build only the dependencies, not the workspace members. + #[builder(default = false)] + only_deps: bool, + + /// Ignore the project's lockfile and don't create one. + #[builder(default = false)] + no_lock: bool, + + /// Whether to validate the integrity of installed packages (`Frozen` mode). + validate_integrity: Option, +} + +impl SyncBuilder<'_, State> +where + State: sync_builder::State + sync_builder::IsComplete, +{ + pub async fn sync(self) -> Result { + let args = self._build(); + match args.mode { + SyncMode::Open => sync_open(&args).await, + SyncMode::Frozen => sync_frozen(&args).await, + } + } +} + +async fn sync_open(args: &Sync<'_>) -> Result { + let config = args.config; + let workspace = args.workspace; + + let tree = workspace.tree(config)?; + let build_tree = tree.build_tree(config)?; + + let mut regular = gather_dependencies(workspace, DependencyKind::Regular, None)?; + regular.retain(|spec| !tree.match_rocks(&spec.package).is_ok_and(|m| m.is_found())); + let build = gather_dependencies(workspace, DependencyKind::Build, None)?; + let build: Vec<_> = build + .into_iter() + .filter(|spec| { + !build_tree + .match_rocks(&spec.package) + .is_ok_and(|m| m.is_found()) + }) + .collect(); + + let test_tree = tree.test_tree(config)?; + let mut test = if args.targets.test { + gather_dependencies( + workspace, + DependencyKind::Test, + args.targets.members.as_deref(), + )? + } else { + Vec::new() + }; + test.retain(|spec| { + !test_tree + .match_rocks(&spec.package) + .is_ok_and(|m| m.is_found()) + }); + + let package_db = if !test.is_empty() || !regular.is_empty() || !build.is_empty() { + Some( + RemotePackageDB::from_config(config) + .await? + .with_locks(workspace.lockfile()?.local_pkg_locks()), + ) + } else { + None + }; + + let mut install = InstallPackages::new(config, &tree) + .packages(regular) + .build_packages(build) + .maybe_package_db(package_db); + if !test.is_empty() { + install = install.test_packages(test); + } + + let (mut added, lockfile) = install.install().await?; + + if !args.no_lock { + let mut workspace_lockfile = workspace.lockfile()?.write_guard(); + lockfile.commit(&mut workspace_lockfile); + if args.targets.test { + workspace_lockfile.sync( + test_tree.lockfile()?.local_pkg_lock(), + &LockedPackageLockType::Test, + ); + } + } + + if !args.only_deps { + let projects = match &args.targets.members { + Some(members) => members + .iter() + .map(|name| workspace.select_member(name)) + .collect::, _>>()?, + None => workspace.members().iter().collect(), + }; + for project in projects { + added.push( + InstallProject::new() + .project(project) + .config(config) + .tree(&tree) + .behaviour(args.behaviour) + .workspace(workspace) + .build() + .await?, + ); + } + } + + if !args.no_lock { + GenLuaRc::new() + .config(config) + .workspace(workspace) + .generate_luarc() + .await?; + } + + Ok(SyncReport { + added, + removed: Vec::new(), + }) +} + +#[derive(Clone, Copy)] +enum DependencyKind { + Regular, + Build, + Test, +} + +fn gather_dependencies( + workspace: &Workspace, + kind: DependencyKind, + members: Option<&[PackageName]>, +) -> Result, SyncError> { + let mut packages = Vec::new(); + for project in workspace.members() { + if let Some(members) = members { + if !members.contains(project.toml().package()) { + continue; + } + } + let toml = project.toml().into_local()?; + if let DependencyKind::Test = kind { + let explicit = toml.test_dependencies().current_platform(); + for dependency in toml.test().current_platform().test_dependencies(project) { + if explicit.iter().any(|dep| dep.name() == dependency.name()) { + continue; + } + packages.push( + PackageInstallSpec::new(dependency, EntryType::Entrypoint) + .build_behaviour(BuildBehaviour::Ignore) + .build(), + ); + } + for dependency in explicit { + if dependency.name().eq(&PackageName::new("lua".into())) { + continue; + } + packages.push( + PackageInstallSpec::new( + dependency.package_req().clone(), + EntryType::Entrypoint, + ) + .build_behaviour(BuildBehaviour::Ignore) + .pin(*dependency.pin()) + .opt(*dependency.opt()) + .maybe_source(dependency.source.clone()) + .build(), + ); + } + } else { + let dependencies = match kind { + DependencyKind::Regular => toml.dependencies().current_platform().clone(), + _ => toml.build_dependencies().current_platform().clone(), + }; + for dependency in dependencies { + // From the perspective of a project, dependencies are entrypoints. + packages.push( + PackageInstallSpec::new( + dependency.package_req().clone(), + EntryType::Entrypoint, + ) + .build_behaviour(BuildBehaviour::Ignore) + .maybe_source(dependency.source().clone()) + .build(), + ); + } + } + } + Ok(packages) +} + +#[cfg(all(test, feature = "impure_tests"))] +mod tests { + use std::path::PathBuf; + + use assert_fs::prelude::{FileWriteStr, PathChild, PathCopy}; + + use crate::{ + config::ConfigBuilder, lockfile::LockedPackageLockType, lua_version::LuaVersion, + package::PackageName, remote_package_db::RemotePackageDB, tree::InstallTree, + workspace::Workspace, + }; + + use super::{Sync, SyncMode, TargetSet}; + + fn no_members() -> TargetSet { + TargetSet { + test: false, + members: Some(Vec::new()), + } + } + + #[tokio::test] + async fn installs_workspace_dependencies() { + let sample = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("resources/test/sample-projects/dependencies/"); + let temp = assert_fs::TempDir::new().unwrap(); + temp.copy_from(sample, &["**"]).unwrap(); + + let workspace = Workspace::from_exact(temp.path()).unwrap().unwrap(); + let config = ConfigBuilder::new() + .unwrap() + .lua_version(Some(LuaVersion::Lua51)) + .build() + .unwrap(); + + Sync::new(&workspace, &config) + .mode(SyncMode::Open) + .targets(no_members()) + .sync() + .await + .unwrap(); + + let lockfile = workspace.lockfile().unwrap(); + let rocks = lockfile.rocks(&LockedPackageLockType::Regular); + assert!(rocks + .values() + .any(|pkg| pkg.name().to_string() == "lua-cjson")); + assert!(rocks + .values() + .any(|pkg| pkg.name().to_string() == "plenary.nvim")); + } + + #[tokio::test] + async fn install_honors_locked_version() { + let temp = assert_fs::TempDir::new().unwrap(); + let toml = temp.child("lux.toml"); + toml.write_str( + r#" + package = "locktest" + version = "0.1.0" + lua = ">=5.1" + + [build] + type = "builtin" + + [dependencies] + argparse = "==0.7.1-1" + "#, + ) + .unwrap(); + + let config = ConfigBuilder::new() + .unwrap() + .lua_version(Some(LuaVersion::Lua51)) + .build() + .unwrap(); + let name = PackageName::from("argparse"); + + let workspace = Workspace::from_exact(temp.path()).unwrap().unwrap(); + Sync::new(&workspace, &config) + .mode(SyncMode::Open) + .targets(no_members()) + .sync() + .await + .unwrap(); + let tree = workspace.tree(&config).unwrap(); + assert_eq!( + tree.lockfile() + .unwrap() + .entrypoint(&name) + .unwrap() + .version() + .to_string(), + "0.7.1-1" + ); + + toml.write_str( + r#" + package = "locktest" + version = "0.1.0" + lua = ">=5.1" + + [build] + type = "builtin" + + [dependencies] + argparse = ">=0.7.1" + "#, + ) + .unwrap(); + std::fs::remove_dir_all(tree.root()).unwrap(); + + let workspace = Workspace::from_exact(temp.path()).unwrap().unwrap(); + Sync::new(&workspace, &config) + .mode(SyncMode::Open) + .targets(no_members()) + .sync() + .await + .unwrap(); + let tree = workspace.tree(&config).unwrap(); + assert_eq!( + tree.lockfile() + .unwrap() + .entrypoint(&name) + .unwrap() + .version() + .to_string(), + "0.7.1-1" + ); + } + + #[tokio::test] + async fn install_without_lock_resolves_latest() { + let temp = assert_fs::TempDir::new().unwrap(); + temp.child("lux.toml") + .write_str( + r#" + package = "locktest" + version = "0.1.0" + lua = ">=5.1" + + [build] + type = "builtin" + + [dependencies] + argparse = ">=0.7.1" + "#, + ) + .unwrap(); + + let config = ConfigBuilder::new() + .unwrap() + .lua_version(Some(LuaVersion::Lua51)) + .build() + .unwrap(); + let name = PackageName::from("argparse"); + + let workspace = Workspace::from_exact(temp.path()).unwrap().unwrap(); + Sync::new(&workspace, &config) + .mode(SyncMode::Open) + .targets(no_members()) + .sync() + .await + .unwrap(); + let tree = workspace.tree(&config).unwrap(); + let installed = tree + .lockfile() + .unwrap() + .entrypoint(&name) + .unwrap() + .version() + .clone(); + let latest = RemotePackageDB::from_config(&config) + .await + .unwrap() + .latest_version(&name) + .unwrap(); + assert_eq!(installed, latest); + assert_ne!(installed.to_string(), "0.7.1-1"); + } +} + +async fn sync_frozen(args: &Sync<'_>) -> Result { + let report = reconcile::Sync::new(args.workspace, args.config) + .validate_integrity(args.validate_integrity.unwrap_or(true)) + .test(args.targets.test) + .sync() + .await?; + Ok(report) +} diff --git a/lux-lib/src/operations/sync.rs b/lux-lib/src/drivers/sync/reconcile.rs similarity index 96% rename from lux-lib/src/operations/sync.rs rename to lux-lib/src/drivers/sync/reconcile.rs index 927a32f7f..d27993c35 100644 --- a/lux-lib/src/operations/sync.rs +++ b/lux-lib/src/drivers/sync/reconcile.rs @@ -1,9 +1,10 @@ use std::io; -use super::{ - pipeline::install_packages::{InstallPackages, InstallPackagesError}, - PackageInstallSpec, RemoveError, Uninstall, -}; +use crate::operations::{PackageInstallSpec, RemoveError, Uninstall}; + +pub use crate::drivers::sync::SyncReport; + +use crate::drivers::install_packages::{InstallPackages, InstallPackagesError}; use crate::{ build::BuildBehaviour, config::Config, @@ -13,8 +14,9 @@ use crate::{ PackageSyncSpec, ReadOnly, ReadWrite, SyncStrategy, WorkspaceLockfile, }, luarocks::luarocks_installation::LUAROCKS_VERSION, - operations::{self, pipeline::resolve::luarocks_build_backend_name, GenLuaRcError}, + operations::{self, GenLuaRcError}, package::{PackageName, PackageReq}, + pipeline::resolve::luarocks_build_backend_name, project::{project_toml::LocalProjectTomlValidationError, ProjectError}, rockspec::{lua_dependency::LuaDependencySpec, Rockspec}, tree::{self, InstallTree, TreeError}, @@ -115,26 +117,6 @@ where } } -#[derive(Debug)] -pub struct SyncReport { - pub(crate) added: Vec, - pub(crate) removed: Vec, -} - -impl SyncReport { - pub fn added(&self) -> &[LockedPackage] { - &self.added - } - pub fn removed(&self) -> &[LockedPackage] { - &self.removed - } - - fn merge(&mut self, other: SyncReport) { - self.added.extend(other.added); - self.removed.extend(other.removed); - } -} - #[derive(Error, Debug, Diagnostic)] pub enum SyncError { #[error(transparent)] @@ -216,6 +198,10 @@ async fn do_sync( let package_db = workspace_lockfile.local_pkg_locks().into(); let (to_add, mut report) = reconcile_locks(&workspace_lockfile, &dest_lockfile, lock_type); + let member_names = args.workspace.member_names(); + report + .removed + .retain(|package| !member_names.contains(package.name())); let packages_to_install = install_specs_to_force(&to_add); report .added @@ -241,8 +227,18 @@ async fn do_sync( .remove() .await?; + let member_packages = install_tree_lockfile + .rocks() + .values() + .filter(|package| member_names.contains(package.name())) + .cloned() + .collect_vec(); + install_tree_lockfile.map_then_flush(|lockfile| { lockfile.sync(workspace_lockfile.local_pkg_lock(lock_type)); + for package in &member_packages { + lockfile.add_entrypoint(package); + } Ok::<_, io::Error>(()) })?; diff --git a/lux-lib/src/lib.rs b/lux-lib/src/lib.rs index b13d41a2b..3e05394aa 100644 --- a/lux-lib/src/lib.rs +++ b/lux-lib/src/lib.rs @@ -1,5 +1,6 @@ pub mod build; pub mod config; +pub mod drivers; pub mod fs; pub mod git; pub mod hash; @@ -13,6 +14,7 @@ pub mod manifest; pub mod operations; pub mod package; pub mod path; +pub mod pipeline; pub mod progress; pub mod project; pub mod remote_package_db; diff --git a/lux-lib/src/lockfile/mod.rs b/lux-lib/src/lockfile/mod.rs index 36bd07c28..ffb817ca8 100644 --- a/lux-lib/src/lockfile/mod.rs +++ b/lux-lib/src/lockfile/mod.rs @@ -677,6 +677,21 @@ impl PackageLock { lock } + pub(crate) fn merge(&mut self, other: &PackageLock) { + self.entrypoints.extend( + other + .entrypoints + .iter() + .map(|(name, id)| (name.clone(), id.clone())), + ); + self.rocks.extend( + other + .rocks + .iter() + .map(|(id, package)| (id.clone(), package.clone())), + ); + } + fn get(&self, id: &LockedPackageId) -> Option<&LockedPackage> { self.rocks.get(id) } @@ -1488,6 +1503,14 @@ impl WorkspaceLockfile { } } } + + pub(crate) fn merge(&mut self, lock: &PackageLock, deps: &LockedPackageLockType) { + match deps { + LockedPackageLockType::Regular => self.dependencies.merge(lock), + LockedPackageLockType::Test => self.test_dependencies.merge(lock), + LockedPackageLockType::Build => self.build_dependencies.merge(lock), + } + } } /// Flushes a lockfile automatically when it goes out of scope diff --git a/lux-lib/src/luarocks/install_binary_rock.rs b/lux-lib/src/luarocks/install_binary_rock.rs index b04ec0938..c81574ca7 100644 --- a/lux-lib/src/luarocks/install_binary_rock.rs +++ b/lux-lib/src/luarocks/install_binary_rock.rs @@ -379,7 +379,7 @@ mod tests { .install() .await .unwrap(); - let layout = tree.layout_for(&local_package); + let layout = tree.layout_for(local_package.spec()); assert!(layout.lib.join("toml_edit.so").is_file()); let orig_install_tree_integrity = layout.root.hash().await.unwrap(); @@ -438,7 +438,7 @@ mod tests { .install() .await .unwrap(); - let layout = tree.layout_for(&local_package); + let layout = tree.layout_for(local_package.spec()); assert!(layout.rockspec_path().is_file()); let new_install_tree_integrity = layout.root.hash().await.unwrap(); assert_eq!(orig_install_tree_integrity, new_install_tree_integrity); diff --git a/lux-lib/src/luarocks/luarocks_installation.rs b/lux-lib/src/luarocks/luarocks_installation.rs index f86c031d7..f5263fd32 100644 --- a/lux-lib/src/luarocks/luarocks_installation.rs +++ b/lux-lib/src/luarocks/luarocks_installation.rs @@ -26,7 +26,7 @@ use crate::tree::{self, Tree, TreeError}; use crate::tree::{Tree, TreeError}; #[cfg(target_family = "unix")] -use crate::operations::pipeline::build_local::Build; +use crate::pipeline::build_local::Build; #[cfg(target_family = "unix")] const LUAROCKS_EXE: &str = "luarocks"; diff --git a/lux-lib/src/operations/build_workspace.rs b/lux-lib/src/operations/build_workspace.rs index cec18cb2c..10cbf8aaa 100644 --- a/lux-lib/src/operations/build_workspace.rs +++ b/lux-lib/src/operations/build_workspace.rs @@ -1,16 +1,10 @@ use crate::{ build::BuildBehaviour, config::Config, + drivers::sync::{Sync, SyncError, SyncMode, TargetSet}, lockfile::LockedPackage, - lua_installation::LuaInstallationError, - operations::{ - pipeline::install_workspace::{ - InstallWorkspaceDependencies, InstallWorkspaceDependenciesError, - }, - GenLuaRc, GenLuaRcError, InstallProject, InstallProjectError, - }, package::PackageName, - workspace::{Workspace, WorkspaceError, WorkspaceTreeError}, + workspace::Workspace, }; use bon::Builder; use thiserror::Error; @@ -21,22 +15,7 @@ use tracing::{info_span, Instrument}; pub enum BuildWorkspaceError { #[error(transparent)] #[diagnostic(transparent)] - InstallWorkspaceDependencies(#[from] InstallWorkspaceDependenciesError), - #[error(transparent)] - #[diagnostic(transparent)] - InstallProject(#[from] InstallProjectError), - #[error(transparent)] - #[diagnostic(transparent)] - WorkspaceTree(#[from] WorkspaceTreeError), - #[error(transparent)] - #[diagnostic(transparent)] - LuaInstallation(#[from] LuaInstallationError), - #[error(transparent)] - #[diagnostic(transparent)] - GenLuaRc(#[from] GenLuaRcError), - #[error(transparent)] - #[diagnostic(transparent)] - Workspace(#[from] WorkspaceError), + Sync(#[from] SyncError), } #[derive(Builder)] @@ -71,52 +50,18 @@ impl info_span!("Building workspace"), }; async move { - InstallWorkspaceDependencies::new(build.config, build.workspace) + let report = Sync::new(build.workspace, build.config) + .mode(SyncMode::Open) + .targets(TargetSet { + test: false, + members: build.package.map(|package| vec![package]), + }) + .behaviour(build.behaviour) + .only_deps(build.only_deps) .no_lock(build.no_lock) - .install() + .sync() .await?; - - let mut built = Vec::new(); - if !build.only_deps { - let workspace_tree = build.workspace.tree(build.config)?; - match &build.package { - Some(package) => { - let project = build.workspace.select_member(package)?; - built.push( - InstallProject::new() - .project(project) - .config(build.config) - .tree(&workspace_tree) - .behaviour(build.behaviour) - .build() - .await?, - ); - } - None => { - for project in build.workspace.members() { - built.push( - InstallProject::new() - .project(project) - .config(build.config) - .tree(&workspace_tree) - .behaviour(build.behaviour) - .build() - .await?, - ); - } - } - } - } - - if !build.no_lock { - GenLuaRc::new() - .config(build.config) - .workspace(build.workspace) - .generate_luarc() - .await?; - } - - Ok(built) + Ok(report.added().to_vec()) } .instrument(span) .await diff --git a/lux-lib/src/operations/exec.rs b/lux-lib/src/operations/exec.rs index 1cd6cc069..3132f9ca5 100644 --- a/lux-lib/src/operations/exec.rs +++ b/lux-lib/src/operations/exec.rs @@ -4,12 +4,10 @@ use tokio::process::Command; use crate::{ build::BuildBehaviour, config::Config, + drivers::install_packages::{InstallPackages, InstallPackagesError}, lua_rockspec::LuaVersionError, lua_version::{LuaVersion, LuaVersionUnset}, - operations::{ - pipeline::install_packages::{InstallPackages, InstallPackagesError}, - BuildWorkspace, BuildWorkspaceError, - }, + operations::{BuildWorkspace, BuildWorkspaceError}, package::{PackageReq, PackageVersionReqError}, path::{Paths, PathsError}, remote_package_db::RemotePackageDBError, diff --git a/lux-lib/src/operations/install_project.rs b/lux-lib/src/operations/install_project.rs index 4aab8fa65..52ebc7173 100644 --- a/lux-lib/src/operations/install_project.rs +++ b/lux-lib/src/operations/install_project.rs @@ -5,20 +5,17 @@ use thiserror::Error; use crate::{ build::{BuildBehaviour, BuildError}, config::Config, + drivers::install_packages::{InstallPackages, InstallPackagesError}, hash::HasIntegrity, lockfile::LockedPackage, - operations::{ - pipeline::{ - discover::FoundPackage, - install_packages::{InstallPackages, InstallPackagesError}, - }, - PackageInstallSpec, - }, + operations::PackageInstallSpec, package::{PackageName, PackageReq}, + pipeline::discover::FoundPackage, project::{IntoLocalRockspecError, Project, ProjectError}, remote_package_db::{RemotePackageDB, RemotePackageDBError}, rockspec::Rockspec, tree::{self, InstallTree, TreeError}, + workspace::{Workspace, WorkspaceError}, }; #[derive(Debug, Error, Diagnostic)] @@ -45,6 +42,9 @@ pub enum InstallProjectError { PackageNotInstalled(PackageName), #[error("failed to hash the project's sources")] Hash(#[source] std::io::Error), + #[error(transparent)] + #[diagnostic(transparent)] + Workspace(#[from] WorkspaceError), } impl From for InstallProjectError { @@ -76,6 +76,9 @@ where #[builder(default = BuildBehaviour::Force)] behaviour: BuildBehaviour, + + /// Workspace whose lockfile is preferred during dependency resolution. + workspace: Option<&'a Workspace>, } impl< @@ -116,8 +119,13 @@ impl< }; let root = FoundPackage::from_project_root(rockspec, project.root().to_path_buf()); + let locks = match args.workspace { + Some(workspace) => workspace.lockfile()?.local_pkg_locks(), + None => Vec::new(), + }; let package_db = RemotePackageDB::from_config(config) .await? + .with_locks(locks) .with_local(vec![root]); let install_spec = diff --git a/lux-lib/src/operations/install_rockspec.rs b/lux-lib/src/operations/install_rockspec.rs index 565e5fe6a..06c1b3aa6 100644 --- a/lux-lib/src/operations/install_rockspec.rs +++ b/lux-lib/src/operations/install_rockspec.rs @@ -5,18 +5,14 @@ use thiserror::Error; use crate::{ build::{BuildBehaviour, BuildError}, config::Config, + drivers::install_packages::{InstallPackages, InstallPackagesError}, lockfile::{LockedPackage, OptState, PinnedState}, lua_installation::LuaInstallationError, lua_rockspec::{LuaVersionError, RemoteLuaRockspec}, luarocks::luarocks_installation::{LuaRocksError, LuaRocksInstallError}, - operations::{ - pipeline::{ - discover::FoundPackage, - install_packages::{InstallPackages, InstallPackagesError}, - }, - PackageInstallSpec, - }, + operations::PackageInstallSpec, package::{PackageName, PackageReq}, + pipeline::discover::FoundPackage, remote_package_db::{RemotePackageDB, RemotePackageDBError}, rockspec::{LuaVersionCompatibility, Rockspec}, tree::{self, InstallTree, TreeError}, diff --git a/lux-lib/src/operations/mod.rs b/lux-lib/src/operations/mod.rs index e3137c5b7..dc98bf181 100644 --- a/lux-lib/src/operations/mod.rs +++ b/lux-lib/src/operations/mod.rs @@ -13,16 +13,15 @@ mod install_project; mod install_rockspec; mod pack; mod pin; -pub mod pipeline; mod run; mod run_lua; -mod sync; mod test; mod uninstall; mod unpack; mod update; mod vendor; +pub use crate::drivers::sync::SyncReport; pub use build_lua::*; pub use build_workspace::*; pub use dist_bin::*; @@ -38,7 +37,6 @@ pub use pack::*; pub use pin::*; pub use run::*; pub use run_lua::*; -pub use sync::*; pub use test::*; pub use uninstall::*; pub use unpack::*; diff --git a/lux-lib/src/operations/pipeline/install_workspace.rs b/lux-lib/src/operations/pipeline/install_workspace.rs deleted file mode 100644 index 853da0fe3..000000000 --- a/lux-lib/src/operations/pipeline/install_workspace.rs +++ /dev/null @@ -1,189 +0,0 @@ -use bon::Builder; -use miette::Diagnostic; -use thiserror::Error; - -use crate::{ - build::BuildBehaviour, - config::Config, - lockfile::LockedPackage, - operations::PackageInstallSpec, - project::project_toml::LocalProjectTomlValidationError, - rockspec::Rockspec, - tree::{EntryType, InstallTree, TreeError}, - workspace::{Workspace, WorkspaceError, WorkspaceTreeError}, -}; - -use super::install_packages::{InstallPackages, InstallPackagesError}; - -/// Installs all of a workspace's dependencies into its regular, build and test trees. -#[derive(Builder)] -#[builder(start_fn = new, finish_fn(name = _build, vis = ""))] -pub struct InstallWorkspaceDependencies<'a> { - #[builder(start_fn)] - config: &'a Config, - #[builder(start_fn)] - workspace: &'a Workspace, - /// Additional packages to install alongside the workspace's own dependencies. - #[builder(field)] - packages: Vec, - /// Also install the workspace's test dependencies. - test: Option, - /// Ignore the project's lockfile and don't create one. - no_lock: Option, -} - -impl InstallWorkspaceDependenciesBuilder<'_, State> -where - State: install_workspace_dependencies_builder::State, -{ - pub fn packages(mut self, packages: Vec) -> Self { - self.packages = packages; - self - } -} - -impl InstallWorkspaceDependenciesBuilder<'_, State> -where - State: install_workspace_dependencies_builder::State - + install_workspace_dependencies_builder::IsComplete, -{ - pub async fn install(self) -> Result, InstallWorkspaceDependenciesError> { - let args = self._build(); - let config = args.config; - let workspace = args.workspace; - - let tree = workspace.tree(config)?; - let build_tree = tree.build_tree(config)?; - - let mut regular = gather_dependencies(workspace, DependencyKind::Regular)?; - regular.retain(|spec| !tree.match_rocks(&spec.package).is_ok_and(|m| m.is_found())); - regular.extend(args.packages); - let build = gather_dependencies(workspace, DependencyKind::Build)?; - let build = build - .into_iter() - .filter(|spec| { - !build_tree - .match_rocks(&spec.package) - .is_ok_and(|m| m.is_found()) - }) - .collect(); - - let mut install = InstallPackages::new(config, &tree) - .packages(regular) - .build_packages(build); - if args.test.unwrap_or(false) { - install = install.test_packages(gather_dependencies(workspace, DependencyKind::Test)?); - } - - let (built, lockfile) = install.install().await?; - - // Commit the workspace lockfile only once installation has succeeded. - if !args.no_lock.unwrap_or(false) { - lockfile.commit(&mut workspace.lockfile()?.write_guard()); - } - - Ok(built) - } -} - -/// The errors that may occur while installing a workspace's dependencies. -#[derive(Error, Debug, Diagnostic)] -#[non_exhaustive] -pub enum InstallWorkspaceDependenciesError { - #[error(transparent)] - #[diagnostic(transparent)] - Workspace(#[from] WorkspaceError), - #[error(transparent)] - #[diagnostic(transparent)] - WorkspaceTree(#[from] WorkspaceTreeError), - #[error(transparent)] - #[diagnostic(transparent)] - Tree(#[from] TreeError), - #[error(transparent)] - #[diagnostic(transparent)] - Project(#[from] LocalProjectTomlValidationError), - #[error(transparent)] - #[diagnostic(transparent)] - InstallPackages(#[from] Box), -} - -impl From for InstallWorkspaceDependenciesError { - fn from(source: InstallPackagesError) -> Self { - Self::InstallPackages(Box::new(source)) - } -} - -#[derive(Clone, Copy)] -enum DependencyKind { - Regular, - Build, - Test, -} - -fn gather_dependencies( - workspace: &Workspace, - kind: DependencyKind, -) -> Result, InstallWorkspaceDependenciesError> { - let mut packages = Vec::new(); - for project in workspace.members() { - let toml = project.toml().into_local()?; - let dependencies = match kind { - DependencyKind::Regular => toml.dependencies().current_platform().clone(), - DependencyKind::Build => toml.build_dependencies().current_platform().clone(), - DependencyKind::Test => toml.test_dependencies().current_platform().clone(), - }; - for dependency in dependencies { - // From the perspective of a project, dependencies are entrypoints. - packages.push( - PackageInstallSpec::new(dependency.package_req().clone(), EntryType::Entrypoint) - .build_behaviour(BuildBehaviour::Ignore) - .maybe_source(dependency.source().clone()) - .build(), - ); - } - } - Ok(packages) -} - -#[cfg(all(test, feature = "impure_tests"))] -mod tests { - use std::path::PathBuf; - - use assert_fs::prelude::PathCopy; - - use crate::{ - config::ConfigBuilder, lockfile::LockedPackageLockType, lua_version::LuaVersion, - workspace::Workspace, - }; - - use super::InstallWorkspaceDependencies; - - #[tokio::test] - async fn installs_workspace_dependencies() { - let sample = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("resources/test/sample-projects/dependencies/"); - let temp = assert_fs::TempDir::new().unwrap(); - temp.copy_from(sample, &["**"]).unwrap(); - - let workspace = Workspace::from_exact(temp.path()).unwrap().unwrap(); - let config = ConfigBuilder::new() - .unwrap() - .lua_version(Some(LuaVersion::Lua51)) - .build() - .unwrap(); - - InstallWorkspaceDependencies::new(&config, &workspace) - .install() - .await - .unwrap(); - - let lockfile = workspace.lockfile().unwrap(); - let rocks = lockfile.rocks(&LockedPackageLockType::Regular); - assert!(rocks - .values() - .any(|pkg| pkg.name().to_string() == "lua-cjson")); - assert!(rocks - .values() - .any(|pkg| pkg.name().to_string() == "plenary.nvim")); - } -} diff --git a/lux-lib/src/operations/test.rs b/lux-lib/src/operations/test.rs index 1498261b5..b75187448 100644 --- a/lux-lib/src/operations/test.rs +++ b/lux-lib/src/operations/test.rs @@ -1,9 +1,6 @@ use std::{io, ops::Deref, path::PathBuf, process::Command}; -use super::{ - pipeline::install_packages::{InstallPackages, InstallPackagesError}, - BuildWorkspace, BuildWorkspaceError, PackageInstallSpec, Sync, SyncError, -}; +use crate::drivers::sync::{Sync, SyncError, SyncMode, TargetSet}; use crate::fs; use crate::tree::InstallTree; use crate::workspace::{WorkspaceError, WorkspaceTreeError}; @@ -12,11 +9,10 @@ use crate::{ config::{Config, ConfigError}, lua_installation::{LuaBinary, LuaBinaryError}, lua_rockspec::{LuaVersionError, TestSpecError, ValidatedTestSpec}, - package::{PackageName, PackageVersionReqError}, + package::PackageName, path::{Paths, PathsError}, project::{project_toml::LocalProjectTomlValidationError, Project, ProjectError}, rockspec::Rockspec, - tree::{self, TreeError}, workspace::Workspace, }; use bon::Builder; @@ -84,12 +80,9 @@ pub enum RunTestsError { #[error(transparent)] #[diagnostic(transparent)] Config(#[from] ConfigError), - #[error(transparent)] - #[diagnostic(transparent)] - InstallTestDependencies(#[from] Box), #[error("build failed")] #[diagnostic(forward(0))] - BuildWorkspace(#[from] Box), + Sync(#[from] Box), #[error("tests failed!")] #[diagnostic(help("see the test runner's output for details"))] TestFailure, @@ -118,9 +111,6 @@ pub enum RunTestsError { #[error(transparent)] #[diagnostic(transparent)] ProjectTomlValidation(#[from] LocalProjectTomlValidationError), - #[error("failed to sync dependencies")] - #[diagnostic(forward(0))] - Sync(#[from] Box), #[error(transparent)] #[diagnostic(transparent)] TestSpec(#[from] TestSpecError), @@ -132,18 +122,6 @@ pub enum RunTestsError { LuaBinary(#[from] LuaBinaryError), } -impl From for RunTestsError { - fn from(source: InstallTestDependenciesError) -> Self { - Self::InstallTestDependencies(Box::new(source)) - } -} - -impl From for RunTestsError { - fn from(source: BuildWorkspaceError) -> Self { - Self::BuildWorkspace(Box::new(source)) - } -} - impl From for RunTestsError { fn from(source: SyncError) -> Self { Self::Sync(Box::new(source)) @@ -186,19 +164,16 @@ async fn run_project_tests( let test_spec = rocks.test().current_platform().to_validated(project)?; let test_config = test_spec.test_config(config)?; - if no_lock { - let rockspec = project.toml().into_local()?; - ensure_test_dependencies(workspace, project, rockspec, &test_config).await?; - } else { - Sync::new(workspace, &test_config).test(true).sync().await?; - } - Box::pin( - BuildWorkspace::new(workspace, &test_config) - .package(project.toml().package().clone()) + Sync::new(workspace, &test_config) + .mode(SyncMode::Open) + .targets(TargetSet { + test: true, + members: Some(vec![project.toml().package().clone()]), + }) .no_lock(no_lock) - .only_deps(false) - .build(), + .behaviour(BuildBehaviour::Ignore) + .sync(), ) .await?; @@ -285,93 +260,6 @@ async fn run_project_tests( } } -#[derive(Error, Debug, Diagnostic)] -#[error("error installing test dependencies")] -#[diagnostic(forward(0))] -pub enum InstallTestDependenciesError { - WorkspaceTree(#[from] WorkspaceTreeError), - Tree(#[from] TreeError), - Install(#[from] Box), - PackageVersionReq(#[from] PackageVersionReqError), -} - -impl From for InstallTestDependenciesError { - fn from(source: InstallPackagesError) -> Self { - Self::Install(Box::new(source)) - } -} - -/// Ensure test dependencies are installed -/// This defaults to the local project tree if cwd is a project root. -async fn ensure_test_dependencies( - workspace: &Workspace, - project: &Project, - rockspec: impl Rockspec, - config: &Config, -) -> Result<(), InstallTestDependenciesError> { - let test_tree = workspace.test_tree(config)?; - let rockspec_dependencies = rockspec.test_dependencies().current_platform(); - let test_dependencies = rockspec - .test() - .current_platform() - .test_dependencies(project) - .iter() - .filter(|test_dep| { - !rockspec_dependencies - .iter() - .any(|dep| dep.name() == test_dep.name()) - }) - .filter_map(|dep| { - let build_behaviour = if test_tree - .match_rocks(dep) - .is_ok_and(|matches| matches.is_found()) - { - Some(BuildBehaviour::Ignore) - } else { - Some(BuildBehaviour::Force) - }; - build_behaviour.map(|build_behaviour| { - PackageInstallSpec::new(dep.clone(), tree::EntryType::Entrypoint) - .build_behaviour(build_behaviour) - .build() - }) - }) - .chain( - rockspec_dependencies - .iter() - .filter(|req| !req.name().eq(&PackageName::new("lua".into()))) - .filter_map(|dep| { - let build_behaviour = if test_tree - .match_rocks(dep.package_req()) - .is_ok_and(|matches| matches.is_found()) - { - Some(BuildBehaviour::Ignore) - } else { - Some(BuildBehaviour::Force) - }; - build_behaviour.map(|build_behaviour| { - PackageInstallSpec::new( - dep.package_req().clone(), - tree::EntryType::Entrypoint, - ) - .build_behaviour(build_behaviour) - .pin(*dep.pin()) - .opt(*dep.opt()) - .maybe_source(dep.source.clone()) - .build() - }) - }), - ) - .collect(); - - InstallPackages::new(config, &test_tree) - .packages(test_dependencies) - .install() - .await?; - - Ok(()) -} - #[cfg(test)] mod tests { use std::path::Path; diff --git a/lux-lib/src/operations/update.rs b/lux-lib/src/operations/update.rs index 14b4cf62a..cda39ae0d 100644 --- a/lux-lib/src/operations/update.rs +++ b/lux-lib/src/operations/update.rs @@ -19,10 +19,10 @@ use itertools::Itertools; use miette::Diagnostic; use thiserror::Error; -use super::{ - pipeline::install_packages::{InstallPackages, InstallPackagesError}, - PackageInstallSpec, RemoveError, SyncError, Uninstall, -}; +use super::{PackageInstallSpec, RemoveError, Uninstall}; + +use crate::drivers::install_packages::{InstallPackages, InstallPackagesError}; +use crate::drivers::sync::{Sync, SyncError, SyncMode, TargetSet}; #[derive(Error, Debug, Diagnostic)] pub enum UpdateError { @@ -159,9 +159,13 @@ async fn update_workspace( let mut project_lockfile = workspace.lockfile()?.write_guard(); let tree = workspace.tree(args.config)?; - let sync_report = super::Sync::new(&workspace, args.config) + let sync_report = Sync::new(&workspace, args.config) + .mode(SyncMode::Frozen) + .targets(TargetSet { + test: true, + members: None, + }) .validate_integrity(args.validate_integrity.unwrap_or(false)) - .test(true) .sync() .await?; diff --git a/lux-lib/src/operations/vendor.rs b/lux-lib/src/operations/vendor.rs index 8cfa2a2af..670ceb878 100644 --- a/lux-lib/src/operations/vendor.rs +++ b/lux-lib/src/operations/vendor.rs @@ -22,19 +22,15 @@ use crate::{ fs, lockfile::LockedPackageLockType, lua_rockspec::{BuildBackendSpec, RemoteLuaRockspec}, - operations::{ - self, - pipeline::{ - download_sources_and_hash::{ - DownloadSourcesAndHash, DownloadSourcesAndHashError, DownloadedPackage, - PackageSource, - }, - resolve::{luarocks_build_backend_name, ResolveError, ResolvePackageDependencies}, - Artifacts, + operations::{self, PackageInstallSpec, UnpackError}, + package::{PackageReq, PackageSpec}, + pipeline::{ + download_sources_and_hash::{ + DownloadSourcesAndHash, DownloadSourcesAndHashError, DownloadedPackage, PackageSource, }, - PackageInstallSpec, UnpackError, + resolve::{luarocks_build_backend_name, ResolveError, ResolvePackageDependencies}, + Artifacts, }, - package::{PackageReq, PackageSpec}, project::project_toml::LocalProjectTomlValidationError, remote_package_db::{RemotePackageDB, RemotePackageDBError}, rockspec::Rockspec, diff --git a/lux-lib/src/operations/pipeline/build.rs b/lux-lib/src/pipeline/build.rs similarity index 100% rename from lux-lib/src/operations/pipeline/build.rs rename to lux-lib/src/pipeline/build.rs diff --git a/lux-lib/src/operations/pipeline/build_local.rs b/lux-lib/src/pipeline/build_local.rs similarity index 100% rename from lux-lib/src/operations/pipeline/build_local.rs rename to lux-lib/src/pipeline/build_local.rs diff --git a/lux-lib/src/operations/pipeline/discover.rs b/lux-lib/src/pipeline/discover.rs similarity index 100% rename from lux-lib/src/operations/pipeline/discover.rs rename to lux-lib/src/pipeline/discover.rs diff --git a/lux-lib/src/operations/pipeline/download_sources_and_hash.rs b/lux-lib/src/pipeline/download_sources_and_hash.rs similarity index 100% rename from lux-lib/src/operations/pipeline/download_sources_and_hash.rs rename to lux-lib/src/pipeline/download_sources_and_hash.rs diff --git a/lux-lib/src/operations/pipeline/emit_lockfile.rs b/lux-lib/src/pipeline/emit_lockfile.rs similarity index 96% rename from lux-lib/src/operations/pipeline/emit_lockfile.rs rename to lux-lib/src/pipeline/emit_lockfile.rs index 5265c0d1e..d76e45148 100644 --- a/lux-lib/src/operations/pipeline/emit_lockfile.rs +++ b/lux-lib/src/pipeline/emit_lockfile.rs @@ -31,7 +31,7 @@ impl LockfileHandle { pub(crate) fn commit(&self, lockfile: &mut WorkspaceLockfile) { self.0.iter().for_each(|(section, lock)| { if let Some(lock) = lock { - lockfile.sync(lock, §ion); + lockfile.merge(lock, §ion); } }); } diff --git a/lux-lib/src/operations/pipeline/mod.rs b/lux-lib/src/pipeline/mod.rs similarity index 97% rename from lux-lib/src/operations/pipeline/mod.rs rename to lux-lib/src/pipeline/mod.rs index 51dc6ff19..d06e93fe3 100644 --- a/lux-lib/src/operations/pipeline/mod.rs +++ b/lux-lib/src/pipeline/mod.rs @@ -5,8 +5,6 @@ pub mod build_local; pub mod discover; pub mod download_sources_and_hash; pub mod emit_lockfile; -pub mod install_packages; -pub mod install_workspace; pub mod resolve; /// Packages grouped by the lockfile section they belong to. diff --git a/lux-lib/src/operations/pipeline/resolve.rs b/lux-lib/src/pipeline/resolve.rs similarity index 100% rename from lux-lib/src/operations/pipeline/resolve.rs rename to lux-lib/src/pipeline/resolve.rs diff --git a/lux-lib/src/remote_package_db/mod.rs b/lux-lib/src/remote_package_db/mod.rs index efb0cffa8..357533cba 100644 --- a/lux-lib/src/remote_package_db/mod.rs +++ b/lux-lib/src/remote_package_db/mod.rs @@ -2,17 +2,15 @@ use crate::{ config::{Config, ConfigError}, lockfile::{LockfileIntegrityError, PackageLock}, manifest::{Manifest, ManifestError}, - operations::{ - pipeline::{ - discover::{DiscoverError, FoundPackage}, - download_sources_and_hash::PackageSource, - }, - Download, FetchVendored, PackageInstallSpec, RemoteRockDownload, - }, + operations::{Download, FetchVendored, PackageInstallSpec, RemoteRockDownload}, package::{ PackageName, PackageReq, PackageSpec, PackageVersion, RemotePackage, RemotePackageTypeFilterSpec, }, + pipeline::{ + discover::{DiscoverError, FoundPackage}, + download_sources_and_hash::PackageSource, + }, }; use itertools::Itertools; @@ -82,6 +80,12 @@ impl RemotePackageDB { self } + /// Prepends locked packages to the lookup order, ahead of manifests. + pub(crate) fn with_locks(mut self, locks: Vec) -> Self { + self.0.insert(0, RemoteSource::LockedPackageLocks(locks)); + self + } + fn find_in_source( source: &RemoteSource, package_req: &PackageReq, diff --git a/lux-lib/src/workspace/mod.rs b/lux-lib/src/workspace/mod.rs index 43ed1b784..25e7f6984 100644 --- a/lux-lib/src/workspace/mod.rs +++ b/lux-lib/src/workspace/mod.rs @@ -159,6 +159,14 @@ impl Workspace { &mut self.members } + /// The package names of all workspace members. + pub fn member_names(&self) -> Vec { + self.members() + .iter() + .map(|project| project.toml().package.clone()) + .collect() + } + /// Get a workspace member, defaulting to the first one if none is specified. /// Fails if a package name is specified, but not found. pub fn single_member_or_select( diff --git a/lux-lib/tests/build.rs b/lux-lib/tests/build.rs index 0c876c11e..4a224635f 100644 --- a/lux-lib/tests/build.rs +++ b/lux-lib/tests/build.rs @@ -12,7 +12,7 @@ use lux_lib::{ config::ConfigBuilder, lua_installation::{detect_installed_lua_version, LuaInstallation}, lua_rockspec::RemoteLuaRockspec, - operations::pipeline::build_local::Build, + pipeline::build_local::Build, tree, }; use tokio::runtime::Builder; diff --git a/lux-lib/tests/install.rs b/lux-lib/tests/install.rs index 323870ae7..08836c02c 100644 --- a/lux-lib/tests/install.rs +++ b/lux-lib/tests/install.rs @@ -4,11 +4,12 @@ use itertools::Itertools; use lux_lib::{ build::BuildBehaviour, config::ConfigBuilder, + drivers::install_packages::InstallPackages, git::{GitRef, GitSource}, lua_installation::detect_installed_lua_version, lua_rockspec::RockSourceSpec, lua_version::LuaVersion, - operations::{pipeline::install_packages::InstallPackages, Exec, PackageInstallSpec}, + operations::{Exec, PackageInstallSpec}, package::{PackageName, PackageReq}, tree::{EntryType, InstallTree}, }; diff --git a/lux-lib/tests/sync.rs b/lux-lib/tests/sync.rs index 55e5af640..811a1a35b 100644 --- a/lux-lib/tests/sync.rs +++ b/lux-lib/tests/sync.rs @@ -2,7 +2,7 @@ use assert_fs::{prelude::PathCopy, TempDir}; use flaky_test::flaky_test; use lux_lib::{ config::ConfigBuilder, - operations::Sync, + drivers::sync::{Sync, SyncMode, TargetSet}, tree::{InstallTree, RockMatches}, workspace::Workspace, }; @@ -22,9 +22,12 @@ async fn sync_test_dependencies_empty_project() { String::from_utf8(tokio::fs::read(workspace.lockfile_path()).await.unwrap()); Sync::new(&workspace, &config) + .mode(SyncMode::Frozen) + .targets(TargetSet { + test: true, + members: None, + }) .validate_integrity(cfg!(not(target_os = "windows"))) - .test(true) - .fast(true) .sync() .await .unwrap(); @@ -71,9 +74,12 @@ async fn sync_multi_projects_same_dependencies() { let config = ConfigBuilder::new().unwrap().build().unwrap(); Sync::new(&workspace, &config) + .mode(SyncMode::Frozen) + .targets(TargetSet { + test: true, + members: None, + }) .validate_integrity(cfg!(not(target_os = "windows"))) - .test(true) - .fast(true) .sync() .await .unwrap(); @@ -121,6 +127,7 @@ fallo = "2.2.0" let workspace_lockfile = project_dir.join("lux.lock"); Sync::new(&workspace, &config) + .mode(SyncMode::Frozen) .validate_integrity(cfg!(not(target_os = "windows"))) .sync() .await @@ -133,6 +140,7 @@ fallo = "2.2.0" tokio::fs::remove_dir_all(workspace_tree).await.unwrap(); Sync::new(&workspace, &config) + .mode(SyncMode::Frozen) .validate_integrity(cfg!(not(target_os = "windows"))) .sync() .await @@ -158,6 +166,7 @@ async fn sync_dependencies_adds_luarocks_build_backend() { let config = ConfigBuilder::new().unwrap().build().unwrap(); Sync::new(&workspace, &config) + .mode(SyncMode::Frozen) .validate_integrity(cfg!(not(target_os = "windows"))) .sync() .await @@ -185,6 +194,7 @@ async fn sync_dependencies_adds_transitive_build_dependencies() { let config = ConfigBuilder::new().unwrap().build().unwrap(); Sync::new(&workspace, &config) + .mode(SyncMode::Frozen) .validate_integrity(cfg!(not(target_os = "windows"))) .sync() .await diff --git a/lux-lua/src/operations.rs b/lux-lua/src/operations.rs index e77a82d20..b7be0b1e9 100644 --- a/lux-lua/src/operations.rs +++ b/lux-lua/src/operations.rs @@ -7,11 +7,13 @@ use itertools::Itertools; use lux_lib::tree::InstallTree; use lux_lib::{ build::BuildBehaviour, + drivers::install_packages::InstallPackages, + drivers::sync::{Sync, SyncMode, TargetSet}, lockfile::LockedPackageId, lua::lua_runtime, operations::{ - pipeline::install_packages::InstallPackages, set_pinned_state, BuildWorkspace, - DistProjectBin, Download, PackageInstallSpec, Sync, Uninstall, Update, + set_pinned_state, BuildWorkspace, DistProjectBin, Download, PackageInstallSpec, Uninstall, + Update, }, package::{PackageName, PackageReq}, remote_package_db::RemotePackageDB, @@ -219,7 +221,11 @@ impl TypedUserData for OperationsModule { |_, (workspace, config): (WorkspaceLua, ConfigLua)| async move { let _runtime = lua_runtime().enter(); Sync::new(&workspace.0, &config.0) - .test(true) + .mode(SyncMode::Frozen) + .targets(TargetSet { + test: true, + members: None, + }) .sync() .await .into_lua_err() @@ -235,6 +241,11 @@ impl TypedUserData for OperationsModule { |_, (workspace, config): (WorkspaceLua, ConfigLua)| async move { let _runtime = lua_runtime().enter(); Sync::new(&workspace.0, &config.0) + .mode(SyncMode::Frozen) + .targets(TargetSet { + test: false, + members: None, + }) .sync() .await .into_lua_err() From 74fc1fd23e81a6c8bdc96cf15e2a59038cc75354 Mon Sep 17 00:00:00 2001 From: vhyrro Date: Tue, 6 Oct 2026 20:37:07 +0200 Subject: [PATCH 27/39] fix: don't include `.luarc.json` and others in project hashes --- lux-lib/src/drivers/sync/reconcile.rs | 10 ---------- lux-lib/src/hash.rs | 18 +++++++++++++----- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/lux-lib/src/drivers/sync/reconcile.rs b/lux-lib/src/drivers/sync/reconcile.rs index d27993c35..9fac4089f 100644 --- a/lux-lib/src/drivers/sync/reconcile.rs +++ b/lux-lib/src/drivers/sync/reconcile.rs @@ -50,16 +50,6 @@ pub struct Sync<'a> { fast: Option, } -impl SyncBuilder<'_, State> -where - State: sync_builder::State, -{ - pub fn add_package(mut self, package: PackageReq) -> Self { - self.extra_packages.push(package); - self - } -} - impl SyncBuilder<'_, State> where State: sync_builder::State + sync_builder::IsComplete, diff --git a/lux-lib/src/hash.rs b/lux-lib/src/hash.rs index 7044e23f5..c1e0e34ad 100644 --- a/lux-lib/src/hash.rs +++ b/lux-lib/src/hash.rs @@ -64,10 +64,9 @@ fn hash_file(path: &Path, integrity_opts: &mut IntegrityOpts) -> io::Result<()> Ok(()) } -/// A [`FileSystem`] that excludes VCS directories (e.g. `.git`, `.jj`) and -/// files ignored by the project's `.gitignore` from the encoded -/// NAR, as their contents are not deterministic and are not part of a -/// package's sources. +/// A [`FileSystem`] that excludes VCS directories (e.g. `.git`, `.jj`, files from `.gitignore`) and +/// lux-generated files (e.g. `.lux`, `lux.lock`, `.luarc.json`) from the NAR, as their contents are +/// not deterministic and are not part of a package's sources. struct VcsExcludingFileSystem { gitignore: Gitignore, } @@ -89,6 +88,13 @@ fn is_vcs_dir(name: &str) -> bool { matches!(name, ".git" | ".jj" | ".hg" | "_darcs" | ".svn" | ".bzr") } +fn is_lux_generated(name: &str) -> bool { + matches!( + name, + crate::workspace::LUX_DIR_NAME | "lux.lock" | ".luarc.json" | ".emmyrc.json" + ) +} + impl FileSystem for VcsExcludingFileSystem { type File = std::fs::File; @@ -99,7 +105,7 @@ impl FileSystem for VcsExcludingFileSystem { fn read_dir(&self, path: &Utf8Path) -> io::Result> { Ok(FileSystem::read_dir(&NativeFileSystem {}, path)? .into_iter() - .filter(|name| !is_vcs_dir(name) && name != crate::workspace::LUX_DIR_NAME) + .filter(|name| !is_vcs_dir(name) && !is_lux_generated(name)) .filter(|name| { let full = path.join(name); let is_dir = full.as_std_path().is_dir(); @@ -254,6 +260,8 @@ mod tests { write(temp.child(".jj/repo").path(), "nondeterministic").unwrap(); temp.child(".lux").create_dir_all().unwrap(); write(temp.child(".lux/tree").path(), "nondeterministic").unwrap(); + write(temp.child("lux.lock").path(), "nondeterministic").unwrap(); + write(temp.child(".luarc.json").path(), "nondeterministic").unwrap(); let hash_with_vcs = temp.path().to_path_buf().hash().await.unwrap(); From 035c961866fca4c5b9b3f76b0e5c6bbda2883f1b Mon Sep 17 00:00:00 2001 From: vhyrro Date: Wed, 7 Oct 2026 08:22:25 +0200 Subject: [PATCH 28/39] fix: `lint` command reinstalling `luacheck` each time --- lux-lib/src/operations/exec.rs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/lux-lib/src/operations/exec.rs b/lux-lib/src/operations/exec.rs index 3132f9ca5..71aea62b7 100644 --- a/lux-lib/src/operations/exec.rs +++ b/lux-lib/src/operations/exec.rs @@ -222,13 +222,19 @@ To suppress this warning, set the `--no-loader` option."# /// Ensure that a command is installed. /// This defaults to the local project tree if cwd is a project root. async fn install_command(command: &str, config: &Config) -> Result<(), InstallCommandError> { - let install_spec = PackageInstallSpec::new( - PackageReq::new(command.into(), None)?, - tree::EntryType::Entrypoint, - ) - .build_behaviour(BuildBehaviour::Force) - .build(); + let package = PackageReq::new(command.into(), None)?; let tree = config.user_tree(LuaVersion::from(config)?.clone())?; + // INVESTIGATE(vhyrro): maybe InstallPackages should do this for us, just ensure + // that it doesn't pull down the manifest each time. + if tree + .match_rocks(&package) + .is_ok_and(|matches| matches.is_found()) + { + return Ok(()); + } + let install_spec = PackageInstallSpec::new(package, tree::EntryType::Entrypoint) + .build_behaviour(BuildBehaviour::Force) + .build(); InstallPackages::new(config, &tree) .package(install_spec) .install() From 9cfdbd08cffc4c2043391c886b9ab3cb7044a178 Mon Sep 17 00:00:00 2001 From: vhyrro Date: Wed, 7 Oct 2026 10:27:42 +0200 Subject: [PATCH 29/39] fix(outdated): don't resync the project --- lux-cli/src/outdated.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lux-cli/src/outdated.rs b/lux-cli/src/outdated.rs index 6f7483447..bf0670c3e 100644 --- a/lux-cli/src/outdated.rs +++ b/lux-cli/src/outdated.rs @@ -10,7 +10,7 @@ use lux_lib::{ use miette::{IntoDiagnostic, Result}; use text_trees::{FormatCharacters, StringTreeNode, TreeFormatting}; -use crate::{args::OutputFormat, workspace::sync_dependencies_if_locked}; +use crate::args::OutputFormat; #[derive(Args)] pub struct Outdated { @@ -23,11 +23,7 @@ pub struct Outdated { pub async fn outdated(outdated_data: Outdated, config: Config) -> Result<()> { let workspace = Workspace::current()?; let tree = match &workspace { - Some(project) => { - // Make sure dependencies are synced if in a project - sync_dependencies_if_locked(project, &config).await?; - project.tree(&config)? - } + Some(project) => project.tree(&config)?, None => { let lua_version = LuaVersion::from(&config)?.clone(); config.user_tree(lua_version)? @@ -95,5 +91,7 @@ pub async fn outdated(outdated_data: Outdated, config: Config) -> Result<()> { } } + println!("\nRun `lx update` to update all outdated rocks."); + Ok(()) } From e55f4ef87b7f98d137baf6a0432618378550435e Mon Sep 17 00:00:00 2001 From: vhyrro Date: Wed, 7 Oct 2026 12:06:00 +0200 Subject: [PATCH 30/39] fix(build): force install unavailable packages (allows overwrites and prevents conflicting entrypoints by removing the old version of the package and replacing it with a new one) --- lux-lib/src/drivers/sync.rs | 116 +++++++++++++++++++++++--- lux-lib/src/drivers/sync/reconcile.rs | 11 +++ 2 files changed, 116 insertions(+), 11 deletions(-) diff --git a/lux-lib/src/drivers/sync.rs b/lux-lib/src/drivers/sync.rs index aa83d6520..57c3a1a3b 100644 --- a/lux-lib/src/drivers/sync.rs +++ b/lux-lib/src/drivers/sync.rs @@ -155,33 +155,49 @@ async fn sync_open(args: &Sync<'_>) -> Result { let tree = workspace.tree(config)?; let build_tree = tree.build_tree(config)?; - let mut regular = gather_dependencies(workspace, DependencyKind::Regular, None)?; - regular.retain(|spec| !tree.match_rocks(&spec.package).is_ok_and(|m| m.is_found())); - let build = gather_dependencies(workspace, DependencyKind::Build, None)?; - let build: Vec<_> = build + let regular = gather_dependencies(workspace, DependencyKind::Regular, None)? + .into_iter() + .filter(|spec| !tree.match_rocks(&spec.package).is_ok_and(|m| m.is_found())) + .map(|spec| PackageInstallSpec { + build_behaviour: BuildBehaviour::Force, + ..spec + }) + .collect::>(); + + let build = gather_dependencies(workspace, DependencyKind::Build, None)? .into_iter() .filter(|spec| { !build_tree .match_rocks(&spec.package) .is_ok_and(|m| m.is_found()) }) - .collect(); + .map(|spec| PackageInstallSpec { + build_behaviour: BuildBehaviour::Force, + ..spec + }) + .collect::>(); let test_tree = tree.test_tree(config)?; - let mut test = if args.targets.test { + let test = if args.targets.test { gather_dependencies( workspace, DependencyKind::Test, args.targets.members.as_deref(), )? + .into_iter() + .filter(|spec| { + !test_tree + .match_rocks(&spec.package) + .is_ok_and(|m| m.is_found()) + }) + .map(|spec| PackageInstallSpec { + build_behaviour: BuildBehaviour::Force, + ..spec + }) + .collect::>() } else { Vec::new() }; - test.retain(|spec| { - !test_tree - .match_rocks(&spec.package) - .is_ok_and(|m| m.is_found()) - }); let package_db = if !test.is_empty() || !regular.is_empty() || !build.is_empty() { Some( @@ -500,6 +516,84 @@ mod tests { assert_eq!(installed, latest); assert_ne!(installed.to_string(), "0.7.1-1"); } + + #[tokio::test] + async fn replaces_dependency_on_version_change() { + let temp = assert_fs::TempDir::new().unwrap(); + let toml = temp.child("lux.toml"); + toml.write_str( + r#" + package = "versionchange" + version = "0.1.0" + lua = ">=5.1" + + [build] + type = "builtin" + + [dependencies] + argparse = "==0.7.1-1" + "#, + ) + .unwrap(); + + let config = ConfigBuilder::new() + .unwrap() + .lua_version(Some(LuaVersion::Lua51)) + .build() + .unwrap(); + let name = PackageName::from("argparse"); + + let workspace = Workspace::from_exact(temp.path()).unwrap().unwrap(); + Sync::new(&workspace, &config) + .mode(SyncMode::Open) + .targets(no_members()) + .sync() + .await + .unwrap(); + let tree = workspace.tree(&config).unwrap(); + assert_eq!( + tree.lockfile() + .unwrap() + .entrypoint(&name) + .unwrap() + .version() + .to_string(), + "0.7.1-1" + ); + + toml.write_str( + r#" + package = "versionchange" + version = "0.1.0" + lua = ">=5.1" + + [build] + type = "builtin" + + [dependencies] + argparse = "==0.7.2-1" + "#, + ) + .unwrap(); + + let workspace = Workspace::from_exact(temp.path()).unwrap().unwrap(); + Sync::new(&workspace, &config) + .mode(SyncMode::Open) + .targets(no_members()) + .sync() + .await + .unwrap(); + let tree = workspace.tree(&config).unwrap(); + assert_eq!( + tree.lockfile() + .unwrap() + .entrypoint(&name) + .unwrap() + .version() + .to_string(), + "0.7.2-1" + ); + } } async fn sync_frozen(args: &Sync<'_>) -> Result { diff --git a/lux-lib/src/drivers/sync/reconcile.rs b/lux-lib/src/drivers/sync/reconcile.rs index 9fac4089f..ee1b3c03c 100644 --- a/lux-lib/src/drivers/sync/reconcile.rs +++ b/lux-lib/src/drivers/sync/reconcile.rs @@ -50,6 +50,17 @@ pub struct Sync<'a> { fast: Option, } +#[cfg(all(test, feature = "impure_tests"))] +impl SyncBuilder<'_, State> +where + State: sync_builder::State, +{ + pub fn add_package(mut self, package: PackageReq) -> Self { + self.extra_packages.push(package); + self + } +} + impl SyncBuilder<'_, State> where State: sync_builder::State + sync_builder::IsComplete, From 3747485f614f7be872b090122bde0c397975d177 Mon Sep 17 00:00:00 2001 From: vhyrro Date: Wed, 7 Oct 2026 18:38:11 +0200 Subject: [PATCH 31/39] fix(TargetSet): use constructors instead of explicit construction --- lux-cli/src/build.rs | 6 +- lux-cli/src/pin.rs | 5 +- lux-cli/src/sync.rs | 5 +- lux-cli/src/workspace.rs | 5 +- lux-lib/src/build/mod.rs | 3 +- lux-lib/src/drivers/sync.rs | 120 +++++++++++----------- lux-lib/src/operations/build_workspace.rs | 6 +- lux-lib/src/operations/exec.rs | 3 +- lux-lib/src/operations/test.rs | 5 +- lux-lib/src/operations/update.rs | 5 +- lux-lib/src/workspace/mod.rs | 13 ++- lux-lib/tests/sync.rs | 14 ++- lux-lua/src/operations.rs | 10 +- 13 files changed, 89 insertions(+), 111 deletions(-) diff --git a/lux-cli/src/build.rs b/lux-cli/src/build.rs index 284fa06d7..3d2b1530a 100644 --- a/lux-cli/src/build.rs +++ b/lux-cli/src/build.rs @@ -37,9 +37,9 @@ pub async fn build_with_behaviour( let workspace = Workspace::current_or_err()?; let report = Sync::new(&workspace, &config) .mode(SyncMode::Open) - .targets(TargetSet { - test: false, - members: data.package.map(|package| vec![package]), + .targets(match data.package { + Some(package) => TargetSet::member(package, false), + None => TargetSet::all(&workspace, false), }) .behaviour(behaviour) .only_deps(data.only_deps) diff --git a/lux-cli/src/pin.rs b/lux-cli/src/pin.rs index c64d414c0..3f04a2dfd 100644 --- a/lux-cli/src/pin.rs +++ b/lux-cli/src/pin.rs @@ -85,10 +85,7 @@ pub async fn set_pinned_state(data: ChangePin, config: Config, pin: PinnedState) } Sync::new(&workspace, &config) .mode(SyncMode::Frozen) - .targets(TargetSet { - test: !test_packages.is_empty(), - members: None, - }) + .targets(TargetSet::all(&workspace, !test_packages.is_empty())) .sync() .await .wrap_err("syncing dependencies with the workspace lockfile failed.")?; diff --git a/lux-cli/src/sync.rs b/lux-cli/src/sync.rs index e031b05c1..70684562d 100644 --- a/lux-cli/src/sync.rs +++ b/lux-cli/src/sync.rs @@ -20,10 +20,7 @@ pub async fn sync(args: SyncProject, config: Config) -> Result<()> { let report = Sync::new(&workspace, &config) .mode(SyncMode::Frozen) - .targets(TargetSet { - test: true, - members: None, - }) + .targets(TargetSet::all(&workspace, true)) .validate_integrity(!args.no_integrity_check) .sync() .await?; diff --git a/lux-cli/src/workspace.rs b/lux-cli/src/workspace.rs index 1ca81d801..075923e53 100644 --- a/lux-cli/src/workspace.rs +++ b/lux-cli/src/workspace.rs @@ -86,10 +86,7 @@ pub async fn sync_dependencies_if_locked(workspace: &Workspace, config: &Config) // Otherwise, the next `lx build` will remove the packages. Sync::new(workspace, config) .mode(SyncMode::Frozen) - .targets(TargetSet { - test: true, - members: None, - }) + .targets(TargetSet::all(workspace, true)) .sync() .await .wrap_err("syncing dependencies with the workspace lockfile failed.")?; diff --git a/lux-lib/src/build/mod.rs b/lux-lib/src/build/mod.rs index 69df100e2..0bab6c130 100644 --- a/lux-lib/src/build/mod.rs +++ b/lux-lib/src/build/mod.rs @@ -293,7 +293,8 @@ pub(crate) async fn deploy( } } - // FIXME(vhyrro): Maybe make prepare/finalize a struct with Drop behaviour? If it makes sense only. + // TODO(vhyrro): When we implement transactionality make this a Drop guard + // or something else. tree.prepare(&package.spec)?; let layout = tree.layout_for(&package.spec); diff --git a/lux-lib/src/drivers/sync.rs b/lux-lib/src/drivers/sync.rs index 57c3a1a3b..02b045913 100644 --- a/lux-lib/src/drivers/sync.rs +++ b/lux-lib/src/drivers/sync.rs @@ -1,5 +1,6 @@ use bon::Builder; use miette::Diagnostic; +use nonempty::NonEmpty; use thiserror::Error; mod reconcile; @@ -20,7 +21,6 @@ use crate::{ workspace::{Workspace, WorkspaceError, WorkspaceTreeError}, }; -/// How a [`Sync`] reconciles the workspace's install trees with its lockfile. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum SyncMode { /// The lockfile may be rewritten. Missing dependencies are resolved from @@ -30,21 +30,24 @@ pub enum SyncMode { Frozen, } -/// The set of targets a [`Sync`] should realize. #[derive(Clone, Debug)] pub struct TargetSet { - /// Also install test dependencies. pub test: bool, - /// Members to build. `None` builds every workspace member, `Some(vec![])` - /// builds none. - pub members: Option>, + pub members: NonEmpty, } -impl Default for TargetSet { - fn default() -> Self { +impl TargetSet { + pub fn all(workspace: &Workspace, test: bool) -> Self { Self { - test: false, - members: None, + test, + members: workspace.member_names(), + } + } + + pub fn member(member: PackageName, test: bool) -> Self { + Self { + test, + members: NonEmpty::new(member), } } } @@ -117,7 +120,6 @@ pub struct Sync<'a> { mode: SyncMode, - #[builder(default = TargetSet::default())] targets: TargetSet, #[builder(default = BuildBehaviour::Force)] @@ -155,16 +157,20 @@ async fn sync_open(args: &Sync<'_>) -> Result { let tree = workspace.tree(config)?; let build_tree = tree.build_tree(config)?; - let regular = gather_dependencies(workspace, DependencyKind::Regular, None)? - .into_iter() - .filter(|spec| !tree.match_rocks(&spec.package).is_ok_and(|m| m.is_found())) - .map(|spec| PackageInstallSpec { - build_behaviour: BuildBehaviour::Force, - ..spec - }) - .collect::>(); + let regular = gather_dependencies( + workspace, + DependencyKind::Regular, + &workspace.member_names(), + )? + .into_iter() + .filter(|spec| !tree.match_rocks(&spec.package).is_ok_and(|m| m.is_found())) + .map(|spec| PackageInstallSpec { + build_behaviour: BuildBehaviour::Force, + ..spec + }) + .collect::>(); - let build = gather_dependencies(workspace, DependencyKind::Build, None)? + let build = gather_dependencies(workspace, DependencyKind::Build, &workspace.member_names())? .into_iter() .filter(|spec| { !build_tree @@ -179,22 +185,18 @@ async fn sync_open(args: &Sync<'_>) -> Result { let test_tree = tree.test_tree(config)?; let test = if args.targets.test { - gather_dependencies( - workspace, - DependencyKind::Test, - args.targets.members.as_deref(), - )? - .into_iter() - .filter(|spec| { - !test_tree - .match_rocks(&spec.package) - .is_ok_and(|m| m.is_found()) - }) - .map(|spec| PackageInstallSpec { - build_behaviour: BuildBehaviour::Force, - ..spec - }) - .collect::>() + gather_dependencies(workspace, DependencyKind::Test, &args.targets.members)? + .into_iter() + .filter(|spec| { + !test_tree + .match_rocks(&spec.package) + .is_ok_and(|m| m.is_found()) + }) + .map(|spec| PackageInstallSpec { + build_behaviour: BuildBehaviour::Force, + ..spec + }) + .collect::>() } else { Vec::new() }; @@ -231,13 +233,12 @@ async fn sync_open(args: &Sync<'_>) -> Result { } if !args.only_deps { - let projects = match &args.targets.members { - Some(members) => members - .iter() - .map(|name| workspace.select_member(name)) - .collect::, _>>()?, - None => workspace.members().iter().collect(), - }; + let projects = args + .targets + .members + .iter() + .map(|name| workspace.select_member(name)) + .collect::, _>>()?; for project in projects { added.push( InstallProject::new() @@ -276,14 +277,12 @@ enum DependencyKind { fn gather_dependencies( workspace: &Workspace, kind: DependencyKind, - members: Option<&[PackageName]>, + members: &NonEmpty, ) -> Result, SyncError> { let mut packages = Vec::new(); for project in workspace.members() { - if let Some(members) = members { - if !members.contains(project.toml().package()) { - continue; - } + if !members.contains(project.toml().package()) { + continue; } let toml = project.toml().into_local()?; if let DependencyKind::Test = kind { @@ -350,13 +349,6 @@ mod tests { use super::{Sync, SyncMode, TargetSet}; - fn no_members() -> TargetSet { - TargetSet { - test: false, - members: Some(Vec::new()), - } - } - #[tokio::test] async fn installs_workspace_dependencies() { let sample = PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -373,7 +365,8 @@ mod tests { Sync::new(&workspace, &config) .mode(SyncMode::Open) - .targets(no_members()) + .targets(TargetSet::all(&workspace, false)) + .only_deps(true) .sync() .await .unwrap(); @@ -417,7 +410,8 @@ mod tests { let workspace = Workspace::from_exact(temp.path()).unwrap().unwrap(); Sync::new(&workspace, &config) .mode(SyncMode::Open) - .targets(no_members()) + .targets(TargetSet::all(&workspace, false)) + .only_deps(true) .sync() .await .unwrap(); @@ -451,7 +445,8 @@ mod tests { let workspace = Workspace::from_exact(temp.path()).unwrap().unwrap(); Sync::new(&workspace, &config) .mode(SyncMode::Open) - .targets(no_members()) + .targets(TargetSet::all(&workspace, false)) + .only_deps(true) .sync() .await .unwrap(); @@ -496,7 +491,8 @@ mod tests { let workspace = Workspace::from_exact(temp.path()).unwrap().unwrap(); Sync::new(&workspace, &config) .mode(SyncMode::Open) - .targets(no_members()) + .targets(TargetSet::all(&workspace, false)) + .only_deps(true) .sync() .await .unwrap(); @@ -546,7 +542,8 @@ mod tests { let workspace = Workspace::from_exact(temp.path()).unwrap().unwrap(); Sync::new(&workspace, &config) .mode(SyncMode::Open) - .targets(no_members()) + .targets(TargetSet::all(&workspace, false)) + .only_deps(true) .sync() .await .unwrap(); @@ -579,7 +576,8 @@ mod tests { let workspace = Workspace::from_exact(temp.path()).unwrap().unwrap(); Sync::new(&workspace, &config) .mode(SyncMode::Open) - .targets(no_members()) + .targets(TargetSet::all(&workspace, false)) + .only_deps(true) .sync() .await .unwrap(); diff --git a/lux-lib/src/operations/build_workspace.rs b/lux-lib/src/operations/build_workspace.rs index 10cbf8aaa..340028fd0 100644 --- a/lux-lib/src/operations/build_workspace.rs +++ b/lux-lib/src/operations/build_workspace.rs @@ -52,9 +52,9 @@ impl TargetSet::member(package, false), + None => TargetSet::all(build.workspace, false), }) .behaviour(build.behaviour) .only_deps(build.only_deps) diff --git a/lux-lib/src/operations/exec.rs b/lux-lib/src/operations/exec.rs index 71aea62b7..cc1fd4c38 100644 --- a/lux-lib/src/operations/exec.rs +++ b/lux-lib/src/operations/exec.rs @@ -224,8 +224,7 @@ To suppress this warning, set the `--no-loader` option."# async fn install_command(command: &str, config: &Config) -> Result<(), InstallCommandError> { let package = PackageReq::new(command.into(), None)?; let tree = config.user_tree(LuaVersion::from(config)?.clone())?; - // INVESTIGATE(vhyrro): maybe InstallPackages should do this for us, just ensure - // that it doesn't pull down the manifest each time. + if tree .match_rocks(&package) .is_ok_and(|matches| matches.is_found()) diff --git a/lux-lib/src/operations/test.rs b/lux-lib/src/operations/test.rs index b75187448..1fcc59b41 100644 --- a/lux-lib/src/operations/test.rs +++ b/lux-lib/src/operations/test.rs @@ -167,10 +167,7 @@ async fn run_project_tests( Box::pin( Sync::new(workspace, &test_config) .mode(SyncMode::Open) - .targets(TargetSet { - test: true, - members: Some(vec![project.toml().package().clone()]), - }) + .targets(TargetSet::member(project.toml().package().clone(), true)) .no_lock(no_lock) .behaviour(BuildBehaviour::Ignore) .sync(), diff --git a/lux-lib/src/operations/update.rs b/lux-lib/src/operations/update.rs index cda39ae0d..e519fcc40 100644 --- a/lux-lib/src/operations/update.rs +++ b/lux-lib/src/operations/update.rs @@ -161,10 +161,7 @@ async fn update_workspace( let sync_report = Sync::new(&workspace, args.config) .mode(SyncMode::Frozen) - .targets(TargetSet { - test: true, - members: None, - }) + .targets(TargetSet::all(&workspace, true)) .validate_integrity(args.validate_integrity.unwrap_or(false)) .sync() .await?; diff --git a/lux-lib/src/workspace/mod.rs b/lux-lib/src/workspace/mod.rs index 25e7f6984..41805f544 100644 --- a/lux-lib/src/workspace/mod.rs +++ b/lux-lib/src/workspace/mod.rs @@ -160,11 +160,14 @@ impl Workspace { } /// The package names of all workspace members. - pub fn member_names(&self) -> Vec { - self.members() - .iter() - .map(|project| project.toml().package.clone()) - .collect() + pub fn member_names(&self) -> NonEmpty { + NonEmpty::from_vec( + self.members() + .iter() + .map(|project| project.toml().package.clone()) + .collect(), + ) + .expect("workspace has at least one member") } /// Get a workspace member, defaulting to the first one if none is specified. diff --git a/lux-lib/tests/sync.rs b/lux-lib/tests/sync.rs index 811a1a35b..b9fe0e3c0 100644 --- a/lux-lib/tests/sync.rs +++ b/lux-lib/tests/sync.rs @@ -23,10 +23,7 @@ async fn sync_test_dependencies_empty_project() { Sync::new(&workspace, &config) .mode(SyncMode::Frozen) - .targets(TargetSet { - test: true, - members: None, - }) + .targets(TargetSet::all(&workspace, true)) .validate_integrity(cfg!(not(target_os = "windows"))) .sync() .await @@ -75,10 +72,7 @@ async fn sync_multi_projects_same_dependencies() { Sync::new(&workspace, &config) .mode(SyncMode::Frozen) - .targets(TargetSet { - test: true, - members: None, - }) + .targets(TargetSet::all(&workspace, true)) .validate_integrity(cfg!(not(target_os = "windows"))) .sync() .await @@ -128,6 +122,7 @@ fallo = "2.2.0" Sync::new(&workspace, &config) .mode(SyncMode::Frozen) + .targets(TargetSet::all(&workspace, false)) .validate_integrity(cfg!(not(target_os = "windows"))) .sync() .await @@ -141,6 +136,7 @@ fallo = "2.2.0" Sync::new(&workspace, &config) .mode(SyncMode::Frozen) + .targets(TargetSet::all(&workspace, false)) .validate_integrity(cfg!(not(target_os = "windows"))) .sync() .await @@ -167,6 +163,7 @@ async fn sync_dependencies_adds_luarocks_build_backend() { Sync::new(&workspace, &config) .mode(SyncMode::Frozen) + .targets(TargetSet::all(&workspace, false)) .validate_integrity(cfg!(not(target_os = "windows"))) .sync() .await @@ -195,6 +192,7 @@ async fn sync_dependencies_adds_transitive_build_dependencies() { Sync::new(&workspace, &config) .mode(SyncMode::Frozen) + .targets(TargetSet::all(&workspace, false)) .validate_integrity(cfg!(not(target_os = "windows"))) .sync() .await diff --git a/lux-lua/src/operations.rs b/lux-lua/src/operations.rs index b7be0b1e9..d9f2b2365 100644 --- a/lux-lua/src/operations.rs +++ b/lux-lua/src/operations.rs @@ -222,10 +222,7 @@ impl TypedUserData for OperationsModule { let _runtime = lua_runtime().enter(); Sync::new(&workspace.0, &config.0) .mode(SyncMode::Frozen) - .targets(TargetSet { - test: true, - members: None, - }) + .targets(TargetSet::all(&workspace.0, true)) .sync() .await .into_lua_err() @@ -242,10 +239,7 @@ impl TypedUserData for OperationsModule { let _runtime = lua_runtime().enter(); Sync::new(&workspace.0, &config.0) .mode(SyncMode::Frozen) - .targets(TargetSet { - test: false, - members: None, - }) + .targets(TargetSet::all(&workspace.0, false)) .sync() .await .into_lua_err() From 9683dc20ea12f71d0c987e7ca301ee89557b149e Mon Sep 17 00:00:00 2001 From: vhyrro Date: Thu, 8 Oct 2026 10:33:17 +0200 Subject: [PATCH 32/39] refactor: `reconcile` -> `sync::frozen` --- .../drivers/sync/{reconcile.rs => frozen.rs} | 0 lux-lib/src/drivers/{sync.rs => sync/mod.rs} | 22 +++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) rename lux-lib/src/drivers/sync/{reconcile.rs => frozen.rs} (100%) rename lux-lib/src/drivers/{sync.rs => sync/mod.rs} (99%) diff --git a/lux-lib/src/drivers/sync/reconcile.rs b/lux-lib/src/drivers/sync/frozen.rs similarity index 100% rename from lux-lib/src/drivers/sync/reconcile.rs rename to lux-lib/src/drivers/sync/frozen.rs diff --git a/lux-lib/src/drivers/sync.rs b/lux-lib/src/drivers/sync/mod.rs similarity index 99% rename from lux-lib/src/drivers/sync.rs rename to lux-lib/src/drivers/sync/mod.rs index 02b045913..249fbf1ae 100644 --- a/lux-lib/src/drivers/sync.rs +++ b/lux-lib/src/drivers/sync/mod.rs @@ -3,7 +3,7 @@ use miette::Diagnostic; use nonempty::NonEmpty; use thiserror::Error; -mod reconcile; +mod frozen; use crate::{ build::BuildBehaviour, @@ -100,7 +100,7 @@ pub enum SyncError { GenLuaRc(#[from] GenLuaRcError), #[error(transparent)] #[diagnostic(transparent)] - Frozen(#[from] reconcile::SyncError), + Frozen(#[from] frozen::SyncError), } impl From for SyncError { @@ -335,6 +335,15 @@ fn gather_dependencies( Ok(packages) } +async fn sync_frozen(args: &Sync<'_>) -> Result { + let report = frozen::Sync::new(args.workspace, args.config) + .validate_integrity(args.validate_integrity.unwrap_or(true)) + .test(args.targets.test) + .sync() + .await?; + Ok(report) +} + #[cfg(all(test, feature = "impure_tests"))] mod tests { use std::path::PathBuf; @@ -593,12 +602,3 @@ mod tests { ); } } - -async fn sync_frozen(args: &Sync<'_>) -> Result { - let report = reconcile::Sync::new(args.workspace, args.config) - .validate_integrity(args.validate_integrity.unwrap_or(true)) - .test(args.targets.test) - .sync() - .await?; - Ok(report) -} From 4504d0cd4e3af389191037aef5b0c662a8562c7c Mon Sep 17 00:00:00 2001 From: vhyrro Date: Thu, 8 Oct 2026 19:37:49 +0200 Subject: [PATCH 33/39] refactor!: `RemotePackageDB` -> `PackageDB` --- lux-cli/src/add.rs | 4 +- lux-cli/src/outdated.rs | 4 +- lux-cli/src/search.rs | 4 +- lux-cli/src/update.rs | 4 +- lux-cli/src/upload.rs | 8 +- lux-lib/src/drivers/install_packages.rs | 8 +- lux-lib/src/drivers/sync/mod.rs | 15 +- lux-lib/src/lib.rs | 2 +- lux-lib/src/operations/download.rs | 26 +- lux-lib/src/operations/exec.rs | 4 +- lux-lib/src/operations/fetch_vendored.rs | 4 +- lux-lib/src/operations/install_project.rs | 17 +- lux-lib/src/operations/install_rockspec.rs | 8 +- lux-lib/src/operations/update.rs | 16 +- lux-lib/src/operations/vendor.rs | 12 +- lux-lib/src/package/outdated.rs | 6 +- lux-lib/src/package_db/mod.rs | 355 +++++++++++++++++++++ lux-lib/src/pipeline/discover.rs | 2 +- lux-lib/src/pipeline/resolve.rs | 17 +- lux-lib/src/project/mod.rs | 8 +- lux-lib/src/remote_package_db/mod.rs | 332 ------------------- lux-lib/src/upload/mod.rs | 6 +- lux-lua/src/lua_impls.rs | 26 +- lux-lua/src/operations.rs | 10 +- 24 files changed, 453 insertions(+), 445 deletions(-) create mode 100644 lux-lib/src/package_db/mod.rs delete mode 100644 lux-lib/src/remote_package_db/mod.rs diff --git a/lux-cli/src/add.rs b/lux-cli/src/add.rs index d07e8525e..b33534a28 100644 --- a/lux-cli/src/add.rs +++ b/lux-cli/src/add.rs @@ -2,7 +2,7 @@ use itertools::{Either, Itertools}; use lux_lib::{ config::Config, package::PackageName, - remote_package_db::RemotePackageDB, + package_db::PackageDB, rockspec::lua_dependency::{self}, workspace::Workspace, }; @@ -45,7 +45,7 @@ pub struct Add { pub async fn add(data: Add, config: Config) -> Result<()> { let mut workspace = Workspace::current_or_err()?; let project = workspace.single_member_or_select_mut(&data.package)?; - let db = RemotePackageDB::from_config(&config).await?; + let db = PackageDB::from_config(&config).await?; let (dependencies, git_dependencies): (Vec<_>, Vec<_>) = data.package_req.iter().partition_map(|req| match req { diff --git a/lux-cli/src/outdated.rs b/lux-cli/src/outdated.rs index bf0670c3e..dd4f9e9fe 100644 --- a/lux-cli/src/outdated.rs +++ b/lux-cli/src/outdated.rs @@ -4,7 +4,7 @@ use clap::Args; use itertools::Itertools; use lux_lib::{ config::Config, lockfile::LockedPackage, lua_version::LuaVersion, package::PackageVersion, - remote_package_db::RemotePackageDB, workspace::Workspace, + package_db::PackageDB, workspace::Workspace, }; use miette::{IntoDiagnostic, Result}; @@ -30,7 +30,7 @@ pub async fn outdated(outdated_data: Outdated, config: Config) -> Result<()> { } }; - let package_db = RemotePackageDB::from_config(&config).await?; + let package_db = PackageDB::from_config(&config).await?; // NOTE: This will display all installed versions and each possible upgrade. // However, this should also take into account dependency constraints made by other rocks. diff --git a/lux-cli/src/search.rs b/lux-cli/src/search.rs index 15f82a44b..3664d77ff 100644 --- a/lux-cli/src/search.rs +++ b/lux-cli/src/search.rs @@ -8,7 +8,7 @@ use text_trees::{FormatCharacters, StringTreeNode, TreeFormatting}; use lux_lib::{ config::Config, package::{PackageName, PackageReq, PackageVersion}, - remote_package_db::RemotePackageDB, + package_db::PackageDB, }; use crate::args::OutputFormat; @@ -24,7 +24,7 @@ pub struct Search { pub async fn search(data: Search, config: Config) -> Result<()> { let formatting = TreeFormatting::dir_tree(FormatCharacters::box_chars()); - let package_db = RemotePackageDB::from_config(&config).await?; + let package_db = PackageDB::from_config(&config).await?; let lua_package_req = data.lua_package_req; diff --git a/lux-cli/src/update.rs b/lux-cli/src/update.rs index 58d835e1b..114bae969 100644 --- a/lux-cli/src/update.rs +++ b/lux-cli/src/update.rs @@ -2,7 +2,7 @@ use clap::Args; use itertools::Itertools; use lux_lib::package::{PackageName, PackageReq}; -use lux_lib::remote_package_db::RemotePackageDB; +use lux_lib::package_db::PackageDB; use lux_lib::rockspec::lua_dependency::LuaDependencyType; use lux_lib::workspace::Workspace; use lux_lib::{config::Config, operations}; @@ -42,7 +42,7 @@ pub async fn update(args: Update, config: Config) -> Result<()> { if args.toml { let mut workspace = Workspace::current_or_err()?; - let db = RemotePackageDB::from_config(&config).await?; + let db = PackageDB::from_config(&config).await?; let package_names = to_package_names(args.packages.as_ref())?; let mut upgrade_all = true; if let Some(packages) = package_names { diff --git a/lux-cli/src/upload.rs b/lux-cli/src/upload.rs index fc6304d77..5841519a4 100644 --- a/lux-cli/src/upload.rs +++ b/lux-cli/src/upload.rs @@ -1,7 +1,7 @@ use clap::Args; use lux_lib::{ - config::Config, package::PackageName, remote_package_db::RemotePackageDB, - upload::ProjectUpload, workspace::Workspace, + config::Config, package::PackageName, package_db::PackageDB, upload::ProjectUpload, + workspace::Workspace, }; use miette::{IntoDiagnostic, Result}; @@ -31,7 +31,7 @@ pub struct Upload { pub async fn upload(data: Upload, config: Config) -> Result<()> { let workspace = Workspace::current_or_err()?; - let package_db = RemotePackageDB::from_config(&config).await?; + let package_db = PackageDB::from_config(&config).await?; let tfa_code = tfa_code_from_args_or_secret(&data, &config)?; if let Some(package) = data.package { let project = workspace.select_member(&package)?; @@ -62,7 +62,7 @@ pub async fn upload(data: Upload, config: Config) -> Result<()> { #[cfg(not(feature = "gpgme"))] pub async fn upload(data: Upload, config: Config) -> Result<()> { let workspace = Workspace::current_or_err()?; - let package_db = RemotePackageDB::from_config(&config).await?; + let package_db = PackageDB::from_config(&config).await?; let tfa_code = tfa_code_from_args_or_secret(&data, &config)?; if let Some(package) = data.package { let project = workspace.select_member(&package)?; diff --git a/lux-lib/src/drivers/install_packages.rs b/lux-lib/src/drivers/install_packages.rs index e20fda2db..8300ea951 100644 --- a/lux-lib/src/drivers/install_packages.rs +++ b/lux-lib/src/drivers/install_packages.rs @@ -16,7 +16,7 @@ use crate::{ lua_installation::LuaInstallationError, luarocks::luarocks_installation::{LuaRocksError, LuaRocksInstallError}, package::PackageNameList, - remote_package_db::{RemotePackageDB, RemotePackageDBError}, + package_db::{PackageDB, PackageDBError}, tree::{self, InstallTree, TreeError}, workspace::WorkspaceTreeError, }; @@ -50,7 +50,7 @@ where test_packages: Option>, #[builder(field)] packages: Vec, - package_db: Option, + package_db: Option, } impl<'a, T, State> InstallPackagesBuilder<'a, T, State> @@ -112,7 +112,7 @@ where { let package_db = match install.package_db { Some(db) => db, - None => RemotePackageDB::from_config(install.config).await?, + None => PackageDB::from_config(install.config).await?, }; let duplicate_entrypoints = install @@ -301,7 +301,7 @@ pub enum InstallPackagesError { LuaRocksInstall(#[from] Box), #[error("error initialising remote package DB")] #[diagnostic(forward(0))] - RemotePackageDB(#[from] RemotePackageDBError), + PackageDB(#[from] PackageDBError), #[error("cannot install duplicate entrypoints:\n{0}")] DuplicateEntrypoints(PackageNameList), #[error("cannot install conflicting entrypoints:\n{0}")] diff --git a/lux-lib/src/drivers/sync/mod.rs b/lux-lib/src/drivers/sync/mod.rs index 249fbf1ae..71aae41b1 100644 --- a/lux-lib/src/drivers/sync/mod.rs +++ b/lux-lib/src/drivers/sync/mod.rs @@ -14,8 +14,8 @@ use crate::{ GenLuaRc, GenLuaRcError, InstallProject, InstallProjectError, PackageInstallSpec, }, package::PackageName, + package_db::{PackageDB, PackageDBError}, project::project_toml::LocalProjectTomlValidationError, - remote_package_db::{RemotePackageDB, RemotePackageDBError}, rockspec::Rockspec, tree::{EntryType, InstallTree, TreeError}, workspace::{Workspace, WorkspaceError, WorkspaceTreeError}, @@ -94,7 +94,7 @@ pub enum SyncError { Project(#[from] LocalProjectTomlValidationError), #[error(transparent)] #[diagnostic(transparent)] - RemotePackageDB(#[from] RemotePackageDBError), + PackageDB(#[from] PackageDBError), #[error(transparent)] #[diagnostic(transparent)] GenLuaRc(#[from] GenLuaRcError), @@ -202,11 +202,7 @@ async fn sync_open(args: &Sync<'_>) -> Result { }; let package_db = if !test.is_empty() || !regular.is_empty() || !build.is_empty() { - Some( - RemotePackageDB::from_config(config) - .await? - .with_locks(workspace.lockfile()?.local_pkg_locks()), - ) + Some(PackageDB::for_open(config, workspace).await?) } else { None }; @@ -352,8 +348,7 @@ mod tests { use crate::{ config::ConfigBuilder, lockfile::LockedPackageLockType, lua_version::LuaVersion, - package::PackageName, remote_package_db::RemotePackageDB, tree::InstallTree, - workspace::Workspace, + package::PackageName, package_db::PackageDB, tree::InstallTree, workspace::Workspace, }; use super::{Sync, SyncMode, TargetSet}; @@ -513,7 +508,7 @@ mod tests { .unwrap() .version() .clone(); - let latest = RemotePackageDB::from_config(&config) + let latest = PackageDB::from_config(&config) .await .unwrap() .latest_version(&name) diff --git a/lux-lib/src/lib.rs b/lux-lib/src/lib.rs index 3e05394aa..2365c5376 100644 --- a/lux-lib/src/lib.rs +++ b/lux-lib/src/lib.rs @@ -13,11 +13,11 @@ pub mod luarocks; pub mod manifest; pub mod operations; pub mod package; +pub mod package_db; pub mod path; pub mod pipeline; pub mod progress; pub mod project; -pub mod remote_package_db; pub mod rockspec; pub mod toolchains; pub mod tree; diff --git a/lux-lib/src/operations/download.rs b/lux-lib/src/operations/download.rs index 740eb3d54..7df176aa5 100644 --- a/lux-lib/src/operations/download.rs +++ b/lux-lib/src/operations/download.rs @@ -21,7 +21,7 @@ use crate::{ PackageName, PackageReq, PackageSpec, PackageSpecFromPackageReqError, PackageVersion, RemotePackageTypeFilterSpec, }, - remote_package_db::{RemotePackageDB, RemotePackageDBError, SearchError}, + package_db::{PackageDB, PackageDBError, SearchError}, remote_package_source::RemotePackageSource, reqwest::{RequestBuilderExt, RequestError}, rockspec::Rockspec, @@ -30,7 +30,7 @@ use crate::{ /// Builder for a rock downloader. pub struct Download<'a> { package_req: &'a PackageReq, - package_db: Option<&'a RemotePackageDB>, + package_db: Option<&'a PackageDB>, config: &'a Config, } @@ -46,7 +46,7 @@ impl<'a> Download<'a> { /// Sets the package database to use for searching for packages. /// Instantiated from the config if not set. - pub fn package_db(self, package_db: &'a RemotePackageDB) -> Self { + pub fn package_db(self, package_db: &'a PackageDB) -> Self { Self { package_db: Some(package_db), ..self @@ -58,7 +58,7 @@ impl<'a> Download<'a> { match self.package_db { Some(db) => download_rockspec(self.package_req, db, self.config).await, None => { - let db = RemotePackageDB::from_config(self.config).await?; + let db = PackageDB::from_config(self.config).await?; download_rockspec(self.package_req, &db, self.config).await } } @@ -75,7 +75,7 @@ impl<'a> Download<'a> { download_src_rock_to_file(self.package_req, destination_dir, db, self.config).await } None => { - let db = RemotePackageDB::from_config(self.config).await?; + let db = PackageDB::from_config(self.config).await?; download_src_rock_to_file(self.package_req, destination_dir, &db, self.config).await } } @@ -88,7 +88,7 @@ impl<'a> Download<'a> { match self.package_db { Some(db) => search_and_download_src_rock(self.package_req, db, self.config).await, None => { - let db = RemotePackageDB::from_config(self.config).await?; + let db = PackageDB::from_config(self.config).await?; search_and_download_src_rock(self.package_req, &db, self.config).await } } @@ -100,7 +100,7 @@ impl<'a> Download<'a> { match self.package_db { Some(db) => download_remote_rock(self.package_req, db, self.config).await, None => { - let db = RemotePackageDB::from_config(self.config).await?; + let db = PackageDB::from_config(self.config).await?; download_remote_rock(self.package_req, &db, self.config).await } } @@ -211,7 +211,7 @@ if the issue persists, the server may be temporarily unavailable."# ResponseConversion(#[from] FromUtf8Error), #[error("error initialising remote package DB")] #[diagnostic(forward(0))] - RemotePackageDB(#[from] RemotePackageDBError), + PackageDB(#[from] PackageDBError), #[error(transparent)] #[diagnostic(transparent)] DownloadSrcRock(#[from] DownloadSrcRockError), @@ -226,7 +226,7 @@ impl From for DownloadRockspecError { /// Find and download a rockspec for a given package requirement async fn download_rockspec( package_req: &PackageReq, - package_db: &RemotePackageDB, + package_db: &PackageDB, config: &Config, ) -> Result { let rockspec = match download_remote_rock(package_req, package_db, config).await? { @@ -253,7 +253,7 @@ async fn download_rockspec( )] async fn download_remote_rock( package_req: &PackageReq, - package_db: &RemotePackageDB, + package_db: &PackageDB, config: &Config, ) -> Result { let remote_package = package_db.find(package_req, None)?; @@ -373,7 +373,7 @@ if the issue persists, the server may be temporarily unavailable."# Rockspec(Box), #[error("error initialising remote package DB")] #[diagnostic(forward(0))] - RemotePackageDB(#[from] RemotePackageDBError), + PackageDB(#[from] PackageDBError), #[error("failed to read packed rock {0}:\n{1}")] #[diagnostic(help( r#"the downloaded rock may be corrupted. @@ -433,7 +433,7 @@ impl From for SearchAndDownloadError { async fn search_and_download_src_rock( package_req: &PackageReq, - package_db: &RemotePackageDB, + package_db: &PackageDB, config: &Config, ) -> Result { let filter = Some(RemotePackageTypeFilterSpec { @@ -500,7 +500,7 @@ pub(crate) async fn download_binary_rock( async fn download_src_rock_to_file( package_req: &PackageReq, destination_dir: Option, - package_db: &RemotePackageDB, + package_db: &PackageDB, config: &Config, ) -> Result { let rock = search_and_download_src_rock(package_req, package_db, config).await?; diff --git a/lux-lib/src/operations/exec.rs b/lux-lib/src/operations/exec.rs index cc1fd4c38..d5d06322e 100644 --- a/lux-lib/src/operations/exec.rs +++ b/lux-lib/src/operations/exec.rs @@ -9,8 +9,8 @@ use crate::{ lua_version::{LuaVersion, LuaVersionUnset}, operations::{BuildWorkspace, BuildWorkspaceError}, package::{PackageReq, PackageVersionReqError}, + package_db::PackageDBError, path::{Paths, PathsError}, - remote_package_db::RemotePackageDBError, tree::{self, InstallTree, TreeError}, workspace::{Workspace, WorkspaceTreeError}, }; @@ -121,7 +121,7 @@ impl From for ExecError { pub enum InstallCommandError { InstallError(#[from] Box), PackageVersionReqError(#[from] PackageVersionReqError), - RemotePackageDBError(#[from] RemotePackageDBError), + PackageDBError(#[from] PackageDBError), Tree(#[from] TreeError), LuaVersionUnset(#[from] LuaVersionUnset), } diff --git a/lux-lib/src/operations/fetch_vendored.rs b/lux-lib/src/operations/fetch_vendored.rs index dde0bc716..1b807d71b 100644 --- a/lux-lib/src/operations/fetch_vendored.rs +++ b/lux-lib/src/operations/fetch_vendored.rs @@ -6,7 +6,7 @@ use crate::{ lua_rockspec::{LuaRockspecError, RemoteLuaRockspec}, operations::{DownloadedRockspec, RemoteRockDownload}, package::{PackageReq, PackageSpec}, - remote_package_db::{RemotePackageDB, SearchError}, + package_db::{PackageDB, SearchError}, remote_package_source::RemotePackageSource, }; use bon::Builder; @@ -19,7 +19,7 @@ use thiserror::Error; pub(crate) struct FetchVendored<'a> { vendor_dir: &'a Path, package: &'a PackageReq, - package_db: &'a RemotePackageDB, + package_db: &'a PackageDB, } #[derive(Error, Debug, Diagnostic)] diff --git a/lux-lib/src/operations/install_project.rs b/lux-lib/src/operations/install_project.rs index 52ebc7173..49fced9ff 100644 --- a/lux-lib/src/operations/install_project.rs +++ b/lux-lib/src/operations/install_project.rs @@ -10,9 +10,9 @@ use crate::{ lockfile::LockedPackage, operations::PackageInstallSpec, package::{PackageName, PackageReq}, + package_db::{PackageDB, PackageDBError}, pipeline::discover::FoundPackage, project::{IntoLocalRockspecError, Project, ProjectError}, - remote_package_db::{RemotePackageDB, RemotePackageDBError}, rockspec::Rockspec, tree::{self, InstallTree, TreeError}, workspace::{Workspace, WorkspaceError}, @@ -28,7 +28,7 @@ pub enum InstallProjectError { LocalRockspec(#[from] IntoLocalRockspecError), #[error(transparent)] #[diagnostic(transparent)] - RemotePackageDB(#[from] RemotePackageDBError), + PackageDB(#[from] PackageDBError), #[error(transparent)] #[diagnostic(transparent)] Tree(#[from] TreeError), @@ -119,14 +119,11 @@ impl< }; let root = FoundPackage::from_project_root(rockspec, project.root().to_path_buf()); - let locks = match args.workspace { - Some(workspace) => workspace.lockfile()?.local_pkg_locks(), - None => Vec::new(), - }; - let package_db = RemotePackageDB::from_config(config) - .await? - .with_locks(locks) - .with_local(vec![root]); + let package_db = match args.workspace { + Some(workspace) => PackageDB::for_open(config, workspace).await?, + None => PackageDB::from_config(config).await?, + } + .with_local(vec![root]); let install_spec = PackageInstallSpec::new(PackageReq::from(name.clone()), tree::EntryType::Entrypoint) diff --git a/lux-lib/src/operations/install_rockspec.rs b/lux-lib/src/operations/install_rockspec.rs index 06c1b3aa6..113092295 100644 --- a/lux-lib/src/operations/install_rockspec.rs +++ b/lux-lib/src/operations/install_rockspec.rs @@ -12,8 +12,8 @@ use crate::{ luarocks::luarocks_installation::{LuaRocksError, LuaRocksInstallError}, operations::PackageInstallSpec, package::{PackageName, PackageReq}, + package_db::{PackageDB, PackageDBError}, pipeline::discover::FoundPackage, - remote_package_db::{RemotePackageDB, RemotePackageDBError}, rockspec::{LuaVersionCompatibility, Rockspec}, tree::{self, InstallTree, TreeError}, }; @@ -27,7 +27,7 @@ pub enum InstallRockspecError { #[diagnostic(transparent)] LuaVersion(#[from] LuaVersionError), #[diagnostic(transparent)] - RemotePackageDB(#[from] RemotePackageDBError), + PackageDB(#[from] PackageDBError), #[diagnostic(transparent)] Tree(#[from] TreeError), #[diagnostic(transparent)] @@ -92,9 +92,7 @@ impl< let name = rockspec.package().clone(); let root = FoundPackage::from_rockspec(rockspec); - let package_db = RemotePackageDB::from_config(config) - .await? - .with_local(vec![root]); + let package_db = PackageDB::from_config(config).await?.with_local(vec![root]); let install_spec = PackageInstallSpec::new(PackageReq::from(name.clone()), tree::EntryType::Entrypoint) diff --git a/lux-lib/src/operations/update.rs b/lux-lib/src/operations/update.rs index e519fcc40..2db731e02 100644 --- a/lux-lib/src/operations/update.rs +++ b/lux-lib/src/operations/update.rs @@ -9,7 +9,7 @@ use crate::{ }, lua_version::{LuaVersion, LuaVersionUnset}, package::{PackageReq, RockConstraintUnsatisfied}, - remote_package_db::{RemotePackageDB, RemotePackageDBError}, + package_db::{PackageDB, PackageDBError}, remote_package_source::RemotePackageSource, tree::{self, InstallTree, Tree, TreeError}, workspace::{Workspace, WorkspaceError, WorkspaceTreeError}, @@ -37,7 +37,7 @@ pub enum UpdateError { Remove(#[from] RemoveError), #[error("error initialising remote package DB")] #[diagnostic(forward(0))] - RemotePackageDB(#[from] RemotePackageDBError), + PackageDB(#[from] PackageDBError), #[error("error loading the workspace")] #[diagnostic(forward(0))] Workspace(#[from] WorkspaceError), @@ -97,7 +97,7 @@ pub struct Update<'a> { /// Whether to validate the integrity when syncing the project lockfile. validate_integrity: Option, - package_db: Option, + package_db: Option, } impl UpdateBuilder<'_, State> { @@ -134,7 +134,7 @@ impl UpdateBuilder<'_, State> { let package_db = match &args.package_db { Some(db) => db.clone(), None => { - let db = RemotePackageDB::from_config(args.config).await?; + let db = PackageDB::from_config(args.config).await?; db } }; @@ -154,7 +154,7 @@ impl UpdateBuilder<'_, State> { async fn update_workspace( workspace: Workspace, args: Update<'_>, - package_db: RemotePackageDB, + package_db: PackageDB, ) -> Result, UpdateError> { let mut project_lockfile = workspace.lockfile()?.write_guard(); let tree = workspace.tree(args.config)?; @@ -213,7 +213,7 @@ async fn update_dependency_tree( tree: Tree, project_lockfile: &mut WorkspaceLockfile, lock_type: LockedPackageLockType, - package_db: RemotePackageDB, + package_db: PackageDB, config: &Config, packages: &Option>, ) -> Result, UpdateError> { @@ -244,7 +244,7 @@ fn is_included( async fn update_install_tree( args: Update<'_>, - package_db: RemotePackageDB, + package_db: PackageDB, ) -> Result, UpdateError> { let tree = args .config @@ -259,7 +259,7 @@ async fn update_install_tree( async fn update( packages: Vec<(LockedPackage, PackageReq)>, - package_db: RemotePackageDB, + package_db: PackageDB, tree: Tree, lockfile: &Lockfile, config: &Config, diff --git a/lux-lib/src/operations/vendor.rs b/lux-lib/src/operations/vendor.rs index 670ceb878..c3ea60e3e 100644 --- a/lux-lib/src/operations/vendor.rs +++ b/lux-lib/src/operations/vendor.rs @@ -24,6 +24,7 @@ use crate::{ lua_rockspec::{BuildBackendSpec, RemoteLuaRockspec}, operations::{self, PackageInstallSpec, UnpackError}, package::{PackageReq, PackageSpec}, + package_db::{PackageDB, PackageDBError}, pipeline::{ download_sources_and_hash::{ DownloadSourcesAndHash, DownloadSourcesAndHashError, DownloadedPackage, PackageSource, @@ -32,7 +33,6 @@ use crate::{ Artifacts, }, project::project_toml::LocalProjectTomlValidationError, - remote_package_db::{RemotePackageDB, RemotePackageDBError}, rockspec::Rockspec, tree::EntryType, workspace::{Workspace, WorkspaceError}, @@ -78,7 +78,7 @@ pub enum VendorError { LocalProjectTomlValidation(#[from] LocalProjectTomlValidationError), #[error("error initialising remote package DB")] #[diagnostic(forward(0))] - RemotePackageDB(#[from] RemotePackageDBError), + PackageDB(#[from] PackageDBError), #[error(transparent)] #[diagnostic(transparent)] Resolve(#[from] ResolveError), @@ -122,7 +122,7 @@ const CARGO_VENDOR_SUBDIR: &str = "cargo"; /// Resolves the requested packages and downloads (and hashes) their sources. async fn resolve_and_download( config: &Config, - package_db: RemotePackageDB, + package_db: PackageDB, install_specs: Artifacts>, ) -> Result, VendorError> { let Artifacts { @@ -225,15 +225,15 @@ async fn gather_install_specs( no_lock: bool, target: &VendorTarget, config: &Config, -) -> Result<(RemotePackageDB, Artifacts>), VendorError> { +) -> Result<(PackageDB, Artifacts>), VendorError> { // Resolve against the project's lockfile if present, otherwise fall back to // the remote package DB (e.g. for a project that has not yet generated a lockfile). let package_db = match target { VendorTarget::Workspace(workspace) => match workspace.try_lockfile()? { Some(lockfile) if !no_lock => lockfile.local_pkg_locks().into(), - _ => RemotePackageDB::from_config(config).await?, + _ => PackageDB::from_config(config).await?, }, - VendorTarget::Rockspec(_) => RemotePackageDB::from_config(config).await?, + VendorTarget::Rockspec(_) => PackageDB::from_config(config).await?, }; let mut install_specs: Artifacts> = Artifacts::default(); diff --git a/lux-lib/src/package/outdated.rs b/lux-lib/src/package/outdated.rs index 226ffe821..a6391ca38 100644 --- a/lux-lib/src/package/outdated.rs +++ b/lux-lib/src/package/outdated.rs @@ -1,6 +1,6 @@ use std::fmt::Display; -use crate::remote_package_db::RemotePackageDB; +use crate::package_db::PackageDB; use miette::Diagnostic; use thiserror::Error; @@ -22,7 +22,7 @@ impl PackageSpec { /// Returns the latest version if found. pub fn has_update( &self, - package_db: &RemotePackageDB, + package_db: &PackageDB, ) -> Result, RockNotFound> { let latest_version = package_db .latest_version(&self.name) @@ -40,7 +40,7 @@ impl PackageSpec { pub fn has_update_with( &self, constraint: &PackageReq, - package_db: &RemotePackageDB, + package_db: &PackageDB, ) -> Result, RockConstraintUnsatisfied> { let latest_version = package_db diff --git a/lux-lib/src/package_db/mod.rs b/lux-lib/src/package_db/mod.rs new file mode 100644 index 000000000..4577b7788 --- /dev/null +++ b/lux-lib/src/package_db/mod.rs @@ -0,0 +1,355 @@ +use crate::{ + config::{Config, ConfigError}, + lockfile::{LockfileIntegrityError, PackageLock}, + manifest::{Manifest, ManifestError}, + operations::{Download, FetchVendored, PackageInstallSpec, RemoteRockDownload}, + package::{ + PackageName, PackageReq, PackageSpec, PackageVersion, RemotePackage, + RemotePackageTypeFilterSpec, + }, + pipeline::{ + discover::{DiscoverError, FoundPackage}, + download_sources_and_hash::PackageSource, + }, + workspace::{Workspace, WorkspaceError}, +}; +use itertools::Itertools; + +use miette::Diagnostic; +use thiserror::Error; + +/// Package database, used to look up packages. +/// Lookup order: local packages, locked packages, manifests. +#[derive(Clone, Debug)] +pub struct PackageDB { + local: Vec, + locks: Vec, + remote_manifests: Vec, +} + +#[derive(Error, Debug, Diagnostic)] +pub enum PackageDBError { + #[error(transparent)] + #[diagnostic(transparent)] + ManifestError(#[from] ManifestError), + #[error(transparent)] + #[diagnostic(transparent)] + ConfigError(#[from] ConfigError), + #[error(transparent)] + #[diagnostic(transparent)] + Workspace(#[from] WorkspaceError), +} + +#[derive(Error, Debug, Diagnostic)] +#[non_exhaustive] +pub enum SearchError { + #[error("no rock that matches '{0}' found")] + RockNotFound(PackageReq), + #[error("no rock that matches '{0}' found in the lockfile.")] + RockNotFoundInLockfile(PackageReq), + #[error("error when pulling manifest")] + #[diagnostic(forward(0))] + Manifest(#[from] ManifestError), +} + +#[derive(Error, Debug, Diagnostic)] +#[non_exhaustive] +pub enum RemotePackageDbIntegrityError { + #[error(transparent)] + #[diagnostic(transparent)] + Lockfile(#[from] LockfileIntegrityError), +} + +impl PackageDB { + #[tracing::instrument(level = "trace", skip_all)] + pub async fn from_config(config: &Config) -> Result { + let mut manifests = Vec::new(); + for server in config.enabled_dev_servers()? { + let manifest = Manifest::from_config(server, config).await?; + manifests.push(manifest); + } + for server in config.extra_servers() { + let manifest = Manifest::from_config(server.clone(), config).await?; + manifests.push(manifest); + } + manifests.push(Manifest::from_config(config.server().clone(), config).await?); + Ok(Self { + local: Vec::new(), + locks: Vec::new(), + remote_manifests: manifests, + }) + } + + /// Builds a package database that prefers the workspace lockfile, falling + /// back to the manifests for requirements the lockfile does not satisfy. + pub(crate) async fn for_open( + config: &Config, + workspace: &Workspace, + ) -> Result { + Ok(Self::from_config(config) + .await? + .with_locks(workspace.lockfile()?.local_pkg_locks())) + } + + #[cfg(test)] + pub(crate) fn from_local(packages: Vec) -> Self { + Self { + local: packages, + locks: Vec::new(), + remote_manifests: Vec::new(), + } + } + + /// Sets the local packages to look up first. + pub(crate) fn with_local(mut self, packages: Vec) -> Self { + self.local = packages; + self + } + + /// Sets the locked packages to look up after local packages, ahead of manifests. + pub(crate) fn with_locks(mut self, locks: Vec) -> Self { + self.locks = locks; + self + } + + fn find_in_local(&self, package_req: &PackageReq) -> Option { + self.local + .iter() + .find(|package| package.package.package.name() == package_req.name()) + .map(|package| package.package.clone()) + } + + fn find_in_locks( + &self, + package_req: &PackageReq, + filter: Option, + ) -> Option { + self.locks + .iter() + .filter_map(|lock| lock.has_rock(package_req, filter.clone())) + .map(|local_package| { + RemotePackage::new( + local_package.to_package(), + local_package.source().clone(), + local_package.source_url().cloned(), + ) + }) + .next() + } + + fn find_in_manifests( + &self, + package_req: &PackageReq, + filter: Option, + ) -> Option { + self.remote_manifests + .iter() + .find_map(|manifest| manifest.find(package_req, filter.clone())) + } + + /// Find a remote package that matches the requirement, returning the latest match. + #[tracing::instrument( + name = "Searching tree-sitter parser", + level = "info", + skip_all, + fields(package = package_req.to_string()), + )] + pub(crate) fn find( + &self, + package_req: &PackageReq, + filter: Option, + ) -> Result { + if let Some(package) = self.find_in_local(package_req) { + return Ok(package); + } + if let Some(package) = self.find_in_locks(package_req, filter.clone()) { + return Ok(package); + } + if let Some(package) = self.find_in_manifests(package_req, filter) { + return Ok(package); + } + if !self.remote_manifests.is_empty() { + Err(SearchError::RockNotFound(package_req.clone())) + } else if !self.locks.is_empty() { + Err(SearchError::RockNotFoundInLockfile(package_req.clone())) + } else { + Err(SearchError::RockNotFound(package_req.clone())) + } + } + + /// Resolves a package install spec into a fully discovered package. + pub(crate) async fn resolve( + &self, + spec: &PackageInstallSpec, + config: &Config, + ) -> Result { + if let Some(source) = &spec.source { + let download = RemoteRockDownload::from_package_req_and_source_spec( + spec.package.clone(), + source.clone(), + ) + .map_err(|err| DiscoverError::Download(spec.package.clone(), Box::new(err)))?; + let rockspec = download.rockspec().clone(); + let package = RemotePackage::new( + spec.package.clone().try_into().map_err(|err| { + DiscoverError::Download( + spec.package.clone(), + Box::new(crate::operations::SearchAndDownloadError::from(err)), + ) + })?, + download.rockspec_download().source.clone(), + download.rockspec_download().source_url.clone(), + ); + return Ok(FoundPackage { + package, + rockspec, + artifact: None, + }); + } + + if let Some(package) = self + .local + .iter() + .find(|package| package.package.package.name() == spec.package.name()) + { + return Ok(package.clone()); + } + + if !self.locks.is_empty() || !self.remote_manifests.is_empty() { + let mut package = self.find(&spec.package, None)?; + let download = if let Some(vendor_dir) = config.vendor_dir() { + FetchVendored::new() + .vendor_dir(vendor_dir) + .package(&spec.package) + .package_db(self) + .fetch_vendored_rock() + .await + .map_err(|err| { + DiscoverError::FetchVendored(spec.package.clone(), Box::new(err)) + })? + } else { + Download::new(&spec.package, config) + .package_db(self) + .download_remote_rock() + .await + .map_err(|err| DiscoverError::Download(spec.package.clone(), Box::new(err)))? + }; + let rockspec = download.rockspec().clone(); + let artifact = match &download { + RemoteRockDownload::SrcRock { + src_rock, + source_url, + .. + } => { + package.source_url = Some(source_url.clone()); + Some(PackageSource::PackedRock(src_rock.clone())) + } + RemoteRockDownload::BinaryRock { packed_rock, .. } => { + Some(PackageSource::PackedRock(packed_rock.clone())) + } + RemoteRockDownload::RockspecOnly { .. } => None, + }; + return Ok(FoundPackage { + package, + rockspec, + artifact, + }); + } + + Err(DiscoverError::Search(SearchError::RockNotFound( + spec.package.clone(), + ))) + } + + /// Search for all packages that match the requirement. + pub fn search(&self, package_req: &PackageReq) -> Vec<(&PackageName, Vec<&PackageVersion>)> { + self.local + .iter() + .filter_map(|package| { + let name = package.package.package.name(); + if name.to_string().contains(&package_req.name().to_string()) + && package_req + .version_req() + .matches(package.package.package.version()) + { + Some((name, vec![package.package.package.version()])) + } else { + None + } + }) + .chain( + self.locks + .iter() + .flat_map(|lock| lock.rocks().values()) + .filter_map(|package| { + // NOTE: This doesn't group packages by name, but we don't care for now, + // as we shouldn't need to use this function with a lockfile. + let name = package.name(); + if name.to_string().contains(&package_req.name().to_string()) { + Some((name, vec![package.version()])) + } else { + None + } + }), + ) + .chain(self.remote_manifests.iter().flat_map(|manifest| { + manifest + .metadata() + .repository + .iter() + .filter_map(|(name, elements)| { + if name.to_string().contains(&package_req.name().to_string()) { + Some(( + name, + elements + .keys() + .filter(|version| package_req.version_req().matches(version)) + .sorted_by(|a, b| Ord::cmp(b, a)) + .collect_vec(), + )) + } else { + None + } + }) + })) + .collect() + } + + /// Find the latest version for a package by name. + pub(crate) fn latest_version(&self, rock_name: &PackageName) -> Option { + self.latest_match(&rock_name.clone().into(), None) + .map(|result| result.version().clone()) + } + + /// Find the latest package that matches the requirement. + pub fn latest_match( + &self, + package_req: &PackageReq, + filter: Option, + ) -> Option { + match self.find(package_req, filter) { + Ok(result) => Some(result.package), + Err(_) => None, + } + } +} + +impl From for PackageDB { + fn from(manifest: Manifest) -> Self { + Self { + local: Vec::new(), + locks: Vec::new(), + remote_manifests: vec![manifest], + } + } +} + +impl From> for PackageDB { + fn from(locks: Vec) -> Self { + Self { + local: Vec::new(), + locks, + remote_manifests: Vec::new(), + } + } +} diff --git a/lux-lib/src/pipeline/discover.rs b/lux-lib/src/pipeline/discover.rs index 38e94aa77..ccfbecedf 100644 --- a/lux-lib/src/pipeline/discover.rs +++ b/lux-lib/src/pipeline/discover.rs @@ -5,7 +5,7 @@ use crate::{ lua_rockspec::RemoteLuaRockspec, operations, package::{PackageReq, PackageSpec, RemotePackage}, - remote_package_db::SearchError, + package_db::SearchError, remote_package_source::RemotePackageSource, rockspec::Rockspec, }; diff --git a/lux-lib/src/pipeline/resolve.rs b/lux-lib/src/pipeline/resolve.rs index d12049ca7..71815dea7 100644 --- a/lux-lib/src/pipeline/resolve.rs +++ b/lux-lib/src/pipeline/resolve.rs @@ -17,7 +17,7 @@ use crate::{ lua_rockspec::{BuildBackendSpec, RemoteLuaRockspec}, operations::PackageInstallSpec, package::{PackageName, PackageReq}, - remote_package_db::RemotePackageDB, + package_db::PackageDB, remote_package_source::RemotePackageSource, rockspec::Rockspec, tree::EntryType, @@ -91,7 +91,7 @@ pub enum ResolveError { #[builder(start_fn = new, finish_fn(name = _build, vis = ""))] pub(crate) struct ResolvePackageDependencies<'a> { #[builder(start_fn)] - pub(crate) package_db: RemotePackageDB, + pub(crate) package_db: PackageDB, #[builder(start_fn)] pub(crate) config: &'a Config, #[builder(field)] @@ -165,7 +165,7 @@ struct Node { } struct Resolver { - package_db: RemotePackageDB, + package_db: PackageDB, config: Arc, max_concurrent: usize, joinset: JoinSet<( @@ -183,7 +183,7 @@ struct Resolver { impl Resolver { fn new( - package_db: RemotePackageDB, + package_db: PackageDB, config: &Config, installed: HashMap>, ) -> Self { @@ -196,7 +196,7 @@ impl Resolver { } fn with_max_concurrent( - package_db: RemotePackageDB, + package_db: PackageDB, config: Arc, max_concurrent: usize, installed: HashMap>, @@ -549,7 +549,6 @@ mod tests { use crate::config::ConfigBuilder; use crate::lockfile::{LockConstraint, LockedPackageHashes}; use crate::package::{PackageSpec, RemotePackage}; - use crate::remote_package_db::RemoteSource; fn rockspec(package: &str, dependencies: &[&str]) -> String { let dependencies = if dependencies.is_empty() { @@ -567,8 +566,8 @@ mod tests { ) } - fn local_packages(rockspecs: &[String]) -> RemotePackageDB { - RemotePackageDB(vec![RemoteSource::Local( + fn local_packages(rockspecs: &[String]) -> PackageDB { + PackageDB::from_local( rockspecs .iter() .map(|content| { @@ -590,7 +589,7 @@ mod tests { } }) .collect(), - )]) + ) } fn install_spec(name: &str) -> PackageInstallSpec { diff --git a/lux-lib/src/project/mod.rs b/lux-lib/src/project/mod.rs index 4d12b41b7..0ffe322dd 100644 --- a/lux-lib/src/project/mod.rs +++ b/lux-lib/src/project/mod.rs @@ -29,7 +29,7 @@ use crate::{ }, lua_version::LuaVersion, package::SpecRev, - remote_package_db::RemotePackageDB, + package_db::PackageDB, rockspec::{ lua_dependency::{DependencyType, LuaDependencySpec, LuaDependencyType}, LuaVersionCompatibility, @@ -323,7 +323,7 @@ impl Project { pub async fn add( &mut self, dependencies: DependencyType<&PackageReq>, - package_db: &RemotePackageDB, + package_db: &PackageDB, ) -> Result<(), ProjectEditError> { let mut project_toml = toml_edit::DocumentMut::from_str(&fs::tokio::read_to_string(self.toml_path()).await?)?; @@ -477,7 +477,7 @@ impl Project { pub async fn upgrade( &mut self, dependencies: LuaDependencyType<&PackageName>, - package_db: &RemotePackageDB, + package_db: &PackageDB, config: &Config, ) -> Result<(), ProjectEditError> { let mut project_toml = @@ -568,7 +568,7 @@ impl Project { pub async fn upgrade_all( &mut self, - package_db: &RemotePackageDB, + package_db: &PackageDB, config: &Config, ) -> Result<(), ProjectEditError> { if let Some(dependencies) = &self.toml().dependencies { diff --git a/lux-lib/src/remote_package_db/mod.rs b/lux-lib/src/remote_package_db/mod.rs deleted file mode 100644 index 357533cba..000000000 --- a/lux-lib/src/remote_package_db/mod.rs +++ /dev/null @@ -1,332 +0,0 @@ -use crate::{ - config::{Config, ConfigError}, - lockfile::{LockfileIntegrityError, PackageLock}, - manifest::{Manifest, ManifestError}, - operations::{Download, FetchVendored, PackageInstallSpec, RemoteRockDownload}, - package::{ - PackageName, PackageReq, PackageSpec, PackageVersion, RemotePackage, - RemotePackageTypeFilterSpec, - }, - pipeline::{ - discover::{DiscoverError, FoundPackage}, - download_sources_and_hash::PackageSource, - }, -}; -use itertools::Itertools; - -use miette::Diagnostic; -use thiserror::Error; - -/// Package database, used to look up remote rocks -#[derive(Clone, Debug)] -pub struct RemotePackageDB(pub(crate) Vec); - -#[derive(Clone, Debug)] -pub(crate) enum RemoteSource { - LuarocksManifests(Vec), - LockedPackageLocks(Vec), - Local(Vec), -} - -#[derive(Error, Debug, Diagnostic)] -pub enum RemotePackageDBError { - #[error(transparent)] - #[diagnostic(transparent)] - ManifestError(#[from] ManifestError), - #[error(transparent)] - #[diagnostic(transparent)] - ConfigError(#[from] ConfigError), -} - -#[derive(Error, Debug, Diagnostic)] -#[non_exhaustive] -pub enum SearchError { - #[error("no rock that matches '{0}' found")] - RockNotFound(PackageReq), - #[error("no rock that matches '{0}' found in the lockfile.")] - RockNotFoundInLockfile(PackageReq), - #[error("error when pulling manifest")] - #[diagnostic(forward(0))] - Manifest(#[from] ManifestError), -} - -#[derive(Error, Debug, Diagnostic)] -#[non_exhaustive] -pub enum RemotePackageDbIntegrityError { - #[error(transparent)] - #[diagnostic(transparent)] - Lockfile(#[from] LockfileIntegrityError), -} - -impl RemotePackageDB { - #[tracing::instrument(level = "trace", skip_all)] - pub async fn from_config(config: &Config) -> Result { - let mut manifests = Vec::new(); - for server in config.enabled_dev_servers()? { - let manifest = Manifest::from_config(server, config).await?; - manifests.push(manifest); - } - for server in config.extra_servers() { - let manifest = Manifest::from_config(server.clone(), config).await?; - manifests.push(manifest); - } - manifests.push(Manifest::from_config(config.server().clone(), config).await?); - Ok(Self(vec![RemoteSource::LuarocksManifests(manifests)])) - } - - /// Prepends local packages to the lookup order. - pub(crate) fn with_local(mut self, packages: Vec) -> Self { - self.0.insert(0, RemoteSource::Local(packages)); - self - } - - /// Prepends locked packages to the lookup order, ahead of manifests. - pub(crate) fn with_locks(mut self, locks: Vec) -> Self { - self.0.insert(0, RemoteSource::LockedPackageLocks(locks)); - self - } - - fn find_in_source( - source: &RemoteSource, - package_req: &PackageReq, - filter: Option, - ) -> Result, SearchError> { - match source { - RemoteSource::LuarocksManifests(manifests) => Ok(manifests - .iter() - .find_map(|manifest| manifest.find(package_req, filter.clone()))), - RemoteSource::LockedPackageLocks(locks) => Ok(locks - .iter() - .filter_map(|lock| lock.has_rock(package_req, filter.clone())) - .map(|local_package| { - RemotePackage::new( - local_package.to_package(), - local_package.source().clone(), - local_package.source_url().cloned(), - ) - }) - .next()), - RemoteSource::Local(packages) => Ok(packages - .iter() - .find(|package| package.package.package.name() == package_req.name()) - .map(|package| package.package.clone())), - } - } - - /// Find a remote package that matches the requirement, returning the latest match. - #[tracing::instrument( - name = "Searching tree-sitter parser", - level = "info", - skip_all, - fields(package = package_req.to_string()), - )] - pub(crate) fn find( - &self, - package_req: &PackageReq, - filter: Option, - ) -> Result { - for source in &self.0 { - if let Some(package) = Self::find_in_source(source, package_req, filter.clone())? { - return Ok(package); - } - } - if self - .0 - .iter() - .any(|source| matches!(source, RemoteSource::LuarocksManifests(_))) - { - Err(SearchError::RockNotFound(package_req.clone())) - } else if self - .0 - .iter() - .any(|source| matches!(source, RemoteSource::LockedPackageLocks(_))) - { - Err(SearchError::RockNotFoundInLockfile(package_req.clone())) - } else { - Err(SearchError::RockNotFound(package_req.clone())) - } - } - - /// Resolves a package install spec into a fully discovered package. - pub(crate) async fn resolve( - &self, - spec: &PackageInstallSpec, - config: &Config, - ) -> Result { - if let Some(source) = &spec.source { - let download = RemoteRockDownload::from_package_req_and_source_spec( - spec.package.clone(), - source.clone(), - ) - .map_err(|err| DiscoverError::Download(spec.package.clone(), Box::new(err)))?; - let rockspec = download.rockspec().clone(); - let package = RemotePackage::new( - spec.package.clone().try_into().map_err(|err| { - DiscoverError::Download( - spec.package.clone(), - Box::new(crate::operations::SearchAndDownloadError::from(err)), - ) - })?, - download.rockspec_download().source.clone(), - download.rockspec_download().source_url.clone(), - ); - return Ok(FoundPackage { - package, - rockspec, - artifact: None, - }); - } - - for source in &self.0 { - match source { - RemoteSource::Local(packages) => { - if let Some(package) = packages - .iter() - .find(|package| package.package.package.name() == spec.package.name()) - { - return Ok(package.clone()); - } - } - RemoteSource::LuarocksManifests(_) | RemoteSource::LockedPackageLocks(_) => { - let mut package = self.find(&spec.package, None)?; - let download = if let Some(vendor_dir) = config.vendor_dir() { - FetchVendored::new() - .vendor_dir(vendor_dir) - .package(&spec.package) - .package_db(self) - .fetch_vendored_rock() - .await - .map_err(|err| { - DiscoverError::FetchVendored(spec.package.clone(), Box::new(err)) - })? - } else { - Download::new(&spec.package, config) - .package_db(self) - .download_remote_rock() - .await - .map_err(|err| { - DiscoverError::Download(spec.package.clone(), Box::new(err)) - })? - }; - let rockspec = download.rockspec().clone(); - let artifact = match &download { - RemoteRockDownload::SrcRock { - src_rock, - source_url, - .. - } => { - package.source_url = Some(source_url.clone()); - Some(PackageSource::PackedRock(src_rock.clone())) - } - RemoteRockDownload::BinaryRock { packed_rock, .. } => { - Some(PackageSource::PackedRock(packed_rock.clone())) - } - RemoteRockDownload::RockspecOnly { .. } => None, - }; - return Ok(FoundPackage { - package, - rockspec, - artifact, - }); - } - } - } - - Err(DiscoverError::Search(SearchError::RockNotFound( - spec.package.clone(), - ))) - } - - /// Search for all packages that match the requirement. - pub fn search(&self, package_req: &PackageReq) -> Vec<(&PackageName, Vec<&PackageVersion>)> { - self.0 - .iter() - .flat_map(|source| match source { - RemoteSource::LuarocksManifests(manifests) => manifests - .iter() - .flat_map(|manifest| { - manifest - .metadata() - .repository - .iter() - .filter_map(|(name, elements)| { - if name.to_string().contains(&package_req.name().to_string()) { - Some(( - name, - elements - .keys() - .filter(|version| { - package_req.version_req().matches(version) - }) - .sorted_by(|a, b| Ord::cmp(b, a)) - .collect_vec(), - )) - } else { - None - } - }) - }) - .collect_vec(), - RemoteSource::LockedPackageLocks(locks) => locks - .iter() - .flat_map(|lock| lock.rocks().values()) - .filter_map(|package| { - // NOTE: This doesn't group packages by name, but we don't care for now, - // as we shouldn't need to use this function with a lockfile. - let name = package.name(); - if name.to_string().contains(&package_req.name().to_string()) { - Some((name, vec![package.version()])) - } else { - None - } - }) - .collect_vec(), - RemoteSource::Local(packages) => packages - .iter() - .filter_map(|package| { - let name = package.package.package.name(); - if name.to_string().contains(&package_req.name().to_string()) - && package_req - .version_req() - .matches(package.package.package.version()) - { - Some((name, vec![package.package.package.version()])) - } else { - None - } - }) - .collect_vec(), - }) - .collect() - } - - /// Find the latest version for a package by name. - pub(crate) fn latest_version(&self, rock_name: &PackageName) -> Option { - self.latest_match(&rock_name.clone().into(), None) - .map(|result| result.version().clone()) - } - - /// Find the latest package that matches the requirement. - pub fn latest_match( - &self, - package_req: &PackageReq, - filter: Option, - ) -> Option { - match self.find(package_req, filter) { - Ok(result) => Some(result.package), - Err(_) => None, - } - } -} - -impl From for RemotePackageDB { - fn from(manifest: Manifest) -> Self { - Self(vec![RemoteSource::LuarocksManifests(vec![manifest])]) - } -} - -impl From> for RemotePackageDB { - fn from(locks: Vec) -> Self { - Self(vec![RemoteSource::LockedPackageLocks(locks)]) - } -} diff --git a/lux-lib/src/upload/mod.rs b/lux-lib/src/upload/mod.rs index b54a94a56..7a10d6710 100644 --- a/lux-lib/src/upload/mod.rs +++ b/lux-lib/src/upload/mod.rs @@ -2,8 +2,8 @@ use std::{env, io}; use crate::operations::SearchAndDownloadError; use crate::package::SpecRevIterator; +use crate::package_db::PackageDB; use crate::project::project_toml::{ProjectTomlError, RemoteProjectTomlValidationError}; -use crate::remote_package_db::RemotePackageDB; use crate::rockspec::Rockspec; use crate::TOOL_VERSION; use crate::{config::Config, project::Project}; @@ -36,7 +36,7 @@ pub struct ProjectUpload<'a> { #[cfg(feature = "gpgme")] sign_protocol: SignatureProtocol, config: &'a Config, - package_db: &'a RemotePackageDB, + package_db: &'a PackageDB, } impl ProjectUploadBuilder<'_, State> @@ -510,7 +510,7 @@ mod helpers { client: &Client, api_key: &ApiKey, config: &Config, - package_db: &RemotePackageDB, + package_db: &PackageDB, ) -> Result<(RemoteProjectToml, String), UploadError> { for specrev in SpecRevIterator::new() { let rockspec = project.toml().into_remote(Some(specrev))?; diff --git a/lux-lua/src/lua_impls.rs b/lux-lua/src/lua_impls.rs index 25b0baa8b..91f48ceed 100644 --- a/lux-lua/src/lua_impls.rs +++ b/lux-lua/src/lua_impls.rs @@ -35,11 +35,11 @@ use lux_lib::{ lua_version::LuaVersion, operations::{DownloadedRockspec, PackageInstallSpec, SyncReport}, package::{PackageName, PackageReq, PackageSpec, PackageVersion, PackageVersionReq, SpecRev}, + package_db::PackageDB, project::{ project_toml::{LocalProjectToml, PartialProjectToml, RemoteProjectToml}, Project, }, - remote_package_db::RemotePackageDB, rockspec::{ lua_dependency::{DependencyType, LuaDependencySpec, LuaDependencyType}, Rockspec, @@ -2742,28 +2742,28 @@ impl mlua::UserData for RemoteProjectTomlLua { } #[derive(Debug, Clone)] -pub struct RemotePackageDBLua(pub RemotePackageDB); +pub struct PackageDBLua(pub PackageDB); -impl Typed for RemotePackageDBLua { +impl Typed for PackageDBLua { fn ty() -> Type { - Type::named("RemotePackageDB") + Type::named("PackageDB") } } -impl FromLua for RemotePackageDBLua { +impl FromLua for PackageDBLua { fn from_lua(value: LuaValue, _lua: &Lua) -> LuaResult { match value { - LuaValue::UserData(ud) => Ok(ud.borrow::()?.clone()), + LuaValue::UserData(ud) => Ok(ud.borrow::()?.clone()), v => Err(LuaError::FromLuaConversionError { from: v.type_name(), - to: "RemotePackageDBLua".to_string(), + to: "PackageDBLua".to_string(), message: None, }), } } } -impl mlua::UserData for RemotePackageDBLua { +impl mlua::UserData for PackageDBLua { fn add_fields>(fields: &mut F) { let mut wrapper = mlua_extras::typed::WrappedBuilder::new(fields); ::add_fields(&mut wrapper); @@ -2775,7 +2775,7 @@ impl mlua::UserData for RemotePackageDBLua { } } -impl TypedUserData for RemotePackageDBLua { +impl TypedUserData for PackageDBLua { fn add_methods>(methods: &mut M) { methods.document("Search for all packages that match the requirement"); methods.param( @@ -3133,7 +3133,7 @@ impl TypedUserData for ProjectLua { let _guard = lux_lib::lua::lua_runtime().enter(); let deps = map_dependency_type(deps.0); let package_db = - RemotePackageDB::from_config(&config.0) + PackageDB::from_config(&config.0) .await .into_lua_err()?; this.0.add(deps.as_ref(), &package_db).await.into_lua_err() @@ -3222,8 +3222,8 @@ mod definitions_registry { InstallSpecLua, LocalLuaRockspecLua, LocalProjectTomlLua, LockedPackageHashesLua, LockedPackageLua, LockfileGuardLua, LockfileReadOnlyLua, LockfileReadWriteLua, LuaDependencySpecLua, LuaScriptTestSpecLua, MakeBuildSpecLua, ModulePathsLua, - NvimLayoutLua, PackageReqLua, PackageSpecLua, PartialLuaRockspecLua, PartialProjectTomlLua, - PlatformSupportLua, ProjectLua, RemoteLuaRockspecLua, RemotePackageDBLua, + NvimLayoutLua, PackageDBLua, PackageReqLua, PackageSpecLua, PartialLuaRockspecLua, + PartialProjectTomlLua, PlatformSupportLua, ProjectLua, RemoteLuaRockspecLua, RemoteProjectTomlLua, RemoteRockSourceLua, RockDescriptionLua, RockLayoutLua, RustBinaryBuildSpecLua, RustMluaBuildSpecLua, TreeLua, TreesitterParserBuildSpecLua, WorkspaceLua, @@ -3276,7 +3276,7 @@ mod definitions_registry { "PartialProjectToml" => PartialProjectTomlLua, "LocalProjectToml" => LocalProjectTomlLua, "RemoteProjectToml" => RemoteProjectTomlLua, - "RemotePackageDB" => RemotePackageDBLua, + "PackageDB" => PackageDBLua, "LockfileReadOnly" => LockfileReadOnlyLua, "LockfileGuard" => LockfileGuardLua, "LockfileReadWrite" => LockfileReadWriteLua, diff --git a/lux-lua/src/operations.rs b/lux-lua/src/operations.rs index d9f2b2365..ca83d0041 100644 --- a/lux-lua/src/operations.rs +++ b/lux-lua/src/operations.rs @@ -16,7 +16,7 @@ use lux_lib::{ Update, }, package::{PackageName, PackageReq}, - remote_package_db::RemotePackageDB, + package_db::PackageDB, rockspec::lua_dependency::DependencyType, tree::{EntryType, RockMatches, Tree}, }; @@ -142,9 +142,7 @@ impl TypedUserData for OperationsModule { )| async move { let _runtime = lua_runtime().enter(); let deps = lua_impls::map_dependency_type(deps.0); - let package_db = RemotePackageDB::from_config(&config.0) - .await - .into_lua_err()?; + let package_db = PackageDB::from_config(&config.0).await.into_lua_err()?; let install_specs = deps_to_specs(&deps); @@ -356,9 +354,7 @@ mod definitions_registry { } async fn search(query: String, config: ConfigLua) -> mlua::Result>> { - let remote_db = RemotePackageDB::from_config(&config.0) - .await - .into_lua_err()?; + let remote_db = PackageDB::from_config(&config.0).await.into_lua_err()?; Ok(remote_db .search(&query.parse().into_lua_err()?) From 9d2714e484bfcd1dd84d84a656f3f101d6541f1f Mon Sep 17 00:00:00 2001 From: vhyrro Date: Sat, 10 Oct 2026 17:57:48 +0200 Subject: [PATCH 34/39] fix: resolve nitpicks and PR suggestions --- lux-cli/src/dist/flat_archive.rs | 2 ++ lux-cli/src/install.rs | 2 ++ lux-cli/src/outdated.rs | 9 +++++- lux-cli/src/pack.rs | 2 ++ lux-cli/src/uninstall.rs | 5 +++- lux-lib/src/drivers/install_packages.rs | 7 ++--- lux-lib/src/drivers/sync/frozen.rs | 12 ++++++-- lux-lib/src/drivers/sync/mod.rs | 17 ++++++------ lux-lib/src/fs/sync.rs | 34 +++++++++++------------ lux-lib/src/operations/exec.rs | 3 +- lux-lib/src/operations/install_project.rs | 2 +- lux-lib/src/operations/test.rs | 16 +++++------ lux-lib/src/operations/update.rs | 2 +- lux-lib/src/package_db/mod.rs | 2 +- lux-lib/src/workspace/mod.rs | 4 +-- lux-lib/tests/install.rs | 3 ++ lux-lua/src/operations.rs | 3 ++ 17 files changed, 74 insertions(+), 51 deletions(-) diff --git a/lux-cli/src/dist/flat_archive.rs b/lux-cli/src/dist/flat_archive.rs index 71b6e9b67..149744266 100644 --- a/lux-cli/src/dist/flat_archive.rs +++ b/lux-cli/src/dist/flat_archive.rs @@ -14,6 +14,7 @@ use lux_lib::{ lua_version::LuaVersion, operations::{InstallProject, PackageInstallSpec}, package::{PackageName, PackageReq}, + package_db::PackageDB, pipeline::build_local::Build, tree::{self, FlatDistTree, InstallTree}, workspace::Workspace, @@ -166,6 +167,7 @@ async fn install_package( .build_behaviour(BuildBehaviour::Force) .build(), ) + .package_db(PackageDB::from_config(config).await?) .install() .await? .0; diff --git a/lux-cli/src/install.rs b/lux-cli/src/install.rs index 235694dc6..c4ae60319 100644 --- a/lux-cli/src/install.rs +++ b/lux-cli/src/install.rs @@ -7,6 +7,7 @@ use lux_lib::{ lua_version::LuaVersion, operations, package::{PackageName, PackageReq}, + package_db::PackageDB, workspace::{Workspace, WorkspaceError}, }; @@ -81,6 +82,7 @@ async fn install_remote(data: Install, config: Config) -> Result<()> { // TODO(vhyrro): If the tree doesn't exist then error out. InstallPackages::new(&config, &tree) .packages(packages) + .package_db(PackageDB::from_config(&config).await?) .install() .await?; diff --git a/lux-cli/src/outdated.rs b/lux-cli/src/outdated.rs index dd4f9e9fe..eee85cc62 100644 --- a/lux-cli/src/outdated.rs +++ b/lux-cli/src/outdated.rs @@ -22,6 +22,9 @@ pub struct Outdated { /// If in a project, this lists rocks in the project tree pub async fn outdated(outdated_data: Outdated, config: Config) -> Result<()> { let workspace = Workspace::current()?; + // NOTE: We deliberately do not sync the workspace before listing outdated + // rocks. `Sync` may install or remove packages as a side effect, which would + // make this otherwise read-only command mutate the install tree. let tree = match &workspace { Some(project) => project.tree(&config)?, None => { @@ -53,6 +56,8 @@ pub async fn outdated(outdated_data: Outdated, config: Config) -> Result<()> { .sorted_by_key(|(rock, _)| rock.name().to_owned()) .into_group_map_by(|(rock, _)| rock.name().to_owned()); + let has_outdated = !rock_list.is_empty(); + match outdated_data.output_format { OutputFormat::Json => { let jsonified_rock_list = rock_list @@ -91,7 +96,9 @@ pub async fn outdated(outdated_data: Outdated, config: Config) -> Result<()> { } } - println!("\nRun `lx update` to update all outdated rocks."); + if has_outdated { + println!("\nRun `lx update` to update all outdated rocks."); + } Ok(()) } diff --git a/lux-cli/src/pack.rs b/lux-cli/src/pack.rs index 048a84432..c9a47c810 100644 --- a/lux-cli/src/pack.rs +++ b/lux-cli/src/pack.rs @@ -12,6 +12,7 @@ use lux_lib::{ lua_version::LuaVersion, operations::{self, PackageInstallSpec}, package::PackageName, + package_db::PackageDB, pipeline::build_local::Build, rockspec::Rockspec as _, tree::{self, InstallTree}, @@ -115,6 +116,7 @@ pub async fn pack(args: Pack, config: Config) -> Result<()> { .build_behaviour(BuildBehaviour::Force) .build(), ) + .package_db(PackageDB::from_config(&temp_config).await?) .install() .await? .0; diff --git a/lux-cli/src/uninstall.rs b/lux-cli/src/uninstall.rs index 34d077bef..91cd8b307 100644 --- a/lux-cli/src/uninstall.rs +++ b/lux-cli/src/uninstall.rs @@ -4,10 +4,12 @@ use itertools::Itertools; use lux_lib::{ build::BuildBehaviour, config::Config, + drivers::install_packages::InstallPackages, lockfile::LockedPackageId, lua_version::LuaVersion, operations::{self, PackageInstallSpec}, package::PackageReq, + package_db::PackageDB, tree::{self, InstallTree, RockMatches, TreeError}, }; @@ -151,7 +153,8 @@ Reinstall? .packages(dependencies) .remove() .await?; - lux_lib::drivers::install_packages::InstallPackages::new(&config, &tree) + InstallPackages::new(&config, &tree) + .package_db(PackageDB::from_config(&config).await?) .packages(reinstall_specs) .install() .await?; diff --git a/lux-lib/src/drivers/install_packages.rs b/lux-lib/src/drivers/install_packages.rs index 8300ea951..86640ad19 100644 --- a/lux-lib/src/drivers/install_packages.rs +++ b/lux-lib/src/drivers/install_packages.rs @@ -50,7 +50,7 @@ where test_packages: Option>, #[builder(field)] packages: Vec, - package_db: Option, + package_db: PackageDB, } impl<'a, T, State> InstallPackagesBuilder<'a, T, State> @@ -110,10 +110,7 @@ async fn install_packages( where T: InstallTree + Send + Sync, { - let package_db = match install.package_db { - Some(db) => db, - None => PackageDB::from_config(install.config).await?, - }; + let package_db = install.package_db; let duplicate_entrypoints = install .packages diff --git a/lux-lib/src/drivers/sync/frozen.rs b/lux-lib/src/drivers/sync/frozen.rs index ee1b3c03c..b88a02aa8 100644 --- a/lux-lib/src/drivers/sync/frozen.rs +++ b/lux-lib/src/drivers/sync/frozen.rs @@ -199,7 +199,9 @@ async fn do_sync( let package_db = workspace_lockfile.local_pkg_locks().into(); let (to_add, mut report) = reconcile_locks(&workspace_lockfile, &dest_lockfile, lock_type); - let member_names = args.workspace.member_names(); + // Workspace members are the workspace's own entrypoints, not dependencies, + // so they must never be reported (or uninstalled) as removed by a sync. + let member_names = args.workspace.member_names()?; report .removed .retain(|package| !member_names.contains(package.name())); @@ -236,6 +238,9 @@ async fn do_sync( .collect_vec(); install_tree_lockfile.map_then_flush(|lockfile| { + // `sync` replaces the lockfile's contents whole, so we synchronize the + // workspace lockfile first and only then re-add the workspace members as + // entrypoints. Adding them beforehand would have them wiped out. lockfile.sync(workspace_lockfile.local_pkg_lock(lock_type)); for package in &member_packages { lockfile.add_entrypoint(package); @@ -285,7 +290,7 @@ impl<'a, 'b> SyncPackages<'a, 'b> { } } - let mut extra_packages = self.args.extra_packages.iter().cloned().collect_vec(); + let mut extra_packages = self.args.extra_packages.clone(); match self.lock_type { LockedPackageLockType::Build => { for project in self.args.workspace.members() { @@ -402,7 +407,7 @@ fn validate_integrity( } /// Installs packages that were newly added to the workspace lockfile but are not yet present in -/// the install tree, using the default package database. +/// the install tree, using the workspace lockfile as the package database. async fn install_missing_packages( args: &Sync<'_>, tree: &T, @@ -433,6 +438,7 @@ where .collect(); let added = InstallPackages::new(args.config, tree) + .package_db(workspace_lockfile.local_pkg_locks().into()) .packages(missing_packages) .install() .await? diff --git a/lux-lib/src/drivers/sync/mod.rs b/lux-lib/src/drivers/sync/mod.rs index 71aae41b1..5d5cbae25 100644 --- a/lux-lib/src/drivers/sync/mod.rs +++ b/lux-lib/src/drivers/sync/mod.rs @@ -40,7 +40,10 @@ impl TargetSet { pub fn all(workspace: &Workspace, test: bool) -> Self { Self { test, - members: workspace.member_names(), + members: workspace + .members() + .clone() + .map(|project| project.toml().package.clone()), } } @@ -160,7 +163,7 @@ async fn sync_open(args: &Sync<'_>) -> Result { let regular = gather_dependencies( workspace, DependencyKind::Regular, - &workspace.member_names(), + &workspace.member_names()?, )? .into_iter() .filter(|spec| !tree.match_rocks(&spec.package).is_ok_and(|m| m.is_found())) @@ -170,7 +173,7 @@ async fn sync_open(args: &Sync<'_>) -> Result { }) .collect::>(); - let build = gather_dependencies(workspace, DependencyKind::Build, &workspace.member_names())? + let build = gather_dependencies(workspace, DependencyKind::Build, &workspace.member_names()?)? .into_iter() .filter(|spec| { !build_tree @@ -201,16 +204,12 @@ async fn sync_open(args: &Sync<'_>) -> Result { Vec::new() }; - let package_db = if !test.is_empty() || !regular.is_empty() || !build.is_empty() { - Some(PackageDB::for_open(config, workspace).await?) - } else { - None - }; + let package_db = PackageDB::open(config, workspace).await?; let mut install = InstallPackages::new(config, &tree) .packages(regular) .build_packages(build) - .maybe_package_db(package_db); + .package_db(package_db); if !test.is_empty() { install = install.test_packages(test); } diff --git a/lux-lib/src/fs/sync.rs b/lux-lib/src/fs/sync.rs index 6ee75c339..c14fc9e5c 100644 --- a/lux-lib/src/fs/sync.rs +++ b/lux-lib/src/fs/sync.rs @@ -23,6 +23,11 @@ pub(crate) fn write(path: impl AsRef, contents: impl AsRef<[u8]>) -> Resul }) } +/// Atomically writes `contents` to `path`. +/// +/// The contents are written to a temporary file in the destination directory +/// and then renamed into place, so readers never observe a partially-written +/// file. The temporary file is flushed to disk before the rename. pub(crate) fn write_atomic( path: impl AsRef, contents: impl AsRef<[u8]>, @@ -30,31 +35,26 @@ pub(crate) fn write_atomic( use std::io::Write; let path = path.as_ref(); - let parent = match path.parent() { - Some(parent) if !parent.as_os_str().is_empty() => parent, - _ => Path::new("."), - }; + let dir = path + .parent() + .filter(|dir| !dir.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); - let mut file = - tempfile::NamedTempFile::new_in(parent).map_err(|source| FsError::FileCreate { - path: path.to_path_buf(), - source, - })?; + let mut file = tempfile::NamedTempFile::new_in(dir).map_err(|source| FsError::FileCreate { + path: path.to_path_buf(), + source, + })?; file.write_all(contents.as_ref()) + .and_then(|()| file.as_file().sync_all()) .map_err(|source| FsError::Write { path: path.to_path_buf(), source, })?; - file.as_file().sync_all().map_err(|source| FsError::Write { + file.persist(path).map_err(|err| FsError::Write { path: path.to_path_buf(), - source, + source: err.error, })?; - file.persist(path) - .map_err(|err| FsError::Write { - path: path.to_path_buf(), - source: err.error, - }) - .map(|_| ()) + Ok(()) } /// Wrapped [`fs::copy`]. diff --git a/lux-lib/src/operations/exec.rs b/lux-lib/src/operations/exec.rs index d5d06322e..c316dd44f 100644 --- a/lux-lib/src/operations/exec.rs +++ b/lux-lib/src/operations/exec.rs @@ -9,7 +9,7 @@ use crate::{ lua_version::{LuaVersion, LuaVersionUnset}, operations::{BuildWorkspace, BuildWorkspaceError}, package::{PackageReq, PackageVersionReqError}, - package_db::PackageDBError, + package_db::{PackageDB, PackageDBError}, path::{Paths, PathsError}, tree::{self, InstallTree, TreeError}, workspace::{Workspace, WorkspaceTreeError}, @@ -236,6 +236,7 @@ async fn install_command(command: &str, config: &Config) -> Result<(), InstallCo .build(); InstallPackages::new(config, &tree) .package(install_spec) + .package_db(PackageDB::from_config(config).await?) .install() .await?; Ok(()) diff --git a/lux-lib/src/operations/install_project.rs b/lux-lib/src/operations/install_project.rs index 49fced9ff..1932e1d5b 100644 --- a/lux-lib/src/operations/install_project.rs +++ b/lux-lib/src/operations/install_project.rs @@ -120,7 +120,7 @@ impl< let root = FoundPackage::from_project_root(rockspec, project.root().to_path_buf()); let package_db = match args.workspace { - Some(workspace) => PackageDB::for_open(config, workspace).await?, + Some(workspace) => PackageDB::open(config, workspace).await?, None => PackageDB::from_config(config).await?, } .with_local(vec![root]); diff --git a/lux-lib/src/operations/test.rs b/lux-lib/src/operations/test.rs index 1fcc59b41..f82f61003 100644 --- a/lux-lib/src/operations/test.rs +++ b/lux-lib/src/operations/test.rs @@ -164,15 +164,13 @@ async fn run_project_tests( let test_spec = rocks.test().current_platform().to_validated(project)?; let test_config = test_spec.test_config(config)?; - Box::pin( - Sync::new(workspace, &test_config) - .mode(SyncMode::Open) - .targets(TargetSet::member(project.toml().package().clone(), true)) - .no_lock(no_lock) - .behaviour(BuildBehaviour::Ignore) - .sync(), - ) - .await?; + Sync::new(workspace, &test_config) + .mode(SyncMode::Open) + .targets(TargetSet::member(project.toml().package().clone(), true)) + .no_lock(no_lock) + .behaviour(BuildBehaviour::Ignore) + .sync() + .await?; let lua_version = project.lua_version(&test_config)?; let project_tree = workspace.lua_version_tree(lua_version, &test_config)?; diff --git a/lux-lib/src/operations/update.rs b/lux-lib/src/operations/update.rs index 2db731e02..448875d30 100644 --- a/lux-lib/src/operations/update.rs +++ b/lux-lib/src/operations/update.rs @@ -294,7 +294,7 @@ async fn update( .map(|updatable| mk_install_spec(updatable, lockfile)) .collect(), ) - .maybe_package_db(Some(package_db)) + .package_db(package_db) .install() .await?; Ok(updated_packages) diff --git a/lux-lib/src/package_db/mod.rs b/lux-lib/src/package_db/mod.rs index 4577b7788..24ee64d78 100644 --- a/lux-lib/src/package_db/mod.rs +++ b/lux-lib/src/package_db/mod.rs @@ -82,7 +82,7 @@ impl PackageDB { /// Builds a package database that prefers the workspace lockfile, falling /// back to the manifests for requirements the lockfile does not satisfy. - pub(crate) async fn for_open( + pub(crate) async fn open( config: &Config, workspace: &Workspace, ) -> Result { diff --git a/lux-lib/src/workspace/mod.rs b/lux-lib/src/workspace/mod.rs index 41805f544..ae034517c 100644 --- a/lux-lib/src/workspace/mod.rs +++ b/lux-lib/src/workspace/mod.rs @@ -160,14 +160,14 @@ impl Workspace { } /// The package names of all workspace members. - pub fn member_names(&self) -> NonEmpty { + pub fn member_names(&self) -> Result, WorkspaceError> { NonEmpty::from_vec( self.members() .iter() .map(|project| project.toml().package.clone()) .collect(), ) - .expect("workspace has at least one member") + .ok_or_else(|| WorkspaceError::EmptyWorkspace(self.root.0.clone())) } /// Get a workspace member, defaulting to the first one if none is specified. diff --git a/lux-lib/tests/install.rs b/lux-lib/tests/install.rs index 08836c02c..af3e4ef55 100644 --- a/lux-lib/tests/install.rs +++ b/lux-lib/tests/install.rs @@ -11,6 +11,7 @@ use lux_lib::{ lua_version::LuaVersion, operations::{Exec, PackageInstallSpec}, package::{PackageName, PackageReq}, + package_db::PackageDB, tree::{EntryType, InstallTree}, }; use std::path::PathBuf; @@ -80,6 +81,7 @@ async fn install_and_use(package: PackageReq, module_name: &str) { .unwrap(); let installed = InstallPackages::new(&config, &tree) .package(install_spec) + .package_db(PackageDB::from_config(&config).await.unwrap()) .install() .await .unwrap() @@ -134,6 +136,7 @@ async fn test_install(install_spec: PackageInstallSpec) { .unwrap(); let installed = InstallPackages::new(&config, &tree) .package(install_spec) + .package_db(PackageDB::from_config(&config).await.unwrap()) .install() .await .unwrap() diff --git a/lux-lua/src/operations.rs b/lux-lua/src/operations.rs index ca83d0041..e2a5a0e22 100644 --- a/lux-lua/src/operations.rs +++ b/lux-lua/src/operations.rs @@ -64,7 +64,9 @@ impl TypedUserData for OperationsModule { |_, (packages, tree, config): (Vec, TreeLua, ConfigLua)| async move { let _runtime = lua_runtime().enter(); let specs = packages.into_iter().map(|p| p.0).collect(); + let package_db = PackageDB::from_config(&config.0).await.into_lua_err()?; InstallPackages::new(&config.0, &tree.0) + .package_db(package_db) .packages(specs) .install() .await @@ -159,6 +161,7 @@ impl TypedUserData for OperationsModule { let tree = workspace.0.tree(&config.0).into_lua_err()?; InstallPackages::new(&config.0, &tree) + .package_db(package_db) .packages(install_specs) .install() .await From 320a30785bfd4beb449fcaecbc3bdb0071340cad Mon Sep 17 00:00:00 2001 From: vhyrro Date: Sat, 10 Oct 2026 22:06:24 +0200 Subject: [PATCH 35/39] fix: make CI happy --- lux-lib/src/drivers/sync/frozen.rs | 14 ++++++++++++-- lux-lib/src/drivers/sync/mod.rs | 16 ++++++++++++++-- lux-lib/src/operations/build_workspace.rs | 8 +++++++- lux-lib/src/operations/vendor.rs | 9 --------- lux-lib/src/package_db/mod.rs | 8 +++++++- lux-lib/tests/install.rs | 4 ++-- 6 files changed, 42 insertions(+), 17 deletions(-) diff --git a/lux-lib/src/drivers/sync/frozen.rs b/lux-lib/src/drivers/sync/frozen.rs index b88a02aa8..f177adbee 100644 --- a/lux-lib/src/drivers/sync/frozen.rs +++ b/lux-lib/src/drivers/sync/frozen.rs @@ -16,6 +16,7 @@ use crate::{ luarocks::luarocks_installation::LUAROCKS_VERSION, operations::{self, GenLuaRcError}, package::{PackageName, PackageReq}, + package_db::{PackageDB, PackageDBError}, pipeline::resolve::luarocks_build_backend_name, project::{project_toml::LocalProjectTomlValidationError, ProjectError}, rockspec::{lua_dependency::LuaDependencySpec, Rockspec}, @@ -156,6 +157,9 @@ pub enum SyncError { #[error("failed to generate `.luarc.json`")] #[diagnostic(forward(0))] GenLuaRc(#[from] GenLuaRcError), + #[error(transparent)] + #[diagnostic(transparent)] + PackageDB(Box), } impl From for SyncError { @@ -164,6 +168,12 @@ impl From for SyncError { } } +impl From for SyncError { + fn from(source: PackageDBError) -> Self { + Self::PackageDB(Box::new(source)) + } +} + #[tracing::instrument(name = "Syncing dependencies", skip_all)] async fn do_sync( args: &Sync<'_>, @@ -407,7 +417,7 @@ fn validate_integrity( } /// Installs packages that were newly added to the workspace lockfile but are not yet present in -/// the install tree, using the workspace lockfile as the package database. +/// the install tree, resolving them from the configured package database. async fn install_missing_packages( args: &Sync<'_>, tree: &T, @@ -438,7 +448,7 @@ where .collect(); let added = InstallPackages::new(args.config, tree) - .package_db(workspace_lockfile.local_pkg_locks().into()) + .package_db(PackageDB::from_config(args.config).await?) .packages(missing_packages) .install() .await? diff --git a/lux-lib/src/drivers/sync/mod.rs b/lux-lib/src/drivers/sync/mod.rs index 5d5cbae25..7d4b6b297 100644 --- a/lux-lib/src/drivers/sync/mod.rs +++ b/lux-lib/src/drivers/sync/mod.rs @@ -82,7 +82,7 @@ pub enum SyncError { InstallPackages(#[from] Box), #[error(transparent)] #[diagnostic(transparent)] - InstallProject(#[from] InstallProjectError), + InstallProject(Box), #[error(transparent)] #[diagnostic(transparent)] Workspace(#[from] WorkspaceError), @@ -103,7 +103,7 @@ pub enum SyncError { GenLuaRc(#[from] GenLuaRcError), #[error(transparent)] #[diagnostic(transparent)] - Frozen(#[from] frozen::SyncError), + Frozen(Box), } impl From for SyncError { @@ -112,6 +112,18 @@ impl From for SyncError { } } +impl From for SyncError { + fn from(source: InstallProjectError) -> Self { + Self::InstallProject(Box::new(source)) + } +} + +impl From for SyncError { + fn from(source: frozen::SyncError) -> Self { + Self::Frozen(Box::new(source)) + } +} + /// Reconciles a workspace's install trees with its dependencies. #[derive(Builder)] #[builder(start_fn = new, finish_fn(name = _build, vis = ""))] diff --git a/lux-lib/src/operations/build_workspace.rs b/lux-lib/src/operations/build_workspace.rs index 340028fd0..a38536978 100644 --- a/lux-lib/src/operations/build_workspace.rs +++ b/lux-lib/src/operations/build_workspace.rs @@ -15,7 +15,13 @@ use tracing::{info_span, Instrument}; pub enum BuildWorkspaceError { #[error(transparent)] #[diagnostic(transparent)] - Sync(#[from] SyncError), + Sync(Box), +} + +impl From for BuildWorkspaceError { + fn from(source: SyncError) -> Self { + Self::Sync(Box::new(source)) + } } #[derive(Builder)] diff --git a/lux-lib/src/operations/vendor.rs b/lux-lib/src/operations/vendor.rs index c3ea60e3e..b402171d7 100644 --- a/lux-lib/src/operations/vendor.rs +++ b/lux-lib/src/operations/vendor.rs @@ -503,7 +503,6 @@ async fn cargo_vendor( mod tests { use std::path::PathBuf; - use bytes::Bytes; use tempfile::TempDir; use super::*; @@ -511,7 +510,6 @@ mod tests { build::BuildBehaviour, config::ConfigBuilder, lockfile::{LockConstraint, LockedPackage, LockedPackageSpec, OptState, PinnedState}, - operations::unpack_rockspec, remote_package_source::RemotePackageSource, }; use assert_fs::prelude::PathCopy; @@ -524,13 +522,6 @@ mod tests { .unwrap() } - fn fixture_bytes(name: &str) -> Bytes { - let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("resources/test") - .join(name); - Bytes::from(std::fs::read(path).unwrap()) - } - fn make_downloaded_package( rockspec: RemoteLuaRockspec, source: RemotePackageSource, diff --git a/lux-lib/src/package_db/mod.rs b/lux-lib/src/package_db/mod.rs index 24ee64d78..afeff7c37 100644 --- a/lux-lib/src/package_db/mod.rs +++ b/lux-lib/src/package_db/mod.rs @@ -37,7 +37,13 @@ pub enum PackageDBError { ConfigError(#[from] ConfigError), #[error(transparent)] #[diagnostic(transparent)] - Workspace(#[from] WorkspaceError), + Workspace(Box), +} + +impl From for PackageDBError { + fn from(err: WorkspaceError) -> Self { + Self::Workspace(Box::new(err)) + } } #[derive(Error, Debug, Diagnostic)] diff --git a/lux-lib/tests/install.rs b/lux-lib/tests/install.rs index af3e4ef55..df10185c4 100644 --- a/lux-lib/tests/install.rs +++ b/lux-lib/tests/install.rs @@ -10,9 +10,9 @@ use lux_lib::{ lua_rockspec::RockSourceSpec, lua_version::LuaVersion, operations::{Exec, PackageInstallSpec}, - package::{PackageName, PackageReq}, + package::PackageReq, package_db::PackageDB, - tree::{EntryType, InstallTree}, + tree::EntryType, }; use std::path::PathBuf; use walkdir::WalkDir; From e4bee7fe4e36ddd4dbb0bd0969a1a8daa499f6f1 Mon Sep 17 00:00:00 2001 From: vhyrro Date: Sun, 11 Oct 2026 11:31:10 +0200 Subject: [PATCH 36/39] fix: properly gate impure tests for CI --- lux-lib/src/build/mod.rs | 16 ++++++++++++++-- lux-lib/src/operations/build_workspace.rs | 2 +- lux-lib/src/operations/dist_bin.rs | 8 ++++++-- lux-lib/src/operations/test.rs | 2 +- lux-lib/src/operations/vendor.rs | 8 ++++---- 5 files changed, 26 insertions(+), 10 deletions(-) diff --git a/lux-lib/src/build/mod.rs b/lux-lib/src/build/mod.rs index 0bab6c130..9b6fd97ac 100644 --- a/lux-lib/src/build/mod.rs +++ b/lux-lib/src/build/mod.rs @@ -454,11 +454,14 @@ mod tests { lockfile::{LockConstraint, LockedPackageHashes}, lua_installation::{detect_installed_lua_version, LuaInstallation}, lua_version::LuaVersion, - operations::{unpack_rockspec, DownloadedPackedRockBytes}, package::PackageSpec, - pipeline::build_local::Build, project::Project, remote_package_source::RemotePackageSource, + }; + #[cfg(feature = "impure_tests")] + use crate::{ + operations::{unpack_rockspec, DownloadedPackedRockBytes}, + pipeline::build_local::Build, tree::Tree, }; @@ -542,11 +545,15 @@ mod tests { bin_file.assert(predicate::str::contains("echo \"Hello\"")); } + #[cfg(feature = "impure_tests")] const LUATEST_SRC_ROCK: &str = "resources/test/luatest-0.2-1.src.rock"; + #[cfg(feature = "impure_tests")] const LUATEST_SRC_ROCK_SOURCE_HASH: &str = "sha256-2jS0XOq0iIVhsZJ3BVqXSlKsx2vsqCAaaYyqcBEb7RI="; + #[cfg(feature = "impure_tests")] const LUATEST_ROCKSPEC_HASH: &str = "sha256-NljJ20A+VadUyhhBjrRnojeQjSqRpQy7FWcgjUt2Fdc="; + #[cfg(feature = "impure_tests")] fn luatest_src_rock_bytes() -> Bytes { Bytes::copy_from_slice( &std::fs::read(PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(LUATEST_SRC_ROCK)) @@ -554,6 +561,7 @@ mod tests { ) } + #[cfg(feature = "impure_tests")] fn luatest_config(dir: &assert_fs::TempDir) -> Config { ConfigBuilder::new() .unwrap() @@ -563,6 +571,7 @@ mod tests { .unwrap() } + #[cfg(feature = "impure_tests")] async fn build_luatest( config: &Config, tree: &Tree, @@ -601,6 +610,7 @@ mod tests { .unwrap() } + #[cfg(feature = "impure_tests")] #[tokio::test] async fn luatest_src_rock_produces_expected_hashes() { let dir = assert_fs::TempDir::new().unwrap(); @@ -625,6 +635,7 @@ mod tests { assert!(tree.bin().join("luatest").is_file()); } + #[cfg(feature = "impure_tests")] #[tokio::test] async fn force_build_is_deterministic() { let dir1 = assert_fs::TempDir::new().unwrap(); @@ -654,6 +665,7 @@ mod tests { assert_eq!(package1.hashes(), package2.hashes()); } + #[cfg(feature = "impure_tests")] #[tokio::test] async fn noforce_skips_rebuild_when_package_is_locked() { let dir = assert_fs::TempDir::new().unwrap(); diff --git a/lux-lib/src/operations/build_workspace.rs b/lux-lib/src/operations/build_workspace.rs index a38536978..94bacacde 100644 --- a/lux-lib/src/operations/build_workspace.rs +++ b/lux-lib/src/operations/build_workspace.rs @@ -74,7 +74,7 @@ impl Config { ConfigBuilder::new() .unwrap() @@ -630,6 +629,7 @@ mod tests { assert!(stale_dir.join("hello.lua").is_file()); } + #[cfg(feature = "impure_tests")] #[tokio::test] async fn vendor_workspace_no_delete_preserves_existing_contents() { let sample_project_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")) From 8e5e0480e719f2aad23c077366cab55ba0dcf2b2 Mon Sep 17 00:00:00 2001 From: Marc Jakobi Date: Mon, 5 Oct 2026 21:41:52 +0200 Subject: [PATCH 37/39] feat(lockfile): auto-migrate lockfiles with default layout (#1998) --- lux-lib/src/lockfile/mod.rs | 222 +++++++++++++++++++++++++++++++----- 1 file changed, 193 insertions(+), 29 deletions(-) diff --git a/lux-lib/src/lockfile/mod.rs b/lux-lib/src/lockfile/mod.rs index ffb817ca8..34f5bd7ca 100644 --- a/lux-lib/src/lockfile/mod.rs +++ b/lux-lib/src/lockfile/mod.rs @@ -928,6 +928,9 @@ pub enum LockfileError { expected: String, help: String, }, + #[error("cannot migrate lockfile '{}'", lockfile.display())] + #[diagnostic(code(lux_lib::lockfile::conflicting_entrypoints), help("{help}"))] + ConflictingEntrypoints { lockfile: PathBuf, help: String }, } #[derive(Error, Debug, Diagnostic)] @@ -1265,7 +1268,7 @@ impl Lockfile { remove the tree at '{}' and reinstall all packages it contained.", tree_dir.display() ); - let mut lockfile: Lockfile = parse_lockfile(&content, &filepath, &help)?; + let mut lockfile: Lockfile = load_lockfile(&content, &filepath, &help)?; lockfile.filepath = filepath; Ok(lockfile) } @@ -1366,7 +1369,7 @@ impl WorkspaceLockfile { #[tracing::instrument(level = "trace")] pub fn load(filepath: PathBuf) -> Result, LockfileError> { let content = fs::sync::read_to_string(&filepath)?; - let mut lockfile: WorkspaceLockfile = parse_lockfile( + let mut lockfile: WorkspaceLockfile = load_lockfile( &content, &filepath, "this lockfile was created by an incompatible version of Lux. @@ -1598,36 +1601,133 @@ impl Drop for ProjectLockfileGuard { } } -fn parse_lockfile(content: &str, lockfile: &Path, help: &str) -> Result +fn load_lockfile(content: &str, lockfile: &Path, help: &str) -> Result where T: serde::de::DeserializeOwned, { - let value: serde_json::Value = - serde_json::from_str(content).map_err(LockfileError::ParseJson)?; + let mut value = parse_lockfile_json(content)?; + if lockfile_version(&value, lockfile)?.major == 1 { + migrate_v1_lockfile(&mut value, lockfile)?; + } + deserialize_lockfile(value, lockfile, help) +} + +fn parse_lockfile_json( + content: &str, +) -> Result, LockfileError> { + serde_json::from_str(content).map_err(LockfileError::ParseJson) +} +fn lockfile_version( + value: &serde_json::Map, + lockfile: &Path, +) -> Result { let version = value .get("version") .and_then(|version| version.as_str()) - .ok_or_else(|| LockfileError::MissingVersion { + .ok_or(LockfileError::MissingVersion { lockfile: lockfile.to_path_buf(), })?; + semver::Version::parse(version).map_err(|source| LockfileError::InvalidVersion { + version: version.to_string(), + source, + }) +} - let found = - semver::Version::parse(version).map_err(|source| LockfileError::InvalidVersion { - version: version.to_string(), - source, - })?; - +fn deserialize_lockfile( + value: serde_json::Map, + lockfile: &Path, + help: &str, +) -> Result +where + T: serde::de::DeserializeOwned, +{ + let found = lockfile_version(&value, lockfile)?; if found.major != LOCKFILE_VERSION { return Err(LockfileError::IncompatibleVersion { lockfile: lockfile.to_path_buf(), - found: version.into(), + found: found.to_string(), expected: LOCKFILE_VERSION_STR.into(), help: help.into(), }); } + serde_json::from_value(serde_json::Value::Object(value)).map_err(LockfileError::ParseJson) +} - serde_json::from_value(value).map_err(LockfileError::ParseJson) +fn migrate_v1_lockfile( + value: &mut serde_json::Map, + lockfile: &Path, +) -> Result<(), LockfileError> { + let tree_dir = lockfile.parent().unwrap_or(lockfile); + let found = lockfile_version(value, lockfile)?; + if value.contains_key("entrypoint_layout") { + return Err(LockfileError::IncompatibleVersion { + lockfile: lockfile.to_path_buf(), + found: found.to_string(), + expected: LOCKFILE_VERSION_STR.into(), + help: format!( + "this install tree uses a custom entrypoint layout that cannot be migrated automatically. +remove the tree at '{}' and reinstall all packages it contained.", + tree_dir.display() + ), + }); + } + + let conflict_help = format!( + "this lockfile contains entrypoints that cannot be represented in the current format. +remove the tree at '{}' and reinstall all packages it contained.", + tree_dir.display() + ); + + if value.contains_key("entrypoints") { + migrate_entrypoints(value, lockfile, &conflict_help)?; + } + for section in ["dependencies", "test_dependencies", "build_dependencies"] { + if let Some(section) = value + .get_mut(section) + .and_then(|section| section.as_object_mut()) + { + migrate_entrypoints(section, lockfile, &conflict_help)?; + } + } + value.insert( + "version".into(), + serde_json::Value::String(LOCKFILE_VERSION_STR.into()), + ); + Ok(()) +} + +fn migrate_entrypoints( + section: &mut serde_json::Map, + lockfile: &Path, + help: &str, +) -> Result<(), LockfileError> { + let conflicting_entrypoints = || LockfileError::ConflictingEntrypoints { + lockfile: lockfile.to_path_buf(), + help: help.to_string(), + }; + + let rocks = section.get("rocks").and_then(|rocks| rocks.as_object()); + let mut entrypoints = serde_json::Map::new(); + if let Some(old_entrypoints) = section.get("entrypoints") { + for id in old_entrypoints + .as_array() + .ok_or_else(&conflicting_entrypoints)? + { + let id = id.as_str().ok_or_else(&conflicting_entrypoints)?; + let name = rocks + .and_then(|rocks| rocks.get(id)) + .and_then(|package| package.get("name")) + .and_then(|name| name.as_str()) + .ok_or_else(&conflicting_entrypoints)?; + if entrypoints.contains_key(name) { + return Err(conflicting_entrypoints()); + } + entrypoints.insert(name.to_string(), serde_json::Value::String(id.to_string())); + } + } + section.insert("entrypoints".into(), serde_json::Value::Object(entrypoints)); + Ok(()) } fn integrity_err_not_found(package: &LockedPackage) -> LockfileIntegrityError { @@ -1942,21 +2042,85 @@ mod tests { #[test] fn verify_lockfile_version() { - for version in ["1.0.0", "9999.0.0", "not-a-version"] { - let lockfile = format!( - r#" - {{ - "version": {version} - }} - "# - ); + let deserialize = |version: &str| { + let content = format!(r#"{{"version": "{version}"}}"#); + parse_lockfile_json(&content).and_then(|value| { + deserialize_lockfile::>(value, Path::new("lux.lock"), "") + }) + }; - assert!(super::parse_lockfile::>( - &lockfile, - Path::new("lux.lock"), - "" - ) - .is_err()); - } + assert!(matches!( + deserialize("3.0.0"), + Err(LockfileError::IncompatibleVersion { .. }) + )); + assert!(matches!( + deserialize("not-a-version"), + Err(LockfileError::InvalidVersion { .. }) + )); + } + + #[test] + fn migrate_v1_tree_lockfile() { + let sample = + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("resources/test/lockfile-v1/tree.lock"); + + let temp = assert_fs::TempDir::new().unwrap(); + let lockfile_path = temp.path().join("lux.lock"); + std::fs::copy(&sample, &lockfile_path).unwrap(); + + let lockfile = Lockfile::load(lockfile_path.clone()).unwrap(); + + let on_disk: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(&lockfile_path).unwrap()).unwrap(); + assert_eq!(on_disk["version"], "1.0.0"); + assert!(on_disk["entrypoints"].is_array()); + + assert_eq!(lockfile.version, "2.0.0"); + assert_eq!(lockfile.lock.entrypoints.len(), 3); + } + + #[test] + fn refuse_migration_for_custom_layout() { + let lockfile = r#"{"version":"1.0.0","entrypoint_layout":{"root":"site/pack/lux"}}"#; + assert!(matches!( + load_lockfile::>(lockfile, Path::new("lux.lock"), ""), + Err(LockfileError::IncompatibleVersion { .. }) + )); + } + + #[test] + fn refuse_migration_for_conflicting_entrypoints() { + let lockfile = r#"{ + "version": "1.0.0", + "rocks": { + "a": { "name": "foo" }, + "b": { "name": "foo" } + }, + "entrypoints": ["a", "b"] + }"#; + assert!(matches!( + load_lockfile::>(lockfile, Path::new("lux.lock"), ""), + Err(LockfileError::ConflictingEntrypoints { .. }) + )); + } + + #[test] + fn migrate_v1_workspace_lockfile() { + let sample = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("resources/test/lockfile-v1/workspace.lock"); + + let temp = assert_fs::TempDir::new().unwrap(); + let lockfile_path = temp.path().join("lux.lock"); + std::fs::copy(&sample, &lockfile_path).unwrap(); + + let lockfile = WorkspaceLockfile::load(lockfile_path.clone()).unwrap(); + + let on_disk: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(&lockfile_path).unwrap()).unwrap(); + assert_eq!(on_disk["version"], "1.0.0"); + assert!(on_disk["dependencies"]["entrypoints"].is_array()); + + assert_eq!(lockfile.version, "2.0.0"); + assert_eq!(lockfile.dependencies.entrypoints.len(), 3); } } From 0c3962835690a271e93dbb7664b3f90638ffb746 Mon Sep 17 00:00:00 2001 From: vhyrro Date: Sun, 11 Oct 2026 12:36:10 +0200 Subject: [PATCH 38/39] fix(lux-lua): mark loader test as impure --- lux-lua/src/loader.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lux-lua/src/loader.rs b/lux-lua/src/loader.rs index e01ef1d9b..f90f36681 100644 --- a/lux-lua/src/loader.rs +++ b/lux-lua/src/loader.rs @@ -442,7 +442,7 @@ mod tests { assert_eq!(result, "nested"); } - #[cfg(target_os = "linux")] + #[cfg(all(target_os = "linux", feature = "impure_tests"))] #[tokio::test] async fn test_load_c_module_from_built_project() { use lux_lib::{ From 786cfa2adf30310efb211a3bb5a8102598ad3f5a Mon Sep 17 00:00:00 2001 From: vhyrro Date: Sun, 11 Oct 2026 15:00:47 +0200 Subject: [PATCH 39/39] fix(ci): make CI happy --- lux-lib/src/operations/dist_bin.rs | 3 +++ nix/README.md | 4 ++-- nix/tests/build-lux-helpers.nix | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lux-lib/src/operations/dist_bin.rs b/lux-lib/src/operations/dist_bin.rs index d311005fb..3d509d663 100644 --- a/lux-lib/src/operations/dist_bin.rs +++ b/lux-lib/src/operations/dist_bin.rs @@ -537,14 +537,17 @@ local unpack = unpack or table.unpack #[cfg(test)] mod tests { + #[cfg(any(target_os = "linux", feature = "impure_tests"))] use super::*; #[cfg(target_os = "linux")] use assert_fs::prelude::{PathChild, PathCreateDir}; + #[cfg(any(target_os = "linux", feature = "impure_tests"))] use assert_fs::TempDir; #[cfg(feature = "impure_tests")] use {crate::lua_installation::detect_installed_lua_version, assert_fs::fixture::PathCopy}; + #[cfg(any(target_os = "linux", feature = "impure_tests"))] use crate::{config::ConfigBuilder, lua_version::LuaVersion, tree::FlatDistTree}; #[cfg(target_os = "linux")] use crate::{ diff --git a/nix/README.md b/nix/README.md index e140e4888..677be36bb 100644 --- a/nix/README.md +++ b/nix/README.md @@ -37,7 +37,7 @@ buildLuxPackage { hash = "sha256-imyKEDvbn9EWJ8Cm54v5sUKZV2fZuBTT3du+1ipKgW4="; }; - luxHash = "sha256-DpkDMK6AV7Iy6YFbKbx+EEkDeGaAFbszNkLPhdVr+qY="; + luxHash = "sha256-XaA9FtwHtm8BSvjTyRHvAkgCZpbo48gNI5yfNQuRL78="; meta = { description = "Modern and ergonomic error handling for Lua"; @@ -70,7 +70,7 @@ buildLuxApplication { hash = "sha256-imyKEDvbn9EWJ8Cm54v5sUKZV2fZuBTT3du+1ipKgW4="; }; - luxHash = "sha256-DpkDMK6AV7Iy6YFbKbx+EEkDeGaAFbszNkLPhdVr+qY="; + luxHash = "sha256-XaA9FtwHtm8BSvjTyRHvAkgCZpbo48gNI5yfNQuRL78="; } ``` diff --git a/nix/tests/build-lux-helpers.nix b/nix/tests/build-lux-helpers.nix index 9a928e826..96ae15d46 100644 --- a/nix/tests/build-lux-helpers.nix +++ b/nix/tests/build-lux-helpers.nix @@ -74,7 +74,7 @@ ''; in { fetchLuxDeps = mkTest "build-lux-helpers-test" { - luxHash = "sha256-DpkDMK6AV7Iy6YFbKbx+EEkDeGaAFbszNkLPhdVr+qY="; + luxHash = "sha256-XaA9FtwHtm8BSvjTyRHvAkgCZpbo48gNI5yfNQuRL78="; }; importLuxLock = mkTest "build-lux-helpers-import-lock-test" { @@ -98,7 +98,7 @@ in { pname = "fallo"; version = "0-unstable-2026-03-25"; inherit src; - luxHash = "sha256-DpkDMK6AV7Iy6YFbKbx+EEkDeGaAFbszNkLPhdVr+qY="; + luxHash = "sha256-XaA9FtwHtm8BSvjTyRHvAkgCZpbo48gNI5yfNQuRL78="; }; in pkgs.runCommandLocal "build-lux-helpers-with-packages-test" {} ''