diff --git a/lux-cli/src/add.rs b/lux-cli/src/add.rs index d07e8525e..b33534a28 100644 --- a/lux-cli/src/add.rs +++ b/lux-cli/src/add.rs @@ -2,7 +2,7 @@ use itertools::{Either, Itertools}; use lux_lib::{ config::Config, package::PackageName, - remote_package_db::RemotePackageDB, + package_db::PackageDB, rockspec::lua_dependency::{self}, workspace::Workspace, }; @@ -45,7 +45,7 @@ pub struct Add { pub async fn add(data: Add, config: Config) -> Result<()> { let mut workspace = Workspace::current_or_err()?; let project = workspace.single_member_or_select_mut(&data.package)?; - let db = RemotePackageDB::from_config(&config).await?; + let db = PackageDB::from_config(&config).await?; let (dependencies, git_dependencies): (Vec<_>, Vec<_>) = data.package_req.iter().partition_map(|req| match req { diff --git a/lux-cli/src/build.rs b/lux-cli/src/build.rs index 8d15533b8..3d2b1530a 100644 --- a/lux-cli/src/build.rs +++ b/lux-cli/src/build.rs @@ -1,8 +1,9 @@ use clap::Args; use lux_lib::{ + build::BuildBehaviour, config::Config, - lockfile::LocalPackage, - operations::{self}, + drivers::sync::{Sync, SyncMode, TargetSet}, + lockfile::LockedPackage, package::PackageName, workspace::Workspace, }; @@ -24,15 +25,28 @@ pub struct Build { } /// Returns `Some` if the `only_deps` arg is set to `false`. -pub async fn build(data: Build, config: Config) -> Result> { +pub async fn build(data: Build, config: Config) -> Result> { + build_with_behaviour(data, config, BuildBehaviour::Force).await +} + +pub async fn build_with_behaviour( + data: Build, + config: Config, + behaviour: BuildBehaviour, +) -> Result> { let workspace = Workspace::current_or_err()?; - let result = operations::BuildWorkspace::new(&workspace, &config) - .maybe_package(data.package) - .no_lock(data.no_lock) + let report = Sync::new(&workspace, &config) + .mode(SyncMode::Open) + .targets(match data.package { + Some(package) => TargetSet::member(package, false), + None => TargetSet::all(&workspace, false), + }) + .behaviour(behaviour) .only_deps(data.only_deps) - .build() + .no_lock(data.no_lock) + .sync() .await?; - Ok(result) + Ok(report.added().to_vec()) } #[cfg(test)] diff --git a/lux-cli/src/dist/flat_archive.rs b/lux-cli/src/dist/flat_archive.rs index 04c1d6cae..149744266 100644 --- a/lux-cli/src/dist/flat_archive.rs +++ b/lux-cli/src/dist/flat_archive.rs @@ -5,14 +5,17 @@ use std::{ use clap::Args; use lux_lib::{ - build::{Build, BuildBehaviour}, + build::BuildBehaviour, config::{Config, ConfigBuilder}, - lockfile::LocalPackage, + drivers::install_packages::InstallPackages, + lockfile::LockedPackage, lua_installation::LuaInstallation, lua_rockspec::RemoteLuaRockspec, lua_version::LuaVersion, - operations::{Install, InstallProject, PackageInstallSpec}, + operations::{InstallProject, PackageInstallSpec}, package::{PackageName, PackageReq}, + package_db::PackageDB, + pipeline::build_local::Build, tree::{self, FlatDistTree, InstallTree}, workspace::Workspace, }; @@ -132,7 +135,7 @@ async fn install_project( package: Option<&PackageName>, staging_dir: &TempDir, config: &Config, -) -> Result<(LocalPackage, PathBuf)> { +) -> Result<(LockedPackage, PathBuf)> { let workspace = Workspace::current_or_err()?; let project = match package { Some(package) => workspace.select_member(package)?, @@ -155,18 +158,19 @@ async fn install_package( package: &PackageReq, staging_dir: &TempDir, config: &Config, -) -> Result<(LocalPackage, PathBuf)> { +) -> Result<(LockedPackage, PathBuf)> { let lua_version = LuaVersion::from(config)?.clone(); let tree = FlatDistTree::new(staging_dir.path().to_path_buf(), lua_version, config)?; - let packages = Install::new(config) + let packages = InstallPackages::new(config, &tree) .package( PackageInstallSpec::new(package.clone(), tree::EntryType::Entrypoint) .build_behaviour(BuildBehaviour::Force) .build(), ) - .tree(tree.clone()) + .package_db(PackageDB::from_config(config).await?) .install() - .await?; + .await? + .0; let package = packages .into_iter() .find(|pkg| pkg.name() == package.name()) @@ -178,7 +182,7 @@ async fn install_rockspec( rockspec_path: &Path, staging_dir: &TempDir, config: &Config, -) -> Result<(LocalPackage, PathBuf)> { +) -> Result<(LockedPackage, PathBuf)> { let content = tokio::fs::read_to_string(&rockspec_path) .await .into_diagnostic()?; @@ -203,6 +207,7 @@ async fn install_rockspec( .tree(&tree) .entry_type(tree::EntryType::Entrypoint) .config(config) + .behaviour(BuildBehaviour::Force) .build() .await?; Ok((package, tree.root())) diff --git a/lux-cli/src/doc.rs b/lux-cli/src/doc.rs index 9666f8053..aae94230b 100644 --- a/lux-cli/src/doc.rs +++ b/lux-cli/src/doc.rs @@ -3,7 +3,7 @@ use inquire::{Confirm, Select}; use itertools::Itertools; use lux_lib::{ config::Config, - lockfile::LocalPackage, + lockfile::LockedPackage, lua_rockspec::RemoteLuaRockspec, lua_version::LuaVersion, package::PackageReq, @@ -50,7 +50,7 @@ pub async fn doc(args: Doc, config: Config) -> Result<()> { } } -async fn open_homepage(pkg: LocalPackage, tree: &Tree) -> Result<()> { +async fn open_homepage(pkg: LockedPackage, tree: &Tree) -> Result<()> { let homepage = match get_homepage(&pkg, tree)? { Some(homepage) => Ok(homepage), None => Err(miette!( @@ -62,9 +62,9 @@ async fn open_homepage(pkg: LocalPackage, tree: &Tree) -> Result<()> { Ok(()) } -fn get_homepage(pkg: &LocalPackage, tree: &Tree) -> Result> { +fn get_homepage(pkg: &LockedPackage, tree: &Tree) -> Result> { let rockspec_content = - std::fs::read_to_string(tree.layout_for(pkg).rockspec_path()).into_diagnostic()?; + std::fs::read_to_string(tree.layout_for(pkg.spec()).rockspec_path()).into_diagnostic()?; let rockspec = RemoteLuaRockspec::new(&rockspec_content)?; Ok(rockspec.description().homepage.clone()) } @@ -88,8 +88,8 @@ fn open_doc_file(path: &Path) -> Result<()> { Ok(()) } -async fn open_local_docs(pkg: LocalPackage, tree: &Tree, config: &Config) -> Result<()> { - let doc_dir = tree.layout_for(&pkg).doc; +async fn open_local_docs(pkg: LockedPackage, tree: &Tree, config: &Config) -> Result<()> { + let doc_dir = tree.layout_for(pkg.spec()).doc; let files: Vec = WalkDir::new(&doc_dir) .into_iter() .filter_map_ok(|file| { diff --git a/lux-cli/src/install.rs b/lux-cli/src/install.rs index a604bb305..c4ae60319 100644 --- a/lux-cli/src/install.rs +++ b/lux-cli/src/install.rs @@ -2,10 +2,12 @@ use std::path::PathBuf; use lux_lib::{ config::Config, + drivers::install_packages::InstallPackages, lockfile::PinnedState, lua_version::LuaVersion, operations, package::{PackageName, PackageReq}, + package_db::PackageDB, workspace::{Workspace, WorkspaceError}, }; @@ -62,6 +64,7 @@ async fn install_from_path( .project(project) .config(&config) .tree(&tree) + .workspace(&workspace) .build() .await?; @@ -77,9 +80,9 @@ async fn install_remote(data: Install, config: Config) -> Result<()> { let packages = apply_build_behaviour(data.package_req, pin, data.force, &tree, &config)?; // TODO(vhyrro): If the tree doesn't exist then error out. - operations::Install::new(&config) + InstallPackages::new(&config, &tree) .packages(packages) - .tree(tree) + .package_db(PackageDB::from_config(&config).await?) .install() .await?; diff --git a/lux-cli/src/list.rs b/lux-cli/src/list.rs index 3e21e4d17..36532915d 100644 --- a/lux-cli/src/list.rs +++ b/lux-cli/src/list.rs @@ -4,7 +4,7 @@ use clap::Args; use itertools::Itertools as _; use lux_lib::{ config::Config, - lockfile::{LocalPackageId, PinnedState}, + lockfile::{LockedPackageId, PinnedState}, lua_version::LuaVersion, tree::InstallTree, }; @@ -33,7 +33,7 @@ pub fn list_installed(list_data: ListCmd, config: Config) -> Result<()> { let lockfile = tree.lockfile()?; let mut available_rocks = tree.list()?; if list_data.orphans { - let reachable: HashSet = lockfile + let reachable: HashSet = lockfile .reachable_rocks() .into_iter() .map(|package| package.id()) diff --git a/lux-cli/src/outdated.rs b/lux-cli/src/outdated.rs index cd848a991..eee85cc62 100644 --- a/lux-cli/src/outdated.rs +++ b/lux-cli/src/outdated.rs @@ -3,14 +3,14 @@ use std::collections::HashMap; use clap::Args; use itertools::Itertools; use lux_lib::{ - config::Config, lockfile::LocalPackage, lua_version::LuaVersion, package::PackageVersion, - remote_package_db::RemotePackageDB, workspace::Workspace, + config::Config, lockfile::LockedPackage, lua_version::LuaVersion, package::PackageVersion, + package_db::PackageDB, workspace::Workspace, }; use miette::{IntoDiagnostic, Result}; use text_trees::{FormatCharacters, StringTreeNode, TreeFormatting}; -use crate::{args::OutputFormat, workspace::sync_dependencies_if_locked}; +use crate::args::OutputFormat; #[derive(Args)] pub struct Outdated { @@ -22,19 +22,18 @@ pub struct Outdated { /// If in a project, this lists rocks in the project tree pub async fn outdated(outdated_data: Outdated, config: Config) -> Result<()> { let workspace = Workspace::current()?; + // NOTE: We deliberately do not sync the workspace before listing outdated + // rocks. `Sync` may install or remove packages as a side effect, which would + // make this otherwise read-only command mutate the install tree. let tree = match &workspace { - Some(project) => { - // Make sure dependencies are synced if in a project - sync_dependencies_if_locked(project, &config).await?; - project.tree(&config)? - } + Some(project) => project.tree(&config)?, None => { let lua_version = LuaVersion::from(&config)?.clone(); config.user_tree(lua_version)? } }; - let package_db = RemotePackageDB::from_config(&config).await?; + let package_db = PackageDB::from_config(&config).await?; // NOTE: This will display all installed versions and each possible upgrade. // However, this should also take into account dependency constraints made by other rocks. @@ -43,19 +42,22 @@ pub async fn outdated(outdated_data: Outdated, config: Config) -> Result<()> { let rock_list = tree.as_rock_list()?; let rock_list = rock_list .iter() - .map(|rock| { + .filter_map(|rock| { rock.to_package() .has_update(&package_db) - .map(|mb_version| mb_version.map(|version| (rock, version))) + .ok() + .flatten() + .map(|version| (rock, version)) }) - .filter_map_ok(|mb_tuple| mb_tuple) - .try_collect::<_, Vec<(&LocalPackage, PackageVersion)>, _>()?; + .collect::>(); let rock_list = rock_list .iter() .sorted_by_key(|(rock, _)| rock.name().to_owned()) .into_group_map_by(|(rock, _)| rock.name().to_owned()); + let has_outdated = !rock_list.is_empty(); + match outdated_data.output_format { OutputFormat::Json => { let jsonified_rock_list = rock_list @@ -94,5 +96,9 @@ pub async fn outdated(outdated_data: Outdated, config: Config) -> Result<()> { } } + if has_outdated { + println!("\nRun `lx update` to update all outdated rocks."); + } + Ok(()) } diff --git a/lux-cli/src/pack.rs b/lux-cli/src/pack.rs index 1bca82a05..c9a47c810 100644 --- a/lux-cli/src/pack.rs +++ b/lux-cli/src/pack.rs @@ -4,13 +4,16 @@ use crate::{args::PackageOrRockspec, build, workspace::exists_matching_workspace use clap::Args; use itertools::Itertools; use lux_lib::{ - build::{Build, BuildBehaviour}, + build::BuildBehaviour, config::Config, + drivers::install_packages::InstallPackages, lua_installation::LuaInstallation, lua_rockspec::RemoteLuaRockspec, lua_version::LuaVersion, - operations::{self, Install, PackageInstallSpec}, + operations::{self, PackageInstallSpec}, package::PackageName, + package_db::PackageDB, + pipeline::build_local::Build, rockspec::Rockspec as _, tree::{self, InstallTree}, workspace::Workspace, @@ -107,15 +110,16 @@ pub async fn pack(args: Pack, config: Config) -> Result<()> { let temp_dir = tempdir().into_diagnostic()?; let temp_config = config.with_tree(temp_dir.path().to_path_buf()); let tree = temp_config.user_tree(lua_version.clone())?; - let packages = Install::new(&temp_config) + let packages = InstallPackages::new(&temp_config, &tree) .package( PackageInstallSpec::new(package_req, tree::EntryType::Entrypoint) .build_behaviour(BuildBehaviour::Force) .build(), ) - .tree(tree.clone()) + .package_db(PackageDB::from_config(&temp_config).await?) .install() - .await?; + .await? + .0; let package = packages .first() .ok_or_else(|| miette!("no packages installed"))?; @@ -174,6 +178,7 @@ pub async fn pack(args: Pack, config: Config) -> Result<()> { .tree(&tree) .entry_type(tree::EntryType::Entrypoint) .config(&config) + .behaviour(BuildBehaviour::Force) .build() .await?; let rock_path = operations::Pack::new(dest_dir, tree, package) diff --git a/lux-cli/src/pin.rs b/lux-cli/src/pin.rs index cbdecfac3..3f04a2dfd 100644 --- a/lux-cli/src/pin.rs +++ b/lux-cli/src/pin.rs @@ -1,6 +1,7 @@ use clap::Args; use itertools::Itertools; use lux_lib::config::Config; +use lux_lib::drivers::sync::{Sync, SyncMode, TargetSet}; use lux_lib::lockfile::PinnedState; use lux_lib::lua_version::LuaVersion; use lux_lib::operations; @@ -82,8 +83,9 @@ pub async fn set_pinned_state(data: ChangePin, config: Config, pin: PinnedState) ) .await?; } - operations::Sync::new(&workspace, &config) - .test(!test_packages.is_empty()) + Sync::new(&workspace, &config) + .mode(SyncMode::Frozen) + .targets(TargetSet::all(&workspace, !test_packages.is_empty())) .sync() .await .wrap_err("syncing dependencies with the workspace lockfile failed.")?; diff --git a/lux-cli/src/run.rs b/lux-cli/src/run.rs index e3a9caed7..5b7a254ac 100644 --- a/lux-cli/src/run.rs +++ b/lux-cli/src/run.rs @@ -1,7 +1,7 @@ use std::path::PathBuf; use clap::Args; -use lux_lib::{config::Config, operations, workspace::Workspace}; +use lux_lib::{build::BuildBehaviour, config::Config, operations, workspace::Workspace}; use miette::Result; use crate::build::{self, Build}; @@ -31,7 +31,7 @@ pub async fn run(run_args: Run, config: Config) -> Result<()> { let workspace = Workspace::current_or_err()?; let package = run_args.build.package.clone(); - build::build(run_args.build, config.clone()).await?; + build::build_with_behaviour(run_args.build, config.clone(), BuildBehaviour::Ignore).await?; operations::Run::new() .workspace(&workspace) diff --git a/lux-cli/src/run_lua.rs b/lux-cli/src/run_lua.rs index 6d0272736..171af0375 100644 --- a/lux-cli/src/run_lua.rs +++ b/lux-cli/src/run_lua.rs @@ -17,6 +17,8 @@ use miette::{IntoDiagnostic, Result, WrapErr}; use crate::build::{self, Build}; +use lux_lib::build::BuildBehaviour; + #[derive(Args, Default)] #[clap(disable_help_flag = true)] pub struct RunLua { @@ -102,7 +104,8 @@ To exit type 'exit()' or . } if workspace.is_some() { - build::build(run_lua.build_args, config.clone()).await?; + build::build_with_behaviour(run_lua.build_args, config.clone(), BuildBehaviour::Ignore) + .await?; } let args = &run_lua.args.unwrap_or_default(); diff --git a/lux-cli/src/search.rs b/lux-cli/src/search.rs index 15f82a44b..3664d77ff 100644 --- a/lux-cli/src/search.rs +++ b/lux-cli/src/search.rs @@ -8,7 +8,7 @@ use text_trees::{FormatCharacters, StringTreeNode, TreeFormatting}; use lux_lib::{ config::Config, package::{PackageName, PackageReq, PackageVersion}, - remote_package_db::RemotePackageDB, + package_db::PackageDB, }; use crate::args::OutputFormat; @@ -24,7 +24,7 @@ pub struct Search { pub async fn search(data: Search, config: Config) -> Result<()> { let formatting = TreeFormatting::dir_tree(FormatCharacters::box_chars()); - let package_db = RemotePackageDB::from_config(&config).await?; + let package_db = PackageDB::from_config(&config).await?; let lua_package_req = data.lua_package_req; diff --git a/lux-cli/src/sync.rs b/lux-cli/src/sync.rs index 9e439fb88..70684562d 100644 --- a/lux-cli/src/sync.rs +++ b/lux-cli/src/sync.rs @@ -1,5 +1,9 @@ use clap::Args; -use lux_lib::{config::Config, operations::Sync, workspace::Workspace}; +use lux_lib::{ + config::Config, + drivers::sync::{Sync, SyncMode, TargetSet}, + workspace::Workspace, +}; use miette::Result; @@ -15,8 +19,9 @@ pub async fn sync(args: SyncProject, config: Config) -> Result<()> { let workspace = Workspace::current_or_err()?; let report = Sync::new(&workspace, &config) + .mode(SyncMode::Frozen) + .targets(TargetSet::all(&workspace, true)) .validate_integrity(!args.no_integrity_check) - .test(true) .sync() .await?; diff --git a/lux-cli/src/uninstall.rs b/lux-cli/src/uninstall.rs index ba10bf535..91cd8b307 100644 --- a/lux-cli/src/uninstall.rs +++ b/lux-cli/src/uninstall.rs @@ -4,10 +4,12 @@ use itertools::Itertools; use lux_lib::{ build::BuildBehaviour, config::Config, - lockfile::LocalPackageId, + drivers::install_packages::InstallPackages, + lockfile::LockedPackageId, lua_version::LuaVersion, operations::{self, PackageInstallSpec}, package::PackageReq, + package_db::PackageDB, tree::{self, InstallTree, RockMatches, TreeError}, }; @@ -83,7 +85,7 @@ multiple packages satisfying your version requirements were found: )); } - let (dependencies, entrypoints): (Vec, Vec) = packages + let (dependencies, entrypoints): (Vec, Vec) = packages .iter() .cloned() .partition(|pkg_id| lockfile.is_dependency(pkg_id)); @@ -151,9 +153,9 @@ Reinstall? .packages(dependencies) .remove() .await?; - operations::Install::new(&config) + InstallPackages::new(&config, &tree) + .package_db(PackageDB::from_config(&config).await?) .packages(reinstall_specs) - .tree(tree) .install() .await?; } else { diff --git a/lux-cli/src/update.rs b/lux-cli/src/update.rs index 58d835e1b..114bae969 100644 --- a/lux-cli/src/update.rs +++ b/lux-cli/src/update.rs @@ -2,7 +2,7 @@ use clap::Args; use itertools::Itertools; use lux_lib::package::{PackageName, PackageReq}; -use lux_lib::remote_package_db::RemotePackageDB; +use lux_lib::package_db::PackageDB; use lux_lib::rockspec::lua_dependency::LuaDependencyType; use lux_lib::workspace::Workspace; use lux_lib::{config::Config, operations}; @@ -42,7 +42,7 @@ pub async fn update(args: Update, config: Config) -> Result<()> { if args.toml { let mut workspace = Workspace::current_or_err()?; - let db = RemotePackageDB::from_config(&config).await?; + let db = PackageDB::from_config(&config).await?; let package_names = to_package_names(args.packages.as_ref())?; let mut upgrade_all = true; if let Some(packages) = package_names { diff --git a/lux-cli/src/upload.rs b/lux-cli/src/upload.rs index fc6304d77..5841519a4 100644 --- a/lux-cli/src/upload.rs +++ b/lux-cli/src/upload.rs @@ -1,7 +1,7 @@ use clap::Args; use lux_lib::{ - config::Config, package::PackageName, remote_package_db::RemotePackageDB, - upload::ProjectUpload, workspace::Workspace, + config::Config, package::PackageName, package_db::PackageDB, upload::ProjectUpload, + workspace::Workspace, }; use miette::{IntoDiagnostic, Result}; @@ -31,7 +31,7 @@ pub struct Upload { pub async fn upload(data: Upload, config: Config) -> Result<()> { let workspace = Workspace::current_or_err()?; - let package_db = RemotePackageDB::from_config(&config).await?; + let package_db = PackageDB::from_config(&config).await?; let tfa_code = tfa_code_from_args_or_secret(&data, &config)?; if let Some(package) = data.package { let project = workspace.select_member(&package)?; @@ -62,7 +62,7 @@ pub async fn upload(data: Upload, config: Config) -> Result<()> { #[cfg(not(feature = "gpgme"))] pub async fn upload(data: Upload, config: Config) -> Result<()> { let workspace = Workspace::current_or_err()?; - let package_db = RemotePackageDB::from_config(&config).await?; + let package_db = PackageDB::from_config(&config).await?; let tfa_code = tfa_code_from_args_or_secret(&data, &config)?; if let Some(package) = data.package { let project = workspace.select_member(&package)?; diff --git a/lux-cli/src/workspace.rs b/lux-cli/src/workspace.rs index f9bb588f3..075923e53 100644 --- a/lux-cli/src/workspace.rs +++ b/lux-cli/src/workspace.rs @@ -2,8 +2,13 @@ use std::{collections::HashSet, path::PathBuf, str::FromStr}; use itertools::Itertools; use lux_lib::{ - config::Config, git::shorthand::RemoteGitUrlShorthand, lua_version::LuaVersion, - operations::Sync, package::PackageReq, tree::Tree, workspace::Workspace, + config::Config, + drivers::sync::{Sync, SyncMode, TargetSet}, + git::shorthand::RemoteGitUrlShorthand, + lua_version::LuaVersion, + package::PackageReq, + tree::Tree, + workspace::Workspace, }; use miette::{Context, Result}; use walkdir::WalkDir; @@ -80,7 +85,8 @@ pub async fn sync_dependencies_if_locked(workspace: &Workspace, config: &Config) // NOTE: We only update the lockfile if one exists. // Otherwise, the next `lx build` will remove the packages. Sync::new(workspace, config) - .test(true) + .mode(SyncMode::Frozen) + .targets(TargetSet::all(workspace, true)) .sync() .await .wrap_err("syncing dependencies with the workspace lockfile failed.")?; diff --git a/lux-lib/resources/test/lux.lock b/lux-lib/resources/test/lux.lock index c8a9283c3..261910019 100644 --- a/lux-lib/resources/test/lux.lock +++ b/lux-lib/resources/test/lux.lock @@ -109,10 +109,6 @@ "name": "lua-cjson", "version": "2.1.0-1", "constraint": "==2.1.0", - "binaries": [ - "json2lua", - "lua2json" - ], "source": "luarocks_rockspec+https://luarocks.org/", "source_url": { "type": "url", diff --git a/lux-lib/resources/test/sample-projects/busted-with-lockfile/lux.lock b/lux-lib/resources/test/sample-projects/busted-with-lockfile/lux.lock index a433b5905..1596a2272 100644 --- a/lux-lib/resources/test/sample-projects/busted-with-lockfile/lux.lock +++ b/lux-lib/resources/test/sample-projects/busted-with-lockfile/lux.lock @@ -98,13 +98,6 @@ "name": "lua_cliargs", "version": "3.0.2-1", "constraint": ">=3.0", - "binaries": [ - "watch-tests.sh", - "release", - "lint", - "docs", - "coverage" - ], "source": "luarocks_rockspec+https://luarocks.org/", "source_url": { "type": "git", @@ -158,10 +151,6 @@ "2bd1c8e4fea66a88aeb65c1c8a3d0efdf188d681d2d47f163b976a771545aac9", "01a3c364614bddff7370223a5a9c4580f8e62d144384148444c518ec5367a59b" ], - "binaries": [ - "busted", - "busted" - ], "source": "luarocks_rockspec+https://luarocks.org/", "source_url": { "type": "git", diff --git a/lux-lib/resources/test/sample-projects/lockfile-missing-deps/lux.lock b/lux-lib/resources/test/sample-projects/lockfile-missing-deps/lux.lock index 098155682..efafe79a2 100644 --- a/lux-lib/resources/test/sample-projects/lockfile-missing-deps/lux.lock +++ b/lux-lib/resources/test/sample-projects/lockfile-missing-deps/lux.lock @@ -178,10 +178,6 @@ "e7c4c9fcd3cb6fe33c1c0a2ab8cdbbe8ee81d11334277d1f5631c80317770769": { "name": "lua-cjson", "version": "2.1.0-1", - "binaries": [ - "lua2json", - "json2lua" - ], "source": "luarocks_rockspec+https://luarocks.org/", "source_url": { "type": "url", diff --git a/lux-lib/resources/test/sample-tree/5.1/lux.lock b/lux-lib/resources/test/sample-tree/5.1/lux.lock index 9449cac51..8d515dbf6 100644 --- a/lux-lib/resources/test/sample-tree/5.1/lux.lock +++ b/lux-lib/resources/test/sample-tree/5.1/lux.lock @@ -108,10 +108,6 @@ "name": "lua-cjson", "version": "2.1.0-1", "constraint": "==2.1.0", - "binaries": [ - "json2lua", - "lua2json" - ], "source": "luarocks_rockspec+https://luarocks.org/", "source_url": { "type": "url", diff --git a/lux-lib/src/build/backend.rs b/lux-lib/src/build/backend.rs index f889d1c21..23eba0fc6 100644 --- a/lux-lib/src/build/backend.rs +++ b/lux-lib/src/build/backend.rs @@ -7,14 +7,15 @@ use std::{ use bon::Builder; use crate::{ - build::external_dependency::ExternalDependencyInfo, config::Config, lockfile::LocalPackage, - lua_installation::LuaInstallation, lua_rockspec::DeploySpec, tree::InstallTree, + build::external_dependency::ExternalDependencyInfo, config::Config, + lockfile::LockedPackageSpec, lua_installation::LuaInstallation, lua_rockspec::DeploySpec, + tree::InstallTree, }; #[derive(Builder)] #[builder(start_fn(name = "new"))] pub(crate) struct RunBuildArgs<'a, T: InstallTree> { - pub(crate) package: &'a LocalPackage, + pub(crate) package: &'a LockedPackageSpec, pub(crate) no_install: bool, pub(crate) lua: &'a LuaInstallation, pub(crate) external_dependencies: &'a HashMap, diff --git a/lux-lib/src/build/command.rs b/lux-lib/src/build/command.rs index 5bfef0f71..d7f73e6be 100644 --- a/lux-lib/src/build/command.rs +++ b/lux-lib/src/build/command.rs @@ -50,7 +50,9 @@ pub enum CommandError { ShellNotFoundError(#[from] which::Error), #[error("error executing command:\n{command}\n\nerror: {err}")] Io { err: io::Error, command: String }, - #[error("failed to execute command:\n{command}\n\nstatus: {status}\nstdout: {stdout}\nstderr: {stderr}")] + #[error( + "failed to execute command:\n{command}\n\nstatus: {status}\nstdout: {stdout}\nstderr: {stderr}" + )] CommandFailure { command: String, status: ExitStatus, @@ -151,7 +153,7 @@ async fn run_command( return Err(CommandError::Io { err, command: substituted_cmd, - }) + }); } Ok(child) => match child .wait_with_output() @@ -174,7 +176,7 @@ async fn run_command( return Err(CommandError::Io { err, command: substituted_cmd, - }) + }); } }, } diff --git a/lux-lib/src/build/luarocks.rs b/lux-lib/src/build/luarocks.rs index 0fb665d70..dcd6fdd6a 100644 --- a/lux-lib/src/build/luarocks.rs +++ b/lux-lib/src/build/luarocks.rs @@ -10,7 +10,7 @@ use std::path::Path; use crate::{ config::Config, - lockfile::LocalPackage, + lockfile::LockedPackageSpec, luarocks::luarocks_installation::{ExecLuaRocksError, LuaRocksError, LuaRocksInstallation}, }; @@ -34,9 +34,8 @@ pub enum LuarocksBuildError { #[error(transparent)] #[diagnostic(transparent)] Tree(#[from] TreeError), - #[error(transparent)] - #[diagnostic(transparent)] - Rockspec(Box), + #[error("failed to serialize rockspec: {0}")] + Rockspec(String), #[error("error installing luarocks compatibility layer")] #[diagnostic(forward(0))] LuaVersion(#[from] LuaVersionError), @@ -68,7 +67,7 @@ pub(crate) async fn build( &rockspec_file, rockspec .to_lua_remote_rockspec_string() - .map_err(|err| LuarocksBuildError::Rockspec(Box::new(err)))?, + .map_err(|err| LuarocksBuildError::Rockspec(err.to_string()))?, ) .await?; let luarocks = LuaRocksInstallation::new(config, tree.build_tree(config)?)?; @@ -83,7 +82,7 @@ async fn install( rockspec: &R, luarocks_tree: &Path, tree: &T, - package: &LocalPackage, + package: &LockedPackageSpec, config: &Config, ) -> Result { let layout = tree.layout_for(package); diff --git a/lux-lib/src/build/make.rs b/lux-lib/src/build/make.rs index 82b26c248..6f136cd0c 100644 --- a/lux-lib/src/build/make.rs +++ b/lux-lib/src/build/make.rs @@ -173,7 +173,7 @@ impl BuildBackend for MakeBuildSpec { status: output.status, stdout: String::from_utf8_lossy(&output.stdout).into(), stderr: String::from_utf8_lossy(&output.stderr).into(), - }) + }); } Err(err) => return Err(MakeError::Io(err)), } diff --git a/lux-lib/src/build/mod.rs b/lux-lib/src/build/mod.rs index f3f2b4753..9b6fd97ac 100644 --- a/lux-lib/src/build/mod.rs +++ b/lux-lib/src/build/mod.rs @@ -1,15 +1,14 @@ use crate::build::backend::{BuildBackend, BuildInfo, RunBuildArgs}; use crate::fs; -use crate::lockfile::{LockfileError, OptState, RemotePackageSourceUrl}; +use crate::lockfile::{LockfileError, RemotePackageSourceUrl}; use crate::lua_installation::LuaInstallationError; use crate::lua_rockspec::LuaVersionError; -use crate::operations::{RemotePackageSourceMetadata, UnpackError}; -use crate::rockspec::{LuaVersionCompatibility, Rockspec}; -use crate::tree::{self, EntryType, InstallTree, TreeError}; +use crate::operations::UnpackError; +use crate::rockspec::Rockspec; +use crate::tree::{EntryType, InstallTree, TreeError}; use bytes::Bytes; use std::collections::HashMap; use std::fs::DirEntry; -use std::io::Cursor; use std::path::PathBuf; use std::{io, path::Path}; use tracing::Instrument; @@ -17,14 +16,11 @@ use tracing::Instrument; use crate::{ config::Config, hash::HasIntegrity, - lockfile::{LocalPackage, LocalPackageHashes, LockConstraint, PinnedState}, + lockfile::{LockedPackage, LockedPackageSpec}, lua_installation::LuaInstallation, lua_rockspec::BuildBackendSpec, - operations::{self, FetchSrcError}, - package::PackageSpec, - remote_package_source::RemotePackageSource, + operations::FetchSrcError, }; -use bon::Builder; use builtin::BuiltinBuildError; use cmake::CMakeError; use command::CommandError; @@ -60,34 +56,6 @@ pub(crate) mod utils; pub mod external_dependency; -/// A rocks package builder, providing fine-grained control -/// over how a package should be built. -#[derive(Builder)] -#[builder(start_fn = new, finish_fn(name = _build, vis = ""))] -pub struct Build<'a, R: Rockspec + HasIntegrity, T: InstallTree> { - rockspec: &'a R, - tree: &'a T, - entry_type: tree::EntryType, - config: &'a Config, - lua: &'a LuaInstallation, - - #[builder(default)] - pin: PinnedState, - #[builder(default)] - opt: OptState, - #[builder(default)] - constraint: LockConstraint, - #[builder(default)] - behaviour: BuildBehaviour, - - #[builder(setters(vis = "pub(crate)"))] - source_spec: Option, - - // TODO(vhyrro): Remove this and enforce that this is provided at a type level. - #[builder(setters(vis = "pub(crate)"))] - source: Option, -} - #[derive(Debug)] pub(crate) enum RemotePackageSourceSpec { RockSpec(Option), @@ -101,22 +69,6 @@ pub(crate) struct SrcRockSource { pub source_url: RemotePackageSourceUrl, } -// Overwrite the `build()` function to use our own instead. -impl BuildBuilder<'_, R, T, State> -where - State: build_builder::State + build_builder::IsComplete, -{ - pub async fn build(self) -> Result { - let build = self._build(); - let span = tracing::info_span!( - "Building", - package = build.rockspec.package().to_string(), - version = build.rockspec.version().to_string(), - ); - do_build(build).instrument(span).await - } -} - #[derive(Error, Debug, Diagnostic)] #[non_exhaustive] pub enum BuildError { @@ -209,12 +161,15 @@ impl From for BuildError { } } -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, Default)] +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] pub enum BuildBehaviour { - /// Don't force a rebuild if the package is already installed - #[default] - NoForce, - /// Force a rebuild if the package is already installed + /// If a package with the same version is already installed, do nothing. + /// Otherwise, behave like [`BuildBehaviour::Conflict`]. + Ignore, + /// Error if an entrypoint with the same name is already installed, even if + /// it is the same version. + Conflict, + /// Remove any installed entrypoint with the same name and rebuild. Force, } @@ -250,7 +205,7 @@ async fn run_build( async fn install( rockspec: &R, tree: &T, - package: &LocalPackage, + package: &LockedPackageSpec, lua: &LuaInstallation, build_dir: &Path, entry_type: &EntryType, @@ -320,144 +275,102 @@ async fn install( Ok(()) } -#[tracing::instrument(level = "trace", skip_all)] -async fn do_build(build: Build<'_, R, T>) -> Result -where - R: Rockspec + HasIntegrity, - T: InstallTree + Sync, -{ - let rockspec = build.rockspec; - let lua = build.lua; - - rockspec.validate_lua_version(&lua.version)?; - - let tree = build.tree; - - let temp_dir = fs::tempfile::tempdir()?; - - let source_metadata = match build.source_spec { - Some(RemotePackageSourceSpec::SrcRock(SrcRockSource { bytes, source_url })) => { - let hash = bytes.hash().await?; - let cursor = Cursor::new(bytes); - operations::unpack_src_rock(cursor, temp_dir.path().to_path_buf()) - .await - .map_err(BuildError::UnpackSrcRock)?; - RemotePackageSourceMetadata { hash, source_url } - } - Some(RemotePackageSourceSpec::RockSpec(source_url)) => { - operations::FetchSrc::new(temp_dir.path(), rockspec, build.config) - .maybe_source_url(source_url) - .fetch_internal() - .await? - } - None => { - operations::FetchSrc::new(temp_dir.path(), rockspec, build.config) - .fetch_internal() - .await? +// TODO(vhyrro): break apart deployment into a separate step (once we implement transactions) +#[allow(clippy::too_many_arguments)] +pub(crate) async fn deploy( + rockspec: &R, + tree: &T, + package: LockedPackage, + lua: &LuaInstallation, + source_root: &Path, + entry_type: EntryType, + config: &Config, + behaviour: BuildBehaviour, +) -> Result { + if matches!(behaviour, BuildBehaviour::Ignore | BuildBehaviour::Conflict) { + if let Some(existing) = tree.lockfile()?.get(&package.id()) { + return Ok(existing.clone()); } - }; - - let hashes = LocalPackageHashes { - rockspec: rockspec.hash().await?, - source: source_metadata.hash.clone(), - }; - - let mut package = LocalPackage::from( - &PackageSpec::new(rockspec.package().clone(), rockspec.version().clone()), - build.constraint, - rockspec.binaries(), - build - .source - .map(Result::Ok) - .unwrap_or_else(|| { - rockspec - .to_lua_remote_rockspec_string() - .map(RemotePackageSource::RockspecContent) - }) - .unwrap_or(RemotePackageSource::Local), - Some(source_metadata.source_url.clone()), - hashes, - ); - package.spec.pinned = build.pin; - package.spec.opt = build.opt; - - match tree.lockfile()?.get(&package.id()) { - Some(package) if build.behaviour == BuildBehaviour::NoForce => Ok(package.clone()), - _ => { - tree.prepare(&package)?; - let layout = tree.layout_for(&package); - - let rock_source = rockspec.source().current_platform(); - let build_dir = resolve_source_dir( - temp_dir.path(), - rock_source.unpack_dir.as_deref(), - &rockspec.build().current_platform().copy_directories, - )?; - - Patch::new(&build_dir, &rockspec.build().current_platform().patches).apply()?; - - let external_dependencies = rockspec - .external_dependencies() - .current_platform() - .iter() - .map(|(name, dep)| { - ExternalDependencyInfo::probe(name, dep, build.config.external_deps()) - .map(|info| (name.clone(), info)) - }) - .try_collect::<_, HashMap<_, _>, _>()?; - - let output = run_build( - rockspec, - RunBuildArgs::new() - .package(&package) - .no_install(false) - .lua(lua) - .external_dependencies(&external_dependencies) - .deploy(rockspec.deploy().current_platform()) - .config(build.config) - .tree(tree) - .build_dir(&build_dir) - .build(), - ) - .await?; - - package.spec.binaries.extend(output.binaries); + } - install( - rockspec, - tree, - &package, - lua, - &build_dir, - &build.entry_type, - build.config, - ) - .await?; - - for directory in rockspec - .build() - .current_platform() - .copy_directories - .iter() - .filter(|dir| { - dir.file_name() - .is_some_and(|name| name != "doc" && name != "docs") - }) - { - recursive_copy_dir(&build_dir.join(directory), &layout.etc.join(directory)).await?; - } + // TODO(vhyrro): When we implement transactionality make this a Drop guard + // or something else. + tree.prepare(&package.spec)?; + let layout = tree.layout_for(&package.spec); + + let rock_source = rockspec.source().current_platform(); + let build_dir = resolve_source_dir( + source_root, + rock_source.unpack_dir.as_deref(), + &rockspec.build().current_platform().copy_directories, + )?; + + Patch::new(&build_dir, &rockspec.build().current_platform().patches).apply()?; + + let external_dependencies = rockspec + .external_dependencies() + .current_platform() + .iter() + .map(|(name, dep)| { + ExternalDependencyInfo::probe(name, dep, config.external_deps()) + .map(|info| (name.clone(), info)) + }) + .try_collect::<_, HashMap<_, _>, _>()?; + + let output = run_build( + rockspec, + RunBuildArgs::new() + .package(&package.spec) + .no_install(false) + .lua(lua) + .external_dependencies(&external_dependencies) + .deploy(rockspec.deploy().current_platform()) + .config(config) + .tree(tree) + .build_dir(&build_dir) + .build(), + ) + .await?; + + let mut binaries = rockspec.binaries(); + binaries.extend(output.binaries); + tree.lockfile()? + .write_guard() + .set_binaries(&package, binaries); + + install( + rockspec, + tree, + &package.spec, + lua, + &build_dir, + &entry_type, + config, + ) + .await?; + + for directory in rockspec + .build() + .current_platform() + .copy_directories + .iter() + .filter(|dir| { + dir.file_name() + .is_some_and(|name| name != "doc" && name != "docs") + }) + { + recursive_copy_dir(&build_dir.join(directory), &layout.etc.join(directory)).await?; + } - recursive_copy_doc_dir(&layout.doc, &build_dir).await?; + recursive_copy_doc_dir(&layout.doc, &build_dir).await?; - if let Ok(rockspec_str) = rockspec.to_lua_remote_rockspec_string() { - fs::sync::write(layout.rockspec_path(), rockspec_str)?; - } + if let Ok(rockspec_str) = rockspec.to_lua_remote_rockspec_string() { + fs::sync::write(layout.rockspec_path(), rockspec_str)?; + } - tree.finalize(&package, build.entry_type)?; + tree.finalize(&package, entry_type)?; - Ok(package) - } - } + Ok(package) } fn is_source_or_etc_dir(dir: &DirEntry, copy_dirs: &[PathBuf]) -> bool { @@ -538,10 +451,18 @@ mod tests { use crate::{ config::ConfigBuilder, + lockfile::{LockConstraint, LockedPackageHashes}, lua_installation::{detect_installed_lua_version, LuaInstallation}, lua_version::LuaVersion, + package::PackageSpec, project::Project, - rockspec::RockBinaries, + remote_package_source::RemotePackageSource, + }; + #[cfg(feature = "impure_tests")] + use crate::{ + operations::{unpack_rockspec, DownloadedPackedRockBytes}, + pipeline::build_local::Build, + tree::Tree, }; #[tokio::test] @@ -565,13 +486,12 @@ mod tests { let lua = LuaInstallation::new(lua_version, &config).await.unwrap(); let project = Project::from_exact(&project_root).unwrap().unwrap(); let rockspec = project.toml().into_remote(None).unwrap(); - let package = LocalPackage::from( + let package = LockedPackage::from( &PackageSpec::new(rockspec.package().clone(), rockspec.version().clone()), LockConstraint::Unconstrained, - RockBinaries::default(), RemotePackageSource::Test, None, - LocalPackageHashes { + LockedPackageHashes { rockspec: "sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=" .parse() .unwrap(), @@ -580,12 +500,12 @@ mod tests { .unwrap(), }, ); - tree.prepare(&package).unwrap(); - let src_dir = tree.layout_for(&package).src; + tree.prepare(&package.spec).unwrap(); + let src_dir = tree.layout_for(&package.spec).src; run_build( &rockspec, RunBuildArgs::new() - .package(&package) + .package(&package.spec) .no_install(false) .lua(&lua) .external_dependencies(&HashMap::default()) @@ -624,4 +544,159 @@ mod tests { bin_file.assert(predicate::str::contains("#!/usr/bin/env bash")); bin_file.assert(predicate::str::contains("echo \"Hello\"")); } + + #[cfg(feature = "impure_tests")] + const LUATEST_SRC_ROCK: &str = "resources/test/luatest-0.2-1.src.rock"; + #[cfg(feature = "impure_tests")] + const LUATEST_SRC_ROCK_SOURCE_HASH: &str = + "sha256-2jS0XOq0iIVhsZJ3BVqXSlKsx2vsqCAaaYyqcBEb7RI="; + #[cfg(feature = "impure_tests")] + const LUATEST_ROCKSPEC_HASH: &str = "sha256-NljJ20A+VadUyhhBjrRnojeQjSqRpQy7FWcgjUt2Fdc="; + + #[cfg(feature = "impure_tests")] + fn luatest_src_rock_bytes() -> Bytes { + Bytes::copy_from_slice( + &std::fs::read(PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(LUATEST_SRC_ROCK)) + .unwrap(), + ) + } + + #[cfg(feature = "impure_tests")] + fn luatest_config(dir: &assert_fs::TempDir) -> Config { + ConfigBuilder::new() + .unwrap() + .user_tree(Some(dir.to_path_buf())) + .lua_version(Some(LuaVersion::Lua51)) + .build() + .unwrap() + } + + #[cfg(feature = "impure_tests")] + async fn build_luatest( + config: &Config, + tree: &Tree, + entry_type: EntryType, + behaviour: BuildBehaviour, + ) -> LockedPackage { + let bytes = luatest_src_rock_bytes(); + let rock = DownloadedPackedRockBytes { + name: "luatest".into(), + version: "0.2-1".parse().unwrap(), + bytes: bytes.clone(), + file_name: "luatest-0.2-1.src.rock".into(), + url: "https://example.org/luatest-0.2-1.src.rock" + .parse() + .unwrap(), + }; + let rockspec = unpack_rockspec(&rock).await.unwrap(); + let lua = LuaInstallation::new_from_config(config).await.unwrap(); + Build::new() + .rockspec(&rockspec) + .lua(&lua) + .tree(tree) + .entry_type(entry_type) + .config(config) + .behaviour(behaviour) + .source_spec(RemotePackageSourceSpec::SrcRock(SrcRockSource { + bytes, + source_url: RemotePackageSourceUrl::Url { + url: "https://example.org/luatest-0.2-1.src.rock" + .parse() + .unwrap(), + }, + })) + .build() + .await + .unwrap() + } + + #[cfg(feature = "impure_tests")] + #[tokio::test] + async fn luatest_src_rock_produces_expected_hashes() { + let dir = assert_fs::TempDir::new().unwrap(); + let config = luatest_config(&dir); + let tree = config.user_tree(LuaVersion::Lua51).unwrap(); + + let package = + build_luatest(&config, &tree, EntryType::Entrypoint, BuildBehaviour::Force).await; + + assert_eq!( + package.hashes().source, + LUATEST_SRC_ROCK_SOURCE_HASH.parse().unwrap() + ); + assert_eq!( + package.hashes().rockspec, + LUATEST_ROCKSPEC_HASH.parse().unwrap() + ); + assert_eq!( + package.hashes().source, + luatest_src_rock_bytes().hash().await.unwrap() + ); + assert!(tree.bin().join("luatest").is_file()); + } + + #[cfg(feature = "impure_tests")] + #[tokio::test] + async fn force_build_is_deterministic() { + let dir1 = assert_fs::TempDir::new().unwrap(); + let config1 = luatest_config(&dir1); + let tree1 = config1.user_tree(LuaVersion::Lua51).unwrap(); + + let dir2 = assert_fs::TempDir::new().unwrap(); + let config2 = luatest_config(&dir2); + let tree2 = config2.user_tree(LuaVersion::Lua51).unwrap(); + + let package1 = build_luatest( + &config1, + &tree1, + EntryType::Entrypoint, + BuildBehaviour::Force, + ) + .await; + let package2 = build_luatest( + &config2, + &tree2, + EntryType::Entrypoint, + BuildBehaviour::Force, + ) + .await; + + assert_eq!(package1, package2); + assert_eq!(package1.hashes(), package2.hashes()); + } + + #[cfg(feature = "impure_tests")] + #[tokio::test] + async fn noforce_skips_rebuild_when_package_is_locked() { + let dir = assert_fs::TempDir::new().unwrap(); + let config = luatest_config(&dir); + let tree = config.user_tree(LuaVersion::Lua51).unwrap(); + + let package = + build_luatest(&config, &tree, EntryType::Entrypoint, BuildBehaviour::Force).await; + let bin = tree.bin().join("luatest"); + assert!(bin.is_file()); + + // Simulate a committed install + { + let mut lockfile = tree.lockfile().unwrap().write_guard(); + lockfile.add_entrypoint(&package); + } + + // Simulate a partially broken install. + std::fs::remove_file(&bin).unwrap(); + + let skipped = build_luatest( + &config, + &tree, + EntryType::Entrypoint, + BuildBehaviour::Ignore, + ) + .await; + assert_eq!(skipped.id(), package.id()); + assert!(!bin.is_file()); + + build_luatest(&config, &tree, EntryType::Entrypoint, BuildBehaviour::Force).await; + assert!(bin.is_file()); + } } diff --git a/lux-lib/src/config/mod.rs b/lux-lib/src/config/mod.rs index ba2904607..81e7b4aa4 100644 --- a/lux-lib/src/config/mod.rs +++ b/lux-lib/src/config/mod.rs @@ -821,7 +821,9 @@ impl ConfigBuilder { no_progress: self.no_progress.unwrap_or(false), no_prompt: self.no_prompt.unwrap_or(false), timeout: self.timeout.unwrap_or_else(|| Duration::from_secs(30)), - max_jobs: match self.max_jobs.unwrap_or(usize::MAX) { + max_jobs: match self.max_jobs.unwrap_or_else(|| { + std::thread::available_parallelism().map_or(1, |available| available.get()) + }) { 0 => usize::MAX, max_jobs => max_jobs, }, diff --git a/lux-lib/src/drivers/install_packages.rs b/lux-lib/src/drivers/install_packages.rs new file mode 100644 index 000000000..86640ad19 --- /dev/null +++ b/lux-lib/src/drivers/install_packages.rs @@ -0,0 +1,394 @@ +use std::{collections::HashMap, io}; + +use bon::Builder; +use itertools::Itertools; +use miette::Diagnostic; +use thiserror::Error; +use tracing::Instrument; + +use crate::{ + build::BuildBehaviour, + config::Config, + lockfile::{ + FlushLockfileError, LockedPackage, LockedPackageId, LockedPackageLockType, Lockfile, + ReadWrite, + }, + lua_installation::LuaInstallationError, + luarocks::luarocks_installation::{LuaRocksError, LuaRocksInstallError}, + package::PackageNameList, + package_db::{PackageDB, PackageDBError}, + tree::{self, InstallTree, TreeError}, + workspace::WorkspaceTreeError, +}; + +use crate::pipeline::{ + build::{Build as PipelineBuild, BuildError as PipelineBuildError}, + download_sources_and_hash::{DownloadSourcesAndHash, DownloadSourcesAndHashError}, + emit_lockfile::LockfileHandle, + resolve::{ResolveError, ResolvePackageDependencies}, +}; + +use crate::operations::PackageInstallSpec; + +/// Installs a set of [`PackageInstallSpec`]s into a tree. +/// +/// Returns a [`LockfileHandle`] so changes can be committed into +/// the correct lockfile. +#[derive(Builder)] +#[builder(start_fn = new, finish_fn(name = _build, vis = ""))] +pub struct InstallPackages<'a, T> +where + T: InstallTree + Send + Sync, +{ + #[builder(start_fn)] + config: &'a Config, + #[builder(start_fn)] + tree: &'a T, + #[builder(field)] + build_packages: Vec, + #[builder(field)] + test_packages: Option>, + #[builder(field)] + packages: Vec, + package_db: PackageDB, +} + +impl<'a, T, State> InstallPackagesBuilder<'a, T, State> +where + T: InstallTree + Send + Sync, + State: install_packages_builder::State, +{ + pub fn packages(mut self, packages: Vec) -> Self { + self.packages = packages; + self + } + + pub fn package(mut self, package: PackageInstallSpec) -> Self { + self.packages.push(package); + self + } + + pub fn build_packages(mut self, packages: Vec) -> Self { + self.build_packages = packages; + self + } + + pub fn test_packages(mut self, packages: Vec) -> Self { + self.test_packages = Some(packages); + self + } +} + +impl InstallPackagesBuilder<'_, T, State> +where + State: install_packages_builder::State + install_packages_builder::IsComplete, + T: InstallTree + Send + Sync, +{ + pub async fn install( + self, + ) -> Result<(Vec, LockfileHandle), InstallPackagesError> { + let args = self._build(); + if args.packages.is_empty() + && args.build_packages.is_empty() + && args.test_packages.is_none() + { + return Ok((Vec::new(), LockfileHandle::default())); + } + let span = match args.packages.as_slice() { + [install_spec] => { + tracing::info_span!("Installing", package = install_spec.package.to_string()) + } + packages => tracing::info_span!("Installing", count = packages.len()), + }; + install_packages(args).instrument(span).await + } +} + +async fn install_packages( + install: InstallPackages<'_, T>, +) -> Result<(Vec, LockfileHandle), InstallPackagesError> +where + T: InstallTree + Send + Sync, +{ + let package_db = install.package_db; + + let duplicate_entrypoints = install + .packages + .iter() + .filter(|pkg| pkg.entry_type == tree::EntryType::Entrypoint) + .map(|pkg| pkg.package.name()) + .duplicates() + .cloned() + .collect_vec(); + + if !duplicate_entrypoints.is_empty() { + return Err(InstallPackagesError::DuplicateEntrypoints( + PackageNameList::new(duplicate_entrypoints), + )); + } + + let packages = install.packages; + let config = install.config; + let tree = install.tree; + + let lockfile = tree.lockfile()?; + let build_tree = tree.build_tree(config)?; + let build_lockfile = build_tree.lockfile()?; + + let mut installed: HashMap> = HashMap::new(); + installed.insert( + LockedPackageLockType::Regular, + lockfile.rocks().values().cloned().collect_vec(), + ); + installed.insert( + LockedPackageLockType::Build, + build_lockfile.rocks().values().cloned().collect_vec(), + ); + if install.test_packages.is_some() { + installed.insert( + LockedPackageLockType::Test, + tree.test_tree(config)? + .lockfile()? + .rocks() + .values() + .cloned() + .collect_vec(), + ); + } + + let mut resolve = ResolvePackageDependencies::new(package_db, config) + .installed(installed) + .packages(packages) + .build_packages(install.build_packages); + if let Some(test_packages) = install.test_packages { + resolve = resolve.test_packages(test_packages); + } + let resolved = resolve.resolve().await?; + let artifacts = DownloadSourcesAndHash::new(config) + .resolved(resolved) + .download_sources_and_hash() + .await?; + let handle = LockfileHandle::from_artifacts(&artifacts); + + let build_packages = artifacts + .build + .unwrap_or_default() + .into_values() + .collect_vec(); + let regular_packages = artifacts + .regular + .unwrap_or_default() + .into_values() + .collect_vec(); + + // Entrypoints that are already installed. The requested build behaviour decides how to + // react to them. + let mut conflicting_entrypoints: HashMap = HashMap::new(); + let mut unforced_conflicts = Vec::new(); + for package in regular_packages + .iter() + .filter(|package| package.entry_type == tree::EntryType::Entrypoint) + { + let Some(existing) = lockfile.entrypoint(package.package.name()) else { + continue; + }; + match package.build_behaviour { + // Forced installs replace the existing entrypoint. + BuildBehaviour::Force => { + conflicting_entrypoints.insert(existing.id(), existing.clone()); + } + // `Ignore` tolerates re-installing the exact same version. + BuildBehaviour::Ignore if existing.version() == package.package.version() => {} + // Any other existing entrypoint is a conflict. + BuildBehaviour::Ignore | BuildBehaviour::Conflict => { + unforced_conflicts.push(format!("{}@{}", existing.name(), existing.version())); + } + } + } + + if !unforced_conflicts.is_empty() { + return Err(InstallPackagesError::ConflictingEntrypoints( + unforced_conflicts.join("\n"), + )); + } + + // FIXME(vhyrro): non-transactional. If an error occurs this removes the conflicting package + // without installing the substitute. Implement transactions at some point. + let conflicting_entrypoints = conflicting_entrypoints.into_values().collect_vec(); + for package in &conflicting_entrypoints { + tree.cleanup(package, tree::EntryType::Entrypoint)?; + } + + let regular_entry_types: HashMap = regular_packages + .iter() + .map(|pkg| (pkg.package.spec.id(), pkg.entry_type)) + .collect(); + + // Build dependencies first, then the packages themselves. + let built_build_deps = PipelineBuild::new(config, &build_tree) + .packages(build_packages) + .build() + .await?; + let built = PipelineBuild::new(config, tree) + .packages(regular_packages) + .build() + .await?; + if let Some(test_packages) = artifacts.test { + let test_tree = tree.test_tree(config)?; + PipelineBuild::new(config, &test_tree) + .packages(test_packages.into_values().collect_vec()) + .build() + .await?; + } + + let mut installed_packages: HashMap = lockfile + .rocks() + .iter() + .map(|(id, package)| (id.clone(), package.clone())) + .collect(); + installed_packages.extend(built.iter().map(|pkg| (pkg.spec().id(), pkg.clone()))); + let mut installed_build_deps: HashMap = build_lockfile + .rocks() + .iter() + .map(|(id, package)| (id.clone(), package.clone())) + .collect(); + installed_build_deps.extend( + built_build_deps + .iter() + .map(|pkg| (pkg.spec().id(), pkg.clone())), + ); + + lockfile.map_then_flush(|lockfile| { + for package in &conflicting_entrypoints { + lockfile.remove_by_id(&package.id()); + } + for package in &built { + let entry_type = regular_entry_types[&package.spec().id()]; + lockfile.add_dependencies(package, entry_type, &installed_packages)?; + lockfile.add_build_dependencies(package, &installed_build_deps)?; + } + Ok::<_, io::Error>(()) + })?; + + Ok((built, handle)) +} + +#[derive(Error, Debug, Diagnostic)] +pub enum InstallPackagesError { + #[error(transparent)] + #[diagnostic(transparent)] + LuaVersionUnset(#[from] crate::lua_version::LuaVersionUnset), + #[error(transparent)] + #[diagnostic(transparent)] + LuaInstallation(#[from] LuaInstallationError), + #[error(transparent)] + #[diagnostic(transparent)] + FlushLockfile(#[from] FlushLockfileError), + #[error(transparent)] + #[diagnostic(transparent)] + Tree(#[from] TreeError), + #[error(transparent)] + #[diagnostic(transparent)] + WorkspaceTree(#[from] WorkspaceTreeError), + #[error("error instantiating LuaRocks compatibility layer")] + #[diagnostic(forward(0))] + LuaRocks(#[from] LuaRocksError), + #[error("error installing LuaRocks compatibility layer")] + #[diagnostic(forward(0))] + LuaRocksInstall(#[from] Box), + #[error("error initialising remote package DB")] + #[diagnostic(forward(0))] + PackageDB(#[from] PackageDBError), + #[error("cannot install duplicate entrypoints:\n{0}")] + DuplicateEntrypoints(PackageNameList), + #[error("cannot install conflicting entrypoints:\n{0}")] + #[diagnostic(help( + "only a single entrypoint per package is allowed.\nretry with `--force` to overwrite the existing entrypoint" + ))] + ConflictingEntrypoints(String), + #[error(transparent)] + #[diagnostic(transparent)] + Resolve(#[from] ResolveError), + #[error(transparent)] + #[diagnostic(transparent)] + Download(#[from] DownloadSourcesAndHashError), + #[error(transparent)] + #[diagnostic(transparent)] + Build(#[from] PipelineBuildError), +} + +impl From for InstallPackagesError { + fn from(source: LuaRocksInstallError) -> Self { + Self::LuaRocksInstall(Box::new(source)) + } +} + +trait LockfileExt { + fn add_dependencies( + self, + pkg: &LockedPackage, + entry_type: tree::EntryType, + installed_packages: &HashMap, + ) -> io::Result<()>; + + fn add_build_dependencies( + self, + pkg: &LockedPackage, + build_dependencies: &HashMap, + ) -> io::Result<()>; +} + +impl LockfileExt for &mut Lockfile { + fn add_dependencies( + self, + pkg: &LockedPackage, + entry_type: tree::EntryType, + installed_packages: &HashMap, + ) -> io::Result<()> { + if entry_type == tree::EntryType::Entrypoint { + self.add_entrypoint(pkg); + } + + for dependency_id in pkg.spec().dependencies() { + self.add_dependency( + pkg, + installed_packages + .get(dependency_id) + .ok_or(io::Error::other( + r#" +error writing dependencies to the lockfile. +A required dependency was not installed correctly. +This is likely a bug in Lux. + +[THIS IS A BUG!] +"#, + ))?, + ); + } + Ok(()) + } + + fn add_build_dependencies( + self, + pkg: &LockedPackage, + build_dependencies: &HashMap, + ) -> io::Result<()> { + for dependency_id in pkg.spec().build_dependencies() { + self.add_build_dependency( + pkg, + build_dependencies + .get(dependency_id) + .ok_or(io::Error::other( + r#" +error writing build dependencies to the lockfile. +A required build dependency was not installed correctly. +This is likely a bug in Lux. + +[THIS IS A BUG!] +"#, + ))?, + ); + } + Ok(()) + } +} diff --git a/lux-lib/src/drivers/mod.rs b/lux-lib/src/drivers/mod.rs new file mode 100644 index 000000000..8b0a230b1 --- /dev/null +++ b/lux-lib/src/drivers/mod.rs @@ -0,0 +1,2 @@ +pub mod install_packages; +pub mod sync; diff --git a/lux-lib/src/operations/sync.rs b/lux-lib/src/drivers/sync/frozen.rs similarity index 58% rename from lux-lib/src/operations/sync.rs rename to lux-lib/src/drivers/sync/frozen.rs index bfc89c9d4..f177adbee 100644 --- a/lux-lib/src/operations/sync.rs +++ b/lux-lib/src/drivers/sync/frozen.rs @@ -1,19 +1,25 @@ use std::io; -use super::{Install, InstallError, PackageInstallSpec, RemoveError, Uninstall}; +use crate::operations::{PackageInstallSpec, RemoveError, Uninstall}; + +pub use crate::drivers::sync::SyncReport; + +use crate::drivers::install_packages::{InstallPackages, InstallPackagesError}; use crate::{ build::BuildBehaviour, config::Config, fs, lockfile::{ - FlushLockfileError, LocalPackage, LocalPackageLockType, LockfileIntegrityError, - SyncStrategy, + FlushLockfileError, LockedPackage, LockedPackageLockType, Lockfile, LockfileIntegrityError, + PackageSyncSpec, ReadOnly, ReadWrite, SyncStrategy, WorkspaceLockfile, }, luarocks::luarocks_installation::LUAROCKS_VERSION, operations::{self, GenLuaRcError}, package::{PackageName, PackageReq}, + package_db::{PackageDB, PackageDBError}, + pipeline::resolve::luarocks_build_backend_name, project::{project_toml::LocalProjectTomlValidationError, ProjectError}, - rockspec::Rockspec, + rockspec::{lua_dependency::LuaDependencySpec, Rockspec}, tree::{self, InstallTree, TreeError}, workspace::{Workspace, WorkspaceError, WorkspaceTreeError}, }; @@ -45,6 +51,7 @@ pub struct Sync<'a> { fast: Option, } +#[cfg(all(test, feature = "impure_tests"))] impl SyncBuilder<'_, State> where State: sync_builder::State, @@ -62,11 +69,11 @@ where pub async fn sync(self) -> Result { let mut args = self._build(); let test_report = if args.test.unwrap_or(false) { - Some(do_sync(&args, &LocalPackageLockType::Test).await?) + Some(do_sync(&args, &LockedPackageLockType::Test).await?) } else { None }; - let mut report = do_sync(&args, &LocalPackageLockType::Regular).await?; + let mut report = do_sync(&args, &LockedPackageLockType::Regular).await?; // `do_sync` removes dependencies that aren't listed in the workspace lockfile // or in `args.extra_packages`. @@ -91,11 +98,11 @@ where .flat_map(|rock| rock.build_dependencies()) .filter_map(|dep_id| test_build_lockfile.get(dep_id)), ) - .map(|pkg| pkg.spec.as_package_req()) + .map(|pkg| pkg.spec().as_package_req()) .collect_vec(); args.extra_packages.extend(transitive_build_dependencies); - let build_report = do_sync(&args, &LocalPackageLockType::Build).await?; + let build_report = do_sync(&args, &LockedPackageLockType::Build).await?; operations::GenLuaRc::new() .config(args.config) @@ -112,26 +119,6 @@ where } } -#[derive(Debug)] -pub struct SyncReport { - pub(crate) added: Vec, - pub(crate) removed: Vec, -} - -impl SyncReport { - pub fn added(&self) -> &[LocalPackage] { - &self.added - } - pub fn removed(&self) -> &[LocalPackage] { - &self.removed - } - - fn merge(&mut self, other: SyncReport) { - self.added.extend(other.added); - self.removed.extend(other.removed); - } -} - #[derive(Error, Debug, Diagnostic)] pub enum SyncError { #[error(transparent)] @@ -145,7 +132,7 @@ pub enum SyncError { Tree(#[from] TreeError), #[error(transparent)] #[diagnostic(transparent)] - Install(#[from] Box), + Install(#[from] Box), #[error(transparent)] #[diagnostic(transparent)] Remove(#[from] RemoveError), @@ -170,104 +157,43 @@ pub enum SyncError { #[error("failed to generate `.luarc.json`")] #[diagnostic(forward(0))] GenLuaRc(#[from] GenLuaRcError), + #[error(transparent)] + #[diagnostic(transparent)] + PackageDB(Box), } -impl From for SyncError { - fn from(source: InstallError) -> Self { +impl From for SyncError { + fn from(source: InstallPackagesError) -> Self { Self::Install(Box::new(source)) } } +impl From for SyncError { + fn from(source: PackageDBError) -> Self { + Self::PackageDB(Box::new(source)) + } +} + #[tracing::instrument(name = "Syncing dependencies", skip_all)] async fn do_sync( args: &Sync<'_>, - lock_type: &LocalPackageLockType, + lock_type: &LockedPackageLockType, ) -> Result { // NOTE(vhyrro): tools like cc and pkg-config leak cargo:rerun-if-env-changed // stdout calls, therefore gag all standard output during sync. let _stdout_gag = gag::Gag::stdout(); let tree = match lock_type { - LocalPackageLockType::Regular => args.workspace.tree(args.config)?, - LocalPackageLockType::Test => args.workspace.test_tree(args.config)?, - LocalPackageLockType::Build => args.workspace.build_tree(args.config)?, + LockedPackageLockType::Regular => args.workspace.tree(args.config)?, + LockedPackageLockType::Test => args.workspace.test_tree(args.config)?, + LockedPackageLockType::Build => args.workspace.build_tree(args.config)?, }; fs::sync::create_dir_all(tree.root())?; let mut workspace_lockfile = args.workspace.lockfile()?.write_guard(); let dest_lockfile = tree.lockfile()?; - let mut packages = Vec::new(); - for project in args.workspace.members() { - match lock_type { - LocalPackageLockType::Regular => packages.extend( - project - .toml() - .into_local()? - .dependencies() - .current_platform() - .clone(), - ), - LocalPackageLockType::Build => packages.extend( - project - .toml() - .into_local()? - .build_dependencies() - .current_platform() - .clone(), - ), - LocalPackageLockType::Test => packages.extend( - project - .toml() - .into_local()? - .test_dependencies() - .current_platform() - .clone(), - ), - } - } - - let mut extra_packages = args.extra_packages.iter().cloned().collect_vec(); - if lock_type == &LocalPackageLockType::Build { - for project in args.workspace.members() { - let toml = project.toml().into_local()?; - if let Some(backend) = operations::resolve::luarocks_build_backend_name(&toml) { - extra_packages.push(backend.into()); - if cfg!(target_family = "unix") { - let luarocks = unsafe { - PackageReq::new_unchecked("luarocks".into(), Some(LUAROCKS_VERSION.into())) - }; - extra_packages.push(luarocks); - } - } - } - } else if lock_type == &LocalPackageLockType::Test { - for project in args.workspace.members() { - let toml = project.toml().into_local()?; - for test_dep in toml - .test() - .current_platform() - .test_dependencies(project) - .iter() - .filter(|test_dep| { - !toml - .test_dependencies() - .current_platform() - .iter() - .any(|dep| dep.name() == test_dep.name()) - }) - .cloned() - { - extra_packages.push(test_dep); - } - } - } - - let packages = packages - .into_iter() - .chain(extra_packages.into_iter().unique().map_into()) - .collect_vec(); - + let packages = SyncPackages::new(args, lock_type).gather()?; let strategy = if args.fast.unwrap_or(false) { SyncStrategy::LockfileOnly } else { @@ -280,12 +206,163 @@ async fn do_sync( .iter() .for_each(|pkg| workspace_lockfile.remove(pkg, lock_type)); - let mut to_add: Vec<(tree::EntryType, LocalPackage)> = Vec::new(); + let package_db = workspace_lockfile.local_pkg_locks().into(); + + let (to_add, mut report) = reconcile_locks(&workspace_lockfile, &dest_lockfile, lock_type); + // Workspace members are the workspace's own entrypoints, not dependencies, + // so they must never be reported (or uninstalled) as removed by a sync. + let member_names = args.workspace.member_names()?; + report + .removed + .retain(|package| !member_names.contains(package.name())); + let packages_to_install = install_specs_to_force(&to_add); + report + .added + .extend(to_add.iter().map(|(_, pkg)| pkg).cloned()); + + InstallPackages::new(args.config, &tree) + .package_db(package_db) + .packages(packages_to_install) + .install() + .await?; + + // Read the destination lockfile after installing + let install_tree_lockfile = tree.lockfile()?; + + validate_integrity(args, &to_add, &install_tree_lockfile)?; + + let packages_to_remove = report.removed.iter().map(|pkg| pkg.id()).collect_vec(); + + Uninstall::new() + .config(args.config) + .packages(packages_to_remove) + .tree(tree.clone()) + .remove() + .await?; + + let member_packages = install_tree_lockfile + .rocks() + .values() + .filter(|package| member_names.contains(package.name())) + .cloned() + .collect_vec(); + + install_tree_lockfile.map_then_flush(|lockfile| { + // `sync` replaces the lockfile's contents whole, so we synchronize the + // workspace lockfile first and only then re-add the workspace members as + // entrypoints. Adding them beforehand would have them wiped out. + lockfile.sync(workspace_lockfile.local_pkg_lock(lock_type)); + for package in &member_packages { + lockfile.add_entrypoint(package); + } + Ok::<_, io::Error>(()) + })?; + + install_missing_packages( + args, + &tree, + &mut workspace_lockfile, + lock_type, + &package_sync_spec, + &mut report, + ) + .await?; + + Ok(report) +} + +/// Collects the packages which must be synchronised for a given lockfile type: +/// workspace member dependencies plus any extra build backends or test dependencies. +struct SyncPackages<'a, 'b> { + args: &'a Sync<'b>, + lock_type: &'a LockedPackageLockType, +} + +impl<'a, 'b> SyncPackages<'a, 'b> { + fn new(args: &'a Sync<'b>, lock_type: &'a LockedPackageLockType) -> Self { + Self { args, lock_type } + } + + fn gather(self) -> Result, SyncError> { + let mut packages = Vec::new(); + for project in self.args.workspace.members() { + let toml = project.toml().into_local()?; + match self.lock_type { + LockedPackageLockType::Regular => { + packages.extend(toml.dependencies().current_platform().clone()) + } + LockedPackageLockType::Build => { + packages.extend(toml.build_dependencies().current_platform().clone()) + } + LockedPackageLockType::Test => { + packages.extend(toml.test_dependencies().current_platform().clone()) + } + } + } + + let mut extra_packages = self.args.extra_packages.clone(); + match self.lock_type { + LockedPackageLockType::Build => { + for project in self.args.workspace.members() { + let toml = project.toml().into_local()?; + if let Some(backend) = luarocks_build_backend_name(&toml) { + extra_packages.push(backend.into()); + if cfg!(target_family = "unix") { + let luarocks = unsafe { + PackageReq::new_unchecked( + "luarocks".into(), + Some(LUAROCKS_VERSION.into()), + ) + }; + extra_packages.push(luarocks); + } + } + } + } + LockedPackageLockType::Test => { + for project in self.args.workspace.members() { + let toml = project.toml().into_local()?; + for test_dep in toml + .test() + .current_platform() + .test_dependencies(project) + .iter() + .filter(|test_dep| { + !toml + .test_dependencies() + .current_platform() + .iter() + .any(|dep| dep.name() == test_dep.name()) + }) + .cloned() + { + extra_packages.push(test_dep); + } + } + } + LockedPackageLockType::Regular => {} + } + + Ok(packages + .into_iter() + .chain(extra_packages.into_iter().unique().map_into()) + .collect()) + } +} +/// Determines which packages are present in one lockfile but not the other, producing the +/// packages to add (with their entry type) and the report of packages that must be removed. +fn reconcile_locks( + workspace_lockfile: &WorkspaceLockfile, + dest_lockfile: &Lockfile, + lock_type: &LockedPackageLockType, +) -> (Vec<(tree::EntryType, LockedPackage)>, SyncReport) { + let mut to_add: Vec<(tree::EntryType, LockedPackage)> = Vec::new(); let mut report = SyncReport { added: Vec::new(), removed: Vec::new(), }; + for (id, local_package) in workspace_lockfile.rocks(lock_type) { if dest_lockfile.get(id).is_none() { let entry_type = if workspace_lockfile.is_entrypoint(&local_package.id(), lock_type) { @@ -302,7 +379,11 @@ async fn do_sync( } } - let packages_to_install = to_add + (to_add, report) +} + +fn install_specs_to_force(to_add: &[(tree::EntryType, LockedPackage)]) -> Vec { + to_add .iter() .map(|(entry_type, pkg)| { PackageInstallSpec::new(pkg.clone().into_package_req(), *entry_type) @@ -313,85 +394,80 @@ async fn do_sync( .build() }) .unique() - .collect_vec(); - report - .added - .extend(to_add.iter().map(|(_, pkg)| pkg).cloned()); - - let package_db = workspace_lockfile.local_pkg_locks().into(); - - Install::new(args.config) - .package_db(package_db) - .packages(packages_to_install) - .tree(tree.clone()) - .install() - .await?; - - // Read the destination lockfile after installing - let install_tree_lockfile = tree.lockfile()?; + .collect() +} - if args.validate_integrity.unwrap_or(true) { - for (_, package) in &to_add { - install_tree_lockfile - .validate_integrity(package) - .map_err(|source| SyncError::Integrity { - package: package.name().clone(), - source, - })?; - } +fn validate_integrity( + args: &Sync<'_>, + to_add: &[(tree::EntryType, LockedPackage)], + install_tree_lockfile: &Lockfile, +) -> Result<(), SyncError> { + if !args.validate_integrity.unwrap_or(true) { + return Ok(()); } + for (_, package) in to_add { + install_tree_lockfile + .validate_integrity(package) + .map_err(|source| SyncError::Integrity { + package: package.name().clone(), + source, + })?; + } + Ok(()) +} - let packages_to_remove = report.removed.iter().map(|pkg| pkg.id()).collect_vec(); - - Uninstall::new() - .config(args.config) - .packages(packages_to_remove) - .tree(tree.clone()) - .remove() - .await?; +/// Installs packages that were newly added to the workspace lockfile but are not yet present in +/// the install tree, resolving them from the configured package database. +async fn install_missing_packages( + args: &Sync<'_>, + tree: &T, + workspace_lockfile: &mut WorkspaceLockfile, + lock_type: &LockedPackageLockType, + package_sync_spec: &PackageSyncSpec, + report: &mut SyncReport, +) -> Result<(), SyncError> +where + T: InstallTree + Clone + Send + std::marker::Sync + 'static, +{ + if package_sync_spec.to_add.is_empty() { + return Ok(()); + } - install_tree_lockfile.map_then_flush(|lockfile| { - lockfile.sync(workspace_lockfile.local_pkg_lock(lock_type)); - Ok::<_, io::Error>(()) - })?; + let missing_packages = package_sync_spec + .to_add + .iter() + .map(|dep| { + PackageInstallSpec::new(dep.package_req().clone(), tree::EntryType::Entrypoint) + .build_behaviour(BuildBehaviour::Force) + .pin(*dep.pin()) + .opt(*dep.opt()) + .maybe_source(dep.source.clone()) + .build() + }) + .unique() + .collect(); - if !package_sync_spec.to_add.is_empty() { - // Install missing packages using the default package_db. - let missing_packages = package_sync_spec - .to_add - .into_iter() - .map(|dep| { - PackageInstallSpec::new(dep.package_req().clone(), tree::EntryType::Entrypoint) - .build_behaviour(BuildBehaviour::Force) - .pin(*dep.pin()) - .opt(*dep.opt()) - .maybe_source(dep.source.clone()) - .build() - }) - .unique() - .collect(); - - let added = Install::new(args.config) - .packages(missing_packages) - .tree(tree.clone()) - .install() - .await?; + let added = InstallPackages::new(args.config, tree) + .package_db(PackageDB::from_config(args.config).await?) + .packages(missing_packages) + .install() + .await? + .0; - report.added.extend(added); + report.added.extend(added); - // Sync the newly added packages back to the workspace lockfile - let dest_lockfile = tree.lockfile()?; - workspace_lockfile.sync(dest_lockfile.local_pkg_lock(), lock_type); - } + // Sync the newly added packages back to the workspace lockfile + let dest_lockfile = tree.lockfile()?; + workspace_lockfile.sync(dest_lockfile.local_pkg_lock(), lock_type); - Ok(report) + Ok(()) } #[cfg(all(test, feature = "impure_tests"))] mod tests { use super::Sync; use crate::{ - config::ConfigBuilder, lockfile::LocalPackageLockType, package::PackageReq, + config::ConfigBuilder, lockfile::LockedPackageLockType, package::PackageReq, workspace::Workspace, }; use assert_fs::{prelude::PathCopy, TempDir}; @@ -416,7 +492,7 @@ mod tests { let lockfile_after_sync = workspace.lockfile().unwrap(); assert!(!lockfile_after_sync - .rocks(&LocalPackageLockType::Regular) + .rocks(&LockedPackageLockType::Regular) .is_empty()); } @@ -448,7 +524,7 @@ mod tests { } let lockfile_after_sync = workspace.lockfile().unwrap(); assert!(!lockfile_after_sync - .rocks(&LocalPackageLockType::Regular) + .rocks(&LockedPackageLockType::Regular) .is_empty()); } @@ -481,7 +557,7 @@ mod tests { } let lockfile_after_sync = workspace.lockfile().unwrap(); assert!(!lockfile_after_sync - .rocks(&LocalPackageLockType::Regular) + .rocks(&LockedPackageLockType::Regular) .is_empty()); } @@ -509,7 +585,7 @@ mod tests { let lockfile_after_sync = workspace.lockfile().unwrap(); assert!(!lockfile_after_sync - .rocks(&LocalPackageLockType::Regular) + .rocks(&LockedPackageLockType::Regular) .is_empty()); } } diff --git a/lux-lib/src/drivers/sync/mod.rs b/lux-lib/src/drivers/sync/mod.rs new file mode 100644 index 000000000..7d4b6b297 --- /dev/null +++ b/lux-lib/src/drivers/sync/mod.rs @@ -0,0 +1,610 @@ +use bon::Builder; +use miette::Diagnostic; +use nonempty::NonEmpty; +use thiserror::Error; + +mod frozen; + +use crate::{ + build::BuildBehaviour, + config::Config, + drivers::install_packages::{InstallPackages, InstallPackagesError}, + lockfile::{LockedPackage, LockedPackageLockType}, + operations::{ + GenLuaRc, GenLuaRcError, InstallProject, InstallProjectError, PackageInstallSpec, + }, + package::PackageName, + package_db::{PackageDB, PackageDBError}, + project::project_toml::LocalProjectTomlValidationError, + rockspec::Rockspec, + tree::{EntryType, InstallTree, TreeError}, + workspace::{Workspace, WorkspaceError, WorkspaceTreeError}, +}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SyncMode { + /// The lockfile may be rewritten. Missing dependencies are resolved from + /// manifests, preferring locked versions. + Open, + /// The lockfile is authoritative. Nothing is resolved from manifests. + Frozen, +} + +#[derive(Clone, Debug)] +pub struct TargetSet { + pub test: bool, + pub members: NonEmpty, +} + +impl TargetSet { + pub fn all(workspace: &Workspace, test: bool) -> Self { + Self { + test, + members: workspace + .members() + .clone() + .map(|project| project.toml().package.clone()), + } + } + + pub fn member(member: PackageName, test: bool) -> Self { + Self { + test, + members: NonEmpty::new(member), + } + } +} + +#[derive(Debug)] +pub struct SyncReport { + pub(crate) added: Vec, + pub(crate) removed: Vec, +} + +impl SyncReport { + pub fn added(&self) -> &[LockedPackage] { + &self.added + } + pub fn removed(&self) -> &[LockedPackage] { + &self.removed + } + + pub(crate) fn merge(&mut self, other: SyncReport) { + self.added.extend(other.added); + self.removed.extend(other.removed); + } +} + +#[derive(Error, Debug, Diagnostic)] +pub enum SyncError { + #[error(transparent)] + #[diagnostic(transparent)] + InstallPackages(#[from] Box), + #[error(transparent)] + #[diagnostic(transparent)] + InstallProject(Box), + #[error(transparent)] + #[diagnostic(transparent)] + Workspace(#[from] WorkspaceError), + #[error(transparent)] + #[diagnostic(transparent)] + WorkspaceTree(#[from] WorkspaceTreeError), + #[error(transparent)] + #[diagnostic(transparent)] + Tree(#[from] TreeError), + #[error(transparent)] + #[diagnostic(transparent)] + Project(#[from] LocalProjectTomlValidationError), + #[error(transparent)] + #[diagnostic(transparent)] + PackageDB(#[from] PackageDBError), + #[error(transparent)] + #[diagnostic(transparent)] + GenLuaRc(#[from] GenLuaRcError), + #[error(transparent)] + #[diagnostic(transparent)] + Frozen(Box), +} + +impl From for SyncError { + fn from(source: InstallPackagesError) -> Self { + Self::InstallPackages(Box::new(source)) + } +} + +impl From for SyncError { + fn from(source: InstallProjectError) -> Self { + Self::InstallProject(Box::new(source)) + } +} + +impl From for SyncError { + fn from(source: frozen::SyncError) -> Self { + Self::Frozen(Box::new(source)) + } +} + +/// Reconciles a workspace's install trees with its dependencies. +#[derive(Builder)] +#[builder(start_fn = new, finish_fn(name = _build, vis = ""))] +pub struct Sync<'a> { + #[builder(start_fn)] + workspace: &'a Workspace, + #[builder(start_fn)] + config: &'a Config, + + mode: SyncMode, + + targets: TargetSet, + + #[builder(default = BuildBehaviour::Force)] + behaviour: BuildBehaviour, + + /// Build only the dependencies, not the workspace members. + #[builder(default = false)] + only_deps: bool, + + /// Ignore the project's lockfile and don't create one. + #[builder(default = false)] + no_lock: bool, + + /// Whether to validate the integrity of installed packages (`Frozen` mode). + validate_integrity: Option, +} + +impl SyncBuilder<'_, State> +where + State: sync_builder::State + sync_builder::IsComplete, +{ + pub async fn sync(self) -> Result { + let args = self._build(); + match args.mode { + SyncMode::Open => sync_open(&args).await, + SyncMode::Frozen => sync_frozen(&args).await, + } + } +} + +async fn sync_open(args: &Sync<'_>) -> Result { + let config = args.config; + let workspace = args.workspace; + + let tree = workspace.tree(config)?; + let build_tree = tree.build_tree(config)?; + + let regular = gather_dependencies( + workspace, + DependencyKind::Regular, + &workspace.member_names()?, + )? + .into_iter() + .filter(|spec| !tree.match_rocks(&spec.package).is_ok_and(|m| m.is_found())) + .map(|spec| PackageInstallSpec { + build_behaviour: BuildBehaviour::Force, + ..spec + }) + .collect::>(); + + let build = gather_dependencies(workspace, DependencyKind::Build, &workspace.member_names()?)? + .into_iter() + .filter(|spec| { + !build_tree + .match_rocks(&spec.package) + .is_ok_and(|m| m.is_found()) + }) + .map(|spec| PackageInstallSpec { + build_behaviour: BuildBehaviour::Force, + ..spec + }) + .collect::>(); + + let test_tree = tree.test_tree(config)?; + let test = if args.targets.test { + gather_dependencies(workspace, DependencyKind::Test, &args.targets.members)? + .into_iter() + .filter(|spec| { + !test_tree + .match_rocks(&spec.package) + .is_ok_and(|m| m.is_found()) + }) + .map(|spec| PackageInstallSpec { + build_behaviour: BuildBehaviour::Force, + ..spec + }) + .collect::>() + } else { + Vec::new() + }; + + let package_db = PackageDB::open(config, workspace).await?; + + let mut install = InstallPackages::new(config, &tree) + .packages(regular) + .build_packages(build) + .package_db(package_db); + if !test.is_empty() { + install = install.test_packages(test); + } + + let (mut added, lockfile) = install.install().await?; + + if !args.no_lock { + let mut workspace_lockfile = workspace.lockfile()?.write_guard(); + lockfile.commit(&mut workspace_lockfile); + if args.targets.test { + workspace_lockfile.sync( + test_tree.lockfile()?.local_pkg_lock(), + &LockedPackageLockType::Test, + ); + } + } + + if !args.only_deps { + let projects = args + .targets + .members + .iter() + .map(|name| workspace.select_member(name)) + .collect::, _>>()?; + for project in projects { + added.push( + InstallProject::new() + .project(project) + .config(config) + .tree(&tree) + .behaviour(args.behaviour) + .workspace(workspace) + .build() + .await?, + ); + } + } + + if !args.no_lock { + GenLuaRc::new() + .config(config) + .workspace(workspace) + .generate_luarc() + .await?; + } + + Ok(SyncReport { + added, + removed: Vec::new(), + }) +} + +#[derive(Clone, Copy)] +enum DependencyKind { + Regular, + Build, + Test, +} + +fn gather_dependencies( + workspace: &Workspace, + kind: DependencyKind, + members: &NonEmpty, +) -> Result, SyncError> { + let mut packages = Vec::new(); + for project in workspace.members() { + if !members.contains(project.toml().package()) { + continue; + } + let toml = project.toml().into_local()?; + if let DependencyKind::Test = kind { + let explicit = toml.test_dependencies().current_platform(); + for dependency in toml.test().current_platform().test_dependencies(project) { + if explicit.iter().any(|dep| dep.name() == dependency.name()) { + continue; + } + packages.push( + PackageInstallSpec::new(dependency, EntryType::Entrypoint) + .build_behaviour(BuildBehaviour::Ignore) + .build(), + ); + } + for dependency in explicit { + if dependency.name().eq(&PackageName::new("lua".into())) { + continue; + } + packages.push( + PackageInstallSpec::new( + dependency.package_req().clone(), + EntryType::Entrypoint, + ) + .build_behaviour(BuildBehaviour::Ignore) + .pin(*dependency.pin()) + .opt(*dependency.opt()) + .maybe_source(dependency.source.clone()) + .build(), + ); + } + } else { + let dependencies = match kind { + DependencyKind::Regular => toml.dependencies().current_platform().clone(), + _ => toml.build_dependencies().current_platform().clone(), + }; + for dependency in dependencies { + // From the perspective of a project, dependencies are entrypoints. + packages.push( + PackageInstallSpec::new( + dependency.package_req().clone(), + EntryType::Entrypoint, + ) + .build_behaviour(BuildBehaviour::Ignore) + .maybe_source(dependency.source().clone()) + .build(), + ); + } + } + } + Ok(packages) +} + +async fn sync_frozen(args: &Sync<'_>) -> Result { + let report = frozen::Sync::new(args.workspace, args.config) + .validate_integrity(args.validate_integrity.unwrap_or(true)) + .test(args.targets.test) + .sync() + .await?; + Ok(report) +} + +#[cfg(all(test, feature = "impure_tests"))] +mod tests { + use std::path::PathBuf; + + use assert_fs::prelude::{FileWriteStr, PathChild, PathCopy}; + + use crate::{ + config::ConfigBuilder, lockfile::LockedPackageLockType, lua_version::LuaVersion, + package::PackageName, package_db::PackageDB, tree::InstallTree, workspace::Workspace, + }; + + use super::{Sync, SyncMode, TargetSet}; + + #[tokio::test] + async fn installs_workspace_dependencies() { + let sample = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("resources/test/sample-projects/dependencies/"); + let temp = assert_fs::TempDir::new().unwrap(); + temp.copy_from(sample, &["**"]).unwrap(); + + let workspace = Workspace::from_exact(temp.path()).unwrap().unwrap(); + let config = ConfigBuilder::new() + .unwrap() + .lua_version(Some(LuaVersion::Lua51)) + .build() + .unwrap(); + + Sync::new(&workspace, &config) + .mode(SyncMode::Open) + .targets(TargetSet::all(&workspace, false)) + .only_deps(true) + .sync() + .await + .unwrap(); + + let lockfile = workspace.lockfile().unwrap(); + let rocks = lockfile.rocks(&LockedPackageLockType::Regular); + assert!(rocks + .values() + .any(|pkg| pkg.name().to_string() == "lua-cjson")); + assert!(rocks + .values() + .any(|pkg| pkg.name().to_string() == "plenary.nvim")); + } + + #[tokio::test] + async fn install_honors_locked_version() { + let temp = assert_fs::TempDir::new().unwrap(); + let toml = temp.child("lux.toml"); + toml.write_str( + r#" + package = "locktest" + version = "0.1.0" + lua = ">=5.1" + + [build] + type = "builtin" + + [dependencies] + argparse = "==0.7.1-1" + "#, + ) + .unwrap(); + + let config = ConfigBuilder::new() + .unwrap() + .lua_version(Some(LuaVersion::Lua51)) + .build() + .unwrap(); + let name = PackageName::from("argparse"); + + let workspace = Workspace::from_exact(temp.path()).unwrap().unwrap(); + Sync::new(&workspace, &config) + .mode(SyncMode::Open) + .targets(TargetSet::all(&workspace, false)) + .only_deps(true) + .sync() + .await + .unwrap(); + let tree = workspace.tree(&config).unwrap(); + assert_eq!( + tree.lockfile() + .unwrap() + .entrypoint(&name) + .unwrap() + .version() + .to_string(), + "0.7.1-1" + ); + + toml.write_str( + r#" + package = "locktest" + version = "0.1.0" + lua = ">=5.1" + + [build] + type = "builtin" + + [dependencies] + argparse = ">=0.7.1" + "#, + ) + .unwrap(); + std::fs::remove_dir_all(tree.root()).unwrap(); + + let workspace = Workspace::from_exact(temp.path()).unwrap().unwrap(); + Sync::new(&workspace, &config) + .mode(SyncMode::Open) + .targets(TargetSet::all(&workspace, false)) + .only_deps(true) + .sync() + .await + .unwrap(); + let tree = workspace.tree(&config).unwrap(); + assert_eq!( + tree.lockfile() + .unwrap() + .entrypoint(&name) + .unwrap() + .version() + .to_string(), + "0.7.1-1" + ); + } + + #[tokio::test] + async fn install_without_lock_resolves_latest() { + let temp = assert_fs::TempDir::new().unwrap(); + temp.child("lux.toml") + .write_str( + r#" + package = "locktest" + version = "0.1.0" + lua = ">=5.1" + + [build] + type = "builtin" + + [dependencies] + argparse = ">=0.7.1" + "#, + ) + .unwrap(); + + let config = ConfigBuilder::new() + .unwrap() + .lua_version(Some(LuaVersion::Lua51)) + .build() + .unwrap(); + let name = PackageName::from("argparse"); + + let workspace = Workspace::from_exact(temp.path()).unwrap().unwrap(); + Sync::new(&workspace, &config) + .mode(SyncMode::Open) + .targets(TargetSet::all(&workspace, false)) + .only_deps(true) + .sync() + .await + .unwrap(); + let tree = workspace.tree(&config).unwrap(); + let installed = tree + .lockfile() + .unwrap() + .entrypoint(&name) + .unwrap() + .version() + .clone(); + let latest = PackageDB::from_config(&config) + .await + .unwrap() + .latest_version(&name) + .unwrap(); + assert_eq!(installed, latest); + assert_ne!(installed.to_string(), "0.7.1-1"); + } + + #[tokio::test] + async fn replaces_dependency_on_version_change() { + let temp = assert_fs::TempDir::new().unwrap(); + let toml = temp.child("lux.toml"); + toml.write_str( + r#" + package = "versionchange" + version = "0.1.0" + lua = ">=5.1" + + [build] + type = "builtin" + + [dependencies] + argparse = "==0.7.1-1" + "#, + ) + .unwrap(); + + let config = ConfigBuilder::new() + .unwrap() + .lua_version(Some(LuaVersion::Lua51)) + .build() + .unwrap(); + let name = PackageName::from("argparse"); + + let workspace = Workspace::from_exact(temp.path()).unwrap().unwrap(); + Sync::new(&workspace, &config) + .mode(SyncMode::Open) + .targets(TargetSet::all(&workspace, false)) + .only_deps(true) + .sync() + .await + .unwrap(); + let tree = workspace.tree(&config).unwrap(); + assert_eq!( + tree.lockfile() + .unwrap() + .entrypoint(&name) + .unwrap() + .version() + .to_string(), + "0.7.1-1" + ); + + toml.write_str( + r#" + package = "versionchange" + version = "0.1.0" + lua = ">=5.1" + + [build] + type = "builtin" + + [dependencies] + argparse = "==0.7.2-1" + "#, + ) + .unwrap(); + + let workspace = Workspace::from_exact(temp.path()).unwrap().unwrap(); + Sync::new(&workspace, &config) + .mode(SyncMode::Open) + .targets(TargetSet::all(&workspace, false)) + .only_deps(true) + .sync() + .await + .unwrap(); + let tree = workspace.tree(&config).unwrap(); + assert_eq!( + tree.lockfile() + .unwrap() + .entrypoint(&name) + .unwrap() + .version() + .to_string(), + "0.7.2-1" + ); + } +} diff --git a/lux-lib/src/fs/sync.rs b/lux-lib/src/fs/sync.rs index b6d17db79..c14fc9e5c 100644 --- a/lux-lib/src/fs/sync.rs +++ b/lux-lib/src/fs/sync.rs @@ -23,6 +23,40 @@ pub(crate) fn write(path: impl AsRef, contents: impl AsRef<[u8]>) -> Resul }) } +/// Atomically writes `contents` to `path`. +/// +/// The contents are written to a temporary file in the destination directory +/// and then renamed into place, so readers never observe a partially-written +/// file. The temporary file is flushed to disk before the rename. +pub(crate) fn write_atomic( + path: impl AsRef, + contents: impl AsRef<[u8]>, +) -> Result<(), FsError> { + use std::io::Write; + + let path = path.as_ref(); + let dir = path + .parent() + .filter(|dir| !dir.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); + + let mut file = tempfile::NamedTempFile::new_in(dir).map_err(|source| FsError::FileCreate { + path: path.to_path_buf(), + source, + })?; + file.write_all(contents.as_ref()) + .and_then(|()| file.as_file().sync_all()) + .map_err(|source| FsError::Write { + path: path.to_path_buf(), + source, + })?; + file.persist(path).map_err(|err| FsError::Write { + path: path.to_path_buf(), + source: err.error, + })?; + Ok(()) +} + /// Wrapped [`fs::copy`]. pub(crate) fn copy(from: impl AsRef, to: impl AsRef) -> Result { let from = from.as_ref(); diff --git a/lux-lib/src/fs/tokio.rs b/lux-lib/src/fs/tokio.rs index b8da8510e..cafa1d3fd 100644 --- a/lux-lib/src/fs/tokio.rs +++ b/lux-lib/src/fs/tokio.rs @@ -60,6 +60,32 @@ pub(crate) async fn create_dir_all(path: impl AsRef) -> Result<(), FsError }) } +/// Recursively copy the contents of a directory into `dest`. +pub(crate) async fn copy_dir_all(src: &Path, dest: &Path) -> Result<(), FsError> { + Box::pin(async move { + create_dir_all(dest).await?; + let mut entries = read_dir(src).await?; + while let Some(entry) = entries + .next_entry() + .await + .map_err(|source| FsError::ReadDir { + path: src.to_path_buf(), + source, + })? + { + let entry_path = entry.path(); + let target = dest.join(entry.file_name()); + if entry_path.is_dir() { + copy_dir_all(&entry_path, &target).await?; + } else { + copy(&entry_path, &target).await?; + } + } + Ok(()) + }) + .await +} + /// Wrapped [`fs::remove_file`]. pub(crate) async fn remove_file(path: impl AsRef) -> Result<(), FsError> { let path = path.as_ref(); diff --git a/lux-lib/src/hash.rs b/lux-lib/src/hash.rs index cb471b757..c1e0e34ad 100644 --- a/lux-lib/src/hash.rs +++ b/lux-lib/src/hash.rs @@ -64,10 +64,9 @@ fn hash_file(path: &Path, integrity_opts: &mut IntegrityOpts) -> io::Result<()> Ok(()) } -/// A [`FileSystem`] that excludes VCS directories (e.g. `.git`, `.jj`) and -/// files ignored by the project's `.gitignore` from the encoded -/// NAR, as their contents are not deterministic and are not part of a -/// package's sources. +/// A [`FileSystem`] that excludes VCS directories (e.g. `.git`, `.jj`, files from `.gitignore`) and +/// lux-generated files (e.g. `.lux`, `lux.lock`, `.luarc.json`) from the NAR, as their contents are +/// not deterministic and are not part of a package's sources. struct VcsExcludingFileSystem { gitignore: Gitignore, } @@ -89,6 +88,13 @@ fn is_vcs_dir(name: &str) -> bool { matches!(name, ".git" | ".jj" | ".hg" | "_darcs" | ".svn" | ".bzr") } +fn is_lux_generated(name: &str) -> bool { + matches!( + name, + crate::workspace::LUX_DIR_NAME | "lux.lock" | ".luarc.json" | ".emmyrc.json" + ) +} + impl FileSystem for VcsExcludingFileSystem { type File = std::fs::File; @@ -99,7 +105,7 @@ impl FileSystem for VcsExcludingFileSystem { fn read_dir(&self, path: &Utf8Path) -> io::Result> { Ok(FileSystem::read_dir(&NativeFileSystem {}, path)? .into_iter() - .filter(|name| !is_vcs_dir(name)) + .filter(|name| !is_vcs_dir(name) && !is_lux_generated(name)) .filter(|name| { let full = path.join(name); let is_dir = full.as_std_path().is_dir(); @@ -252,6 +258,10 @@ mod tests { write(temp.child(".git/config").path(), "nondeterministic").unwrap(); temp.child(".jj").create_dir_all().unwrap(); write(temp.child(".jj/repo").path(), "nondeterministic").unwrap(); + temp.child(".lux").create_dir_all().unwrap(); + write(temp.child(".lux/tree").path(), "nondeterministic").unwrap(); + write(temp.child("lux.lock").path(), "nondeterministic").unwrap(); + write(temp.child(".luarc.json").path(), "nondeterministic").unwrap(); let hash_with_vcs = temp.path().to_path_buf().hash().await.unwrap(); diff --git a/lux-lib/src/lib.rs b/lux-lib/src/lib.rs index b13d41a2b..2365c5376 100644 --- a/lux-lib/src/lib.rs +++ b/lux-lib/src/lib.rs @@ -1,5 +1,6 @@ pub mod build; pub mod config; +pub mod drivers; pub mod fs; pub mod git; pub mod hash; @@ -12,10 +13,11 @@ pub mod luarocks; pub mod manifest; pub mod operations; pub mod package; +pub mod package_db; pub mod path; +pub mod pipeline; pub mod progress; pub mod project; -pub mod remote_package_db; pub mod rockspec; pub mod toolchains; pub mod tree; diff --git a/lux-lib/src/lockfile/mod.rs b/lux-lib/src/lockfile/mod.rs index f71bef614..34f5bd7ca 100644 --- a/lux-lib/src/lockfile/mod.rs +++ b/lux-lib/src/lockfile/mod.rs @@ -161,16 +161,15 @@ impl<'de> Deserialize<'de> for OptState { } #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub(crate) struct LocalPackageSpec { - pub name: PackageName, - pub version: PackageVersion, - pub pinned: PinnedState, - pub opt: OptState, - pub dependencies: Vec, - pub build_dependencies: Vec, +pub struct LockedPackageSpec { + name: PackageName, + version: PackageVersion, + pinned: PinnedState, + opt: OptState, + dependencies: Vec, + build_dependencies: Vec, // TODO: Deserialize this directly into a `LuaPackageReq` - pub constraint: Option, - pub binaries: RockBinaries, + constraint: Option, } /// ID of a local package, a hash that is comprised of: @@ -180,9 +179,9 @@ pub(crate) struct LocalPackageSpec { /// - opt state /// - lock constraint #[derive(Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize, Clone)] -pub struct LocalPackageId(String); +pub struct LockedPackageId(String); -impl LocalPackageId { +impl LockedPackageId { pub fn new( name: &PackageName, version: &PackageVersion, @@ -223,23 +222,22 @@ impl LocalPackageId { } } -impl Display for LocalPackageId { +impl Display for LockedPackageId { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { self.0.fmt(f) } } -impl LocalPackageSpec { +impl LockedPackageSpec { #[allow(clippy::too_many_arguments)] pub(crate) fn new( name: &PackageName, version: &PackageVersion, constraint: LockConstraint, - dependencies: Vec, - build_dependencies: Vec, + dependencies: Vec, + build_dependencies: Vec, pinned: &PinnedState, opt: &OptState, - binaries: RockBinaries, ) -> Self { Self { name: name.clone(), @@ -252,12 +250,11 @@ impl LocalPackageSpec { LockConstraint::Unconstrained => None, LockConstraint::Constrained(version_req) => Some(version_req.to_string()), }, - binaries, } } - pub fn id(&self) -> LocalPackageId { - LocalPackageId::new( + pub fn id(&self) -> LockedPackageId { + LockedPackageId::new( self.name(), self.version(), self.pinned, @@ -290,18 +287,14 @@ impl LocalPackageSpec { self.opt } - pub fn dependencies(&self) -> Vec<&LocalPackageId> { + pub fn dependencies(&self) -> Vec<&LockedPackageId> { self.dependencies.iter().collect() } - pub fn build_dependencies(&self) -> Vec<&LocalPackageId> { + pub fn build_dependencies(&self) -> Vec<&LockedPackageId> { self.build_dependencies.iter().collect() } - pub fn binaries(&self) -> Vec<&PathBuf> { - self.binaries.iter().collect() - } - pub fn to_package(&self) -> PackageSpec { PackageSpec::new(self.name.clone(), self.version.clone()) } @@ -362,27 +355,126 @@ impl Display for RemotePackageSourceUrl { // TODO(vhyrro): Move to `package/local.rs` -/// A locally installed rock +/// A package whose source has been downloaded, hashed and which is installed in a tree. #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct LocalPackage { - pub(crate) spec: LocalPackageSpec, +pub struct LockedPackage { + pub(crate) spec: LockedPackageSpec, pub(crate) source: RemotePackageSource, pub(crate) source_url: Option, - hashes: LocalPackageHashes, + pub(crate) hashes: LockedPackageHashes, } -impl LocalPackage { +impl LockedPackage { + pub(crate) fn repin(mut self, pinned: PinnedState) -> Self { + self.spec.pinned = pinned; + self + } + + #[cfg(test)] + pub(crate) fn from( + package: &PackageSpec, + constraint: LockConstraint, + source: RemotePackageSource, + source_url: Option, + hashes: LockedPackageHashes, + ) -> Self { + Self::new( + LockedPackageSpec::new( + package.name(), + package.version(), + constraint, + Vec::default(), + Vec::default(), + &PinnedState::Unpinned, + &OptState::Required, + ), + source, + source_url, + hashes, + ) + } + + pub(crate) fn new( + spec: LockedPackageSpec, + source: RemotePackageSource, + source_url: Option, + hashes: LockedPackageHashes, + ) -> Self { + Self { + spec, + source, + source_url, + hashes, + } + } + + pub fn spec(&self) -> &LockedPackageSpec { + &self.spec + } + pub fn into_package_spec(self) -> PackageSpec { - PackageSpec::new(self.spec.name, self.spec.version) + self.spec.to_package() } pub fn as_package_spec(&self) -> PackageSpec { - PackageSpec::new(self.spec.name.clone(), self.spec.version.clone()) + self.spec.to_package() + } + + pub fn id(&self) -> LockedPackageId { + self.spec.id() + } + + pub fn name(&self) -> &PackageName { + self.spec.name() + } + + pub fn version(&self) -> &PackageVersion { + self.spec.version() + } + + pub fn pinned(&self) -> PinnedState { + self.spec.pinned() + } + + pub fn opt(&self) -> OptState { + self.spec.opt() + } + + pub fn dependencies(&self) -> Vec<&LockedPackageId> { + self.spec.dependencies() + } + + pub fn build_dependencies(&self) -> Vec<&LockedPackageId> { + self.spec.build_dependencies() + } + + pub fn constraint(&self) -> LockConstraint { + self.spec.constraint() + } + + pub(crate) fn source(&self) -> &RemotePackageSource { + &self.source + } + + pub(crate) fn source_url(&self) -> Option<&RemotePackageSourceUrl> { + self.source_url.as_ref() + } + + pub fn hashes(&self) -> &LockedPackageHashes { + &self.hashes + } + + pub fn to_package(&self) -> PackageSpec { + self.spec.to_package() + } + + pub fn into_package_req(self) -> PackageReq { + self.spec.into_package_req() } } #[derive(Debug, Serialize, Deserialize, Clone)] -struct LocalPackageIntermediate { +struct LockedPackageIntermediate { name: PackageName, version: PackageVersion, #[serde(default, skip_serializing_if = "PinnedState::is_default")] @@ -390,26 +482,24 @@ struct LocalPackageIntermediate { #[serde(default, skip_serializing_if = "OptState::is_default")] opt: OptState, #[serde(default, skip_serializing_if = "Vec::is_empty")] - dependencies: Vec, + dependencies: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] - build_dependencies: Vec, + build_dependencies: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] constraint: Option, - #[serde(default, skip_serializing_if = "RockBinaries::is_default")] - binaries: RockBinaries, source: RemotePackageSource, #[serde(default, skip_serializing_if = "Option::is_none")] source_url: Option, - hashes: LocalPackageHashes, + hashes: LockedPackageHashes, } -impl TryFrom for LocalPackage { +impl TryFrom for LockedPackage { type Error = LockConstraintParseError; - fn try_from(value: LocalPackageIntermediate) -> Result { + fn try_from(value: LockedPackageIntermediate) -> Result { let constraint = LockConstraint::try_from(&value.constraint)?; Ok(Self { - spec: LocalPackageSpec::new( + spec: LockedPackageSpec::new( &value.name, &value.version, constraint, @@ -417,7 +507,6 @@ impl TryFrom for LocalPackage { value.build_dependencies, &value.pinned, &value.opt, - value.binaries, ), source: value.source, source_url: value.source_url, @@ -426,8 +515,8 @@ impl TryFrom for LocalPackage { } } -impl From<&LocalPackage> for LocalPackageIntermediate { - fn from(value: &LocalPackage) -> Self { +impl From<&LockedPackage> for LockedPackageIntermediate { + fn from(value: &LockedPackage) -> Self { Self { name: value.spec.name.clone(), version: value.spec.version.clone(), @@ -436,7 +525,6 @@ impl From<&LocalPackage> for LocalPackageIntermediate { dependencies: value.spec.dependencies.clone(), build_dependencies: value.spec.build_dependencies.clone(), constraint: value.spec.constraint.clone(), - binaries: value.spec.binaries.clone(), source: value.source.clone(), source_url: value.source_url.clone(), hashes: value.hashes.clone(), @@ -444,107 +532,32 @@ impl From<&LocalPackage> for LocalPackageIntermediate { } } -impl<'de> Deserialize<'de> for LocalPackage { +impl<'de> Deserialize<'de> for LockedPackage { fn deserialize(deserializer: D) -> Result where D: serde::Deserializer<'de>, { - LocalPackage::try_from(LocalPackageIntermediate::deserialize(deserializer)?) + LockedPackage::try_from(LockedPackageIntermediate::deserialize(deserializer)?) .map_err(de::Error::custom) } } -impl Serialize for LocalPackage { +impl Serialize for LockedPackage { fn serialize(&self, serializer: S) -> Result where S: serde::Serializer, { - LocalPackageIntermediate::from(self).serialize(serializer) - } -} - -impl LocalPackage { - pub(crate) fn from( - package: &PackageSpec, - constraint: LockConstraint, - binaries: RockBinaries, - source: RemotePackageSource, - source_url: Option, - hashes: LocalPackageHashes, - ) -> Self { - Self { - spec: LocalPackageSpec::new( - package.name(), - package.version(), - constraint, - Vec::default(), - Vec::default(), - &PinnedState::Unpinned, - &OptState::Required, - binaries, - ), - source, - source_url, - hashes, - } - } - - pub fn id(&self) -> LocalPackageId { - self.spec.id() - } - - pub fn name(&self) -> &PackageName { - self.spec.name() - } - - pub fn version(&self) -> &PackageVersion { - self.spec.version() - } - - pub fn pinned(&self) -> PinnedState { - self.spec.pinned() - } - - pub fn opt(&self) -> OptState { - self.spec.opt() - } - - pub(crate) fn source(&self) -> &RemotePackageSource { - &self.source - } - - pub fn dependencies(&self) -> Vec<&LocalPackageId> { - self.spec.dependencies() - } - - pub fn build_dependencies(&self) -> Vec<&LocalPackageId> { - self.spec.build_dependencies() - } - - pub fn constraint(&self) -> LockConstraint { - self.spec.constraint() - } - - pub fn hashes(&self) -> &LocalPackageHashes { - &self.hashes - } - - pub fn to_package(&self) -> PackageSpec { - self.spec.to_package() - } - - pub fn into_package_req(self) -> PackageReq { - self.spec.into_package_req() + LockedPackageIntermediate::from(self).serialize(serializer) } } #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Hash)] -pub struct LocalPackageHashes { +pub struct LockedPackageHashes { pub rockspec: Integrity, pub source: Integrity, } -impl Ord for LocalPackageHashes { +impl Ord for LockedPackageHashes { fn cmp(&self, other: &Self) -> std::cmp::Ordering { let a = (self.rockspec.to_hex().1, self.source.to_hex().1); let b = (other.rockspec.to_hex().1, other.source.to_hex().1); @@ -552,7 +565,7 @@ impl Ord for LocalPackageHashes { } } -impl PartialOrd for LocalPackageHashes { +impl PartialOrd for LockedPackageHashes { fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } @@ -638,17 +651,48 @@ impl LockfilePermissions for ReadOnly {} impl LockfilePermissions for ReadWrite {} #[derive(Clone, Debug, Serialize, Deserialize, Default)] -pub(crate) struct LocalPackageLock { +pub(crate) struct PackageLock { // NOTE: We cannot directly serialize to a `Sha256` object as they don't implement serde traits. // NOTE: We want to retain ordering of rocks when de/serializing. - rocks: BTreeMap, + rocks: BTreeMap, /// The entrypoints of the tree. Enforces one entrypoint per package name. - entrypoints: BTreeMap, + entrypoints: BTreeMap, } -impl LocalPackageLock { - fn get(&self, id: &LocalPackageId) -> Option<&LocalPackage> { +impl PackageLock { + pub(crate) fn from_packages( + packages: impl Iterator, + ) -> Self { + let mut lock = Self::default(); + + for (package, entry_type) in packages { + if matches!(entry_type, EntryType::Entrypoint) { + lock.entrypoints + .insert(package.name().clone(), package.id()); + } + lock.rocks.insert(package.id(), package); + } + + lock + } + + pub(crate) fn merge(&mut self, other: &PackageLock) { + self.entrypoints.extend( + other + .entrypoints + .iter() + .map(|(name, id)| (name.clone(), id.clone())), + ); + self.rocks.extend( + other + .rocks + .iter() + .map(|(id, package)| (id.clone(), package.clone())), + ); + } + + fn get(&self, id: &LockedPackageId) -> Option<&LockedPackage> { self.rocks.get(id) } @@ -656,27 +700,27 @@ impl LocalPackageLock { self.rocks.is_empty() } - pub(crate) fn rocks(&self) -> &BTreeMap { + pub(crate) fn rocks(&self) -> &BTreeMap { &self.rocks } - fn is_entrypoint(&self, package: &LocalPackageId) -> bool { + fn is_entrypoint(&self, package: &LockedPackageId) -> bool { self.entrypoints.values().any(|id| id == package) } /// Returns the entrypoint with the given package name, if any. - pub(crate) fn entrypoint(&self, name: &PackageName) -> Option<&LocalPackage> { + pub(crate) fn entrypoint(&self, name: &PackageName) -> Option<&LockedPackage> { self.entrypoints.get(name).and_then(|id| self.get(id)) } - fn is_dependency(&self, package: &LocalPackageId) -> bool { + fn is_dependency(&self, package: &LockedPackageId) -> bool { self.rocks .values() .flat_map(|rock| rock.dependencies()) .any(|dep_id| dep_id == package) } - fn list(&self) -> HashMap> { + fn list(&self) -> HashMap> { self.rocks() .values() .cloned() @@ -684,11 +728,11 @@ impl LocalPackageLock { .into_group_map() } - fn remove(&mut self, target: &LocalPackage) { + fn remove(&mut self, target: &LockedPackage) { self.remove_by_id(&target.id()) } - fn remove_by_id(&mut self, target: &LocalPackageId) { + fn remove_by_id(&mut self, target: &LockedPackageId) { self.rocks.remove(target); self.entrypoints.retain(|_, id| id != target); } @@ -697,14 +741,14 @@ impl LocalPackageLock { &self, req: &PackageReq, filter: Option, - ) -> Option { + ) -> Option { self.list() .get(req.name()) .map(|packages| { packages .iter() .filter(|package| match &filter { - Some(filter_spec) => match package.source { + Some(filter_spec) => match &package.source { RemotePackageSource::LuarocksRockspec(_) => filter_spec.rockspec, RemotePackageSource::LuarocksSrcRock(_) => filter_spec.src, RemotePackageSource::LuarocksBinaryRock(_) => filter_spec.binary, @@ -721,7 +765,7 @@ impl LocalPackageLock { .cloned() } - fn has_rock_with_equal_constraint(&self, req: &LuaDependencySpec) -> Option { + fn has_rock_with_equal_constraint(&self, req: &LuaDependencySpec) -> Option { self.list() .get(req.name()) .map(|packages| { @@ -744,12 +788,12 @@ impl LocalPackageLock { packages: &[LuaDependencySpec], strategy: &SyncStrategy<'_>, ) -> PackageSyncSpec { - let pkg_dir_exists = |pkg: &LocalPackage| match strategy { + let pkg_dir_exists = |pkg: &LockedPackage| match strategy { SyncStrategy::LockfileOnly => true, - SyncStrategy::EnsureInstalled(tree) => tree.layout_for(pkg).root.is_dir(), + SyncStrategy::EnsureInstalled(tree) => tree.layout_for(pkg.spec()).root.is_dir(), }; - let entrypoints_to_keep: HashSet = self + let entrypoints_to_keep: HashSet = self .entrypoints .values() .filter_map(|id| self.get(id)) @@ -763,7 +807,7 @@ impl LocalPackageLock { .cloned() .collect(); - let packages_to_keep: HashSet<&LocalPackage> = entrypoints_to_keep + let packages_to_keep: HashSet<&LockedPackage> = entrypoints_to_keep .iter() .flat_map(|local_pkg| self.get_all_dependencies(&local_pkg.id())) .collect(); @@ -789,7 +833,7 @@ impl LocalPackageLock { } /// Return all dependencies of a package, including itself - fn get_all_dependencies(&self, id: &LocalPackageId) -> HashSet<&LocalPackage> { + fn get_all_dependencies(&self, id: &LockedPackageId) -> HashSet<&LockedPackage> { let mut packages = HashSet::new(); if let Some(local_pkg) = self.get(id) { packages.insert(local_pkg); @@ -805,7 +849,7 @@ impl LocalPackageLock { /// The set of all packages reachable from the entrypoints via the /// dependency graph. - fn reachable(&self) -> HashSet<&LocalPackage> { + fn reachable(&self) -> HashSet<&LockedPackage> { self.entrypoints .values() .flat_map(|id| self.get_all_dependencies(id)) @@ -822,12 +866,14 @@ pub struct Lockfile { _marker: PhantomData

, // TODO: Serialize this directly into a `Version` version: String, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + binaries: BTreeMap, #[serde(flatten)] - lock: LocalPackageLock, + lock: PackageLock, } -#[derive(EnumIter, Debug, PartialEq, Eq)] -pub enum LocalPackageLockType { +#[derive(EnumIter, Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum LockedPackageLockType { Regular, Test, Build, @@ -841,12 +887,12 @@ pub struct WorkspaceLockfile { #[serde(skip)] _marker: PhantomData

, version: String, - #[serde(default, skip_serializing_if = "LocalPackageLock::is_empty")] - dependencies: LocalPackageLock, - #[serde(default, skip_serializing_if = "LocalPackageLock::is_empty")] - test_dependencies: LocalPackageLock, - #[serde(default, skip_serializing_if = "LocalPackageLock::is_empty")] - build_dependencies: LocalPackageLock, + #[serde(default, skip_serializing_if = "PackageLock::is_empty")] + dependencies: PackageLock, + #[serde(default, skip_serializing_if = "PackageLock::is_empty")] + test_dependencies: PackageLock, + #[serde(default, skip_serializing_if = "PackageLock::is_empty")] + build_dependencies: PackageLock, } #[derive(Error, Debug, Diagnostic)] @@ -923,7 +969,9 @@ check the source URL, then rerun the command with `lx --no-lock` to update the h expected: Integrity, got: Integrity, }, - #[error("package {0} version {1} with pinned state {2} and constraint {3} not found in the lockfile.")] + #[error( + "package {0} version {1} with pinned state {2} and constraint {3} not found in the lockfile." + )] PackageNotFound(PackageName, Box, PinnedState, String), } @@ -940,7 +988,7 @@ pub struct FlushLockfileError { #[derive(Debug, Default)] pub(crate) struct PackageSyncSpec { pub to_add: Vec, - pub to_remove: Vec, + pub to_remove: Vec, } /// Controls how `package_sync_spec` determines whether a package exists. @@ -957,30 +1005,30 @@ impl Lockfile

{ &self.version } - pub fn rocks(&self) -> &BTreeMap { + pub fn rocks(&self) -> &BTreeMap { self.lock.rocks() } - pub fn is_dependency(&self, package: &LocalPackageId) -> bool { + pub fn is_dependency(&self, package: &LockedPackageId) -> bool { self.lock.is_dependency(package) } - pub fn is_entrypoint(&self, package: &LocalPackageId) -> bool { + pub fn is_entrypoint(&self, package: &LockedPackageId) -> bool { self.lock.is_entrypoint(package) } /// Returns the entrypoint with the given package name, if any. - pub fn entrypoint(&self, name: &PackageName) -> Option<&LocalPackage> { + pub fn entrypoint(&self, name: &PackageName) -> Option<&LockedPackage> { self.lock.entrypoint(name) } /// Returns all rocks in the lockfile that are reachable from an entrypoint /// via the dependency graph. - pub fn reachable_rocks(&self) -> Vec { + pub fn reachable_rocks(&self) -> Vec { self.lock.reachable().into_iter().cloned().collect() } - pub fn entry_type(&self, package: &LocalPackageId) -> EntryType { + pub fn entry_type(&self, package: &LockedPackageId) -> EntryType { if self.lock.is_entrypoint(package) { EntryType::Entrypoint } else { @@ -988,11 +1036,16 @@ impl Lockfile

{ } } - pub(crate) fn local_pkg_lock(&self) -> &LocalPackageLock { + /// The binaries installed in the tree for a given package. + pub(crate) fn binaries(&self, id: &LockedPackageId) -> Option<&RockBinaries> { + self.binaries.get(id) + } + + pub(crate) fn local_pkg_lock(&self) -> &PackageLock { &self.lock } - pub fn get(&self, id: &LocalPackageId) -> Option<&LocalPackage> { + pub fn get(&self, id: &LockedPackageId) -> Option<&LockedPackage> { self.lock.get(id) } @@ -1001,11 +1054,11 @@ impl Lockfile

{ /// # Safety /// /// Ensure that the package is present in the lockfile before calling this function. - pub unsafe fn get_unchecked(&self, id: &LocalPackageId) -> &LocalPackage { + pub unsafe fn get_unchecked(&self, id: &LockedPackageId) -> &LockedPackage { self.lock.get(id).unwrap_unchecked() } - pub(crate) fn list(&self) -> HashMap> { + pub(crate) fn list(&self) -> HashMap> { self.lock.list() } @@ -1013,12 +1066,12 @@ impl Lockfile

{ &self, req: &PackageReq, filter: Option, - ) -> Option { + ) -> Option { self.lock.has_rock(req, filter) } /// Find all rocks that match the requirement - pub(crate) fn find_rocks(&self, req: &PackageReq) -> Vec { + pub(crate) fn find_rocks(&self, req: &PackageReq) -> Vec { match self.list().get(req.name()) { Some(packages) => packages .iter() @@ -1033,7 +1086,7 @@ impl Lockfile

{ /// Validate the integrity of an installed package with the entry in this lockfile. pub(crate) fn validate_integrity( &self, - expected_package: &LocalPackage, + expected_package: &LockedPackage, ) -> Result<(), LockfileIntegrityError> { // NOTE: We can't query by ID, because when installing from a lockfile (e.g. during sync), // the constraint is always `==`. @@ -1085,7 +1138,7 @@ impl Lockfile

{ source: io::Error::other(err), })?; - fs::sync::write(&self.filepath, content).map_err(|err| FlushLockfileError { + fs::sync::write_atomic(&self.filepath, content).map_err(|err| FlushLockfileError { filepath: self.filepath.to_path_buf(), source: io::Error::other(err), }) @@ -1095,67 +1148,67 @@ impl Lockfile

{ impl WorkspaceLockfile

{ pub(crate) fn rocks( &self, - deps: &LocalPackageLockType, - ) -> &BTreeMap { + deps: &LockedPackageLockType, + ) -> &BTreeMap { match deps { - LocalPackageLockType::Regular => self.dependencies.rocks(), - LocalPackageLockType::Test => self.test_dependencies.rocks(), - LocalPackageLockType::Build => self.build_dependencies.rocks(), + LockedPackageLockType::Regular => self.dependencies.rocks(), + LockedPackageLockType::Test => self.test_dependencies.rocks(), + LockedPackageLockType::Build => self.build_dependencies.rocks(), } } pub(crate) fn get( &self, - id: &LocalPackageId, - deps: &LocalPackageLockType, - ) -> Option<&LocalPackage> { + id: &LockedPackageId, + deps: &LockedPackageLockType, + ) -> Option<&LockedPackage> { match deps { - LocalPackageLockType::Regular => self.dependencies.get(id), - LocalPackageLockType::Test => self.test_dependencies.get(id), - LocalPackageLockType::Build => self.build_dependencies.get(id), + LockedPackageLockType::Regular => self.dependencies.get(id), + LockedPackageLockType::Test => self.test_dependencies.get(id), + LockedPackageLockType::Build => self.build_dependencies.get(id), } } pub(crate) fn is_entrypoint( &self, - package: &LocalPackageId, - deps: &LocalPackageLockType, + package: &LockedPackageId, + deps: &LockedPackageLockType, ) -> bool { match deps { - LocalPackageLockType::Regular => self.dependencies.is_entrypoint(package), - LocalPackageLockType::Test => self.test_dependencies.is_entrypoint(package), - LocalPackageLockType::Build => self.build_dependencies.is_entrypoint(package), + LockedPackageLockType::Regular => self.dependencies.is_entrypoint(package), + LockedPackageLockType::Test => self.test_dependencies.is_entrypoint(package), + LockedPackageLockType::Build => self.build_dependencies.is_entrypoint(package), } } pub(crate) fn package_sync_spec( &self, packages: &[LuaDependencySpec], - deps: &LocalPackageLockType, + deps: &LockedPackageLockType, strategy: &SyncStrategy<'_>, ) -> PackageSyncSpec { match deps { - LocalPackageLockType::Regular => { + LockedPackageLockType::Regular => { self.dependencies.package_sync_spec(packages, strategy) } - LocalPackageLockType::Test => { + LockedPackageLockType::Test => { self.test_dependencies.package_sync_spec(packages, strategy) } - LocalPackageLockType::Build => self + LockedPackageLockType::Build => self .build_dependencies .package_sync_spec(packages, strategy), } } - pub(crate) fn local_pkg_lock(&self, deps: &LocalPackageLockType) -> &LocalPackageLock { + pub(crate) fn local_pkg_lock(&self, deps: &LockedPackageLockType) -> &PackageLock { match deps { - LocalPackageLockType::Regular => &self.dependencies, - LocalPackageLockType::Test => &self.test_dependencies, - LocalPackageLockType::Build => &self.build_dependencies, + LockedPackageLockType::Regular => &self.dependencies, + LockedPackageLockType::Test => &self.test_dependencies, + LockedPackageLockType::Build => &self.build_dependencies, } } - pub(crate) fn local_pkg_locks(&self) -> Vec { + pub(crate) fn local_pkg_locks(&self) -> Vec { vec![ self.dependencies.clone(), self.test_dependencies.clone(), @@ -1166,7 +1219,7 @@ impl WorkspaceLockfile

{ fn flush(&self) -> io::Result<()> { let content = serde_json::to_string_pretty(&self)?; - fs::sync::write(&self.filepath, content).map_err(io::Error::other)?; + fs::sync::write_atomic(&self.filepath, content).map_err(io::Error::other)?; Ok(()) } @@ -1183,7 +1236,8 @@ impl Lockfile { filepath: filepath.clone(), _marker: PhantomData, version: LOCKFILE_VERSION_STR.into(), - lock: LocalPackageLock::default(), + binaries: BTreeMap::new(), + lock: PackageLock::default(), }; let json_str = serde_json::to_string(&empty_lockfile).map_err(LockfileError::WriteJson)?; @@ -1197,7 +1251,7 @@ impl Lockfile { return Err(LockfileError::Fs(fs::FsError::FileOpen { path: filepath.to_path_buf(), source, - })) + })); } } @@ -1225,6 +1279,7 @@ remove the tree at '{}' and reinstall all packages it contained.", _marker: PhantomData, filepath: self.filepath, version: self.version, + binaries: self.binaries, lock: self.lock, } } @@ -1287,9 +1342,9 @@ impl WorkspaceLockfile { filepath: filepath.clone(), _marker: PhantomData, version: LOCKFILE_VERSION_STR.into(), - dependencies: LocalPackageLock::default(), - test_dependencies: LocalPackageLock::default(), - build_dependencies: LocalPackageLock::default(), + dependencies: PackageLock::default(), + test_dependencies: PackageLock::default(), + build_dependencies: PackageLock::default(), }; let json_str = serde_json::to_string(&empty_lockfile).map_err(LockfileError::WriteJson)?; @@ -1303,7 +1358,7 @@ impl WorkspaceLockfile { return Err(LockfileError::Fs(fs::FsError::FileOpen { path: filepath.to_path_buf(), source, - })) + })); } } @@ -1346,12 +1401,12 @@ remove the `lux.lock` and run `lx sync` to regenerate it.", } impl Lockfile { - pub(crate) fn add_entrypoint(&mut self, rock: &LocalPackage) { + pub(crate) fn add_entrypoint(&mut self, rock: &LockedPackage) { self.add(rock); self.lock.entrypoints.insert(rock.name().clone(), rock.id()); } - fn add(&mut self, rock: &LocalPackage) { + pub(crate) fn add(&mut self, rock: &LockedPackage) { // Since rocks entries are mutable, we only add the dependency if it // has not already been added. self.lock @@ -1361,7 +1416,7 @@ impl Lockfile { } /// Add a dependency for a package. - pub(crate) fn add_dependency(&mut self, target: &LocalPackage, dependency: &LocalPackage) { + pub(crate) fn add_dependency(&mut self, target: &LockedPackage, dependency: &LockedPackage) { self.lock .rocks .entry(target.id()) @@ -1385,8 +1440,8 @@ impl Lockfile { /// Add a build dependency for a package. pub(crate) fn add_build_dependency( &mut self, - target: &LocalPackage, - dependency: &LocalPackage, + target: &LockedPackage, + dependency: &LockedPackage, ) { self.lock .rocks @@ -1407,15 +1462,22 @@ impl Lockfile { }); } - pub(crate) fn remove(&mut self, target: &LocalPackage) { + /// Record the binaries installed for `package` into the tree lockfile. + pub(crate) fn set_binaries(&mut self, package: &LockedPackage, binaries: RockBinaries) { + self.binaries.insert(package.id(), binaries); + } + + pub(crate) fn remove(&mut self, target: &LockedPackage) { + self.binaries.remove(&target.id()); self.lock.remove(target) } - pub(crate) fn remove_by_id(&mut self, target: &LocalPackageId) { + pub(crate) fn remove_by_id(&mut self, target: &LockedPackageId) { + self.binaries.remove(target); self.lock.remove_by_id(target) } - pub(crate) fn sync(&mut self, lock: &LocalPackageLock) { + pub(crate) fn sync(&mut self, lock: &PackageLock) { self.lock = lock.clone(); } @@ -1423,27 +1485,35 @@ impl Lockfile { } impl WorkspaceLockfile { - pub(crate) fn remove(&mut self, target: &LocalPackage, deps: &LocalPackageLockType) { + pub(crate) fn remove(&mut self, target: &LockedPackage, deps: &LockedPackageLockType) { match deps { - LocalPackageLockType::Regular => self.dependencies.remove(target), - LocalPackageLockType::Test => self.test_dependencies.remove(target), - LocalPackageLockType::Build => self.build_dependencies.remove(target), + LockedPackageLockType::Regular => self.dependencies.remove(target), + LockedPackageLockType::Test => self.test_dependencies.remove(target), + LockedPackageLockType::Build => self.build_dependencies.remove(target), } } - pub(crate) fn sync(&mut self, lock: &LocalPackageLock, deps: &LocalPackageLockType) { + pub(crate) fn sync(&mut self, lock: &PackageLock, deps: &LockedPackageLockType) { match deps { - LocalPackageLockType::Regular => { + LockedPackageLockType::Regular => { self.dependencies = lock.clone(); } - LocalPackageLockType::Test => { + LockedPackageLockType::Test => { self.test_dependencies = lock.clone(); } - LocalPackageLockType::Build => { + LockedPackageLockType::Build => { self.build_dependencies = lock.clone(); } } } + + pub(crate) fn merge(&mut self, lock: &PackageLock, deps: &LockedPackageLockType) { + match deps { + LockedPackageLockType::Regular => self.dependencies.merge(lock), + LockedPackageLockType::Test => self.test_dependencies.merge(lock), + LockedPackageLockType::Build => self.build_dependencies.merge(lock), + } + } } /// Flushes a lockfile automatically when it goes out of scope @@ -1660,7 +1730,7 @@ fn migrate_entrypoints( Ok(()) } -fn integrity_err_not_found(package: &LocalPackage) -> LockfileIntegrityError { +fn integrity_err_not_found(package: &LockedPackage) -> LockfileIntegrityError { LockfileIntegrityError::PackageNotFound( package.name().clone(), Box::new(package.version().clone()), @@ -1727,7 +1797,7 @@ mod tests { ) .unwrap(); - let mock_hashes = LocalPackageHashes { + let mock_hashes = LockedPackageHashes { rockspec: "sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=" .parse() .unwrap(), @@ -1745,10 +1815,9 @@ mod tests { let mut lockfile = tree.lockfile().unwrap().write_guard(); let test_package = PackageSpec::parse("test1".to_string(), "0.1.0".to_string()).unwrap(); - let test_local_package = LocalPackage::from( + let test_local_package = LockedPackage::from( &test_package, crate::lockfile::LockConstraint::Unconstrained, - RockBinaries::default(), RemotePackageSource::Test, None, mock_hashes.clone(), @@ -1757,10 +1826,9 @@ mod tests { let test_dep_package = PackageSpec::parse("test2".to_string(), "0.1.0".to_string()).unwrap(); - let mut test_local_dep_package = LocalPackage::from( + let mut test_local_dep_package = LockedPackage::from( &test_dep_package, crate::lockfile::LockConstraint::Constrained(">= 1.0.0".parse().unwrap()), - RockBinaries::default(), RemotePackageSource::Test, None, mock_hashes.clone(), diff --git a/lux-lib/src/lockfile/snapshots/lux_lib__lockfile__tests__add_rocks.snap b/lux-lib/src/lockfile/snapshots/lux_lib__lockfile__tests__add_rocks.snap index 720fa4fc7..a112ab549 100644 --- a/lux-lib/src/lockfile/snapshots/lux_lib__lockfile__tests__add_rocks.snap +++ b/lux-lib/src/lockfile/snapshots/lux_lib__lockfile__tests__add_rocks.snap @@ -1,6 +1,5 @@ --- source: lux-lib/src/lockfile/mod.rs -assertion_line: 1494 expression: lockfile --- { @@ -136,10 +135,6 @@ expression: lockfile "name": "lua-cjson", "version": "2.1.0-1", "constraint": "==2.1.0", - "binaries": [ - "json2lua", - "lua2json" - ], "source": "luarocks_rockspec+https://luarocks.org/", "source_url": { "type": "url", diff --git a/lux-lib/src/lockfile/snapshots/lux_lib__lockfile__tests__parse_lockfile.snap b/lux-lib/src/lockfile/snapshots/lux_lib__lockfile__tests__parse_lockfile.snap index 88b3e09af..729d525a0 100644 --- a/lux-lib/src/lockfile/snapshots/lux_lib__lockfile__tests__parse_lockfile.snap +++ b/lux-lib/src/lockfile/snapshots/lux_lib__lockfile__tests__parse_lockfile.snap @@ -1,6 +1,5 @@ --- source: lux-lib/src/lockfile/mod.rs -assertion_line: 1441 expression: lockfile --- { @@ -113,10 +112,6 @@ expression: lockfile "name": "lua-cjson", "version": "2.1.0-1", "constraint": "==2.1.0", - "binaries": [ - "json2lua", - "lua2json" - ], "source": "luarocks_rockspec+https://luarocks.org/", "source_url": { "type": "url", diff --git a/lux-lib/src/lua_rockspec/mod.rs b/lux-lib/src/lua_rockspec/mod.rs index 8896c89a2..89f24fc4d 100644 --- a/lux-lib/src/lua_rockspec/mod.rs +++ b/lux-lib/src/lua_rockspec/mod.rs @@ -206,6 +206,44 @@ impl<'gc> HasRockspecKey<'gc> for ottavino::Table<'gc> { } impl LocalLuaRockspec { + /// Builds a [`LocalLuaRockspec`] from an already-parsed project. + /// FIXME(vhyrro): investigate if necessary or can be deduplicated + pub(crate) fn from_project_toml( + project: &crate::project::project_toml::LocalProjectToml, + ) -> Self { + use crate::rockspec::Rockspec; + + Self { + rockspec_format: project.format().clone(), + package: project.package().clone(), + version: project.version().clone(), + description: project.description().clone(), + supported_platforms: project.supported_platforms().clone(), + lua: project.lua().clone(), + dependencies: project.dependencies().clone(), + build_dependencies: project.build_dependencies().clone(), + external_dependencies: project.external_dependencies().clone(), + test_dependencies: project.test_dependencies().clone(), + build: project.build().clone(), + source: project.source().clone(), + test: project.test().clone(), + deploy: project.deploy().clone(), + raw_content: String::new(), + } + } + + /// Wraps this rockspec into a [`RemoteLuaRockspec`], overriding its source + /// with the given `source`. + pub fn into_remote_with_source( + self, + source: PerPlatform, + ) -> RemoteLuaRockspec { + RemoteLuaRockspec { + local: self, + source, + } + } + pub fn new( rockspec_content: &str, project_root: ProjectRoot, diff --git a/lux-lib/src/luarocks/install_binary_rock.rs b/lux-lib/src/luarocks/install_binary_rock.rs index 62532ca27..c81574ca7 100644 --- a/lux-lib/src/luarocks/install_binary_rock.rs +++ b/lux-lib/src/luarocks/install_binary_rock.rs @@ -13,11 +13,11 @@ use crate::{ config::Config, hash::HasIntegrity, lockfile::{ - LocalPackage, LocalPackageHashes, LockConstraint, LockfileError, OptState, PinnedState, + LockConstraint, LockedPackage, LockedPackageHashes, LockedPackageSpec, LockfileError, + OptState, PinnedState, }, lua_rockspec::{LuaVersionError, RemoteLuaRockspec}, luarocks::rock_manifest::RockManifest, - package::PackageSpec, remote_package_source::RemotePackageSource, rockspec::Rockspec, tree::{self, InstallTree, TreeError}, @@ -51,7 +51,9 @@ pub enum InstallBinaryRockError { LuaVersionError(#[from] LuaVersionError), #[error("failed to unpack packed rock")] Zip(#[from] zip::result::ZipError), - #[error("rock_manifest not found. Cannot install rock files that were packed using LuaRocks version 1")] + #[error( + "rock_manifest not found. Cannot install rock files that were packed using LuaRocks version 1" + )] RockManifestNotFound, #[error(transparent)] #[diagnostic(transparent)] @@ -97,7 +99,7 @@ where config, tree, constraint: LockConstraint::default(), - behaviour: BuildBehaviour::default(), + behaviour: BuildBehaviour::Ignore, pin: PinnedState::default(), opt: OptState::default(), entry_type, @@ -121,7 +123,7 @@ where } #[tracing::instrument(name = "Installing binary rock", skip_all)] - pub(crate) async fn install(self) -> Result { + pub(crate) async fn install(self) -> Result { let rockspec = self.rockspec; for (name, dep) in rockspec.external_dependencies().current_platform() { let _ = ExternalDependencyInfo::probe(name, dep, self.config.external_deps())?; @@ -129,7 +131,7 @@ where rockspec.validate_lua_version_from_config(self.config)?; - let hashes = LocalPackageHashes { + let hashes = LockedPackageHashes { rockspec: rockspec.hash().await?, source: self.rock_bytes.hash().await?, }; @@ -139,18 +141,25 @@ where } _ => None, }; - let mut package = LocalPackage::from( - &PackageSpec::new(rockspec.package().clone(), rockspec.version().clone()), + let spec = LockedPackageSpec::new( + rockspec.package(), + rockspec.version(), self.constraint, - rockspec.binaries(), - self.source, - source_url, - hashes, + Vec::new(), + Vec::new(), + &self.pin, + &self.opt, ); - package.spec.pinned = self.pin; - package.spec.opt = self.opt; + let package = LockedPackage::new(spec, self.source, source_url, hashes); match self.tree.lockfile()?.get(&package.id()) { - Some(package) if self.behaviour == BuildBehaviour::NoForce => Ok(package.clone()), + Some(package) + if matches!( + self.behaviour, + BuildBehaviour::Ignore | BuildBehaviour::Conflict + ) => + { + Ok(package.clone()) + } _ => { let unpack_dir = fs::tempfile::tempdir()?; let cursor = Cursor::new(self.rock_bytes); @@ -166,8 +175,8 @@ where return Err(InstallBinaryRockError::RockManifestNotFound); } let rock_manifest_content = fs::tokio::read_to_string(rock_manifest_file).await?; - self.tree.prepare(&package)?; - let layout = self.tree.layout_for(&package); + self.tree.prepare(&package.spec)?; + let layout = self.tree.layout_for(&package.spec); let rock_manifest = RockManifest::new(&rock_manifest_content)?; install_manifest_entries( &rock_manifest.lib.entries, @@ -209,6 +218,10 @@ where fs::tokio::remove_file(&rockspec_path).await?; } self.tree.finalize(&package, self.entry_type)?; + self.tree + .lockfile()? + .write_guard() + .set_binaries(&package, rockspec.binaries()); Ok(package) } } @@ -297,8 +310,14 @@ mod tests { .install() .await .unwrap(); + assert_eq!( + local_package.hashes().source, + "sha256-zDmnjE2b8GxX+6j5vTCWUEBxe0aFXFVhC4JOrnAgQ2g=" + .parse() + .unwrap() + ); let foo_bar_module = tree - .layout_for(&local_package) + .layout_for(&local_package.spec) .src .join("foo") .join("bar.lua"); @@ -360,7 +379,7 @@ mod tests { .install() .await .unwrap(); - let layout = tree.layout_for(&local_package); + let layout = tree.layout_for(local_package.spec()); assert!(layout.lib.join("toml_edit.so").is_file()); let orig_install_tree_integrity = layout.root.hash().await.unwrap(); @@ -419,7 +438,7 @@ mod tests { .install() .await .unwrap(); - let layout = tree.layout_for(&local_package); + let layout = tree.layout_for(local_package.spec()); assert!(layout.rockspec_path().is_file()); let new_install_tree_integrity = layout.root.hash().await.unwrap(); assert_eq!(orig_install_tree_integrity, new_install_tree_integrity); diff --git a/lux-lib/src/luarocks/luarocks_installation.rs b/lux-lib/src/luarocks/luarocks_installation.rs index e5b29f243..f5263fd32 100644 --- a/lux-lib/src/luarocks/luarocks_installation.rs +++ b/lux-lib/src/luarocks/luarocks_installation.rs @@ -26,7 +26,7 @@ use crate::tree::{self, Tree, TreeError}; use crate::tree::{Tree, TreeError}; #[cfg(target_family = "unix")] -use crate::build::Build; +use crate::pipeline::build_local::Build; #[cfg(target_family = "unix")] const LUAROCKS_EXE: &str = "luarocks"; @@ -97,7 +97,9 @@ pub enum ExecLuaRocksError { #[error(transparent)] #[diagnostic(transparent)] LuaVersionUnset(#[from] LuaVersionUnset), - #[error("could not substitute '$(LUA_LIBDIR)' and '$(LUA_INCDIR)' variables in the luarocks config template")] + #[error( + "could not substitute '$(LUA_LIBDIR)' and '$(LUA_INCDIR)' variables in the luarocks config template" + )] #[diagnostic(forward(0))] VariableSubstitutionInConfig(#[from] VariableSubstitutionError), #[error("failed to run luarocks")] @@ -165,6 +167,7 @@ impl LuaRocksInstallation { .tree(&self.tree) .entry_type(tree::EntryType::Entrypoint) .config(&self.config) + .behaviour(build::BuildBehaviour::Ignore) .constraint(luarocks_req.version_req().clone().into()) .build() .await?; diff --git a/lux-lib/src/manifest/metadata.rs b/lux-lib/src/manifest/metadata.rs index da5c10665..b18751aab 100644 --- a/lux-lib/src/manifest/metadata.rs +++ b/lux-lib/src/manifest/metadata.rs @@ -223,4 +223,43 @@ mod tests { }; assert!(metadata.latest_match(&package_req, &filter).is_none()); } + + #[tokio::test] + pub async fn latest_match_prefers_binary_then_rockspec_then_src() { + let manifest = r#" + commands = {} + modules = {} + repository = { + ['testpkg'] = { + ['1.0-1'] = { + { arch = "rockspec" }, { arch = "src" }, { arch = "all" } + } + } + } + "# + .to_string(); + let metadata = ManifestMetadata::new(&manifest).unwrap(); + let package_req: PackageReq = "testpkg == 1.0-1".parse().unwrap(); + + let (_, rock_type) = metadata + .latest_match(&package_req, &Default::default()) + .unwrap(); + assert_eq!(rock_type, RemotePackageType::Binary); + + let no_binary = RemotePackageTypeFilterSpec { + rockspec: true, + src: true, + binary: false, + }; + let (_, rock_type) = metadata.latest_match(&package_req, &no_binary).unwrap(); + assert_eq!(rock_type, RemotePackageType::Rockspec); + + let src_only = RemotePackageTypeFilterSpec { + rockspec: false, + src: true, + binary: false, + }; + let (_, rock_type) = metadata.latest_match(&package_req, &src_only).unwrap(); + assert_eq!(rock_type, RemotePackageType::Src); + } } diff --git a/lux-lib/src/operations/build_workspace.rs b/lux-lib/src/operations/build_workspace.rs index a5cc04889..94bacacde 100644 --- a/lux-lib/src/operations/build_workspace.rs +++ b/lux-lib/src/operations/build_workspace.rs @@ -1,70 +1,26 @@ use crate::{ - build::{Build, BuildBehaviour, BuildError}, + build::BuildBehaviour, config::Config, - lockfile::LocalPackage, - lua_installation::{LuaInstallation, LuaInstallationError}, - luarocks::luarocks_installation::{LuaRocksError, LuaRocksInstallError, LuaRocksInstallation}, - operations::{install_dependencies::prepare_dependencies_for_build, InstallDependencies}, + drivers::sync::{Sync, SyncError, SyncMode, TargetSet}, + lockfile::LockedPackage, package::PackageName, - project::{project_toml::LocalProjectTomlValidationError, Project}, - tree::{self, InstallTree, TreeError}, - workspace::{Workspace, WorkspaceError, WorkspaceTreeError}, + workspace::Workspace, }; use bon::Builder; -use itertools::Itertools; -use miette::Diagnostic; use thiserror::Error; use tracing::{info_span, Instrument}; -use super::{InstallError, Sync, SyncError}; - -#[derive(Debug, Error, Diagnostic)] +#[derive(Debug, Error, miette::Diagnostic)] #[non_exhaustive] pub enum BuildWorkspaceError { #[error(transparent)] #[diagnostic(transparent)] - LocalProjectTomlValidation(#[from] LocalProjectTomlValidationError), - #[error(transparent)] - #[diagnostic(transparent)] - Workspace(#[from] WorkspaceError), - #[error(transparent)] - #[diagnostic(transparent)] - WorkspaceTree(#[from] WorkspaceTreeError), - #[error(transparent)] - #[diagnostic(transparent)] - LuaInstallation(#[from] LuaInstallationError), - #[error(transparent)] - #[diagnostic(transparent)] - Tree(#[from] TreeError), - #[error(transparent)] - #[diagnostic(transparent)] - LuaRocks(#[from] LuaRocksError), - #[error(transparent)] - #[diagnostic(transparent)] - LuaRocksInstall(#[from] Box), - #[error("error installind dependencies")] - #[diagnostic(forward(0))] - InstallDependencies(#[source] Box), - #[error("error installind build dependencies")] - #[diagnostic(forward(0))] - InstallBuildDependencies(#[source] Box), - #[error("syncing dependencies with the workspace lockfile failed")] - #[diagnostic(forward(0))] - SyncDependencies(#[source] Box), - #[error("error building the workspace")] - #[diagnostic(forward(0))] - Build(#[from] Box), + Sync(Box), } -impl From for BuildWorkspaceError { - fn from(source: LuaRocksInstallError) -> Self { - Self::LuaRocksInstall(Box::new(source)) - } -} - -impl From for BuildWorkspaceError { - fn from(source: BuildError) -> Self { - Self::Build(Box::new(source)) +impl From for BuildWorkspaceError { + fn from(source: SyncError) -> Self { + Self::Sync(Box::new(source)) } } @@ -85,147 +41,46 @@ pub struct BuildWorkspace<'a> { /// Build only the dependencies only_deps: bool, + + #[builder(default = BuildBehaviour::Force)] + behaviour: BuildBehaviour, } impl BuildWorkspaceBuilder<'_, State> { - pub async fn build(self) -> Result, BuildWorkspaceError> { + pub async fn build(self) -> Result, BuildWorkspaceError> { let build = self._build(); let span = match &build.package { Some(package) => info_span!("Building workspace", package = package.to_string()), None => info_span!("Building workspace"), }; - do_build(build).instrument(span).await - } -} - -async fn do_build(args: BuildWorkspace<'_>) -> Result, BuildWorkspaceError> { - let config = args.config; - let workspace = args.workspace; - let workspace_tree = workspace.tree(config)?; - let build_tree = workspace.build_tree(config)?; - let lua = LuaInstallation::new_from_config(config).await?; - if !args.no_lock { - Sync::new(workspace, config) - .sync() - .await - .map_err(|err| BuildWorkspaceError::SyncDependencies(Box::new(err)))?; - } else { - let luarocks = LuaRocksInstallation::new(config, build_tree.clone())?; - let mut dependencies_to_install = Vec::new(); - let mut build_dependencies_to_install = Vec::new(); - if let Some(package) = &args.package { - let project = workspace.select_member(package)?; - let project_toml = project.toml().into_local()?; - prepare_dependencies_for_build( - &project_toml, - &workspace_tree, - &mut dependencies_to_install, - &mut build_dependencies_to_install, - tree::EntryType::Entrypoint, - ); - } else { - for project in workspace.members() { - let project_toml = project.toml().into_local()?; - prepare_dependencies_for_build( - &project_toml, - &workspace_tree, - &mut dependencies_to_install, - &mut build_dependencies_to_install, - tree::EntryType::Entrypoint, - ); - } - } - - let tree = workspace.tree(config)?; - - InstallDependencies::new() - .dependencies(dependencies_to_install.into_iter().unique().collect_vec()) - .build_dependencies( - build_dependencies_to_install - .into_iter() - .unique() - .collect_vec(), - ) - .tree(&tree) - .lua(&lua) - .luarocks(&luarocks) - .config(config) - .build() - .await - .map_err(|err| BuildWorkspaceError::InstallBuildDependencies(Box::new(err)))?; - } - - let mut packages = Vec::new(); - if !args.only_deps { - if let Some(package) = &args.package { - let project = workspace.select_member(package)?; - let pkg = build_project(project, workspace, &lua, config).await?; - packages.push(pkg); - } else { - for project in workspace.members() { - let pkg = build_project(project, workspace, &lua, config).await?; - packages.push(pkg); - } + async move { + let report = Sync::new(build.workspace, build.config) + .mode(SyncMode::Open) + .targets(match build.package { + Some(package) => TargetSet::member(package, false), + None => TargetSet::all(build.workspace, false), + }) + .behaviour(build.behaviour) + .only_deps(build.only_deps) + .no_lock(build.no_lock) + .sync() + .await?; + Ok(report.added().to_vec()) } + .instrument(span) + .await } - Ok(packages) -} - -async fn build_project( - project: &Project, - workspace: &Workspace, - lua: &LuaInstallation, - config: &Config, -) -> Result { - let workspace_tree = workspace.tree(config)?; - let project_toml = project.toml().into_local()?; - - let package = Build::new() - .rockspec(&project_toml) - .lua(lua) - .tree(&workspace_tree) - .entry_type(tree::EntryType::Entrypoint) - .config(config) - .behaviour(BuildBehaviour::Force) - .build() - .await?; - - let lockfile = workspace_tree.lockfile()?; - let dependencies = lockfile - .rocks() - .iter() - .filter_map(|(pkg_id, value)| { - if lockfile.is_entrypoint(pkg_id) { - Some(value) - } else { - None - } - }) - .cloned() - .collect_vec(); - - let build_lockfile = workspace.build_tree(config)?.lockfile()?; - - let mut lockfile = lockfile.write_guard(); - lockfile.add_entrypoint(&package); - for dep in dependencies { - lockfile.add_dependency(&package, &dep); - } - for dep in build_lockfile.rocks().values() { - lockfile.add_build_dependency(&package, dep); - } - Ok(package) } -#[cfg(test)] +#[cfg(all(test, feature = "impure_tests"))] mod tests { use super::*; use crate::{ config::ConfigBuilder, fs, lua_installation::detect_installed_lua_version, - lua_version::LuaVersion, + lua_version::LuaVersion, tree::InstallTree, }; use assert_fs::prelude::PathCopy; use std::path::PathBuf; @@ -302,7 +157,7 @@ mod tests { .unwrap(); let package = package.first().unwrap(); let tree = workspace.tree(&config).unwrap(); - let src_dir = tree.layout_for(package).src; + let src_dir = tree.layout_for(&package.spec).src; assert!(src_dir.join("init.lua").is_file()); } } diff --git a/lux-lib/src/operations/dist_bin.rs b/lux-lib/src/operations/dist_bin.rs index db11ab4ee..3d509d663 100644 --- a/lux-lib/src/operations/dist_bin.rs +++ b/lux-lib/src/operations/dist_bin.rs @@ -154,7 +154,7 @@ where std::env::current_dir()?.join(output) }; - let lib_root = args.tree.layout_for(&package).lib; + let lib_root = args.tree.layout_for(&package.spec).lib; let c_src = generate_c_source(&entrypoint_module, &files, &lib_root).await?; let work_dir = fs::tempfile::tempdir()?; @@ -173,7 +173,7 @@ fn collect_installed_files(tree: &impl InstallTree) -> Result LocalPackage { - let hashes = LocalPackageHashes { + fn mk_dummy_package(spec: PackageSpec) -> LockedPackage { + let hashes = LockedPackageHashes { rockspec: "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" .parse() .unwrap(), @@ -565,10 +567,9 @@ mod tests { .parse() .unwrap(), }; - LocalPackage::from( + LockedPackage::from( &spec, LockConstraint::Unconstrained, - RockBinaries::default(), RemotePackageSource::Test, None, hashes, @@ -587,8 +588,8 @@ mod tests { let tree = FlatDistTree::new(staging.to_path_buf(), LuaVersion::Lua51, &config).unwrap(); let pkg_a = mk_dummy_package(PackageSpec::new("foo".into(), "1.0.0-1".parse().unwrap())); - tree.prepare(&pkg_a).unwrap(); - let layout_a = tree.layout_for(&pkg_a); + tree.prepare(&pkg_a.spec).unwrap(); + let layout_a = tree.layout_for(&pkg_a.spec); staging .child(layout_a.src.strip_prefix(staging.path()).unwrap()) .create_dir_all() @@ -598,8 +599,8 @@ mod tests { .unwrap(); let pkg_b = mk_dummy_package(PackageSpec::new("bar".into(), "2.0.0-1".parse().unwrap())); - tree.prepare(&pkg_b).unwrap(); - let layout_b = tree.layout_for(&pkg_b); + tree.prepare(&pkg_b.spec).unwrap(); + let layout_b = tree.layout_for(&pkg_b.spec); staging .child(layout_b.src.strip_prefix(staging.path()).unwrap()) .create_dir_all() @@ -639,6 +640,7 @@ mod tests { .is_some_and(|e| e == c_dylib_extension())); } + #[cfg(feature = "impure_tests")] #[tokio::test] async fn test_collect_installed_files_from_sample_project() { let sample = PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -684,17 +686,20 @@ mod tests { ); } + #[cfg(feature = "impure_tests")] #[tokio::test] async fn test_dist_bin_from_lua_source_compiles_and_runs() { test_dist_bin_compiles_and_runs("resources/test/sample-projects/only-src/", "1").await } + #[cfg(feature = "impure_tests")] #[tokio::test] #[cfg(target_os = "linux")] async fn test_dist_bin_from_c_source_compiles_and_runs() { test_dist_bin_compiles_and_runs("resources/test/sample-projects/c-src/", "OK").await } + #[cfg(feature = "impure_tests")] async fn test_dist_bin_compiles_and_runs(sample_project_path: &str, expected_output: &str) { let sample = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(sample_project_path); let project_dir = TempDir::new().unwrap(); diff --git a/lux-lib/src/operations/download.rs b/lux-lib/src/operations/download.rs index 740eb3d54..7df176aa5 100644 --- a/lux-lib/src/operations/download.rs +++ b/lux-lib/src/operations/download.rs @@ -21,7 +21,7 @@ use crate::{ PackageName, PackageReq, PackageSpec, PackageSpecFromPackageReqError, PackageVersion, RemotePackageTypeFilterSpec, }, - remote_package_db::{RemotePackageDB, RemotePackageDBError, SearchError}, + package_db::{PackageDB, PackageDBError, SearchError}, remote_package_source::RemotePackageSource, reqwest::{RequestBuilderExt, RequestError}, rockspec::Rockspec, @@ -30,7 +30,7 @@ use crate::{ /// Builder for a rock downloader. pub struct Download<'a> { package_req: &'a PackageReq, - package_db: Option<&'a RemotePackageDB>, + package_db: Option<&'a PackageDB>, config: &'a Config, } @@ -46,7 +46,7 @@ impl<'a> Download<'a> { /// Sets the package database to use for searching for packages. /// Instantiated from the config if not set. - pub fn package_db(self, package_db: &'a RemotePackageDB) -> Self { + pub fn package_db(self, package_db: &'a PackageDB) -> Self { Self { package_db: Some(package_db), ..self @@ -58,7 +58,7 @@ impl<'a> Download<'a> { match self.package_db { Some(db) => download_rockspec(self.package_req, db, self.config).await, None => { - let db = RemotePackageDB::from_config(self.config).await?; + let db = PackageDB::from_config(self.config).await?; download_rockspec(self.package_req, &db, self.config).await } } @@ -75,7 +75,7 @@ impl<'a> Download<'a> { download_src_rock_to_file(self.package_req, destination_dir, db, self.config).await } None => { - let db = RemotePackageDB::from_config(self.config).await?; + let db = PackageDB::from_config(self.config).await?; download_src_rock_to_file(self.package_req, destination_dir, &db, self.config).await } } @@ -88,7 +88,7 @@ impl<'a> Download<'a> { match self.package_db { Some(db) => search_and_download_src_rock(self.package_req, db, self.config).await, None => { - let db = RemotePackageDB::from_config(self.config).await?; + let db = PackageDB::from_config(self.config).await?; search_and_download_src_rock(self.package_req, &db, self.config).await } } @@ -100,7 +100,7 @@ impl<'a> Download<'a> { match self.package_db { Some(db) => download_remote_rock(self.package_req, db, self.config).await, None => { - let db = RemotePackageDB::from_config(self.config).await?; + let db = PackageDB::from_config(self.config).await?; download_remote_rock(self.package_req, &db, self.config).await } } @@ -211,7 +211,7 @@ if the issue persists, the server may be temporarily unavailable."# ResponseConversion(#[from] FromUtf8Error), #[error("error initialising remote package DB")] #[diagnostic(forward(0))] - RemotePackageDB(#[from] RemotePackageDBError), + PackageDB(#[from] PackageDBError), #[error(transparent)] #[diagnostic(transparent)] DownloadSrcRock(#[from] DownloadSrcRockError), @@ -226,7 +226,7 @@ impl From for DownloadRockspecError { /// Find and download a rockspec for a given package requirement async fn download_rockspec( package_req: &PackageReq, - package_db: &RemotePackageDB, + package_db: &PackageDB, config: &Config, ) -> Result { let rockspec = match download_remote_rock(package_req, package_db, config).await? { @@ -253,7 +253,7 @@ async fn download_rockspec( )] async fn download_remote_rock( package_req: &PackageReq, - package_db: &RemotePackageDB, + package_db: &PackageDB, config: &Config, ) -> Result { let remote_package = package_db.find(package_req, None)?; @@ -373,7 +373,7 @@ if the issue persists, the server may be temporarily unavailable."# Rockspec(Box), #[error("error initialising remote package DB")] #[diagnostic(forward(0))] - RemotePackageDB(#[from] RemotePackageDBError), + PackageDB(#[from] PackageDBError), #[error("failed to read packed rock {0}:\n{1}")] #[diagnostic(help( r#"the downloaded rock may be corrupted. @@ -433,7 +433,7 @@ impl From for SearchAndDownloadError { async fn search_and_download_src_rock( package_req: &PackageReq, - package_db: &RemotePackageDB, + package_db: &PackageDB, config: &Config, ) -> Result { let filter = Some(RemotePackageTypeFilterSpec { @@ -500,7 +500,7 @@ pub(crate) async fn download_binary_rock( async fn download_src_rock_to_file( package_req: &PackageReq, destination_dir: Option, - package_db: &RemotePackageDB, + package_db: &PackageDB, config: &Config, ) -> Result { let rock = search_and_download_src_rock(package_req, package_db, config).await?; diff --git a/lux-lib/src/operations/exec.rs b/lux-lib/src/operations/exec.rs index d630976da..c316dd44f 100644 --- a/lux-lib/src/operations/exec.rs +++ b/lux-lib/src/operations/exec.rs @@ -2,13 +2,15 @@ use std::io; use tokio::process::Command; use crate::{ + build::BuildBehaviour, config::Config, + drivers::install_packages::{InstallPackages, InstallPackagesError}, lua_rockspec::LuaVersionError, lua_version::{LuaVersion, LuaVersionUnset}, - operations::{BuildWorkspace, BuildWorkspaceError, Install}, + operations::{BuildWorkspace, BuildWorkspaceError}, package::{PackageReq, PackageVersionReqError}, + package_db::{PackageDB, PackageDBError}, path::{Paths, PathsError}, - remote_package_db::RemotePackageDBError, tree::{self, InstallTree, TreeError}, workspace::{Workspace, WorkspaceTreeError}, }; @@ -18,7 +20,7 @@ use miette::Diagnostic; use thiserror::Error; use which::which; -use super::{InstallError, PackageInstallSpec}; +use super::PackageInstallSpec; /// Rocks package runner, providing fine-grained control /// over how a package should be run. @@ -91,7 +93,7 @@ pub enum ExecError { LuaVersionError(#[from] LuaVersionError), #[error(transparent)] #[diagnostic(transparent)] - BuildProject(#[from] Box), + BuildWorkspace(#[from] Box), #[error(transparent)] #[diagnostic(transparent)] InstallCommand(#[from] Box), @@ -104,7 +106,7 @@ pub enum ExecError { impl From for ExecError { fn from(source: BuildWorkspaceError) -> Self { - Self::BuildProject(Box::new(source)) + Self::BuildWorkspace(Box::new(source)) } } @@ -117,15 +119,15 @@ impl From for ExecError { #[derive(Error, Debug, Diagnostic)] #[error(transparent)] pub enum InstallCommandError { - InstallError(#[from] Box), + InstallError(#[from] Box), PackageVersionReqError(#[from] PackageVersionReqError), - RemotePackageDBError(#[from] RemotePackageDBError), + PackageDBError(#[from] PackageDBError), Tree(#[from] TreeError), LuaVersionUnset(#[from] LuaVersionUnset), } -impl From for InstallCommandError { - fn from(source: InstallError) -> Self { +impl From for InstallCommandError { + fn from(source: InstallPackagesError) -> Self { Self::InstallError(Box::new(source)) } } @@ -220,15 +222,21 @@ To suppress this warning, set the `--no-loader` option."# /// Ensure that a command is installed. /// This defaults to the local project tree if cwd is a project root. async fn install_command(command: &str, config: &Config) -> Result<(), InstallCommandError> { - let install_spec = PackageInstallSpec::new( - PackageReq::new(command.into(), None)?, - tree::EntryType::Entrypoint, - ) - .build(); + let package = PackageReq::new(command.into(), None)?; let tree = config.user_tree(LuaVersion::from(config)?.clone())?; - Install::new(config) + + if tree + .match_rocks(&package) + .is_ok_and(|matches| matches.is_found()) + { + return Ok(()); + } + let install_spec = PackageInstallSpec::new(package, tree::EntryType::Entrypoint) + .build_behaviour(BuildBehaviour::Force) + .build(); + InstallPackages::new(config, &tree) .package(install_spec) - .tree(tree) + .package_db(PackageDB::from_config(config).await?) .install() .await?; Ok(()) diff --git a/lux-lib/src/operations/fetch.rs b/lux-lib/src/operations/fetch.rs index 330759f9c..23a128e1f 100644 --- a/lux-lib/src/operations/fetch.rs +++ b/lux-lib/src/operations/fetch.rs @@ -117,6 +117,9 @@ pub enum FetchSrcError { #[error("failed to clone rock source")] #[diagnostic(help("check your network connection and verify the git URL is correct."))] GitClone(#[from] git2::Error), + #[error("git operation failed")] + #[diagnostic(help("this is a bug, please report it."))] + GitJoin(#[from] tokio::task::JoinError), #[error("failed to parse git URL")] #[diagnostic(forward(0))] GitUrlParse(#[from] RemoteGitUrlParseError), @@ -257,8 +260,15 @@ async fn fetch_src_impl( let url = git.url.to_string(); tracing::debug!(message = format!("Cloning {url}").as_str()); - let (checkout_ref, has_submodules) = { - let mut auth = if config.no_prompt() { + let clone_url = url.clone(); + let clone_dir = dest_dir.to_path_buf(); + let host = git.url.host().to_string(); + let no_prompt = config.no_prompt(); + let access_token = config.access_token(&host); + let git_ref = git.git_ref.clone(); + + let (checkout_ref, has_submodules) = tokio::task::spawn_blocking(move || { + let mut auth = if no_prompt { GitAuthenticator::default() .try_password_prompt(0) .prompt_ssh_key_password(false) @@ -266,12 +276,10 @@ async fn fetch_src_impl( } else { GitAuthenticator::default() }; - if let Some(access_token) = config.access_token(git.url.host()) { - auth = auth.add_plaintext_credentials( - git.url.host(), - access_token.username(), - unsafe { access_token.password() }, - ); + if let Some(access_token) = access_token { + auth = auth.add_plaintext_credentials(&host, access_token.username(), unsafe { + access_token.password() + }); } let git_config = git2::Config::open_default()?; let mut callbacks = RemoteCallbacks::new(); @@ -279,7 +287,7 @@ async fn fetch_src_impl( let mut fetch_options = FetchOptions::new(); fetch_options.update_fetchhead(false); fetch_options.remote_callbacks(callbacks); - let checkout_ref = match &git.git_ref { + let checkout_ref = match &git_ref { Some(GitRef::Tag(tag)) => Some(tag.as_str()), Some(GitRef::Branch(branch)) => Some(branch.as_str()), None => None, @@ -289,11 +297,11 @@ async fn fetch_src_impl( }; let mut repo_builder = RepoBuilder::new(); repo_builder.fetch_options(fetch_options); - let repo = repo_builder.clone(&url, dest_dir)?; + let repo = repo_builder.clone(&clone_url, &clone_dir)?; let checkout_ref = match checkout_ref { Some(checkout_ref) => { - let ref_name = if let Some(GitRef::Branch(branch)) = &git.git_ref { + let ref_name = if let Some(GitRef::Branch(branch)) = &git_ref { format!("origin/{branch}") } else { checkout_ref.to_string() @@ -311,8 +319,10 @@ async fn fetch_src_impl( let submodule_paths = init_submodules(&repo, &auth, &git_config)?; - (checkout_ref, !submodule_paths.is_empty()) - }; + Ok::<_, FetchSrcError>((checkout_ref, !submodule_paths.is_empty())) + }) + .await??; + // The .git directory is not deterministic remove_dir_all(dest_dir.join(".git")).map_err(FetchSrcError::CleanGitDir)?; let hash = dest_dir.hash().await.map_err(FetchSrcError::Hash)?; diff --git a/lux-lib/src/operations/fetch_vendored.rs b/lux-lib/src/operations/fetch_vendored.rs index dde0bc716..1b807d71b 100644 --- a/lux-lib/src/operations/fetch_vendored.rs +++ b/lux-lib/src/operations/fetch_vendored.rs @@ -6,7 +6,7 @@ use crate::{ lua_rockspec::{LuaRockspecError, RemoteLuaRockspec}, operations::{DownloadedRockspec, RemoteRockDownload}, package::{PackageReq, PackageSpec}, - remote_package_db::{RemotePackageDB, SearchError}, + package_db::{PackageDB, SearchError}, remote_package_source::RemotePackageSource, }; use bon::Builder; @@ -19,7 +19,7 @@ use thiserror::Error; pub(crate) struct FetchVendored<'a> { vendor_dir: &'a Path, package: &'a PackageReq, - package_db: &'a RemotePackageDB, + package_db: &'a PackageDB, } #[derive(Error, Debug, Diagnostic)] diff --git a/lux-lib/src/operations/gen_luarc.rs b/lux-lib/src/operations/gen_luarc.rs index 8c359442a..9a852b066 100644 --- a/lux-lib/src/operations/gen_luarc.rs +++ b/lux-lib/src/operations/gen_luarc.rs @@ -1,6 +1,6 @@ use crate::config::Config; use crate::fs; -use crate::lockfile::LocalPackageLockType; +use crate::lockfile::LockedPackageLockType; use crate::tree::InstallTree; use crate::workspace::Workspace; use crate::workspace::WorkspaceError; @@ -35,7 +35,7 @@ pub enum GenLuaRcError { #[derive(Builder)] #[builder(start_fn = new, finish_fn(name = _build, vis = ""))] -pub(crate) struct GenLuaRc<'a> { +pub struct GenLuaRc<'a> { config: &'a Config, workspace: &'a Workspace, } @@ -84,19 +84,19 @@ async fn do_generate_luarc(args: GenLuaRc<'_>) -> Result<(), GenLuaRcError> { let dependency_tree = workspace.tree(config)?; let dependency_dirs = lockfile - .local_pkg_lock(&LocalPackageLockType::Regular) + .local_pkg_lock(&LockedPackageLockType::Regular) .rocks() .values() - .map(|dependency| dependency_tree.layout_for(dependency).src) + .map(|dependency| dependency_tree.layout_for(&dependency.spec).src) .filter(|dir| dir.is_dir()) .filter_map(|dependency_dir| diff_paths(dependency_dir, workspace.root())); let test_dependency_tree = workspace.test_tree(config)?; let test_dependency_dirs = lockfile - .local_pkg_lock(&LocalPackageLockType::Test) + .local_pkg_lock(&LockedPackageLockType::Test) .rocks() .values() - .map(|dependency| test_dependency_tree.layout_for(dependency).src) + .map(|dependency| test_dependency_tree.layout_for(&dependency.spec).src) .filter(|dir| dir.is_dir()) .filter_map(|test_dependency_dir| diff_paths(test_dependency_dir, workspace.root())); diff --git a/lux-lib/src/operations/install/mod.rs b/lux-lib/src/operations/install/mod.rs index f98a132ab..2a6ed0923 100644 --- a/lux-lib/src/operations/install/mod.rs +++ b/lux-lib/src/operations/install/mod.rs @@ -1,844 +1,3 @@ -use std::{ - collections::{HashMap, HashSet}, - io, - sync::Arc, -}; - -use crate::{ - build::{Build, BuildBehaviour, BuildError, RemotePackageSourceSpec, SrcRockSource}, - config::Config, - lockfile::{ - FlushLockfileError, LocalPackage, LocalPackageId, LockConstraint, Lockfile, OptState, - PinnedState, ReadOnly, ReadWrite, - }, - lua_installation::{LuaInstallation, LuaInstallationError}, - lua_rockspec::BuildBackendSpec, - lua_version::LuaVersionUnset, - luarocks::{ - install_binary_rock::{BinaryRockInstall, InstallBinaryRockError}, - luarocks_installation::{LuaRocksError, LuaRocksInstallError, LuaRocksInstallation}, - }, - operations::resolve::{ - build_dependencies_to_install, PackageInstallData, Resolve, ResolveDependenciesError, - }, - package::{PackageName, PackageNameList, PackageReq}, - remote_package_db::{RemotePackageDB, RemotePackageDBError}, - rockspec::Rockspec, - tree::{self, InstallTree, Tree, TreeError}, - workspace::{Workspace, WorkspaceTreeError}, -}; - pub use crate::operations::install::spec::PackageInstallSpec; -use super::{DownloadedRockspec, RemoteRockDownload}; -use bon::Builder; -use bytes::Bytes; -use futures::stream::FuturesUnordered; -use futures::StreamExt; -use itertools::Itertools; -use miette::Diagnostic; -use thiserror::Error; -use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; -use tokio::task::{JoinError, JoinHandle}; - -use tracing::Instrument; pub mod spec; - -/// A rocks package installer, providing fine-grained control -/// over how packages should be installed. -/// Can install multiple packages in parallel. -#[derive(Builder)] -#[builder(start_fn = new, finish_fn(name = _build, vis = ""))] -pub struct Install<'a, T> -where - T: InstallTree + Clone + Send + Sync, -{ - #[builder(start_fn)] - config: &'a Config, - #[builder(field)] - packages: Vec, - #[builder(setters(name = "_tree", vis = ""))] - tree: T, - package_db: Option, -} - -impl<'a, State> InstallBuilder<'a, Tree, State> -where - State: install_builder::State, -{ - pub fn workspace( - self, - workspace: &'a Workspace, - ) -> Result>, WorkspaceTreeError> - where - State::Tree: install_builder::IsUnset, - { - let config = self.config; - Ok(self._tree(workspace.tree(config)?)) - } -} - -impl<'a, T, State> InstallBuilder<'a, T, State> -where - State: install_builder::State, - T: InstallTree + Clone + Send + Sync, -{ - pub fn tree(self, tree: T) -> InstallBuilder<'a, T, install_builder::SetTree> - where - State::Tree: install_builder::IsUnset, - { - self._tree(tree) - } - - pub fn packages(self, packages: Vec) -> Self { - Self { packages, ..self } - } - - pub fn package(self, package: PackageInstallSpec) -> Self { - Self { - packages: self - .packages - .into_iter() - .chain(std::iter::once(package)) - .collect(), - ..self - } - } -} - -impl InstallBuilder<'_, T, State> -where - State: install_builder::State + install_builder::IsComplete, - T: InstallTree + Clone + Send + Sync + 'static, -{ - /// Install the packages. - pub async fn install(self) -> Result, InstallError> { - let install_built = self._build(); - if install_built.packages.is_empty() { - return Ok(Vec::default()); - } - let count = install_built.packages.len(); - let span = if count > 1 { - tracing::info_span!("Installing", count) - } else { - let install_spec = &install_built.packages[0]; - tracing::info_span!("Installing", package = install_spec.package.to_string()) - }; - install_impl(install_built).instrument(span).await - } -} - -type InstallWorkerOutput = Result<(LocalPackageId, (LocalPackage, tree::EntryType)), InstallError>; - -#[derive(Error, Debug, Diagnostic)] -pub enum InstallError { - #[error("unable to resolve dependencies")] - #[diagnostic(forward(0))] - ResolveDependencies(#[from] Box), - #[error(transparent)] - #[diagnostic(transparent)] - LuaVersionUnset(#[from] LuaVersionUnset), - #[error(transparent)] - #[diagnostic(transparent)] - LuaInstallation(#[from] LuaInstallationError), - #[error(transparent)] - #[diagnostic(transparent)] - FlushLockfile(#[from] FlushLockfileError), - #[error(transparent)] - #[diagnostic(transparent)] - Tree(#[from] TreeError), - #[error(transparent)] - #[diagnostic(transparent)] - WorkspaceTree(#[from] WorkspaceTreeError), - #[error("error instantiating LuaRocks compatibility layer")] - #[diagnostic(forward(0))] - LuaRocks(#[from] LuaRocksError), - #[error("error installing LuaRocks compatibility layer")] - #[diagnostic(forward(0))] - LuaRocksInstall(#[from] Box), - #[error("failed to build {0}")] - Build(PackageName, #[source] Box), - #[error("failed to install build depencency {0}")] - BuildDependency(PackageName, #[source] Box), - #[error("error initialising remote package DB")] - #[diagnostic(forward(0))] - RemotePackageDB(#[from] RemotePackageDBError), - #[error("failed to install pre-built rock {0}")] - InstallBinaryRock(PackageName, #[source] Box), - #[error("cannot install duplicate entrypoints:\n{0}")] - DuplicateEntrypoints(PackageNameList), - #[error("cannot install conflicting entrypoints:\n{0}")] - #[diagnostic(help( - "only a single entrypoint per package is allowed.\nretry with `--force` to overwrite the existing entrypoint" - ))] - ConflictingEntrypoints(String), - #[error("install worker panicked")] - #[diagnostic(help( - r#"this is a bug in Lux, please report it, ideally with `RUST_BACKTRACE=1`. -retrying with fewer parallel jobs (`--max-jobs`) may avoid the panic in the meantime"# - ))] - Join(#[from] tokio::task::JoinError), -} - -impl From for InstallError { - fn from(source: ResolveDependenciesError) -> Self { - Self::ResolveDependencies(Box::new(source)) - } -} - -impl From for InstallError { - fn from(source: LuaRocksInstallError) -> Self { - Self::LuaRocksInstall(Box::new(source)) - } -} - -async fn install_impl(install: Install<'_, T>) -> Result, InstallError> -where - T: InstallTree + Clone + Send + Sync + 'static, -{ - let package_db = match install.package_db { - Some(db) => db, - None => RemotePackageDB::from_config(install.config).await?, - }; - - let duplicate_entrypoints = install - .packages - .iter() - .filter(|pkg| pkg.entry_type == tree::EntryType::Entrypoint) - .map(|pkg| pkg.package.name()) - .duplicates() - .cloned() - .collect_vec(); - - if !duplicate_entrypoints.is_empty() { - return Err(InstallError::DuplicateEntrypoints(PackageNameList::new( - duplicate_entrypoints, - ))); - } - - let packages = install.packages; - let package_db = Arc::new(package_db); - let config = install.config; - let tree = &install.tree; - - let (dep_tx, mut dep_rx) = tokio::sync::mpsc::unbounded_channel(); - let (build_dep_tx, build_dep_rx) = tokio::sync::mpsc::unbounded_channel(); - let (build_dep_install_done_tx, mut build_dep_install_done_rx) = - tokio::sync::mpsc::unbounded_channel::<(LocalPackage, PackageInstallData)>(); - - let lockfile = tree.lockfile()?; - let build_lockfile = tree.build_tree(config)?.lockfile()?; - - let entrypoint_specs = packages - .iter() - .filter(|spec| spec.entry_type == tree::EntryType::Entrypoint) - .collect_vec(); - - // Entrypoints already installed that conflict with another entrypoint. This enumerates both - // packages that are planned to be removed (`--force`), as well as those which are - // unintentionally causing conflicts. - let conflicting_entrypoints: HashMap = entrypoint_specs - .iter() - .filter_map(|spec| lockfile.entrypoint(spec.package.name()).cloned()) - .map(|package| (package.name().clone(), package)) - .collect(); - - let unforced_conflicts = entrypoint_specs - .iter() - .filter(|spec| spec.build_behaviour != BuildBehaviour::Force) - .filter_map(|spec| conflicting_entrypoints.get(spec.package.name())) - .map(|existing| format!("{}@{}", existing.name(), existing.version())) - .collect_vec(); - - if !unforced_conflicts.is_empty() { - return Err(InstallError::ConflictingEntrypoints( - unforced_conflicts.join("\n"), - )); - } - - // Forced overwrites: remove the conflicting entrypoints from the tree. - // NOTE: non-transactional. If an error occurs, this removes the conflicting package without - // installing the substitute. Fix when transactions are implemented. - let conflicting_entrypoints = conflicting_entrypoints.into_values().collect_vec(); - for package in &conflicting_entrypoints { - tree.cleanup(package, tree::EntryType::Entrypoint)?; - } - - let lua = Arc::new(LuaInstallation::new_from_config(config).await?); - - let mut resolve_worker = spawn_resolve_worker( - config, - packages, - package_db, - lockfile.clone(), - build_lockfile.clone(), - dep_tx, - build_dep_tx, - ); - let mut build_deps_worker = spawn_build_deps_worker( - config, - tree, - lua.clone(), - build_dep_rx, - build_dep_install_done_tx, - ); - - let mut all_packages: HashMap = HashMap::new(); - let mut scheduled_packages: HashSet = HashSet::new(); - let mut installed_packages: HashMap = - HashMap::new(); - let mut installed_build_deps: HashMap = - HashMap::new(); - let mut ongoing_installs: FuturesUnordered< - tracing::instrument::Instrumented>, - > = FuturesUnordered::new(); - let mut resolve_done = false; - let mut build_deps_done = false; - let mut dep_rx_drained = false; - let mut build_dep_rx_drained = false; - let mut install_loop_result: Result<(), InstallError> = Ok(()); - let max_jobs = config.max_jobs(); - - 'install: loop { - if resolve_done && build_deps_done && dep_rx_drained && build_dep_rx_drained { - break; - } - tokio::select! { - resolve_result = &mut resolve_worker, if !resolve_done => { - if let Err(err) = worker_result(resolve_result) { - install_loop_result = Err(*err); - break 'install; - } - resolve_done = true; - } - build_deps_result = &mut build_deps_worker, if !build_deps_done => { - if let Err(err) = worker_result(build_deps_result) { - install_loop_result = Err(*err); - break 'install; - } - build_deps_done = true; - } - build_dep = build_dep_install_done_rx.recv(), if !build_dep_rx_drained => { - if let Some((build_dep, install_data)) = build_dep { - installed_build_deps.insert(build_dep.spec.id(), (build_dep, install_data)); - } else { - build_dep_rx_drained = true; - } - } - dep = dep_rx.recv(), if !dep_rx_drained => { - if let Some(dep) = dep { - all_packages.insert(dep.spec.id(), dep); - } else { - dep_rx_drained = true; - } - } - } - - for (package_id, package_install_data) in ready_to_install( - &all_packages, - &scheduled_packages, - &build_lockfile, - &installed_build_deps - .values() - .map(|(pkg, _)| pkg.name().clone()) - .collect(), - ) { - if max_jobs > 0 && ongoing_installs.len() >= max_jobs { - if let Err(err) = - wait_for_next_install(&mut ongoing_installs, &mut installed_packages).await - { - install_loop_result = Err(err); - break 'install; - } - } - scheduled_packages.insert(package_id); - ongoing_installs.push(spawn_install_worker( - package_install_data, - &lua, - tree, - config, - )); - } - } - - match install_loop_result { - Ok(_) => { - while wait_for_next_install(&mut ongoing_installs, &mut installed_packages).await? {} - - lockfile.map_then_flush(|lockfile| { - for package in &conflicting_entrypoints { - lockfile.remove_by_id(&package.id()); - } - for (package_id, (package, is_entrypoint)) in installed_packages.iter().unique() { - lockfile.add_dependencies( - package_id, - package, - *is_entrypoint, - &all_packages, - &installed_packages, - )?; - lockfile.add_build_dependencies( - package_id, - package, - &all_packages, - &installed_build_deps, - )?; - } - Ok::<_, io::Error>(()) - })?; - - Ok(installed_packages - .into_values() - .map(|(pkg, _)| pkg) - .collect_vec()) - } - Err(err) => { - resolve_worker.into_inner().abort(); - build_deps_worker.into_inner().abort(); - for install in ongoing_installs { - install.into_inner().abort(); - } - Err(err) - } - } -} - -fn spawn_resolve_worker( - config: &Config, - packages: Vec, - package_db: Arc, - lockfile: Lockfile, - build_lockfile: Lockfile, - dep_tx: UnboundedSender, - build_dep_tx: UnboundedSender, -) -> tracing::instrument::Instrumented>> { - tokio::spawn({ - let config = config.clone(); - let lockfile = Arc::new(lockfile); - let build_lockfile = Arc::new(build_lockfile); - async move { - Resolve::new() - .dependencies_tx(dep_tx) - .build_dependencies_tx(build_dep_tx) - .packages(packages) - .package_db(package_db) - .lockfile(lockfile) - .build_lockfile(build_lockfile) - .config(&config) - .get_all_dependencies() - .await?; - Ok::<(), InstallError>(()) - } - }) - .instrument(tracing::trace_span!("resolve_worker")) -} - -fn spawn_build_deps_worker( - config: &Config, - tree: &T, - lua: Arc, - mut build_dep_rx: UnboundedReceiver, - build_dep_install_done_tx: UnboundedSender<(LocalPackage, PackageInstallData)>, -) -> tracing::instrument::Instrumented>> -where - T: InstallTree + Clone + Send + Sync + 'static, -{ - tokio::spawn({ - let config = config.clone(); - let tree = tree.clone(); - let lua = lua.clone(); - async move { - while let Some(build_dep_spec) = build_dep_rx.recv().await { - let rockspec = build_dep_spec.downloaded_rock.rockspec(); - let package = rockspec.package().clone(); - let span = tracing::info_span!( - "Installing build dependency", - package = package.to_string(), - version = rockspec.version().to_string() - ); - let pkg = async { - let build_tree = tree.build_tree(&config)?; - let mut build_lockfile = build_tree.lockfile()?.write_guard(); - let pkg = Build::new() - .rockspec(rockspec) - .lua(&lua) - .tree(&build_tree) - .entry_type(tree::EntryType::Entrypoint) - .config(&config) - .constraint(build_dep_spec.spec.constraint()) - .behaviour(build_dep_spec.build_behaviour) - .build() - .await - .map_err(|err| InstallError::BuildDependency(package, Box::new(err)))?; - build_lockfile.add_entrypoint(&pkg); - Ok::<_, InstallError>(pkg) - } - .instrument(span) - .await?; - let _ = build_dep_install_done_tx.send((pkg, build_dep_spec)); - } - Ok::<(), InstallError>(()) - } - }) - .instrument(tracing::trace_span!("build_deps_worker")) -} - -fn ready_to_install( - all_packages: &HashMap, - scheduled: &HashSet, - build_lockfile: &Lockfile, - installed_build_deps: &HashSet, -) -> Vec<(LocalPackageId, PackageInstallData)> { - all_packages - .iter() - .filter(|(id, data)| { - !scheduled.contains(*id) - && match &data.downloaded_rock { - RemoteRockDownload::BinaryRock { .. } => true, - _ => build_dependencies_ready( - data.downloaded_rock.rockspec(), - data.build_behaviour, - build_lockfile, - installed_build_deps, - ), - } - }) - .map(|(id, data)| (id.clone(), data.clone())) - .collect() -} - -fn spawn_install_worker( - data: PackageInstallData, - lua: &Arc, - tree: &T, - config: &Config, -) -> tracing::instrument::Instrumented> -where - T: InstallTree + Clone + Send + Sync + 'static, -{ - let config = config.clone(); - let tree = tree.clone(); - let lua = lua.clone(); - let entry_type = data.entry_type; - tokio::spawn(async move { - let pkg = install_package(data, &lua, &tree, &config).await?; - Ok::<_, InstallError>((pkg.id(), (pkg, entry_type))) - }) - .instrument(tracing::trace_span!("install_worker")) -} - -#[tracing::instrument(level = "trace", skip_all)] -async fn install_package( - data: PackageInstallData, - lua: &Arc, - tree: &T, - config: &Config, -) -> Result -where - T: InstallTree + Sync, -{ - match data.downloaded_rock { - RemoteRockDownload::RockspecOnly { rockspec_download } => { - install_rockspec( - rockspec_download, - None, - data.spec.constraint(), - data.build_behaviour, - data.pin, - data.opt, - data.entry_type, - lua, - tree, - config, - ) - .await - } - RemoteRockDownload::BinaryRock { - rockspec_download, - packed_rock, - } => { - install_binary_rock( - rockspec_download, - packed_rock, - data.spec.constraint(), - data.build_behaviour, - data.pin, - data.opt, - data.entry_type, - config, - tree, - ) - .await - } - RemoteRockDownload::SrcRock { - rockspec_download, - src_rock, - source_url, - } => { - let src_rock_source = SrcRockSource { - bytes: src_rock, - source_url, - }; - install_rockspec( - rockspec_download, - Some(src_rock_source), - data.spec.constraint(), - data.build_behaviour, - data.pin, - data.opt, - data.entry_type, - lua, - tree, - config, - ) - .await - } - } -} - -fn worker_result( - result: Result, JoinError>, -) -> Result<(), Box> { - match result { - Ok(Ok(())) => Ok(()), - Ok(Err(err)) => Err(err.into()), - Err(join) => Err(InstallError::from(join).into()), - } -} - -async fn wait_for_next_install( - ongoing_installs: &mut FuturesUnordered< - tracing::instrument::Instrumented>, - >, - installed_packages: &mut HashMap, -) -> Result { - if let Some(result) = ongoing_installs.next().await { - match result { - Ok(Ok((id, installed))) => { - installed_packages.insert(id, installed); - Ok(true) - } - Ok(Err(err)) => Err(err), - Err(join) => Err(InstallError::from(join)), - } - } else { - Ok(false) - } -} -trait LockfileExt { - fn add_dependencies( - self, - id: &LocalPackageId, - pkg: &LocalPackage, - entry_type: tree::EntryType, - all_packages: &HashMap, - installed_packages: &HashMap, - ) -> io::Result<()>; - - fn add_build_dependencies( - self, - id: &LocalPackageId, - pkg: &LocalPackage, - all_packages: &HashMap, - build_dependencies: &HashMap, - ) -> io::Result<()>; -} - -impl LockfileExt for &mut Lockfile { - fn add_dependencies( - self, - id: &LocalPackageId, - pkg: &LocalPackage, - entry_type: tree::EntryType, - all_packages: &HashMap, - installed_packages: &HashMap, - ) -> io::Result<()> { - if entry_type == tree::EntryType::Entrypoint { - self.add_entrypoint(pkg); - } - - for dependency_id in all_packages - .get(id) - .map(|pkg| pkg.spec.dependencies()) - .unwrap_or_default() - .into_iter() - { - self.add_dependency( - pkg, - installed_packages - .get(dependency_id) - .map(|(pkg, _)| pkg) - .ok_or(io::Error::other( - r#" -error writing dependencies to the lockfile. -A required dependency was not installed correctly. -This is likely because an install thread panicked and was interrupted unexpectedly. - -[THIS IS A BUG!] -"#, - ))?, - ); - } - Ok(()) - } - - fn add_build_dependencies( - self, - id: &LocalPackageId, - pkg: &LocalPackage, - all_packages: &HashMap, - build_dependencies: &HashMap, - ) -> io::Result<()> { - for dependency_id in all_packages - .get(id) - .map(|pkg| pkg.spec.build_dependencies()) - .unwrap_or_default() - .into_iter() - { - self.add_build_dependency( - pkg, - build_dependencies - .get(dependency_id) - .map(|(pkg, _)| pkg) - .ok_or(io::Error::other( - r#" -error writing build dependencies to the lockfile. -A required build dependency was not installed correctly. -This is likely because an install thread panicked and was interrupted unexpectedly. - -[THIS IS A BUG!] -"#, - ))?, - ); - } - Ok(()) - } -} - -/// Whether all build dependencies of the given rockspec have been installed -/// into the build tree, so that the package can start building. -/// -/// A build dependency is considered ready when it has been freshly installed -/// into the build tree, or when it was already present in the build lockfile -/// and satisfies the dependency constraint. -fn build_dependencies_ready( - rockspec: &impl Rockspec, - behaviour: BuildBehaviour, - build_lockfile: &Lockfile, - installed: &HashSet, -) -> bool { - let build_deps = rockspec.build_dependencies().current_platform(); - build_dependencies_to_install(rockspec).iter().all(|name| { - installed.contains(name) - || (behaviour != BuildBehaviour::Force - && build_lockfile - .has_rock( - &build_deps - .iter() - .find(|dep| dep.name() == name) - .map(|dep| dep.package_req().clone()) - .unwrap_or_else(|| PackageReq::from(name.clone())), - None, - ) - .is_some()) - }) -} - -#[allow(clippy::too_many_arguments)] -#[tracing::instrument( - name = "Installing", - level = "info", - skip_all, - fields( - package = rockspec_download.rockspec.package().to_string(), - version = rockspec_download.rockspec.version().to_string(), - ), -)] -async fn install_rockspec( - rockspec_download: DownloadedRockspec, - src_rock_source: Option, - constraint: LockConstraint, - behaviour: BuildBehaviour, - pin: PinnedState, - opt: OptState, - entry_type: tree::EntryType, - lua: &LuaInstallation, - tree: &T, - config: &Config, -) -> Result -where - T: InstallTree + Sync, -{ - let package = rockspec_download.rockspec.package().clone(); - let rockspec = rockspec_download.rockspec; - let source = rockspec_download.source; - - if let Some(BuildBackendSpec::LuaRock(_)) = &rockspec.build().current_platform().build_backend { - let luarocks_tree = tree.build_tree(config)?; - let luarocks = LuaRocksInstallation::new(config, luarocks_tree)?; - luarocks.ensure_installed(lua).await?; - } - - let source_spec = match src_rock_source { - Some(src_rock_source) => RemotePackageSourceSpec::SrcRock(src_rock_source), - None => RemotePackageSourceSpec::RockSpec(rockspec_download.source_url), - }; - - let pkg = Build::new() - .rockspec(&rockspec) - .lua(lua) - .tree(tree) - .entry_type(entry_type) - .config(config) - .pin(pin) - .opt(opt) - .constraint(constraint) - .behaviour(behaviour) - .source(source) - .source_spec(source_spec) - .build() - .await - .map_err(|err| InstallError::Build(package, Box::new(err)))?; - Ok(pkg) -} - -#[allow(clippy::too_many_arguments)] -#[tracing::instrument( - name = "Installing (pre-built)", - level = "info", - skip_all, - fields( - package = rockspec_download.rockspec.package().to_string(), - version = rockspec_download.rockspec.version().to_string(), - ), -)] -async fn install_binary_rock( - rockspec_download: DownloadedRockspec, - packed_rock: Bytes, - constraint: LockConstraint, - behaviour: BuildBehaviour, - pin: PinnedState, - opt: OptState, - entry_type: tree::EntryType, - config: &Config, - tree: &impl InstallTree, -) -> Result { - let rockspec = rockspec_download.rockspec; - let package = rockspec.package().clone(); - let pkg = BinaryRockInstall::new( - &rockspec, - rockspec_download.source, - packed_rock, - entry_type, - config, - tree, - ) - .pin(pin) - .opt(opt) - .constraint(constraint) - .behaviour(behaviour) - .install() - .await - .map_err(|err| InstallError::InstallBinaryRock(package, Box::new(err)))?; - Ok(pkg) -} diff --git a/lux-lib/src/operations/install/spec.rs b/lux-lib/src/operations/install/spec.rs index 712a76709..6ba76764f 100644 --- a/lux-lib/src/operations/install/spec.rs +++ b/lux-lib/src/operations/install/spec.rs @@ -16,7 +16,6 @@ pub struct PackageInstallSpec { pub(crate) package: PackageReq, #[builder(start_fn)] pub(crate) entry_type: tree::EntryType, - #[builder(default)] pub(crate) build_behaviour: BuildBehaviour, #[builder(default)] pub(crate) pin: PinnedState, diff --git a/lux-lib/src/operations/install_dependencies.rs b/lux-lib/src/operations/install_dependencies.rs deleted file mode 100644 index 16e9332a5..000000000 --- a/lux-lib/src/operations/install_dependencies.rs +++ /dev/null @@ -1,118 +0,0 @@ -use bon::Builder; -use itertools::Itertools; - -use crate::{ - config::Config, - lockfile::LocalPackage, - lua_installation::LuaInstallation, - luarocks::luarocks_installation::LuaRocksInstallation, - operations::{Install, InstallError}, - project::project_toml::LocalProjectToml, - rockspec::Rockspec, - tree::{self, InstallTree}, -}; - -use super::PackageInstallSpec; - -#[derive(Builder)] -#[builder(start_fn = new, finish_fn(name = _build, vis = ""))] -pub(crate) struct InstallDependencies<'a, T> -where - T: InstallTree, -{ - dependencies: Vec, - build_dependencies: Vec, - tree: &'a T, - - lua: &'a LuaInstallation, - luarocks: &'a LuaRocksInstallation, - config: &'a Config, -} - -impl< - T: InstallTree + Sync + Send + Clone + 'static, - State: install_dependencies_builder::State + install_dependencies_builder::IsComplete, - > InstallDependenciesBuilder<'_, T, State> -{ - /// Installs the configured dependencies and build dependencies into [`Self::tree`], - /// returning the installed regular dependencies. - pub(crate) async fn build(self) -> Result, InstallError> { - let args = self._build(); - let config = args.config; - let dependencies = args.dependencies; - let build_dependencies = args.build_dependencies; - let tree = args.tree; - let build_tree = tree.build_tree(config)?; - let lua = args.lua; - let luarocks = args.luarocks; - if !build_dependencies.is_empty() { - luarocks.ensure_installed(lua).await?; - Install::new(config) - .packages(build_dependencies.into_iter().unique().collect_vec()) - .tree(build_tree.clone()) - .install() - .await?; - } - // for some reason, cargo can't infer the type - let dependencies = Install::new(config) - .packages(dependencies.into_iter().unique().collect_vec()) - .tree(tree.clone()) - .install() - .await?; - Ok(dependencies) - } -} - -pub(crate) fn prepare_dependencies_for_build( - project_toml: &LocalProjectToml, - workspace_tree: &impl InstallTree, - dependencies_to_install: &mut Vec, - build_dependencies_to_install: &mut Vec, - entry_type: tree::EntryType, -) { - let dependencies = project_toml - .dependencies() - .current_platform() - .iter() - .cloned() - .collect_vec(); - - let build_dependencies = project_toml - .build_dependencies() - .current_platform() - .iter() - .cloned() - .collect_vec(); - - dependencies - .into_iter() - .filter(|dep| { - workspace_tree - .match_rocks(dep.package_req()) - .is_ok_and(|rock_match| !rock_match.is_found()) - }) - .map(|dep| { - PackageInstallSpec::new(dep.clone().into_package_req(), entry_type) - .pin(*dep.pin()) - .opt(*dep.opt()) - .maybe_source(dep.source().clone()) - .build() - }) - .for_each(|dep| dependencies_to_install.push(dep)); - - build_dependencies - .into_iter() - .filter(|dep| { - workspace_tree - .match_rocks(dep.package_req()) - .is_ok_and(|rock_match| !rock_match.is_found()) - }) - .map(|dep| { - PackageInstallSpec::new(dep.clone().into_package_req(), entry_type) - .pin(*dep.pin()) - .opt(*dep.opt()) - .maybe_source(dep.source().clone()) - .build() - }) - .for_each(|dep| build_dependencies_to_install.push(dep)); -} diff --git a/lux-lib/src/operations/install_project.rs b/lux-lib/src/operations/install_project.rs index fa2bf851e..1932e1d5b 100644 --- a/lux-lib/src/operations/install_project.rs +++ b/lux-lib/src/operations/install_project.rs @@ -1,54 +1,55 @@ -use crate::{ - build::{Build, BuildBehaviour, BuildError}, - config::Config, - lockfile::LocalPackage, - lua_installation::{LuaInstallation, LuaInstallationError}, - luarocks::luarocks_installation::{LuaRocksError, LuaRocksInstallError, LuaRocksInstallation}, - operations::{install_dependencies::prepare_dependencies_for_build, InstallDependencies}, - project::{project_toml::LocalProjectTomlValidationError, Project, ProjectError}, - tree::{self, InstallTree, TreeError}, -}; use bon::Builder; -use itertools::Itertools; use miette::Diagnostic; use thiserror::Error; -use super::InstallError; +use crate::{ + build::{BuildBehaviour, BuildError}, + config::Config, + drivers::install_packages::{InstallPackages, InstallPackagesError}, + hash::HasIntegrity, + lockfile::LockedPackage, + operations::PackageInstallSpec, + package::{PackageName, PackageReq}, + package_db::{PackageDB, PackageDBError}, + pipeline::discover::FoundPackage, + project::{IntoLocalRockspecError, Project, ProjectError}, + rockspec::Rockspec, + tree::{self, InstallTree, TreeError}, + workspace::{Workspace, WorkspaceError}, +}; #[derive(Debug, Error, Diagnostic)] pub enum InstallProjectError { #[error(transparent)] #[diagnostic(transparent)] - LocalProjectTomlValidation(#[from] LocalProjectTomlValidationError), + Project(#[from] ProjectError), #[error(transparent)] #[diagnostic(transparent)] - Project(#[from] ProjectError), + LocalRockspec(#[from] IntoLocalRockspecError), #[error(transparent)] #[diagnostic(transparent)] - LuaInstallation(#[from] LuaInstallationError), + PackageDB(#[from] PackageDBError), #[error(transparent)] #[diagnostic(transparent)] Tree(#[from] TreeError), #[error(transparent)] #[diagnostic(transparent)] - LuaRocks(#[from] LuaRocksError), + Install(#[from] Box), #[error(transparent)] #[diagnostic(transparent)] - LuaRocksInstall(#[from] Box), - #[error("error installind dependencies")] - #[diagnostic(forward(0))] - InstallDependencies(Box), - #[error("error installind build dependencies")] - #[diagnostic(forward(0))] - InstallBuildDependencies(Box), - #[error("error building project")] - #[diagnostic(forward(0))] Build(#[from] Box), + #[error("package '{0}' was not installed")] + PackageNotInstalled(PackageName), + #[error("failed to hash the project's sources")] + Hash(#[source] std::io::Error), + #[error(transparent)] + #[diagnostic(transparent)] + Workspace(#[from] WorkspaceError), } -impl From for InstallProjectError { - fn from(source: LuaRocksInstallError) -> Self { - Self::LuaRocksInstall(Box::new(source)) +impl From for InstallProjectError { + fn from(source: InstallPackagesError) -> Self { + Self::Install(Box::new(source)) } } @@ -72,70 +73,71 @@ where config: &'a Config, tree: &'a T, + + #[builder(default = BuildBehaviour::Force)] + behaviour: BuildBehaviour, + + /// Workspace whose lockfile is preferred during dependency resolution. + workspace: Option<&'a Workspace>, } impl< - T: InstallTree + Sync + Send + Clone + 'static, + T: InstallTree + Sync + Send, State: install_project_builder::State + install_project_builder::IsComplete, > InstallProjectBuilder<'_, T, State> { - /// Returns `Some` if the `only_deps` option is set to `false`. - pub async fn build(self) -> Result { + /// Builds the project's root package, installing its dependencies through + /// the pipeline. Returns the installed root package. + pub async fn build(self) -> Result { let args = self._build(); let config = args.config; let project = args.project; let tree = args.tree; - let build_tree = tree.build_tree(config)?; - let lua = LuaInstallation::new_from_config(config).await?; - let luarocks = LuaRocksInstallation::new(config, build_tree.clone())?; - let mut dependencies_to_install = Vec::new(); - let mut build_dependencies_to_install = Vec::new(); - let project_toml = project.toml().into_local()?; - prepare_dependencies_for_build( - &project_toml, - tree, - &mut dependencies_to_install, - &mut build_dependencies_to_install, - tree::EntryType::DependencyOnly, - ); - let dependencies = InstallDependencies::new() - .dependencies(dependencies_to_install.into_iter().unique().collect_vec()) - .build_dependencies( - build_dependencies_to_install - .into_iter() - .unique() - .collect_vec(), - ) - .tree(tree) - .lua(&lua) - .luarocks(&luarocks) - .config(config) - .build() - .await - .map_err(|err| InstallProjectError::InstallBuildDependencies(Box::new(err)))?; + let rockspec = project.local_remote_rockspec()?; + let name = rockspec.package().clone(); - let package = Build::new() - .rockspec(&project_toml) - .lua(&lua) - .tree(tree) - .entry_type(tree::EntryType::Entrypoint) - .config(config) - .behaviour(BuildBehaviour::Force) - .build() - .await?; + let behaviour = if matches!(args.behaviour, BuildBehaviour::Ignore) { + match tree.lockfile()?.entrypoint(&name) { + Some(existing) => { + let source_hash = project + .root() + .hash() + .await + .map_err(InstallProjectError::Hash)?; + if existing.version() == rockspec.version() + && existing.hashes().source == source_hash + { + return Ok(existing.clone()); + } + BuildBehaviour::Force + } + None => args.behaviour, + } + } else { + args.behaviour + }; - let lockfile = tree.lockfile()?; - let mut lockfile = lockfile.write_guard(); - let build_lockfile = tree.build_tree(config)?.lockfile()?; - - lockfile.add_entrypoint(&package); - for dep in dependencies { - lockfile.add_dependency(&package, &dep); - } - for dep in build_lockfile.rocks().values() { - lockfile.add_build_dependency(&package, dep); + let root = FoundPackage::from_project_root(rockspec, project.root().to_path_buf()); + let package_db = match args.workspace { + Some(workspace) => PackageDB::open(config, workspace).await?, + None => PackageDB::from_config(config).await?, } - Ok(package) + .with_local(vec![root]); + + let install_spec = + PackageInstallSpec::new(PackageReq::from(name.clone()), tree::EntryType::Entrypoint) + .build_behaviour(behaviour) + .build(); + + InstallPackages::new(config, tree) + .package_db(package_db) + .package(install_spec) + .install() + .await? + .0 + .into_iter() + .find(|package| package.name() == &name) + .ok_or(InstallProjectError::PackageNotInstalled(name)) } } diff --git a/lux-lib/src/operations/install_rockspec.rs b/lux-lib/src/operations/install_rockspec.rs index ae74ad96d..113092295 100644 --- a/lux-lib/src/operations/install_rockspec.rs +++ b/lux-lib/src/operations/install_rockspec.rs @@ -3,13 +3,17 @@ use miette::Diagnostic; use thiserror::Error; use crate::{ - build::{self, BuildBehaviour, BuildError}, + build::{BuildBehaviour, BuildError}, config::Config, - lockfile::{LocalPackage, OptState, PinnedState}, - lua_installation::{LuaInstallation, LuaInstallationError}, - lua_rockspec::{BuildBackendSpec, LuaVersionError, RemoteLuaRockspec}, - luarocks::luarocks_installation::{LuaRocksError, LuaRocksInstallError, LuaRocksInstallation}, - operations::{Install, InstallError, PackageInstallSpec}, + drivers::install_packages::{InstallPackages, InstallPackagesError}, + lockfile::{LockedPackage, OptState, PinnedState}, + lua_installation::LuaInstallationError, + lua_rockspec::{LuaVersionError, RemoteLuaRockspec}, + luarocks::luarocks_installation::{LuaRocksError, LuaRocksInstallError}, + operations::PackageInstallSpec, + package::{PackageName, PackageReq}, + package_db::{PackageDB, PackageDBError}, + pipeline::discover::FoundPackage, rockspec::{LuaVersionCompatibility, Rockspec}, tree::{self, InstallTree, TreeError}, }; @@ -23,19 +27,23 @@ pub enum InstallRockspecError { #[diagnostic(transparent)] LuaVersion(#[from] LuaVersionError), #[diagnostic(transparent)] + PackageDB(#[from] PackageDBError), + #[diagnostic(transparent)] Tree(#[from] TreeError), #[diagnostic(transparent)] - Install(#[from] Box), + Install(#[from] Box), #[diagnostic(transparent)] LuaRocks(#[from] LuaRocksError), #[diagnostic(transparent)] LuaRocksInstall(#[from] Box), #[diagnostic(transparent)] Build(#[from] Box), + #[error("package '{0}' was not installed")] + PackageNotInstalled(PackageName), } -impl From for InstallRockspecError { - fn from(source: InstallError) -> Self { +impl From for InstallRockspecError { + fn from(source: InstallPackagesError) -> Self { Self::Install(Box::new(source)) } } @@ -73,106 +81,34 @@ impl< State: install_rockspec_builder::State + install_rockspec_builder::IsComplete, > InstallRockspecBuilder<'_, T, State> { - pub async fn install(self) -> Result { + pub async fn install(self) -> Result { let args = self._build(); let rockspec = args.rockspec; let pin = args.pin; let config = args.config; let tree = args.tree; - let lua_version = rockspec.lua_version_matches(config)?; - let lua = LuaInstallation::new(&lua_version, config).await?; - - // Ensure all dependencies and build dependencies are installed first - - let build_dependencies = rockspec.build_dependencies().current_platform(); - - let build_dependencies_to_install = build_dependencies - .iter() - .filter(|dep| { - // Exclude luarocks build backends that we have implemented in lux - !matches!( - dep.name().to_string().as_str(), - "luarocks-build-rust-mlua" - | "luarocks-build-rust-binary" - | "luarocks-build-treesitter-parser" - ) - }) - .filter(|dep| { - tree.match_rocks(dep.package_req()) - .is_ok_and(|rock_match| rock_match.is_found()) - }) - .map(|dep| { - PackageInstallSpec::new(dep.package_req().clone(), tree::EntryType::Entrypoint) - .build_behaviour(BuildBehaviour::NoForce) - .pin(pin) - .opt(OptState::Required) - .maybe_source(dep.source().clone()) - .build() - }) - .collect(); - - Install::new(config) - .packages(build_dependencies_to_install) - .tree(tree.build_tree(config)?) - .install() - .await?; + rockspec.lua_version_matches(config)?; - let dependencies = rockspec.dependencies().current_platform(); + let name = rockspec.package().clone(); + let root = FoundPackage::from_rockspec(rockspec); + let package_db = PackageDB::from_config(config).await?.with_local(vec![root]); - let mut dependencies_to_install = Vec::new(); - for dep in dependencies { - let rock_match = tree.match_rocks(dep.package_req())?; - if !rock_match.is_found() { - let dep = PackageInstallSpec::new( - dep.package_req().clone(), - tree::EntryType::DependencyOnly, - ) - .build_behaviour(BuildBehaviour::NoForce) + let install_spec = + PackageInstallSpec::new(PackageReq::from(name.clone()), tree::EntryType::Entrypoint) + .build_behaviour(BuildBehaviour::Force) .pin(pin) .opt(OptState::Required) - .maybe_source(dep.source().clone()) .build(); - dependencies_to_install.push(dep); - } - } - let dependencies = Install::new(config) - .packages(dependencies_to_install) - .tree(tree.clone()) + InstallPackages::new(config, tree) + .package_db(package_db) + .package(install_spec) .install() - .await?; - - if let Some(BuildBackendSpec::LuaRock(_)) = - &rockspec.build().current_platform().build_backend - { - let build_tree = tree.build_tree(config)?; - let luarocks = LuaRocksInstallation::new(config, build_tree)?; - luarocks.ensure_installed(&lua).await?; - } - - let package = build::Build::new() - .rockspec(&rockspec) - .tree(tree) - .lua(&lua) - .entry_type(tree::EntryType::Entrypoint) - .config(config) - .pin(pin) - .behaviour(BuildBehaviour::Force) - .build() - .await?; - - let lockfile = tree.lockfile()?; - let build_lockfile = tree.build_tree(config)?.lockfile()?; - - let mut lockfile = lockfile.write_guard(); - lockfile.add_entrypoint(&package); - for dep in dependencies { - lockfile.add_dependency(&package, &dep); - } - for dep in build_lockfile.rocks().values() { - lockfile.add_build_dependency(&package, dep); - } - Ok(package) + .await? + .0 + .into_iter() + .find(|package| package.name() == &name) + .ok_or(InstallRockspecError::PackageNotInstalled(name)) } } diff --git a/lux-lib/src/operations/mod.rs b/lux-lib/src/operations/mod.rs index 580481149..dc98bf181 100644 --- a/lux-lib/src/operations/mod.rs +++ b/lux-lib/src/operations/mod.rs @@ -9,21 +9,19 @@ mod fetch; mod fetch_vendored; mod gen_luarc; pub mod install; -mod install_dependencies; mod install_project; mod install_rockspec; mod pack; mod pin; -mod resolve; mod run; mod run_lua; -mod sync; mod test; mod uninstall; mod unpack; mod update; mod vendor; +pub use crate::drivers::sync::SyncReport; pub use build_lua::*; pub use build_workspace::*; pub use dist_bin::*; @@ -31,16 +29,14 @@ pub use download::*; pub use exec::*; pub use fetch::*; pub(crate) use fetch_vendored::*; -pub use gen_luarc::*; +pub(crate) use gen_luarc::*; pub use install::*; -pub(crate) use install_dependencies::*; pub use install_project::*; pub use install_rockspec::*; pub use pack::*; pub use pin::*; pub use run::*; pub use run_lua::*; -pub use sync::*; pub use test::*; pub use uninstall::*; pub use unpack::*; diff --git a/lux-lib/src/operations/pack.rs b/lux-lib/src/operations/pack.rs index 23dfee32c..a227c109c 100644 --- a/lux-lib/src/operations/pack.rs +++ b/lux-lib/src/operations/pack.rs @@ -1,7 +1,7 @@ use crate::build::utils; use crate::build::utils::c_dylib_extension; use crate::fs; -use crate::lockfile::LocalPackage; +use crate::lockfile::LockedPackage; use crate::luarocks; use crate::luarocks::rock_manifest::DirOrFileEntry; use crate::luarocks::rock_manifest::RockManifest; @@ -13,6 +13,7 @@ use crate::luarocks::rock_manifest::RockManifestLua; use crate::luarocks::rock_manifest::RockManifestRoot; use crate::tree::InstallTree; use crate::tree::Tree; +use crate::tree::TreeError; use bon::Builder; use clean_path::Clean; use itertools::Itertools; @@ -42,7 +43,7 @@ pub struct Pack { #[builder(start_fn)] tree: Tree, #[builder(start_fn)] - package: LocalPackage, + package: LockedPackage, } impl PackBuilder @@ -63,6 +64,9 @@ pub enum PackError { #[diagnostic(transparent)] Fs(#[from] fs::FsError), Walkdir(#[from] walkdir::Error), + #[error(transparent)] + #[diagnostic(transparent)] + Tree(#[from] TreeError), #[error("expected a `package.rockspec` in the package root.")] MissingRockspec, } @@ -70,7 +74,7 @@ pub enum PackError { async fn do_pack(args: Pack) -> Result { let package = args.package; let tree = args.tree; - let layout = tree.layout_for(&package); + let layout = tree.layout_for(&package.spec); let suffix = if is_binary_rock(&layout.lib) { format!("{}.rock", luarocks::current_platform_luarocks_identifier()) } else { @@ -109,7 +113,12 @@ async fn do_pack(args: Pack) -> Result { fs::tokio::copy(layout.rockspec_path(), &renamed_rockspec_entry).await?; let root_entries = add_rock_entries(&mut zip, temp_root_dir.path(), "".into())?; let mut bin_entries = HashMap::new(); - for relative_binary_path in package.spec.binaries() { + let lockfile = tree.lockfile()?; + for relative_binary_path in lockfile + .binaries(&package.id()) + .into_iter() + .flat_map(|binaries| binaries.iter()) + { if let Some(binary_name) = relative_binary_path.clean().file_name() { let binary_path = tree.bin().join(binary_name); if binary_path.is_file() { diff --git a/lux-lib/src/operations/pin.rs b/lux-lib/src/operations/pin.rs index af755579e..3f6a1834d 100644 --- a/lux-lib/src/operations/pin.rs +++ b/lux-lib/src/operations/pin.rs @@ -2,7 +2,7 @@ use std::io; use crate::{ fs, - lockfile::{FlushLockfileError, LocalPackageId, PinnedState}, + lockfile::{FlushLockfileError, LockedPackageId, PinnedState}, package::PackageSpec, tree::{InstallTree, Tree, TreeError}, }; @@ -11,13 +11,13 @@ use itertools::Itertools; use miette::Diagnostic; use thiserror::Error; -// TODO(vhyrro): Differentiate pinned LocalPackages at the type level? +// TODO(vhyrro): Differentiate pinned LockedPackages at the type level? #[derive(Error, Debug, Diagnostic)] pub enum PinError { #[error("package with ID '{0}' not found in the lockfile")] #[diagnostic(help("this is probably a bug"))] - PackageNotFound(LocalPackageId), + PackageNotFound(LockedPackageId), #[error("rock {rock} is already {}pinned!", if *.pin_state == PinnedState::Unpinned { "un" } else { "" })] PinStateUnchanged { pin_state: PinnedState, @@ -38,7 +38,9 @@ pub enum PinError { #[diagnostic(help("make sure Lux has write access to the install directory"))] MoveItemsFailure(#[from] fs_extra::error::Error), #[error("cannot change pin state of {rock}, because it is not an entrypoint")] - #[diagnostic(help("Lux does not allow pinning dependencies, as doing so could break the version requirement in a future update."))] + #[diagnostic(help( + "Lux does not allow pinning dependencies, as doing so could break the version requirement in a future update." + ))] NotAnEntrypoint { rock: PackageSpec }, #[error(transparent)] #[diagnostic(transparent)] @@ -46,12 +48,12 @@ pub enum PinError { } pub fn set_pinned_state( - package_id: &LocalPackageId, + package_id: &LockedPackageId, tree: &Tree, pin: PinnedState, ) -> Result<(), PinError> { let lockfile = tree.lockfile()?; - let mut package = lockfile + let package = lockfile .get(package_id) .ok_or_else(|| PinError::PackageNotFound(package_id.clone()))? .clone(); @@ -70,13 +72,13 @@ pub fn set_pinned_state( } let old_package = package.clone(); - let layout = tree.layout_for(&package); + let layout = tree.layout_for(&package.spec); let items = fs::sync::read_dir(&layout.root)? .filter_map(Result::ok) .map(|dir| dir.path()) .collect_vec(); - package.spec.pinned = pin; + let package = package.repin(pin); if lockfile.get(&package.id()).is_some() { return Err(PinError::PinStateConflict { @@ -90,8 +92,12 @@ pub fn set_pinned_state( fs_extra::move_items(&items, layout.root, &CopyOptions::new())?; lockfile.map_then_flush(|lockfile| { + let binaries = lockfile.binaries(&old_package.id()).cloned(); lockfile.remove(&old_package); lockfile.add_entrypoint(&package); + if let Some(binaries) = binaries { + lockfile.set_binaries(&package, binaries); + } Ok::<_, io::Error>(()) })?; diff --git a/lux-lib/src/operations/resolve.rs b/lux-lib/src/operations/resolve.rs deleted file mode 100644 index 3bcf3c5bb..000000000 --- a/lux-lib/src/operations/resolve.rs +++ /dev/null @@ -1,385 +0,0 @@ -use std::{fmt::Display, sync::Arc}; - -use async_recursion::async_recursion; -use bon::Builder; -use futures::StreamExt; -use itertools::Itertools; -use miette::Diagnostic; -use thiserror::Error; -use tokio::sync::mpsc::UnboundedSender; -use tracing::Instrument; - -use crate::{ - build::BuildBehaviour, - config::Config, - lockfile::{ - LocalPackageId, LocalPackageSpec, Lockfile, LockfilePermissions, OptState, PinnedState, - }, - lua_rockspec::BuildBackendSpec, - operations::{FetchVendored, FetchVendoredError}, - package::{PackageName, PackageReq}, - remote_package_db::RemotePackageDB, - rockspec::Rockspec, - tree, -}; - -use super::{Download, PackageInstallSpec, RemoteRockDownload, SearchAndDownloadError}; - -#[derive(Error, Debug, Diagnostic)] -#[non_exhaustive] -pub enum ResolveDependenciesError { - #[error(transparent)] - #[diagnostic(transparent)] - SearchAndDownload(#[from] Box), - #[error("cyclic dependency detected:\n{0}")] - CyclicDependency(DependencyCycle), - #[error("error processing resolved dependency:\n{0}")] - ChannelSend(String), - #[error("error fetching vendored dependency '{0}'")] - FetchVendored(PackageReq, #[source] Box), -} - -impl From for ResolveDependenciesError { - fn from(source: SearchAndDownloadError) -> Self { - Self::SearchAndDownload(Box::new(source)) - } -} - -#[derive(Debug)] -pub struct DependencyCycle(Vec); - -impl Display for DependencyCycle { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - self.0.iter().join(" -> ").fmt(f) - } -} - -#[derive(Clone, Debug)] -pub(crate) struct PackageInstallData { - pub build_behaviour: BuildBehaviour, - pub pin: PinnedState, - pub opt: OptState, - pub downloaded_rock: RemoteRockDownload, - pub spec: LocalPackageSpec, - pub entry_type: tree::EntryType, -} - -#[derive(Builder)] -#[builder(start_fn = new, finish_fn(name = _build, vis = ""))] -pub(crate) struct Resolve<'a, P> -where - P: LockfilePermissions + Send + Sync + 'static, -{ - dependencies_tx: UnboundedSender, - build_dependencies_tx: UnboundedSender, - packages: Vec, - package_db: Arc, - parent_packages: Option>>, - lockfile: Option>>, - build_lockfile: Option>>, - config: &'a Config, -} - -impl ResolveBuilder<'_, P, State> -where - P: LockfilePermissions + Send + Sync + 'static, - State: resolve_builder::State + resolve_builder::IsComplete, -{ - pub(crate) async fn get_all_dependencies( - self, - ) -> Result, ResolveDependenciesError> { - let args = self._build(); - do_get_all_dependencies(args).await - } -} - -/// The names of the build dependencies to install, including the -/// build backend rock (if any), excluding the luarocks build backends -/// that Lux implements natively. -pub(crate) fn build_dependencies_to_install(rockspec: &R) -> Vec { - let mut names = rockspec - .build_dependencies() - .current_platform() - .iter() - .filter(|dep| { - !matches!( - dep.name().to_string().as_str(), - "luarocks-build-rust-mlua" - | "luarocks-build-rust-binary" - | "luarocks-build-treesitter-parser" - ) - }) - .map(|dep| dep.name().clone()) - .collect_vec(); - - if let Some(backend) = luarocks_build_backend_name(rockspec) { - names.insert(0, backend); - } - names -} - -/// The name of the luarocks build backend rock required to build this rockspec (if any). -pub(crate) fn luarocks_build_backend_name(rockspec: &R) -> Option { - match &rockspec.build().current_platform().build_backend { - Some(BuildBackendSpec::LuaRock(backend)) => { - Some(PackageName::new(format!("luarocks-build-{backend}"))) - } - _ => None, - } -} - -#[tracing::instrument(name = "Resolving dependencies", skip_all)] -#[async_recursion] -async fn do_get_all_dependencies<'a, P>( - args: Resolve<'a, P>, -) -> Result, ResolveDependenciesError> -where - 'a: 'async_recursion, - P: LockfilePermissions + Send + Sync + 'static, -{ - let dependencies_tx = args.dependencies_tx; - let build_dependencies_tx = args.build_dependencies_tx; - let packages = args.packages; - let parent_packages = args.parent_packages.unwrap_or_default(); - let package_db = args.package_db; - let lockfile = args.lockfile; - let build_lockfile = args.build_lockfile; - let config = args.config; - futures::stream::iter( - packages - .into_iter() - // If there is a lockfile, exclude packages that are already installed - .filter( - |PackageInstallSpec { - package, - build_behaviour, - .. - }| { - *build_behaviour == BuildBehaviour::Force - || lockfile - .as_ref() - .is_none_or(|lockfile| lockfile.has_rock(package, None).is_none()) - }, - ) - .map( - // NOTE: we propagate build_behaviour, pin and opt to all dependencies - |PackageInstallSpec { - package, - build_behaviour, - pin, - opt, - entry_type, - constraint, - source, - }| { - let config = config.clone(); - let dependencies_tx = dependencies_tx.clone(); - let build_dependencies_tx = build_dependencies_tx.clone(); - let parent_packages = Arc::clone(&parent_packages); - let package_db = Arc::clone(&package_db); - let lockfile = match &lockfile { - Some(lockfile) => Some(Arc::clone(lockfile)), - None => None, - }; - let build_lockfile = match &build_lockfile { - Some(lockfile) => Some(Arc::clone(lockfile)), - None => None, - }; - - tokio::spawn( - async move { - let downloaded_rock = if let Some(source) = source { - RemoteRockDownload::from_package_req_and_source_spec( - package.clone(), - source, - )? - } else if let Some(vendor_dir) = config.vendor_dir() { - FetchVendored::new() - .vendor_dir(vendor_dir) - .package(&package) - .package_db(&package_db) - .fetch_vendored_rock() - .await - .map_err(|err| { - ResolveDependenciesError::FetchVendored( - package.clone(), - Box::new(err), - ) - })? - } else { - Download::new(&package, &config) - .package_db(&package_db) - .download_remote_rock() - .await? - }; - - let constraint = - constraint.unwrap_or(package.version_req().clone().into()); - - let rockspec = downloaded_rock.rockspec(); - - if parent_packages.contains(rockspec.package()) { - return Err(ResolveDependenciesError::CyclicDependency( - DependencyCycle( - parent_packages - .iter() - .cloned() - .chain(std::iter::once(rockspec.package().clone())) - .collect_vec(), - ), - )); - } - - // NOTE: We don't need to install build dependencies to install binary rocks. - let build_dependencies = if !matches!( - downloaded_rock, - RemoteRockDownload::BinaryRock { .. } - ) { - let packages = build_dependencies_to_install(rockspec) - .into_iter() - .filter_map(|name| { - rockspec - .build_dependencies() - .current_platform() - .iter() - .find(|dep| dep.name() == &name) - .map(|dep| { - // We always install build dependencies as entrypoints - // with regard to the build tree - let entry_type = tree::EntryType::Entrypoint; - PackageInstallSpec::new( - dep.package_req().clone(), - entry_type, - ) - .build_behaviour(build_behaviour) - .pin(pin) - .opt(opt) - .maybe_source(dep.source().clone()) - .build() - }) - // The build backend rock (e.g. `luarocks-build-foo`) - // is not declared as a build dependency in the rockspec. - .or_else(|| { - PackageInstallSpec::new( - PackageReq::from(name), - tree::EntryType::Entrypoint, - ) - .build_behaviour(build_behaviour) - .pin(pin) - .opt(opt) - .build() - .into() - }) - }) - .collect_vec(); - - // NOTE: We treat transitive regular dependencies of build dependencies - // as build dependencies - Resolve::new() - .dependencies_tx(build_dependencies_tx.clone()) - .build_dependencies_tx(build_dependencies_tx.clone()) - .packages(packages) - .parent_packages(Arc::new( - parent_packages - .iter() - .cloned() - .chain(std::iter::once(rockspec.package().clone())) - .collect_vec(), - )) - .package_db(package_db.clone()) - .maybe_lockfile(build_lockfile.clone()) - .maybe_build_lockfile(build_lockfile.clone()) - .config(&config) - .get_all_dependencies() - .await? - } else { - Vec::new() - }; - - let dependencies = rockspec - .dependencies() - .current_platform() - .iter() - .map(|dep| { - // If we're forcing a rebuild, retain the `EntryType` - // of existing dependencies - let entry_type = if build_behaviour == BuildBehaviour::Force - && lockfile.as_ref().is_some_and(|lockfile| { - let installed_rock = - lockfile.has_rock(dep.package_req(), None); - installed_rock.is_some_and(|installed_rock| { - lockfile.is_entrypoint(&installed_rock.id()) - }) - }) { - tree::EntryType::Entrypoint - } else { - tree::EntryType::DependencyOnly - }; - - PackageInstallSpec::new(dep.package_req().clone(), entry_type) - .build_behaviour(build_behaviour) - .pin(pin) - .opt(opt) - .maybe_source(dep.source().clone()) - .build() - }) - .collect_vec(); - - let dependencies = Resolve::new() - .dependencies_tx(dependencies_tx.clone()) - .build_dependencies_tx(build_dependencies_tx) - .packages(dependencies) - .parent_packages(Arc::new( - parent_packages - .iter() - .cloned() - .chain(std::iter::once(rockspec.package().clone())) - .collect_vec(), - )) - .package_db(package_db) - .maybe_lockfile(lockfile) - .maybe_build_lockfile(build_lockfile) - .config(&config) - .get_all_dependencies() - .await?; - - let rockspec = downloaded_rock.rockspec(); - let local_spec = LocalPackageSpec::new( - rockspec.package(), - rockspec.version(), - constraint, - dependencies, - build_dependencies, - &pin, - &opt, - rockspec.binaries(), - ); - - let install_spec = PackageInstallData { - build_behaviour, - pin, - opt, - spec: local_spec.clone(), - downloaded_rock, - entry_type, - }; - - dependencies_tx.send(install_spec).map_err(|err| { - ResolveDependenciesError::ChannelSend(err.to_string()) - })?; - - Ok::<_, ResolveDependenciesError>(local_spec.id()) - } - .instrument(tracing::trace_span!("resolve_worker")), - ) - }, - ), - ) - .buffered(config.max_jobs()) - .collect::>() - .instrument(tracing::trace_span!("resolve_collector")) - .await - .into_iter() - .flatten() - .try_collect() -} diff --git a/lux-lib/src/operations/test.rs b/lux-lib/src/operations/test.rs index f3fe0b28b..b33d1dc07 100644 --- a/lux-lib/src/operations/test.rs +++ b/lux-lib/src/operations/test.rs @@ -1,8 +1,6 @@ use std::{io, ops::Deref, path::PathBuf, process::Command}; -use super::{ - BuildWorkspace, BuildWorkspaceError, Install, InstallError, PackageInstallSpec, Sync, SyncError, -}; +use crate::drivers::sync::{Sync, SyncError, SyncMode, TargetSet}; use crate::fs; use crate::tree::InstallTree; use crate::workspace::{WorkspaceError, WorkspaceTreeError}; @@ -11,11 +9,10 @@ use crate::{ config::{Config, ConfigError}, lua_installation::{LuaBinary, LuaBinaryError}, lua_rockspec::{LuaVersionError, TestSpecError, ValidatedTestSpec}, - package::{PackageName, PackageVersionReqError}, + package::PackageName, path::{Paths, PathsError}, project::{project_toml::LocalProjectTomlValidationError, Project, ProjectError}, rockspec::Rockspec, - tree::{self, TreeError}, workspace::Workspace, }; use bon::Builder; @@ -83,12 +80,9 @@ pub enum RunTestsError { #[error(transparent)] #[diagnostic(transparent)] Config(#[from] ConfigError), - #[error(transparent)] - #[diagnostic(transparent)] - InstallTestDependencies(#[from] Box), #[error("build failed")] #[diagnostic(forward(0))] - BuildWorkspace(#[from] Box), + Sync(#[from] Box), #[error("tests failed!")] #[diagnostic(help("see the test runner's output for details"))] TestFailure, @@ -117,9 +111,6 @@ pub enum RunTestsError { #[error(transparent)] #[diagnostic(transparent)] ProjectTomlValidation(#[from] LocalProjectTomlValidationError), - #[error("failed to sync dependencies")] - #[diagnostic(forward(0))] - Sync(#[from] Box), #[error(transparent)] #[diagnostic(transparent)] TestSpec(#[from] TestSpecError), @@ -131,18 +122,6 @@ pub enum RunTestsError { LuaBinary(#[from] LuaBinaryError), } -impl From for RunTestsError { - fn from(source: InstallTestDependenciesError) -> Self { - Self::InstallTestDependencies(Box::new(source)) - } -} - -impl From for RunTestsError { - fn from(source: BuildWorkspaceError) -> Self { - Self::BuildWorkspace(Box::new(source)) - } -} - impl From for RunTestsError { fn from(source: SyncError) -> Self { Self::Sync(Box::new(source)) @@ -158,10 +137,16 @@ async fn run_tests(test: Test<'_>) -> Result<(), RunTestsError> { if let Some(package) = test.package { let project = workspace.select_member(&package)?; - run_project_tests(&workspace, project, no_lock, &test.args, &test.env, config).await + Box::pin(run_project_tests( + &workspace, project, no_lock, &test.args, &test.env, config, + )) + .await } else { for project in workspace.members() { - run_project_tests(&workspace, project, no_lock, &test.args, &test.env, config).await?; + Box::pin(run_project_tests( + &workspace, project, no_lock, &test.args, &test.env, config, + )) + .await?; } Ok(()) } @@ -179,18 +164,12 @@ async fn run_project_tests( let test_spec = rocks.test().current_platform().to_validated(project)?; let test_config = test_spec.test_config(config)?; - if no_lock { - let rockspec = project.toml().into_local()?; - ensure_test_dependencies(workspace, project, rockspec, &test_config).await?; - } else { - Sync::new(workspace, &test_config).test(true).sync().await?; - } - - BuildWorkspace::new(workspace, &test_config) - .package(project.toml().package().clone()) + Sync::new(workspace, &test_config) + .mode(SyncMode::Open) + .targets(TargetSet::member(project.toml().package().clone(), true)) .no_lock(no_lock) - .only_deps(false) - .build() + .behaviour(BuildBehaviour::Ignore) + .sync() .await?; let lua_version = project.lua_version(&test_config)?; @@ -276,95 +255,7 @@ async fn run_project_tests( } } -#[derive(Error, Debug, Diagnostic)] -#[error("error installing test dependencies")] -#[diagnostic(forward(0))] -pub enum InstallTestDependenciesError { - WorkspaceTree(#[from] WorkspaceTreeError), - Tree(#[from] TreeError), - Install(#[from] Box), - PackageVersionReq(#[from] PackageVersionReqError), -} - -impl From for InstallTestDependenciesError { - fn from(source: InstallError) -> Self { - Self::Install(Box::new(source)) - } -} - -/// Ensure test dependencies are installed -/// This defaults to the local project tree if cwd is a project root. -async fn ensure_test_dependencies( - workspace: &Workspace, - project: &Project, - rockspec: impl Rockspec, - config: &Config, -) -> Result<(), InstallTestDependenciesError> { - let test_tree = workspace.test_tree(config)?; - let rockspec_dependencies = rockspec.test_dependencies().current_platform(); - let test_dependencies = rockspec - .test() - .current_platform() - .test_dependencies(project) - .iter() - .filter(|test_dep| { - !rockspec_dependencies - .iter() - .any(|dep| dep.name() == test_dep.name()) - }) - .filter_map(|dep| { - let build_behaviour = if test_tree - .match_rocks(dep) - .is_ok_and(|matches| matches.is_found()) - { - Some(BuildBehaviour::NoForce) - } else { - Some(BuildBehaviour::Force) - }; - build_behaviour.map(|build_behaviour| { - PackageInstallSpec::new(dep.clone(), tree::EntryType::Entrypoint) - .build_behaviour(build_behaviour) - .build() - }) - }) - .chain( - rockspec_dependencies - .iter() - .filter(|req| !req.name().eq(&PackageName::new("lua".into()))) - .filter_map(|dep| { - let build_behaviour = if test_tree - .match_rocks(dep.package_req()) - .is_ok_and(|matches| matches.is_found()) - { - Some(BuildBehaviour::NoForce) - } else { - Some(BuildBehaviour::Force) - }; - build_behaviour.map(|build_behaviour| { - PackageInstallSpec::new( - dep.package_req().clone(), - tree::EntryType::Entrypoint, - ) - .build_behaviour(build_behaviour) - .pin(*dep.pin()) - .opt(*dep.opt()) - .maybe_source(dep.source.clone()) - .build() - }) - }), - ) - .collect(); - - Install::new(config) - .packages(test_dependencies) - .tree(test_tree) - .install() - .await?; - - Ok(()) -} - -#[cfg(test)] +#[cfg(all(test, feature = "impure_tests"))] mod tests { use std::path::Path; diff --git a/lux-lib/src/operations/uninstall.rs b/lux-lib/src/operations/uninstall.rs index 0920c9ce1..8a9275633 100644 --- a/lux-lib/src/operations/uninstall.rs +++ b/lux-lib/src/operations/uninstall.rs @@ -1,7 +1,7 @@ use std::io; use crate::fs; -use crate::lockfile::{FlushLockfileError, LocalPackage, LocalPackageId}; +use crate::lockfile::{FlushLockfileError, LockedPackage, LockedPackageId}; use crate::lua_version::{LuaVersion, LuaVersionUnset}; use crate::tree::{EntryType, InstallTree, TreeError}; use crate::{config::Config, tree::Tree}; @@ -31,7 +31,7 @@ pub enum RemoveError { #[builder(start_fn = new, finish_fn(name = _build, vis = ""))] pub struct Uninstall<'a> { #[builder(field)] - packages: Vec, + packages: Vec, config: &'a Config, tree: Option, } @@ -43,7 +43,7 @@ where /// Add packages to remove. pub fn packages(self, packages: I) -> Self where - I: IntoIterator, + I: IntoIterator, { Self { packages: self.packages.into_iter().chain(packages).collect_vec(), @@ -52,7 +52,7 @@ where } /// Add a package to the set of packages to remove. - pub fn package(self, package: LocalPackageId) -> Self { + pub fn package(self, package: LockedPackageId) -> Self { self.packages(std::iter::once(package)) } } @@ -62,7 +62,7 @@ where State: uninstall_builder::State + uninstall_builder::IsComplete, { /// Remove the packages. - pub async fn remove(self) -> Result, RemoveError> { + pub async fn remove(self) -> Result, RemoveError> { let args = self._build(); let tree = args.tree.unwrap_or( args.config @@ -74,10 +74,10 @@ where // TODO: Remove dependencies recursively too! async fn remove( - package_ids: Vec, + package_ids: Vec, tree: Tree, config: &Config, -) -> Result, RemoveError> { +) -> Result, RemoveError> { let lockfile = tree.lockfile()?; let packages = package_ids @@ -121,7 +121,7 @@ async fn remove( ), )] async fn remove_package( - package: LocalPackage, + package: LockedPackage, tree: Tree, entry_type: EntryType, ) -> Result<(), RemoveError> { diff --git a/lux-lib/src/operations/update.rs b/lux-lib/src/operations/update.rs index 276b0a118..448875d30 100644 --- a/lux-lib/src/operations/update.rs +++ b/lux-lib/src/operations/update.rs @@ -1,14 +1,15 @@ use std::io; use crate::{ + build::BuildBehaviour, config::Config, lockfile::{ - LocalPackage, LocalPackageLockType, Lockfile, PinnedState, ReadOnly, ReadWrite, + LockedPackage, LockedPackageLockType, Lockfile, PinnedState, ReadOnly, ReadWrite, WorkspaceLockfile, }, lua_version::{LuaVersion, LuaVersionUnset}, package::{PackageReq, RockConstraintUnsatisfied}, - remote_package_db::{RemotePackageDB, RemotePackageDBError}, + package_db::{PackageDB, PackageDBError}, remote_package_source::RemotePackageSource, tree::{self, InstallTree, Tree, TreeError}, workspace::{Workspace, WorkspaceError, WorkspaceTreeError}, @@ -18,7 +19,10 @@ use itertools::Itertools; use miette::Diagnostic; use thiserror::Error; -use super::{Install, InstallError, PackageInstallSpec, RemoveError, SyncError, Uninstall}; +use super::{PackageInstallSpec, RemoveError, Uninstall}; + +use crate::drivers::install_packages::{InstallPackages, InstallPackagesError}; +use crate::drivers::sync::{Sync, SyncError, SyncMode, TargetSet}; #[derive(Error, Debug, Diagnostic)] pub enum UpdateError { @@ -27,13 +31,13 @@ pub enum UpdateError { RockConstraintUnsatisfied(#[from] RockConstraintUnsatisfied), #[error("failed to update rock")] #[diagnostic(forward(0))] - Install(#[from] Box), + Install(#[from] Box), #[error("failed to remove old rock")] #[diagnostic(forward(0))] Remove(#[from] RemoveError), #[error("error initialising remote package DB")] #[diagnostic(forward(0))] - RemotePackageDB(#[from] RemotePackageDBError), + PackageDB(#[from] PackageDBError), #[error("error loading the workspace")] #[diagnostic(forward(0))] Workspace(#[from] WorkspaceError), @@ -53,8 +57,8 @@ pub enum UpdateError { Sync(#[from] Box), } -impl From for UpdateError { - fn from(source: InstallError) -> Self { +impl From for UpdateError { + fn from(source: InstallPackagesError) -> Self { Self::Install(Box::new(source)) } } @@ -93,7 +97,7 @@ pub struct Update<'a> { /// Whether to validate the integrity when syncing the project lockfile. validate_integrity: Option, - package_db: Option, + package_db: Option, } impl UpdateBuilder<'_, State> { @@ -113,9 +117,7 @@ impl UpdateBuilder<'_, State> { impl UpdateBuilder<'_, State> { #[tracing::instrument(name = "Updating packages", skip_all)] - - /// Returns the packages that were installed or removed - pub async fn update(self) -> Result, UpdateError> + pub async fn update(self) -> Result, UpdateError> where State: update_builder::IsComplete, { @@ -132,7 +134,7 @@ impl UpdateBuilder<'_, State> { let package_db = match &args.package_db { Some(db) => db.clone(), None => { - let db = RemotePackageDB::from_config(args.config).await?; + let db = PackageDB::from_config(args.config).await?; db } }; @@ -143,8 +145,8 @@ impl UpdateBuilder<'_, State> { }; match workspace { - Some(workspace) => update_workspace(workspace, args, package_db).await, - None => update_install_tree(args, package_db).await, + Some(workspace) => Box::pin(update_workspace(workspace, args, package_db)).await, + None => Box::pin(update_install_tree(args, package_db)).await, } } } @@ -152,21 +154,22 @@ impl UpdateBuilder<'_, State> { async fn update_workspace( workspace: Workspace, args: Update<'_>, - package_db: RemotePackageDB, -) -> Result, UpdateError> { + package_db: PackageDB, +) -> Result, UpdateError> { let mut project_lockfile = workspace.lockfile()?.write_guard(); let tree = workspace.tree(args.config)?; - let sync_report = super::Sync::new(&workspace, args.config) + let sync_report = Sync::new(&workspace, args.config) + .mode(SyncMode::Frozen) + .targets(TargetSet::all(&workspace, true)) .validate_integrity(args.validate_integrity.unwrap_or(false)) - .test(true) .sync() .await?; let updated_dependencies = update_dependency_tree( tree, &mut project_lockfile, - LocalPackageLockType::Regular, + LockedPackageLockType::Regular, package_db.clone(), args.config, &args.packages, @@ -178,7 +181,7 @@ async fn update_workspace( let updated_test_dependencies = update_dependency_tree( test_tree, &mut project_lockfile, - LocalPackageLockType::Test, + LockedPackageLockType::Test, package_db.clone(), args.config, &args.test_dependencies, @@ -190,7 +193,7 @@ async fn update_workspace( let updated_build_dependencies = update_dependency_tree( build_tree, &mut project_lockfile, - LocalPackageLockType::Build, + LockedPackageLockType::Build, package_db.clone(), args.config, &args.build_dependencies, @@ -209,11 +212,11 @@ async fn update_workspace( async fn update_dependency_tree( tree: Tree, project_lockfile: &mut WorkspaceLockfile, - lock_type: LocalPackageLockType, - package_db: RemotePackageDB, + lock_type: LockedPackageLockType, + package_db: PackageDB, config: &Config, packages: &Option>, -) -> Result, UpdateError> { +) -> Result, UpdateError> { let lockfile = tree.lockfile()?; let dependencies = updatable_packages(&lockfile) .into_iter() @@ -228,7 +231,7 @@ async fn update_dependency_tree( } fn is_included( - (pkg, _): &(LocalPackage, PackageReq), + (pkg, _): &(LockedPackage, PackageReq), package_reqs: &Option>, ) -> bool { package_reqs.is_none() @@ -241,8 +244,8 @@ fn is_included( async fn update_install_tree( args: Update<'_>, - package_db: RemotePackageDB, -) -> Result, UpdateError> { + package_db: PackageDB, +) -> Result, UpdateError> { let tree = args .config .user_tree(LuaVersion::from(args.config)?.clone())?; @@ -255,12 +258,12 @@ async fn update_install_tree( } async fn update( - packages: Vec<(LocalPackage, PackageReq)>, - package_db: RemotePackageDB, + packages: Vec<(LockedPackage, PackageReq)>, + package_db: PackageDB, tree: Tree, lockfile: &Lockfile, config: &Config, -) -> Result, UpdateError> { +) -> Result, UpdateError> { let updatable = packages .clone() .into_iter() @@ -284,14 +287,13 @@ async fn update( .packages(updatable.iter().map(|(package, _)| package.id())) .remove() .await?; - let updated_packages = Install::new(config) + let (updated_packages, _lockfile) = InstallPackages::new(config, &tree) .packages( updatable .iter() .map(|updatable| mk_install_spec(updatable, lockfile)) .collect(), ) - .tree(tree) .package_db(package_db) .install() .await?; @@ -299,7 +301,7 @@ async fn update( } } -fn updatable_packages(lockfile: &Lockfile) -> Vec<(LocalPackage, PackageReq)> { +fn updatable_packages(lockfile: &Lockfile) -> Vec<(LockedPackage, PackageReq)> { lockfile .rocks() .values() @@ -322,7 +324,7 @@ fn updatable_packages(lockfile: &Lockfile) -> Vec<(LocalPackage, Packa } fn mk_install_spec( - (package, req): &(LocalPackage, PackageReq), + (package, req): &(LockedPackage, PackageReq), lockfile: &Lockfile, ) -> PackageInstallSpec { let entry_type = if lockfile.is_entrypoint(&package.id()) { @@ -331,6 +333,7 @@ fn mk_install_spec( tree::EntryType::DependencyOnly }; PackageInstallSpec::new(req.clone(), entry_type) + .build_behaviour(BuildBehaviour::Force) .pin(PinnedState::Unpinned) .opt(package.opt()) .build() diff --git a/lux-lib/src/operations/vendor.rs b/lux-lib/src/operations/vendor.rs index c5d9363c1..761f6d7ac 100644 --- a/lux-lib/src/operations/vendor.rs +++ b/lux-lib/src/operations/vendor.rs @@ -1,4 +1,5 @@ use std::{ + collections::HashMap, io::{self, Cursor}, path::{Path, PathBuf}, process::ExitStatus, @@ -6,33 +7,32 @@ use std::{ }; use bon::Builder; -use bytes::Bytes; use futures::StreamExt; +use futures::TryStreamExt; use itertools::Itertools; use miette::Diagnostic; use path_slash::PathExt; use strum::IntoEnumIterator; use thiserror::Error; use tokio::io::AsyncWriteExt; -use tracing::Instrument; use crate::{ - build::{resolve_source_dir, RemotePackageSourceSpec, SrcRockSource}, + build::{resolve_source_dir, BuildBehaviour}, config::Config, fs, - lockfile::{LocalPackageLockType, ReadOnly, RemotePackageSourceUrl}, + lockfile::LockedPackageLockType, lua_rockspec::{BuildBackendSpec, RemoteLuaRockspec}, - operations::{ - self, - resolve::{ - luarocks_build_backend_name, PackageInstallData, Resolve, ResolveDependenciesError, + operations::{self, PackageInstallSpec, UnpackError}, + package::{PackageReq, PackageSpec}, + package_db::{PackageDB, PackageDBError}, + pipeline::{ + download_sources_and_hash::{ + DownloadSourcesAndHash, DownloadSourcesAndHashError, DownloadedPackage, PackageSource, }, - DownloadedRockspec, FetchSrcError, PackageInstallSpec, UnpackError, + resolve::{luarocks_build_backend_name, ResolveError, ResolvePackageDependencies}, + Artifacts, }, - package::{PackageReq, PackageSpec}, project::project_toml::LocalProjectTomlValidationError, - remote_package_db::{RemotePackageDB, RemotePackageDBError}, - reqwest::RequestError, rockspec::Rockspec, tree::EntryType, workspace::{Workspace, WorkspaceError}, @@ -78,10 +78,13 @@ pub enum VendorError { LocalProjectTomlValidation(#[from] LocalProjectTomlValidationError), #[error("error initialising remote package DB")] #[diagnostic(forward(0))] - RemotePackageDB(#[from] RemotePackageDBError), - #[error("failed to resolve dependencies")] - #[diagnostic(forward(0))] - ResolveDependencies(#[from] Box), + PackageDB(#[from] PackageDBError), + #[error(transparent)] + #[diagnostic(transparent)] + Resolve(#[from] ResolveError), + #[error(transparent)] + #[diagnostic(transparent)] + Download(#[from] DownloadSourcesAndHashError), #[error(transparent)] #[diagnostic(transparent)] Fs(#[from] fs::FsError), @@ -92,10 +95,7 @@ pub enum VendorError { Unpack(#[from] UnpackError), #[error("failed to fetch rock source")] #[diagnostic(forward(0))] - FetchSrc(#[from] FetchSrcError), - #[error("failed to download rock source")] - #[diagnostic(forward(0))] - Request(#[from] RequestError), + FetchSrc(#[from] crate::operations::FetchSrcError), #[error("failed to run `cargo vendor`")] #[diagnostic(help("ensure cargo is installed"))] CargoVendor { source: io::Error }, @@ -108,12 +108,6 @@ pub enum VendorError { }, } -impl From for VendorError { - fn from(source: ResolveDependenciesError) -> Self { - Self::ResolveDependencies(Box::new(source)) - } -} - impl VendorBuilder<'_, State> where State: vendor_builder::State + vendor_builder::IsComplete, @@ -125,61 +119,101 @@ where const CARGO_VENDOR_SUBDIR: &str = "cargo"; -async fn do_vendor_dependencies(args: Vendor<'_>) -> Result<(), VendorError> { - let vendor_dir = args.vendor_dir; - let no_delete = args.no_delete.unwrap_or(false); - let no_lock = args.no_lock.unwrap_or(false); - let target = args.target; - let config = args.config; - let mut all_packages = Vec::new(); - - for lock_type in LocalPackageLockType::iter() { - let (package_db, install_specs) = - mk_resolve_args(lock_type, no_lock, &target, config).await?; - - let (dep_tx, mut dep_rx) = tokio::sync::mpsc::unbounded_channel(); - Resolve::<'_, ReadOnly>::new() - .dependencies_tx(dep_tx.clone()) - .build_dependencies_tx(dep_tx) - .packages(install_specs) - .package_db(Arc::new(package_db)) - .config(config) - .get_all_dependencies() - .await?; - - while let Some(dep) = dep_rx.recv().await { - all_packages.push(dep); - } +/// Resolves the requested packages and downloads (and hashes) their sources. +async fn resolve_and_download( + config: &Config, + package_db: PackageDB, + install_specs: Artifacts>, +) -> Result, VendorError> { + let Artifacts { + regular, + build, + test, + } = install_specs; + let mut resolve = + ResolvePackageDependencies::new(package_db, config).packages(regular.unwrap_or_default()); + if let Some(build) = build { + resolve = resolve.build_packages(build); } + if let Some(test) = test { + resolve = resolve.test_packages(test); + } + let resolved = resolve.resolve().await?; + + let artifacts = DownloadSourcesAndHash::new(config) + .resolved(resolved) + .download_sources_and_hash() + .await?; // The lockfile may contain the same package (name@version) multiple times, // with different constraints. - let all_packages = all_packages + Ok(artifacts .into_iter() - .unique_by(|pkg| (pkg.spec.name().clone(), pkg.spec.version().clone())) - .collect_vec(); + .flat_map(|(_, packages)| packages) + .flat_map(HashMap::into_values) + .unique_by(|pkg| { + ( + pkg.package.spec.name().clone(), + pkg.package.spec.version().clone(), + ) + }) + .collect()) +} - let cargo_deps: Vec<(PackageSpec, Option, Vec)> = all_packages +/// Vendors the sources of all packages into `vendor_dir`. +async fn vendor_sources( + vendor_dir: &Path, + packages: &[DownloadedPackage], + config: &Config, +) -> Result<(), VendorError> { + futures::stream::iter( + packages + .iter() + .map(|package| vendor_package_sources(vendor_dir, package)), + ) + .buffered(config.max_jobs()) + .try_collect() + .await +} + +/// Cargo-based build backends need their Cargo dependencies vendored too. +fn cargo_dependencies( + packages: &[DownloadedPackage], +) -> Vec<(PackageSpec, Option, Vec)> { + packages .iter() - .filter_map(|pkg| { - let rockspec = pkg.downloaded_rock.rockspec(); + .filter_map(|package| { + let rockspec = &package.rockspec; match rockspec.build().current_platform().build_backend { Some(BuildBackendSpec::RustMlua(_) | BuildBackendSpec::RustBinary(_)) => Some(( - pkg.spec.to_package(), + package.package.spec.to_package(), rockspec.source().current_platform().unpack_dir.clone(), rockspec.build().current_platform().copy_directories.clone(), )), _ => None, } }) - .collect(); + .collect() +} + +async fn do_vendor_dependencies(args: Vendor<'_>) -> Result<(), VendorError> { + let vendor_dir = args.vendor_dir; + let no_delete = args.no_delete.unwrap_or(false); + let no_lock = args.no_lock.unwrap_or(false); + let target = args.target; + let config = args.config; + + let (package_db, install_specs) = gather_install_specs(no_lock, &target, config).await?; + let packages = resolve_and_download(config, package_db, install_specs).await?; + + let cargo_deps = cargo_dependencies(&packages); if !no_delete && vendor_dir.exists() { fs::tokio::remove_dir_all(&vendor_dir).await?; } let vendor_dir = Arc::new(vendor_dir); - vendor_sources(vendor_dir.clone(), config.clone(), all_packages).await?; + vendor_sources(&vendor_dir, &packages, config).await?; vendor_target_cargo_deps(&vendor_dir, &target, config).await?; for (dep, unpack_dir, copy_dirs) in cargo_deps { vendor_package_cargo_deps(&vendor_dir, &dep, &unpack_dir, ©_dirs, config).await?; @@ -187,76 +221,82 @@ async fn do_vendor_dependencies(args: Vendor<'_>) -> Result<(), VendorError> { Ok(()) } -async fn mk_resolve_args( - lock_type: LocalPackageLockType, +async fn gather_install_specs( no_lock: bool, target: &VendorTarget, config: &Config, -) -> Result<(RemotePackageDB, Vec), VendorError> { - match &target { - VendorTarget::Workspace(workspace) => { - // Resolve against the project's lockfile if present, otherwise fall - // back to the remote package DB (e.g. for a project that has not - // yet generated a lockfile). - let lockfile = workspace.try_lockfile()?; - let package_db = match lockfile { - Some(lockfile) if !no_lock => lockfile.local_pkg_locks().into(), - _ => RemotePackageDB::from_config(config).await?, - }; - let mut install_specs = Vec::new(); - for project in workspace.members() { - let toml = project.toml().into_local()?; - push_dependencies(&lock_type, &toml, &mut install_specs)?; - if lock_type == LocalPackageLockType::Test { - for test_spec_dependency in toml - .test() - .current_platform() - .test_dependencies(project) - .iter() - .cloned() - .map(|dep| PackageInstallSpec::new(dep, EntryType::Entrypoint).build()) - { - install_specs.push(test_spec_dependency); +) -> Result<(PackageDB, Artifacts>), VendorError> { + // Resolve against the project's lockfile if present, otherwise fall back to + // the remote package DB (e.g. for a project that has not yet generated a lockfile). + let package_db = match target { + VendorTarget::Workspace(workspace) => match workspace.try_lockfile()? { + Some(lockfile) if !no_lock => lockfile.local_pkg_locks().into(), + _ => PackageDB::from_config(config).await?, + }, + VendorTarget::Rockspec(_) => PackageDB::from_config(config).await?, + }; + + let mut install_specs: Artifacts> = Artifacts::default(); + for lock_type in LockedPackageLockType::iter() { + let specs = install_specs + .get_mut(lock_type) + .get_or_insert_with(Vec::new); + match target { + VendorTarget::Workspace(workspace) => { + for project in workspace.members() { + let toml = project.toml().into_local()?; + push_dependencies(&lock_type, &toml, specs)?; + if lock_type == LockedPackageLockType::Test { + specs.extend( + toml.test() + .current_platform() + .test_dependencies(project) + .iter() + .cloned() + .map(|dep| { + PackageInstallSpec::new(dep, EntryType::Entrypoint) + .build_behaviour(BuildBehaviour::Ignore) + .build() + }), + ); } } } - Ok((package_db, install_specs)) - } - VendorTarget::Rockspec(remote_lua_rockspec) => { - let package_db = RemotePackageDB::from_config(config).await?; - let mut install_specs = Vec::new(); - push_dependencies(&lock_type, remote_lua_rockspec, &mut install_specs)?; - Ok((package_db, install_specs)) + VendorTarget::Rockspec(rockspec) => { + push_dependencies(&lock_type, rockspec, specs)?; + } } } + + Ok((package_db, install_specs)) } fn push_dependencies( - lock_type: &LocalPackageLockType, + lock_type: &LockedPackageLockType, rockspec: &R, install_specs: &mut Vec, ) -> Result<(), LocalProjectTomlValidationError> { let mut dependencies: Vec = match lock_type { - LocalPackageLockType::Regular => rockspec + LockedPackageLockType::Regular => rockspec .dependencies() .current_platform() .iter() .map(|dep| dep.package_req().clone()) .collect_vec(), - LocalPackageLockType::Test => rockspec + LockedPackageLockType::Test => rockspec .test_dependencies() .current_platform() .iter() .map(|dep| dep.package_req().clone()) .collect_vec(), - LocalPackageLockType::Build => rockspec + LockedPackageLockType::Build => rockspec .build_dependencies() .current_platform() .iter() .map(|dep| dep.package_req().clone()) .collect_vec(), }; - if *lock_type == LocalPackageLockType::Build { + if *lock_type == LockedPackageLockType::Build { if let Some(backend) = luarocks_build_backend_name(rockspec) { dependencies.insert(0, backend.into()); } @@ -265,167 +305,62 @@ fn push_dependencies( dependencies .into_iter() .unique() - .map(|dep| PackageInstallSpec::new(dep, EntryType::Entrypoint).build()) + .map(|dep| { + PackageInstallSpec::new(dep, EntryType::Entrypoint) + .build_behaviour(BuildBehaviour::Ignore) + .build() + }) .collect_vec(), ); Ok(()) } -async fn vendor_sources( - vendor_dir: Arc, - config: Config, - packages: Vec, -) -> Result<(), VendorError> { - futures::stream::iter(packages.into_iter().map(|dep| { - let vendor_dir = Arc::clone(&vendor_dir); - let config = config.clone(); - tokio::spawn( - async move { - match dep.downloaded_rock { - crate::operations::RemoteRockDownload::RockspecOnly { rockspec_download } => { - vendor_rockspec_sources(&vendor_dir, rockspec_download, None, &config) - .await? - } - crate::operations::RemoteRockDownload::BinaryRock { - rockspec_download, - packed_rock, - } => vendor_binary_rock(&vendor_dir, rockspec_download, packed_rock).await?, - crate::operations::RemoteRockDownload::SrcRock { - rockspec_download, - src_rock, - source_url, - } => { - let src_rock_source = SrcRockSource { - bytes: src_rock, - source_url, - }; - vendor_rockspec_sources( - &vendor_dir, - rockspec_download, - Some(src_rock_source), - &config, - ) - .await? - } - }; - Ok::<_, VendorError>(()) - } - .instrument(tracing::trace_span!("vendor_worker")), - ) - })) - .buffered(config.max_jobs()) - .collect::>() - .instrument(tracing::trace_span!("vendor_collector")) - .await - .into_iter() - .flatten() - .try_collect() -} - #[tracing::instrument( name = "Vendoring source", level = "info", skip_all, fields( - package = rockspec_download.rockspec.package().to_string(), - version = rockspec_download.rockspec.version().to_string(), + package = package.package.spec.name().to_string(), + version = package.package.spec.version().to_string(), ), )] -async fn vendor_rockspec_sources( +async fn vendor_package_sources( vendor_dir: &Path, - rockspec_download: DownloadedRockspec, - src_rock_source: Option, - config: &Config, + package: &DownloadedPackage, ) -> Result<(), VendorError> { - let rockspec = rockspec_download.rockspec; - let package = rockspec.package(); + let rockspec = &package.rockspec; + let name = rockspec.package(); let version = rockspec.version(); - let package_version_str = format!("{}@{}", package, version); - - let source_spec = match src_rock_source { - Some(src_rock_source) => RemotePackageSourceSpec::SrcRock(src_rock_source), - None => RemotePackageSourceSpec::RockSpec(rockspec_download.source_url), - }; - - let source_path = vendor_dir.join(&package_version_str); fs::tokio::create_dir_all(vendor_dir).await?; let rockspec_lua_content = rockspec .to_lua_remote_rockspec_string() .map_err(|err| VendorError::LuaRockSpec(err.to_string()))?; - - let rockspec_file_name = format!("{}-{}.rockspec", package, version); - let rockspec_path = vendor_dir.join(rockspec_file_name); + let rockspec_path = vendor_dir.join(format!("{}-{}.rockspec", name, version)); fs::tokio::write(&rockspec_path, rockspec_lua_content).await?; - match source_spec { - RemotePackageSourceSpec::SrcRock(SrcRockSource { - bytes, - source_url: _, - }) => { - fs::tokio::write(&source_path, &bytes).await?; + let source_path = vendor_dir.join(format!("{}@{}", name, version)); + match &package.artifact { + PackageSource::SourceDir(path) => { + fs::tokio::remove_dir_all(&source_path).await.ok(); + fs::tokio::copy_dir_all(path, &source_path).await?; + } + PackageSource::PackedRock(bytes) => { + let rock_path = vendor_dir.join(format!("{}@{}.rock", name, version)); + let mut file = fs::tokio::create(&rock_path).await?; + file.write_all(bytes) + .await + .map_err(|source| fs::FsError::Write { + path: rock_path, + source, + })?; } - RemotePackageSourceSpec::RockSpec(source_url) => match source_url { - Some(RemotePackageSourceUrl::Url { url }) => { - let bytes = crate::reqwest::download_bytes(config, &url).await?; - fs::tokio::write(&source_path, &bytes).await?; - } - _ => { - fs::tokio::create_dir_all(&source_path).await?; - operations::FetchSrc::new(&source_path, &rockspec, config) - .maybe_source_url(source_url) - .fetch_internal() - .await?; - } - }, } Ok(()) } -#[tracing::instrument( - name = "Vendoring pre-built binary", - level = "info", - skip_all, - fields( - package = rockspec_download.rockspec.package().to_string(), - version = rockspec_download.rockspec.version().to_string(), - ), -)] -async fn vendor_binary_rock( - vendor_dir: &Path, - rockspec_download: DownloadedRockspec, - packed_rock: Bytes, -) -> Result<(), VendorError> { - let rockspec = rockspec_download.rockspec; - let package = rockspec.package(); - let version = rockspec.version(); - - let file_name = format!("{}@{}.rock", package, version); - - fs::tokio::create_dir_all(&vendor_dir).await?; - - let dest_file = vendor_dir.join(&file_name); - let mut file = fs::tokio::create(&dest_file).await?; - file.write_all(&packed_rock) - .await - .map_err(|source| fs::FsError::Write { - path: dest_file.to_path_buf(), - source, - })?; - - let rockspec_lua_content = rockspec - .to_lua_remote_rockspec_string() - .map_err(|err| VendorError::LuaRockSpec(err.to_string()))?; - - let rockspec_file_name = format!("{}-{}.rockspec", package, version); - let rockspec_path = vendor_dir.join(rockspec_file_name); - fs::tokio::write(&rockspec_path, rockspec_lua_content).await?; - - Ok(()) -} - #[tracing::instrument(name = "Vendoring cargo dependencies", level = "info", skip_all)] async fn vendor_target_cargo_deps( vendor_dir: &Path, @@ -484,8 +419,6 @@ async fn vendor_package_cargo_deps( ) .await?; } else if source_dir.is_file() { - // The vendored source is an archive; extract it so `cargo vendor` can - // read its `Cargo.toml`. Keep the temp dir alive while cargo runs. let temp_dir = fs::tempfile::tempdir()?; extract_source_archive(&source_dir, unpack_dir.as_deref(), temp_dir.path()).await?; cargo_vendor( @@ -565,3 +498,164 @@ async fn cargo_vendor( }) } } + +#[cfg(test)] +mod tests { + use tempfile::TempDir; + + use super::*; + use crate::{ + build::BuildBehaviour, + lockfile::{LockConstraint, LockedPackage, LockedPackageSpec, OptState, PinnedState}, + remote_package_source::RemotePackageSource, + }; + #[cfg(feature = "impure_tests")] + use {crate::config::ConfigBuilder, assert_fs::prelude::PathCopy, std::path::PathBuf}; + + #[cfg(feature = "impure_tests")] + fn test_config() -> Config { + ConfigBuilder::new() + .unwrap() + .lua_version(Some(crate::lua_version::LuaVersion::Lua51)) + .build() + .unwrap() + } + + fn make_downloaded_package( + rockspec: RemoteLuaRockspec, + source: RemotePackageSource, + artifact: PackageSource, + temp_dir: Option, + ) -> DownloadedPackage { + let spec = LockedPackageSpec::new( + rockspec.package(), + rockspec.version(), + LockConstraint::Unconstrained, + Vec::new(), + Vec::new(), + &PinnedState::Unpinned, + &OptState::Required, + ); + let package = LockedPackage::new( + spec, + source, + None, + crate::lockfile::LockedPackageHashes { + rockspec: "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + .parse() + .unwrap(), + source: "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + .parse() + .unwrap(), + }, + ); + DownloadedPackage { + package, + rockspec, + entry_type: EntryType::Entrypoint, + build_behaviour: BuildBehaviour::Ignore, + artifact, + temp_dir, + } + } + + #[tokio::test] + async fn vendor_source_tree_writes_rockspec_and_source_dir() { + let vendor_dir = assert_fs::TempDir::new().unwrap(); + + let src_dir = TempDir::new().unwrap(); + std::fs::write(src_dir.path().join("hello.lua"), "return 'hello'").unwrap(); + let rockspec = RemoteLuaRockspec::from_package_and_source_spec( + "rockspec-only-url@1.0.0".parse().unwrap(), + crate::lua_rockspec::RockSourceSpec::File(src_dir.path().to_path_buf()), + ); + let artifact = PackageSource::SourceDir(src_dir.path().to_path_buf()); + let package = make_downloaded_package( + rockspec, + RemotePackageSource::RockspecContent(String::new()), + artifact, + Some(src_dir), + ); + + vendor_package_sources(vendor_dir.path(), &package) + .await + .unwrap(); + + let rockspec_file = vendor_dir.path().join("rockspec-only-url-1.0.0-1.rockspec"); + assert!(rockspec_file.is_file(), "rockspec not vendored"); + let rockspec_content = std::fs::read_to_string(&rockspec_file).unwrap(); + assert!( + rockspec_content.contains("rockspec-only-url"), + "vendored rockspec is not for rockspec-only-url:\n{rockspec_content}" + ); + + let source_path = vendor_dir.path().join("rockspec-only-url@1.0.0-1"); + assert!(source_path.is_dir(), "source tree not vendored"); + assert!( + source_path.join("hello.lua").is_file(), + "source tree contents not vendored" + ); + } + + #[tokio::test] + async fn vendor_source_tree_replaces_stale_source_dir() { + let vendor_dir = assert_fs::TempDir::new().unwrap(); + let stale_dir = vendor_dir.path().join("rockspec-only-url@1.0.0-1"); + std::fs::create_dir_all(&stale_dir).unwrap(); + std::fs::write(stale_dir.join("stale.lua"), "return 'stale'").unwrap(); + + let src_dir = TempDir::new().unwrap(); + std::fs::write(src_dir.path().join("hello.lua"), "return 'hello'").unwrap(); + let rockspec = RemoteLuaRockspec::from_package_and_source_spec( + "rockspec-only-url@1.0.0".parse().unwrap(), + crate::lua_rockspec::RockSourceSpec::File(src_dir.path().to_path_buf()), + ); + let artifact = PackageSource::SourceDir(src_dir.path().to_path_buf()); + let package = make_downloaded_package( + rockspec, + RemotePackageSource::RockspecContent(String::new()), + artifact, + Some(src_dir), + ); + + vendor_package_sources(vendor_dir.path(), &package) + .await + .unwrap(); + + assert!( + !stale_dir.join("stale.lua").exists(), + "stale vendored source was not replaced" + ); + assert!(stale_dir.join("hello.lua").is_file()); + } + + #[cfg(feature = "impure_tests")] + #[tokio::test] + async fn vendor_workspace_no_delete_preserves_existing_contents() { + let sample_project_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("resources/test/sample-projects/busted-with-lockfile/"); + let temp_dir = assert_fs::TempDir::new().unwrap(); + temp_dir + .copy_from(&sample_project_dir, &["**"]) + .expect("failed to copy sample project"); + let workspace = Workspace::from_exact(temp_dir.path()).unwrap().unwrap(); + let config = test_config(); + let vendor_dir = assert_fs::TempDir::new().unwrap(); + let marker = vendor_dir.path().join("EXISTING_FILE"); + std::fs::write(&marker, "keep me").unwrap(); + + Vendor::new() + .target(VendorTarget::Workspace(workspace)) + .vendor_dir(vendor_dir.to_path_buf()) + .no_delete(true) + .config(&config) + .vendor_dependencies() + .await + .unwrap(); + + assert!( + marker.is_file(), + "no_delete should have preserved existing contents" + ); + } +} diff --git a/lux-lib/src/package/outdated.rs b/lux-lib/src/package/outdated.rs index 226ffe821..a6391ca38 100644 --- a/lux-lib/src/package/outdated.rs +++ b/lux-lib/src/package/outdated.rs @@ -1,6 +1,6 @@ use std::fmt::Display; -use crate::remote_package_db::RemotePackageDB; +use crate::package_db::PackageDB; use miette::Diagnostic; use thiserror::Error; @@ -22,7 +22,7 @@ impl PackageSpec { /// Returns the latest version if found. pub fn has_update( &self, - package_db: &RemotePackageDB, + package_db: &PackageDB, ) -> Result, RockNotFound> { let latest_version = package_db .latest_version(&self.name) @@ -40,7 +40,7 @@ impl PackageSpec { pub fn has_update_with( &self, constraint: &PackageReq, - package_db: &RemotePackageDB, + package_db: &PackageDB, ) -> Result, RockConstraintUnsatisfied> { let latest_version = package_db diff --git a/lux-lib/src/package_db/mod.rs b/lux-lib/src/package_db/mod.rs new file mode 100644 index 000000000..afeff7c37 --- /dev/null +++ b/lux-lib/src/package_db/mod.rs @@ -0,0 +1,361 @@ +use crate::{ + config::{Config, ConfigError}, + lockfile::{LockfileIntegrityError, PackageLock}, + manifest::{Manifest, ManifestError}, + operations::{Download, FetchVendored, PackageInstallSpec, RemoteRockDownload}, + package::{ + PackageName, PackageReq, PackageSpec, PackageVersion, RemotePackage, + RemotePackageTypeFilterSpec, + }, + pipeline::{ + discover::{DiscoverError, FoundPackage}, + download_sources_and_hash::PackageSource, + }, + workspace::{Workspace, WorkspaceError}, +}; +use itertools::Itertools; + +use miette::Diagnostic; +use thiserror::Error; + +/// Package database, used to look up packages. +/// Lookup order: local packages, locked packages, manifests. +#[derive(Clone, Debug)] +pub struct PackageDB { + local: Vec, + locks: Vec, + remote_manifests: Vec, +} + +#[derive(Error, Debug, Diagnostic)] +pub enum PackageDBError { + #[error(transparent)] + #[diagnostic(transparent)] + ManifestError(#[from] ManifestError), + #[error(transparent)] + #[diagnostic(transparent)] + ConfigError(#[from] ConfigError), + #[error(transparent)] + #[diagnostic(transparent)] + Workspace(Box), +} + +impl From for PackageDBError { + fn from(err: WorkspaceError) -> Self { + Self::Workspace(Box::new(err)) + } +} + +#[derive(Error, Debug, Diagnostic)] +#[non_exhaustive] +pub enum SearchError { + #[error("no rock that matches '{0}' found")] + RockNotFound(PackageReq), + #[error("no rock that matches '{0}' found in the lockfile.")] + RockNotFoundInLockfile(PackageReq), + #[error("error when pulling manifest")] + #[diagnostic(forward(0))] + Manifest(#[from] ManifestError), +} + +#[derive(Error, Debug, Diagnostic)] +#[non_exhaustive] +pub enum RemotePackageDbIntegrityError { + #[error(transparent)] + #[diagnostic(transparent)] + Lockfile(#[from] LockfileIntegrityError), +} + +impl PackageDB { + #[tracing::instrument(level = "trace", skip_all)] + pub async fn from_config(config: &Config) -> Result { + let mut manifests = Vec::new(); + for server in config.enabled_dev_servers()? { + let manifest = Manifest::from_config(server, config).await?; + manifests.push(manifest); + } + for server in config.extra_servers() { + let manifest = Manifest::from_config(server.clone(), config).await?; + manifests.push(manifest); + } + manifests.push(Manifest::from_config(config.server().clone(), config).await?); + Ok(Self { + local: Vec::new(), + locks: Vec::new(), + remote_manifests: manifests, + }) + } + + /// Builds a package database that prefers the workspace lockfile, falling + /// back to the manifests for requirements the lockfile does not satisfy. + pub(crate) async fn open( + config: &Config, + workspace: &Workspace, + ) -> Result { + Ok(Self::from_config(config) + .await? + .with_locks(workspace.lockfile()?.local_pkg_locks())) + } + + #[cfg(test)] + pub(crate) fn from_local(packages: Vec) -> Self { + Self { + local: packages, + locks: Vec::new(), + remote_manifests: Vec::new(), + } + } + + /// Sets the local packages to look up first. + pub(crate) fn with_local(mut self, packages: Vec) -> Self { + self.local = packages; + self + } + + /// Sets the locked packages to look up after local packages, ahead of manifests. + pub(crate) fn with_locks(mut self, locks: Vec) -> Self { + self.locks = locks; + self + } + + fn find_in_local(&self, package_req: &PackageReq) -> Option { + self.local + .iter() + .find(|package| package.package.package.name() == package_req.name()) + .map(|package| package.package.clone()) + } + + fn find_in_locks( + &self, + package_req: &PackageReq, + filter: Option, + ) -> Option { + self.locks + .iter() + .filter_map(|lock| lock.has_rock(package_req, filter.clone())) + .map(|local_package| { + RemotePackage::new( + local_package.to_package(), + local_package.source().clone(), + local_package.source_url().cloned(), + ) + }) + .next() + } + + fn find_in_manifests( + &self, + package_req: &PackageReq, + filter: Option, + ) -> Option { + self.remote_manifests + .iter() + .find_map(|manifest| manifest.find(package_req, filter.clone())) + } + + /// Find a remote package that matches the requirement, returning the latest match. + #[tracing::instrument( + name = "Searching tree-sitter parser", + level = "info", + skip_all, + fields(package = package_req.to_string()), + )] + pub(crate) fn find( + &self, + package_req: &PackageReq, + filter: Option, + ) -> Result { + if let Some(package) = self.find_in_local(package_req) { + return Ok(package); + } + if let Some(package) = self.find_in_locks(package_req, filter.clone()) { + return Ok(package); + } + if let Some(package) = self.find_in_manifests(package_req, filter) { + return Ok(package); + } + if !self.remote_manifests.is_empty() { + Err(SearchError::RockNotFound(package_req.clone())) + } else if !self.locks.is_empty() { + Err(SearchError::RockNotFoundInLockfile(package_req.clone())) + } else { + Err(SearchError::RockNotFound(package_req.clone())) + } + } + + /// Resolves a package install spec into a fully discovered package. + pub(crate) async fn resolve( + &self, + spec: &PackageInstallSpec, + config: &Config, + ) -> Result { + if let Some(source) = &spec.source { + let download = RemoteRockDownload::from_package_req_and_source_spec( + spec.package.clone(), + source.clone(), + ) + .map_err(|err| DiscoverError::Download(spec.package.clone(), Box::new(err)))?; + let rockspec = download.rockspec().clone(); + let package = RemotePackage::new( + spec.package.clone().try_into().map_err(|err| { + DiscoverError::Download( + spec.package.clone(), + Box::new(crate::operations::SearchAndDownloadError::from(err)), + ) + })?, + download.rockspec_download().source.clone(), + download.rockspec_download().source_url.clone(), + ); + return Ok(FoundPackage { + package, + rockspec, + artifact: None, + }); + } + + if let Some(package) = self + .local + .iter() + .find(|package| package.package.package.name() == spec.package.name()) + { + return Ok(package.clone()); + } + + if !self.locks.is_empty() || !self.remote_manifests.is_empty() { + let mut package = self.find(&spec.package, None)?; + let download = if let Some(vendor_dir) = config.vendor_dir() { + FetchVendored::new() + .vendor_dir(vendor_dir) + .package(&spec.package) + .package_db(self) + .fetch_vendored_rock() + .await + .map_err(|err| { + DiscoverError::FetchVendored(spec.package.clone(), Box::new(err)) + })? + } else { + Download::new(&spec.package, config) + .package_db(self) + .download_remote_rock() + .await + .map_err(|err| DiscoverError::Download(spec.package.clone(), Box::new(err)))? + }; + let rockspec = download.rockspec().clone(); + let artifact = match &download { + RemoteRockDownload::SrcRock { + src_rock, + source_url, + .. + } => { + package.source_url = Some(source_url.clone()); + Some(PackageSource::PackedRock(src_rock.clone())) + } + RemoteRockDownload::BinaryRock { packed_rock, .. } => { + Some(PackageSource::PackedRock(packed_rock.clone())) + } + RemoteRockDownload::RockspecOnly { .. } => None, + }; + return Ok(FoundPackage { + package, + rockspec, + artifact, + }); + } + + Err(DiscoverError::Search(SearchError::RockNotFound( + spec.package.clone(), + ))) + } + + /// Search for all packages that match the requirement. + pub fn search(&self, package_req: &PackageReq) -> Vec<(&PackageName, Vec<&PackageVersion>)> { + self.local + .iter() + .filter_map(|package| { + let name = package.package.package.name(); + if name.to_string().contains(&package_req.name().to_string()) + && package_req + .version_req() + .matches(package.package.package.version()) + { + Some((name, vec![package.package.package.version()])) + } else { + None + } + }) + .chain( + self.locks + .iter() + .flat_map(|lock| lock.rocks().values()) + .filter_map(|package| { + // NOTE: This doesn't group packages by name, but we don't care for now, + // as we shouldn't need to use this function with a lockfile. + let name = package.name(); + if name.to_string().contains(&package_req.name().to_string()) { + Some((name, vec![package.version()])) + } else { + None + } + }), + ) + .chain(self.remote_manifests.iter().flat_map(|manifest| { + manifest + .metadata() + .repository + .iter() + .filter_map(|(name, elements)| { + if name.to_string().contains(&package_req.name().to_string()) { + Some(( + name, + elements + .keys() + .filter(|version| package_req.version_req().matches(version)) + .sorted_by(|a, b| Ord::cmp(b, a)) + .collect_vec(), + )) + } else { + None + } + }) + })) + .collect() + } + + /// Find the latest version for a package by name. + pub(crate) fn latest_version(&self, rock_name: &PackageName) -> Option { + self.latest_match(&rock_name.clone().into(), None) + .map(|result| result.version().clone()) + } + + /// Find the latest package that matches the requirement. + pub fn latest_match( + &self, + package_req: &PackageReq, + filter: Option, + ) -> Option { + match self.find(package_req, filter) { + Ok(result) => Some(result.package), + Err(_) => None, + } + } +} + +impl From for PackageDB { + fn from(manifest: Manifest) -> Self { + Self { + local: Vec::new(), + locks: Vec::new(), + remote_manifests: vec![manifest], + } + } +} + +impl From> for PackageDB { + fn from(locks: Vec) -> Self { + Self { + local: Vec::new(), + locks, + remote_manifests: Vec::new(), + } + } +} diff --git a/lux-lib/src/path.rs b/lux-lib/src/path.rs index d5670f441..d41cb108a 100644 --- a/lux-lib/src/path.rs +++ b/lux-lib/src/path.rs @@ -45,7 +45,7 @@ impl Paths { pub fn new(tree: &impl InstallTree) -> Result { let mut paths = Self::default(tree); for package in tree.list()?.values().flatten() { - let layout = tree.layout_for(package); + let layout = tree.layout_for(&package.spec); paths.src.0.push(layout.src.join("?.lua")); paths.src.0.push(layout.src.join("?").join("init.lua")); paths diff --git a/lux-lib/src/pipeline/build.rs b/lux-lib/src/pipeline/build.rs new file mode 100644 index 000000000..3dfb2f57a --- /dev/null +++ b/lux-lib/src/pipeline/build.rs @@ -0,0 +1,207 @@ +use std::collections::{HashMap, HashSet}; + +use bon::Builder; +use miette::Diagnostic; +use thiserror::Error; + +use crate::{ + build::deploy, + config::Config, + lockfile::{LockedPackage, LockedPackageId}, + lua_installation::{LuaInstallation, LuaInstallationError}, + lua_rockspec::BuildBackendSpec, + luarocks::{ + install_binary_rock::{BinaryRockInstall, InstallBinaryRockError}, + luarocks_installation::{LuaRocksError, LuaRocksInstallError, LuaRocksInstallation}, + }, + rockspec::Rockspec, + tree::{InstallTree, TreeError}, +}; + +use super::download_sources_and_hash::{DownloadedPackage, PackageSource}; + +#[derive(Error, Debug, Diagnostic)] +#[non_exhaustive] +pub enum BuildError { + #[error("failed to build '{0}'")] + Build(String, #[source] Box), + #[error(transparent)] + #[diagnostic(transparent)] + Tree(#[from] TreeError), + #[error("failed to install pre-built rock '{0}'")] + InstallBinaryRock(String, #[source] Box), + #[error(transparent)] + #[diagnostic(transparent)] + LuaInstallation(#[from] LuaInstallationError), + #[error("error instantiating LuaRocks compatibility layer")] + #[diagnostic(forward(0))] + LuaRocks(#[from] LuaRocksError), + #[error("error installing LuaRocks compatibility layer")] + #[diagnostic(forward(0))] + LuaRocksInstall(#[from] Box), +} + +impl From for BuildError { + fn from(source: LuaRocksInstallError) -> Self { + Self::LuaRocksInstall(Box::new(source)) + } +} + +#[derive(Builder)] +#[builder(start_fn = new, finish_fn(name = _build, vis = ""))] +pub(crate) struct Build<'a, T: InstallTree + Send + Sync> { + #[builder(start_fn)] + pub(crate) config: &'a Config, + #[builder(start_fn)] + pub(crate) tree: &'a T, + #[builder(field)] + pub(crate) packages: Vec, +} + +impl BuildBuilder<'_, T, State> +where + T: InstallTree + Send + Sync, + State: build_builder::State, +{ + pub(crate) fn packages(mut self, packages: Vec) -> Self { + self.packages = packages; + self + } +} + +impl BuildBuilder<'_, T, State> +where + T: InstallTree + Send + Sync, + State: build_builder::State + build_builder::IsComplete, +{ + pub(crate) async fn build(self) -> Result, BuildError> { + let args = self._build(); + let lua = LuaInstallation::new_from_config(args.config).await?; + + let needs_luarocks = args.packages.iter().any(|package| { + matches!( + package.rockspec.build().current_platform().build_backend, + Some(BuildBackendSpec::LuaRock(_)) + ) + }); + if needs_luarocks { + LuaRocksInstallation::new(args.config, args.tree.build_tree(args.config)?)? + .ensure_installed(&lua) + .await?; + } + + let mut installed = Vec::new(); + + for downloaded in order_by_build_dependencies(args.packages) { + let DownloadedPackage { + package, + rockspec, + entry_type, + build_behaviour: package_behaviour, + artifact, + temp_dir: _temp_dir, + } = downloaded; + + let is_entrypoint = entry_type.is_entrypoint(); + let constraint = package.spec.constraint(); + let pin = package.spec.pinned(); + let opt = package.spec.opt(); + + let pkg = match artifact { + PackageSource::SourceDir(path) => deploy( + &rockspec, + args.tree, + package, + &lua, + &path, + entry_type, + args.config, + package_behaviour, + ) + .await + .map_err(|err| BuildError::Build(rockspec.package().to_string(), Box::new(err)))?, + PackageSource::PackedRock(bytes) => BinaryRockInstall::new( + &rockspec, + package.source().clone(), + bytes, + entry_type, + args.config, + args.tree, + ) + .pin(pin) + .opt(opt) + .constraint(constraint) + .behaviour(package_behaviour) + .install() + .await + .map_err(|err| { + BuildError::InstallBinaryRock(rockspec.package().to_string(), Box::new(err)) + })?, + }; + + // Record the installed package in the tree lockfile so that its paths are visible to + // the packages built after it (e.g. LuaRocks build backends), as well as future builds. + { + let mut lockfile = args.tree.lockfile()?.write_guard(); + if is_entrypoint { + lockfile.add_entrypoint(&pkg); + } else { + lockfile.add(&pkg); + } + } + + installed.push(pkg); + } + + Ok(installed) + } +} + +fn order_by_build_dependencies(packages: Vec) -> Vec { + let mut by_id: HashMap = packages + .into_iter() + .map(|package| (package.package.id(), package)) + .collect(); + + let mut ordered = Vec::new(); + let mut visited = HashSet::new(); + + let ids: Vec = by_id.keys().cloned().collect(); + for id in ids { + visit(id, &mut by_id, &mut visited, &mut ordered); + } + + ordered +} + +fn visit( + id: LockedPackageId, + by_id: &mut HashMap, + visited: &mut HashSet, + ordered: &mut Vec, +) { + if !visited.insert(id.clone()) { + return; + } + + let build_dependencies: Vec = by_id + .get(&id) + .map(|package| { + package + .package + .spec + .build_dependencies() + .into_iter() + .cloned() + .collect() + }) + .unwrap_or_default(); + + for dependency in build_dependencies { + visit(dependency, by_id, visited, ordered); + } + + if let Some(package) = by_id.remove(&id) { + ordered.push(package); + } +} diff --git a/lux-lib/src/pipeline/build_local.rs b/lux-lib/src/pipeline/build_local.rs new file mode 100644 index 000000000..e5b55ef78 --- /dev/null +++ b/lux-lib/src/pipeline/build_local.rs @@ -0,0 +1,100 @@ +use bon::Builder; +use tracing::Instrument; + +use crate::{ + build::{deploy, BuildBehaviour, BuildError, RemotePackageSourceSpec}, + config::Config, + hash::HasIntegrity, + lockfile::{LockConstraint, LockedPackage, LockedPackageSpec, OptState, PinnedState}, + lua_installation::LuaInstallation, + remote_package_source::RemotePackageSource, + rockspec::{LuaVersionCompatibility, Rockspec}, + tree::{EntryType, InstallTree}, +}; + +use super::download_sources_and_hash::fetch_and_hash_source; + +/// A rocks package builder, providing fine-grained control +/// over how a package should be built. +#[derive(Builder)] +#[builder(start_fn = new, finish_fn(name = _build, vis = ""))] +pub struct Build<'a, R: Rockspec + HasIntegrity, T: InstallTree> { + rockspec: &'a R, + tree: &'a T, + entry_type: EntryType, + config: &'a Config, + lua: &'a LuaInstallation, + + #[builder(default)] + pin: PinnedState, + #[builder(default)] + opt: OptState, + #[builder(default)] + constraint: LockConstraint, + behaviour: BuildBehaviour, + + #[builder(setters(vis = "pub(crate)"))] + source_spec: Option, + + // TODO(vhyrro): Remove this and enforce that this is provided at a type level. + #[builder(setters(vis = "pub(crate)"))] + source: Option, +} + +impl BuildBuilder<'_, R, T, State> +where + State: build_builder::State + build_builder::IsComplete, +{ + pub async fn build(self) -> Result { + let build = self._build(); + let span = tracing::info_span!( + "Building", + package = build.rockspec.package().to_string(), + version = build.rockspec.version().to_string(), + ); + do_build(build).instrument(span).await + } +} + +#[tracing::instrument(level = "trace", skip_all)] +async fn do_build(build: Build<'_, R, T>) -> Result +where + R: Rockspec + HasIntegrity, + T: InstallTree + Sync, +{ + let rockspec = build.rockspec; + let lua = build.lua; + + rockspec.validate_lua_version(&lua.version)?; + + let spec = LockedPackageSpec::new( + rockspec.package(), + rockspec.version(), + build.constraint, + Vec::new(), + Vec::new(), + &build.pin, + &build.opt, + ); + // Pin the futures otherwise they fill up the stack quite a bit + let (package, temp_dir) = Box::pin(fetch_and_hash_source( + rockspec, + spec, + build.source_spec, + build.source, + build.config, + )) + .await?; + + Box::pin(deploy( + rockspec, + build.tree, + package, + lua, + temp_dir.path(), + build.entry_type, + build.config, + build.behaviour, + )) + .await +} diff --git a/lux-lib/src/pipeline/discover.rs b/lux-lib/src/pipeline/discover.rs new file mode 100644 index 000000000..ccfbecedf --- /dev/null +++ b/lux-lib/src/pipeline/discover.rs @@ -0,0 +1,95 @@ +use miette::Diagnostic; +use thiserror::Error; + +use crate::{ + lua_rockspec::RemoteLuaRockspec, + operations, + package::{PackageReq, PackageSpec, RemotePackage}, + package_db::SearchError, + remote_package_source::RemotePackageSource, + rockspec::Rockspec, +}; +use std::path::PathBuf; + +use super::download_sources_and_hash::PackageSource; + +#[derive(Clone, Debug)] +pub(crate) struct FoundPackage { + pub(crate) package: RemotePackage, + pub(crate) rockspec: RemoteLuaRockspec, + pub(crate) artifact: Option, +} + +impl FoundPackage { + /// Builds a discovered package from an already-parsed rockspec, so it can + /// be resolved as a root without contacting a remote package database. + pub(crate) fn from_rockspec(rockspec: RemoteLuaRockspec) -> Self { + let source = rockspec + .to_lua_remote_rockspec_string() + .map(RemotePackageSource::RockspecContent) + .unwrap_or(RemotePackageSource::Local); + let package = RemotePackage::new( + PackageSpec::new(rockspec.package().clone(), rockspec.version().clone()), + source, + None, + ); + Self { + package, + rockspec, + artifact: None, + } + } + + /// Builds a discovered package from a project root rockspec, using the + /// project's own directory as the source. The source is hashed and built in + /// place, so no download happens. + pub(crate) fn from_project_root(rockspec: RemoteLuaRockspec, root_dir: PathBuf) -> Self { + let package = RemotePackage::new( + PackageSpec::new(rockspec.package().clone(), rockspec.version().clone()), + RemotePackageSource::Local, + None, + ); + Self { + package, + rockspec, + artifact: Some(PackageSource::SourceDir(root_dir)), + } + } +} + +#[derive(Error, Debug, Diagnostic)] +#[non_exhaustive] +pub enum DiscoverError { + #[error(transparent)] + #[diagnostic(transparent)] + Search(#[from] SearchError), + #[error("failed to acquire metadata for '{0}'")] + #[diagnostic(forward(1))] + Download(PackageReq, Box), + #[error("failed to fetch vendored package for '{0}'")] + #[diagnostic(forward(1))] + FetchVendored(PackageReq, Box), +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn from_rockspec_builds_local_found_package() { + let content = "rockspec_format = '1.0'\npackage = 'foo'\nversion = '1.0.0-1'\nsource = { url = 'https://example.com/foo.zip' }"; + let Ok(rockspec) = RemoteLuaRockspec::new(content) else { + panic!("invalid test rockspec"); + }; + + let found = FoundPackage::from_rockspec(rockspec); + + assert_eq!(found.package.package.name().to_string(), "foo"); + assert_eq!(found.package.package.version().to_string(), "1.0.0-1"); + assert!(matches!( + found.package.source, + RemotePackageSource::RockspecContent(_) + )); + assert!(found.artifact.is_none()); + } +} diff --git a/lux-lib/src/pipeline/download_sources_and_hash.rs b/lux-lib/src/pipeline/download_sources_and_hash.rs new file mode 100644 index 000000000..2dc1f1fd9 --- /dev/null +++ b/lux-lib/src/pipeline/download_sources_and_hash.rs @@ -0,0 +1,275 @@ +use std::collections::HashMap; +use std::io; +use std::io::Cursor; +use std::path::PathBuf; + +use bon::Builder; +use bytes::Bytes; +use futures::{StreamExt, TryStreamExt}; +use miette::Diagnostic; +use tempfile::TempDir; +use thiserror::Error; + +use crate::{ + build::{BuildError, RemotePackageSourceSpec, SrcRockSource}, + config::Config, + fs::{self, FsError}, + hash::HasIntegrity, + lockfile::{ + LockedPackage, LockedPackageHashes, LockedPackageId, LockedPackageSpec, + RemotePackageSourceUrl, + }, + lua_rockspec::RemoteLuaRockspec, + operations::{unpack_src_rock, FetchSrc, RemotePackageSourceMetadata}, + remote_package_source::RemotePackageSource, + rockspec::Rockspec, + tree::EntryType, +}; + +use super::{ + resolve::{ResolvedArtifacts, ResolvedPackage}, + Artifacts, +}; + +#[derive(Clone, Debug)] +pub(crate) enum PackageSource { + SourceDir(PathBuf), + PackedRock(Bytes), +} + +pub(crate) struct DownloadedPackage { + pub(crate) package: LockedPackage, + pub(crate) rockspec: RemoteLuaRockspec, + pub(crate) entry_type: EntryType, + pub(crate) build_behaviour: crate::build::BuildBehaviour, + pub(crate) artifact: PackageSource, + pub(crate) temp_dir: Option, +} + +pub(crate) type DownloadSourcesAndHashArtifacts = + Artifacts>; + +#[derive(Error, Debug, Diagnostic)] +#[non_exhaustive] +pub enum DownloadSourcesAndHashError { + #[error(transparent)] + #[diagnostic(transparent)] + Fs(#[from] FsError), + #[error(transparent)] + #[diagnostic(transparent)] + Build(#[from] Box), + #[error("failed to hash the source for '{0}'")] + Hash(String, #[source] io::Error), + #[error("missing source url for '{0}'")] + MissingSourceUrl(String), +} + +impl From for DownloadSourcesAndHashError { + fn from(source: BuildError) -> Self { + Self::Build(Box::new(source)) + } +} + +#[derive(Builder)] +#[builder(start_fn = new, finish_fn(name = _build, vis = ""))] +pub(crate) struct DownloadSourcesAndHash<'a> { + #[builder(start_fn)] + pub(crate) config: &'a Config, + pub(crate) resolved: ResolvedArtifacts, +} + +impl DownloadSourcesAndHashBuilder<'_, State> +where + State: download_sources_and_hash_builder::State + download_sources_and_hash_builder::IsComplete, +{ + pub(crate) async fn download_sources_and_hash( + self, + ) -> Result { + let args = self._build(); + let config = args.config; + + let artifacts: DownloadSourcesAndHashArtifacts = args + .resolved + .iter() + .map(|(section, packages)| (section, packages.map(|_| HashMap::new()))) + .collect(); + + let jobs = args.resolved.into_iter().flat_map(|(section, packages)| { + packages + .into_iter() + .flatten() + .map(move |(id, package)| (section, id, package)) + }); + + futures::stream::iter(jobs) + .map(|(section, id, package)| async move { + download_sources_and_hash(config, package) + .await + .map(|package| (section, id, package)) + }) + .buffered(config.max_jobs()) + .try_fold( + artifacts, + |mut artifacts, (section, id, package)| async move { + artifacts + .get_mut(section) + .get_or_insert_with(HashMap::new) + .insert(id, package); + Ok(artifacts) + }, + ) + .await + } +} + +async fn download_sources_and_hash( + config: &Config, + resolved: ResolvedPackage, +) -> Result { + let ResolvedPackage { + spec, + rockspec, + source, + source_url, + entry_type, + build_behaviour, + artifact, + } = resolved; + + let name = spec.name().to_string(); + + let (package, artifact, temp_dir) = match artifact { + Some(PackageSource::PackedRock(bytes)) => match &source { + RemotePackageSource::LuarocksBinaryRock(url) => { + let binary_url = url.clone(); + let rockspec_hash = rockspec + .hash() + .await + .map_err(|err| DownloadSourcesAndHashError::Hash(name.clone(), err))?; + let source_hash = bytes + .hash() + .await + .map_err(|err| DownloadSourcesAndHashError::Hash(name.clone(), err))?; + let hashes = LockedPackageHashes { + rockspec: rockspec_hash, + source: source_hash, + }; + let package = LockedPackage::new( + spec, + source, + Some(RemotePackageSourceUrl::Url { url: binary_url }), + hashes, + ); + (package, PackageSource::PackedRock(bytes), None) + } + _ => { + let source_url = source_url + .clone() + .ok_or_else(|| DownloadSourcesAndHashError::MissingSourceUrl(name.clone()))?; + let (package, dir) = fetch_and_hash_source( + &rockspec, + spec, + Some(RemotePackageSourceSpec::SrcRock(SrcRockSource { + bytes, + source_url, + })), + Some(source), + config, + ) + .await?; + let artifact = PackageSource::SourceDir(dir.path().to_path_buf()); + (package, artifact, Some(dir)) + } + }, + Some(PackageSource::SourceDir(path)) => { + let rockspec_hash = rockspec + .hash() + .await + .map_err(|err| DownloadSourcesAndHashError::Hash(name.clone(), err))?; + let source_hash = path + .hash() + .await + .map_err(|err| DownloadSourcesAndHashError::Hash(name.clone(), err))?; + let hashes = LockedPackageHashes { + rockspec: rockspec_hash, + source: source_hash, + }; + let package = LockedPackage::new(spec, source, source_url, hashes); + (package, PackageSource::SourceDir(path), None) + } + None => { + let (package, dir) = fetch_and_hash_source( + &rockspec, + spec, + Some(RemotePackageSourceSpec::RockSpec(source_url)), + Some(source), + config, + ) + .await?; + let artifact = PackageSource::SourceDir(dir.path().to_path_buf()); + (package, artifact, Some(dir)) + } + }; + + Ok(DownloadedPackage { + package, + rockspec, + entry_type, + build_behaviour, + artifact, + temp_dir, + }) +} + +pub(crate) async fn fetch_and_hash_source( + rockspec: &R, + spec: LockedPackageSpec, + source_spec: Option, + source: Option, + config: &Config, +) -> Result<(LockedPackage, TempDir), BuildError> { + let temp_dir = fs::tempfile::tempdir()?; + + let source_metadata = match source_spec { + Some(RemotePackageSourceSpec::SrcRock(SrcRockSource { bytes, source_url })) => { + let hash = bytes.hash().await?; + let cursor = Cursor::new(bytes); + unpack_src_rock(cursor, temp_dir.path().to_path_buf()) + .await + .map_err(BuildError::UnpackSrcRock)?; + RemotePackageSourceMetadata { hash, source_url } + } + Some(RemotePackageSourceSpec::RockSpec(source_url)) => { + FetchSrc::new(temp_dir.path(), rockspec, config) + .maybe_source_url(source_url) + .fetch_internal() + .await? + } + None => { + FetchSrc::new(temp_dir.path(), rockspec, config) + .fetch_internal() + .await? + } + }; + + let hashes = LockedPackageHashes { + rockspec: rockspec.hash().await?, + source: source_metadata.hash.clone(), + }; + + let package = LockedPackage::new( + spec, + source + .map(Result::Ok) + .unwrap_or_else(|| { + rockspec + .to_lua_remote_rockspec_string() + .map(RemotePackageSource::RockspecContent) + }) + .unwrap_or(RemotePackageSource::Local), + Some(source_metadata.source_url.clone()), + hashes, + ); + + Ok((package, temp_dir)) +} diff --git a/lux-lib/src/pipeline/emit_lockfile.rs b/lux-lib/src/pipeline/emit_lockfile.rs new file mode 100644 index 000000000..d76e45148 --- /dev/null +++ b/lux-lib/src/pipeline/emit_lockfile.rs @@ -0,0 +1,38 @@ +use crate::lockfile::{PackageLock, ReadWrite, WorkspaceLockfile}; + +use super::{download_sources_and_hash::DownloadSourcesAndHashArtifacts, Artifacts}; + +/// The lockfile entries resolved by the pipeline, grouped by section. +/// Allows for transactionality when committing to the lockfile. +#[derive(Default)] +pub struct LockfileHandle(Artifacts); + +impl LockfileHandle { + pub(crate) fn from_artifacts(artifacts: &DownloadSourcesAndHashArtifacts) -> Self { + Self( + artifacts + .iter() + .map(|(section, packages)| { + ( + section, + packages.map(|packages| { + PackageLock::from_packages( + packages + .values() + .map(|pkg| (pkg.package.clone(), pkg.entry_type)), + ) + }), + ) + }) + .collect(), + ) + } + + pub(crate) fn commit(&self, lockfile: &mut WorkspaceLockfile) { + self.0.iter().for_each(|(section, lock)| { + if let Some(lock) = lock { + lockfile.merge(lock, §ion); + } + }); + } +} diff --git a/lux-lib/src/pipeline/mod.rs b/lux-lib/src/pipeline/mod.rs new file mode 100644 index 000000000..d06e93fe3 --- /dev/null +++ b/lux-lib/src/pipeline/mod.rs @@ -0,0 +1,74 @@ +use crate::lockfile::LockedPackageLockType; + +pub mod build; +pub mod build_local; +pub mod discover; +pub mod download_sources_and_hash; +pub mod emit_lockfile; +pub mod resolve; + +/// Packages grouped by the lockfile section they belong to. +/// `None` means that that section shouldn't be modified. +pub(crate) struct Artifacts { + pub(crate) regular: Option, + pub(crate) build: Option, + pub(crate) test: Option, +} + +impl Default for Artifacts { + fn default() -> Self { + Self { + regular: None, + build: None, + test: None, + } + } +} + +impl Artifacts { + pub(crate) fn get_mut(&mut self, section: LockedPackageLockType) -> &mut Option { + match section { + LockedPackageLockType::Regular => &mut self.regular, + LockedPackageLockType::Build => &mut self.build, + LockedPackageLockType::Test => &mut self.test, + } + } + + pub(crate) fn iter(&self) -> impl Iterator)> { + [ + (LockedPackageLockType::Regular, self.regular.as_ref()), + (LockedPackageLockType::Build, self.build.as_ref()), + (LockedPackageLockType::Test, self.test.as_ref()), + ] + .into_iter() + } +} + +impl Extend<(LockedPackageLockType, Option)> for Artifacts { + fn extend)>>(&mut self, iter: I) { + iter.into_iter() + .for_each(|(section, value)| *self.get_mut(section) = value); + } +} + +impl FromIterator<(LockedPackageLockType, Option)> for Artifacts { + fn from_iter)>>(iter: I) -> Self { + let mut artifacts = Artifacts::default(); + artifacts.extend(iter); + artifacts + } +} + +impl IntoIterator for Artifacts { + type Item = (LockedPackageLockType, Option); + type IntoIter = std::array::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + [ + (LockedPackageLockType::Regular, self.regular), + (LockedPackageLockType::Build, self.build), + (LockedPackageLockType::Test, self.test), + ] + .into_iter() + } +} diff --git a/lux-lib/src/pipeline/resolve.rs b/lux-lib/src/pipeline/resolve.rs new file mode 100644 index 000000000..71815dea7 --- /dev/null +++ b/lux-lib/src/pipeline/resolve.rs @@ -0,0 +1,688 @@ +use std::collections::{HashMap, HashSet, VecDeque}; +use std::sync::Arc; + +use bon::Builder; +use itertools::Itertools; +use miette::Diagnostic; +use thiserror::Error; +use tokio::task::JoinSet; + +use crate::{ + build::BuildBehaviour, + config::Config, + lockfile::{ + LockedPackage, LockedPackageId, LockedPackageLockType, LockedPackageSpec, OptState, + PinnedState, RemotePackageSourceUrl, + }, + lua_rockspec::{BuildBackendSpec, RemoteLuaRockspec}, + operations::PackageInstallSpec, + package::{PackageName, PackageReq}, + package_db::PackageDB, + remote_package_source::RemotePackageSource, + rockspec::Rockspec, + tree::EntryType, +}; + +use super::{ + discover::{DiscoverError, FoundPackage}, + download_sources_and_hash::PackageSource, + Artifacts, +}; + +/// The build dependencies of a rockspec that still need to be installed, with the +/// LuaRocks build backend (if any) first. +pub(crate) fn build_dependencies_to_install(rockspec: &R) -> Vec { + let mut names = rockspec + .build_dependencies() + .current_platform() + .iter() + .filter(|dep| { + !matches!( + dep.name().to_string().as_str(), + "luarocks-build-rust-mlua" + | "luarocks-build-rust-binary" + | "luarocks-build-treesitter-parser" + ) + }) + .map(|dep| dep.name().clone()) + .collect_vec(); + + if let Some(backend) = luarocks_build_backend_name(rockspec) { + names.insert(0, backend); + } + names +} + +/// The name of the luarocks build backend rock required to build this rockspec (if any). +pub(crate) fn luarocks_build_backend_name(rockspec: &R) -> Option { + match &rockspec.build().current_platform().build_backend { + Some(BuildBackendSpec::LuaRock(backend)) => { + Some(PackageName::new(format!("luarocks-build-{backend}"))) + } + _ => None, + } +} + +pub(crate) struct ResolvedPackage { + pub(crate) spec: LockedPackageSpec, + pub(crate) rockspec: RemoteLuaRockspec, + pub(crate) source: RemotePackageSource, + pub(crate) source_url: Option, + pub(crate) entry_type: EntryType, + pub(crate) build_behaviour: crate::build::BuildBehaviour, + pub(crate) artifact: Option, +} + +pub(crate) type ResolvedArtifacts = Artifacts>; + +#[derive(Error, Debug, Diagnostic)] +#[non_exhaustive] +pub enum ResolveError { + #[error(transparent)] + #[diagnostic(transparent)] + Discover(#[from] DiscoverError), + #[error("cyclic dependency detected:\n{0}")] + CyclicDependency(String), + #[error(transparent)] + Join(#[from] tokio::task::JoinError), +} + +#[derive(Builder)] +#[builder(start_fn = new, finish_fn(name = _build, vis = ""))] +pub(crate) struct ResolvePackageDependencies<'a> { + #[builder(start_fn)] + pub(crate) package_db: PackageDB, + #[builder(start_fn)] + pub(crate) config: &'a Config, + #[builder(field)] + pub(crate) packages: Artifacts>, + #[builder(default)] + pub(crate) installed: HashMap>, +} + +impl ResolvePackageDependenciesBuilder<'_, State> +where + State: resolve_package_dependencies_builder::State, +{ + fn set(mut self, section: LockedPackageLockType, packages: Vec) -> Self { + *self.packages.get_mut(section) = Some(packages); + self + } + + pub(crate) fn packages(self, packages: Vec) -> Self { + self.set(LockedPackageLockType::Regular, packages) + } + + pub(crate) fn build_packages(self, packages: Vec) -> Self { + self.set(LockedPackageLockType::Build, packages) + } + + pub(crate) fn test_packages(self, packages: Vec) -> Self { + self.set(LockedPackageLockType::Test, packages) + } +} + +impl ResolvePackageDependenciesBuilder<'_, State> +where + State: resolve_package_dependencies_builder::State + + resolve_package_dependencies_builder::IsComplete, +{ + pub(crate) async fn resolve(self) -> Result { + let args = self._build(); + Resolver::new(args.package_db, args.config, args.installed) + .run(args.packages) + .await + } +} + +/// Identifies a single discovery request. Two requests with the same key resolve to the +/// same package, so the diamond problem is solved by only discovering each key once. +#[derive(Clone, PartialEq, Eq, Hash)] +struct RequestKey { + section: LockedPackageLockType, + package: PackageReq, + pin: PinnedState, + opt: OptState, +} + +impl RequestKey { + fn from_install_spec(section: LockedPackageLockType, spec: &PackageInstallSpec) -> Self { + Self { + section, + package: spec.package.clone(), + pin: spec.pin, + opt: spec.opt, + } + } +} + +/// A discovered node in the dependency graph. +struct Node { + install_spec: PackageInstallSpec, + id: LockedPackageId, + found: FoundPackage, + children: Vec<(bool, RequestKey)>, +} + +struct Resolver { + package_db: PackageDB, + config: Arc, + max_concurrent: usize, + joinset: JoinSet<( + RequestKey, + PackageInstallSpec, + Result, + )>, + pending: VecDeque<(RequestKey, PackageInstallSpec)>, + seen: HashSet, + nodes: HashMap, + resolved: ResolvedArtifacts, + installed: HashMap>, + installed_ids: HashMap, +} + +impl Resolver { + fn new( + package_db: PackageDB, + config: &Config, + installed: HashMap>, + ) -> Self { + Self::with_max_concurrent( + package_db, + Arc::new(config.clone()), + config.max_jobs(), + installed, + ) + } + + fn with_max_concurrent( + package_db: PackageDB, + config: Arc, + max_concurrent: usize, + installed: HashMap>, + ) -> Self { + Self { + package_db, + config, + max_concurrent, + joinset: JoinSet::new(), + pending: VecDeque::new(), + seen: HashSet::new(), + nodes: HashMap::new(), + resolved: ResolvedArtifacts::default(), + installed, + installed_ids: HashMap::new(), + } + } + + async fn run( + mut self, + packages: Artifacts>, + ) -> Result { + for (section, specs) in packages { + let Some(specs) = specs else { + continue; + }; + // Mark the section as requested even if it resolves to nothing. + self.resolved + .get_mut(section) + .get_or_insert_with(HashMap::new); + for spec in specs { + let key = RequestKey::from_install_spec(section, &spec); + self.enqueue(key, spec); + } + } + + self.spawn_ready(); + while let Some(joined) = self.joinset.join_next().await { + let (key, spec, result) = joined.map_err(ResolveError::Join)?; + self.on_discovered(key, spec, result?); + self.spawn_ready(); + } + + detect_cycles(&self.nodes)?; + Ok(self.assemble()) + } + + /// Registers a request for discovery, unless it has already been seen. + fn enqueue(&mut self, key: RequestKey, spec: PackageInstallSpec) { + if self.seen.insert(key.clone()) { + self.pending.push_back((key, spec)); + } + } + + /// Spawns queued discoveries up to the configured concurrency limit. + fn spawn_ready(&mut self) { + while self.joinset.len() < self.max_concurrent { + let Some((key, spec)) = self.pending.pop_front() else { + break; + }; + let package_db = self.package_db.clone(); + let config = Arc::clone(&self.config); + let task_spec = spec.clone(); + self.joinset.spawn(async move { + let result = package_db + .resolve(&task_spec, &config) + .await + .map_err(ResolveError::Discover); + (key, spec, result) + }); + } + } + + /// Records a discovered package and queues any children it introduces. + fn on_discovered(&mut self, key: RequestKey, spec: PackageInstallSpec, found: FoundPackage) { + let section = key.section; + let is_binary = matches!( + found.package.source, + RemotePackageSource::LuarocksBinaryRock(_) + ); + let constraint = spec + .constraint + .clone() + .unwrap_or_else(|| spec.package.version_req().clone().into()); + let id = LockedPackageId::new( + found.package.package.name(), + found.package.package.version(), + spec.pin, + spec.opt, + constraint, + ); + + let mut child_specs: Vec<(bool, RequestKey, PackageInstallSpec)> = Vec::new(); + if !is_binary { + for child_spec in build_dependency_specs(&found.rockspec, &spec) { + push_child(&mut child_specs, true, section, child_spec); + } + } + for dependency in found.rockspec.dependencies().current_platform() { + let child_spec = PackageInstallSpec::new( + dependency.package_req().clone(), + EntryType::DependencyOnly, + ) + .build_behaviour(BuildBehaviour::Ignore) + .pin(spec.pin) + .opt(spec.opt) + .maybe_source(dependency.source().clone()) + .build(); + push_child(&mut child_specs, false, section, child_spec); + } + + let children = child_specs + .iter() + .map(|(is_build, child_key, _)| (*is_build, child_key.clone())) + .collect(); + self.resolved + .get_mut(section) + .get_or_insert_with(HashMap::new); + self.nodes.insert( + key, + Node { + install_spec: spec, + id, + found, + children, + }, + ); + + for (is_build, child_key, child_spec) in child_specs { + let child_section = if is_build { + LockedPackageLockType::Build + } else { + section + }; + if let Some(installed_id) = self.installed_match(child_section, &child_spec) { + self.installed_ids.insert(child_key, installed_id); + } else { + self.enqueue(child_key, child_spec); + } + } + } + + fn installed_match( + &self, + section: LockedPackageLockType, + spec: &PackageInstallSpec, + ) -> Option { + if !matches!(spec.build_behaviour, BuildBehaviour::Ignore) || spec.source.is_some() { + return None; + } + self.installed + .get(§ion)? + .iter() + .rev() + .find(|package| { + package.name() == spec.package.name() + && spec.package.version_req().matches(package.version()) + }) + .map(|package| package.id()) + } + + /// Computes the final specs and assembles the resolved packages now that discovery is + /// complete and every child id is known. + fn assemble(self) -> ResolvedArtifacts { + let Resolver { + nodes, + mut resolved, + installed_ids, + .. + } = self; + + let mut ids: HashMap = nodes + .iter() + .map(|(key, node)| (key.clone(), node.id.clone())) + .collect(); + for (key, id) in installed_ids { + ids.entry(key).or_insert(id); + } + + for (key, node) in nodes { + let Node { + install_spec, + id, + found, + children, + } = node; + let dependencies = children + .iter() + .filter(|(is_build, _)| !is_build) + .map(|(_, child)| ids[child].clone()) + .collect_vec(); + let build_dependencies = children + .iter() + .filter(|(is_build, _)| *is_build) + .map(|(_, child)| ids[child].clone()) + .collect_vec(); + let constraint = install_spec + .constraint + .clone() + .unwrap_or_else(|| install_spec.package.version_req().clone().into()); + + let spec = LockedPackageSpec::new( + found.package.package.name(), + found.package.package.version(), + constraint, + dependencies, + build_dependencies, + &install_spec.pin, + &install_spec.opt, + ); + let package = ResolvedPackage { + spec, + rockspec: found.rockspec, + source: found.package.source, + source_url: found.package.source_url, + entry_type: install_spec.entry_type, + build_behaviour: install_spec.build_behaviour, + artifact: found.artifact, + }; + + resolved + .get_mut(key.section) + .get_or_insert_with(HashMap::new) + .entry(id) + .or_insert(package); + } + + resolved + } +} + +/// Adds a child request unless an equivalent request is already present. +fn push_child( + children: &mut Vec<(bool, RequestKey, PackageInstallSpec)>, + is_build: bool, + section: LockedPackageLockType, + spec: PackageInstallSpec, +) { + let child_section = if is_build { + LockedPackageLockType::Build + } else { + section + }; + let child_key = RequestKey::from_install_spec(child_section, &spec); + if !children + .iter() + .any(|(_, existing, _)| existing == &child_key) + { + children.push((is_build, child_key, spec)); + } +} + +fn detect_cycles(nodes: &HashMap) -> Result<(), ResolveError> { + #[derive(Clone, Copy, PartialEq, Eq)] + enum Color { + White, + Grey, + Black, + } + + let mut colors: HashMap<&RequestKey, Color> = HashMap::with_capacity(nodes.len()); + + for start in nodes.keys() { + if colors.get(start).copied().unwrap_or(Color::White) != Color::White { + continue; + } + colors.insert(start, Color::Grey); + let mut stack: Vec<(&RequestKey, usize)> = vec![(start, 0)]; + let mut path: Vec<&RequestKey> = vec![start]; + + while let Some(&(node, index)) = stack.last() { + let children = &nodes[node].children; + if index < children.len() { + if let Some(top) = stack.last_mut() { + top.1 += 1; + } + let child = &children[index].1; + if !nodes.contains_key(child) { + continue; + } + match colors.get(child).copied().unwrap_or(Color::White) { + Color::White => { + colors.insert(child, Color::Grey); + path.push(child); + stack.push((child, 0)); + } + Color::Grey => { + let Some(position) = path.iter().position(|key| *key == child) else { + continue; + }; + let chain = path[position..] + .iter() + .map(|key| key.package.name().to_string()) + .chain(std::iter::once(child.package.name().to_string())) + .collect_vec() + .join(" -> "); + return Err(ResolveError::CyclicDependency(chain)); + } + Color::Black => {} + } + } else { + colors.insert(node, Color::Black); + stack.pop(); + path.pop(); + } + } + } + + Ok(()) +} + +fn build_dependency_specs( + rockspec: &RemoteLuaRockspec, + parent: &PackageInstallSpec, +) -> Vec { + build_dependencies_to_install(rockspec) + .into_iter() + .filter_map(|name| { + rockspec + .build_dependencies() + .current_platform() + .iter() + .find(|dep| dep.name() == &name) + .map(|dep| { + PackageInstallSpec::new(dep.package_req().clone(), EntryType::Entrypoint) + .build_behaviour(BuildBehaviour::Ignore) + .pin(parent.pin) + .opt(parent.opt) + .maybe_source(dep.source().clone()) + .build() + }) + .or_else(|| { + Some( + PackageInstallSpec::new(PackageReq::from(name), EntryType::Entrypoint) + .build_behaviour(BuildBehaviour::Ignore) + .pin(parent.pin) + .opt(parent.opt) + .build(), + ) + }) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use super::*; + use crate::config::ConfigBuilder; + use crate::lockfile::{LockConstraint, LockedPackageHashes}; + use crate::package::{PackageSpec, RemotePackage}; + + fn rockspec(package: &str, dependencies: &[&str]) -> String { + let dependencies = if dependencies.is_empty() { + String::new() + } else { + let list = dependencies + .iter() + .map(|dependency| format!("'{dependency}'")) + .collect::>() + .join(", "); + format!("\ndependencies = {{ {list} }}") + }; + format!( + "rockspec_format = '1.0'\npackage = '{package}'\nversion = '1.0.0-1'\nsource = {{ url = 'https://example.com/{package}.zip' }}{dependencies}" + ) + } + + fn local_packages(rockspecs: &[String]) -> PackageDB { + PackageDB::from_local( + rockspecs + .iter() + .map(|content| { + let rockspec = match RemoteLuaRockspec::new(content) { + Ok(rockspec) => rockspec, + Err(err) => panic!("invalid test rockspec: {err}"), + }; + FoundPackage { + package: RemotePackage::new( + PackageSpec::new( + rockspec.package().clone(), + rockspec.version().clone(), + ), + RemotePackageSource::RockspecContent(content.clone()), + None, + ), + rockspec, + artifact: None, + } + }) + .collect(), + ) + } + + fn install_spec(name: &str) -> PackageInstallSpec { + let package = match name.parse::() { + Ok(package) => package, + Err(err) => panic!("invalid package req: {err}"), + }; + PackageInstallSpec::new(package, EntryType::Entrypoint) + .build_behaviour(BuildBehaviour::Ignore) + .build() + } + + fn installed_package(name: &str) -> LockedPackage { + let spec = PackageSpec::parse(name.to_string(), "1.0.0-1".to_string()).unwrap(); + LockedPackage::from( + &spec, + LockConstraint::Unconstrained, + RemotePackageSource::Test, + None, + LockedPackageHashes { + rockspec: "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + .parse() + .unwrap(), + source: "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + .parse() + .unwrap(), + }, + ) + } + + fn roots(spec: PackageInstallSpec) -> Artifacts> { + Artifacts { + regular: Some(vec![spec]), + build: None, + test: None, + } + } + + fn test_config() -> Arc { + let builder = match ConfigBuilder::new() { + Ok(builder) => builder, + Err(err) => panic!("invalid test config: {err}"), + }; + let config = match builder.build() { + Ok(config) => config, + Err(err) => panic!("invalid test config: {err}"), + }; + Arc::new(config) + } + + #[tokio::test] + async fn diamond_problem() { + let packages = local_packages(&[ + rockspec("a", &["b", "c"]), + rockspec("b", &["d"]), + rockspec("c", &["d"]), + rockspec("d", &[]), + ]); + + let resolved = + match Resolver::with_max_concurrent(packages, test_config(), 8, HashMap::new()) + .run(roots(install_spec("a"))) + .await + { + Ok(resolved) => resolved, + Err(err) => panic!("resolution failed: {err}"), + }; + + assert_eq!(resolved.regular.as_ref().map(|rocks| rocks.len()), Some(4)); + } + + #[tokio::test] + async fn installed_dependency_is_reused() { + let packages = local_packages(&[rockspec("a", &["d"])]); + let installed = + HashMap::from([(LockedPackageLockType::Regular, vec![installed_package("d")])]); + + let resolved = Resolver::with_max_concurrent(packages, test_config(), 8, installed) + .run(roots(install_spec("a"))) + .await + .unwrap(); + + assert_eq!(resolved.regular.as_ref().map(|rocks| rocks.len()), Some(1)); + } + + #[tokio::test] + async fn cyclic_dependencies_are_rejected() { + let packages = local_packages(&[rockspec("a", &["b"]), rockspec("b", &["a"])]); + + let result = Resolver::with_max_concurrent(packages, test_config(), 8, HashMap::new()) + .run(roots(install_spec("a"))) + .await; + + assert!(matches!(result, Err(ResolveError::CyclicDependency(_)))); + } +} diff --git a/lux-lib/src/project/mod.rs b/lux-lib/src/project/mod.rs index 14ee361cd..0ffe322dd 100644 --- a/lux-lib/src/project/mod.rs +++ b/lux-lib/src/project/mod.rs @@ -29,7 +29,7 @@ use crate::{ }, lua_version::LuaVersion, package::SpecRev, - remote_package_db::RemotePackageDB, + package_db::PackageDB, rockspec::{ lua_dependency::{DependencyType, LuaDependencySpec, LuaDependencyType}, LuaVersionCompatibility, @@ -285,6 +285,16 @@ impl Project { .map_err(|err| IntoLocalRockspecError::RockspecError(Box::new(err))) } + /// Returns this project's rockspec as a [`RemoteLuaRockspec`] whose source points at the + /// project's own directory, so it can be built like any other package. + pub fn local_remote_rockspec(&self) -> Result { + self.toml() + .into_local() + .map_err(|err| IntoLocalRockspecError::LocalProjectTomlValidationError(Box::new(err)))? + .to_local_rockspec() + .map_err(|err| IntoLocalRockspecError::RockspecError(Box::new(err))) + } + pub fn remote_rockspec( &self, specrev: Option, @@ -313,7 +323,7 @@ impl Project { pub async fn add( &mut self, dependencies: DependencyType<&PackageReq>, - package_db: &RemotePackageDB, + package_db: &PackageDB, ) -> Result<(), ProjectEditError> { let mut project_toml = toml_edit::DocumentMut::from_str(&fs::tokio::read_to_string(self.toml_path()).await?)?; @@ -467,7 +477,7 @@ impl Project { pub async fn upgrade( &mut self, dependencies: LuaDependencyType<&PackageName>, - package_db: &RemotePackageDB, + package_db: &PackageDB, config: &Config, ) -> Result<(), ProjectEditError> { let mut project_toml = @@ -558,7 +568,7 @@ impl Project { pub async fn upgrade_all( &mut self, - package_db: &RemotePackageDB, + package_db: &PackageDB, config: &Config, ) -> Result<(), ProjectEditError> { if let Some(dependencies) = &self.toml().dependencies { diff --git a/lux-lib/src/project/project_toml.rs b/lux-lib/src/project/project_toml.rs index 9c1905d7a..bc853b172 100644 --- a/lux-lib/src/project/project_toml.rs +++ b/lux-lib/src/project/project_toml.rs @@ -25,11 +25,11 @@ use std::{collections::HashMap, path::PathBuf}; use crate::{ config::Config, lua_rockspec::{ - BuildSpec, BuildSpecInternal, BuildSpecInternalError, DisplayAsLuaKV, ExternalDependencies, - ExternalDependencySpec, LuaVersionError, PartialLuaRockspec, PerPlatform, - PlatformIdentifier, PlatformSupport, PlatformValidationError, RemoteRockSource, - RockDescription, RockSourceError, RockspecFormat, TestSpec, TestSpecDecodeError, - TestSpecInternal, + BuildSpec, BuildSpecInternal, BuildSpecInternalError, DisplayAsLuaKV, DisplayLuaKV, + ExternalDependencies, ExternalDependencySpec, LuaVersionError, PartialLuaRockspec, + PerPlatform, PlatformIdentifier, PlatformSupport, PlatformValidationError, + RemoteRockSource, RockDescription, RockSourceError, RockspecFormat, TestSpec, + TestSpecDecodeError, TestSpecInternal, }, package::{ BuildDependencies, Dependencies, PackageName, PackageReq, PackageVersion, PackageVersionReq, @@ -709,6 +709,103 @@ impl LocalProjectToml { ProjectRoot(self.internal.lux_toml_dir.clone()), ) } + + /// Renders the rockspec for local builds, using the rockspec's own version + /// and its source pointing at the project root. + pub fn to_local_rockspec(&self) -> Result { + Ok(LocalLuaRockspec::from_project_toml(self).into_remote_with_source(self.source.clone())) + } + + fn render_rockspec( + &self, + version: &PackageVersion, + source: Option, + ) -> Result { + let starter = format!( + r#" +rockspec_format = "{}" +package = "{}" +version = "{}""#, + self.rockspec_format + .as_ref() + .unwrap_or(&RockspecFormat::default()), + self.package, + version + ); + + let mut template = Vec::new(); + + if self.description != RockDescription::default() { + template.push(self.description.display_lua()); + } + + if self.supported_platforms != PlatformSupport::default() { + template.push(self.supported_platforms.display_lua()); + } + + { + let mut dependencies = self.internal.dependencies.clone().unwrap_or_default(); + dependencies.insert( + 0, + PackageReq { + name: "lua".into(), + version_req: self.lua.clone(), + } + .into(), + ); + template.push(Dependencies(&dependencies).display_lua()); + } + + let mut build_dependencies = self + .internal + .build_dependencies + .as_ref() + .cloned() + .unwrap_or_default(); + + let build_backend_dependency = self + .internal + .build + .build_type + .as_ref() + .and_then(|build_type| build_type.luarocks_build_backend()); + + if let Some(build_backend_dependency) = build_backend_dependency { + build_dependencies.push(build_backend_dependency); + } + + if !build_dependencies.is_empty() { + template.push(BuildDependencies(&build_dependencies).display_lua()); + } + + match self.internal.external_dependencies { + Some(ref external_dependencies) if !external_dependencies.is_empty() => { + template.push(ExternalDependencies(external_dependencies).display_lua()); + } + _ => {} + } + + if let Some(source) = source { + template.push(source); + } + + template.push(self.internal.build.display_lua()); + + let unformatted_code = std::iter::once(starter) + .chain(template.into_iter().map(|kv| kv.to_string())) + .join("\n\n"); + let result = match stylua_lib::format_code( + &unformatted_code, + stylua_lib::Config::default(), + None, + stylua_lib::OutputVerification::Full, + ) { + Ok(formatted_code) => formatted_code, + Err(_) => unformatted_code, + }; + validate_generated_lua(&result)?; + Ok(result) + } } impl Rockspec for LocalProjectToml { @@ -792,92 +889,11 @@ impl Rockspec for LocalProjectToml { .internal .version_template .try_generate(lux_toml_dir, None)?; - let starter = format!( - r#" -rockspec_format = "{}" -package = "{}" -version = "{}""#, - self.rockspec_format - .as_ref() - .unwrap_or(&RockspecFormat::default()), - self.package, - version - ); - - let mut template = Vec::new(); - - if self.description != RockDescription::default() { - template.push(self.description.display_lua()); - } - - if self.supported_platforms != PlatformSupport::default() { - template.push(self.supported_platforms.display_lua()); - } - - { - let mut dependencies = self.internal.dependencies.clone().unwrap_or_default(); - dependencies.insert( - 0, - PackageReq { - name: "lua".into(), - version_req: self.lua.clone(), - } - .into(), - ); - template.push(Dependencies(&dependencies).display_lua()); - } - - let mut build_dependencies = self - .internal - .build_dependencies - .as_ref() - .cloned() - .unwrap_or_default(); - - let build_backend_dependency = self - .internal - .build - .build_type - .as_ref() - .and_then(|build_type| build_type.luarocks_build_backend()); - - if let Some(build_backend_dependency) = build_backend_dependency { - build_dependencies.push(build_backend_dependency); - } - - if !build_dependencies.is_empty() { - template.push(BuildDependencies(&build_dependencies).display_lua()); - } - - match self.internal.external_dependencies { - Some(ref external_dependencies) if !external_dependencies.is_empty() => { - template.push(ExternalDependencies(external_dependencies).display_lua()); - } - _ => {} - } - let source = self.internal .source_template .try_generate(lux_toml_dir, &self.package, &version)?; - template.push(source.display_lua()); - - template.push(self.internal.build.display_lua()); - - let unformatted_code = std::iter::once(starter) - .chain(template.into_iter().map(|kv| kv.to_string())) - .join("\n\n"); - let result = match stylua_lib::format_code( - &unformatted_code, - stylua_lib::Config::default(), - None, - stylua_lib::OutputVerification::Full, - ) { - Ok(formatted_code) => formatted_code, - Err(_) => unformatted_code, - }; - validate_generated_lua(&result)?; - Ok(result) + self.render_rockspec(&version, Some(source.display_lua())) } } diff --git a/lux-lib/src/remote_package_db/mod.rs b/lux-lib/src/remote_package_db/mod.rs deleted file mode 100644 index bc45a6fbf..000000000 --- a/lux-lib/src/remote_package_db/mod.rs +++ /dev/null @@ -1,187 +0,0 @@ -use crate::{ - config::{Config, ConfigError}, - lockfile::{LocalPackageLock, LockfileIntegrityError}, - manifest::{Manifest, ManifestError}, - package::{ - PackageName, PackageReq, PackageSpec, PackageVersion, RemotePackage, - RemotePackageTypeFilterSpec, - }, -}; -use itertools::Itertools; - -use miette::Diagnostic; -use thiserror::Error; - -/// Package database, used to look up remote rocks -#[derive(Clone, Debug)] -pub struct RemotePackageDB(Impl); - -#[derive(Clone, Debug)] -enum Impl { - LuarocksManifests(Vec), - LocalPackageLocks(Vec), -} - -#[derive(Error, Debug, Diagnostic)] -pub enum RemotePackageDBError { - #[error(transparent)] - #[diagnostic(transparent)] - ManifestError(#[from] ManifestError), - #[error(transparent)] - #[diagnostic(transparent)] - ConfigError(#[from] ConfigError), -} - -#[derive(Error, Debug, Diagnostic)] -#[non_exhaustive] -pub enum SearchError { - #[error("no rock that matches '{0}' found")] - RockNotFound(PackageReq), - #[error("no rock that matches '{0}' found in the lockfile.")] - RockNotFoundInLockfile(PackageReq), - #[error("error when pulling manifest")] - #[diagnostic(forward(0))] - Manifest(#[from] ManifestError), -} - -#[derive(Error, Debug, Diagnostic)] -#[non_exhaustive] -pub enum RemotePackageDbIntegrityError { - #[error(transparent)] - #[diagnostic(transparent)] - Lockfile(#[from] LockfileIntegrityError), -} - -impl RemotePackageDB { - #[tracing::instrument(level = "trace", skip_all)] - pub async fn from_config(config: &Config) -> Result { - let mut manifests = Vec::new(); - for server in config.enabled_dev_servers()? { - let manifest = Manifest::from_config(server, config).await?; - manifests.push(manifest); - } - for server in config.extra_servers() { - let manifest = Manifest::from_config(server.clone(), config).await?; - manifests.push(manifest); - } - manifests.push(Manifest::from_config(config.server().clone(), config).await?); - Ok(Self(Impl::LuarocksManifests(manifests))) - } - - /// Find a remote package that matches the requirement, returning the latest match. - #[tracing::instrument( - name = "Searching tree-sitter parser", - level = "info", - skip_all, - fields(package = package_req.to_string()), - )] - pub(crate) fn find( - &self, - package_req: &PackageReq, - filter: Option, - ) -> Result { - match &self.0 { - Impl::LuarocksManifests(manifests) => { - match manifests - .iter() - .find_map(|manifest| manifest.find(package_req, filter.clone())) - { - Some(package) => Ok(package), - None => Err(SearchError::RockNotFound(package_req.clone())), - } - } - Impl::LocalPackageLocks(locks) => { - match locks - .iter() - .filter_map(|lock| lock.has_rock(package_req, filter.clone())) - .map(|local_package| { - RemotePackage::new( - PackageSpec::new(local_package.spec.name, local_package.spec.version), - local_package.source, - local_package.source_url, - ) - }) - .next() - { - Some(package) => Ok(package), - None => Err(SearchError::RockNotFoundInLockfile(package_req.clone())), - } - } - } - } - - /// Search for all packages that match the requirement. - pub fn search(&self, package_req: &PackageReq) -> Vec<(&PackageName, Vec<&PackageVersion>)> { - match &self.0 { - Impl::LuarocksManifests(manifests) => manifests - .iter() - .flat_map(|manifest| { - manifest - .metadata() - .repository - .iter() - .filter_map(|(name, elements)| { - if name.to_string().contains(&package_req.name().to_string()) { - Some(( - name, - elements - .keys() - .filter(|version| { - package_req.version_req().matches(version) - }) - .sorted_by(|a, b| Ord::cmp(b, a)) - .collect_vec(), - )) - } else { - None - } - }) - }) - .collect(), - Impl::LocalPackageLocks(locks) => locks - .iter() - .flat_map(|lock| lock.rocks().values()) - .filter_map(|package| { - // NOTE: This doesn't group packages by name, but we don't care for now, - // as we shouldn't need to use this function with a lockfile. - let name = package.name(); - if name.to_string().contains(&package_req.name().to_string()) { - Some((name, vec![package.version()])) - } else { - None - } - }) - .collect_vec(), - } - } - - /// Find the latest version for a package by name. - pub(crate) fn latest_version(&self, rock_name: &PackageName) -> Option { - self.latest_match(&rock_name.clone().into(), None) - .map(|result| result.version().clone()) - } - - /// Find the latest package that matches the requirement. - pub fn latest_match( - &self, - package_req: &PackageReq, - filter: Option, - ) -> Option { - match self.find(package_req, filter) { - Ok(result) => Some(result.package), - Err(_) => None, - } - } -} - -impl From for RemotePackageDB { - fn from(manifest: Manifest) -> Self { - Self(Impl::LuarocksManifests(vec![manifest])) - } -} - -impl From> for RemotePackageDB { - fn from(locks: Vec) -> Self { - Self(Impl::LocalPackageLocks(locks)) - } -} diff --git a/lux-lib/src/reqwest.rs b/lux-lib/src/reqwest.rs index 0c04792d4..1349a149c 100644 --- a/lux-lib/src/reqwest.rs +++ b/lux-lib/src/reqwest.rs @@ -1,6 +1,5 @@ use std::sync::OnceLock; -use bytes::Bytes; use miette::Diagnostic; use reqwest::{header::AUTHORIZATION, Client, RequestBuilder}; use thiserror::Error; @@ -110,18 +109,6 @@ pub(crate) fn http_client(config: &Config) -> Result<&Client, reqwest::Error> { client(false, config) } -pub(crate) async fn download_bytes(config: &Config, url: &Url) -> Result { - let bytes = http_client(config)? - .get(url.clone()) - .apply_access_token(config, url) - .send() - .await? - .error_for_status()? - .bytes() - .await?; - Ok(bytes) -} - mod private { pub trait Sealed {} impl Sealed for reqwest::RequestBuilder {} diff --git a/lux-lib/src/rockspec/mod.rs b/lux-lib/src/rockspec/mod.rs index 30bc4dc04..c967c67e6 100644 --- a/lux-lib/src/rockspec/mod.rs +++ b/lux-lib/src/rockspec/mod.rs @@ -136,12 +136,6 @@ impl LuaVersionCompatibility for T { #[derive(Debug, Default, Serialize, Deserialize, Clone, PartialOrd, Ord, Hash, PartialEq, Eq)] pub struct RockBinaries(Vec); -impl RockBinaries { - pub(crate) fn is_default(&self) -> bool { - &Self::default() == self - } -} - impl Deref for RockBinaries { type Target = Vec; diff --git a/lux-lib/src/tree/dist.rs b/lux-lib/src/tree/dist.rs index c193b2e36..8c2a34089 100644 --- a/lux-lib/src/tree/dist.rs +++ b/lux-lib/src/tree/dist.rs @@ -4,7 +4,7 @@ use super::{EntryType, InstallTree, RockLayout, Tree, TreeError}; use crate::{ config::Config, fs, - lockfile::{LocalPackage, Lockfile, ReadOnly}, + lockfile::{LockedPackage, LockedPackageSpec, Lockfile, ReadOnly}, lua_version::LuaVersion, package::{PackageName, PackageVersion}, }; @@ -42,7 +42,7 @@ impl FlatDistTree { Ok(Self(tree)) } - fn guard_no_conflicting_package(&self, package: &LocalPackage) -> Result<(), io::Error> { + fn guard_no_conflicting_package(&self, package: &LockedPackageSpec) -> Result<(), io::Error> { let lockfile = self.lockfile().map_err(io::Error::other)?; match lockfile.has_rock(&package.clone().into_package_req(), None) { Some(existing_package) => { @@ -91,7 +91,7 @@ impl InstallTree for FlatDistTree { self.0.root() } - fn layout_for(&self, _package: &LocalPackage) -> RockLayout { + fn layout_for(&self, _package: &LockedPackageSpec) -> RockLayout { RockLayout::new(self.0.root(), self.0.bin()) } @@ -103,7 +103,7 @@ impl InstallTree for FlatDistTree { self.0.unwrapped_bin() } - fn prepare(&self, package: &LocalPackage) -> Result<(), TreeError> { + fn prepare(&self, package: &LockedPackageSpec) -> Result<(), TreeError> { self.guard_no_conflicting_package(package) .map_err(TreeError::Io)?; let layout = self.layout_for(package); @@ -113,7 +113,7 @@ impl InstallTree for FlatDistTree { Ok(()) } - fn cleanup(&self, _package: &LocalPackage, _entry_type: EntryType) -> Result<(), TreeError> { + fn cleanup(&self, _package: &LockedPackage, _entry_type: EntryType) -> Result<(), TreeError> { unreachable!("dist trees do not undergo cleanup for packages") } @@ -133,7 +133,7 @@ impl InstallTree for FlatDistTree { self.0.test_tree(config) } - fn list(&self) -> Result>, TreeError> { + fn list(&self) -> Result>, TreeError> { self.0.list() } diff --git a/lux-lib/src/tree/layout.rs b/lux-lib/src/tree/layout.rs index 5e83bcfb9..025caba13 100644 --- a/lux-lib/src/tree/layout.rs +++ b/lux-lib/src/tree/layout.rs @@ -1,6 +1,6 @@ use crate::{ fs::{self, FsError}, - lockfile::{LocalPackage, OptState}, + lockfile::{LockedPackageSpec, OptState}, tree::{InstallTree, Tree}, }; use std::path::{Path, PathBuf}; @@ -8,9 +8,9 @@ use std::path::{Path, PathBuf}; /// A custom layout for entrypoint packages. /// Implementations arrange symlinks so external tools can find files at the locations they expect. pub trait CustomRockLayout: std::fmt::Debug + Send + Sync { - fn make_symlinks(&self, tree: &Tree, package: &LocalPackage) -> fs::Result<()>; + fn make_symlinks(&self, tree: &Tree, package: &LockedPackageSpec) -> fs::Result<()>; - fn remove_symlinks(&self, tree: &Tree, package: &LocalPackage) -> fs::Result<()>; + fn remove_symlinks(&self, tree: &Tree, package: &LockedPackageSpec) -> fs::Result<()>; } /// A [`CustomRockLayout`] for Neovim plugins. Packages are exposed under `/site/pack/lux/{start,opt}/` @@ -18,8 +18,8 @@ pub trait CustomRockLayout: std::fmt::Debug + Send + Sync { pub struct NvimLayout; impl NvimLayout { - fn target_for(tree: &Tree, package: &LocalPackage) -> PathBuf { - let subdir = match package.spec.opt { + fn target_for(tree: &Tree, package: &LockedPackageSpec) -> PathBuf { + let subdir = match package.opt() { OptState::Required => "start", OptState::Optional => "opt", }; @@ -31,7 +31,7 @@ impl NvimLayout { } impl CustomRockLayout for NvimLayout { - fn make_symlinks(&self, tree: &Tree, package: &LocalPackage) -> fs::Result<()> { + fn make_symlinks(&self, tree: &Tree, package: &LockedPackageSpec) -> fs::Result<()> { let custom_dir = Self::target_for(tree, package); fs::sync::create_dir_all(&custom_dir)?; @@ -49,7 +49,7 @@ impl CustomRockLayout for NvimLayout { Ok(()) } - fn remove_symlinks(&self, tree: &Tree, package: &LocalPackage) -> fs::Result<()> { + fn remove_symlinks(&self, tree: &Tree, package: &LockedPackageSpec) -> fs::Result<()> { let target = Self::target_for(tree, package); if target.is_dir() { // SAFETY: does not follow symlinks, only removes them @@ -93,16 +93,15 @@ mod tests { use crate::{ config::ConfigBuilder, - lockfile::{LocalPackage, LocalPackageHashes, LockConstraint}, + lockfile::{LockConstraint, LockedPackage, LockedPackageHashes}, lua_version::LuaVersion, package::PackageSpec, remote_package_source::RemotePackageSource, - rockspec::RockBinaries, tree::{EntryType, InstallTree, NvimLayout}, }; - fn mock_hashes() -> LocalPackageHashes { - LocalPackageHashes { + fn mock_hashes() -> LockedPackageHashes { + LockedPackageHashes { rockspec: "sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=" .parse() .unwrap(), @@ -128,11 +127,10 @@ mod tests { (temp, tree_path, tree) } - fn sample_package() -> LocalPackage { - LocalPackage::from( + fn sample_package() -> LockedPackage { + LockedPackage::from( &PackageSpec::parse("neorg".into(), "8.0.0-1".into()).unwrap(), LockConstraint::Unconstrained, - RockBinaries::default(), RemotePackageSource::Test, None, mock_hashes(), @@ -144,7 +142,7 @@ mod tests { let (_temp, tree_path, tree) = sample_tree(); let package = sample_package(); - tree.prepare(&package).unwrap(); + tree.prepare(&package.spec).unwrap(); tree.finalize(&package, EntryType::Entrypoint).unwrap(); let custom_dir = tree_path @@ -161,9 +159,9 @@ mod tests { let (_temp, tree_path, tree) = sample_tree(); let package = sample_package(); - tree.prepare(&package).unwrap(); + tree.prepare(&package.spec).unwrap(); - let etc = tree.layout_for(&package).etc; + let etc = tree.layout_for(&package.spec).etc; std::fs::create_dir_all(etc.join("plugin")).unwrap(); std::fs::write(etc.join("plugin/foo.vim"), "lua _G.foo = 1\n").unwrap(); @@ -180,7 +178,7 @@ mod tests { let (_temp, tree_path, tree) = sample_tree(); let package = sample_package(); - tree.prepare(&package).unwrap(); + tree.prepare(&package.spec).unwrap(); tree.finalize(&package, EntryType::DependencyOnly).unwrap(); let custom_dir = tree_path @@ -194,7 +192,7 @@ mod tests { let (_temp, tree_path, tree) = sample_tree(); let package = sample_package(); - tree.prepare(&package).unwrap(); + tree.prepare(&package.spec).unwrap(); tree.finalize(&package, EntryType::Entrypoint).unwrap(); let custom_dir = tree_path .join("5.1/site/pack/lux/start") diff --git a/lux-lib/src/tree/list.rs b/lux-lib/src/tree/list.rs index 68f0ed70d..da0790ea0 100644 --- a/lux-lib/src/tree/list.rs +++ b/lux-lib/src/tree/list.rs @@ -1,4 +1,4 @@ -use crate::{lockfile::LocalPackage, tree::InstallTree}; +use crate::{lockfile::LockedPackage, tree::InstallTree}; use super::{Tree, TreeError}; @@ -8,7 +8,7 @@ use super::{Tree, TreeError}; // Cloning isn't destructive, but it's sure expensive. impl Tree { - pub fn as_rock_list(&self) -> Result, TreeError> { + pub fn as_rock_list(&self) -> Result, TreeError> { let rock_list = self.list()?; Ok(rock_list.values().flatten().cloned().collect()) diff --git a/lux-lib/src/tree/mod.rs b/lux-lib/src/tree/mod.rs index 9cb42df6b..194e74a36 100644 --- a/lux-lib/src/tree/mod.rs +++ b/lux-lib/src/tree/mod.rs @@ -2,7 +2,9 @@ use crate::{ build::utils::format_path, config::Config, fs, - lockfile::{LocalPackage, LocalPackageId, Lockfile, LockfileError, ReadOnly}, + lockfile::{ + LockedPackage, LockedPackageId, LockedPackageSpec, Lockfile, LockfileError, ReadOnly, + }, lua_version::LuaVersion, package::{PackageName, PackageReq}, variables::{GetVariableError, HasVariables}, @@ -42,7 +44,7 @@ pub trait InstallTree { /// Where unwrapped package binaries are installed fn unwrapped_bin(&self) -> PathBuf; /// The standard install layout for a package. - fn layout_for(&self, package: &LocalPackage) -> RockLayout; + fn layout_for(&self, package: &LockedPackageSpec) -> RockLayout; /// Create a [`Lockfile`] for this tree. fn lockfile(&self) -> Result, TreeError>; /// Get this tree's lockfile path. @@ -52,22 +54,22 @@ pub trait InstallTree { /// The tree in which to install test dependencies. fn test_tree(&self, config: &Config) -> Result; /// List the packages that are installed in this tree. - fn list(&self) -> Result>, TreeError>; + fn list(&self) -> Result>, TreeError>; /// Find installed rocks that match the given [`PackageReq`]. fn match_rocks(&self, req: &PackageReq) -> Result; /// Create the standard directories (src, lib, etc.) for a package. - fn prepare(&self, package: &LocalPackage) -> Result<(), TreeError>; + fn prepare(&self, package: &LockedPackageSpec) -> Result<(), TreeError>; /// Apply the custom layout once a package has been fully installed. /// /// For entrypoints, this creates the symlinks for the custom layout, if one is configured. /// Must be called after the package's files have been written to the standard layout. - fn finalize(&self, _package: &LocalPackage, _entry_type: EntryType) -> Result<(), TreeError> { + fn finalize(&self, _package: &LockedPackage, _entry_type: EntryType) -> Result<(), TreeError> { Ok(()) } /// Remove the install layout directories. /// /// For entrypoints, this also applies the custom layout, if one is configured. - fn cleanup(&self, package: &LocalPackage, entry_type: EntryType) -> Result<(), TreeError>; + fn cleanup(&self, package: &LockedPackage, entry_type: EntryType) -> Result<(), TreeError>; } /// The standard install layout for a rock. @@ -128,7 +130,7 @@ impl RockLayout { } /// A Lux install tree that supports multiple versions of the same dependency, -/// with packages addressed by their [`LocalPackageId`] +/// with packages addressed by their [`LockedPackageId`] #[derive(Clone, Debug)] pub struct Tree { /// The Lua version of the tree. @@ -214,7 +216,7 @@ impl Tree { pub fn match_rocks_and(&self, req: &PackageReq, filter: F) -> Result where - F: Fn(&LocalPackage) -> bool, + F: Fn(&LockedPackage) -> bool, { match self.list()?.get(req.name()) { Some(packages) => { @@ -252,7 +254,7 @@ impl InstallTree for Tree { self.root_parent.join(self.version.to_string()) } - fn prepare(&self, package: &LocalPackage) -> Result<(), TreeError> { + fn prepare(&self, package: &LockedPackageSpec) -> Result<(), TreeError> { let layout = self.layout_for(package); fs::sync::create_dir_all(&layout.root)?; fs::sync::create_dir_all(&layout.lib)?; @@ -262,28 +264,33 @@ impl InstallTree for Tree { Ok(()) } - fn finalize(&self, package: &LocalPackage, entry_type: EntryType) -> Result<(), TreeError> { + fn finalize(&self, package: &LockedPackage, entry_type: EntryType) -> Result<(), TreeError> { if entry_type.is_entrypoint() { if let Some(custom_layout) = &self.entrypoint_layout { - custom_layout.make_symlinks(self, package)?; + custom_layout.make_symlinks(self, &package.spec)?; } } Ok(()) } - fn cleanup(&self, package: &LocalPackage, entry_type: EntryType) -> Result<(), TreeError> { + fn cleanup(&self, package: &LockedPackage, entry_type: EntryType) -> Result<(), TreeError> { if entry_type.is_entrypoint() { if let Some(layout) = &self.entrypoint_layout { - layout.remove_symlinks(self, package)?; + layout.remove_symlinks(self, &package.spec)?; } } - let layout = self.layout_for(package); + let layout = self.layout_for(&package.spec); fs::sync::remove_dir_all(&layout.etc)?; fs::sync::remove_dir_all(&layout.root)?; - for relative_binary_path in package.spec.binaries() { + let lockfile = self.lockfile()?; + for relative_binary_path in lockfile + .binaries(&package.id()) + .into_iter() + .flat_map(|binaries| binaries.iter()) + { if let Some(binary_file_name) = relative_binary_path.file_name() { let binary_path = self.bin().join(binary_file_name); if binary_path.is_file() { @@ -308,7 +315,7 @@ impl InstallTree for Tree { self.root().join(LOCKFILE_NAME) } - fn layout_for(&self, package: &LocalPackage) -> RockLayout { + fn layout_for(&self, package: &LockedPackageSpec) -> RockLayout { RockLayout::new( self.root().join(format!( "{}-{}@{}", @@ -352,7 +359,7 @@ impl InstallTree for Tree { ) } - fn list(&self) -> Result>, TreeError> { + fn list(&self) -> Result>, TreeError> { Ok(self.lockfile()?.list()) } @@ -386,8 +393,8 @@ impl EntryType { #[derive(Clone, Debug)] pub enum RockMatches { NotFound(PackageReq), - Single(LocalPackageId), - Many(NonEmpty), + Single(LockedPackageId), + Many(NonEmpty), } // Loosely mimic the Option functions. @@ -407,11 +414,10 @@ mod tests { use crate::{ config::ConfigBuilder, - lockfile::{LocalPackage, LocalPackageHashes, LockConstraint}, + lockfile::{LockConstraint, LockedPackage, LockedPackageHashes}, lua_version::LuaVersion, package::{PackageName, PackageSpec, PackageVersion}, remote_package_source::RemotePackageSource, - rockspec::RockBinaries, tree::{InstallTree, RockLayout}, variables, }; @@ -432,7 +438,7 @@ mod tests { .unwrap(); let tree = config.user_tree(LuaVersion::Lua51).unwrap(); - let mock_hashes = LocalPackageHashes { + let mock_hashes = LockedPackageHashes { rockspec: "sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=" .parse() .unwrap(), @@ -441,20 +447,19 @@ mod tests { .unwrap(), }; - let package = LocalPackage::from( + let package = LockedPackage::from( &PackageSpec::parse("neorg".into(), "8.0.0-1".into()).unwrap(), LockConstraint::Unconstrained, - RockBinaries::default(), RemotePackageSource::Test, None, mock_hashes.clone(), ); let id = package.id(); - tree.prepare(&package).unwrap(); + tree.prepare(&package.spec).unwrap(); assert_eq!( - tree.layout_for(&package), + tree.layout_for(&package.spec), RockLayout { bin: tree_path.join("5.1/bin"), root: tree_path.join(format!("5.1/{id}-neorg@8.0.0-1")), @@ -492,7 +497,7 @@ mod tests { name, package .into_iter() - .map(|package| package.spec.version) + .map(|package| package.version().clone()) .sorted() .collect_vec(), ) @@ -518,7 +523,7 @@ mod tests { .unwrap(); let tree = config.user_tree(LuaVersion::Lua51).unwrap(); - let mock_hashes = LocalPackageHashes { + let mock_hashes = LockedPackageHashes { rockspec: "sha256-uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=" .parse() .unwrap(), @@ -527,15 +532,14 @@ mod tests { .unwrap(), }; - let package = LocalPackage::from( + let package = LockedPackage::from( &PackageSpec::parse("neorg".into(), "8.0.0-1-1".into()).unwrap(), LockConstraint::Unconstrained, - RockBinaries::default(), RemotePackageSource::Test, None, mock_hashes.clone(), ); - let layout = tree.layout_for(&package); + let layout = tree.layout_for(&package.spec); let build_variables = vec![ "$(PREFIX)", "$(LIBDIR)", diff --git a/lux-lib/src/upload/mod.rs b/lux-lib/src/upload/mod.rs index b54a94a56..7a10d6710 100644 --- a/lux-lib/src/upload/mod.rs +++ b/lux-lib/src/upload/mod.rs @@ -2,8 +2,8 @@ use std::{env, io}; use crate::operations::SearchAndDownloadError; use crate::package::SpecRevIterator; +use crate::package_db::PackageDB; use crate::project::project_toml::{ProjectTomlError, RemoteProjectTomlValidationError}; -use crate::remote_package_db::RemotePackageDB; use crate::rockspec::Rockspec; use crate::TOOL_VERSION; use crate::{config::Config, project::Project}; @@ -36,7 +36,7 @@ pub struct ProjectUpload<'a> { #[cfg(feature = "gpgme")] sign_protocol: SignatureProtocol, config: &'a Config, - package_db: &'a RemotePackageDB, + package_db: &'a PackageDB, } impl ProjectUploadBuilder<'_, State> @@ -510,7 +510,7 @@ mod helpers { client: &Client, api_key: &ApiKey, config: &Config, - package_db: &RemotePackageDB, + package_db: &PackageDB, ) -> Result<(RemoteProjectToml, String), UploadError> { for specrev in SpecRevIterator::new() { let rockspec = project.toml().into_remote(Some(specrev))?; diff --git a/lux-lib/src/which/mod.rs b/lux-lib/src/which/mod.rs index 0d5466901..871082de1 100644 --- a/lux-lib/src/which/mod.rs +++ b/lux-lib/src/which/mod.rs @@ -86,7 +86,7 @@ fn do_search(which: Which<'_>) -> Result { local_packages .into_iter() .filter_map(|pkg| { - let layout = tree.layout_for(&pkg); + let layout = tree.layout_for(&pkg.spec); let lib_path = layout.lib.join(which.module.to_lib_path()); if lib_path.is_file() { return Some(lib_path); diff --git a/lux-lib/src/workspace/mod.rs b/lux-lib/src/workspace/mod.rs index 43ed1b784..ae034517c 100644 --- a/lux-lib/src/workspace/mod.rs +++ b/lux-lib/src/workspace/mod.rs @@ -159,6 +159,17 @@ impl Workspace { &mut self.members } + /// The package names of all workspace members. + pub fn member_names(&self) -> Result, WorkspaceError> { + NonEmpty::from_vec( + self.members() + .iter() + .map(|project| project.toml().package.clone()) + .collect(), + ) + .ok_or_else(|| WorkspaceError::EmptyWorkspace(self.root.0.clone())) + } + /// Get a workspace member, defaulting to the first one if none is specified. /// Fails if a package name is specified, but not found. pub fn single_member_or_select( diff --git a/lux-lib/tests/build.rs b/lux-lib/tests/build.rs index 9f170e62a..4a224635f 100644 --- a/lux-lib/tests/build.rs +++ b/lux-lib/tests/build.rs @@ -8,10 +8,11 @@ use lux_lib::rockspec::Rockspec; use lux_lib::tree::InstallTree; use lux_lib::workspace::Workspace; use lux_lib::{ - build::{Build, BuildBehaviour::Force}, + build::BuildBehaviour::Force, config::ConfigBuilder, lua_installation::{detect_installed_lua_version, LuaInstallation}, lua_rockspec::RemoteLuaRockspec, + pipeline::build_local::Build, tree, }; use tokio::runtime::Builder; @@ -193,7 +194,7 @@ async fn treesitter_parser_build() { .await .unwrap(); - let etc = tree.layout_for(&package).etc; + let etc = tree.layout_for(package.spec()).etc; let folds_query = etc.join("queries").join("rust").join("folds.scm"); assert!(folds_query.is_file()); @@ -239,7 +240,7 @@ async fn treesitter_parser_build_source_queries() { .await .unwrap(); - let etc = tree.layout_for(&package).etc; + let etc = tree.layout_for(package.spec()).etc; let highlights_query = etc.join("queries").join("tmux").join("highlights.scm"); assert!(highlights_query.is_file()); @@ -297,7 +298,7 @@ async fn test_build_local_project_no_source() { .await .unwrap(); - let layout = tree.layout_for(&package); + let layout = tree.layout_for(package.spec()); let conf_file = layout.conf.join("foo").join("bar.toml"); assert!(conf_file.is_file()); @@ -338,7 +339,7 @@ async fn test_build_local_project_only_src() { .await .unwrap(); - let src = tree.layout_for(&pkg).src; + let src = tree.layout_for(pkg.spec()).src; assert!(src.is_dir()); assert!(src.join("main.lua").is_file()); assert!(src.join("foo.lua").is_file()); @@ -475,7 +476,7 @@ async fn test_multiline_command_build() { .await .unwrap(); - let success_dir = tree.layout_for(&package).src.join("success"); + let success_dir = tree.layout_for(package.spec()).src.join("success"); assert!(success_dir.is_dir()); } diff --git a/lux-lib/tests/build_workspace.rs b/lux-lib/tests/build_workspace.rs index 0306175f1..60854aff0 100644 --- a/lux-lib/tests/build_workspace.rs +++ b/lux-lib/tests/build_workspace.rs @@ -5,6 +5,8 @@ use assert_fs::TempDir; use flaky_test::flaky_test; use lux_lib::lua_version::LuaVersion; use lux_lib::operations::BuildWorkspace; +use lux_lib::package::PackageName; +use lux_lib::tree::InstallTree; use lux_lib::workspace::Workspace; use lux_lib::{config::ConfigBuilder, lua_installation::detect_installed_lua_version}; @@ -29,4 +31,33 @@ async fn test_build_multi_workspace_local_dependencies() { .build() .await .unwrap(); + + // Building an already-built workspace must be idempotent + BuildWorkspace::new(&workspace, &config) + .no_lock(false) + .only_deps(false) + .build() + .await + .unwrap(); + + let lockfile = workspace.tree(&config).unwrap().lockfile().unwrap(); + + let foo = lockfile + .entrypoint(&PackageName::new("foo".into())) + .expect("foo must be recorded as an entrypoint"); + let bar = lockfile + .entrypoint(&PackageName::new("bar".into())) + .expect("bar must be recorded as an entrypoint"); + + assert!( + foo.dependencies().is_empty(), + "foo declares no dependencies and must not be linked to its sibling member bar" + ); + assert!( + bar.dependencies() + .into_iter() + .filter_map(|id| lockfile.get(id)) + .any(|pkg| pkg.name() == foo.name()), + "bar's path dependency on foo must be recorded as a tree edge" + ); } diff --git a/lux-lib/tests/install.rs b/lux-lib/tests/install.rs index 26ccb7a1f..df10185c4 100644 --- a/lux-lib/tests/install.rs +++ b/lux-lib/tests/install.rs @@ -2,13 +2,16 @@ use assert_fs::TempDir; use flaky_test::flaky_test; use itertools::Itertools; use lux_lib::{ + build::BuildBehaviour, config::ConfigBuilder, + drivers::install_packages::InstallPackages, git::{GitRef, GitSource}, lua_installation::detect_installed_lua_version, lua_rockspec::RockSourceSpec, lua_version::LuaVersion, - operations::{Exec, Install, PackageInstallSpec}, + operations::{Exec, PackageInstallSpec}, package::PackageReq, + package_db::PackageDB, tree::EntryType, }; use std::path::PathBuf; @@ -27,6 +30,7 @@ async fn install_git_package() { .unwrap(), git_ref: Some(GitRef::Tag("v6.0.3".into())), })) + .build_behaviour(BuildBehaviour::Force) .build(); test_install(install_spec).await } @@ -40,7 +44,9 @@ async fn install_http_package() { // See https://github.com/wahern/luaossl/issues/220#issuecomment-3401472124 std::env::set_var("CFLAGS", "-Wno-error=incompatible-pointer-types"); let install_spec = - PackageInstallSpec::new("http@0.4-0".parse().unwrap(), EntryType::Entrypoint).build(); + PackageInstallSpec::new("http@0.4-0".parse().unwrap(), EntryType::Entrypoint) + .build_behaviour(BuildBehaviour::Force) + .build(); test_install(install_spec).await; std::env::set_var("CFLAGS", cflags); } @@ -57,7 +63,9 @@ async fn install_and_use_toml_edit() { #[cfg(test)] async fn install_and_use(package: PackageReq, module_name: &str) { - let install_spec = PackageInstallSpec::new(package, EntryType::Entrypoint).build(); + let install_spec = PackageInstallSpec::new(package, EntryType::Entrypoint) + .build_behaviour(BuildBehaviour::Force) + .build(); let dir = TempDir::new().unwrap(); let lua_version = detect_installed_lua_version().or(Some(LuaVersion::Lua51)); @@ -71,12 +79,13 @@ async fn install_and_use(package: PackageReq, module_name: &str) { let tree = config .user_tree(LuaVersion::from(&config).unwrap().clone()) .unwrap(); - let installed = Install::new(&config) + let installed = InstallPackages::new(&config, &tree) .package(install_spec) - .tree(tree) + .package_db(PackageDB::from_config(&config).await.unwrap()) .install() .await - .unwrap(); + .unwrap() + .0; assert!(!installed.is_empty()); Exec::new("lua", None, &config) @@ -97,7 +106,9 @@ async fn no_build_artifacts_in_cwd() { .map(|entry| entry.into_path()) .collect_vec(); let install_spec = - PackageInstallSpec::new("bit32@5.3.5".parse().unwrap(), EntryType::Entrypoint).build(); + PackageInstallSpec::new("bit32@5.3.5".parse().unwrap(), EntryType::Entrypoint) + .build_behaviour(BuildBehaviour::Force) + .build(); test_install(install_spec).await; let build_artifacts = WalkDir::new(&cwd) .into_iter() @@ -123,11 +134,12 @@ async fn test_install(install_spec: PackageInstallSpec) { let tree = config .user_tree(LuaVersion::from(&config).unwrap().clone()) .unwrap(); - let installed = Install::new(&config) + let installed = InstallPackages::new(&config, &tree) .package(install_spec) - .tree(tree) + .package_db(PackageDB::from_config(&config).await.unwrap()) .install() .await - .unwrap(); + .unwrap() + .0; assert!(!installed.is_empty()); } diff --git a/lux-lib/tests/sync.rs b/lux-lib/tests/sync.rs index 55e5af640..b9fe0e3c0 100644 --- a/lux-lib/tests/sync.rs +++ b/lux-lib/tests/sync.rs @@ -2,7 +2,7 @@ use assert_fs::{prelude::PathCopy, TempDir}; use flaky_test::flaky_test; use lux_lib::{ config::ConfigBuilder, - operations::Sync, + drivers::sync::{Sync, SyncMode, TargetSet}, tree::{InstallTree, RockMatches}, workspace::Workspace, }; @@ -22,9 +22,9 @@ async fn sync_test_dependencies_empty_project() { String::from_utf8(tokio::fs::read(workspace.lockfile_path()).await.unwrap()); Sync::new(&workspace, &config) + .mode(SyncMode::Frozen) + .targets(TargetSet::all(&workspace, true)) .validate_integrity(cfg!(not(target_os = "windows"))) - .test(true) - .fast(true) .sync() .await .unwrap(); @@ -71,9 +71,9 @@ async fn sync_multi_projects_same_dependencies() { let config = ConfigBuilder::new().unwrap().build().unwrap(); Sync::new(&workspace, &config) + .mode(SyncMode::Frozen) + .targets(TargetSet::all(&workspace, true)) .validate_integrity(cfg!(not(target_os = "windows"))) - .test(true) - .fast(true) .sync() .await .unwrap(); @@ -121,6 +121,8 @@ fallo = "2.2.0" let workspace_lockfile = project_dir.join("lux.lock"); Sync::new(&workspace, &config) + .mode(SyncMode::Frozen) + .targets(TargetSet::all(&workspace, false)) .validate_integrity(cfg!(not(target_os = "windows"))) .sync() .await @@ -133,6 +135,8 @@ fallo = "2.2.0" tokio::fs::remove_dir_all(workspace_tree).await.unwrap(); Sync::new(&workspace, &config) + .mode(SyncMode::Frozen) + .targets(TargetSet::all(&workspace, false)) .validate_integrity(cfg!(not(target_os = "windows"))) .sync() .await @@ -158,6 +162,8 @@ async fn sync_dependencies_adds_luarocks_build_backend() { let config = ConfigBuilder::new().unwrap().build().unwrap(); Sync::new(&workspace, &config) + .mode(SyncMode::Frozen) + .targets(TargetSet::all(&workspace, false)) .validate_integrity(cfg!(not(target_os = "windows"))) .sync() .await @@ -185,6 +191,8 @@ async fn sync_dependencies_adds_transitive_build_dependencies() { let config = ConfigBuilder::new().unwrap().build().unwrap(); Sync::new(&workspace, &config) + .mode(SyncMode::Frozen) + .targets(TargetSet::all(&workspace, false)) .validate_integrity(cfg!(not(target_os = "windows"))) .sync() .await diff --git a/lux-lib/tests/vendor.rs b/lux-lib/tests/vendor.rs index ff9e2b4af..61d66b400 100644 --- a/lux-lib/tests/vendor.rs +++ b/lux-lib/tests/vendor.rs @@ -7,6 +7,7 @@ use assert_fs::{ use flaky_test::flaky_test; use lux_lib::{ config::ConfigBuilder, + lua_rockspec::RemoteLuaRockspec, operations::{Vendor, VendorTarget}, workspace::Workspace, }; @@ -20,6 +21,22 @@ async fn vendor_dependencies() { let temp_dir = assert_fs::TempDir::new().unwrap(); temp_dir.copy_from(sample_project_dir, &["**"]).unwrap(); let workspace = Workspace::from_exact(temp_dir.path()).unwrap().unwrap(); + + // Every rock in the sample project's committed lockfile must be vendored. + // (These are the `test_dependencies` of `busted-with-lockfile`.) + let expected_packages = [ + "luassert@1.9.0-1", + "lua_cliargs@3.0.2-1", + "penlight@1.15.0-1", + "lua-term@0.8-1", + "dkjson@2.11-1", + "mediator_lua@1.1.2-0", + "luafilesystem@1.9.0-1", + "busted@2.3.0-1", + "luasystem@0.7.1-1", + "say@1.4.1-3", + ]; + let config = ConfigBuilder::new().unwrap().build().unwrap(); let vendor_dir = assert_fs::TempDir::new().unwrap(); @@ -31,16 +48,34 @@ async fn vendor_dependencies() { .await .unwrap(); - let busted_rockspec = vendor_dir.child("busted-2.3.0-1.rockspec"); - busted_rockspec.assert(predicate::path::is_file()); + for package in expected_packages { + let (name, version) = package.split_once('@').unwrap(); + + // The rockspec must be vendored and parse back into a valid rockspec. + let rockspec_file = vendor_dir + .to_path_buf() + .join(format!("{name}-{version}.rockspec")); + assert!(rockspec_file.is_file(), "missing rockspec for {package}"); + let rockspec_content = std::fs::read_to_string(&rockspec_file).unwrap(); + RemoteLuaRockspec::new(&rockspec_content) + .unwrap_or_else(|err| panic!("vendored rockspec for {package} failed to parse: {err}")); + + let source_path = vendor_dir.to_path_buf().join(package); + assert!( + source_path.is_dir() || source_path.is_file(), + "missing vendored source for {package}" + ); + } + + // The sample lockfile's dependencies are all `rockspec` sources, so the + // vendored sources must be non-empty directories containing their contents. let busted_dir = vendor_dir.child("busted@2.3.0-1"); busted_dir.assert(predicate::path::is_dir()); - let luasystem_rockspec = vendor_dir.child("luasystem-0.7.1-1.rockspec"); - luasystem_rockspec.assert(predicate::path::is_file()); - let luasystem_dir = vendor_dir.child("luasystem@0.7.1-1"); - luasystem_dir.assert(predicate::path::is_dir()); - let say_rockspec = vendor_dir.child("say-1.4.1-3.rockspec"); - say_rockspec.assert(predicate::path::is_file()); - let say_dir = vendor_dir.child("say@1.4.1-3"); - say_dir.assert(predicate::path::is_dir()); + assert!( + std::fs::read_dir(busted_dir.path()) + .unwrap() + .next() + .is_some(), + "vendored source tree for busted is empty" + ); } diff --git a/lux-lua/src/loader.rs b/lux-lua/src/loader.rs index 977853a67..f90f36681 100644 --- a/lux-lua/src/loader.rs +++ b/lux-lua/src/loader.rs @@ -5,7 +5,7 @@ use std::{ rc::Rc, }; -use lux_lib::lockfile::{LocalPackage, LocalPackageId, Lockfile, ReadOnly}; +use lux_lib::lockfile::{LockedPackage, LockedPackageId, Lockfile, ReadOnly}; use mlua::prelude::*; use path_absolutize::Absolutize; @@ -47,8 +47,8 @@ fn current_file(lua: &Lua) -> Option { fn rock_layout( tree_root: &Path, - package_id: &LocalPackageId, - package: &LocalPackage, + package_id: &LockedPackageId, + package: &LockedPackage, ) -> (PathBuf, PathBuf) { let rock_path = tree_root.join(format!( "{}-{}@{}", @@ -218,7 +218,7 @@ fn load_from_workspace_tree( // with the lux tree and malform the package hash. In this case, this // should never cause any security-related problems anyway, as we'll // crash right after this function returns None. - let owner_id = unsafe { LocalPackageId::from_unchecked(module_hash.to_string()) }; + let owner_id = unsafe { LockedPackageId::from_unchecked(module_hash.to_string()) }; let Some(package) = lockfile.get(&owner_id) else { return Ok(None); }; @@ -442,7 +442,7 @@ mod tests { assert_eq!(result, "nested"); } - #[cfg(target_os = "linux")] + #[cfg(all(target_os = "linux", feature = "impure_tests"))] #[tokio::test] async fn test_load_c_module_from_built_project() { use lux_lib::{ diff --git a/lux-lua/src/lua_impls.rs b/lux-lua/src/lua_impls.rs index 5405f2180..91f48ceed 100644 --- a/lux-lua/src/lua_impls.rs +++ b/lux-lua/src/lua_impls.rs @@ -21,8 +21,8 @@ use lux_lib::{ config::{Config, ConfigBuilder}, git::{GitRef, GitSource}, lockfile::{ - LocalPackage, LocalPackageHashes, LocalPackageId, LockConstraint, Lockfile, LockfileGuard, - OptState, PinnedState, ReadOnly, ReadWrite, + LockConstraint, LockedPackage, LockedPackageHashes, LockedPackageId, Lockfile, + LockfileGuard, OptState, PinnedState, ReadOnly, ReadWrite, }, lua_rockspec::{ BuildBackendSpec, BuildSpec, BuiltinBuildSpec, BustedTestSpec, CMakeBuildSpec, @@ -35,11 +35,11 @@ use lux_lib::{ lua_version::LuaVersion, operations::{DownloadedRockspec, PackageInstallSpec, SyncReport}, package::{PackageName, PackageReq, PackageSpec, PackageVersion, PackageVersionReq, SpecRev}, + package_db::PackageDB, project::{ project_toml::{LocalProjectToml, PartialProjectToml, RemoteProjectToml}, Project, }, - remote_package_db::RemotePackageDB, rockspec::{ lua_dependency::{DependencyType, LuaDependencySpec, LuaDependencyType}, Rockspec, @@ -264,23 +264,23 @@ impl IntoLua for OptStateLua { } #[derive(Debug, Clone)] -pub struct LocalPackageIdLua(pub LocalPackageId); +pub struct LockedPackageIdLua(pub LockedPackageId); -impl Typed for LocalPackageIdLua { +impl Typed for LockedPackageIdLua { fn ty() -> Type { Type::string() } } -impl FromLua for LocalPackageIdLua { +impl FromLua for LockedPackageIdLua { fn from_lua(value: LuaValue, lua: &Lua) -> LuaResult { - Ok(LocalPackageIdLua(unsafe { - LocalPackageId::from_unchecked(String::from_lua(value, lua)?) + Ok(LockedPackageIdLua(unsafe { + LockedPackageId::from_unchecked(String::from_lua(value, lua)?) })) } } -impl IntoLua for LocalPackageIdLua { +impl IntoLua for LockedPackageIdLua { fn into_lua(self, lua: &Lua) -> LuaResult { self.0.into_string().into_lua(lua) } @@ -565,10 +565,10 @@ impl IntoLua for RockMatchesLua { table.set("is_found", lua.create_function(move |_, ()| Ok(is_found))?)?; match self.0 { RockMatches::NotFound(req) => table.set("not_found", PackageReqLua(req))?, - RockMatches::Single(id) => table.set("single", LocalPackageIdLua(id))?, + RockMatches::Single(id) => table.set("single", LockedPackageIdLua(id))?, RockMatches::Many(ids) => table.set( "many", - ids.into_iter().map(LocalPackageIdLua).collect::>(), + ids.into_iter().map(LockedPackageIdLua).collect::>(), )?, } Ok(LuaValue::Table(table)) @@ -712,7 +712,7 @@ impl IntoLua for SyncReportLua { .added() .iter() .cloned() - .map(LocalPackageLua) + .map(LockedPackageLua) .collect::>(), )?; table.set( @@ -721,7 +721,7 @@ impl IntoLua for SyncReportLua { .removed() .iter() .cloned() - .map(LocalPackageLua) + .map(LockedPackageLua) .collect::>(), )?; Ok(LuaValue::Table(table)) @@ -863,17 +863,17 @@ impl mlua::UserData for PackageReqLua { } #[derive(Debug, Clone)] -pub struct LocalPackageHashesLua(pub LocalPackageHashes); +pub struct LockedPackageHashesLua(pub LockedPackageHashes); -impl Typed for LocalPackageHashesLua { +impl Typed for LockedPackageHashesLua { fn ty() -> Type { - Type::named("LocalPackageHashes") + Type::named("LockedPackageHashes") } } -impl_from_lua_userdata!(LocalPackageHashesLua); +impl_from_lua_userdata!(LockedPackageHashesLua); -impl TypedUserData for LocalPackageHashesLua { +impl TypedUserData for LockedPackageHashesLua { fn add_methods>(methods: &mut M) { methods.param("rockspec", "sha256sum of the rockspec"); methods.add_method("rockspec", |_, this, ()| Ok(this.0.rockspec.to_hex().1)); @@ -885,7 +885,7 @@ impl TypedUserData for LocalPackageHashesLua { } } -impl mlua::UserData for LocalPackageHashesLua { +impl mlua::UserData for LockedPackageHashesLua { fn add_fields>(fields: &mut F) { let mut wrapper = mlua_extras::typed::WrappedBuilder::new(fields); ::add_fields(&mut wrapper); @@ -898,30 +898,30 @@ impl mlua::UserData for LocalPackageHashesLua { } #[derive(Debug, Clone)] -pub struct LocalPackageLua(pub LocalPackage); +pub struct LockedPackageLua(pub LockedPackage); -impl Typed for LocalPackageLua { +impl Typed for LockedPackageLua { fn ty() -> Type { - Type::named("LocalPackage") + Type::named("LockedPackage") } } -impl FromLua for LocalPackageLua { +impl FromLua for LockedPackageLua { fn from_lua(value: LuaValue, _lua: &Lua) -> LuaResult { match value { - LuaValue::UserData(ud) => Ok(ud.borrow::()?.clone()), + LuaValue::UserData(ud) => Ok(ud.borrow::()?.clone()), v => Err(LuaError::FromLuaConversionError { from: v.type_name(), - to: "LocalPackageLua".to_string(), + to: "LockedPackageLua".to_string(), message: None, }), } } } -impl TypedUserData for LocalPackageLua { +impl TypedUserData for LockedPackageLua { fn add_methods>(methods: &mut M) { - methods.add_method("id", |_, this, ()| Ok(LocalPackageIdLua(this.0.id()))); + methods.add_method("id", |_, this, ()| Ok(LockedPackageIdLua(this.0.id()))); methods.add_method("name", |_, this, ()| { Ok(PackageNameLua(this.0.name().clone())) }); @@ -934,14 +934,14 @@ impl TypedUserData for LocalPackageLua { .0 .dependencies() .into_iter() - .map(|id| LocalPackageIdLua(id.clone())) + .map(|id| LockedPackageIdLua(id.clone())) .collect::>()) }); methods.add_method("constraint", |_, this, ()| { Ok(LockConstraintLua(this.0.constraint())) }); methods.add_method("hashes", |_, this, ()| { - Ok(LocalPackageHashesLua(this.0.hashes().clone())) + Ok(LockedPackageHashesLua(this.0.hashes().clone())) }); methods.add_method("to_package", |_, this, ()| { Ok(PackageSpecLua(this.0.to_package())) @@ -955,7 +955,7 @@ impl TypedUserData for LocalPackageLua { } } -impl mlua::UserData for LocalPackageLua { +impl mlua::UserData for LockedPackageLua { fn add_fields>(fields: &mut F) { let mut wrapper = mlua_extras::typed::WrappedBuilder::new(fields); ::add_fields(&mut wrapper); @@ -1064,10 +1064,10 @@ impl TypedUserData for TreeLua { }); methods.document("The root directory of a package in this tree"); methods.param("package", ""); - methods.add_method("root_for", |_, this, package: LocalPackageLua| { + methods.add_method("root_for", |_, this, package: LockedPackageLua| { Ok(this .0 - .layout_for(&package.0) + .layout_for(package.0.spec()) .root .to_slash_lossy() .into_owned()) @@ -1078,8 +1078,8 @@ impl TypedUserData for TreeLua { }); methods.document("Get the `RockLayout` for an installed package."); methods.param("package", ""); - methods.add_method("rock_layout", |_, this, package: LocalPackageLua| { - Ok(RockLayoutLua(this.0.layout_for(&package.0))) + methods.add_method("rock_layout", |_, this, package: LockedPackageLua| { + Ok(RockLayoutLua(this.0.layout_for(package.0.spec()))) }); methods.document("Find installed rocks that match the given `PackageReq`"); methods.param("req", ""); @@ -2742,28 +2742,28 @@ impl mlua::UserData for RemoteProjectTomlLua { } #[derive(Debug, Clone)] -pub struct RemotePackageDBLua(pub RemotePackageDB); +pub struct PackageDBLua(pub PackageDB); -impl Typed for RemotePackageDBLua { +impl Typed for PackageDBLua { fn ty() -> Type { - Type::named("RemotePackageDB") + Type::named("PackageDB") } } -impl FromLua for RemotePackageDBLua { +impl FromLua for PackageDBLua { fn from_lua(value: LuaValue, _lua: &Lua) -> LuaResult { match value { - LuaValue::UserData(ud) => Ok(ud.borrow::()?.clone()), + LuaValue::UserData(ud) => Ok(ud.borrow::()?.clone()), v => Err(LuaError::FromLuaConversionError { from: v.type_name(), - to: "RemotePackageDBLua".to_string(), + to: "PackageDBLua".to_string(), message: None, }), } } } -impl mlua::UserData for RemotePackageDBLua { +impl mlua::UserData for PackageDBLua { fn add_fields>(fields: &mut F) { let mut wrapper = mlua_extras::typed::WrappedBuilder::new(fields); ::add_fields(&mut wrapper); @@ -2775,7 +2775,7 @@ impl mlua::UserData for RemotePackageDBLua { } } -impl TypedUserData for RemotePackageDBLua { +impl TypedUserData for PackageDBLua { fn add_methods>(methods: &mut M) { methods.document("Search for all packages that match the requirement"); methods.param( @@ -2834,15 +2834,15 @@ impl TypedUserData for LockfileReadOnlyLua { .map(|(id, rock)| { ( id.clone().into_string().clone(), - LocalPackageLua(rock.clone()), + LockedPackageLua(rock.clone()), ) }) .collect::>()) }); methods.param("id", ""); - methods.add_method("get", |_, this, id: LocalPackageIdLua| { - Ok(this.0.get(&id.0).cloned().map(LocalPackageLua)) + methods.add_method("get", |_, this, id: LockedPackageIdLua| { + Ok(this.0.get(&id.0).cloned().map(LockedPackageLua)) }); methods.document( @@ -2891,15 +2891,15 @@ impl TypedUserData for LockfileGuardLua { .map(|(id, rock)| { ( id.clone().into_string().clone(), - LocalPackageLua(rock.clone()), + LockedPackageLua(rock.clone()), ) }) .collect::>()) }); methods.param("id", ""); - methods.add_method("get", |_, this, id: LocalPackageIdLua| { - Ok(this.0.get(&id.0).cloned().map(LocalPackageLua)) + methods.add_method("get", |_, this, id: LockedPackageIdLua| { + Ok(this.0.get(&id.0).cloned().map(LockedPackageLua)) }); } fn add_documentation>(docs: &mut F) { @@ -2939,7 +2939,7 @@ impl TypedUserData for LockfileReadWriteLua { .map(|(id, rock)| { ( id.clone().into_string().clone(), - LocalPackageLua(rock.clone()), + LockedPackageLua(rock.clone()), ) }) .collect::>()) @@ -2949,9 +2949,9 @@ impl TypedUserData for LockfileReadWriteLua { methods.add_method("get", |_, this, id: String| { Ok(this .0 - .get(unsafe { &LocalPackageId::from_unchecked(id) }) + .get(unsafe { &LockedPackageId::from_unchecked(id) }) .cloned() - .map(LocalPackageLua)) + .map(LockedPackageLua)) }); } fn add_documentation>(docs: &mut F) { @@ -3133,7 +3133,7 @@ impl TypedUserData for ProjectLua { let _guard = lux_lib::lua::lua_runtime().enter(); let deps = map_dependency_type(deps.0); let package_db = - RemotePackageDB::from_config(&config.0) + PackageDB::from_config(&config.0) .await .into_lua_err()?; this.0.add(deps.as_ref(), &package_db).await.into_lua_err() @@ -3219,11 +3219,11 @@ mod definitions_registry { use super::{ BuildSpecLua, BustedTestSpecLua, CMakeBuildSpecLua, CommandBuildSpecLua, CommandTestSpecLua, ConfigBuilderLua, ConfigLua, DownloadedRockspecLua, GitSourceLua, - InstallSpecLua, LocalLuaRockspecLua, LocalPackageHashesLua, LocalPackageLua, - LocalProjectTomlLua, LockfileGuardLua, LockfileReadOnlyLua, LockfileReadWriteLua, + InstallSpecLua, LocalLuaRockspecLua, LocalProjectTomlLua, LockedPackageHashesLua, + LockedPackageLua, LockfileGuardLua, LockfileReadOnlyLua, LockfileReadWriteLua, LuaDependencySpecLua, LuaScriptTestSpecLua, MakeBuildSpecLua, ModulePathsLua, - NvimLayoutLua, PackageReqLua, PackageSpecLua, PartialLuaRockspecLua, PartialProjectTomlLua, - PlatformSupportLua, ProjectLua, RemoteLuaRockspecLua, RemotePackageDBLua, + NvimLayoutLua, PackageDBLua, PackageReqLua, PackageSpecLua, PartialLuaRockspecLua, + PartialProjectTomlLua, PlatformSupportLua, ProjectLua, RemoteLuaRockspecLua, RemoteProjectTomlLua, RemoteRockSourceLua, RockDescriptionLua, RockLayoutLua, RustBinaryBuildSpecLua, RustMluaBuildSpecLua, TreeLua, TreesitterParserBuildSpecLua, WorkspaceLua, @@ -3246,8 +3246,8 @@ mod definitions_registry { submit_definitions! { "PackageSpec" => PackageSpecLua, "PackageReq" => PackageReqLua, - "LocalPackageHashes" => LocalPackageHashesLua, - "LocalPackage" => LocalPackageLua, + "LockedPackageHashes" => LockedPackageHashesLua, + "LockedPackage" => LockedPackageLua, "RockLayout" => RockLayoutLua, "Tree" => TreeLua, "NvimLayout" => NvimLayoutLua, @@ -3276,7 +3276,7 @@ mod definitions_registry { "PartialProjectToml" => PartialProjectTomlLua, "LocalProjectToml" => LocalProjectTomlLua, "RemoteProjectToml" => RemoteProjectTomlLua, - "RemotePackageDB" => RemotePackageDBLua, + "PackageDB" => PackageDBLua, "LockfileReadOnly" => LockfileReadOnlyLua, "LockfileGuard" => LockfileGuardLua, "LockfileReadWrite" => LockfileReadWriteLua, @@ -3329,14 +3329,16 @@ impl From for EntryType { #[serde(rename_all = "snake_case")] enum BuildBehaviourLua { #[default] - NoForce, + Ignore, + Conflict, Force, } impl From for BuildBehaviour { fn from(val: BuildBehaviourLua) -> Self { match val { - BuildBehaviourLua::NoForce => Self::NoForce, + BuildBehaviourLua::Ignore => Self::Ignore, + BuildBehaviourLua::Conflict => Self::Conflict, BuildBehaviourLua::Force => Self::Force, } } @@ -3346,7 +3348,7 @@ impl From for BuildBehaviour { /// ```lua /// "say >= 1.3" /// -/// { package = "say >= 1.3", entry_type = "entrypoint", pin = false, opt = false, build_behaviour = "no_force" } +/// { package = "say >= 1.3", entry_type = "entrypoint", pin = false, opt = false, build_behaviour = "ignore" } /// ``` #[derive(Deserialize)] #[serde(untagged)] diff --git a/lux-lua/src/operations.rs b/lux-lua/src/operations.rs index 16817ce95..e2a5a0e22 100644 --- a/lux-lua/src/operations.rs +++ b/lux-lua/src/operations.rs @@ -6,14 +6,17 @@ use std::{collections::HashMap, path::PathBuf}; use itertools::Itertools; use lux_lib::tree::InstallTree; use lux_lib::{ - lockfile::LocalPackageId, + build::BuildBehaviour, + drivers::install_packages::InstallPackages, + drivers::sync::{Sync, SyncMode, TargetSet}, + lockfile::LockedPackageId, lua::lua_runtime, operations::{ - set_pinned_state, BuildWorkspace, DistProjectBin, Download, Install, PackageInstallSpec, - Sync, Uninstall, Update, + set_pinned_state, BuildWorkspace, DistProjectBin, Download, PackageInstallSpec, Uninstall, + Update, }, package::{PackageName, PackageReq}, - remote_package_db::RemotePackageDB, + package_db::PackageDB, rockspec::lua_dependency::DependencyType, tree::{EntryType, RockMatches, Tree}, }; @@ -22,9 +25,9 @@ use mlua_extras::typed::{Type, Typed, TypedDataMethods, TypedUserData}; use path_slash::PathBufExt; use crate::lua_impls::{ - self, ConfigLua, DependencyTypeLua, DownloadedRockspecLua, LocalPackageIdLua, LocalPackageLua, - PackageInstallSpecLua, PackageNameLua, PackageReqLua, PinnedStateLua, ProjectLua, - SyncReportLua, TreeLua, WorkspaceLua, + self, ConfigLua, DependencyTypeLua, DownloadedRockspecLua, LockedPackageIdLua, + LockedPackageLua, PackageInstallSpecLua, PackageNameLua, PackageReqLua, PinnedStateLua, + ProjectLua, SyncReportLua, TreeLua, WorkspaceLua, }; #[derive(Clone)] @@ -61,13 +64,14 @@ impl TypedUserData for OperationsModule { |_, (packages, tree, config): (Vec, TreeLua, ConfigLua)| async move { let _runtime = lua_runtime().enter(); let specs = packages.into_iter().map(|p| p.0).collect(); - Install::new(&config.0) + let package_db = PackageDB::from_config(&config.0).await.into_lua_err()?; + InstallPackages::new(&config.0, &tree.0) + .package_db(package_db) .packages(specs) - .tree(tree.0) .install() .await .into_lua_err() - .map(|pkgs| pkgs.into_iter().map(LocalPackageLua).collect::>()) + .map(|(pkgs, _)| pkgs.into_iter().map(LockedPackageLua).collect::>()) }, ); @@ -77,7 +81,7 @@ impl TypedUserData for OperationsModule { methods.param("config", "Lux config"); methods.add_async_function( "uninstall", - |_, (packages, tree, config): (Vec, Option, ConfigLua)| async move { + |_, (packages, tree, config): (Vec, Option, ConfigLua)| async move { let _runtime = lua_runtime().enter(); let ids = packages.into_iter().map(|p| p.0); Uninstall::new() @@ -87,7 +91,7 @@ impl TypedUserData for OperationsModule { .remove() .await .into_lua_err() - .map(|ids| ids.into_iter().map(LocalPackageIdLua).collect::>()) + .map(|ids| ids.into_iter().map(LockedPackageIdLua).collect::>()) }, ); @@ -118,7 +122,7 @@ impl TypedUserData for OperationsModule { .update() .await .into_lua_err() - .map(|pkgs| pkgs.into_iter().map(LocalPackageLua).collect::>()) + .map(|pkgs| pkgs.into_iter().map(LockedPackageLua).collect::>()) }, ); @@ -140,9 +144,7 @@ impl TypedUserData for OperationsModule { )| async move { let _runtime = lua_runtime().enter(); let deps = lua_impls::map_dependency_type(deps.0); - let package_db = RemotePackageDB::from_config(&config.0) - .await - .into_lua_err()?; + let package_db = PackageDB::from_config(&config.0).await.into_lua_err()?; let install_specs = deps_to_specs(&deps); @@ -157,13 +159,14 @@ impl TypedUserData for OperationsModule { .await .into_lua_err()?; - Install::new(&config.0) + let tree = workspace.0.tree(&config.0).into_lua_err()?; + InstallPackages::new(&config.0, &tree) + .package_db(package_db) .packages(install_specs) - .tree(workspace.0.tree(&config.0).into_lua_err()?) .install() .await .into_lua_err() - .map(|pkgs| pkgs.into_iter().map(LocalPackageLua).collect::>()) + .map(|(pkgs, _)| pkgs.into_iter().map(LockedPackageLua).collect::>()) }, ); @@ -187,7 +190,7 @@ impl TypedUserData for OperationsModule { let _runtime = lua_runtime().enter(); let deps = lua_impls::map_dependency_type_names(deps.0); let tree = workspace.0.tree(&config.0).into_lua_err()?; - let ids_to_remove: Vec = deps_to_ids(&deps, &tree); + let ids_to_remove: Vec = deps_to_ids(&deps, &tree); if ids_to_remove.is_empty() { return Ok(Vec::new()); @@ -207,7 +210,7 @@ impl TypedUserData for OperationsModule { .remove() .await .into_lua_err() - .map(|ids| ids.into_iter().map(LocalPackageIdLua).collect::>()) + .map(|ids| ids.into_iter().map(LockedPackageIdLua).collect::>()) }, ); @@ -219,7 +222,8 @@ impl TypedUserData for OperationsModule { |_, (workspace, config): (WorkspaceLua, ConfigLua)| async move { let _runtime = lua_runtime().enter(); Sync::new(&workspace.0, &config.0) - .test(true) + .mode(SyncMode::Frozen) + .targets(TargetSet::all(&workspace.0, true)) .sync() .await .into_lua_err() @@ -235,6 +239,8 @@ impl TypedUserData for OperationsModule { |_, (workspace, config): (WorkspaceLua, ConfigLua)| async move { let _runtime = lua_runtime().enter(); Sync::new(&workspace.0, &config.0) + .mode(SyncMode::Frozen) + .targets(TargetSet::all(&workspace.0, false)) .sync() .await .into_lua_err() @@ -261,7 +267,7 @@ impl TypedUserData for OperationsModule { .build() .await .into_lua_err() - .map(|packages: Vec<_>| packages.into_iter().map(LocalPackageLua).collect_vec()) + .map(|packages: Vec<_>| packages.into_iter().map(LockedPackageLua).collect_vec()) }, ); @@ -312,7 +318,7 @@ impl TypedUserData for OperationsModule { methods.param("pin_state", "The pinned state to set"); methods.add_function( "pin", - |_, (package_id, tree, pin_state): (LocalPackageIdLua, TreeLua, PinnedStateLua)| { + |_, (package_id, tree, pin_state): (LockedPackageIdLua, TreeLua, PinnedStateLua)| { set_pinned_state(&package_id.0, &tree.0, pin_state.0).into_lua_err() }, ); @@ -351,9 +357,7 @@ mod definitions_registry { } async fn search(query: String, config: ConfigLua) -> mlua::Result>> { - let remote_db = RemotePackageDB::from_config(&config.0) - .await - .into_lua_err()?; + let remote_db = PackageDB::from_config(&config.0).await.into_lua_err()?; Ok(remote_db .search(&query.parse().into_lua_err()?) @@ -374,11 +378,15 @@ fn deps_to_specs(deps: &DependencyType) -> Vec { }; reqs.iter() - .map(|r| PackageInstallSpec::new(r.clone(), EntryType::Entrypoint).build()) + .map(|r| { + PackageInstallSpec::new(r.clone(), EntryType::Entrypoint) + .build_behaviour(BuildBehaviour::Force) + .build() + }) .collect() } -fn deps_to_ids(deps: &DependencyType, tree: &Tree) -> Vec { +fn deps_to_ids(deps: &DependencyType, tree: &Tree) -> Vec { let names = match deps { DependencyType::Regular(d) | DependencyType::Build(d) | DependencyType::Test(d) => d, DependencyType::External(_) => &Vec::default(), @@ -477,7 +485,7 @@ type = "builtin" fn test_package_install_spec_from_full_table() { let lua = Lua::new(); let value = lua - .load(r#"{ package = "busted >= 2.0", entry_type = "dependency_only", pin = true, opt = true, build_behaviour = "no_force" }"#) + .load(r#"{ package = "busted >= 2.0", entry_type = "dependency_only", pin = true, opt = true, build_behaviour = "ignore" }"#) .eval() .unwrap(); PackageInstallSpecLua::from_lua(value, &lua).unwrap(); diff --git a/nix/README.md b/nix/README.md index e140e4888..677be36bb 100644 --- a/nix/README.md +++ b/nix/README.md @@ -37,7 +37,7 @@ buildLuxPackage { hash = "sha256-imyKEDvbn9EWJ8Cm54v5sUKZV2fZuBTT3du+1ipKgW4="; }; - luxHash = "sha256-DpkDMK6AV7Iy6YFbKbx+EEkDeGaAFbszNkLPhdVr+qY="; + luxHash = "sha256-XaA9FtwHtm8BSvjTyRHvAkgCZpbo48gNI5yfNQuRL78="; meta = { description = "Modern and ergonomic error handling for Lua"; @@ -70,7 +70,7 @@ buildLuxApplication { hash = "sha256-imyKEDvbn9EWJ8Cm54v5sUKZV2fZuBTT3du+1ipKgW4="; }; - luxHash = "sha256-DpkDMK6AV7Iy6YFbKbx+EEkDeGaAFbszNkLPhdVr+qY="; + luxHash = "sha256-XaA9FtwHtm8BSvjTyRHvAkgCZpbo48gNI5yfNQuRL78="; } ``` diff --git a/nix/tests/build-lux-helpers.nix b/nix/tests/build-lux-helpers.nix index 9a928e826..96ae15d46 100644 --- a/nix/tests/build-lux-helpers.nix +++ b/nix/tests/build-lux-helpers.nix @@ -74,7 +74,7 @@ ''; in { fetchLuxDeps = mkTest "build-lux-helpers-test" { - luxHash = "sha256-DpkDMK6AV7Iy6YFbKbx+EEkDeGaAFbszNkLPhdVr+qY="; + luxHash = "sha256-XaA9FtwHtm8BSvjTyRHvAkgCZpbo48gNI5yfNQuRL78="; }; importLuxLock = mkTest "build-lux-helpers-import-lock-test" { @@ -98,7 +98,7 @@ in { pname = "fallo"; version = "0-unstable-2026-03-25"; inherit src; - luxHash = "sha256-DpkDMK6AV7Iy6YFbKbx+EEkDeGaAFbszNkLPhdVr+qY="; + luxHash = "sha256-XaA9FtwHtm8BSvjTyRHvAkgCZpbo48gNI5yfNQuRL78="; }; in pkgs.runCommandLocal "build-lux-helpers-with-packages-test" {} ''