diff --git a/boot/components/core/all.go b/boot/components/core/all.go index 7e1571e48..de0956cc6 100644 --- a/boot/components/core/all.go +++ b/boot/components/core/all.go @@ -13,6 +13,7 @@ func All() []boot.Component { Dispatcher(), WASMIsolation(), Profiler(), + Artifacts(), Registry(), Finder(), Security(), diff --git a/boot/components/core/artifact.go b/boot/components/core/artifact.go new file mode 100644 index 000000000..7a3fb6d7f --- /dev/null +++ b/boot/components/core/artifact.go @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: MPL-2.0 + +package core + +import ( + "context" + + "github.com/wippyai/runtime/api/boot" + "github.com/wippyai/runtime/boot/deps/artifact" + "github.com/wippyai/runtime/boot/deps/artifact/standard" +) + +// Artifacts composes the artifact formats available to dependency lifecycle +// operations. Formats are explicit boot dependencies rather than globals. +func Artifacts() boot.Component { + return boot.New(boot.P{ + Name: ArtifactName, + Load: func(ctx context.Context) (context.Context, error) { + registry, err := standard.NewRegistry() + if err != nil { + return ctx, err + } + return artifact.WithRegistry(ctx, registry), nil + }, + }) +} diff --git a/boot/components/core/artifact_test.go b/boot/components/core/artifact_test.go new file mode 100644 index 000000000..d613c3062 --- /dev/null +++ b/boot/components/core/artifact_test.go @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: MPL-2.0 + +package core + +import ( + "testing" + + "github.com/stretchr/testify/require" + bootpkg "github.com/wippyai/runtime/boot" + "github.com/wippyai/runtime/boot/deps/artifact" + "go.uber.org/zap" +) + +func TestArtifactsBootRegistration(t *testing.T) { + ctx, err := bootpkg.NewBootstrapContext(zap.NewNop(), nil) + require.NoError(t, err) + loader, err := bootpkg.NewLoader(Artifacts()) + require.NoError(t, err) + ctx, err = loader.Load(ctx) + require.NoError(t, err) + + registry := artifact.GetRegistry(ctx) + require.NotNil(t, registry) + _, registered := registry.Resolve("node-package") + require.True(t, registered) +} diff --git a/boot/components/core/constants.go b/boot/components/core/constants.go index e4c11747b..2069f1448 100644 --- a/boot/components/core/constants.go +++ b/boot/components/core/constants.go @@ -2,13 +2,17 @@ package core -import "github.com/wippyai/runtime/api/boot" +import ( + "github.com/wippyai/runtime/api/boot" + "github.com/wippyai/runtime/boot/deps/artifact" +) const ( // PIDGenName is the name for the PID generator component PIDGenName boot.Name = "pidgen" SecurityName boot.Name = "security" SecurityPolicyName boot.Name = "security.policy" + ArtifactName boot.Name = artifact.ConfigName RegistryName boot.Name = "registry" FinderName boot.Name = "finder" SupervisorName boot.Name = "supervisor" diff --git a/boot/components/core/core_test.go b/boot/components/core/core_test.go index 1c1e3b4ff..a54a91491 100644 --- a/boot/components/core/core_test.go +++ b/boot/components/core/core_test.go @@ -82,7 +82,7 @@ func TestCorePlugins(t *testing.T) { t.Error("PID generator not available in context") } - lifecycleLoader, err := bootpkg.NewLoader(Registry(), Supervisor()) + lifecycleLoader, err := bootpkg.NewLoader(Artifacts(), Registry(), Supervisor()) require.NoError(t, err) ctx, err = lifecycleLoader.Load(ctx) require.NoError(t, err) diff --git a/boot/components/core/registry.go b/boot/components/core/registry.go index 3f2f665af..bb8869638 100644 --- a/boot/components/core/registry.go +++ b/boot/components/core/registry.go @@ -16,6 +16,7 @@ import ( logapi "github.com/wippyai/runtime/api/logs" regapi "github.com/wippyai/runtime/api/registry" bootpkg "github.com/wippyai/runtime/boot" + "github.com/wippyai/runtime/boot/deps/artifact" hubdeps "github.com/wippyai/runtime/boot/deps/hub" "github.com/wippyai/runtime/boot/deps/lock" "github.com/wippyai/runtime/system/registry" @@ -33,7 +34,7 @@ func Registry() boot.Component { return boot.New(boot.P{ Name: RegistryName, - DependsOn: []boot.Name{}, + DependsOn: []boot.Name{ArtifactName}, Load: func(ctx context.Context) (context.Context, error) { logger := logapi.GetLogger(ctx).Named("registry") bus := event.GetBus(ctx) @@ -119,7 +120,7 @@ func Registry() boot.Component { registryOpts := []registry.Option{} - depHandler, err := newDependencyHandler(cfg, logger.Named("dependency"), resolver) + depHandler, err := newDependencyHandler(ctx, cfg, logger.Named("dependency"), resolver) if err != nil { logger.Warn("dependency handler disabled", zap.Error(err)) } else if depHandler != nil { @@ -220,6 +221,7 @@ func readKindSlice(cfg boot.Config, key boot.Name) ([]regapi.Kind, bool) { } func newDependencyHandler( + ctx context.Context, cfg boot.Config, logger *zap.Logger, resolver regapi.DependencyResolver, @@ -233,6 +235,11 @@ func newDependencyHandler( Logger: logger, Resolver: resolver, } + artifactRegistry := artifact.GetRegistry(ctx) + if artifactRegistry == nil { + return nil, fmt.Errorf("artifact registry is not initialized") + } + opts.Artifacts = artifactRegistry workspaceReplacements, err := lock.WorkspaceReplacements(cfg) if err != nil { return nil, fmt.Errorf("load workspace replacements: %w", err) @@ -245,6 +252,7 @@ func newDependencyHandler( opts.LockPath = registryCfg.GetString(RegistryDependencyLockPath, "") opts.VendorDir = registryCfg.GetString(RegistryDependencyVendorDir, "") } + opts.ArtifactRoot = artifact.ConfiguredRoot(cfg, "") return hubdeps.NewDependencyHandler(opts) } diff --git a/boot/components/core/registry_test.go b/boot/components/core/registry_test.go index d0fd5878e..470239bdd 100644 --- a/boot/components/core/registry_test.go +++ b/boot/components/core/registry_test.go @@ -67,7 +67,7 @@ func TestRegistryPostgresHistoryRequiresDSN(t *testing.T) { ctx, err := bootpkg.NewBootstrapContext(zap.NewNop(), cfg) require.NoError(t, err) - loader, err := bootpkg.NewLoader(Registry()) + loader, err := bootpkg.NewLoader(Artifacts(), Registry()) require.NoError(t, err) _, err = loader.Load(ctx) diff --git a/boot/deps/artifact/config.go b/boot/deps/artifact/config.go new file mode 100644 index 000000000..7887234b0 --- /dev/null +++ b/boot/deps/artifact/config.go @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: MPL-2.0 + +package artifact + +import "github.com/wippyai/runtime/api/boot" + +const ( + // ConfigName is the application configuration section owned by artifacts. + ConfigName boot.Name = "artifact" + // ConfigMaterializationRoot overrides the application root for materialized artifacts. + ConfigMaterializationRoot boot.Name = "materialization_root" +) + +// ConfiguredRoot returns the configured materialization root or fallback. +func ConfiguredRoot(cfg boot.Config, fallback string) string { + if cfg == nil { + return fallback + } + return cfg.Sub(ConfigName).GetString(ConfigMaterializationRoot, fallback) +} diff --git a/boot/deps/artifact/config_test.go b/boot/deps/artifact/config_test.go new file mode 100644 index 000000000..bf353d5df --- /dev/null +++ b/boot/deps/artifact/config_test.go @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: MPL-2.0 + +package artifact + +import ( + "testing" + + "github.com/wippyai/runtime/api/boot" +) + +func TestConfiguredRoot(t *testing.T) { + if got := ConfiguredRoot(nil, ".wippy"); got != ".wippy" { + t.Fatalf("nil config root = %q", got) + } + + cfg := boot.NewConfig(boot.WithSection(ConfigName, map[string]any{ + ConfigMaterializationRoot: "build/resources", + })) + if got := ConfiguredRoot(cfg, ".wippy"); got != "build/resources" { + t.Fatalf("configured root = %q", got) + } +} diff --git a/boot/deps/artifact/context.go b/boot/deps/artifact/context.go new file mode 100644 index 000000000..c52f21749 --- /dev/null +++ b/boot/deps/artifact/context.go @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: MPL-2.0 + +package artifact + +import ( + "context" + + ctxapi "github.com/wippyai/runtime/api/context" +) + +var registryKey = &ctxapi.Key{Name: "artifact.registry"} + +// WithRegistry stores the boot-composed artifact registry. +func WithRegistry(ctx context.Context, registry *Registry) context.Context { + appCtx := ctxapi.AppFromContext(ctx) + if appCtx == nil { + return ctx + } + if appCtx.Get(registryKey) == nil { + appCtx.With(registryKey, registry) + } + return ctx +} + +// GetRegistry retrieves the boot-composed artifact registry. +func GetRegistry(ctx context.Context) *Registry { + appCtx := ctxapi.AppFromContext(ctx) + if appCtx == nil { + return nil + } + registry, _ := appCtx.Get(registryKey).(*Registry) + return registry +} diff --git a/boot/deps/artifact/directory.go b/boot/deps/artifact/directory.go new file mode 100644 index 000000000..db49ae9f6 --- /dev/null +++ b/boot/deps/artifact/directory.go @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: MPL-2.0 + +package artifact + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/wippyai/runtime/api/payload" + regapi "github.com/wippyai/runtime/api/registry" + dirapi "github.com/wippyai/runtime/api/service/fs/directory" + "github.com/wippyai/wapp" +) + +const moduleMetadataKey = "module" + +// DirectoryResources resolves artifact declarations from selected local +// fs.directory entries. moduleRoots is the authoritative source root for each +// local module; entries outside that set are ignored. +func DirectoryResources( + ctx context.Context, + entries regapi.State, + moduleRoots map[string]string, + moduleVersions map[string]string, +) ([]Resource, error) { + if len(moduleRoots) == 0 { + return nil, nil + } + transcoder := payload.GetTranscoder(ctx) + if transcoder == nil { + return nil, errors.New("payload transcoder is unavailable") + } + + resources := make([]Resource, 0) + for _, entry := range entries { + module := entry.Meta.GetString(moduleMetadataKey, "") + moduleRoot, selected := moduleRoots[module] + if !selected || entry.Kind != dirapi.Kind { + continue + } + _, declared, err := ParseDeclaration(wapp.Metadata(entry.Meta)) + if err != nil { + return nil, fmt.Errorf("local resource %s: %w", entry.ID.String(), err) + } + if !declared { + continue + } + + var cfg dirapi.Config + if err := transcoder.Unmarshal(entry.Data, &cfg); err != nil { + return nil, fmt.Errorf("decode local resource %s: %w", entry.ID.String(), err) + } + if err := cfg.Validate(); err != nil { + return nil, fmt.Errorf("validate local resource %s: %w", entry.ID.String(), err) + } + directory := cfg.Directory + if !dirapi.IsConfiguredPathAbsolute(directory) && cfg.Base != dirapi.BaseProject { + directory = filepath.Join(moduleRoot, directory) + } + info, err := os.Stat(directory) + if err != nil { + return nil, fmt.Errorf("inspect local resource %s: %w", entry.ID.String(), err) + } + if !info.IsDir() { + return nil, fmt.Errorf("local resource %s is not a directory", entry.ID.String()) + } + resources = append(resources, Resource{ + Filesystem: os.DirFS(directory), + Meta: wapp.Metadata(entry.Meta), + ModuleVersion: moduleVersions[module], + ResourceID: wapp.NewID(entry.ID.NS, entry.ID.Name), + Source: directory, + }) + } + return resources, nil +} diff --git a/boot/deps/artifact/effect_state_test.go b/boot/deps/artifact/effect_state_test.go new file mode 100644 index 000000000..377710c38 --- /dev/null +++ b/boot/deps/artifact/effect_state_test.go @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: MPL-2.0 + +package artifact + +import ( + "context" + "os" + "path/filepath" + "testing" +) + +func TestRollbackPendingRetriesRestoration(t *testing.T) { + root := t.TempDir() + destination := filepath.Join(root, "npm") + backup := filepath.Join(root, ".npm.artifact-backup-test") + staging := filepath.Join(root, ".npm.artifact-stage-test") + for path, content := range map[string]string{ + destination: "new", + backup: "old", + staging: "staged", + } { + if err := os.MkdirAll(path, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(path, "value"), []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } + + effect := &Effect{ + root: root, + activated: []activatedRoot{{ + destination: destination, + backup: backup, + hadTarget: true, + }}, + pending: []stagedRoot{{staging: staging}}, + state: wappEffectRollbackPending, + } + if err := effect.Rollback(context.Background()); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(filepath.Join(destination, "value")) + if err != nil { + t.Fatal(err) + } + if string(data) != "old" { + t.Fatalf("restored content = %q", data) + } + if _, err := os.Stat(staging); !os.IsNotExist(err) { + t.Fatalf("pending staging directory remains: %v", err) + } + if effect.state != wappEffectRolledBack { + t.Fatalf("state = %d, want rolled back", effect.state) + } +} diff --git a/boot/deps/artifact/filelock.go b/boot/deps/artifact/filelock.go new file mode 100644 index 000000000..6d03bb078 --- /dev/null +++ b/boot/deps/artifact/filelock.go @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: MPL-2.0 + +package artifact + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "time" +) + +const artifactLockRetry = 25 * time.Millisecond + +func acquireArtifactLock(ctx context.Context, root string) (func() error, error) { + file, err := os.OpenFile( + filepath.Join(root, ".artifacts.lock"), + os.O_CREATE|os.O_RDWR, + 0o600, + ) + if err != nil { + return nil, fmt.Errorf("open artifact lock: %w", err) + } + + for { + unlock, lockErr := tryLockFile(file) + if lockErr == nil { + return func() error { + return errors.Join(unlock(), file.Close()) + }, nil + } + if !errors.Is(lockErr, errLockBusy) { + _ = file.Close() + return nil, fmt.Errorf("lock artifact root: %w", lockErr) + } + + timer := time.NewTimer(artifactLockRetry) + select { + case <-ctx.Done(): + if !timer.Stop() { + <-timer.C + } + _ = file.Close() + return nil, fmt.Errorf("lock artifact root: %w", ctx.Err()) + case <-timer.C: + } + } +} diff --git a/boot/deps/artifact/filelock_test.go b/boot/deps/artifact/filelock_test.go new file mode 100644 index 000000000..e1582addf --- /dev/null +++ b/boot/deps/artifact/filelock_test.go @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: MPL-2.0 + +package artifact + +import ( + "context" + "errors" + "testing" +) + +func TestArtifactLockSerializesMaterializers(t *testing.T) { + root := t.TempDir() + unlock, err := acquireArtifactLock(context.Background(), root) + if err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := acquireArtifactLock(ctx, root); !errors.Is(err, context.Canceled) { + t.Fatalf("second lock error = %v, want context cancellation", err) + } + if err := unlock(); err != nil { + t.Fatal(err) + } + + unlock, err = acquireArtifactLock(context.Background(), root) + if err != nil { + t.Fatalf("reacquire lock: %v", err) + } + if err := unlock(); err != nil { + t.Fatal(err) + } +} diff --git a/boot/deps/artifact/filelock_unix.go b/boot/deps/artifact/filelock_unix.go new file mode 100644 index 000000000..bc89ac924 --- /dev/null +++ b/boot/deps/artifact/filelock_unix.go @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: MPL-2.0 + +//go:build !windows + +package artifact + +import ( + "errors" + "os" + + "golang.org/x/sys/unix" +) + +var errLockBusy = errors.New("artifact lock is busy") + +func tryLockFile(file *os.File) (func() error, error) { + err := unix.Flock(int(file.Fd()), unix.LOCK_EX|unix.LOCK_NB) + if errors.Is(err, unix.EWOULDBLOCK) || errors.Is(err, unix.EAGAIN) { + return nil, errLockBusy + } + if err != nil { + return nil, err + } + return func() error { + return unix.Flock(int(file.Fd()), unix.LOCK_UN) + }, nil +} diff --git a/boot/deps/artifact/filelock_windows.go b/boot/deps/artifact/filelock_windows.go new file mode 100644 index 000000000..bd09ec07b --- /dev/null +++ b/boot/deps/artifact/filelock_windows.go @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: MPL-2.0 + +//go:build windows + +package artifact + +import ( + "errors" + "os" + + "golang.org/x/sys/windows" +) + +var errLockBusy = errors.New("artifact lock is busy") + +func tryLockFile(file *os.File) (func() error, error) { + overlapped := &windows.Overlapped{} + handle := windows.Handle(file.Fd()) + err := windows.LockFileEx( + handle, + windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, + 0, + 1, + 0, + overlapped, + ) + if errors.Is(err, windows.ERROR_LOCK_VIOLATION) { + return nil, errLockBusy + } + if err != nil { + return nil, err + } + return func() error { + return windows.UnlockFileEx(handle, 0, 1, 0, overlapped) + }, nil +} diff --git a/boot/deps/artifact/materialize.go b/boot/deps/artifact/materialize.go new file mode 100644 index 000000000..1f9be8ecc --- /dev/null +++ b/boot/deps/artifact/materialize.go @@ -0,0 +1,319 @@ +// SPDX-License-Identifier: MPL-2.0 + +package artifact + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + "hash" + "io" + "io/fs" + "os" + "path/filepath" + "strconv" + "strings" +) + +// Materialize validates a resource with its format and transactionally mirrors +// it below root at the format-derived path. +func Materialize( + ctx context.Context, + registry *Registry, + declaration Declaration, + input InspectInput, + root string, +) (Descriptor, string, error) { + descriptor, err := registry.Inspect(ctx, declaration, input) + if err != nil { + return Descriptor{}, "", err + } + + rootAbs, err := filepath.Abs(root) + if err != nil { + return Descriptor{}, "", fmt.Errorf("resolve artifact root: %w", err) + } + if err := ensureMaterializationRoot(rootAbs); err != nil { + return Descriptor{}, "", err + } + destination := filepath.Join(rootAbs, filepath.FromSlash(descriptor.RelativePath)) + if err := ensureWithinRoot(rootAbs, destination); err != nil { + return Descriptor{}, "", err + } + unlock, err := acquireArtifactLock(ctx, rootAbs) + if err != nil { + return Descriptor{}, "", err + } + if err := exactMirror(input.Filesystem, rootAbs, destination); err != nil { + return Descriptor{}, "", fmt.Errorf( + "materialize %s: %w", + input.ResourceID.String(), + errors.Join(err, unlock()), + ) + } + if err := unlock(); err != nil { + return Descriptor{}, "", fmt.Errorf("unlock artifact root: %w", err) + } + return descriptor, destination, nil +} + +func ensureWithinRoot(root, destination string) error { + relative, err := filepath.Rel(root, destination) + if err != nil { + return fmt.Errorf("resolve artifact destination: %w", err) + } + if relative == "." || relative == ".." || + strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return errors.New("artifact destination escapes the materialization root") + } + return nil +} + +func ensureMaterializationRoot(root string) error { + info, err := os.Lstat(root) + if errors.Is(err, os.ErrNotExist) { + if err := os.MkdirAll(root, 0o755); err != nil { + return fmt.Errorf("create artifact root: %w", err) + } + info, err = os.Lstat(root) + } + if err != nil { + return fmt.Errorf("inspect artifact root: %w", err) + } + if info.Mode()&os.ModeSymlink != 0 { + return errors.New("artifact root must not be a symlink") + } + if !info.IsDir() { + return errors.New("artifact root must be a directory") + } + return nil +} + +func ensureDirectoryBelowRoot(root, directory string) error { + if filepath.Clean(root) == filepath.Clean(directory) { + return nil + } + if err := ensureWithinRoot(root, directory); err != nil { + return err + } + relative, err := filepath.Rel(root, directory) + if err != nil { + return fmt.Errorf("resolve artifact directory: %w", err) + } + current := root + for _, segment := range strings.Split(relative, string(filepath.Separator)) { + current = filepath.Join(current, segment) + info, statErr := os.Lstat(current) + if errors.Is(statErr, os.ErrNotExist) { + if mkdirErr := os.Mkdir(current, 0o755); mkdirErr != nil && + !errors.Is(mkdirErr, os.ErrExist) { + return fmt.Errorf("create artifact directory: %w", mkdirErr) + } + info, statErr = os.Lstat(current) + } + if statErr != nil { + return fmt.Errorf("inspect artifact directory: %w", statErr) + } + if info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("artifact directory %q is a symlink", current) + } + if !info.IsDir() { + return fmt.Errorf("artifact directory %q is not a directory", current) + } + } + return nil +} + +func exactMirror(source fs.FS, root, destination string) error { + parent := filepath.Dir(destination) + if err := ensureDirectoryBelowRoot(root, parent); err != nil { + return fmt.Errorf("prepare destination parent: %w", err) + } + + stage, err := os.MkdirTemp(parent, "."+filepath.Base(destination)+".stage-*") + if err != nil { + return fmt.Errorf("create staging directory: %w", err) + } + stageActive := true + defer func() { + if stageActive { + _ = os.RemoveAll(stage) + } + }() + + if err := copyTree(source, stage); err != nil { + return err + } + + backup, err := os.MkdirTemp(parent, "."+filepath.Base(destination)+".backup-*") + if err != nil { + return fmt.Errorf("reserve backup path: %w", err) + } + if err := os.Remove(backup); err != nil { + return fmt.Errorf("prepare backup path: %w", err) + } + hadDestination := false + if _, err := os.Lstat(destination); err == nil { + if err := os.Rename(destination, backup); err != nil { + return fmt.Errorf("stage existing destination: %w", err) + } + hadDestination = true + } else if !os.IsNotExist(err) { + return fmt.Errorf("stat destination: %w", err) + } + + if err := os.Rename(stage, destination); err != nil { + if hadDestination { + if restoreErr := os.Rename(backup, destination); restoreErr != nil { + return fmt.Errorf( + "activate staged artifact: %w (restore previous destination: %w)", + err, + restoreErr, + ) + } + } + return fmt.Errorf("activate staged artifact: %w", err) + } + stageActive = false + if hadDestination { + if err := os.RemoveAll(backup); err != nil { + return fmt.Errorf("remove artifact backup: %w", err) + } + } + return nil +} + +func copyTree(source fs.FS, destination string) error { + portablePaths := make(map[string]string) + return fs.WalkDir(source, ".", func(path string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if path == "." { + return nil + } + if err := validatePortablePath(path); err != nil { + return fmt.Errorf("invalid resource path %q: %w", path, err) + } + portableKey := strings.ToLower(path) + if previous, exists := portablePaths[portableKey]; exists && previous != path { + return fmt.Errorf("resource paths %q and %q collide on case-insensitive filesystems", previous, path) + } + portablePaths[portableKey] = path + target := filepath.Join(destination, filepath.FromSlash(path)) + if err := ensureWithinRoot(destination, target); err != nil { + return err + } + + info, err := entry.Info() + if err != nil { + return fmt.Errorf("inspect %q: %w", path, err) + } + if info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("symlink %q is not allowed", path) + } + if entry.IsDir() { + return os.MkdirAll(target, 0o755) + } + if !info.Mode().IsRegular() { + return fmt.Errorf("non-regular file %q is not allowed", path) + } + + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + src, err := source.Open(path) + if err != nil { + return fmt.Errorf("open %q: %w", path, err) + } + + dst, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + _ = src.Close() + return fmt.Errorf("create %q: %w", path, err) + } + _, copyErr := io.Copy(dst, src) + sourceCloseErr := src.Close() + closeErr := dst.Close() + if copyErr != nil { + return fmt.Errorf("copy %q: %w", path, copyErr) + } + if sourceCloseErr != nil { + return fmt.Errorf("close source %q: %w", path, sourceCloseErr) + } + if closeErr != nil { + return fmt.Errorf("close %q: %w", path, closeErr) + } + if err := os.Chmod(target, 0o644); err != nil { //nolint:gosec // Artifacts are shared source files, not secrets. + return fmt.Errorf("set permissions on %q: %w", path, err) + } + return nil + }) +} + +func digestTree(source fs.FS) ([sha256.Size]byte, error) { + digest := sha256.New() + portablePaths := make(map[string]string) + err := fs.WalkDir(source, ".", func(path string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if path == "." { + return nil + } + if err := validatePortablePath(path); err != nil { + return fmt.Errorf("invalid resource path %q: %w", path, err) + } + portableKey := strings.ToLower(path) + if previous, exists := portablePaths[portableKey]; exists && previous != path { + return fmt.Errorf("resource paths %q and %q collide on case-insensitive filesystems", previous, path) + } + portablePaths[portableKey] = path + + info, err := entry.Info() + if err != nil { + return fmt.Errorf("inspect %q: %w", path, err) + } + if info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("symlink %q is not allowed", path) + } + if entry.IsDir() { + writeDigestField(digest, "directory") + writeDigestField(digest, path) + return nil + } + if !info.Mode().IsRegular() { + return fmt.Errorf("non-regular file %q is not allowed", path) + } + writeDigestField(digest, "file") + writeDigestField(digest, path) + writeDigestField(digest, strconv.FormatInt(info.Size(), 10)) + + file, err := source.Open(path) + if err != nil { + return fmt.Errorf("open %q: %w", path, err) + } + _, copyErr := io.Copy(digest, file) + closeErr := file.Close() + if copyErr != nil { + return fmt.Errorf("hash %q: %w", path, copyErr) + } + if closeErr != nil { + return fmt.Errorf("close %q: %w", path, closeErr) + } + return nil + }) + if err != nil { + return [sha256.Size]byte{}, err + } + var result [sha256.Size]byte + copy(result[:], digest.Sum(nil)) + return result, nil +} + +func writeDigestField(digest hash.Hash, value string) { + _, _ = digest.Write([]byte(strconv.Itoa(len(value)))) + _, _ = digest.Write([]byte{':'}) + _, _ = digest.Write([]byte(value)) +} diff --git a/boot/deps/artifact/materialize_test.go b/boot/deps/artifact/materialize_test.go new file mode 100644 index 000000000..52fbf3d4b --- /dev/null +++ b/boot/deps/artifact/materialize_test.go @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: MPL-2.0 + +package artifact + +import ( + "context" + "os" + "path/filepath" + "testing" + "testing/fstest" + + "github.com/wippyai/wapp" +) + +func TestMaterializeCreatesExactMirror(t *testing.T) { + root := t.TempDir() + destination := filepath.Join(root, "npm", "@example", "ui") + if err := os.MkdirAll(destination, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(destination, "stale.txt"), []byte("stale"), 0o644); err != nil { + t.Fatal(err) + } + + registry := NewRegistry() + if err := registry.Register(testFormat{ + name: "test", + root: "npm", + descriptor: Descriptor{ + Identity: "@example/ui", + Version: "1.0.0", + RelativePath: "npm/@example/ui", + }, + }); err != nil { + t.Fatal(err) + } + source := fstest.MapFS{ + "package.json": &fstest.MapFile{Data: []byte(`{"name":"@example/ui"}`)}, + "dist/index.js": &fstest.MapFile{Data: []byte("export {}")}, + } + _, gotDestination, err := Materialize( + context.Background(), + registry, + Declaration{Format: "test"}, + InspectInput{ + Filesystem: source, + ResourceID: wapp.NewID("example.ui", "package"), + }, + root, + ) + if err != nil { + t.Fatalf("materialize: %v", err) + } + if gotDestination != destination { + t.Fatalf("destination = %q, want %q", gotDestination, destination) + } + if _, err := os.Stat(filepath.Join(destination, "stale.txt")); !os.IsNotExist(err) { + t.Fatalf("stale file remains: %v", err) + } + data, err := os.ReadFile(filepath.Join(destination, "dist", "index.js")) + if err != nil { + t.Fatal(err) + } + if string(data) != "export {}" { + t.Fatalf("content = %q", data) + } +} + +func TestMaterializeRejectsNonPortableResourcePaths(t *testing.T) { + for name, source := range map[string]fstest.MapFS{ + "reserved name": { + "CON": &fstest.MapFile{Data: []byte("reserved")}, + }, + "case collision": { + "dist/index.js": &fstest.MapFile{Data: []byte("one")}, + "dist/INDEX.js": &fstest.MapFile{Data: []byte("two")}, + }, + } { + t.Run(name, func(t *testing.T) { + registry := NewRegistry() + if err := registry.Register(testFormat{ + name: "test", + root: "artifacts", + descriptor: Descriptor{ + Identity: "example", + RelativePath: "artifacts/example", + }, + }); err != nil { + t.Fatal(err) + } + _, _, err := Materialize( + context.Background(), + registry, + Declaration{Format: "test"}, + InspectInput{Filesystem: source, ResourceID: wapp.NewID("example", "bad")}, + t.TempDir(), + ) + if err == nil { + t.Fatal("expected non-portable resource path error") + } + }) + } +} + +func TestMaterializeRejectsEscapingFormatPath(t *testing.T) { + registry := NewRegistry() + if err := registry.Register(testFormat{ + name: "test", + root: "artifacts", + descriptor: Descriptor{ + Identity: "escape", + RelativePath: "../escape", + }, + }); err != nil { + t.Fatal(err) + } + _, _, err := Materialize( + context.Background(), + registry, + Declaration{Format: "test"}, + InspectInput{Filesystem: fstest.MapFS{}, ResourceID: wapp.NewID("acme", "bad")}, + t.TempDir(), + ) + if err == nil { + t.Fatal("expected escaping path error") + } +} + +func TestMaterializeRejectsSymlinkedDestinationParent(t *testing.T) { + root := t.TempDir() + outside := t.TempDir() + if err := os.Symlink(outside, filepath.Join(root, "npm")); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + + registry := NewRegistry() + if err := registry.Register(testFormat{ + name: "test", + root: "npm", + descriptor: Descriptor{ + Identity: "package", + RelativePath: "npm/package", + }, + }); err != nil { + t.Fatal(err) + } + _, _, err := Materialize( + context.Background(), + registry, + Declaration{Format: "test"}, + InspectInput{ + Filesystem: fstest.MapFS{ + "package.json": &fstest.MapFile{Data: []byte("{}")}, + }, + ResourceID: wapp.NewID("example", "package"), + }, + root, + ) + if err == nil { + t.Fatal("expected symlinked destination parent error") + } + if _, err := os.Stat(filepath.Join(outside, "package")); !os.IsNotExist(err) { + t.Fatalf("materialized outside root: %v", err) + } +} diff --git a/boot/deps/artifact/nodepackage/format.go b/boot/deps/artifact/nodepackage/format.go new file mode 100644 index 000000000..9313bf369 --- /dev/null +++ b/boot/deps/artifact/nodepackage/format.go @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: MPL-2.0 + +// Package nodepackage implements the build-time node-package artifact format. +package nodepackage + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "path" + "strings" + + "github.com/Masterminds/semver/v3" + "github.com/wippyai/runtime/boot/deps/artifact" +) + +const ( + FormatName = "node-package" + maxPackageJSONBytes = 1 << 20 + maxPackageNameBytes = 214 +) + +type Format struct{} + +func New() *Format { + return &Format{} +} + +func (*Format) Name() string { + return FormatName +} + +func (*Format) Root() string { + return "npm" +} + +type manifest struct { + Scripts map[string]json.RawMessage `json:"scripts"` + Name json.RawMessage `json:"name"` + Version json.RawMessage `json:"version"` +} + +func (*Format) Inspect(_ context.Context, input artifact.InspectInput) (artifact.Descriptor, error) { + if input.Filesystem == nil { + return artifact.Descriptor{}, errors.New("filesystem is nil") + } + data, err := readBoundedFile(input.Filesystem, "package.json", maxPackageJSONBytes) + if err != nil { + return artifact.Descriptor{}, err + } + + var packageManifest manifest + if err := json.Unmarshal(data, &packageManifest); err != nil { + return artifact.Descriptor{}, fmt.Errorf("decode package.json: %w", err) + } + name, err := requiredString(packageManifest.Name, "name") + if err != nil { + return artifact.Descriptor{}, err + } + version, err := requiredString(packageManifest.Version, "version") + if err != nil { + return artifact.Descriptor{}, err + } + if err := validatePackageName(name); err != nil { + return artifact.Descriptor{}, err + } + packageVersion, err := semver.NewVersion(version) + if err != nil { + return artifact.Descriptor{}, fmt.Errorf("package.json version %q is not semantic: %w", version, err) + } + if input.ModuleVersion != "" { + moduleVersion, err := semver.NewVersion(input.ModuleVersion) + if err != nil { + return artifact.Descriptor{}, fmt.Errorf("module version %q is not semantic: %w", input.ModuleVersion, err) + } + if !packageVersion.Equal(moduleVersion) { + return artifact.Descriptor{}, fmt.Errorf( + "package version %s does not match module version %s", version, input.ModuleVersion) + } + } + + for _, script := range []string{"preinstall", "install", "postinstall", "prepare"} { + if _, exists := packageManifest.Scripts[script]; exists { + return artifact.Descriptor{}, fmt.Errorf("package.json lifecycle script %q is not allowed", script) + } + } + + return artifact.Descriptor{ + Identity: name, + Version: version, + RelativePath: path.Join("npm", name), + }, nil +} + +func readBoundedFile(filesystem fs.FS, name string, limit int64) ([]byte, error) { + file, err := filesystem.Open(name) + if err != nil { + return nil, fmt.Errorf("open %s: %w", name, err) + } + defer func() { _ = file.Close() }() + + info, err := file.Stat() + if err != nil { + return nil, fmt.Errorf("stat %s: %w", name, err) + } + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("%s must be a regular file", name) + } + if info.Size() > limit { + return nil, fmt.Errorf("%s exceeds %d bytes", name, limit) + } + data, err := io.ReadAll(io.LimitReader(file, limit+1)) + if err != nil { + return nil, fmt.Errorf("read %s: %w", name, err) + } + if int64(len(data)) > limit { + return nil, fmt.Errorf("%s exceeds %d bytes", name, limit) + } + return data, nil +} + +func requiredString(raw json.RawMessage, field string) (string, error) { + if len(raw) == 0 { + return "", fmt.Errorf("package.json %s is required", field) + } + var value string + if err := json.Unmarshal(raw, &value); err != nil { + return "", fmt.Errorf("package.json %s must be a string", field) + } + value = strings.TrimSpace(value) + if value == "" { + return "", fmt.Errorf("package.json %s must not be empty", field) + } + return value, nil +} + +func validatePackageName(name string) error { + if len(name) > maxPackageNameBytes { + return fmt.Errorf("package name exceeds %d bytes", maxPackageNameBytes) + } + parts := []string{name} + if strings.HasPrefix(name, "@") { + parts = strings.Split(strings.TrimPrefix(name, "@"), "/") + if len(parts) != 2 { + return fmt.Errorf("invalid scoped package name %q", name) + } + } else if strings.Contains(name, "/") { + return fmt.Errorf("invalid package name %q", name) + } + for _, part := range parts { + if part == "" || part == "." || part == ".." || + strings.HasPrefix(part, ".") || strings.HasPrefix(part, "_") { + return fmt.Errorf("invalid package name %q", name) + } + for _, char := range part { + if char >= 'a' && char <= 'z' || char >= '0' && char <= '9' || + char == '-' || char == '_' || char == '.' { + continue + } + return fmt.Errorf("invalid package name %q", name) + } + } + if len(parts) == 1 && (parts[0] == "node_modules" || parts[0] == "favicon.ico") { + return fmt.Errorf("invalid package name %q", name) + } + return nil +} diff --git a/boot/deps/artifact/nodepackage/format_test.go b/boot/deps/artifact/nodepackage/format_test.go new file mode 100644 index 000000000..72b6f1a8b --- /dev/null +++ b/boot/deps/artifact/nodepackage/format_test.go @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: MPL-2.0 + +package nodepackage + +import ( + "context" + "strings" + "testing" + "testing/fstest" + + "github.com/wippyai/runtime/boot/deps/artifact" + "github.com/wippyai/wapp" +) + +func TestInspect(t *testing.T) { + input := artifact.InspectInput{ + Filesystem: fstest.MapFS{ + "package.json": &fstest.MapFile{Data: []byte( + `{"name":"@example/package","version":"0.1.6","scripts":{"build":"tsc"}}`, + )}, + "dist/index.js": &fstest.MapFile{Data: []byte("export {}")}, + }, + ModuleVersion: "0.1.6", + ResourceID: wapp.NewID("example.package", "package"), + } + got, err := New().Inspect(context.Background(), input) + if err != nil { + t.Fatalf("inspect: %v", err) + } + if got.Identity != "@example/package" || got.Version != "0.1.6" { + t.Fatalf("descriptor = %+v", got) + } + if got.RelativePath != "npm/@example/package" { + t.Fatalf("relative path = %q", got.RelativePath) + } +} + +func TestInspectRejectsInvalidPackages(t *testing.T) { + tests := map[string]struct { + manifest string + moduleVersion string + want string + }{ + "missing name": { + manifest: `{"version":"1.0.0"}`, + want: "name is required", + }, + "invalid name": { + manifest: `{"name":"@scope/../escape","version":"1.0.0"}`, + want: "invalid scoped package name", + }, + "bad version": { + manifest: `{"name":"@scope/pkg","version":"latest"}`, + want: "is not semantic", + }, + "module mismatch": { + manifest: `{"name":"@scope/pkg","version":"1.0.0"}`, + moduleVersion: "2.0.0", + want: "does not match module version", + }, + "lifecycle script": { + manifest: `{"name":"@scope/pkg","version":"1.0.0","scripts":{"postinstall":"node setup.js"}}`, + want: `lifecycle script "postinstall" is not allowed`, + }, + } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + _, err := New().Inspect(context.Background(), artifact.InspectInput{ + Filesystem: fstest.MapFS{ + "package.json": &fstest.MapFile{Data: []byte(test.manifest)}, + }, + ModuleVersion: test.moduleVersion, + }) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("error = %v, want containing %q", err, test.want) + } + }) + } +} diff --git a/boot/deps/artifact/registry.go b/boot/deps/artifact/registry.go new file mode 100644 index 000000000..9a0304960 --- /dev/null +++ b/boot/deps/artifact/registry.go @@ -0,0 +1,251 @@ +// SPDX-License-Identifier: MPL-2.0 + +// Package artifact defines format discovery, validation, and materialization +// for filesystem resources carried by WAPPs. +// +// Artifacts remain ordinary WAPP filesystem resources. A resource opts into +// format-specific handling through meta.artifact.format. Module selection, +// downloading, integrity verification, and dependency resolution stay with +// their existing owners. +package artifact + +import ( + "context" + "errors" + "fmt" + "io/fs" + "path" + "path/filepath" + "sort" + "strings" + + "github.com/wippyai/runtime/api/attrs" + "github.com/wippyai/wapp" +) + +const MetadataKey = "artifact" + +var ( + ErrDuplicateFormat = errors.New("artifact format already registered") + ErrUnknownFormat = errors.New("unknown artifact format") +) + +// Declaration is the authored metadata attached to an embedded filesystem. +type Declaration struct { + Format string +} + +// Descriptor is format-derived identity used to choose a stable materialized +// location. Identity and Version come from the artifact contents, not authored +// resource metadata. +type Descriptor struct { + Identity string + Version string + RelativePath string +} + +// InspectInput provides immutable module and resource context to a format. +type InspectInput struct { + Filesystem fs.FS + ModuleVersion string + ResourceID wapp.ID +} + +// Format validates one artifact filesystem and derives its identity and stable +// materialization path. Root names the format-managed subtree below the +// configured artifact root; exact reconciliation may replace that entire +// subtree. Formats do not download modules, invoke package managers, mutate +// locks, or register themselves globally. +type Format interface { + Name() string + Root() string + Inspect(context.Context, InspectInput) (Descriptor, error) +} + +// Registry contains the formats available to one explicitly composed caller. +// Construction is explicit so commands, boot, and tests do not share globals. +type Registry struct { + formats map[string]Format +} + +func NewRegistry() *Registry { + return &Registry{formats: make(map[string]Format)} +} + +func (r *Registry) Register(format Format) error { + if format == nil { + return errors.New("artifact format is nil") + } + name := strings.TrimSpace(format.Name()) + if name == "" { + return errors.New("artifact format name is empty") + } + root := strings.TrimSpace(format.Root()) + if err := validatePortablePath(root); err != nil { + return fmt.Errorf("artifact format %q has invalid root: %w", name, err) + } + + if _, exists := r.formats[name]; exists { + return fmt.Errorf("%w: %s", ErrDuplicateFormat, name) + } + for registeredName, registered := range r.formats { + registeredRoot := path.Clean(registered.Root()) + if root != registeredRoot && + (strings.HasPrefix(root, registeredRoot+"/") || + strings.HasPrefix(registeredRoot, root+"/")) { + return fmt.Errorf( + "artifact format roots overlap: %q owns %q and %q owns %q", + registeredName, registeredRoot, name, root, + ) + } + } + r.formats[name] = format + return nil +} + +func (r *Registry) Resolve(name string) (Format, bool) { + if r == nil { + return nil, false + } + format, ok := r.formats[name] + return format, ok +} + +func (r *Registry) Names() []string { + if r == nil { + return nil + } + names := make([]string, 0, len(r.formats)) + for name := range r.formats { + names = append(names, name) + } + sort.Strings(names) + return names +} + +// Roots returns the non-overlapping materialization subtrees owned by the +// registered formats. Multiple formats may intentionally share one root. +func (r *Registry) Roots() []string { + if r == nil { + return nil + } + unique := make(map[string]struct{}, len(r.formats)) + for _, format := range r.formats { + unique[path.Clean(format.Root())] = struct{}{} + } + roots := make([]string, 0, len(unique)) + for root := range unique { + roots = append(roots, root) + } + sort.Strings(roots) + return roots +} + +// ParseDeclaration reads meta.artifact.format. Metadata without an artifact +// key is not an artifact. Once the key exists, malformed declarations fail +// closed. +func ParseDeclaration(meta wapp.Metadata) (Declaration, bool, error) { + raw, exists := meta[MetadataKey] + if !exists { + return Declaration{}, false, nil + } + + block, ok := stringMap(raw) + if !ok { + return Declaration{}, true, errors.New("meta.artifact must be an object") + } + rawFormat, exists := block["format"] + if !exists { + return Declaration{}, true, errors.New("meta.artifact.format is required") + } + format, ok := rawFormat.(string) + if !ok || strings.TrimSpace(format) == "" { + return Declaration{}, true, errors.New("meta.artifact.format must be a non-empty string") + } + return Declaration{Format: strings.TrimSpace(format)}, true, nil +} + +func stringMap(value any) (map[string]any, bool) { + switch typed := value.(type) { + case map[string]any: + return typed, true + case attrs.Bag: + return map[string]any(typed), true + case wapp.Metadata: + return map[string]any(typed), true + default: + return nil, false + } +} + +// Inspect resolves and invokes the declared format. +func (r *Registry) Inspect(ctx context.Context, declaration Declaration, input InspectInput) (Descriptor, error) { + format, ok := r.Resolve(declaration.Format) + if !ok { + return Descriptor{}, fmt.Errorf("%w %q for artifact %s (registered: %s)", + ErrUnknownFormat, declaration.Format, input.ResourceID.String(), + strings.Join(r.Names(), ", ")) + } + descriptor, err := format.Inspect(ctx, input) + if err != nil { + return Descriptor{}, fmt.Errorf("validate %s artifact %s: %w", + declaration.Format, input.ResourceID.String(), err) + } + if strings.TrimSpace(descriptor.Identity) == "" { + return Descriptor{}, errors.New("artifact format returned empty identity") + } + if err := validatePortablePath(descriptor.RelativePath); err != nil { + return Descriptor{}, fmt.Errorf("artifact format returned invalid relative path: %w", err) + } + descriptor.RelativePath = path.Clean(descriptor.RelativePath) + root := path.Clean(format.Root()) + if descriptor.RelativePath != root && + !strings.HasPrefix(descriptor.RelativePath, root+"/") { + return Descriptor{}, fmt.Errorf( + "artifact format %q returned path %q outside its root %q", + declaration.Format, descriptor.RelativePath, root, + ) + } + return descriptor, nil +} + +func validatePortablePath(value string) error { + if value == "" { + return errors.New("path is empty") + } + if !fs.ValidPath(value) || value == "." { + return errors.New("path must be a canonical relative slash path") + } + if strings.ContainsAny(value, `\:`) { + return errors.New("path contains a platform-specific separator or volume") + } + for _, segment := range strings.Split(value, "/") { + if strings.HasSuffix(segment, ".") || strings.HasSuffix(segment, " ") { + return fmt.Errorf("path segment %q has a non-portable suffix", segment) + } + base := strings.ToLower(strings.SplitN(segment, ".", 2)[0]) + if isWindowsReservedName(base) { + return fmt.Errorf("path segment %q is a reserved name", segment) + } + } + return nil +} + +func isWindowsReservedName(name string) bool { + switch name { + case "con", "prn", "aux", "nul", + "com1", "com2", "com3", "com4", "com5", "com6", "com7", "com8", "com9", + "lpt1", "lpt2", "lpt3", "lpt4", "lpt5", "lpt6", "lpt7", "lpt8", "lpt9": + return true + default: + return false + } +} + +func pathsOverlap(left, right string) bool { + left = strings.ToLower(path.Clean(filepath.ToSlash(left))) + right = strings.ToLower(path.Clean(filepath.ToSlash(right))) + return left == right || + strings.HasPrefix(left, right+"/") || + strings.HasPrefix(right, left+"/") +} diff --git a/boot/deps/artifact/registry_test.go b/boot/deps/artifact/registry_test.go new file mode 100644 index 000000000..e9992d9d6 --- /dev/null +++ b/boot/deps/artifact/registry_test.go @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: MPL-2.0 + +package artifact + +import ( + "context" + "errors" + "os" + "strings" + "testing" + "testing/fstest" + + "github.com/wippyai/wapp" +) + +type testFormat struct { + name string + root string + descriptor Descriptor +} + +func (f testFormat) Name() string { return f.name } +func (f testFormat) Root() string { + if f.root == "" { + return "artifacts" + } + return f.root +} + +func (f testFormat) Inspect(context.Context, InspectInput) (Descriptor, error) { + return f.descriptor, nil +} + +func TestParseDeclaration(t *testing.T) { + t.Run("absent", func(t *testing.T) { + _, declared, err := ParseDeclaration(wapp.Metadata{"comment": "ordinary filesystem"}) + if err != nil || declared { + t.Fatalf("declared=%v err=%v, want false nil", declared, err) + } + }) + + t.Run("valid", func(t *testing.T) { + got, declared, err := ParseDeclaration(wapp.Metadata{ + "artifact": map[string]any{"format": " node-package "}, + }) + if err != nil || !declared || got.Format != "node-package" { + t.Fatalf("got=%+v declared=%v err=%v", got, declared, err) + } + }) + + for name, meta := range map[string]wapp.Metadata{ + "not object": {"artifact": "node-package"}, + "missing format": {"artifact": map[string]any{}}, + "empty format": {"artifact": map[string]any{"format": ""}}, + "wrong type": {"artifact": map[string]any{"format": 1}}, + } { + t.Run(name, func(t *testing.T) { + _, declared, err := ParseDeclaration(meta) + if err == nil || !declared { + t.Fatalf("declared=%v err=%v, want declared error", declared, err) + } + }) + } +} + +func TestRegistryExplicitRegistration(t *testing.T) { + registry := NewRegistry() + format := testFormat{ + name: "test", + descriptor: Descriptor{ + Identity: "identity", + Version: "1.0.0", + RelativePath: "artifacts/identity", + }, + } + if err := registry.Register(format); err != nil { + t.Fatalf("register: %v", err) + } + if err := registry.Register(format); !errors.Is(err, ErrDuplicateFormat) { + t.Fatalf("duplicate error = %v", err) + } + + got, err := registry.Inspect(context.Background(), Declaration{Format: "test"}, InspectInput{ + Filesystem: fstest.MapFS{"file": &fstest.MapFile{Data: []byte("data")}}, + ResourceID: wapp.NewID("acme", "resource"), + }) + if err != nil { + t.Fatalf("inspect: %v", err) + } + if got.Identity != "identity" { + t.Fatalf("identity = %q", got.Identity) + } + + _, err = registry.Inspect( + context.Background(), + Declaration{Format: "missing"}, + InspectInput{ResourceID: wapp.NewID("acme", "missing")}, + ) + if !errors.Is(err, ErrUnknownFormat) { + t.Fatalf("unknown error = %v", err) + } + if !strings.Contains(err.Error(), "acme:missing") { + t.Fatalf("unknown error lacks resource ID: %v", err) + } +} + +func TestInspectResourcesRejectsDestinationCollision(t *testing.T) { + registry := NewRegistry() + if err := registry.Register(testFormat{ + name: "test", + descriptor: Descriptor{ + Identity: "same", + RelativePath: "artifacts/same", + }, + }); err != nil { + t.Fatal(err) + } + + meta := wapp.Metadata{"artifact": map[string]any{"format": "test"}} + _, err := InspectResources(context.Background(), registry, []wapp.ResourceSpec{ + {ID: wapp.NewID("acme", "one"), Meta: meta, FS: fstest.MapFS{}}, + {ID: wapp.NewID("acme", "two"), Meta: meta, FS: fstest.MapFS{}}, + }, "") + if err == nil { + t.Fatal("expected destination collision") + } +} + +func TestRegistryRejectsOverlappingOwnedRoots(t *testing.T) { + registry := NewRegistry() + if err := registry.Register(testFormat{name: "one", root: "generated"}); err != nil { + t.Fatal(err) + } + if err := registry.Register(testFormat{name: "two", root: "generated/nested"}); err == nil { + t.Fatal("expected overlapping root error") + } + if err := registry.Register(testFormat{name: "three", root: "other"}); err != nil { + t.Fatalf("register non-overlapping root: %v", err) + } +} + +func TestRegistryRejectsDescriptorOutsideOwnedRoot(t *testing.T) { + registry := NewRegistry() + if err := registry.Register(testFormat{ + name: "test", + root: "generated", + descriptor: Descriptor{ + Identity: "outside", + RelativePath: "other/outside", + }, + }); err != nil { + t.Fatal(err) + } + _, err := registry.Inspect( + context.Background(), + Declaration{Format: "test"}, + InspectInput{ResourceID: wapp.NewID("example", "outside")}, + ) + if err == nil { + t.Fatal("expected descriptor root error") + } +} + +func TestInspectResourcesRejectsNonRegularArtifactTree(t *testing.T) { + registry := NewRegistry() + if err := registry.Register(testFormat{ + name: "test", + root: "artifacts", + descriptor: Descriptor{ + Identity: "unsafe", + RelativePath: "artifacts/unsafe", + }, + }); err != nil { + t.Fatal(err) + } + _, err := InspectResources( + context.Background(), + registry, + []wapp.ResourceSpec{{ + ID: wapp.NewID("example", "unsafe"), + Meta: wapp.Metadata{"artifact": map[string]any{"format": "test"}}, + FS: fstest.MapFS{ + "link": &fstest.MapFile{Mode: os.ModeSymlink}, + }, + }}, + "", + ) + if err == nil { + t.Fatal("expected non-regular artifact tree error") + } +} diff --git a/boot/deps/artifact/resources.go b/boot/deps/artifact/resources.go new file mode 100644 index 000000000..61a02aa7a --- /dev/null +++ b/boot/deps/artifact/resources.go @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: MPL-2.0 + +package artifact + +import ( + "context" + "fmt" + "strings" + + "github.com/wippyai/wapp" +) + +// InspectedResource is a validated artifact resource. +type InspectedResource struct { + Declaration Declaration + Descriptor Descriptor + ID wapp.ID +} + +// InspectResources validates every resource that opts into artifact semantics. +// Ordinary embedded filesystems pass through untouched. +func InspectResources( + ctx context.Context, + registry *Registry, + resources []wapp.ResourceSpec, + moduleVersion string, +) ([]InspectedResource, error) { + inspected := make([]InspectedResource, 0) + destinations := make(map[string]wapp.ID) + for _, resource := range resources { + declaration, declared, err := ParseDeclaration(resource.Meta) + if err != nil { + return nil, fmt.Errorf("resource %s: %w", resource.ID.String(), err) + } + if !declared { + continue + } + if resource.FS == nil { + return nil, fmt.Errorf("resource %s has no filesystem", resource.ID.String()) + } + if _, err := digestTree(resource.FS); err != nil { + return nil, fmt.Errorf("validate artifact resource %s: %w", resource.ID.String(), err) + } + descriptor, err := registry.Inspect(ctx, declaration, InspectInput{ + Filesystem: resource.FS, + ModuleVersion: moduleVersion, + ResourceID: resource.ID, + }) + if err != nil { + return nil, err + } + destinationKey := strings.ToLower(descriptor.RelativePath) + for previousPath, previous := range destinations { + if pathsOverlap(destinationKey, previousPath) { + return nil, fmt.Errorf( + "artifact resources %s at %q and %s at %q have overlapping outputs", + previous.String(), previousPath, + resource.ID.String(), descriptor.RelativePath, + ) + } + } + destinations[destinationKey] = resource.ID + inspected = append(inspected, InspectedResource{ + ID: resource.ID, + Declaration: declaration, + Descriptor: descriptor, + }) + } + return inspected, nil +} diff --git a/boot/deps/artifact/standard/registry.go b/boot/deps/artifact/standard/registry.go new file mode 100644 index 000000000..4bf2248ce --- /dev/null +++ b/boot/deps/artifact/standard/registry.go @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: MPL-2.0 + +// Package standard composes the artifact formats shipped with Wippy. +package standard + +import ( + "fmt" + + "github.com/wippyai/runtime/boot/deps/artifact" + "github.com/wippyai/runtime/boot/deps/artifact/nodepackage" +) + +func NewRegistry() (*artifact.Registry, error) { + registry := artifact.NewRegistry() + if err := registry.Register(nodepackage.New()); err != nil { + return nil, fmt.Errorf("register built-in artifact format: %w", err) + } + return registry, nil +} diff --git a/boot/deps/artifact/wapp.go b/boot/deps/artifact/wapp.go new file mode 100644 index 000000000..f8bfc80df --- /dev/null +++ b/boot/deps/artifact/wapp.go @@ -0,0 +1,674 @@ +// SPDX-License-Identifier: MPL-2.0 + +package artifact + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/wippyai/wapp" +) + +// WAPP identifies one already-selected module pack. Selection, download, and +// integrity verification belong to the caller. +type WAPP struct { + Path string + ModuleVersion string +} + +// Resource identifies an already-resolved filesystem resource. It is used for +// active local module replacements, where no WAPP transport is involved. +type Resource struct { + Filesystem fs.FS + Meta wapp.Metadata + ModuleVersion string + ResourceID wapp.ID + Source string +} + +// Materialized describes one artifact output owned by a WAPP resource. +type Materialized struct { + Descriptor Descriptor + Destination string + ResourceID wapp.ID + Source string +} + +type wappEffectState uint8 + +const ( + wappEffectPlanned wappEffectState = iota + wappEffectPrepared + wappEffectCommitted + wappEffectRollbackPending + wappEffectRolledBack + wappEffectFinalized +) + +type activatedRoot struct { + destination string + staging string + backup string + hadTarget bool +} + +// Effect materializes artifact resources as a registry-compatible +// transaction effect. It deliberately does not resolve or download modules. +type Effect struct { + registry *Registry + unlock func() error + root string + packs []WAPP + resources []Resource + activated []activatedRoot + pending []stagedRoot + results []Materialized + mu sync.Mutex + exact bool + state wappEffectState +} + +// NewWAPPEffect creates a materialization effect for exact, verified WAPPs. +func NewWAPPEffect(registry *Registry, packs []WAPP, root string) (*Effect, error) { + return NewEffect(registry, packs, nil, root) +} + +// NewEffect creates a materialization effect for exact verified WAPPs and +// already-resolved local replacement resources. +func NewEffect( + registry *Registry, + packs []WAPP, + resources []Resource, + root string, +) (*Effect, error) { + return newEffect(registry, packs, resources, root, true) +} + +// NewPartialEffect overlays selected resources while preserving outputs that +// belong to modules outside a targeted install. +func NewPartialEffect( + registry *Registry, + packs []WAPP, + resources []Resource, + root string, +) (*Effect, error) { + return newEffect(registry, packs, resources, root, false) +} + +func newEffect( + registry *Registry, + packs []WAPP, + resources []Resource, + root string, + exact bool, +) (*Effect, error) { + if registry == nil { + return nil, errors.New("artifact registry is nil") + } + if strings.TrimSpace(root) == "" { + return nil, errors.New("artifact root is empty") + } + return &Effect{ + registry: registry, + packs: append([]WAPP(nil), packs...), + resources: append([]Resource(nil), resources...), + root: root, + exact: exact, + }, nil +} + +// Results returns a copy of the successfully prepared outputs. +func (e *Effect) Results() []Materialized { + e.mu.Lock() + defer e.mu.Unlock() + return append([]Materialized(nil), e.results...) +} + +// Prepare validates every declared artifact before activating any output, then +// retains prior outputs for rollback. +func (e *Effect) Prepare(ctx context.Context) error { + e.mu.Lock() + defer e.mu.Unlock() + + if e.state == wappEffectPrepared || e.state == wappEffectCommitted { + return nil + } + if e.state != wappEffectPlanned { + return fmt.Errorf("prepare artifact effect in state %d", e.state) + } + + candidates, closePacks, err := e.inspect(ctx) + if err != nil { + return err + } + unlock, lockErr := acquireArtifactLock(ctx, e.root) + if lockErr != nil { + closePacks() + return lockErr + } + e.unlock = unlock + staged, stageErr := e.stage(candidates) + closePacks() + if stageErr != nil { + return errors.Join(stageErr, e.releaseLock()) + } + + for i, root := range staged { + activated, activateErr := activateRoot(root) + if activateErr != nil { + cleanupFrom := i + if activated.backup != "" || activated.hadTarget { + e.activated = append(e.activated, activated) + cleanupFrom = i + 1 + } + pending := staged[cleanupFrom:] + cleanupErr := cleanupStagedRoots(pending) + remaining, rollbackErr := rollbackActivated(e.activated) + e.activated = remaining + if cleanupErr == nil && rollbackErr == nil { + e.activated = nil + e.pending = nil + e.results = nil + e.state = wappEffectRolledBack + } else { + if cleanupErr != nil { + e.pending = append(e.pending, pending...) + } + e.state = wappEffectRollbackPending + } + unlockErr := e.releaseLock() + return errors.Join(activateErr, cleanupErr, rollbackErr, unlockErr) + } + e.activated = append(e.activated, activated) + } + for _, candidate := range candidates { + e.results = append(e.results, candidate.result) + } + e.state = wappEffectPrepared + return nil +} + +// Commit marks the prepared outputs as committed while keeping rollback data +// until the surrounding transaction is durable. +func (e *Effect) Commit(context.Context) error { + e.mu.Lock() + defer e.mu.Unlock() + if e.state == wappEffectCommitted { + return nil + } + if e.state != wappEffectPrepared { + return fmt.Errorf("commit artifact effect in state %d", e.state) + } + e.state = wappEffectCommitted + return nil +} + +// Rollback restores every output replaced during Prepare. +func (e *Effect) Rollback(ctx context.Context) error { + e.mu.Lock() + defer e.mu.Unlock() + if e.state == wappEffectRolledBack { + return nil + } + if e.state == wappEffectFinalized { + return errors.New("rollback finalized artifact effect") + } + if e.state == wappEffectPlanned { + e.state = wappEffectRolledBack + return nil + } + if e.state == wappEffectRollbackPending && e.unlock == nil { + unlock, err := acquireArtifactLock(ctx, e.root) + if err != nil { + return err + } + e.unlock = unlock + } + remaining, rollbackErr := rollbackActivated(e.activated) + e.activated = remaining + cleanupErr := cleanupStagedRoots(e.pending) + if cleanupErr == nil { + e.pending = nil + } + unlockErr := e.releaseLock() + if err := errors.Join(rollbackErr, cleanupErr, unlockErr); err != nil { + e.state = wappEffectRollbackPending + return err + } + e.activated = nil + e.pending = nil + e.results = nil + e.state = wappEffectRolledBack + return nil +} + +// Finalize removes rollback data after the surrounding transaction is durable. +func (e *Effect) Finalize(context.Context) error { + e.mu.Lock() + defer e.mu.Unlock() + if e.state == wappEffectFinalized { + return nil + } + if e.state != wappEffectPrepared && e.state != wappEffectCommitted { + return fmt.Errorf("finalize artifact effect in state %d", e.state) + } + var errs []error + remaining := make([]activatedRoot, 0) + for _, activated := range e.activated { + if activated.backup == "" { + continue + } + if err := os.RemoveAll(activated.backup); err != nil { + errs = append(errs, fmt.Errorf("remove artifact backup %s: %w", activated.destination, err)) + remaining = append(remaining, activated) + } + } + err := errors.Join(append(errs, e.releaseLock())...) + e.activated = remaining + if err == nil { + e.state = wappEffectFinalized + } + return err +} + +func (e *Effect) releaseLock() error { + if e.unlock == nil { + return nil + } + unlock := e.unlock + e.unlock = nil + return unlock() +} + +// MaterializeWAPPs runs the same effect lifecycle for non-registry callers. +func MaterializeWAPPs( + ctx context.Context, + registry *Registry, + packs []WAPP, + root string, +) ([]Materialized, error) { + effect, err := NewWAPPEffect(registry, packs, root) + if err != nil { + return nil, err + } + if err := effect.Prepare(ctx); err != nil { + return nil, err + } + if err := effect.Commit(ctx); err != nil { + rollbackErr := effect.Rollback(ctx) + return nil, errors.Join(err, rollbackErr) + } + results := effect.Results() + if err := effect.Finalize(ctx); err != nil { + return nil, err + } + return results, nil +} + +type artifactCandidate struct { + filesystem fs.FS + root string + result Materialized + mutable bool +} + +type stagedRoot struct { + destination string + staging string +} + +func (e *Effect) inspect(ctx context.Context) ([]artifactCandidate, func(), error) { + root, err := filepath.Abs(e.root) + if err != nil { + return nil, func() {}, fmt.Errorf("resolve artifact root: %w", err) + } + if err := ensureMaterializationRoot(root); err != nil { + return nil, func() {}, err + } + + var files []*os.File + closePacks := func() { + for _, file := range files { + _ = file.Close() + } + } + var candidates []artifactCandidate + destinations := make(map[string]wapp.ID) + appendResource := func( + meta wapp.Metadata, + filesystem fs.FS, + moduleVersion string, + resourceID wapp.ID, + source string, + mutable bool, + ) error { + declaration, declared, err := ParseDeclaration(meta) + if err != nil { + return fmt.Errorf("resource %s from %s: %w", resourceID.String(), source, err) + } + if !declared { + return nil + } + if filesystem == nil { + return fmt.Errorf("resource %s from %s has no filesystem", resourceID.String(), source) + } + descriptor, err := e.registry.Inspect(ctx, declaration, InspectInput{ + Filesystem: filesystem, + ModuleVersion: moduleVersion, + ResourceID: resourceID, + }) + if err != nil { + return err + } + format, _ := e.registry.Resolve(declaration.Format) + managedRoot := pathClean(format.Root()) + destination := filepath.Join(root, filepath.FromSlash(descriptor.RelativePath)) + if err := ensureWithinRoot(root, destination); err != nil { + return err + } + key := strings.ToLower(filepath.Clean(destination)) + for previousPath, previous := range destinations { + if pathsOverlap(key, previousPath) { + return fmt.Errorf( + "artifact resources %s at %q and %s at %q have overlapping outputs", + previous.String(), previousPath, + resourceID.String(), descriptor.RelativePath, + ) + } + } + destinations[key] = resourceID + candidates = append(candidates, artifactCandidate{ + filesystem: filesystem, + root: managedRoot, + mutable: mutable, + result: Materialized{ + Descriptor: descriptor, + Destination: destination, + ResourceID: resourceID, + Source: source, + }, + }) + return nil + } + + for _, pack := range e.packs { + file, openErr := os.Open(pack.Path) + if openErr != nil { + closePacks() + return nil, func() {}, fmt.Errorf("open WAPP %s: %w", pack.Path, openErr) + } + files = append(files, file) + reader, readErr := wapp.NewReader(file) + if readErr != nil { + closePacks() + return nil, func() {}, fmt.Errorf("read WAPP %s: %w", pack.Path, readErr) + } + moduleVersion := pack.ModuleVersion + if moduleVersion == "" { + metadata, metadataErr := reader.GetMetadata() + if metadataErr != nil { + closePacks() + return nil, func() {}, fmt.Errorf("read WAPP metadata %s: %w", pack.Path, metadataErr) + } + moduleVersion, _ = metadata["version"].(string) + } + for _, resource := range reader.ListResources() { + filesystem, fsErr := reader.GetFS(resource.ID) + if fsErr != nil { + closePacks() + return nil, func() {}, fmt.Errorf("open resource %s in %s: %w", resource.ID.String(), pack.Path, fsErr) + } + if err := appendResource( + resource.Meta, filesystem, moduleVersion, resource.ID, pack.Path, false, + ); err != nil { + closePacks() + return nil, func() {}, err + } + } + } + for _, resource := range e.resources { + if err := appendResource( + resource.Meta, + resource.Filesystem, + resource.ModuleVersion, + resource.ResourceID, + resource.Source, + true, + ); err != nil { + closePacks() + return nil, func() {}, err + } + } + return candidates, closePacks, nil +} + +func (e *Effect) stage(candidates []artifactCandidate) ([]stagedRoot, error) { + root, err := filepath.Abs(e.root) + if err != nil { + return nil, fmt.Errorf("resolve artifact root: %w", err) + } + if err := ensureMaterializationRoot(root); err != nil { + return nil, err + } + byRoot := make(map[string][]artifactCandidate) + for _, candidate := range candidates { + byRoot[candidate.root] = append(byRoot[candidate.root], candidate) + } + + var staged []stagedRoot + for _, managedRoot := range e.registry.Roots() { + destination := filepath.Join(root, filepath.FromSlash(managedRoot)) + parent := filepath.Dir(destination) + if err := ensureDirectoryBelowRoot(root, parent); err != nil { + _ = cleanupStagedRoots(staged) + return nil, fmt.Errorf("prepare artifact root %q: %w", managedRoot, err) + } + staging, err := os.MkdirTemp(parent, "."+filepath.Base(destination)+".artifact-stage-*") + if err != nil { + _ = cleanupStagedRoots(staged) + return nil, fmt.Errorf("stage artifact root %q: %w", managedRoot, err) + } + item := stagedRoot{destination: destination, staging: staging} + staged = append(staged, item) + if !e.exact { + info, err := os.Lstat(destination) + if err == nil { + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + _ = cleanupStagedRoots(staged) + return nil, fmt.Errorf("managed artifact root %q is not a directory", managedRoot) + } + if err := copyTree(os.DirFS(destination), staging); err != nil { + _ = cleanupStagedRoots(staged) + return nil, fmt.Errorf("preserve managed artifact root %q: %w", managedRoot, err) + } + } else if !errors.Is(err, os.ErrNotExist) { + _ = cleanupStagedRoots(staged) + return nil, fmt.Errorf("inspect managed artifact root %q: %w", managedRoot, err) + } + } + + for _, candidate := range byRoot[managedRoot] { + var sourceBefore [32]byte + if candidate.mutable { + sourceBefore, err = digestTree(candidate.filesystem) + if err != nil { + _ = cleanupStagedRoots(staged) + return nil, fmt.Errorf( + "snapshot local artifact %s: %w", + candidate.result.ResourceID.String(), err, + ) + } + } + relative, err := filepath.Rel(destination, candidate.result.Destination) + if err != nil || relative == ".." || + strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + _ = cleanupStagedRoots(staged) + return nil, fmt.Errorf( + "resolve artifact %s below managed root %q", + candidate.result.ResourceID.String(), managedRoot, + ) + } + target := filepath.Join(staging, relative) + if relative == "." { + if err := clearDirectory(target); err != nil { + _ = cleanupStagedRoots(staged) + return nil, fmt.Errorf("clear staged artifact root: %w", err) + } + } else { + if err := os.RemoveAll(target); err != nil { + _ = cleanupStagedRoots(staged) + return nil, fmt.Errorf("replace staged artifact destination: %w", err) + } + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + _ = cleanupStagedRoots(staged) + return nil, fmt.Errorf("create staged artifact parent: %w", err) + } + if err := os.Mkdir(target, 0o755); err != nil { + _ = cleanupStagedRoots(staged) + return nil, fmt.Errorf("create staged artifact destination: %w", err) + } + } + if err := copyTree(candidate.filesystem, target); err != nil { + _ = cleanupStagedRoots(staged) + return nil, fmt.Errorf( + "stage artifact %s: %w", + candidate.result.ResourceID.String(), err, + ) + } + if candidate.mutable { + stagedDigest, stagedErr := digestTree(os.DirFS(target)) + sourceAfter, sourceErr := digestTree(candidate.filesystem) + if err := errors.Join(stagedErr, sourceErr); err != nil { + _ = cleanupStagedRoots(staged) + return nil, fmt.Errorf( + "verify local artifact snapshot %s: %w", + candidate.result.ResourceID.String(), err, + ) + } + if sourceBefore != stagedDigest || sourceBefore != sourceAfter { + _ = cleanupStagedRoots(staged) + return nil, fmt.Errorf( + "local artifact %s changed while it was being materialized", + candidate.result.ResourceID.String(), + ) + } + } + } + } + return staged, nil +} + +func clearDirectory(directory string) error { + entries, err := os.ReadDir(directory) + if err != nil { + return err + } + var errs []error + for _, entry := range entries { + errs = append(errs, os.RemoveAll(filepath.Join(directory, entry.Name()))) + } + return errors.Join(errs...) +} + +func pathClean(value string) string { + return filepath.ToSlash(filepath.Clean(filepath.FromSlash(value))) +} + +func activateRoot(staged stagedRoot) (activatedRoot, error) { + activated := activatedRoot{ + destination: staged.destination, + staging: staged.staging, + } + destination := staged.destination + parent := filepath.Dir(destination) + if _, err := os.Lstat(destination); err == nil { + backup, reserveErr := reserveSiblingPath(parent, "."+filepath.Base(destination)+".artifact-backup-*") + if reserveErr != nil { + return activated, reserveErr + } + if err := os.Rename(destination, backup); err != nil { + return activated, fmt.Errorf("move existing artifact to backup: %w", err) + } + activated.backup = backup + activated.hadTarget = true + } else if !errors.Is(err, os.ErrNotExist) { + return activated, fmt.Errorf("inspect artifact destination: %w", err) + } + if err := os.Rename(staged.staging, destination); err != nil { + if activated.hadTarget { + restoreErr := os.Rename(activated.backup, destination) + if restoreErr == nil { + activated.backup = "" + activated.hadTarget = false + } + return activated, errors.Join(err, restoreErr) + } + return activated, fmt.Errorf("activate staged artifact root: %w", err) + } + activated.staging = "" + return activated, nil +} + +func rollbackActivated(activated []activatedRoot) ([]activatedRoot, error) { + var errs []error + var remaining []activatedRoot + for i := len(activated) - 1; i >= 0; i-- { + item := activated[i] + destination := item.destination + if destination != "" { + if err := os.RemoveAll(destination); err != nil { + errs = append(errs, fmt.Errorf("remove materialized artifact root %s: %w", destination, err)) + remaining = append(remaining, item) + continue + } + item.destination = "" + } + if item.hadTarget { + if err := os.Rename(item.backup, destination); err != nil { + item.destination = destination + errs = append(errs, fmt.Errorf("restore artifact root %s: %w", destination, err)) + remaining = append(remaining, item) + continue + } + item.hadTarget = false + item.backup = "" + } + if item.staging != "" { + if err := os.RemoveAll(item.staging); err != nil { + errs = append(errs, fmt.Errorf("remove staged artifact root %s: %w", item.staging, err)) + remaining = append(remaining, item) + } + } + } + return remaining, errors.Join(errs...) +} + +func cleanupStagedRoots(staged []stagedRoot) error { + var errs []error + for _, item := range staged { + if item.staging == "" { + continue + } + if err := os.RemoveAll(item.staging); err != nil { + errs = append(errs, fmt.Errorf("remove staged artifact root %s: %w", item.staging, err)) + } + } + return errors.Join(errs...) +} + +func reserveSiblingPath(parent, pattern string) (string, error) { + path, err := os.MkdirTemp(parent, pattern) + if err != nil { + return "", fmt.Errorf("reserve artifact backup: %w", err) + } + if err := os.Remove(path); err != nil { + return "", fmt.Errorf("prepare artifact backup: %w", err) + } + return path, nil +} diff --git a/boot/deps/artifact/wapp_test.go b/boot/deps/artifact/wapp_test.go new file mode 100644 index 000000000..81030eeed --- /dev/null +++ b/boot/deps/artifact/wapp_test.go @@ -0,0 +1,244 @@ +// SPDX-License-Identifier: MPL-2.0 + +package artifact_test + +import ( + "context" + "os" + "path/filepath" + "testing" + "testing/fstest" + + "github.com/wippyai/runtime/boot/deps/artifact" + "github.com/wippyai/runtime/boot/deps/artifact/nodepackage" + "github.com/wippyai/wapp" +) + +func TestWAPPEffectRollbackRestoresPreviousOutput(t *testing.T) { + root := t.TempDir() + packPath := filepath.Join(t.TempDir(), "package.wapp") + writeArtifactWAPP(t, packPath, "@example/package", "1.0.0", "new") + + destination := filepath.Join(root, "npm", "@example", "package") + if err := os.MkdirAll(filepath.Join(destination, "dist"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(destination, "dist", "index.js"), []byte("old"), 0o644); err != nil { + t.Fatal(err) + } + + effect, err := artifact.NewWAPPEffect(testArtifactRegistry(t), []artifact.WAPP{{ + Path: packPath, + ModuleVersion: "1.0.0", + }}, root) + if err != nil { + t.Fatal(err) + } + if err := effect.Prepare(context.Background()); err != nil { + t.Fatalf("prepare: %v", err) + } + assertFileContent(t, filepath.Join(destination, "dist", "index.js"), "new") + if err := effect.Rollback(context.Background()); err != nil { + t.Fatalf("rollback: %v", err) + } + assertFileContent(t, filepath.Join(destination, "dist", "index.js"), "old") +} + +func TestWAPPEffectCommitKeepsOutputAndRemovesBackup(t *testing.T) { + root := t.TempDir() + packPath := filepath.Join(t.TempDir(), "package.wapp") + writeArtifactWAPP(t, packPath, "@example/package", "1.0.0", "new") + + destination := filepath.Join(root, "npm", "@example", "package") + if err := os.MkdirAll(destination, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(destination, "old.txt"), []byte("old"), 0o644); err != nil { + t.Fatal(err) + } + + effect, err := artifact.NewWAPPEffect(testArtifactRegistry(t), []artifact.WAPP{{ + Path: packPath, + }}, root) + if err != nil { + t.Fatal(err) + } + if err := effect.Prepare(context.Background()); err != nil { + t.Fatalf("prepare: %v", err) + } + if err := effect.Commit(context.Background()); err != nil { + t.Fatalf("commit: %v", err) + } + if err := effect.Finalize(context.Background()); err != nil { + t.Fatalf("finalize: %v", err) + } + + assertFileContent(t, filepath.Join(destination, "dist", "index.js"), "new") + if _, err := os.Stat(filepath.Join(destination, "old.txt")); !os.IsNotExist(err) { + t.Fatalf("stale output remains: %v", err) + } + backups, err := filepath.Glob(filepath.Join(filepath.Dir(destination), ".package.artifact-backup-*")) + if err != nil { + t.Fatal(err) + } + if len(backups) != 0 { + t.Fatalf("artifact backups remain: %v", backups) + } +} + +func TestWAPPEffectRejectsPackCollisionBeforeMutation(t *testing.T) { + root := t.TempDir() + first := filepath.Join(t.TempDir(), "first.wapp") + second := filepath.Join(t.TempDir(), "second.wapp") + writeArtifactWAPP(t, first, "@example/package", "1.0.0", "first") + writeArtifactWAPP(t, second, "@example/package", "1.0.0", "second") + + destination := filepath.Join(root, "npm", "@example", "package", "dist", "index.js") + if err := os.MkdirAll(filepath.Dir(destination), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(destination, []byte("old"), 0o644); err != nil { + t.Fatal(err) + } + + effect, err := artifact.NewWAPPEffect(testArtifactRegistry(t), []artifact.WAPP{ + {Path: first}, + {Path: second}, + }, root) + if err != nil { + t.Fatal(err) + } + if err := effect.Prepare(context.Background()); err == nil { + t.Fatal("expected destination collision") + } + assertFileContent(t, destination, "old") +} + +func TestWAPPEffectReconcilesRegisteredRootToExactSet(t *testing.T) { + root := t.TempDir() + stale := filepath.Join(root, "npm", "@example", "stale", "index.js") + if err := os.MkdirAll(filepath.Dir(stale), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(stale, []byte("stale"), 0o644); err != nil { + t.Fatal(err) + } + + effect, err := artifact.NewWAPPEffect(testArtifactRegistry(t), nil, root) + if err != nil { + t.Fatal(err) + } + if err := effect.Prepare(context.Background()); err != nil { + t.Fatalf("prepare: %v", err) + } + if err := effect.Commit(context.Background()); err != nil { + t.Fatalf("commit: %v", err) + } + if err := effect.Finalize(context.Background()); err != nil { + t.Fatalf("finalize: %v", err) + } + + if _, err := os.Stat(stale); !os.IsNotExist(err) { + t.Fatalf("stale artifact remains: %v", err) + } + info, err := os.Stat(filepath.Join(root, "npm")) + if err != nil { + t.Fatal(err) + } + if !info.IsDir() { + t.Fatal("managed artifact root is not a directory") + } +} + +func TestPartialWAPPEffectPreservesUnselectedOutputs(t *testing.T) { + root := t.TempDir() + packPath := filepath.Join(t.TempDir(), "selected.wapp") + writeArtifactWAPP(t, packPath, "@example/selected", "1.0.0", "new") + + unselected := filepath.Join(root, "npm", "@example", "unselected", "index.js") + if err := os.MkdirAll(filepath.Dir(unselected), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(unselected, []byte("keep"), 0o644); err != nil { + t.Fatal(err) + } + + effect, err := artifact.NewPartialEffect( + testArtifactRegistry(t), + []artifact.WAPP{{Path: packPath, ModuleVersion: "1.0.0"}}, + nil, + root, + ) + if err != nil { + t.Fatal(err) + } + if err := effect.Prepare(context.Background()); err != nil { + t.Fatalf("prepare: %v", err) + } + if err := effect.Commit(context.Background()); err != nil { + t.Fatalf("commit: %v", err) + } + if err := effect.Finalize(context.Background()); err != nil { + t.Fatalf("finalize: %v", err) + } + + assertFileContent(t, unselected, "keep") + assertFileContent( + t, + filepath.Join(root, "npm", "@example", "selected", "dist", "index.js"), + "new", + ) +} + +func testArtifactRegistry(t *testing.T) *artifact.Registry { + t.Helper() + registry := artifact.NewRegistry() + if err := registry.Register(nodepackage.New()); err != nil { + t.Fatal(err) + } + return registry +} + +func writeArtifactWAPP(t *testing.T, path, packageName, version, content string) { + t.Helper() + file, err := os.Create(path) + if err != nil { + t.Fatal(err) + } + resourceID := wapp.NewID("example.package", "artifact") + err = wapp.NewWriter().PackWithResources( + wapp.Metadata{"version": version}, + nil, + []wapp.ResourceSpec{{ + ID: resourceID, + Meta: wapp.Metadata{ + "artifact": map[string]any{"format": "node-package"}, + }, + FS: fstest.MapFS{ + "package.json": &fstest.MapFile{Data: []byte( + `{"name":"` + packageName + `","version":"` + version + `"}`, + )}, + "dist/index.js": &fstest.MapFile{Data: []byte(content)}, + }, + }}, + file, + ) + closeErr := file.Close() + if err != nil { + t.Fatal(err) + } + if closeErr != nil { + t.Fatal(closeErr) + } +} + +func assertFileContent(t *testing.T, path, want string) { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(data) != want { + t.Fatalf("%s = %q, want %q", path, data, want) + } +} diff --git a/boot/deps/hub/artifact_effect.go b/boot/deps/hub/artifact_effect.go new file mode 100644 index 000000000..8d9755848 --- /dev/null +++ b/boot/deps/hub/artifact_effect.go @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: MPL-2.0 + +package hub + +import ( + "context" + + regapi "github.com/wippyai/runtime/api/registry" + "github.com/wippyai/runtime/boot/deps/artifact" +) + +var _ regapi.FinalizingEffect = (*artifact.Effect)(nil) + +// buildArtifactEffect binds derived artifact outputs to the same transaction as +// the dependency graph change. Module selection and verification remain owned +// by DependencyHandler; the artifact subsystem only sees exact WAPP paths. +func (h *DependencyHandler) buildArtifactEffect( + ctx context.Context, + resolved []ResolvedModule, + state regapi.State, +) (regapi.Effect, error) { + if h == nil || h.artifacts == nil { + return nil, nil + } + + packs := make([]artifact.WAPP, 0, len(resolved)) + replacementVersions := make(map[string]string) + seen := make(map[string]struct{}, len(resolved)) + for _, module := range resolved { + moduleName := module.Org + "/" + module.Name + if _, replaced := h.replacementPath(moduleName); replaced || + module.Source == moduleSourceReplacementTreeV1 { + replacementVersions[moduleName] = module.Version + continue + } + path, err := h.ensureModuleAvailable(ctx, module) + if err != nil { + return nil, err + } + if _, exists := seen[path]; exists { + continue + } + seen[path] = struct{}{} + packs = append(packs, artifact.WAPP{ + Path: path, + ModuleVersion: module.Version, + }) + } + resources, err := h.replacementArtifactResources(ctx, state, replacementVersions) + if err != nil { + return nil, err + } + return artifact.NewEffect(h.artifacts, packs, resources, h.artifactRoot) +} + +func (h *DependencyHandler) replacementArtifactResources( + ctx context.Context, + state regapi.State, + versions map[string]string, +) ([]artifact.Resource, error) { + if len(versions) == 0 { + return nil, nil + } + roots := make(map[string]string, len(versions)) + for module := range versions { + root, ok := h.replacementPath(module) + if ok { + roots[module] = root + } + } + return artifact.DirectoryResources(ctx, state, roots, versions) +} diff --git a/boot/deps/hub/artifact_effect_test.go b/boot/deps/hub/artifact_effect_test.go new file mode 100644 index 000000000..155019689 --- /dev/null +++ b/boot/deps/hub/artifact_effect_test.go @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: MPL-2.0 + +package hub + +import ( + "context" + "os" + "path/filepath" + "testing" + "testing/fstest" + + "github.com/wippyai/runtime/api/attrs" + "github.com/wippyai/runtime/api/payload" + regapi "github.com/wippyai/runtime/api/registry" + dirapi "github.com/wippyai/runtime/api/service/fs/directory" + "github.com/wippyai/runtime/boot/deps/artifact" + "github.com/wippyai/runtime/boot/deps/artifact/nodepackage" + "github.com/wippyai/runtime/boot/deps/graph" + "github.com/wippyai/runtime/boot/deps/lock" + "github.com/wippyai/wapp" + "go.uber.org/zap" +) + +func TestBuildArtifactEffectMaterializesVerifiedResolvedWAPP(t *testing.T) { + root := t.TempDir() + vendorDir := filepath.Join(root, "vendor") + name := graph.Name{Organization: "example", Module: "package"} + packPath := filepath.Join(vendorDir, lock.WappPath(name, "1.0.0")) + writeDependencyArtifactWAPP(t, packPath) + + registry := artifact.NewRegistry() + if err := registry.Register(nodepackage.New()); err != nil { + t.Fatal(err) + } + handler := &DependencyHandler{ + artifacts: registry, + artifactRoot: root, + vendorDir: vendorDir, + replacements: map[string]lock.Replacement{}, + logger: zap.NewNop(), + } + + effect, err := handler.buildArtifactEffect(context.Background(), []ResolvedModule{{ + Org: name.Organization, + Name: name.Module, + Version: "1.0.0", + }}, nil) + if err != nil { + t.Fatalf("build artifact effect: %v", err) + } + if effect == nil { + t.Fatal("expected artifact effect") + } + if err := effect.Prepare(context.Background()); err != nil { + t.Fatalf("prepare: %v", err) + } + if err := effect.Commit(context.Background()); err != nil { + t.Fatalf("commit: %v", err) + } + + data, err := os.ReadFile(filepath.Join(root, "npm", "@example", "package", "dist", "index.js")) + if err != nil { + t.Fatal(err) + } + if string(data) != "export {}" { + t.Fatalf("materialized content = %q", data) + } +} + +func TestBuildArtifactEffectMaterializesLocalReplacement(t *testing.T) { + root := t.TempDir() + replacement := t.TempDir() + if err := os.WriteFile( + filepath.Join(replacement, "package.json"), + []byte(`{"name":"@example/package","version":"1.0.0"}`), + 0o644, + ); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(replacement, "dist"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(replacement, "dist", "index.js"), []byte("local"), 0o644); err != nil { + t.Fatal(err) + } + + registry := artifact.NewRegistry() + if err := registry.Register(nodepackage.New()); err != nil { + t.Fatal(err) + } + handler := &DependencyHandler{ + artifacts: registry, + artifactRoot: root, + replacements: map[string]lock.Replacement{ + "example/package": {From: "example/package", To: replacement}, + }, + logger: zap.NewNop(), + } + state := regapi.State{{ + ID: regapi.NewID("example.package", "artifact"), + Kind: dirapi.Kind, + Meta: attrs.NewBagFrom(map[string]any{ + metaModuleKey: "example/package", + "artifact": map[string]any{"format": "node-package"}, + }), + Data: payload.New(map[string]any{ + "directory": ".", + "base": dirapi.BaseModule, + }), + }} + effect, err := handler.buildArtifactEffect(newTestContext(), []ResolvedModule{{ + Org: "example", + Name: "package", + Version: "1.0.0", + Source: moduleSourceReplacementTreeV1, + }}, state) + if err != nil { + t.Fatalf("build artifact effect: %v", err) + } + if err := effect.Prepare(context.Background()); err != nil { + t.Fatalf("prepare: %v", err) + } + if err := effect.Commit(context.Background()); err != nil { + t.Fatalf("commit: %v", err) + } + if finalizer, ok := effect.(regapi.FinalizingEffect); ok { + if err := finalizer.Finalize(context.Background()); err != nil { + t.Fatalf("finalize: %v", err) + } + } + + data, err := os.ReadFile(filepath.Join(root, "npm", "@example", "package", "dist", "index.js")) + if err != nil { + t.Fatal(err) + } + if string(data) != "local" { + t.Fatalf("materialized content = %q", data) + } +} + +func TestBuildArtifactEffectRemovesOutputsWhenGraphIsEmpty(t *testing.T) { + root := t.TempDir() + stale := filepath.Join(root, "npm", "@example", "removed", "index.js") + if err := os.MkdirAll(filepath.Dir(stale), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(stale, []byte("stale"), 0o644); err != nil { + t.Fatal(err) + } + + registry := artifact.NewRegistry() + if err := registry.Register(nodepackage.New()); err != nil { + t.Fatal(err) + } + handler := &DependencyHandler{ + artifacts: registry, + artifactRoot: root, + replacements: map[string]lock.Replacement{}, + logger: zap.NewNop(), + } + effect, err := handler.buildArtifactEffect(context.Background(), nil, nil) + if err != nil { + t.Fatal(err) + } + if effect == nil { + t.Fatal("expected exact reconciliation effect") + } + if err := effect.Prepare(context.Background()); err != nil { + t.Fatal(err) + } + if err := effect.Commit(context.Background()); err != nil { + t.Fatal(err) + } + if finalizer, ok := effect.(regapi.FinalizingEffect); ok { + if err := finalizer.Finalize(context.Background()); err != nil { + t.Fatal(err) + } + } + if _, err := os.Stat(stale); !os.IsNotExist(err) { + t.Fatalf("removed artifact remains: %v", err) + } +} + +func writeDependencyArtifactWAPP(t *testing.T, path string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + file, err := os.Create(path) + if err != nil { + t.Fatal(err) + } + err = wapp.NewWriter().PackWithResources( + wapp.Metadata{"version": "1.0.0"}, + nil, + []wapp.ResourceSpec{{ + ID: wapp.NewID("example.package", "artifact"), + Meta: wapp.Metadata{ + "artifact": map[string]any{"format": "node-package"}, + }, + FS: fstest.MapFS{ + "package.json": &fstest.MapFile{Data: []byte( + `{"name":"@example/package","version":"1.0.0"}`, + )}, + "dist/index.js": &fstest.MapFile{Data: []byte("export {}")}, + }, + }}, + file, + ) + closeErr := file.Close() + if err != nil { + t.Fatal(err) + } + if closeErr != nil { + t.Fatal(closeErr) + } +} diff --git a/boot/deps/hub/dependency_handler.go b/boot/deps/hub/dependency_handler.go index 434cce2e4..10ffa239b 100644 --- a/boot/deps/hub/dependency_handler.go +++ b/boot/deps/hub/dependency_handler.go @@ -26,6 +26,7 @@ import ( hubsemver "github.com/wippyai/runtime/api/semver" "github.com/wippyai/runtime/boot/build" "github.com/wippyai/runtime/boot/build/stages" + "github.com/wippyai/runtime/boot/deps/artifact" "github.com/wippyai/runtime/boot/deps/auth" depconfig "github.com/wippyai/runtime/boot/deps/config" "github.com/wippyai/runtime/boot/deps/graph" @@ -48,9 +49,11 @@ const ( type DependencyHandlerOptions struct { Hub HubClient Resolver regapi.DependencyResolver + Artifacts *artifact.Registry Logger *zap.Logger LockPath string VendorDir string + ArtifactRoot string WorkspaceReplacements []lock.Replacement ResolveTimeout time.Duration DownloadTimeout time.Duration @@ -62,6 +65,8 @@ type DependencyHandler struct { manifestCache *ManifestCache logger *zap.Logger lock *lock.Lock + artifacts *artifact.Registry + artifactRoot string replacements map[string]lock.Replacement vendorDir string resolveTimeout time.Duration @@ -136,6 +141,10 @@ func NewDependencyHandler(opts DependencyHandlerOptions) (*DependencyHandler, er if vendorDir == "" { vendorDir = filepath.Join(".wippy", "vendor") } + artifactRoot := opts.ArtifactRoot + if artifactRoot == "" { + artifactRoot = filepath.Dir(vendorDir) + } replacements := make(map[string]lock.Replacement) if lockObj != nil { @@ -149,6 +158,8 @@ func NewDependencyHandler(opts DependencyHandlerOptions) (*DependencyHandler, er manifestCache: NewManifestCache(client), logger: logger, resolver: opts.Resolver, + artifacts: opts.Artifacts, + artifactRoot: artifactRoot, vendorDir: vendorDir, resolveTimeout: opts.ResolveTimeout, downloadTimeout: opts.DownloadTimeout, @@ -353,6 +364,10 @@ func (h *DependencyHandler) expand( } var effects []regapi.Effect + artifactEffect, err := h.buildArtifactEffect(ctx, resolved, combined) + if err != nil { + return regapi.DirectiveResult{}, err + } packEffect, err := h.buildEmbedPackEffect(ctx, resolved, snapshot, controlledModules) if err != nil { return regapi.DirectiveResult{}, err @@ -361,6 +376,9 @@ func (h *DependencyHandler) expand( if err != nil { return regapi.DirectiveResult{}, err } + if artifactEffect != nil { + effects = append(effects, artifactEffect) + } if filesystemEffect != nil { effects = append(effects, filesystemEffect) } @@ -764,10 +782,17 @@ func (h *DependencyHandler) ReconcileResolution( return regapi.DirectiveResult{}, err } var effects []regapi.Effect + artifactEffect, err := h.buildArtifactEffect(ctx, resolved, combined) + if err != nil { + return regapi.DirectiveResult{}, err + } filesystemEffect, err := h.buildModuleFilesystemEffect(resolved, controlled, unpackPlan) if err != nil { return regapi.DirectiveResult{}, err } + if artifactEffect != nil { + effects = append(effects, artifactEffect) + } if filesystemEffect != nil { effects = append(effects, filesystemEffect) } diff --git a/cmd/internal/entries/extract.go b/cmd/internal/entries/extract.go index 89bac36ce..99ebd82a4 100644 --- a/cmd/internal/entries/extract.go +++ b/cmd/internal/entries/extract.go @@ -10,3 +10,9 @@ import "github.com/wippyai/runtime/boot/deps/wappextract" func ExtractWappToDir(wappPath, targetDir string) error { return wappextract.ExtractWappToDir(wappPath, targetDir) } + +// ExtractWappToDirKeepSource extracts a module while retaining its canonical +// WAPP for resource-backed artifact reconciliation and repair. +func ExtractWappToDirKeepSource(wappPath, targetDir string) error { + return wappextract.ExtractWappToDirKeepSource(wappPath, targetDir) +} diff --git a/cmd/internal/entries/loader.go b/cmd/internal/entries/loader.go index ab1211ea1..0a3d20331 100644 --- a/cmd/internal/entries/loader.go +++ b/cmd/internal/entries/loader.go @@ -145,7 +145,7 @@ func ensureModulesInstalledFromLock(ctx context.Context, lockObj *lock.Lock, log // Migrate legacy .wapp to extracted directory when unpack is enabled dirPath := filepath.Join(vendorPath, lock.ModulePath(name)) logger.Info("unpacking .wapp to directory", zap.String("module", mod.Name)) - if err := ExtractWappToDir(resolved.Path, dirPath); err != nil { + if err := ExtractWappToDirKeepSource(resolved.Path, dirPath); err != nil { return NewExtractModuleError(mod.Name, err) } } @@ -210,14 +210,21 @@ func ensureModulesInstalledFromLock(ctx context.Context, lockObj *lock.Lock, log if err := hubClient.DownloadToFile(ctx, downloadInfo.URL, fullWappPath); err != nil { return NewDownloadModuleError(moduleRef, err) } + if err := hub.VerifyDownloadedArtifact( + fullWappPath, downloadInfo.Digest, downloadInfo.Size, + ); err != nil { + _ = os.Remove(fullWappPath) + return NewDownloadModuleError(moduleRef, fmt.Errorf("verify downloaded WAPP: %w", err)) + } if shouldUnpack { - // Extract .wapp to source directory and remove the .wapp file + // Extract to the module directory while retaining the canonical + // WAPP for artifact reconciliation and repair. dirPath := filepath.Join(vendorPath, lock.ModulePath(name)) if err := os.RemoveAll(dirPath); err != nil { return NewExtractModuleError(moduleRef, err) } - if err := ExtractWappToDir(fullWappPath, dirPath); err != nil { + if err := ExtractWappToDirKeepSource(fullWappPath, dirPath); err != nil { return NewExtractModuleError(moduleRef, err) } } diff --git a/cmd/wippy/cmd/artifact_formats.go b/cmd/wippy/cmd/artifact_formats.go new file mode 100644 index 000000000..a90243d20 --- /dev/null +++ b/cmd/wippy/cmd/artifact_formats.go @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: MPL-2.0 + +package cmd + +import ( + "context" + "errors" + "fmt" + + "github.com/wippyai/runtime/boot/deps/artifact" + "github.com/wippyai/runtime/boot/deps/artifact/standard" + "github.com/wippyai/wapp" +) + +func newArtifactRegistry() (*artifact.Registry, error) { + return standard.NewRegistry() +} + +func validateArtifactResources( + ctx context.Context, + resources []wapp.ResourceSpec, + moduleVersion string, +) error { + registry, err := newArtifactRegistry() + if err != nil { + return err + } + _, err = artifact.InspectResources(ctx, registry, resources, moduleVersion) + return err +} + +func materializeArtifacts( + ctx context.Context, + packs []artifact.WAPP, + resources []artifact.Resource, + root string, + exact bool, +) error { + registry, err := newArtifactRegistry() + if err != nil { + return err + } + var effect *artifact.Effect + if exact { + effect, err = artifact.NewEffect(registry, packs, resources, root) + } else { + effect, err = artifact.NewPartialEffect(registry, packs, resources, root) + } + if err != nil { + return fmt.Errorf("prepare module artifacts: %w", err) + } + if err := effect.Prepare(ctx); err != nil { + return fmt.Errorf("materialize module artifacts: %w", err) + } + if err := effect.Commit(ctx); err != nil { + rollbackErr := effect.Rollback(ctx) + return fmt.Errorf("commit module artifacts: %w", errors.Join(err, rollbackErr)) + } + if err := effect.Finalize(ctx); err != nil { + return fmt.Errorf("finalize module artifacts: %w", err) + } + return nil +} diff --git a/cmd/wippy/cmd/artifacts.go b/cmd/wippy/cmd/artifacts.go new file mode 100644 index 000000000..0eb0e6dd8 --- /dev/null +++ b/cmd/wippy/cmd/artifacts.go @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: MPL-2.0 + +package cmd + +import ( + "fmt" + "os" + "strings" + + "github.com/spf13/cobra" + "github.com/wippyai/runtime/boot/deps/artifact" + "github.com/wippyai/wapp" +) + +var artifactsCmd = &cobra.Command{ + Use: "artifacts", + Short: "Work with build-time filesystem artifacts", +} + +var artifactsMaterializeCmd = &cobra.Command{ + Use: "materialize ", + Short: "Validate and materialize an embedded artifact resource", + Long: `Materialize one artifact filesystem from an existing WAPP. + +The resource must declare meta.artifact.format and the format must be registered +in this CLI. The command does not resolve module dependencies, mutate +wippy.lock, invoke package managers, or participate in runtime composition.`, + Args: cobra.ExactArgs(2), + RunE: runArtifactsMaterialize, +} + +func init() { + rootCmd.AddCommand(artifactsCmd) + artifactsCmd.AddCommand(artifactsMaterializeCmd) + artifactsMaterializeCmd.Flags().String("root", ".wippy", "materialization root") +} + +func runArtifactsMaterialize(cmd *cobra.Command, args []string) error { + root, err := cmd.Flags().GetString("root") + if err != nil { + return fmt.Errorf("read artifact root: %w", err) + } + resourceID, err := parseArtifactResourceID(args[1]) + if err != nil { + return err + } + + file, err := os.Open(args[0]) + if err != nil { + return fmt.Errorf("open WAPP %s: %w", args[0], err) + } + defer func() { _ = file.Close() }() + + reader, err := wapp.NewReader(file) + if err != nil { + return fmt.Errorf("read WAPP %s: %w", args[0], err) + } + info, err := findArtifactResource(reader.ListResources(), resourceID) + if err != nil { + return err + } + declaration, declared, err := artifact.ParseDeclaration(info.Meta) + if err != nil { + return fmt.Errorf("resource %s: %w", resourceID.String(), err) + } + if !declared { + return fmt.Errorf("resource %s does not declare meta.artifact.format", resourceID.String()) + } + filesystem, err := reader.GetFS(resourceID) + if err != nil { + return fmt.Errorf("open resource %s: %w", resourceID.String(), err) + } + registry, err := newArtifactRegistry() + if err != nil { + return err + } + + packMetadata, err := reader.GetMetadata() + if err != nil { + return fmt.Errorf("read WAPP metadata: %w", err) + } + descriptor, destination, err := artifact.Materialize( + cmd.Context(), + registry, + declaration, + artifact.InspectInput{ + Filesystem: filesystem, + ModuleVersion: metadataString(packMetadata, "version"), + ResourceID: resourceID, + }, + root, + ) + if err != nil { + return err + } + _, err = fmt.Fprintf( + cmd.OutOrStdout(), + "Materialized %s@%s to %s\n", + descriptor.Identity, + descriptor.Version, + destination, + ) + if err != nil { + return fmt.Errorf("write materialization result: %w", err) + } + return nil +} + +func parseArtifactResourceID(value string) (wapp.ID, error) { + trimmed := strings.TrimSpace(value) + namespace, name, found := strings.Cut(trimmed, ":") + if !found || + namespace == "" || + name == "" || + namespace != strings.TrimSpace(namespace) || + name != strings.TrimSpace(name) || + strings.Contains(name, ":") { + return wapp.ID{}, fmt.Errorf( + "invalid artifact resource %q: expected full namespace:name", value) + } + return wapp.NewID(namespace, name), nil +} + +func findArtifactResource(resources []wapp.ResourceInfo, id wapp.ID) (wapp.ResourceInfo, error) { + for _, resource := range resources { + if resource.ID.Equal(id) { + return resource, nil + } + } + return wapp.ResourceInfo{}, fmt.Errorf("artifact resource %s not found", id.String()) +} + +func metadataString(metadata wapp.Metadata, key string) string { + value, _ := metadata[key].(string) + return value +} diff --git a/cmd/wippy/cmd/artifacts_test.go b/cmd/wippy/cmd/artifacts_test.go new file mode 100644 index 000000000..7e0112b0b --- /dev/null +++ b/cmd/wippy/cmd/artifacts_test.go @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: MPL-2.0 + +package cmd + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + "testing/fstest" + + "github.com/spf13/cobra" + "github.com/wippyai/wapp" +) + +func TestArtifactsMaterializeFromWapp(t *testing.T) { + packPath := filepath.Join(t.TempDir(), "package.wapp") + pack, err := os.Create(packPath) + if err != nil { + t.Fatal(err) + } + resourceID := wapp.NewID("example.package", "package") + err = wapp.NewWriter().PackWithResources( + wapp.Metadata{"name": "package", "version": "0.1.6"}, + nil, + []wapp.ResourceSpec{{ + ID: resourceID, + Meta: wapp.Metadata{ + "artifact": map[string]any{"format": "node-package"}, + }, + FS: fstest.MapFS{ + "package.json": &fstest.MapFile{Data: []byte( + `{"name":"@example/package","version":"0.1.6"}`, + )}, + "dist/index.js": &fstest.MapFile{Data: []byte("export {}")}, + }, + }}, + pack, + ) + if err != nil { + _ = pack.Close() + t.Fatal(err) + } + if err := pack.Close(); err != nil { + t.Fatal(err) + } + + root := t.TempDir() + command := &cobra.Command{} + command.Flags().String("root", root, "") + if err := runArtifactsMaterialize( + command, + []string{packPath, resourceID.String()}, + ); err != nil { + t.Fatalf("materialize: %v", err) + } + data, err := os.ReadFile(filepath.Join(root, "npm", "@example", "package", "dist", "index.js")) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(data, []byte("export {}")) { + t.Fatalf("content = %q", data) + } +} + +func TestParseArtifactResourceIDRequiresFullID(t *testing.T) { + for _, invalid := range []string{ + "", "package", ":package", "ns:", "ns:one:two", "ns: name", "ns :name", + } { + if _, err := parseArtifactResourceID(invalid); err == nil { + t.Fatalf("parse %q succeeded", invalid) + } + } +} + +func TestArtifactsMaterializeRequiresRootFlag(t *testing.T) { + err := runArtifactsMaterialize( + &cobra.Command{}, + []string{"package.wapp", "example:package"}, + ) + if err == nil || !strings.Contains(err.Error(), "read artifact root") { + t.Fatalf("error = %v, want missing root flag error", err) + } +} diff --git a/cmd/wippy/cmd/install.go b/cmd/wippy/cmd/install.go index 5bc70f02b..506008187 100644 --- a/cmd/wippy/cmd/install.go +++ b/cmd/wippy/cmd/install.go @@ -9,6 +9,7 @@ import ( "path/filepath" "github.com/spf13/cobra" + "github.com/wippyai/runtime/boot/deps/artifact" bootauth "github.com/wippyai/runtime/boot/deps/auth" "github.com/wippyai/runtime/boot/deps/graph" "github.com/wippyai/runtime/boot/deps/hub" @@ -104,17 +105,30 @@ func runInstall(cmd *cobra.Command, args []string) error { selection := selectInstallModules(lockObj, args, logger) modules := selection.modules + lockDir := filepath.Dir(lockObj.Path()) + vendorPath := lockObj.GetVendorPath() + vendorDir := lock.ResolveLockPath(lockDir, vendorPath) + artifactRoot := artifact.ConfiguredRoot(runtimeCfg, filepath.Dir(vendorDir)) + shouldUnpack := lockObj.ShouldUnpackModules() if len(args) > 0 && selection.matched == 0 { logger.Warn("no matching modules found in lock file", zap.Strings("requested", args)) return nil } if len(modules) == 0 { if selection.skippedReplaced > 0 { - logger.Info("all selected modules are local replacements; nothing to install", + logger.Info("all selected modules are local replacements", zap.Int("skipped_replaced", selection.skippedReplaced)) } else { logger.Info("no remote modules to install") } + include := requestedModuleSet(args) + packs, resources, err := installedArtifactInputs(app.Ctx, lockObj, vendorDir, logger, include) + if err != nil { + return NewStoreModuleError("artifacts", err) + } + if err := materializeArtifacts(app.Ctx, packs, resources, artifactRoot, len(args) == 0); err != nil { + return NewStoreModuleError("artifacts", err) + } return nil } if len(args) > 0 { @@ -151,11 +165,6 @@ func runInstall(cmd *cobra.Command, args []string) error { return NewCreateHubClientError(fmt.Errorf("registry %s: %w", registryURL, err)) } - lockDir := filepath.Dir(lockObj.Path()) - vendorPath := lockObj.GetVendorPath() - vendorDir := lock.ResolveLockPath(lockDir, vendorPath) - shouldUnpack := lockObj.ShouldUnpackModules() - refresh := shouldBypassInstallCache(cmd) if refresh { logger.Info("refresh enabled, bypassing module cache") @@ -163,6 +172,13 @@ func runInstall(cmd *cobra.Command, args []string) error { installed := 0 cached := 0 + type pendingExtraction struct { + wappPath string + dirPath string + module string + } + var pendingExtractions []pendingExtraction + var installedModules []lock.Module for _, module := range modules { modName, err := graph.ParseName(module.Name) @@ -180,23 +196,35 @@ func runInstall(cmd *cobra.Command, args []string) error { if !refresh { resolved := lock.ResolveModuleDir(vendorDir, modName, module.Version) if resolved.IsWapp { + if err := hub.VerifyDownloadedArtifact(resolved.Path, module.Hash, 0); err != nil { + return NewStoreModuleError(moduleRef, fmt.Errorf("verify cached WAPP: %w", err)) + } if shouldUnpack { - // Unpack .wapp to directory when unpack is enabled logger.Info("unpacking .wapp to directory", zap.String("module", module.Name)) - if err := entries.ExtractWappToDir(resolved.Path, dirPath); err != nil { - return NewExtractModuleError(module.Name, err) - } + pendingExtractions = append(pendingExtractions, pendingExtraction{ + wappPath: resolved.Path, + dirPath: dirPath, + module: moduleRef, + }) } - // When unpack=false, keep .wapp as-is (already installed) cached++ continue } if _, err := os.Stat(resolved.Path); err == nil { - logger.Info("module already installed, skipping download", + wappPath := filepath.Join(vendorDir, lock.WappPath(modName, module.Version)) + if info, statErr := os.Stat(wappPath); statErr == nil && info.Mode().IsRegular() { + if err := hub.VerifyDownloadedArtifact(wappPath, module.Hash, 0); err != nil { + return NewStoreModuleError(moduleRef, fmt.Errorf("verify cached WAPP: %w", err)) + } + logger.Info("module already installed, skipping download", + zap.String("module", module.Name), + zap.String("version", module.Version)) + cached++ + continue + } + logger.Info("installed module is missing its canonical WAPP; repairing", zap.String("module", module.Name), zap.String("version", module.Version)) - cached++ - continue } } @@ -228,17 +256,17 @@ func runInstall(cmd *cobra.Command, args []string) error { if err := downloadWappViaHubOrLegacy(app.Ctx, hubClient, downloadInfo, wappPath); err != nil { return NewDownloadModuleError(moduleRef, err) } - + if err := hub.VerifyDownloadedArtifact(wappPath, downloadInfo.Digest, downloadInfo.Size); err != nil { + _ = os.Remove(wappPath) + return NewDownloadModuleError(moduleRef, fmt.Errorf("verify downloaded WAPP: %w", err)) + } if shouldUnpack { - // Remove old directory (handles version updates) and extract - if err := os.RemoveAll(dirPath); err != nil { - return NewStoreModuleError(moduleRef, err) - } - if err := entries.ExtractWappToDir(wappPath, dirPath); err != nil { - return NewExtractModuleError(moduleRef, err) - } + pendingExtractions = append(pendingExtractions, pendingExtraction{ + wappPath: wappPath, + dirPath: dirPath, + module: moduleRef, + }) } - // When unpack=false, keep .wapp file as-is // Update hash from download info if available if downloadInfo.Digest != "" && module.Hash != downloadInfo.Digest { @@ -246,10 +274,31 @@ func runInstall(cmd *cobra.Command, args []string) error { lockObj.SetModule(module) } + installedModules = append(installedModules, module) + installed++ + } + + for _, pending := range pendingExtractions { + if err := entries.ExtractWappToDirKeepSource(pending.wappPath, pending.dirPath); err != nil { + return NewExtractModuleError(pending.module, err) + } + } + include := requestedModuleSet(args) + artifactPacks, artifactResources, err := installedArtifactInputs( + app.Ctx, lockObj, vendorDir, logger, include, + ) + if err != nil { + return NewStoreModuleError("artifacts", err) + } + if err := materializeArtifacts( + app.Ctx, artifactPacks, artifactResources, artifactRoot, len(args) == 0, + ); err != nil { + return NewStoreModuleError("artifacts", err) + } + for _, module := range installedModules { logger.Info("installed module", zap.String("module", module.Name), zap.String("version", module.Version)) - installed++ } // Save updated lock file @@ -274,6 +323,114 @@ func runInstall(cmd *cobra.Command, args []string) error { return nil } +func installedArtifactPacks( + lockObj *lock.Lock, + vendorDir string, + include map[string]struct{}, +) ([]artifact.WAPP, error) { + if lockObj == nil { + return nil, nil + } + packs := make([]artifact.WAPP, 0, len(lockObj.GetModules())) + for _, module := range lockObj.GetModules() { + if len(include) > 0 { + if _, selected := include[module.Name]; !selected { + continue + } + } + if _, replaced := lockObj.GetReplacement(module.Name); replaced { + continue + } + name, err := graph.ParseName(module.Name) + if err != nil { + return nil, fmt.Errorf("parse module %q: %w", module.Name, err) + } + path := filepath.Join(vendorDir, lock.WappPath(name, module.Version)) + info, err := os.Stat(path) + if err != nil { + return nil, fmt.Errorf( + "canonical WAPP for %s@%s is unavailable; run wippy install --refresh: %w", + module.Name, module.Version, err, + ) + } + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("canonical WAPP for %s@%s is not a regular file", module.Name, module.Version) + } + if err := hub.VerifyDownloadedArtifact(path, module.Hash, 0); err != nil { + return nil, fmt.Errorf("verify canonical WAPP for %s@%s: %w", module.Name, module.Version, err) + } + packs = append(packs, artifact.WAPP{ + Path: path, + ModuleVersion: module.Version, + }) + } + return packs, nil +} + +func installedArtifactInputs( + ctx context.Context, + lockObj *lock.Lock, + vendorDir string, + logger *zap.Logger, + include map[string]struct{}, +) ([]artifact.WAPP, []artifact.Resource, error) { + packs, err := installedArtifactPacks(lockObj, vendorDir, include) + if err != nil { + return nil, nil, err + } + if lockObj == nil { + return packs, nil, nil + } + + var replacementPaths []lock.ModuleLoadPath + versions := make(map[string]string) + for _, module := range lockObj.GetModules() { + versions[module.Name] = module.Version + } + for _, modulePath := range lockObj.GetModuleLoadPaths() { + if len(include) > 0 { + if _, selected := include[modulePath.Module]; !selected { + continue + } + } + if _, replaced := lockObj.GetReplacement(modulePath.Module); replaced { + replacementPaths = append(replacementPaths, modulePath) + } + } + if len(replacementPaths) == 0 { + return packs, nil, nil + } + + loaded, err := entries.LoadEntriesFromModuleLoadPaths(ctx, replacementPaths, logger) + if err != nil { + return nil, nil, fmt.Errorf("load replacement artifacts: %w", err) + } + roots := make(map[string]string, len(replacementPaths)) + for _, modulePath := range replacementPaths { + root := modulePath.SourceRoot + if root == "" { + root = modulePath.Path + } + roots[modulePath.Module] = root + } + resources, err := artifact.DirectoryResources(ctx, loaded, roots, versions) + if err != nil { + return nil, nil, err + } + return packs, resources, nil +} + +func requestedModuleSet(requested []string) map[string]struct{} { + if len(requested) == 0 { + return nil + } + selected := make(map[string]struct{}, len(requested)) + for _, module := range requested { + selected[module] = struct{}{} + } + return selected +} + type installSelection struct { modules []lock.Module matched int diff --git a/cmd/wippy/cmd/pack.go b/cmd/wippy/cmd/pack.go index 787714376..4a59e8763 100644 --- a/cmd/wippy/cmd/pack.go +++ b/cmd/wippy/cmd/pack.go @@ -539,6 +539,9 @@ func performPack(cmd *cobra.Command, args []string, app *appinit.Context, p *tea "count": len(carriedResources), }}) } + if err := validateArtifactResources(app.Ctx, resources, ""); err != nil { + return NewPackWithResourcesError(fmt.Errorf("validate artifacts: %w", err)) + } var resInfos []resourceInfo if len(resources) > 0 { diff --git a/cmd/wippy/cmd/publish.go b/cmd/wippy/cmd/publish.go index 0749740fb..6967280c7 100644 --- a/cmd/wippy/cmd/publish.go +++ b/cmd/wippy/cmd/publish.go @@ -481,6 +481,13 @@ func packModule(ctx context.Context, app *appinit.Context, cfg *config.ModuleCon } resources := stages.GetResources(ctx) + if err := validateArtifactResources( + ctx, + resources, + cfg.Version, + ); err != nil { + return nil, NewPackWithResourcesError(fmt.Errorf("validate artifacts: %w", err)) + } metadata := attrs.Bag{ "name": cfg.ModuleName, diff --git a/cmd/wippy/cmd/update.go b/cmd/wippy/cmd/update.go index 5c8ddc556..3417dbd3c 100644 --- a/cmd/wippy/cmd/update.go +++ b/cmd/wippy/cmd/update.go @@ -215,18 +215,15 @@ func runUpdate(cmd *cobra.Command, args []string) error { if oldLockObj != nil { changes = lock.Diff(oldLockObj, newLockObj) logChanges(logger, changes) - pruneStaleVendorArtifacts(newLockObj, changes, logger) } - if len(resolvedModules) > 0 { - // Run install to download modules - logger.Info("running install to download modules") - if err := runInstall(cmd, []string{}); err != nil { - return NewInstallFailedError(err) - } - } else if len(replacedModules) == 0 { - logger.Info("no modules to install after update") + // Run install even for an empty or replacement-only graph so managed + // artifact roots converge and stale outputs are removed. + logger.Info("running install to converge modules and artifacts") + if err := runInstall(cmd, []string{}); err != nil { + return NewInstallFailedError(err) } + pruneStaleVendorArtifacts(newLockObj, changes, logger) logger.Info("update completed successfully") return nil @@ -472,13 +469,13 @@ func runTargetedUpdate(cmd *cobra.Command, lockFilePath, srcDir, modulesDir stri logger.Info("lock file updated") logChanges(logger, changes) - pruneStaleVendorArtifacts(newLockObj, changes, logger) // Run install logger.Info("running install to download modules") if err := runInstall(cmd, []string{}); err != nil { return NewInstallFailedError(err) } + pruneStaleVendorArtifacts(newLockObj, changes, logger) logger.Info("update completed successfully") return nil @@ -612,7 +609,12 @@ func pruneStaleVendorArtifacts(lockObj *lock.Lock, changes *lock.Changes, logger pruneModuleArtifacts(vendorDir, removed.Name, removed.Version, true, logger) } for _, updated := range changes.Updated { - pruneModuleArtifacts(vendorDir, updated.Name, updated.OldVersion, true, logger) + if updated.OldVersion == updated.NewVersion { + continue + } + // The current extracted directory now belongs to the newly installed + // version. Only versioned storage for the old selection is stale. + pruneModuleArtifacts(vendorDir, updated.Name, updated.OldVersion, false, logger) } } diff --git a/cmd/wippy/cmd/update_test.go b/cmd/wippy/cmd/update_test.go index dfaa3c75e..e293d2712 100644 --- a/cmd/wippy/cmd/update_test.go +++ b/cmd/wippy/cmd/update_test.go @@ -208,11 +208,43 @@ func TestPruneStaleVendorArtifacts_RemovesStaleArtifacts(t *testing.T) { assertPathMissing(t, removedDir) assertPathMissing(t, removedLegacyDir) assertPathMissing(t, removedWapp) - assertPathMissing(t, updatedDir) + if _, err := os.Stat(updatedDir); err != nil { + t.Fatalf("updated current module directory was pruned: %v", err) + } assertPathMissing(t, updatedLegacyDir) assertPathMissing(t, updatedOldWapp) } +func TestPruneStaleVendorArtifacts_PreservesSameVersionUpdate(t *testing.T) { + tmpDir := t.TempDir() + lockObj, err := lock.New(filepath.Join(tmpDir, "wippy.lock")) + if err != nil { + t.Fatalf("create lock: %v", err) + } + + vendorDir := filepath.Join(tmpDir, ".wippy", "vendor") + currentDir := filepath.Join(vendorDir, "demo", "sql") + legacyDir := filepath.Join(vendorDir, "demo", "sql-v1.0.0") + currentWapp := filepath.Join(vendorDir, "demo", "sql-v1.0.0.wapp") + mustWriteFile(t, filepath.Join(currentDir, "current.txt")) + mustWriteFile(t, filepath.Join(legacyDir, "current.txt")) + mustWriteFile(t, currentWapp) + + pruneStaleVendorArtifacts(lockObj, &lock.Changes{ + Updated: []lock.ModuleChange{{ + Name: "demo/sql", + OldVersion: "v1.0.0", + NewVersion: "v1.0.0", + }}, + }, zap.NewNop()) + + for _, artifactPath := range []string{currentDir, legacyDir, currentWapp} { + if _, err := os.Stat(artifactPath); err != nil { + t.Fatalf("same-version artifact %s was pruned: %v", artifactPath, err) + } + } +} + func TestLoadDependencyScanEntriesIncludesReplacementSources(t *testing.T) { ctx := setupLoaderContext(t) ldr := bootapi.GetLoader(ctx)