diff --git a/gobin/coalescer.go b/gobin/coalescer.go deleted file mode 100644 index 5686ec38e..000000000 --- a/gobin/coalescer.go +++ /dev/null @@ -1,45 +0,0 @@ -package gobin - -import ( - "context" - "strings" - - "github.com/quay/claircore" - "github.com/quay/claircore/indexer" -) - -type coalescer struct{} - -func (c *coalescer) Coalesce(ctx context.Context, ls []*indexer.LayerArtifacts) (*claircore.IndexReport, error) { - ir := &claircore.IndexReport{ - Environments: map[string][]*claircore.Environment{}, - Packages: map[string]*claircore.Package{}, - Repositories: map[string]*claircore.Repository{}, - } - for _, l := range ls { - var rid string - for _, r := range l.Repos { - // Magic strings copied out of the osv package. - if r.Name != `go` || r.URI != `https://pkg.go.dev/` { - continue - } - rid = r.ID - ir.Repositories[r.ID] = r - break - } - for _, pkg := range l.Pkgs { - if !strings.HasPrefix(pkg.PackageDB, "go:") { - continue - } - ir.Packages[pkg.ID] = pkg - ir.Environments[pkg.ID] = []*claircore.Environment{ - { - PackageDB: pkg.PackageDB, - IntroducedIn: l.Hash, - RepositoryIDs: []string{rid}, - }, - } - } - } - return ir, nil -} diff --git a/gobin/ecosystem.go b/gobin/ecosystem.go index 294728ad2..1ded67dca 100644 --- a/gobin/ecosystem.go +++ b/gobin/ecosystem.go @@ -4,10 +4,11 @@ import ( "context" "github.com/quay/claircore/indexer" + "github.com/quay/claircore/language" ) // NewEcosystem provides the ecosystem for handling go binaries. -func NewEcosystem(ctx context.Context) *indexer.Ecosystem { +func NewEcosystem(_ context.Context) *indexer.Ecosystem { return &indexer.Ecosystem{ Name: "gobin", PackageScanners: func(context.Context) ([]indexer.PackageScanner, error) { @@ -15,6 +16,6 @@ func NewEcosystem(ctx context.Context) *indexer.Ecosystem { }, DistributionScanners: func(context.Context) ([]indexer.DistributionScanner, error) { return nil, nil }, RepositoryScanners: func(context.Context) ([]indexer.RepositoryScanner, error) { return nil, nil }, - Coalescer: func(context.Context) (indexer.Coalescer, error) { return &coalescer{}, nil }, + Coalescer: language.NewCoalescer, } } diff --git a/gobin/gobin.go b/gobin/gobin.go index 763a35fe4..aa3a9236c 100644 --- a/gobin/gobin.go +++ b/gobin/gobin.go @@ -33,7 +33,7 @@ type Detector struct{} const ( detectorName = `gobin` - detectorVersion = `7` + detectorVersion = `9` detectorKind = `package` ) diff --git a/java/coalescer.go b/java/coalescer.go deleted file mode 100644 index 688f86280..000000000 --- a/java/coalescer.go +++ /dev/null @@ -1,42 +0,0 @@ -package java - -import ( - "context" - - "github.com/quay/claircore" - "github.com/quay/claircore/indexer" -) - -type coalescer struct{} - -func (*coalescer) Coalesce(ctx context.Context, ls []*indexer.LayerArtifacts) (*claircore.IndexReport, error) { - ir := &claircore.IndexReport{ - Environments: map[string][]*claircore.Environment{}, - Packages: map[string]*claircore.Package{}, - Repositories: map[string]*claircore.Repository{}, - } - - for _, l := range ls { - // If we didn't find at least one maven repo in this layer - // no point in searching for packages. - if len(l.Repos) == 0 { - continue - } - rs := make([]string, len(l.Repos)) - for i, r := range l.Repos { - rs[i] = r.ID - ir.Repositories[r.ID] = r - } - for _, pkg := range l.Pkgs { - ir.Packages[pkg.ID] = pkg - ir.Environments[pkg.ID] = []*claircore.Environment{ - { - PackageDB: pkg.PackageDB, - IntroducedIn: l.Hash, - RepositoryIDs: rs, - }, - } - } - } - return ir, nil -} diff --git a/java/ecosystem.go b/java/ecosystem.go index 1cca07347..894ab920c 100644 --- a/java/ecosystem.go +++ b/java/ecosystem.go @@ -4,10 +4,11 @@ import ( "context" "github.com/quay/claircore/indexer" + "github.com/quay/claircore/language" ) // NewEcosystem provides the set of scanners for the java ecosystem. -func NewEcosystem(ctx context.Context) *indexer.Ecosystem { +func NewEcosystem(_ context.Context) *indexer.Ecosystem { return &indexer.Ecosystem{ PackageScanners: func(_ context.Context) ([]indexer.PackageScanner, error) { return []indexer.PackageScanner{&Scanner{}}, nil @@ -16,8 +17,6 @@ func NewEcosystem(ctx context.Context) *indexer.Ecosystem { RepositoryScanners: func(_ context.Context) ([]indexer.RepositoryScanner, error) { return nil, nil }, - Coalescer: func(_ context.Context) (indexer.Coalescer, error) { - return (*coalescer)(nil), nil - }, + Coalescer: language.NewCoalescer, } } diff --git a/java/packagescanner.go b/java/packagescanner.go index af1356cff..daf0b5b27 100644 --- a/java/packagescanner.go +++ b/java/packagescanner.go @@ -75,7 +75,7 @@ type Scanner struct { func (*Scanner) Name() string { return "java" } // Version implements scanner.VersionedScanner. -func (*Scanner) Version() string { return "8" } +func (*Scanner) Version() string { return "9" } // Kind implements scanner.VersionedScanner. func (*Scanner) Kind() string { return "package" } diff --git a/nodejs/coalescer.go b/language/coalescer.go similarity index 55% rename from nodejs/coalescer.go rename to language/coalescer.go index 469cc7446..f99080662 100644 --- a/nodejs/coalescer.go +++ b/language/coalescer.go @@ -1,9 +1,10 @@ -package nodejs +package language import ( "context" - "github.com/quay/claircore" + "slices" + "github.com/quay/claircore" "github.com/quay/claircore/indexer" ) @@ -15,16 +16,22 @@ func NewCoalescer(_ context.Context) (indexer.Coalescer, error) { return &coalescer{}, nil } +type pkgKey struct { + packageDB string + name string +} + func (c *coalescer) Coalesce(_ context.Context, ls []*indexer.LayerArtifacts) (*claircore.IndexReport, error) { ir := &claircore.IndexReport{ Environments: map[string][]*claircore.Environment{}, Packages: map[string]*claircore.Package{}, Repositories: map[string]*claircore.Repository{}, } - - for _, l := range ls { - // If we didn't find at least one npm repo in this layer - // no point in searching for packages. + packages := make(map[pkgKey]*claircore.Package) + // Iterate layers bottom-up so that the topmost (latest) layer wins for duplicate packages. + for _, l := range slices.Backward(ls) { + // If we didn't find at least one repo in this layer, + // there's no point in searching for packages. if len(l.Repos) == 0 { continue } @@ -34,6 +41,17 @@ func (c *coalescer) Coalesce(_ context.Context, ls []*indexer.LayerArtifacts) (* ir.Repositories[r.ID] = r } for _, pkg := range l.Pkgs { + key := pkgKey{packageDB: pkg.PackageDB, name: pkg.Name} + // Delete the previously seen package in favor of the latest. + // If the version is different, they should be considered different packages. + if seen, exists := packages[key]; exists { + if pkg.Version != seen.Version { + continue + } + delete(ir.Packages, seen.ID) + delete(ir.Environments, seen.ID) + } + packages[key] = pkg ir.Packages[pkg.ID] = pkg ir.Environments[pkg.ID] = []*claircore.Environment{ { diff --git a/language/coalescer_test.go b/language/coalescer_test.go new file mode 100644 index 000000000..f83a1c239 --- /dev/null +++ b/language/coalescer_test.go @@ -0,0 +1,210 @@ +package language + +import ( + "strconv" + "testing" + + "github.com/quay/claircore" + "github.com/quay/claircore/indexer" + "github.com/quay/claircore/test" +) + +func TestCoalescer(t *testing.T) { + t.Parallel() + ctx := test.Logging(t) + repoName := "npm" + coalescer := &coalescer{} + pkgs := test.GenUniquePackages(6) + repo := []*claircore.Repository{{ + ID: "1", + Name: repoName, + URI: "https://www.npmjs.com/", + }} + layerArtifacts := []*indexer.LayerArtifacts{ + { + Hash: test.RandomSHA256Digest(t), + Pkgs: pkgs[:1], + }, + { + Hash: test.RandomSHA256Digest(t), + Pkgs: pkgs[:2], + }, + { + Hash: test.RandomSHA256Digest(t), + Pkgs: pkgs[:3], + Repos: repo, + }, + { + Hash: test.RandomSHA256Digest(t), + Pkgs: pkgs[:4], + }, + { + Hash: test.RandomSHA256Digest(t), + Pkgs: pkgs[:5], + Repos: repo, + }, + { + Hash: test.RandomSHA256Digest(t), + Pkgs: pkgs, + }, + } + ir, err := coalescer.Coalesce(ctx, layerArtifacts) + if err != nil { + t.Fatalf("received error from coalesce method: %v", err) + } + // Expect 0-5 to have gotten associated with the repository. + for i := range pkgs[:5] { + es, ok := ir.Environments[strconv.Itoa(i)] + if !ok { + t.Errorf("missing environment for package %d", i) + } + e := es[0] + if len(e.RepositoryIDs) == 0 { + t.Error("expected some repositories") + } + for _, id := range e.RepositoryIDs { + r := ir.Repositories[id] + if got, want := r.Name, repoName; got != want { + t.Errorf("got: %q, want: %q", got, want) + } + } + } + if _, ok := ir.Environments[strconv.Itoa(5)]; ok { + t.Error("expected last package to be excluded (no repo in its layer)") + } +} + +func TestCoalescerPackageOverwrite(t *testing.T) { + t.Parallel() + ctx := test.Logging(t) + coalescer := &coalescer{} + repo := []*claircore.Repository{{ + ID: "1", + Name: "npm", + URI: "https://www.npmjs.com/", + }} + hashes := []claircore.Digest{ + test.RandomSHA256Digest(t), + test.RandomSHA256Digest(t), + test.RandomSHA256Digest(t), + test.RandomSHA256Digest(t), + } + layerArtifacts := []*indexer.LayerArtifacts{ + { + Hash: hashes[0], + Pkgs: []*claircore.Package{ + { + ID: "0", + Name: "semver", + Version: "7.3.8", + PackageDB: "nodejs:usr/local/lib/node_modules/npm/node_modules/semver/package.json", + }, + }, + Repos: repo, + }, + { + Hash: hashes[1], + }, + { + Hash: hashes[2], + Pkgs: []*claircore.Package{ + { + ID: "1", + Name: "semver", + Version: "7.5.2", + PackageDB: "nodejs:usr/local/lib/node_modules/npm/node_modules/semver/package.json", + }, + }, + Repos: repo, + }, + { + Hash: hashes[3], + Pkgs: []*claircore.Package{ + { + ID: "2", + Name: "semver", + Version: "7.5.2", + PackageDB: "nodejs:usr/local/lib/node_modules/npm/node_modules/semver/package.json", + }, + }, + Repos: repo, + }, + } + ir, err := coalescer.Coalesce(ctx, layerArtifacts) + if err != nil { + t.Fatalf("received error from coalesce method: %v", err) + } + if got, want := len(ir.Packages), 1; got != want { + t.Errorf("got %d packages, want %d", got, want) + } + pkg, ok := ir.Packages["1"] + if !ok { + t.Error("expected package 1 to exist") + } + if got, want := pkg.Version, "7.5.2"; got != want { + t.Errorf("got version %s, want %s", got, want) + } + envs, ok := ir.Environments["1"] + if !ok { + t.Error("expected environment for package 1") + } + if got, want := len(envs), 1; got != want { + t.Errorf("got %d environments, want %d", got, want) + } + if got, want := envs[0].IntroducedIn.String(), hashes[2].String(); got != want { + t.Errorf("introduced in %s, want %s", got, want) + } +} + +func TestCoalescerSharedPackageDB(t *testing.T) { + t.Parallel() + ctx := test.Logging(t) + coalescer := &coalescer{} + repo := []*claircore.Repository{{ + ID: "1", + Name: "go", + URI: "https://pkg.go.dev/", + }} + hash := test.RandomSHA256Digest(t) + layerArtifacts := []*indexer.LayerArtifacts{ + { + Hash: hash, + Pkgs: []*claircore.Package{ + { + ID: "0", + Name: "stdlib", + Version: "1.21.0", + PackageDB: "go:/usr/local/bin/app", + }, + { + ID: "1", + Name: "github.com/foo/bar", + Version: "v1.2.3", + PackageDB: "go:/usr/local/bin/app", + }, + { + ID: "2", + Name: "github.com/baz/qux", + Version: "v0.5.0", + PackageDB: "go:/usr/local/bin/app", + }, + }, + Repos: repo, + }, + } + ir, err := coalescer.Coalesce(ctx, layerArtifacts) + if err != nil { + t.Fatalf("received error from coalesce method: %v", err) + } + if got, want := len(ir.Packages), 3; got != want { + t.Errorf("got %d packages, want %d", got, want) + } + for _, id := range []string{"0", "1", "2"} { + if _, ok := ir.Packages[id]; !ok { + t.Errorf("expected package %s", id) + } + if _, ok := ir.Environments[id]; !ok { + t.Errorf("expected environment for package %s", id) + } + } +} diff --git a/nodejs/coalescer_test.go b/nodejs/coalescer_test.go deleted file mode 100644 index d8a55a364..000000000 --- a/nodejs/coalescer_test.go +++ /dev/null @@ -1,68 +0,0 @@ -package nodejs - -import ( - "strconv" - "testing" - - "github.com/quay/claircore" - "github.com/quay/claircore/indexer" - "github.com/quay/claircore/test" -) - -func TestCoalescer(t *testing.T) { - t.Parallel() - ctx := test.Logging(t) - coalescer := &coalescer{} - pkgs := test.GenUniquePackages(6) - repo := []*claircore.Repository{&Repository} - layerArtifacts := []*indexer.LayerArtifacts{ - { - Hash: test.RandomSHA256Digest(t), - Pkgs: pkgs[:1], - }, - { - Hash: test.RandomSHA256Digest(t), - Pkgs: pkgs[:2], - }, - { - Hash: test.RandomSHA256Digest(t), - Pkgs: pkgs[:3], - Repos: repo, - }, - { - Hash: test.RandomSHA256Digest(t), - Pkgs: pkgs[:4], - }, - { - Hash: test.RandomSHA256Digest(t), - Pkgs: pkgs[:5], - Repos: repo, - }, - { - Hash: test.RandomSHA256Digest(t), - Pkgs: pkgs, - }, - } - ir, err := coalescer.Coalesce(ctx, layerArtifacts) - if err != nil { - t.Fatalf("received error from coalesce method: %v", err) - } - // Expect 0-5 to have gotten associated with the repository. - for i := range pkgs { - es, ok := ir.Environments[strconv.Itoa(i)] - if !ok && i == 5 { - // Left out the last package. - continue - } - e := es[0] - if len(e.RepositoryIDs) == 0 { - t.Error("expected some repositories") - } - for _, id := range e.RepositoryIDs { - r := ir.Repositories[id] - if got, want := r.Name, Repository.Name; got != want { - t.Errorf("got: %q, want: %q", got, want) - } - } - } -} diff --git a/nodejs/ecosystem.go b/nodejs/ecosystem.go index de6f84c83..3f958919d 100644 --- a/nodejs/ecosystem.go +++ b/nodejs/ecosystem.go @@ -4,6 +4,7 @@ import ( "context" "github.com/quay/claircore/indexer" + "github.com/quay/claircore/language" ) var scanners = []indexer.PackageScanner{&Scanner{}} @@ -14,6 +15,6 @@ func NewEcosystem(_ context.Context) *indexer.Ecosystem { PackageScanners: func(_ context.Context) ([]indexer.PackageScanner, error) { return scanners, nil }, DistributionScanners: func(_ context.Context) ([]indexer.DistributionScanner, error) { return nil, nil }, RepositoryScanners: func(_ context.Context) ([]indexer.RepositoryScanner, error) { return nil, nil }, - Coalescer: NewCoalescer, + Coalescer: language.NewCoalescer, } } diff --git a/nodejs/packagescanner.go b/nodejs/packagescanner.go index a90396044..9661d0bd6 100644 --- a/nodejs/packagescanner.go +++ b/nodejs/packagescanner.go @@ -46,7 +46,7 @@ type Scanner struct{} func (*Scanner) Name() string { return "nodejs" } // Version implements scanner.VersionedScanner. -func (*Scanner) Version() string { return "3" } +func (*Scanner) Version() string { return "4" } // Kind implements scanner.VersionedScanner. func (*Scanner) Kind() string { return "package" } diff --git a/python/coalescer.go b/python/coalescer.go deleted file mode 100644 index d0d713881..000000000 --- a/python/coalescer.go +++ /dev/null @@ -1,47 +0,0 @@ -package python - -import ( - "context" - - "github.com/quay/claircore" - "github.com/quay/claircore/indexer" -) - -func NewCoalescer(_ context.Context) (indexer.Coalescer, error) { - return &coalescer{}, nil -} - -type coalescer struct { -} - -func (c *coalescer) Coalesce(ctx context.Context, ls []*indexer.LayerArtifacts) (*claircore.IndexReport, error) { - ir := &claircore.IndexReport{ - Environments: map[string][]*claircore.Environment{}, - Packages: map[string]*claircore.Package{}, - Repositories: map[string]*claircore.Repository{}, - } - - for _, l := range ls { - // If we didn't find at least one pip repo in this layer - // no point in searching for packages. - if len(l.Repos) == 0 { - continue - } - rs := make([]string, len(l.Repos)) - for i, r := range l.Repos { - rs[i] = r.ID - ir.Repositories[r.ID] = r - } - for _, pkg := range l.Pkgs { - ir.Packages[pkg.ID] = pkg - ir.Environments[pkg.ID] = []*claircore.Environment{ - &claircore.Environment{ - PackageDB: pkg.PackageDB, - IntroducedIn: l.Hash, - RepositoryIDs: rs, - }, - } - } - } - return ir, nil -} diff --git a/python/ecosystem.go b/python/ecosystem.go index 7458c9edd..d0ead464f 100644 --- a/python/ecosystem.go +++ b/python/ecosystem.go @@ -4,16 +4,17 @@ import ( "context" "github.com/quay/claircore/indexer" + "github.com/quay/claircore/language" ) var scanners = []indexer.PackageScanner{&Scanner{}} // NewEcosystem provides the set of scanners for the python ecosystem. -func NewEcosystem(ctx context.Context) *indexer.Ecosystem { +func NewEcosystem(_ context.Context) *indexer.Ecosystem { return &indexer.Ecosystem{ PackageScanners: func(_ context.Context) ([]indexer.PackageScanner, error) { return scanners, nil }, DistributionScanners: func(_ context.Context) ([]indexer.DistributionScanner, error) { return nil, nil }, RepositoryScanners: func(_ context.Context) ([]indexer.RepositoryScanner, error) { return nil, nil }, - Coalescer: NewCoalescer, + Coalescer: language.NewCoalescer, } } diff --git a/python/packagescanner.go b/python/packagescanner.go index 819d7969a..9abb4a6be 100644 --- a/python/packagescanner.go +++ b/python/packagescanner.go @@ -47,7 +47,7 @@ type Scanner struct{} func (*Scanner) Name() string { return "python" } // Version implements scanner.VersionedScanner. -func (*Scanner) Version() string { return "5" } +func (*Scanner) Version() string { return "6" } // Kind implements scanner.VersionedScanner. func (*Scanner) Kind() string { return "package" } diff --git a/ruby/coalescer.go b/ruby/coalescer.go deleted file mode 100644 index cb0d33ef1..000000000 --- a/ruby/coalescer.go +++ /dev/null @@ -1,46 +0,0 @@ -package ruby - -import ( - "context" - - "github.com/quay/claircore" - "github.com/quay/claircore/indexer" -) - -func NewCoalescer(_ context.Context) (indexer.Coalescer, error) { - return &coalescer{}, nil -} - -type coalescer struct{} - -func (c *coalescer) Coalesce(_ context.Context, ls []*indexer.LayerArtifacts) (*claircore.IndexReport, error) { - ir := &claircore.IndexReport{ - Environments: map[string][]*claircore.Environment{}, - Packages: map[string]*claircore.Package{}, - Repositories: map[string]*claircore.Repository{}, - } - - for _, l := range ls { - // If we didn't find at least one gem repo in this layer - // no point in searching for packages. - if len(l.Repos) == 0 { - continue - } - rs := make([]string, len(l.Repos)) - for i, r := range l.Repos { - rs[i] = r.ID - ir.Repositories[r.ID] = r - } - for _, pkg := range l.Pkgs { - ir.Packages[pkg.ID] = pkg - ir.Environments[pkg.ID] = []*claircore.Environment{ - { - PackageDB: pkg.PackageDB, - IntroducedIn: l.Hash, - RepositoryIDs: rs, - }, - } - } - } - return ir, nil -} diff --git a/ruby/ecosystem.go b/ruby/ecosystem.go index 75f7208cd..985ecdb01 100644 --- a/ruby/ecosystem.go +++ b/ruby/ecosystem.go @@ -4,6 +4,7 @@ import ( "context" "github.com/quay/claircore/indexer" + "github.com/quay/claircore/language" ) var scanners = []indexer.PackageScanner{&Scanner{}} @@ -14,6 +15,6 @@ func NewEcosystem(_ context.Context) *indexer.Ecosystem { PackageScanners: func(_ context.Context) ([]indexer.PackageScanner, error) { return scanners, nil }, DistributionScanners: func(_ context.Context) ([]indexer.DistributionScanner, error) { return nil, nil }, RepositoryScanners: func(_ context.Context) ([]indexer.RepositoryScanner, error) { return nil, nil }, - Coalescer: NewCoalescer, + Coalescer: language.NewCoalescer, } } diff --git a/ruby/packagescanner.go b/ruby/packagescanner.go index 61eea7961..88e056586 100644 --- a/ruby/packagescanner.go +++ b/ruby/packagescanner.go @@ -71,7 +71,7 @@ type Scanner struct{} func (*Scanner) Name() string { return "ruby" } // Version implements scanner.VersionedScanner. -func (*Scanner) Version() string { return "3" } +func (*Scanner) Version() string { return "4" } // Kind implements scanner.VersionedScanner. func (*Scanner) Kind() string { return "package" }