From f392484f43637487e91b8d85987a86fc1899beb4 Mon Sep 17 00:00:00 2001 From: Brad Lugo Date: Thu, 18 Jun 2026 16:21:11 -0700 Subject: [PATCH 1/2] language: consolidate language-specific coalescers All language ecosystem coalescers shared the same logic: skip layers without repos, associate packages with repos. The gobin coalescer had unnecessary repo name/PackageDB prefix filtering since scanners already ensure only relevant packages appear. Signed-off-by: Brad Lugo --- gobin/coalescer.go | 45 ------------------------ gobin/ecosystem.go | 5 +-- gobin/gobin.go | 2 +- java/coalescer.go | 42 ----------------------- java/ecosystem.go | 7 ++-- {nodejs => language}/coalescer.go | 9 +++-- {nodejs => language}/coalescer_test.go | 21 ++++++++---- nodejs/ecosystem.go | 3 +- python/coalescer.go | 47 -------------------------- python/ecosystem.go | 5 +-- ruby/coalescer.go | 46 ------------------------- ruby/ecosystem.go | 3 +- 12 files changed, 32 insertions(+), 203 deletions(-) delete mode 100644 gobin/coalescer.go delete mode 100644 java/coalescer.go rename {nodejs => language}/coalescer.go (88%) rename {nodejs => language}/coalescer_test.go (74%) delete mode 100644 python/coalescer.go delete mode 100644 ruby/coalescer.go diff --git a/gobin/coalescer.go b/gobin/coalescer.go deleted file mode 100644 index 5686ec38e..000000000 --- a/gobin/coalescer.go +++ /dev/null @@ -1,45 +0,0 @@ -package gobin - -import ( - "context" - "strings" - - "github.com/quay/claircore" - "github.com/quay/claircore/indexer" -) - -type coalescer struct{} - -func (c *coalescer) Coalesce(ctx context.Context, ls []*indexer.LayerArtifacts) (*claircore.IndexReport, error) { - ir := &claircore.IndexReport{ - Environments: map[string][]*claircore.Environment{}, - Packages: map[string]*claircore.Package{}, - Repositories: map[string]*claircore.Repository{}, - } - for _, l := range ls { - var rid string - for _, r := range l.Repos { - // Magic strings copied out of the osv package. - if r.Name != `go` || r.URI != `https://pkg.go.dev/` { - continue - } - rid = r.ID - ir.Repositories[r.ID] = r - break - } - for _, pkg := range l.Pkgs { - if !strings.HasPrefix(pkg.PackageDB, "go:") { - continue - } - ir.Packages[pkg.ID] = pkg - ir.Environments[pkg.ID] = []*claircore.Environment{ - { - PackageDB: pkg.PackageDB, - IntroducedIn: l.Hash, - RepositoryIDs: []string{rid}, - }, - } - } - } - return ir, nil -} diff --git a/gobin/ecosystem.go b/gobin/ecosystem.go index 294728ad2..1ded67dca 100644 --- a/gobin/ecosystem.go +++ b/gobin/ecosystem.go @@ -4,10 +4,11 @@ import ( "context" "github.com/quay/claircore/indexer" + "github.com/quay/claircore/language" ) // NewEcosystem provides the ecosystem for handling go binaries. -func NewEcosystem(ctx context.Context) *indexer.Ecosystem { +func NewEcosystem(_ context.Context) *indexer.Ecosystem { return &indexer.Ecosystem{ Name: "gobin", PackageScanners: func(context.Context) ([]indexer.PackageScanner, error) { @@ -15,6 +16,6 @@ func NewEcosystem(ctx context.Context) *indexer.Ecosystem { }, DistributionScanners: func(context.Context) ([]indexer.DistributionScanner, error) { return nil, nil }, RepositoryScanners: func(context.Context) ([]indexer.RepositoryScanner, error) { return nil, nil }, - Coalescer: func(context.Context) (indexer.Coalescer, error) { return &coalescer{}, nil }, + Coalescer: language.NewCoalescer, } } diff --git a/gobin/gobin.go b/gobin/gobin.go index 763a35fe4..b8a80453a 100644 --- a/gobin/gobin.go +++ b/gobin/gobin.go @@ -33,7 +33,7 @@ type Detector struct{} const ( detectorName = `gobin` - detectorVersion = `7` + detectorVersion = `8` detectorKind = `package` ) diff --git a/java/coalescer.go b/java/coalescer.go deleted file mode 100644 index 688f86280..000000000 --- a/java/coalescer.go +++ /dev/null @@ -1,42 +0,0 @@ -package java - -import ( - "context" - - "github.com/quay/claircore" - "github.com/quay/claircore/indexer" -) - -type coalescer struct{} - -func (*coalescer) Coalesce(ctx context.Context, ls []*indexer.LayerArtifacts) (*claircore.IndexReport, error) { - ir := &claircore.IndexReport{ - Environments: map[string][]*claircore.Environment{}, - Packages: map[string]*claircore.Package{}, - Repositories: map[string]*claircore.Repository{}, - } - - for _, l := range ls { - // If we didn't find at least one maven repo in this layer - // no point in searching for packages. - if len(l.Repos) == 0 { - continue - } - rs := make([]string, len(l.Repos)) - for i, r := range l.Repos { - rs[i] = r.ID - ir.Repositories[r.ID] = r - } - for _, pkg := range l.Pkgs { - ir.Packages[pkg.ID] = pkg - ir.Environments[pkg.ID] = []*claircore.Environment{ - { - PackageDB: pkg.PackageDB, - IntroducedIn: l.Hash, - RepositoryIDs: rs, - }, - } - } - } - return ir, nil -} diff --git a/java/ecosystem.go b/java/ecosystem.go index 1cca07347..894ab920c 100644 --- a/java/ecosystem.go +++ b/java/ecosystem.go @@ -4,10 +4,11 @@ import ( "context" "github.com/quay/claircore/indexer" + "github.com/quay/claircore/language" ) // NewEcosystem provides the set of scanners for the java ecosystem. -func NewEcosystem(ctx context.Context) *indexer.Ecosystem { +func NewEcosystem(_ context.Context) *indexer.Ecosystem { return &indexer.Ecosystem{ PackageScanners: func(_ context.Context) ([]indexer.PackageScanner, error) { return []indexer.PackageScanner{&Scanner{}}, nil @@ -16,8 +17,6 @@ func NewEcosystem(ctx context.Context) *indexer.Ecosystem { RepositoryScanners: func(_ context.Context) ([]indexer.RepositoryScanner, error) { return nil, nil }, - Coalescer: func(_ context.Context) (indexer.Coalescer, error) { - return (*coalescer)(nil), nil - }, + Coalescer: language.NewCoalescer, } } diff --git a/nodejs/coalescer.go b/language/coalescer.go similarity index 88% rename from nodejs/coalescer.go rename to language/coalescer.go index 469cc7446..552610124 100644 --- a/nodejs/coalescer.go +++ b/language/coalescer.go @@ -1,9 +1,9 @@ -package nodejs +package language import ( "context" - "github.com/quay/claircore" + "github.com/quay/claircore" "github.com/quay/claircore/indexer" ) @@ -21,10 +21,9 @@ func (c *coalescer) Coalesce(_ context.Context, ls []*indexer.LayerArtifacts) (* Packages: map[string]*claircore.Package{}, Repositories: map[string]*claircore.Repository{}, } - for _, l := range ls { - // If we didn't find at least one npm repo in this layer - // no point in searching for packages. + // If we didn't find at least one repo in this layer, + // there's no point in searching for packages. if len(l.Repos) == 0 { continue } diff --git a/nodejs/coalescer_test.go b/language/coalescer_test.go similarity index 74% rename from nodejs/coalescer_test.go rename to language/coalescer_test.go index d8a55a364..9b17bc9c1 100644 --- a/nodejs/coalescer_test.go +++ b/language/coalescer_test.go @@ -1,4 +1,4 @@ -package nodejs +package language import ( "strconv" @@ -12,9 +12,14 @@ import ( func TestCoalescer(t *testing.T) { t.Parallel() ctx := test.Logging(t) + repoName := "npm" coalescer := &coalescer{} pkgs := test.GenUniquePackages(6) - repo := []*claircore.Repository{&Repository} + repo := []*claircore.Repository{{ + ID: "1", + Name: repoName, + URI: "https://www.npmjs.com/", + }} layerArtifacts := []*indexer.LayerArtifacts{ { Hash: test.RandomSHA256Digest(t), @@ -48,11 +53,10 @@ func TestCoalescer(t *testing.T) { t.Fatalf("received error from coalesce method: %v", err) } // Expect 0-5 to have gotten associated with the repository. - for i := range pkgs { + for i := range pkgs[:5] { es, ok := ir.Environments[strconv.Itoa(i)] - if !ok && i == 5 { - // Left out the last package. - continue + if !ok { + t.Errorf("missing environment for package %d", i) } e := es[0] if len(e.RepositoryIDs) == 0 { @@ -60,9 +64,12 @@ func TestCoalescer(t *testing.T) { } for _, id := range e.RepositoryIDs { r := ir.Repositories[id] - if got, want := r.Name, Repository.Name; got != want { + if got, want := r.Name, repoName; got != want { t.Errorf("got: %q, want: %q", got, want) } } } + if _, ok := ir.Environments[strconv.Itoa(5)]; ok { + t.Error("expected last package to be excluded (no repo in its layer)") + } } diff --git a/nodejs/ecosystem.go b/nodejs/ecosystem.go index de6f84c83..3f958919d 100644 --- a/nodejs/ecosystem.go +++ b/nodejs/ecosystem.go @@ -4,6 +4,7 @@ import ( "context" "github.com/quay/claircore/indexer" + "github.com/quay/claircore/language" ) var scanners = []indexer.PackageScanner{&Scanner{}} @@ -14,6 +15,6 @@ func NewEcosystem(_ context.Context) *indexer.Ecosystem { PackageScanners: func(_ context.Context) ([]indexer.PackageScanner, error) { return scanners, nil }, DistributionScanners: func(_ context.Context) ([]indexer.DistributionScanner, error) { return nil, nil }, RepositoryScanners: func(_ context.Context) ([]indexer.RepositoryScanner, error) { return nil, nil }, - Coalescer: NewCoalescer, + Coalescer: language.NewCoalescer, } } diff --git a/python/coalescer.go b/python/coalescer.go deleted file mode 100644 index d0d713881..000000000 --- a/python/coalescer.go +++ /dev/null @@ -1,47 +0,0 @@ -package python - -import ( - "context" - - "github.com/quay/claircore" - "github.com/quay/claircore/indexer" -) - -func NewCoalescer(_ context.Context) (indexer.Coalescer, error) { - return &coalescer{}, nil -} - -type coalescer struct { -} - -func (c *coalescer) Coalesce(ctx context.Context, ls []*indexer.LayerArtifacts) (*claircore.IndexReport, error) { - ir := &claircore.IndexReport{ - Environments: map[string][]*claircore.Environment{}, - Packages: map[string]*claircore.Package{}, - Repositories: map[string]*claircore.Repository{}, - } - - for _, l := range ls { - // If we didn't find at least one pip repo in this layer - // no point in searching for packages. - if len(l.Repos) == 0 { - continue - } - rs := make([]string, len(l.Repos)) - for i, r := range l.Repos { - rs[i] = r.ID - ir.Repositories[r.ID] = r - } - for _, pkg := range l.Pkgs { - ir.Packages[pkg.ID] = pkg - ir.Environments[pkg.ID] = []*claircore.Environment{ - &claircore.Environment{ - PackageDB: pkg.PackageDB, - IntroducedIn: l.Hash, - RepositoryIDs: rs, - }, - } - } - } - return ir, nil -} diff --git a/python/ecosystem.go b/python/ecosystem.go index 7458c9edd..d0ead464f 100644 --- a/python/ecosystem.go +++ b/python/ecosystem.go @@ -4,16 +4,17 @@ import ( "context" "github.com/quay/claircore/indexer" + "github.com/quay/claircore/language" ) var scanners = []indexer.PackageScanner{&Scanner{}} // NewEcosystem provides the set of scanners for the python ecosystem. -func NewEcosystem(ctx context.Context) *indexer.Ecosystem { +func NewEcosystem(_ context.Context) *indexer.Ecosystem { return &indexer.Ecosystem{ PackageScanners: func(_ context.Context) ([]indexer.PackageScanner, error) { return scanners, nil }, DistributionScanners: func(_ context.Context) ([]indexer.DistributionScanner, error) { return nil, nil }, RepositoryScanners: func(_ context.Context) ([]indexer.RepositoryScanner, error) { return nil, nil }, - Coalescer: NewCoalescer, + Coalescer: language.NewCoalescer, } } diff --git a/ruby/coalescer.go b/ruby/coalescer.go deleted file mode 100644 index cb0d33ef1..000000000 --- a/ruby/coalescer.go +++ /dev/null @@ -1,46 +0,0 @@ -package ruby - -import ( - "context" - - "github.com/quay/claircore" - "github.com/quay/claircore/indexer" -) - -func NewCoalescer(_ context.Context) (indexer.Coalescer, error) { - return &coalescer{}, nil -} - -type coalescer struct{} - -func (c *coalescer) Coalesce(_ context.Context, ls []*indexer.LayerArtifacts) (*claircore.IndexReport, error) { - ir := &claircore.IndexReport{ - Environments: map[string][]*claircore.Environment{}, - Packages: map[string]*claircore.Package{}, - Repositories: map[string]*claircore.Repository{}, - } - - for _, l := range ls { - // If we didn't find at least one gem repo in this layer - // no point in searching for packages. - if len(l.Repos) == 0 { - continue - } - rs := make([]string, len(l.Repos)) - for i, r := range l.Repos { - rs[i] = r.ID - ir.Repositories[r.ID] = r - } - for _, pkg := range l.Pkgs { - ir.Packages[pkg.ID] = pkg - ir.Environments[pkg.ID] = []*claircore.Environment{ - { - PackageDB: pkg.PackageDB, - IntroducedIn: l.Hash, - RepositoryIDs: rs, - }, - } - } - } - return ir, nil -} diff --git a/ruby/ecosystem.go b/ruby/ecosystem.go index 75f7208cd..985ecdb01 100644 --- a/ruby/ecosystem.go +++ b/ruby/ecosystem.go @@ -4,6 +4,7 @@ import ( "context" "github.com/quay/claircore/indexer" + "github.com/quay/claircore/language" ) var scanners = []indexer.PackageScanner{&Scanner{}} @@ -14,6 +15,6 @@ func NewEcosystem(_ context.Context) *indexer.Ecosystem { PackageScanners: func(_ context.Context) ([]indexer.PackageScanner, error) { return scanners, nil }, DistributionScanners: func(_ context.Context) ([]indexer.DistributionScanner, error) { return nil, nil }, RepositoryScanners: func(_ context.Context) ([]indexer.RepositoryScanner, error) { return nil, nil }, - Coalescer: NewCoalescer, + Coalescer: language.NewCoalescer, } } From 804f10ab2ad1d890a314ad15a851c5b2e5eea039 Mon Sep 17 00:00:00 2001 From: Brad Lugo Date: Thu, 2 Jul 2026 11:38:53 -0700 Subject: [PATCH 2/2] language: deduplicate packages across layers When the same package appears at the same path across multiple layers, keep only the latest version. Duplicate entries at the same version are collapsed to the topmost layer. Signed-off-by: Brad Lugo --- gobin/gobin.go | 2 +- java/packagescanner.go | 2 +- language/coalescer.go | 21 +++++- language/coalescer_test.go | 135 +++++++++++++++++++++++++++++++++++++ nodejs/packagescanner.go | 2 +- python/packagescanner.go | 2 +- ruby/packagescanner.go | 2 +- 7 files changed, 160 insertions(+), 6 deletions(-) diff --git a/gobin/gobin.go b/gobin/gobin.go index b8a80453a..aa3a9236c 100644 --- a/gobin/gobin.go +++ b/gobin/gobin.go @@ -33,7 +33,7 @@ type Detector struct{} const ( detectorName = `gobin` - detectorVersion = `8` + detectorVersion = `9` detectorKind = `package` ) diff --git a/java/packagescanner.go b/java/packagescanner.go index af1356cff..daf0b5b27 100644 --- a/java/packagescanner.go +++ b/java/packagescanner.go @@ -75,7 +75,7 @@ type Scanner struct { func (*Scanner) Name() string { return "java" } // Version implements scanner.VersionedScanner. -func (*Scanner) Version() string { return "8" } +func (*Scanner) Version() string { return "9" } // Kind implements scanner.VersionedScanner. func (*Scanner) Kind() string { return "package" } diff --git a/language/coalescer.go b/language/coalescer.go index 552610124..f99080662 100644 --- a/language/coalescer.go +++ b/language/coalescer.go @@ -2,6 +2,7 @@ package language import ( "context" + "slices" "github.com/quay/claircore" "github.com/quay/claircore/indexer" @@ -15,13 +16,20 @@ func NewCoalescer(_ context.Context) (indexer.Coalescer, error) { return &coalescer{}, nil } +type pkgKey struct { + packageDB string + name string +} + func (c *coalescer) Coalesce(_ context.Context, ls []*indexer.LayerArtifacts) (*claircore.IndexReport, error) { ir := &claircore.IndexReport{ Environments: map[string][]*claircore.Environment{}, Packages: map[string]*claircore.Package{}, Repositories: map[string]*claircore.Repository{}, } - for _, l := range ls { + packages := make(map[pkgKey]*claircore.Package) + // Iterate layers bottom-up so that the topmost (latest) layer wins for duplicate packages. + for _, l := range slices.Backward(ls) { // If we didn't find at least one repo in this layer, // there's no point in searching for packages. if len(l.Repos) == 0 { @@ -33,6 +41,17 @@ func (c *coalescer) Coalesce(_ context.Context, ls []*indexer.LayerArtifacts) (* ir.Repositories[r.ID] = r } for _, pkg := range l.Pkgs { + key := pkgKey{packageDB: pkg.PackageDB, name: pkg.Name} + // Delete the previously seen package in favor of the latest. + // If the version is different, they should be considered different packages. + if seen, exists := packages[key]; exists { + if pkg.Version != seen.Version { + continue + } + delete(ir.Packages, seen.ID) + delete(ir.Environments, seen.ID) + } + packages[key] = pkg ir.Packages[pkg.ID] = pkg ir.Environments[pkg.ID] = []*claircore.Environment{ { diff --git a/language/coalescer_test.go b/language/coalescer_test.go index 9b17bc9c1..f83a1c239 100644 --- a/language/coalescer_test.go +++ b/language/coalescer_test.go @@ -73,3 +73,138 @@ func TestCoalescer(t *testing.T) { t.Error("expected last package to be excluded (no repo in its layer)") } } + +func TestCoalescerPackageOverwrite(t *testing.T) { + t.Parallel() + ctx := test.Logging(t) + coalescer := &coalescer{} + repo := []*claircore.Repository{{ + ID: "1", + Name: "npm", + URI: "https://www.npmjs.com/", + }} + hashes := []claircore.Digest{ + test.RandomSHA256Digest(t), + test.RandomSHA256Digest(t), + test.RandomSHA256Digest(t), + test.RandomSHA256Digest(t), + } + layerArtifacts := []*indexer.LayerArtifacts{ + { + Hash: hashes[0], + Pkgs: []*claircore.Package{ + { + ID: "0", + Name: "semver", + Version: "7.3.8", + PackageDB: "nodejs:usr/local/lib/node_modules/npm/node_modules/semver/package.json", + }, + }, + Repos: repo, + }, + { + Hash: hashes[1], + }, + { + Hash: hashes[2], + Pkgs: []*claircore.Package{ + { + ID: "1", + Name: "semver", + Version: "7.5.2", + PackageDB: "nodejs:usr/local/lib/node_modules/npm/node_modules/semver/package.json", + }, + }, + Repos: repo, + }, + { + Hash: hashes[3], + Pkgs: []*claircore.Package{ + { + ID: "2", + Name: "semver", + Version: "7.5.2", + PackageDB: "nodejs:usr/local/lib/node_modules/npm/node_modules/semver/package.json", + }, + }, + Repos: repo, + }, + } + ir, err := coalescer.Coalesce(ctx, layerArtifacts) + if err != nil { + t.Fatalf("received error from coalesce method: %v", err) + } + if got, want := len(ir.Packages), 1; got != want { + t.Errorf("got %d packages, want %d", got, want) + } + pkg, ok := ir.Packages["1"] + if !ok { + t.Error("expected package 1 to exist") + } + if got, want := pkg.Version, "7.5.2"; got != want { + t.Errorf("got version %s, want %s", got, want) + } + envs, ok := ir.Environments["1"] + if !ok { + t.Error("expected environment for package 1") + } + if got, want := len(envs), 1; got != want { + t.Errorf("got %d environments, want %d", got, want) + } + if got, want := envs[0].IntroducedIn.String(), hashes[2].String(); got != want { + t.Errorf("introduced in %s, want %s", got, want) + } +} + +func TestCoalescerSharedPackageDB(t *testing.T) { + t.Parallel() + ctx := test.Logging(t) + coalescer := &coalescer{} + repo := []*claircore.Repository{{ + ID: "1", + Name: "go", + URI: "https://pkg.go.dev/", + }} + hash := test.RandomSHA256Digest(t) + layerArtifacts := []*indexer.LayerArtifacts{ + { + Hash: hash, + Pkgs: []*claircore.Package{ + { + ID: "0", + Name: "stdlib", + Version: "1.21.0", + PackageDB: "go:/usr/local/bin/app", + }, + { + ID: "1", + Name: "github.com/foo/bar", + Version: "v1.2.3", + PackageDB: "go:/usr/local/bin/app", + }, + { + ID: "2", + Name: "github.com/baz/qux", + Version: "v0.5.0", + PackageDB: "go:/usr/local/bin/app", + }, + }, + Repos: repo, + }, + } + ir, err := coalescer.Coalesce(ctx, layerArtifacts) + if err != nil { + t.Fatalf("received error from coalesce method: %v", err) + } + if got, want := len(ir.Packages), 3; got != want { + t.Errorf("got %d packages, want %d", got, want) + } + for _, id := range []string{"0", "1", "2"} { + if _, ok := ir.Packages[id]; !ok { + t.Errorf("expected package %s", id) + } + if _, ok := ir.Environments[id]; !ok { + t.Errorf("expected environment for package %s", id) + } + } +} diff --git a/nodejs/packagescanner.go b/nodejs/packagescanner.go index a90396044..9661d0bd6 100644 --- a/nodejs/packagescanner.go +++ b/nodejs/packagescanner.go @@ -46,7 +46,7 @@ type Scanner struct{} func (*Scanner) Name() string { return "nodejs" } // Version implements scanner.VersionedScanner. -func (*Scanner) Version() string { return "3" } +func (*Scanner) Version() string { return "4" } // Kind implements scanner.VersionedScanner. func (*Scanner) Kind() string { return "package" } diff --git a/python/packagescanner.go b/python/packagescanner.go index 819d7969a..9abb4a6be 100644 --- a/python/packagescanner.go +++ b/python/packagescanner.go @@ -47,7 +47,7 @@ type Scanner struct{} func (*Scanner) Name() string { return "python" } // Version implements scanner.VersionedScanner. -func (*Scanner) Version() string { return "5" } +func (*Scanner) Version() string { return "6" } // Kind implements scanner.VersionedScanner. func (*Scanner) Kind() string { return "package" } diff --git a/ruby/packagescanner.go b/ruby/packagescanner.go index 61eea7961..88e056586 100644 --- a/ruby/packagescanner.go +++ b/ruby/packagescanner.go @@ -71,7 +71,7 @@ type Scanner struct{} func (*Scanner) Name() string { return "ruby" } // Version implements scanner.VersionedScanner. -func (*Scanner) Version() string { return "3" } +func (*Scanner) Version() string { return "4" } // Kind implements scanner.VersionedScanner. func (*Scanner) Kind() string { return "package" }