From f392484f43637487e91b8d85987a86fc1899beb4 Mon Sep 17 00:00:00 2001 From: Brad Lugo Date: Thu, 18 Jun 2026 16:21:11 -0700 Subject: [PATCH] language: consolidate language-specific coalescers All language ecosystem coalescers shared the same logic: skip layers without repos, associate packages with repos. The gobin coalescer had unnecessary repo name/PackageDB prefix filtering since scanners already ensure only relevant packages appear. Signed-off-by: Brad Lugo --- gobin/coalescer.go | 45 ------------------------ gobin/ecosystem.go | 5 +-- gobin/gobin.go | 2 +- java/coalescer.go | 42 ----------------------- java/ecosystem.go | 7 ++-- {nodejs => language}/coalescer.go | 9 +++-- {nodejs => language}/coalescer_test.go | 21 ++++++++---- nodejs/ecosystem.go | 3 +- python/coalescer.go | 47 -------------------------- python/ecosystem.go | 5 +-- ruby/coalescer.go | 46 ------------------------- ruby/ecosystem.go | 3 +- 12 files changed, 32 insertions(+), 203 deletions(-) delete mode 100644 gobin/coalescer.go delete mode 100644 java/coalescer.go rename {nodejs => language}/coalescer.go (88%) rename {nodejs => language}/coalescer_test.go (74%) delete mode 100644 python/coalescer.go delete mode 100644 ruby/coalescer.go diff --git a/gobin/coalescer.go b/gobin/coalescer.go deleted file mode 100644 index 5686ec38e..000000000 --- a/gobin/coalescer.go +++ /dev/null @@ -1,45 +0,0 @@ -package gobin - -import ( - "context" - "strings" - - "github.com/quay/claircore" - "github.com/quay/claircore/indexer" -) - -type coalescer struct{} - -func (c *coalescer) Coalesce(ctx context.Context, ls []*indexer.LayerArtifacts) (*claircore.IndexReport, error) { - ir := &claircore.IndexReport{ - Environments: map[string][]*claircore.Environment{}, - Packages: map[string]*claircore.Package{}, - Repositories: map[string]*claircore.Repository{}, - } - for _, l := range ls { - var rid string - for _, r := range l.Repos { - // Magic strings copied out of the osv package. - if r.Name != `go` || r.URI != `https://pkg.go.dev/` { - continue - } - rid = r.ID - ir.Repositories[r.ID] = r - break - } - for _, pkg := range l.Pkgs { - if !strings.HasPrefix(pkg.PackageDB, "go:") { - continue - } - ir.Packages[pkg.ID] = pkg - ir.Environments[pkg.ID] = []*claircore.Environment{ - { - PackageDB: pkg.PackageDB, - IntroducedIn: l.Hash, - RepositoryIDs: []string{rid}, - }, - } - } - } - return ir, nil -} diff --git a/gobin/ecosystem.go b/gobin/ecosystem.go index 294728ad2..1ded67dca 100644 --- a/gobin/ecosystem.go +++ b/gobin/ecosystem.go @@ -4,10 +4,11 @@ import ( "context" "github.com/quay/claircore/indexer" + "github.com/quay/claircore/language" ) // NewEcosystem provides the ecosystem for handling go binaries. -func NewEcosystem(ctx context.Context) *indexer.Ecosystem { +func NewEcosystem(_ context.Context) *indexer.Ecosystem { return &indexer.Ecosystem{ Name: "gobin", PackageScanners: func(context.Context) ([]indexer.PackageScanner, error) { @@ -15,6 +16,6 @@ func NewEcosystem(ctx context.Context) *indexer.Ecosystem { }, DistributionScanners: func(context.Context) ([]indexer.DistributionScanner, error) { return nil, nil }, RepositoryScanners: func(context.Context) ([]indexer.RepositoryScanner, error) { return nil, nil }, - Coalescer: func(context.Context) (indexer.Coalescer, error) { return &coalescer{}, nil }, + Coalescer: language.NewCoalescer, } } diff --git a/gobin/gobin.go b/gobin/gobin.go index 763a35fe4..b8a80453a 100644 --- a/gobin/gobin.go +++ b/gobin/gobin.go @@ -33,7 +33,7 @@ type Detector struct{} const ( detectorName = `gobin` - detectorVersion = `7` + detectorVersion = `8` detectorKind = `package` ) diff --git a/java/coalescer.go b/java/coalescer.go deleted file mode 100644 index 688f86280..000000000 --- a/java/coalescer.go +++ /dev/null @@ -1,42 +0,0 @@ -package java - -import ( - "context" - - "github.com/quay/claircore" - "github.com/quay/claircore/indexer" -) - -type coalescer struct{} - -func (*coalescer) Coalesce(ctx context.Context, ls []*indexer.LayerArtifacts) (*claircore.IndexReport, error) { - ir := &claircore.IndexReport{ - Environments: map[string][]*claircore.Environment{}, - Packages: map[string]*claircore.Package{}, - Repositories: map[string]*claircore.Repository{}, - } - - for _, l := range ls { - // If we didn't find at least one maven repo in this layer - // no point in searching for packages. - if len(l.Repos) == 0 { - continue - } - rs := make([]string, len(l.Repos)) - for i, r := range l.Repos { - rs[i] = r.ID - ir.Repositories[r.ID] = r - } - for _, pkg := range l.Pkgs { - ir.Packages[pkg.ID] = pkg - ir.Environments[pkg.ID] = []*claircore.Environment{ - { - PackageDB: pkg.PackageDB, - IntroducedIn: l.Hash, - RepositoryIDs: rs, - }, - } - } - } - return ir, nil -} diff --git a/java/ecosystem.go b/java/ecosystem.go index 1cca07347..894ab920c 100644 --- a/java/ecosystem.go +++ b/java/ecosystem.go @@ -4,10 +4,11 @@ import ( "context" "github.com/quay/claircore/indexer" + "github.com/quay/claircore/language" ) // NewEcosystem provides the set of scanners for the java ecosystem. -func NewEcosystem(ctx context.Context) *indexer.Ecosystem { +func NewEcosystem(_ context.Context) *indexer.Ecosystem { return &indexer.Ecosystem{ PackageScanners: func(_ context.Context) ([]indexer.PackageScanner, error) { return []indexer.PackageScanner{&Scanner{}}, nil @@ -16,8 +17,6 @@ func NewEcosystem(ctx context.Context) *indexer.Ecosystem { RepositoryScanners: func(_ context.Context) ([]indexer.RepositoryScanner, error) { return nil, nil }, - Coalescer: func(_ context.Context) (indexer.Coalescer, error) { - return (*coalescer)(nil), nil - }, + Coalescer: language.NewCoalescer, } } diff --git a/nodejs/coalescer.go b/language/coalescer.go similarity index 88% rename from nodejs/coalescer.go rename to language/coalescer.go index 469cc7446..552610124 100644 --- a/nodejs/coalescer.go +++ b/language/coalescer.go @@ -1,9 +1,9 @@ -package nodejs +package language import ( "context" - "github.com/quay/claircore" + "github.com/quay/claircore" "github.com/quay/claircore/indexer" ) @@ -21,10 +21,9 @@ func (c *coalescer) Coalesce(_ context.Context, ls []*indexer.LayerArtifacts) (* Packages: map[string]*claircore.Package{}, Repositories: map[string]*claircore.Repository{}, } - for _, l := range ls { - // If we didn't find at least one npm repo in this layer - // no point in searching for packages. + // If we didn't find at least one repo in this layer, + // there's no point in searching for packages. if len(l.Repos) == 0 { continue } diff --git a/nodejs/coalescer_test.go b/language/coalescer_test.go similarity index 74% rename from nodejs/coalescer_test.go rename to language/coalescer_test.go index d8a55a364..9b17bc9c1 100644 --- a/nodejs/coalescer_test.go +++ b/language/coalescer_test.go @@ -1,4 +1,4 @@ -package nodejs +package language import ( "strconv" @@ -12,9 +12,14 @@ import ( func TestCoalescer(t *testing.T) { t.Parallel() ctx := test.Logging(t) + repoName := "npm" coalescer := &coalescer{} pkgs := test.GenUniquePackages(6) - repo := []*claircore.Repository{&Repository} + repo := []*claircore.Repository{{ + ID: "1", + Name: repoName, + URI: "https://www.npmjs.com/", + }} layerArtifacts := []*indexer.LayerArtifacts{ { Hash: test.RandomSHA256Digest(t), @@ -48,11 +53,10 @@ func TestCoalescer(t *testing.T) { t.Fatalf("received error from coalesce method: %v", err) } // Expect 0-5 to have gotten associated with the repository. - for i := range pkgs { + for i := range pkgs[:5] { es, ok := ir.Environments[strconv.Itoa(i)] - if !ok && i == 5 { - // Left out the last package. - continue + if !ok { + t.Errorf("missing environment for package %d", i) } e := es[0] if len(e.RepositoryIDs) == 0 { @@ -60,9 +64,12 @@ func TestCoalescer(t *testing.T) { } for _, id := range e.RepositoryIDs { r := ir.Repositories[id] - if got, want := r.Name, Repository.Name; got != want { + if got, want := r.Name, repoName; got != want { t.Errorf("got: %q, want: %q", got, want) } } } + if _, ok := ir.Environments[strconv.Itoa(5)]; ok { + t.Error("expected last package to be excluded (no repo in its layer)") + } } diff --git a/nodejs/ecosystem.go b/nodejs/ecosystem.go index de6f84c83..3f958919d 100644 --- a/nodejs/ecosystem.go +++ b/nodejs/ecosystem.go @@ -4,6 +4,7 @@ import ( "context" "github.com/quay/claircore/indexer" + "github.com/quay/claircore/language" ) var scanners = []indexer.PackageScanner{&Scanner{}} @@ -14,6 +15,6 @@ func NewEcosystem(_ context.Context) *indexer.Ecosystem { PackageScanners: func(_ context.Context) ([]indexer.PackageScanner, error) { return scanners, nil }, DistributionScanners: func(_ context.Context) ([]indexer.DistributionScanner, error) { return nil, nil }, RepositoryScanners: func(_ context.Context) ([]indexer.RepositoryScanner, error) { return nil, nil }, - Coalescer: NewCoalescer, + Coalescer: language.NewCoalescer, } } diff --git a/python/coalescer.go b/python/coalescer.go deleted file mode 100644 index d0d713881..000000000 --- a/python/coalescer.go +++ /dev/null @@ -1,47 +0,0 @@ -package python - -import ( - "context" - - "github.com/quay/claircore" - "github.com/quay/claircore/indexer" -) - -func NewCoalescer(_ context.Context) (indexer.Coalescer, error) { - return &coalescer{}, nil -} - -type coalescer struct { -} - -func (c *coalescer) Coalesce(ctx context.Context, ls []*indexer.LayerArtifacts) (*claircore.IndexReport, error) { - ir := &claircore.IndexReport{ - Environments: map[string][]*claircore.Environment{}, - Packages: map[string]*claircore.Package{}, - Repositories: map[string]*claircore.Repository{}, - } - - for _, l := range ls { - // If we didn't find at least one pip repo in this layer - // no point in searching for packages. - if len(l.Repos) == 0 { - continue - } - rs := make([]string, len(l.Repos)) - for i, r := range l.Repos { - rs[i] = r.ID - ir.Repositories[r.ID] = r - } - for _, pkg := range l.Pkgs { - ir.Packages[pkg.ID] = pkg - ir.Environments[pkg.ID] = []*claircore.Environment{ - &claircore.Environment{ - PackageDB: pkg.PackageDB, - IntroducedIn: l.Hash, - RepositoryIDs: rs, - }, - } - } - } - return ir, nil -} diff --git a/python/ecosystem.go b/python/ecosystem.go index 7458c9edd..d0ead464f 100644 --- a/python/ecosystem.go +++ b/python/ecosystem.go @@ -4,16 +4,17 @@ import ( "context" "github.com/quay/claircore/indexer" + "github.com/quay/claircore/language" ) var scanners = []indexer.PackageScanner{&Scanner{}} // NewEcosystem provides the set of scanners for the python ecosystem. -func NewEcosystem(ctx context.Context) *indexer.Ecosystem { +func NewEcosystem(_ context.Context) *indexer.Ecosystem { return &indexer.Ecosystem{ PackageScanners: func(_ context.Context) ([]indexer.PackageScanner, error) { return scanners, nil }, DistributionScanners: func(_ context.Context) ([]indexer.DistributionScanner, error) { return nil, nil }, RepositoryScanners: func(_ context.Context) ([]indexer.RepositoryScanner, error) { return nil, nil }, - Coalescer: NewCoalescer, + Coalescer: language.NewCoalescer, } } diff --git a/ruby/coalescer.go b/ruby/coalescer.go deleted file mode 100644 index cb0d33ef1..000000000 --- a/ruby/coalescer.go +++ /dev/null @@ -1,46 +0,0 @@ -package ruby - -import ( - "context" - - "github.com/quay/claircore" - "github.com/quay/claircore/indexer" -) - -func NewCoalescer(_ context.Context) (indexer.Coalescer, error) { - return &coalescer{}, nil -} - -type coalescer struct{} - -func (c *coalescer) Coalesce(_ context.Context, ls []*indexer.LayerArtifacts) (*claircore.IndexReport, error) { - ir := &claircore.IndexReport{ - Environments: map[string][]*claircore.Environment{}, - Packages: map[string]*claircore.Package{}, - Repositories: map[string]*claircore.Repository{}, - } - - for _, l := range ls { - // If we didn't find at least one gem repo in this layer - // no point in searching for packages. - if len(l.Repos) == 0 { - continue - } - rs := make([]string, len(l.Repos)) - for i, r := range l.Repos { - rs[i] = r.ID - ir.Repositories[r.ID] = r - } - for _, pkg := range l.Pkgs { - ir.Packages[pkg.ID] = pkg - ir.Environments[pkg.ID] = []*claircore.Environment{ - { - PackageDB: pkg.PackageDB, - IntroducedIn: l.Hash, - RepositoryIDs: rs, - }, - } - } - } - return ir, nil -} diff --git a/ruby/ecosystem.go b/ruby/ecosystem.go index 75f7208cd..985ecdb01 100644 --- a/ruby/ecosystem.go +++ b/ruby/ecosystem.go @@ -4,6 +4,7 @@ import ( "context" "github.com/quay/claircore/indexer" + "github.com/quay/claircore/language" ) var scanners = []indexer.PackageScanner{&Scanner{}} @@ -14,6 +15,6 @@ func NewEcosystem(_ context.Context) *indexer.Ecosystem { PackageScanners: func(_ context.Context) ([]indexer.PackageScanner, error) { return scanners, nil }, DistributionScanners: func(_ context.Context) ([]indexer.DistributionScanner, error) { return nil, nil }, RepositoryScanners: func(_ context.Context) ([]indexer.RepositoryScanner, error) { return nil, nil }, - Coalescer: NewCoalescer, + Coalescer: language.NewCoalescer, } }