diff --git a/db_test/20261001064316_test.sql b/db_test/20261001064316_test.sql new file mode 100644 index 00000000000..6fdfd1a6546 --- /dev/null +++ b/db_test/20261001064316_test.sql @@ -0,0 +1,49 @@ +-- +goose Up +CREATE TABLE vulnerabilities ( + id TEXT PRIMARY KEY, -- NB: This should have COLLATE "C" + data JSONB NOT NULL, + proto BYTEA NOT NULL +); +-- vuln_id does not reference vulnerabilities(id) because there is no index on (vuln_id) +-- (to save space/writes), which would cause a full table scan on vulnerability deletes. +-- Use vuln_commits to look up commits and delete by (commit_sha, vuln_id). +CREATE TABLE affected_commits ( + commit_sha BYTEA NOT NULL, + vuln_id TEXT NOT NULL, + PRIMARY KEY (commit_sha, vuln_id) +); +CREATE TABLE vuln_commits ( + vuln_id TEXT PRIMARY KEY REFERENCES vulnerabilities(id) ON DELETE CASCADE, + commits BYTEA[] +); + +CREATE TABLE affected_versions ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + vuln_id TEXT NOT NULL REFERENCES vulnerabilities(id) ON DELETE CASCADE, + name TEXT NOT NULL, + base_ecosystem TEXT NOT NULL, + ecosystems TEXT[] NOT NULL, + coarse_min TEXT COLLATE "C" NOT NULL, + coarse_max TEXT COLLATE "C" NOT NULL, + versions TEXT[], + events JSONB +); + +-- Primary index for typical queries (with ecosystem provided) +CREATE INDEX idx_affected_versions_match +ON affected_versions (name, base_ecosystem, vuln_id, id, coarse_min, coarse_max) +INCLUDE (ecosystems); + +-- Secondary index for queries without ecosystems +CREATE INDEX idx_affected_versions_name_only +ON affected_versions (name, vuln_id, id); + +-- Index for worker updates/deletes by vuln_id +CREATE INDEX idx_affected_versions_vuln_id ON affected_versions (vuln_id); + + +-- +goose Down +DROP TABLE affected_versions; +DROP TABLE vuln_commits; +DROP TABLE affected_commits; +DROP TABLE vulnerabilities; diff --git a/deployment/terraform/environments/temp/.terraform.lock.hcl b/deployment/terraform/environments/temp/.terraform.lock.hcl new file mode 100644 index 00000000000..77d5494874b --- /dev/null +++ b/deployment/terraform/environments/temp/.terraform.lock.hcl @@ -0,0 +1,21 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/google" { + version = "8.5.0" + hashes = [ + "h1:9494xJ0RxmS9t4nUhm8APyQVoRpo7y+4m7pD5BgBf+I=", + "zh:0c731c8c54b1561edec0f31f70056b22af16c6378d95b12a13d1684a4dd332c6", + "zh:12ed657b5c117068662e2db1dd71b77852c965fb747970a0be4c10ee76622dff", + "zh:1cbd63264ea11a323635bf032bbf336992861d62bf407600412c7cc4c78b4f3b", + "zh:577d9f89f09712bedc74033eabdad25e22dd95a0b851ea7fbd217efe66204183", + "zh:716ba4cefceff5bfab38f126d94de644a0b4495a2662cb8ef587eb954a38fa04", + "zh:76556e11b5ada305abfa86ccee29125fc60c46ed2b9556889e7d4b9913b16432", + "zh:9fc00c77ee7c3bdcb732daf4d375ed9ff4993cedea65ac3469c50b7aed566fd3", + "zh:c301a29da6227d40ed9d751f0c354487d124f52dd5b6c769caded32653d61630", + "zh:cbfa0b1e72349251af4e5c279e72b4ba37c909214933b2c25c7cc081335dfd99", + "zh:d8dfaa1319b0de93536ec3cf90832a9a4a45d9def3123c9bfff40f165d2d290a", + "zh:ec552919ffaf783fbe5a4c86080e4badd19b12a4c50b963b0acc4d4f8c28f14c", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/deployment/terraform/environments/temp/main.tf b/deployment/terraform/environments/temp/main.tf new file mode 100644 index 00000000000..e38704df9c2 --- /dev/null +++ b/deployment/terraform/environments/temp/main.tf @@ -0,0 +1,75 @@ +variable "project_id" { + description = "The GCP project ID to deploy the experimental Cloud SQL database to." + type = string + default = "oss-vdb-test" +} + +resource "google_sql_database_instance" "experiment" { + project = var.project_id + name = "experiment-db" + database_version = "POSTGRES_18" + region = "us-west2" + + settings { + tier = "db-perf-optimized-N-4" + availability_type = "ZONAL" + + connection_pool_config { + connection_pooling_enabled = true + } + + ip_configuration { + ipv4_enabled = true + private_network = data.google_compute_network.default.id + enable_private_path_for_google_cloud_services = true + ssl_mode = "ENCRYPTED_ONLY" + } + + database_flags { + name = "cloudsql.iam_authentication" + value = "on" + } + database_flags { + name = "temp_file_limit" + value = "2147483647" # maximum allowed (2TiB, measured in KiB) + } + } + + deletion_protection = false + depends_on = [google_service_networking_connection.private_vpc_connection] +} + +resource "google_sql_database" "osv" { + name = "osv" + instance = google_sql_database_instance.experiment.name + project = var.project_id +} + + +resource "google_project_service" "service_networking" { + project = var.project_id + service = "servicenetworking.googleapis.com" + disable_on_destroy = false +} + +resource "google_compute_global_address" "private_ip_address" { + project = var.project_id + name = "google-managed-services-default" + purpose = "VPC_PEERING" + address_type = "INTERNAL" + prefix_length = 16 + network = data.google_compute_network.default.id +} + +resource "google_service_networking_connection" "private_vpc_connection" { + network = data.google_compute_network.default.id + service = "servicenetworking.googleapis.com" + reserved_peering_ranges = [google_compute_global_address.private_ip_address.name] + + depends_on = [google_project_service.service_networking] +} + +data "google_compute_network" "default" { + project = var.project_id + name = "default" +} diff --git a/go/go.mod b/go/go.mod index 128abba9e51..9a50e2f963c 100644 --- a/go/go.mod +++ b/go/go.mod @@ -20,6 +20,7 @@ require ( github.com/google/go-cmp v0.7.0 github.com/google/osv-scalibr v0.5.3 github.com/hashicorp/go-retryablehttp v0.7.8 + github.com/jackc/pgx/v5 v5.11.0 github.com/klauspost/compress v1.20.1 github.com/microcosm-cc/bluemonday v1.0.27 github.com/nikolalohinski/gonja/v2 v2.9.1 @@ -95,6 +96,8 @@ require ( github.com/gorilla/css v1.0.1 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kevinburke/ssh_config v1.6.0 // indirect github.com/lucasb-eyer/go-colorful v1.4.1 // indirect diff --git a/go/go.sum b/go/go.sum index c59350b1a0c..db7113aa05f 100644 --- a/go/go.sum +++ b/go/go.sum @@ -188,6 +188,14 @@ github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB1 github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVUrx/c8Unxc48= github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= +github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kevinburke/ssh_config v1.6.0 h1:J1FBfmuVosPHf5GRdltRLhPJtJpTlMdKTBjRgTaQBFY= @@ -272,6 +280,7 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= diff --git a/go/internal/database/postgres/affected_versions.go b/go/internal/database/postgres/affected_versions.go new file mode 100644 index 00000000000..b7d895e3491 --- /dev/null +++ b/go/internal/database/postgres/affected_versions.go @@ -0,0 +1,243 @@ +// Package postgres implements the PostgreSQL database store for OSV. +package postgres + +import ( + "encoding/json" + "net/url" + "slices" + "strings" + + "github.com/google/osv.dev/go/internal/osvutil" + "github.com/google/osv.dev/go/osv/ecosystem" + "github.com/ossf/osv-schema/bindings/go/osvschema" +) + +const ( + MinCoarseVersion = "00:00000000.00000000.00000000" + MaxCoarseVersion = "99:99999999.99999999.99999999" +) + +type AffectedEvent struct { + Type string `json:"type"` + Value string `json:"value"` +} + +type AffectedVersionRow struct { + VulnID string + Name string + BaseEcosystem string + Ecosystems []string + CoarseMin string + CoarseMax string + Versions []string + EventsJSON []byte +} + +var ubuntuReplacer = strings.NewReplacer(":Pro", "", ":LTS", "") + +func removeVariants(eco string) string { + if strings.HasPrefix(eco, "Ubuntu") { + return ubuntuReplacer.Replace(eco) + } + + return "" +} + +func normalizeRepo(repoURL string) string { + // Normalize the repo_url for use with GIT AffectedVersions entities. + // Removes the scheme/protocol, the .git extension, and trailing slashes. + if repoURL == "" { + return "" + } + parsed, err := url.Parse(repoURL) + if err != nil { + return repoURL + } + normalized := parsed.Host + parsed.Path + normalized = strings.TrimRight(normalized, "/") + normalized = strings.TrimSuffix(normalized, ".git") + + return normalized +} + +// ComputeAffectedVersions computes the affected_versions rows for a vulnerability. +func ComputeAffectedVersions(vuln *osvschema.Vulnerability) ([]AffectedVersionRow, error) { + if vuln.GetWithdrawn() != nil { + return nil, nil + } + + var res []AffectedVersionRow + + for _, affected := range vuln.GetAffected() { + var allPkgEcosystems []string + var baseEcosystem string + pkgEcosystem := affected.GetPackage().GetEcosystem() + if pkgEcosystem != "" { + allPkgEcosystems = []string{pkgEcosystem} + baseEcosystem, _, _ = strings.Cut(pkgEcosystem, ":") + if baseEcosystem != pkgEcosystem { + allPkgEcosystems = append(allPkgEcosystems, baseEcosystem) + } + if v := removeVariants(pkgEcosystem); v != "" { + allPkgEcosystems = append(allPkgEcosystems, v) + } + + slices.Sort(allPkgEcosystems) + allPkgEcosystems = slices.Compact(allPkgEcosystems) + } + + pkgName := affected.GetPackage().GetName() + eHelper, exists := ecosystem.DefaultProvider.Get(pkgEcosystem) + if exists { + pkgName = ecosystem.NormalizePackageName(eHelper, pkgName) + } + + var gitRepos []string + hasAffected := false + + for _, r := range affected.GetRanges() { + if r.GetType() == osvschema.Range_GIT { + if r.GetRepo() != "" { + gitRepos = append(gitRepos, normalizeRepo(r.GetRepo())) + } + + continue + } + if r.GetType() != osvschema.Range_ECOSYSTEM && r.GetType() != osvschema.Range_SEMVER { + continue + } + if len(r.GetEvents()) == 0 { + continue + } + + hasAffected = true + var events []osvutil.Event + for _, e := range r.GetEvents() { + evt := osvutil.FromSchemaEvent(e) + if evt.Version != "" { + events = append(events, evt) + } + } + + if exists { + _ = osvutil.SortEvents(eHelper, events) + } + + rangeEvents := make([]AffectedEvent, 0, len(events)) + for _, e := range events { + rangeEvents = append(rangeEvents, AffectedEvent{Type: e.Type.String(), Value: e.Version}) + } + + coarseMin := MinCoarseVersion + coarseMax := MaxCoarseVersion + + if exists { + for _, ev := range rangeEvents { + if ev.Type == "introduced" { + if cm, err := eHelper.Coarse(ev.Value); err == nil { + coarseMin = cm + } + last := rangeEvents[len(rangeEvents)-1] + if last.Type != "introduced" { + if cm, err := eHelper.Coarse(last.Value); err == nil { + coarseMax = cm + } + } + + break + } + } + } + + eventsJSON, err := json.Marshal(rangeEvents) + if err != nil { + return nil, err + } + + if len(allPkgEcosystems) > 0 { + res = append(res, AffectedVersionRow{ + VulnID: vuln.GetId(), + Name: pkgName, + BaseEcosystem: baseEcosystem, + Ecosystems: allPkgEcosystems, + CoarseMin: coarseMin, + CoarseMax: coarseMax, + EventsJSON: eventsJSON, + }) + } + } + + versions := affected.GetVersions() + if pkgName != "" && len(versions) > 0 && len(allPkgEcosystems) > 0 { + hasAffected = true + coarseMin := MinCoarseVersion + coarseMax := MaxCoarseVersion + + if exists { + var allCoarse []string + for _, v := range versions { + cm, err := eHelper.Coarse(v) + if err != nil { + break + } + allCoarse = append(allCoarse, cm) + } + if len(allCoarse) == len(versions) { + slices.Sort(allCoarse) + coarseMin = allCoarse[0] + coarseMax = allCoarse[len(allCoarse)-1] + } + } + + res = append(res, AffectedVersionRow{ + VulnID: vuln.GetId(), + Name: pkgName, + BaseEcosystem: baseEcosystem, + Ecosystems: allPkgEcosystems, + CoarseMin: coarseMin, + CoarseMax: coarseMax, + Versions: versions, + }) + } + + if pkgName != "" && !hasAffected && len(allPkgEcosystems) > 0 { + // We have a package that does not have any affected ranges or versions, + // which doesn't really make sense. + // Add an empty AffectedVersions entry so that this vuln is returned when + // querying the API with no version specified. + res = append(res, AffectedVersionRow{ + VulnID: vuln.GetId(), + Name: pkgName, + BaseEcosystem: baseEcosystem, + Ecosystems: allPkgEcosystems, + CoarseMin: MinCoarseVersion, + CoarseMax: MaxCoarseVersion, + }) + } + + if len(gitRepos) > 0 { + // If we have GIT ranges, always add a GIT entry for each unique repository. + // Even if affected.versions is empty, we still want to return this vuln + // for the API queries with no versions specified. + slices.Sort(gitRepos) + gitRepos = slices.Compact(gitRepos) + gitVersions := versions + if gitVersions == nil { + gitVersions = []string{} + } + for _, repo := range gitRepos { + res = append(res, AffectedVersionRow{ + VulnID: vuln.GetId(), + Name: repo, + BaseEcosystem: "GIT", + Ecosystems: []string{"GIT"}, + CoarseMin: MinCoarseVersion, + CoarseMax: MaxCoarseVersion, + Versions: gitVersions, + }) + } + } + } + + return res, nil +} diff --git a/go/internal/database/postgres/batch.go b/go/internal/database/postgres/batch.go new file mode 100644 index 00000000000..bce7d95d99a --- /dev/null +++ b/go/internal/database/postgres/batch.go @@ -0,0 +1,349 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: batch.go + +package postgres + +import ( + "context" + "errors" + + "github.com/jackc/pgx/v5" +) + +var ( + ErrBatchAlreadyClosed = errors.New("batch already closed") +) + +const matchCommits = `-- name: MatchCommits :batchmany +SELECT vuln_id +FROM affected_commits +WHERE commit_sha = $1 + AND vuln_id > $2 +ORDER BY vuln_id +LIMIT $3 +` + +type MatchCommitsBatchResults struct { + br pgx.BatchResults + tot int + closed bool +} + +type MatchCommitsParams struct { + CommitSha []byte + AfterVulnID string + RowLimit int32 +} + +func (q *Queries) MatchCommits(ctx context.Context, arg []MatchCommitsParams) *MatchCommitsBatchResults { + batch := &pgx.Batch{} + for _, a := range arg { + vals := []interface{}{ + a.CommitSha, + a.AfterVulnID, + a.RowLimit, + } + batch.Queue(matchCommits, vals...) + } + br := q.db.SendBatch(ctx, batch) + return &MatchCommitsBatchResults{br, len(arg), false} +} + +func (b *MatchCommitsBatchResults) Query(f func(int, []string, error)) { + defer b.br.Close() + for t := 0; t < b.tot; t++ { + var items []string + if b.closed { + if f != nil { + f(t, items, ErrBatchAlreadyClosed) + } + continue + } + err := func() error { + rows, err := b.br.Query() + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var vuln_id string + if err := rows.Scan(&vuln_id); err != nil { + return err + } + items = append(items, vuln_id) + } + return rows.Err() + }() + if f != nil { + f(t, items, err) + } + } +} + +func (b *MatchCommitsBatchResults) Close() error { + b.closed = true + return b.br.Close() +} + +const matchPackageVersionedNoEcosystem = `-- name: MatchPackageVersionedNoEcosystem :batchmany +SELECT id, vuln_id, base_ecosystem, versions, events +FROM affected_versions +WHERE name = $1 + AND (vuln_id, id) > ($2::text, $3::bigint) +ORDER BY vuln_id, id +LIMIT $4 +` + +type MatchPackageVersionedNoEcosystemBatchResults struct { + br pgx.BatchResults + tot int + closed bool +} + +type MatchPackageVersionedNoEcosystemParams struct { + Name string + AfterVulnID string + AfterRowID int64 + RowLimit int32 +} + +type MatchPackageVersionedNoEcosystemRow struct { + ID int64 + VulnID string + BaseEcosystem string + Versions []string + Events []byte +} + +func (q *Queries) MatchPackageVersionedNoEcosystem(ctx context.Context, arg []MatchPackageVersionedNoEcosystemParams) *MatchPackageVersionedNoEcosystemBatchResults { + batch := &pgx.Batch{} + for _, a := range arg { + vals := []interface{}{ + a.Name, + a.AfterVulnID, + a.AfterRowID, + a.RowLimit, + } + batch.Queue(matchPackageVersionedNoEcosystem, vals...) + } + br := q.db.SendBatch(ctx, batch) + return &MatchPackageVersionedNoEcosystemBatchResults{br, len(arg), false} +} + +func (b *MatchPackageVersionedNoEcosystemBatchResults) Query(f func(int, []MatchPackageVersionedNoEcosystemRow, error)) { + defer b.br.Close() + for t := 0; t < b.tot; t++ { + var items []MatchPackageVersionedNoEcosystemRow + if b.closed { + if f != nil { + f(t, items, ErrBatchAlreadyClosed) + } + continue + } + err := func() error { + rows, err := b.br.Query() + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var i MatchPackageVersionedNoEcosystemRow + if err := rows.Scan( + &i.ID, + &i.VulnID, + &i.BaseEcosystem, + &i.Versions, + &i.Events, + ); err != nil { + return err + } + items = append(items, i) + } + return rows.Err() + }() + if f != nil { + f(t, items, err) + } + } +} + +func (b *MatchPackageVersionedNoEcosystemBatchResults) Close() error { + b.closed = true + return b.br.Close() +} + +const matchPackageVersionedWithEcosystem = `-- name: MatchPackageVersionedWithEcosystem :batchmany +SELECT id, vuln_id, versions, events +FROM affected_versions +WHERE name = $1 + AND base_ecosystem = $2 + AND (vuln_id, id) > ($3::text, $4::bigint) + AND coarse_min <= $5 + AND coarse_max >= $6 + AND $7::text = ANY(ecosystems) +ORDER BY vuln_id, id +LIMIT $8 +` + +type MatchPackageVersionedWithEcosystemBatchResults struct { + br pgx.BatchResults + tot int + closed bool +} + +type MatchPackageVersionedWithEcosystemParams struct { + Name string + BaseEcosystem string + AfterVulnID string + AfterRowID int64 + CoarseUpper string + CoarseLower string + Ecosystem string + RowLimit int32 +} + +type MatchPackageVersionedWithEcosystemRow struct { + ID int64 + VulnID string + Versions []string + Events []byte +} + +func (q *Queries) MatchPackageVersionedWithEcosystem(ctx context.Context, arg []MatchPackageVersionedWithEcosystemParams) *MatchPackageVersionedWithEcosystemBatchResults { + batch := &pgx.Batch{} + for _, a := range arg { + vals := []interface{}{ + a.Name, + a.BaseEcosystem, + a.AfterVulnID, + a.AfterRowID, + a.CoarseUpper, + a.CoarseLower, + a.Ecosystem, + a.RowLimit, + } + batch.Queue(matchPackageVersionedWithEcosystem, vals...) + } + br := q.db.SendBatch(ctx, batch) + return &MatchPackageVersionedWithEcosystemBatchResults{br, len(arg), false} +} + +func (b *MatchPackageVersionedWithEcosystemBatchResults) Query(f func(int, []MatchPackageVersionedWithEcosystemRow, error)) { + defer b.br.Close() + for t := 0; t < b.tot; t++ { + var items []MatchPackageVersionedWithEcosystemRow + if b.closed { + if f != nil { + f(t, items, ErrBatchAlreadyClosed) + } + continue + } + err := func() error { + rows, err := b.br.Query() + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var i MatchPackageVersionedWithEcosystemRow + if err := rows.Scan( + &i.ID, + &i.VulnID, + &i.Versions, + &i.Events, + ); err != nil { + return err + } + items = append(items, i) + } + return rows.Err() + }() + if f != nil { + f(t, items, err) + } + } +} + +func (b *MatchPackageVersionedWithEcosystemBatchResults) Close() error { + b.closed = true + return b.br.Close() +} + +const matchPackageVersionless = `-- name: MatchPackageVersionless :batchmany +SELECT DISTINCT vuln_id +FROM affected_versions +WHERE name = $1 + AND base_ecosystem = $2 + AND vuln_id > $3 + AND $4::text = ANY(ecosystems) +ORDER BY vuln_id +LIMIT $5 +` + +type MatchPackageVersionlessBatchResults struct { + br pgx.BatchResults + tot int + closed bool +} + +type MatchPackageVersionlessParams struct { + Name string + BaseEcosystem string + AfterVulnID string + Ecosystem string + RowLimit int32 +} + +func (q *Queries) MatchPackageVersionless(ctx context.Context, arg []MatchPackageVersionlessParams) *MatchPackageVersionlessBatchResults { + batch := &pgx.Batch{} + for _, a := range arg { + vals := []interface{}{ + a.Name, + a.BaseEcosystem, + a.AfterVulnID, + a.Ecosystem, + a.RowLimit, + } + batch.Queue(matchPackageVersionless, vals...) + } + br := q.db.SendBatch(ctx, batch) + return &MatchPackageVersionlessBatchResults{br, len(arg), false} +} + +func (b *MatchPackageVersionlessBatchResults) Query(f func(int, []string, error)) { + defer b.br.Close() + for t := 0; t < b.tot; t++ { + var items []string + if b.closed { + if f != nil { + f(t, items, ErrBatchAlreadyClosed) + } + continue + } + err := func() error { + rows, err := b.br.Query() + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var vuln_id string + if err := rows.Scan(&vuln_id); err != nil { + return err + } + items = append(items, vuln_id) + } + return rows.Err() + }() + if f != nil { + f(t, items, err) + } + } +} + +func (b *MatchPackageVersionlessBatchResults) Close() error { + b.closed = true + return b.br.Close() +} diff --git a/go/internal/database/postgres/db.go b/go/internal/database/postgres/db.go new file mode 100644 index 00000000000..4ae5d9215d1 --- /dev/null +++ b/go/internal/database/postgres/db.go @@ -0,0 +1,33 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 + +package postgres + +import ( + "context" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" +) + +type DBTX interface { + Exec(context.Context, string, ...interface{}) (pgconn.CommandTag, error) + Query(context.Context, string, ...interface{}) (pgx.Rows, error) + QueryRow(context.Context, string, ...interface{}) pgx.Row + SendBatch(context.Context, *pgx.Batch) pgx.BatchResults +} + +func New(db DBTX) *Queries { + return &Queries{db: db} +} + +type Queries struct { + db DBTX +} + +func (q *Queries) WithTx(tx pgx.Tx) *Queries { + return &Queries{ + db: tx, + } +} diff --git a/go/internal/database/postgres/models.go b/go/internal/database/postgres/models.go new file mode 100644 index 00000000000..f5ecce053a5 --- /dev/null +++ b/go/internal/database/postgres/models.go @@ -0,0 +1,33 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 + +package postgres + +type AffectedCommit struct { + CommitSha []byte + VulnID string +} + +type AffectedVersion struct { + ID int64 + VulnID string + Name string + BaseEcosystem string + Ecosystems []string + CoarseMin string + CoarseMax string + Versions []string + Events []byte +} + +type VulnCommit struct { + VulnID string + Commits [][]byte +} + +type Vulnerability struct { + ID string + Data []byte + Proto []byte +} diff --git a/go/internal/database/postgres/queries.sql b/go/internal/database/postgres/queries.sql new file mode 100644 index 00000000000..f04158d643e --- /dev/null +++ b/go/internal/database/postgres/queries.sql @@ -0,0 +1,37 @@ +-- name: MatchPackageVersionedWithEcosystem :batchmany +SELECT id, vuln_id, versions, events +FROM affected_versions +WHERE name = @name + AND base_ecosystem = @base_ecosystem + AND (vuln_id, id) > (@after_vuln_id::text, @after_row_id::bigint) + AND coarse_min <= @coarse_upper + AND coarse_max >= @coarse_lower + AND @ecosystem::text = ANY(ecosystems) +ORDER BY vuln_id, id +LIMIT @row_limit; + +-- name: MatchPackageVersionedNoEcosystem :batchmany +SELECT id, vuln_id, base_ecosystem, versions, events +FROM affected_versions +WHERE name = @name + AND (vuln_id, id) > (@after_vuln_id::text, @after_row_id::bigint) +ORDER BY vuln_id, id +LIMIT @row_limit; + +-- name: MatchPackageVersionless :batchmany +SELECT DISTINCT vuln_id +FROM affected_versions +WHERE name = @name + AND base_ecosystem = @base_ecosystem + AND vuln_id > @after_vuln_id + AND @ecosystem::text = ANY(ecosystems) +ORDER BY vuln_id +LIMIT @row_limit; + +-- name: MatchCommits :batchmany +SELECT vuln_id +FROM affected_commits +WHERE commit_sha = @commit_sha + AND vuln_id > @after_vuln_id +ORDER BY vuln_id +LIMIT @row_limit; diff --git a/go/internal/database/postgres/queries.sql.go b/go/internal/database/postgres/queries.sql.go new file mode 100644 index 00000000000..764d9c3be70 --- /dev/null +++ b/go/internal/database/postgres/queries.sql.go @@ -0,0 +1,6 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: queries.sql + +package postgres diff --git a/go/internal/database/postgres/sqlc.yaml b/go/internal/database/postgres/sqlc.yaml new file mode 100644 index 00000000000..70489d6814c --- /dev/null +++ b/go/internal/database/postgres/sqlc.yaml @@ -0,0 +1,16 @@ +version: "2" +sql: + - engine: "postgresql" + schema: "../../../../db_test" + queries: "queries.sql" + gen: + go: + package: "postgres" + out: "." + sql_package: "pgx/v5" + overrides: + - db_type: "timestamptz" + go_type: "time.Time" + - db_type: "jsonb" + go_type: + type: "[]byte" diff --git a/go/internal/database/postgres/vulnerability.go b/go/internal/database/postgres/vulnerability.go new file mode 100644 index 00000000000..94215c53419 --- /dev/null +++ b/go/internal/database/postgres/vulnerability.go @@ -0,0 +1,88 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postgres + +import ( + "context" + "time" + + "github.com/google/osv.dev/go/internal/models" + "github.com/google/osv.dev/go/internal/osvutil/batcher" + "github.com/google/osv.dev/go/osv/ecosystem" +) + +const defaultMatchPageSize = 1000 + +// VulnerabilityStore implements models.VulnerabilityStore backed by PostgreSQL. +type VulnerabilityStore struct { + models.UnimplementedVulnerabilityStore + + db DBTX + queries *Queries + ecosystems *ecosystem.Provider + pageSize int32 + + commitsBatcher *batcher.Batcher[MatchCommitsParams, []string] + versionlessBatcher *batcher.Batcher[MatchPackageVersionlessParams, []string] + versionedEcoBatcher *batcher.Batcher[MatchPackageVersionedWithEcosystemParams, []MatchPackageVersionedWithEcosystemRow] + versionedNoEcoBatcher *batcher.Batcher[MatchPackageVersionedNoEcosystemParams, []MatchPackageVersionedNoEcosystemRow] +} + +var _ models.VulnerabilityStore = (*VulnerabilityStore)(nil) + +// VulnStoreConfig configures a PostgreSQL VulnerabilityStore. +type VulnStoreConfig struct { + DB DBTX + Ecosystems *ecosystem.Provider + PageSize int32 + BatchTimeout time.Duration + BatchMaxElements int +} + +// NewVulnerabilityStore creates a new PostgreSQL VulnerabilityStore. +func NewVulnerabilityStore(cfg VulnStoreConfig) *VulnerabilityStore { + ecos := cfg.Ecosystems + if ecos == nil { + ecos = ecosystem.DefaultProvider + } + pageSize := cfg.PageSize + if pageSize <= 0 { + pageSize = defaultMatchPageSize + } + maxElements := cfg.BatchMaxElements + if maxElements <= 0 { + maxElements = 1000 + } + + s := &VulnerabilityStore{ + db: cfg.DB, + queries: New(cfg.DB), + ecosystems: ecos, + pageSize: pageSize, + } + + s.commitsBatcher = batcher.New(cfg.BatchTimeout, maxElements, s.batchMatchCommits) + s.versionlessBatcher = batcher.New(cfg.BatchTimeout, maxElements, s.batchMatchVersionless) + s.versionedEcoBatcher = batcher.New(cfg.BatchTimeout, maxElements, s.batchMatchVersionedWithEco) + s.versionedNoEcoBatcher = batcher.New(cfg.BatchTimeout, maxElements, s.batchMatchVersionedNoEco) + + return s +} + +// Ping verifies connection health to PostgreSQL. +func (s *VulnerabilityStore) Ping(ctx context.Context) error { + _, err := s.db.Exec(ctx, "SELECT 1") + return err +} diff --git a/go/internal/database/postgres/vulnerability_matching.go b/go/internal/database/postgres/vulnerability_matching.go new file mode 100644 index 00000000000..501e7126da4 --- /dev/null +++ b/go/internal/database/postgres/vulnerability_matching.go @@ -0,0 +1,657 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postgres + +import ( + "context" + "encoding/base64" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "iter" + "math" + "net/url" + "regexp" + "slices" + "strings" + + "github.com/google/osv.dev/go/internal/models" + "github.com/google/osv.dev/go/internal/osvutil" + "github.com/google/osv.dev/go/internal/osvutil/batcher" + eco "github.com/google/osv.dev/go/osv/ecosystem" +) + +const ( + cursorTypeCommit byte = 1 + cursorTypePackage byte = 2 +) + +type matchCommitsCursor struct { + afterVulnID string +} + +func (c matchCommitsCursor) String() string { + buf := make([]byte, 0, 1+len(c.afterVulnID)) + buf = append(buf, cursorTypeCommit) + buf = append(buf, c.afterVulnID...) + + return base64.RawURLEncoding.EncodeToString(buf) +} + +func parseMatchCommitsCursor(cursor string) (matchCommitsCursor, error) { + b, err := base64.RawURLEncoding.DecodeString(cursor) + if err != nil { + return matchCommitsCursor{}, err + } + if len(b) < 2 || b[0] != cursorTypeCommit { + return matchCommitsCursor{}, errors.New("cursor not in correct format") + } + + return matchCommitsCursor{afterVulnID: string(b[1:])}, nil +} + +// matchPackagesCursor represents the keyset pagination position (afterVulnID, afterRowID) +// in affected_versions ordered by (vuln_id, id). +// +// Once a vuln_id has been matched, afterRowID is set to math.MaxInt64 so that: +// 1. Any remaining rows for the same vuln_id in the current in-memory page are skipped. +// 2. Resuming from the cursor in SQL via `WHERE (vuln_id, id) > (@after_vuln_id, @after_row_id)` +// skips all remaining rows for afterVulnID in the index and jumps directly to the next vuln_id. +type matchPackagesCursor struct { + afterVulnID string + afterRowID int64 +} + +func (c matchPackagesCursor) String() string { + // Since Postgres IDENTITY row IDs are always >= 1, we encode math.MaxInt64 as 0 + // on the wire so it takes 1 byte in varint encoding instead of 10 bytes. + wireRowID := c.afterRowID + if wireRowID == math.MaxInt64 { + wireRowID = 0 + } + buf := make([]byte, 0, 1+binary.MaxVarintLen64+len(c.afterVulnID)) + buf = append(buf, cursorTypePackage) + buf = binary.AppendVarint(buf, wireRowID) + buf = append(buf, c.afterVulnID...) + + return base64.RawURLEncoding.EncodeToString(buf) +} + +func parseMatchPackagesCursor(cursor string) (matchPackagesCursor, error) { + b, err := base64.RawURLEncoding.DecodeString(cursor) + if err != nil { + return matchPackagesCursor{}, err + } + if len(b) < 2 || b[0] != cursorTypePackage { + return matchPackagesCursor{}, errors.New("cursor not in correct format") + } + + wireRowID, n := binary.Varint(b[1:]) + if n <= 0 || wireRowID < 0 { + return matchPackagesCursor{}, errors.New("cursor not in correct format") + } + afterRowID := wireRowID + if afterRowID == 0 { + afterRowID = math.MaxInt64 + } + + return matchPackagesCursor{ + afterVulnID: string(b[1+n:]), + afterRowID: afterRowID, + }, nil +} + +func (s *VulnerabilityStore) batchMatchCommits(ctx context.Context, params []MatchCommitsParams) []batcher.Result[[]string] { + results := make([]batcher.Result[[]string], len(params)) + br := s.queries.MatchCommits(ctx, params) + br.Query(func(i int, items []string, err error) { + results[i] = batcher.Result[[]string]{Val: items, Err: err} + }) + + return results +} + +func (s *VulnerabilityStore) batchMatchVersionless(ctx context.Context, params []MatchPackageVersionlessParams) []batcher.Result[[]string] { + results := make([]batcher.Result[[]string], len(params)) + br := s.queries.MatchPackageVersionless(ctx, params) + br.Query(func(i int, items []string, err error) { + results[i] = batcher.Result[[]string]{Val: items, Err: err} + }) + + return results +} + +func (s *VulnerabilityStore) batchMatchVersionedWithEco(ctx context.Context, params []MatchPackageVersionedWithEcosystemParams) []batcher.Result[[]MatchPackageVersionedWithEcosystemRow] { + results := make([]batcher.Result[[]MatchPackageVersionedWithEcosystemRow], len(params)) + br := s.queries.MatchPackageVersionedWithEcosystem(ctx, params) + br.Query(func(i int, items []MatchPackageVersionedWithEcosystemRow, err error) { + results[i] = batcher.Result[[]MatchPackageVersionedWithEcosystemRow]{Val: items, Err: err} + }) + + return results +} + +func (s *VulnerabilityStore) batchMatchVersionedNoEco(ctx context.Context, params []MatchPackageVersionedNoEcosystemParams) []batcher.Result[[]MatchPackageVersionedNoEcosystemRow] { + results := make([]batcher.Result[[]MatchPackageVersionedNoEcosystemRow], len(params)) + br := s.queries.MatchPackageVersionedNoEcosystem(ctx, params) + br.Query(func(i int, items []MatchPackageVersionedNoEcosystemRow, err error) { + results[i] = batcher.Result[[]MatchPackageVersionedNoEcosystemRow]{Val: items, Err: err} + }) + + return results +} + +// MatchCommits returns an iterator of vulnerability IDs for commits that match the query. +func (s *VulnerabilityStore) MatchCommits(ctx context.Context, commit []byte, cursor string) iter.Seq2[models.MatchResult, error] { + return func(yield func(models.MatchResult, error) bool) { + var afterVulnID string + if cursor != "" { + parsed, err := parseMatchCommitsCursor(cursor) + if err != nil { + yield(models.MatchResult{}, fmt.Errorf("%w: %w", models.ErrInvalidCursor, err)) + return + } + afterVulnID = parsed.afterVulnID + } + + for { + page, err := s.commitsBatcher.Get(ctx, MatchCommitsParams{ + CommitSha: commit, + AfterVulnID: afterVulnID, + RowLimit: s.pageSize, + }) + if err != nil { + yield(models.MatchResult{}, fmt.Errorf("failed to fetch affected commits: %w", err)) + return + } + for _, vulnID := range page { + afterVulnID = vulnID + cur := matchCommitsCursor{afterVulnID: afterVulnID} + if !yield(models.MatchResult{IsMatch: true, ID: vulnID, Cursor: cur.String}, nil) { + return + } + } + if len(page) < int(s.pageSize) { + return + } + } + } +} + +// MatchCommitsBatch returns iterators of vulnerability IDs for a batch of commit queries. +func (s *VulnerabilityStore) MatchCommitsBatch(ctx context.Context, queries []models.CommitQuery) ([]iter.Seq2[models.MatchResult, error], error) { + results := make([]iter.Seq2[models.MatchResult, error], len(queries)) + for i, q := range queries { + results[i] = s.MatchCommits(ctx, q.Commit, q.Cursor) + } + + return results, nil +} + +// MatchPackages returns an iterator of vulnerability IDs for packages that match the query. +func (s *VulnerabilityStore) MatchPackages(ctx context.Context, ecosystem, name, version, cursor string) iter.Seq2[models.MatchResult, error] { + return func(yield func(models.MatchResult, error) bool) { + if name == "" { + // Bare minimum a package name is needed to perform a match + return + } + if ecosystem == "" && version == "" { + yield(models.MatchResult{}, errors.New("ecosystem or version must be specified")) + return + } + + var curState matchPackagesCursor + if cursor != "" { + var err error + curState, err = parseMatchPackagesCursor(cursor) + if err != nil { + yield(models.MatchResult{}, fmt.Errorf("%w: %w", models.ErrInvalidCursor, err)) + return + } + } + + // Ideally, we'd check both unnormalized and normalized names when no ecosystem is + // provided, but querying multiple names at once would prevent a single ordered index + // scan for keyset pagination. Only normalize when an ecosystem is provided. + if ecosystem == "GIT" { + name = normalizeRepo(name) + } else if ecosystem != "" { + if sys, ok := s.ecosystems.Get(ecosystem); ok { + name = eco.NormalizePackageName(sys, name) + } + } + + baseEco, _, _ := strings.Cut(ecosystem, ":") + + // If no version is specified, match all package records + if version == "" { + params := MatchPackageVersionlessParams{ + Name: name, + BaseEcosystem: baseEco, + AfterVulnID: curState.afterVulnID, + Ecosystem: ecosystem, + RowLimit: s.pageSize, + } + s.yieldVersionlessPages(ctx, yield, params) + + return + } + + if ecosystem != "" { + coarseLower, coarseUpper, isMatch := s.buildVersionedWithEcoMatcher(ecosystem, name, version) + params := MatchPackageVersionedWithEcosystemParams{ + Name: name, + BaseEcosystem: baseEco, + AfterVulnID: curState.afterVulnID, + AfterRowID: curState.afterRowID, + CoarseUpper: coarseUpper, + CoarseLower: coarseLower, + Ecosystem: ecosystem, + RowLimit: s.pageSize, + } + s.yieldVersionedWithEcoPages(ctx, yield, params, curState, isMatch) + + return + } + + // Default for empty ecosystem query: use dynamic version matching + params := MatchPackageVersionedNoEcosystemParams{ + Name: name, + AfterVulnID: curState.afterVulnID, + AfterRowID: curState.afterRowID, + RowLimit: s.pageSize, + } + s.yieldVersionedNoEcoPages(ctx, yield, params, curState, name, version) + } +} + +// MatchPackagesBatch returns iterators of vulnerability IDs for a batch of package queries. +func (s *VulnerabilityStore) MatchPackagesBatch(ctx context.Context, queries []models.PackageQuery) ([]iter.Seq2[models.MatchResult, error], error) { + results := make([]iter.Seq2[models.MatchResult, error], len(queries)) + for i, q := range queries { + results[i] = s.MatchPackages(ctx, q.Ecosystem, q.Name, q.Version, q.Cursor) + } + + return results, nil +} + +func (s *VulnerabilityStore) yieldVersionlessPages( + ctx context.Context, + yield func(models.MatchResult, error) bool, + params MatchPackageVersionlessParams, +) { + for { + page, err := s.versionlessBatcher.Get(ctx, params) + if err != nil { + yield(models.MatchResult{}, fmt.Errorf("failed to fetch affected versions: %w", err)) + return + } + for _, vulnID := range page { + params.AfterVulnID = vulnID + cur := matchPackagesCursor{afterVulnID: vulnID, afterRowID: math.MaxInt64} + if !yield(models.MatchResult{IsMatch: true, ID: vulnID, Cursor: cur.String}, nil) { + return + } + } + if len(page) < int(s.pageSize) { + return + } + } +} + +func (s *VulnerabilityStore) yieldVersionedWithEcoPages( + ctx context.Context, + yield func(models.MatchResult, error) bool, + params MatchPackageVersionedWithEcosystemParams, + curState matchPackagesCursor, + isMatch func(versions []string, events []byte) bool, +) { + for { + params.AfterVulnID = curState.afterVulnID + params.AfterRowID = curState.afterRowID + page, err := s.versionedEcoBatcher.Get(ctx, params) + if err != nil { + yield(models.MatchResult{}, fmt.Errorf("failed to fetch affected versions: %w", err)) + return + } + for _, row := range page { + // If this vuln_id was already matched on an earlier row, keep afterRowID = math.MaxInt64 + // and skip re-evaluating remaining rows for this vuln_id. + if row.VulnID == curState.afterVulnID && curState.afterRowID == math.MaxInt64 { + cur := curState + if !yield(models.MatchResult{IsMatch: false, ID: "", Cursor: cur.String}, nil) { + return + } + + continue + } + curState.afterVulnID = row.VulnID + if !isMatch(row.Versions, row.Events) { + curState.afterRowID = row.ID + cur := curState + if !yield(models.MatchResult{IsMatch: false, ID: "", Cursor: cur.String}, nil) { + return + } + + continue + } + // Mark this vuln_id as matched so subsequent rows for the same vuln_id (in memory or + // in a follow-up paginated query) are skipped. + curState.afterRowID = math.MaxInt64 + cur := curState + if !yield(models.MatchResult{IsMatch: true, ID: row.VulnID, Cursor: cur.String}, nil) { + return + } + } + if len(page) < int(s.pageSize) { + return + } + } +} + +func (s *VulnerabilityStore) yieldVersionedNoEcoPages( + ctx context.Context, + yield func(models.MatchResult, error) bool, + params MatchPackageVersionedNoEcosystemParams, + curState matchPackagesCursor, + pkgName string, + version string, +) { + for { + params.AfterVulnID = curState.afterVulnID + params.AfterRowID = curState.afterRowID + page, err := s.versionedNoEcoBatcher.Get(ctx, params) + if err != nil { + yield(models.MatchResult{}, fmt.Errorf("failed to fetch affected versions: %w", err)) + return + } + for _, row := range page { + // If this vuln_id was already matched on an earlier row, keep afterRowID = math.MaxInt64 + // and skip re-evaluating remaining rows for this vuln_id. + if row.VulnID == curState.afterVulnID && curState.afterRowID == math.MaxInt64 { + cur := curState + if !yield(models.MatchResult{IsMatch: false, ID: "", Cursor: cur.String}, nil) { + return + } + + continue + } + curState.afterVulnID = row.VulnID + if !s.versionIsMatch(version, pkgName, row.BaseEcosystem, row.Versions, row.Events) { + curState.afterRowID = row.ID + cur := curState + if !yield(models.MatchResult{IsMatch: false, ID: "", Cursor: cur.String}, nil) { + return + } + + continue + } + // Mark this vuln_id as matched so subsequent rows for the same vuln_id (in memory or + // in a follow-up paginated query) are skipped. + curState.afterRowID = math.MaxInt64 + cur := curState + if !yield(models.MatchResult{IsMatch: true, ID: row.VulnID, Cursor: cur.String}, nil) { + return + } + } + if len(page) < int(s.pageSize) { + return + } + } +} + +// buildVersionedWithEcoMatcher determines the version comparison function and applies any +// coarse query range optimizations. +func (s *VulnerabilityStore) buildVersionedWithEcoMatcher( + ecosystem string, + pkgName string, + version string, +) (string, string, func(versions []string, events []byte) bool) { + coarseLower := MinCoarseVersion + coarseUpper := MaxCoarseVersion + + // Git versions only match using git-tag string comparisons + if ecosystem == "GIT" { + return coarseLower, coarseUpper, func(versions []string, _ []byte) bool { + return s.versionIsMatchString(version, pkgName, "GIT", versions) + } + } + + // Lookup ecosystem helper for parsing + eHelper, ok := s.ecosystems.Get(ecosystem) + if !ok { + // Fallback to string matching for unsupported ecosystems + return coarseLower, coarseUpper, func(versions []string, _ []byte) bool { + return s.versionIsMatchString(version, pkgName, ecosystem, versions) + } + } + + // Parse version with ecosystem helper + ver, err := eHelper.Parse(version) + if err != nil { + // Fallback to string matching for unparseable version strings + return coarseLower, coarseUpper, func(versions []string, _ []byte) bool { + return s.versionIsMatchString(version, pkgName, ecosystem, versions) + } + } + + // Optimize query using coarse range index scans if supported + if coarse, err := eHelper.Coarse(version); err == nil { + coarseLower = coarse + coarseUpper = coarse + } + + // Fast structured ecosystem matching + if ecosystem == "Alpine" || ecosystem == "Ubuntu" { + // TODO(michaelkedar): We don't support grabbing the release number from all PURLs + // https://github.com/google/osv.dev/issues/3126 + // This causes many false positive matches in Ubuntu and Alpine in particular + // when doing range-based matching. + // We had version enumeration for Alpine, and Ubuntu provides versions for us. + // Just skip range-based matching if they don't have release numbers for now. + return coarseLower, coarseUpper, func(versions []string, _ []byte) bool { + return s.versionIsMatchEcosystemVersionsList(ver, versions, eHelper) + } + } + + return coarseLower, coarseUpper, func(versions []string, events []byte) bool { + return s.versionIsMatchEcosystem(ver, versions, events, eHelper) + } +} + +func (s *VulnerabilityStore) versionIsMatch(version, pkgName, rowEco string, versions []string, eventsJSON []byte) bool { + if rowEco == "GIT" { + return s.versionIsMatchString(version, pkgName, "GIT", versions) + } + e, ok := s.ecosystems.Get(rowEco) + if !ok { + return s.versionIsMatchString(version, pkgName, rowEco, versions) + } + parsed, err := e.Parse(version) + if err != nil { + // this version is unparsable - try falling back to string compare + return s.versionIsMatchString(version, pkgName, rowEco, versions) + } + + return s.versionIsMatchEcosystem(parsed, versions, eventsJSON, e) +} + +func (s *VulnerabilityStore) versionIsMatchString(version, pkgName, rowEco string, versions []string) bool { + // If we are doing string comparison, we must fall back to checking the "versions" field, + // since we can't really interpret the events correctly. + if slices.Contains(versions, version) { + return true + } + // Fall back to fuzzy matching + repo := "" + if rowEco == "GIT" { + repo = pkgName + } + + return s.fuzzyMatchString(version, versions, repo) +} + +func (s *VulnerabilityStore) versionIsMatchEcosystem(version eco.Version, versions []string, eventsJSON []byte, ecosystem eco.Ecosystem) bool { + // Try the fast path of checking if the version is in the versions list + if s.versionIsMatchEcosystemVersionsList(version, versions, ecosystem) { + return true + } + + // Otherwise, check it's placement in the affected ranges + if len(eventsJSON) == 0 { + return false + } + var events []AffectedEvent + if err := json.Unmarshal(eventsJSON, &events); err != nil || len(events) == 0 { + return false + } + + for _, event := range slices.Backward(events) { + parsed, err := ecosystem.Parse(event.Value) + if err != nil { + // if we can't parse an event, we can't really do anything else + // so we'll just assume it's not a match + continue + } + c, err := version.Compare(parsed) + if err != nil { + continue + } + if c < 0 { + continue + } + if event.Type == osvutil.Introduced.String() { + return true + } + if c == 0 { + return event.Type == osvutil.LastAffected.String() + } + + return false + } + + return false +} + +func (s *VulnerabilityStore) versionIsMatchEcosystemVersionsList(version eco.Version, versions []string, ecosystem eco.Ecosystem) bool { + return slices.ContainsFunc(versions, func(versionStr string) bool { + parsed, err := ecosystem.Parse(versionStr) + if err != nil { + return false + } + c, err := version.Compare(parsed) + if err != nil { + return false + } + + return c == 0 + }) +} + +func (s *VulnerabilityStore) fuzzyMatchString(version string, versionStrings []string, repoURL string) bool { + versionStrings = stripRepoPrefix(versionStrings, repoURL) + normed := normalizeTag(version) + canoned := canonicalizeVersion(version) + + return slices.ContainsFunc(versionStrings, func(v string) bool { + if normalizeTag(v) == normed { + return true + } + if canonicalizeVersion(v) == canoned { + return true + } + + return false + }) +} + +func stripRepoPrefix(versionStrings []string, repoURL string) []string { + versionStrings = slices.Clone(versionStrings) + if repoURL != "" { + // Some git tags may be prefixed with the repo name, e.g. + // https://github.com/eclipse-openj9/openj9 tags may be "openj9-0.8.0" + // Due to a bug in the original python implementation, the equivalent code was stripping the org name + // (e.g. eclipse-openj9) instead of the repo name. + // So we'll strip both org and repo names for good measure. + u, err := url.Parse(repoURL) + if err == nil { + path := strings.TrimSuffix(strings.ToLower(u.Path), "/") + parts := strings.Split(path, "/") + repoName := strings.TrimSuffix(parts[len(parts)-1], ".git") + org := "" + if len(parts) >= 2 { + org = strings.TrimSuffix(parts[len(parts)-2], ".git") + } + for i, vs := range versionStrings { + vs = strings.TrimPrefix(vs, org) + vs = strings.TrimPrefix(vs, repoName) + vs = strings.TrimPrefix(vs, "-") + versionStrings[i] = vs + } + } + } + + return versionStrings +} + +// Groups of numbers, or {RC,alpha,beta,preview}. +// We simulate the Python negative lookbehind (?= "a" && firstChar <= "z" { + if start > 0 { + prevChar := v[start-1 : start] + if isWordChar.MatchString(prevChar) { + continue + } + } + } + components = append(components, matchStr) + } + if len(components) == 0 { + return v + } + + return strings.Join(components, "-") +} + +var canonicalizeRegex = regexp.MustCompile(`[-_.]+`) + +func canonicalizeVersion(v string) string { + v = strings.ToLower(v) + v = canonicalizeRegex.ReplaceAllString(v, ".") + v = strings.Trim(v, ".") + + return v +} diff --git a/go/internal/database/postgres/vulnerability_matching_test.go b/go/internal/database/postgres/vulnerability_matching_test.go new file mode 100644 index 00000000000..c65ca5bf948 --- /dev/null +++ b/go/internal/database/postgres/vulnerability_matching_test.go @@ -0,0 +1,686 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postgres + +import ( + "bytes" + "cmp" + "context" + "encoding/json" + "errors" + "iter" + "math" + "slices" + "strings" + "testing" + + gocmp "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/osv.dev/go/internal/models" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" +) + +func TestCursors(t *testing.T) { + t.Run("MatchCommitsCursor roundtrip", func(t *testing.T) { + orig := matchCommitsCursor{afterVulnID: "GHSA-1234-5678-90ab"} + encoded := orig.String() + parsed, err := parseMatchCommitsCursor(encoded) + if err != nil { + t.Fatalf("parseMatchCommitsCursor(%q) unexpected error: %v", encoded, err) + } + if parsed != orig { + t.Errorf("parseMatchCommitsCursor(%q) = %+v, want %+v", encoded, parsed, orig) + } + }) + + t.Run("MatchCommitsCursor invalid", func(t *testing.T) { + pkgCur := matchPackagesCursor{afterVulnID: "CVE-1", afterRowID: math.MaxInt64}.String() + for _, tok := range []string{"invalid", "C:abc", "P:abc:def", "!!!not-base64!!!", pkgCur} { + if _, err := parseMatchCommitsCursor(tok); err == nil { + t.Errorf("parseMatchCommitsCursor(%q) expected error, got nil", tok) + } + } + }) + + t.Run("MatchPackagesCursor roundtrip (matched MaxInt64 and unmatched row ID)", func(t *testing.T) { + cases := []matchPackagesCursor{ + { + afterVulnID: "CVE-2026-0001", + afterRowID: math.MaxInt64, + }, + { + afterVulnID: "CVE-2026-0002", + afterRowID: 42, + }, + { + afterVulnID: "CVE-2026-0002", + afterRowID: 999999, + }, + } + for _, orig := range cases { + encoded := orig.String() + parsed, err := parseMatchPackagesCursor(encoded) + if err != nil { + t.Fatalf("parseMatchPackagesCursor(%q) unexpected error: %v", encoded, err) + } + if parsed != orig { + t.Errorf("parseMatchPackagesCursor(%q) = %+v, want %+v", encoded, parsed, orig) + } + } + }) + + t.Run("MatchPackagesCursor invalid", func(t *testing.T) { + commitCur := matchCommitsCursor{afterVulnID: "GHSA-1234"}.String() + for _, tok := range []string{"invalid", "P:abc:def", "C:abc", "!!!not-base64!!!", commitCur} { + if _, err := parseMatchPackagesCursor(tok); err == nil { + t.Errorf("parseMatchPackagesCursor(%q) expected error, got nil", tok) + } + } + }) +} + +type fakeDB struct { + commits []AffectedCommit + affectedVersions []AffectedVersion +} + +func (f *fakeDB) Exec(context.Context, string, ...any) (pgconn.CommandTag, error) { + return pgconn.CommandTag{}, nil +} + +func (f *fakeDB) Query(context.Context, string, ...any) (pgx.Rows, error) { + return nil, errors.New("not implemented") +} + +func (f *fakeDB) QueryRow(context.Context, string, ...any) pgx.Row { + return nil +} + +func (f *fakeDB) SendBatch(_ context.Context, b *pgx.Batch) pgx.BatchResults { + results := make([][]any, 0, len(b.QueuedQueries)) + for _, q := range b.QueuedQueries { + results = append(results, f.execQueued(q)) + } + + return &fakeBatchResults{results: results} +} + +func (f *fakeDB) execQueued(q *pgx.QueuedQuery) []any { + sql := q.SQL + args := q.Arguments + switch { + case strings.Contains(sql, "FROM affected_commits"): + sha := args[0].([]byte) + afterVuln := args[1].(string) + limit := int(args[2].(int32)) + var matching []string + for _, c := range f.commits { + if bytes.Equal(c.CommitSha, sha) && c.VulnID > afterVuln { + matching = append(matching, c.VulnID) + } + } + slices.Sort(matching) + if len(matching) > limit { + matching = matching[:limit] + } + rows := make([]any, len(matching)) + for i, id := range matching { + rows[i] = []any{id} + } + + return rows + + case strings.Contains(sql, "MatchPackageVersionless"): + name := args[0].(string) + baseEco := args[1].(string) + afterVuln := args[2].(string) + eco := args[3].(string) + limit := int(args[4].(int32)) + var matching []string + for _, av := range f.affectedVersions { + if av.Name == name && av.BaseEcosystem == baseEco && av.VulnID > afterVuln && slices.Contains(av.Ecosystems, eco) { + matching = append(matching, av.VulnID) + } + } + slices.Sort(matching) + matching = slices.Compact(matching) + if len(matching) > limit { + matching = matching[:limit] + } + rows := make([]any, len(matching)) + for i, id := range matching { + rows[i] = []any{id} + } + + return rows + + case strings.Contains(sql, "MatchPackageVersionedNoEcosystem"): + name := args[0].(string) + afterVuln := args[1].(string) + afterID := args[2].(int64) + limit := int(args[3].(int32)) + var matching []AffectedVersion + for _, av := range f.affectedVersions { + if av.Name == name && (av.VulnID > afterVuln || (av.VulnID == afterVuln && av.ID > afterID)) { + matching = append(matching, av) + } + } + slices.SortFunc(matching, func(a, b AffectedVersion) int { + return cmp.Or(strings.Compare(a.VulnID, b.VulnID), cmp.Compare(a.ID, b.ID)) + }) + if len(matching) > limit { + matching = matching[:limit] + } + rows := make([]any, len(matching)) + for i, av := range matching { + rows[i] = []any{av.ID, av.VulnID, av.BaseEcosystem, av.Versions, av.Events} + } + + return rows + + case strings.Contains(sql, "MatchPackageVersionedWithEcosystem"): + name := args[0].(string) + baseEco := args[1].(string) + afterVuln := args[2].(string) + afterID := args[3].(int64) + coarseUpper := args[4].(string) + coarseLower := args[5].(string) + eco := args[6].(string) + limit := int(args[7].(int32)) + var matching []AffectedVersion + for _, av := range f.affectedVersions { + if av.Name == name && + av.BaseEcosystem == baseEco && + (av.VulnID > afterVuln || (av.VulnID == afterVuln && av.ID > afterID)) && + av.CoarseMin <= coarseUpper && + av.CoarseMax >= coarseLower && + slices.Contains(av.Ecosystems, eco) { + matching = append(matching, av) + } + } + slices.SortFunc(matching, func(a, b AffectedVersion) int { + return cmp.Or(strings.Compare(a.VulnID, b.VulnID), cmp.Compare(a.ID, b.ID)) + }) + if len(matching) > limit { + matching = matching[:limit] + } + rows := make([]any, len(matching)) + for i, av := range matching { + rows[i] = []any{av.ID, av.VulnID, av.Versions, av.Events} + } + + return rows + } + + return nil +} + +type fakeBatchResults struct { + results [][]any + idx int +} + +func (b *fakeBatchResults) Exec() (pgconn.CommandTag, error) { + return pgconn.CommandTag{}, nil +} + +func (b *fakeBatchResults) Query() (pgx.Rows, error) { + if b.idx >= len(b.results) { + return &fakeRows{}, nil + } + r := b.results[b.idx] + b.idx++ + + return &fakeRows{rows: r, idx: -1}, nil +} + +func (b *fakeBatchResults) QueryRow() pgx.Row { + return nil +} + +func (b *fakeBatchResults) Close() error { + return nil +} + +type fakeRows struct { + pgx.Rows + + rows []any + idx int +} + +func (r *fakeRows) Close() {} +func (r *fakeRows) Err() error { return nil } + +func (r *fakeRows) Next() bool { + r.idx++ + return r.idx < len(r.rows) +} + +func (r *fakeRows) Scan(dest ...any) error { + cols := r.rows[r.idx].([]any) + for i, d := range dest { + switch ptr := d.(type) { + case *string: + *ptr = cols[i].(string) + case *int64: + *ptr = cols[i].(int64) + case *[]string: + *ptr = cols[i].([]string) + case *[]byte: + *ptr = cols[i].([]byte) + } + } + + return nil +} + +func mustEventsJSON(t *testing.T, events []AffectedEvent) []byte { + t.Helper() + b, err := json.Marshal(events) + if err != nil { + t.Fatalf("json.Marshal events: %v", err) + } + + return b +} + +func TestVulnerabilityStore_MatchPackages_VersionRanges(t *testing.T) { + ctx := context.Background() + db := &fakeDB{ + affectedVersions: []AffectedVersion{ + { + ID: 1, + VulnID: "VULN-1", + BaseEcosystem: "npm", + Ecosystems: []string{"npm"}, + Name: "pkg-bar", + Events: mustEventsJSON(t, []AffectedEvent{ + {Type: "introduced", Value: "1.0.0"}, + {Type: "fixed", Value: "2.0.0"}, + }), + CoarseMin: "00:00000001.00000000.00000000", + CoarseMax: "00:00000002.00000000.00000000", + }, + { + ID: 2, + VulnID: "VULN-2", + BaseEcosystem: "npm", + Ecosystems: []string{"npm"}, + Name: "pkg-bar", + Events: mustEventsJSON(t, []AffectedEvent{ + {Type: "introduced", Value: "3.0.0"}, + }), + CoarseMin: "00:00000003.00000000.00000000", + CoarseMax: MaxCoarseVersion, + }, + { + ID: 3, + VulnID: "VULN-GIT", + BaseEcosystem: "GIT", + Ecosystems: []string{"GIT"}, + Name: "github.com/example/repo", + Versions: []string{"v1.0.0", "v2.0.0"}, + CoarseMin: MinCoarseVersion, + CoarseMax: MaxCoarseVersion, + }, + { + ID: 4, + VulnID: "VULN-UNSUPPORTED", + BaseEcosystem: "UnsupportedEco", + Ecosystems: []string{"UnsupportedEco"}, + Name: "pkg-unsupported", + Versions: []string{"1.0.0", "2.0.0"}, + CoarseMin: MinCoarseVersion, + CoarseMax: MaxCoarseVersion, + }, + { + ID: 5, + VulnID: "VULN-ALPINE-NORELEASE", + BaseEcosystem: "Alpine", + Ecosystems: []string{"Alpine"}, + Name: "pkg-alpine", + Versions: []string{"1.0.0"}, + Events: mustEventsJSON(t, []AffectedEvent{ + {Type: "introduced", Value: "0"}, + {Type: "fixed", Value: "2.0.0"}, + }), + CoarseMin: MinCoarseVersion, + CoarseMax: MaxCoarseVersion, + }, + { + ID: 6, + VulnID: "VULN-ALPINE-RELEASE", + BaseEcosystem: "Alpine", + Ecosystems: []string{"Alpine", "Alpine:v3.19"}, + Name: "pkg-alpine", + Events: mustEventsJSON(t, []AffectedEvent{ + {Type: "introduced", Value: "0"}, + {Type: "fixed", Value: "2.0.0"}, + }), + CoarseMin: MinCoarseVersion, + CoarseMax: MaxCoarseVersion, + }, + { + ID: 7, + VulnID: "VULN-MULTI", + BaseEcosystem: "npm", + Ecosystems: []string{"npm"}, + Name: "pkg-multi", + Events: mustEventsJSON(t, []AffectedEvent{ + {Type: "introduced", Value: "1.0.0"}, + }), + CoarseMin: "00:00000001.00000000.00000000", + CoarseMax: MaxCoarseVersion, + }, + { + ID: 8, + VulnID: "VULN-MULTI", + BaseEcosystem: "npm", + Ecosystems: []string{"npm"}, + Name: "pkg-multi", + Events: mustEventsJSON(t, []AffectedEvent{ + {Type: "introduced", Value: "2.0.0"}, + }), + CoarseMin: "00:00000002.00000000.00000000", + CoarseMax: MaxCoarseVersion, + }, + }, + } + + store := NewVulnerabilityStore(VulnStoreConfig{DB: db, PageSize: 2}) + + tests := []struct { + name string + ecosystem string + pkgName string + version string + want []string + }{ + { + name: "npm version range below bounds", + ecosystem: "npm", + pkgName: "pkg-bar", + version: "0.9.0", + want: nil, + }, + { + name: "npm version range within bounds", + ecosystem: "npm", + pkgName: "pkg-bar", + version: "1.5.0", + want: []string{"VULN-1"}, + }, + { + name: "npm version range at fixed boundary", + ecosystem: "npm", + pkgName: "pkg-bar", + version: "2.0.0", + want: nil, + }, + { + name: "npm version range unbounded introduced", + ecosystem: "npm", + pkgName: "pkg-bar", + version: "3.5.0", + want: []string{"VULN-2"}, + }, + { + name: "GIT ecosystem matching with .git suffix", + ecosystem: "GIT", + pkgName: "https://github.com/example/repo.git", + version: "v1.0.0", + want: []string{"VULN-GIT"}, + }, + { + name: "Unsupported ecosystem fallback", + ecosystem: "UnsupportedEco", + pkgName: "pkg-unsupported", + version: "1.0.0", + want: []string{"VULN-UNSUPPORTED"}, + }, + { + name: "Empty ecosystem query matching", + ecosystem: "", + pkgName: "pkg-bar", + version: "1.5.0", + want: []string{"VULN-1"}, + }, + { + name: "Alpine without release skips range matching", + ecosystem: "Alpine", + pkgName: "pkg-alpine", + version: "1.5.0", + want: nil, + }, + { + name: "Alpine without release matches explicit versions list", + ecosystem: "Alpine", + pkgName: "pkg-alpine", + version: "1.0.0", + want: []string{"VULN-ALPINE-NORELEASE"}, + }, + { + name: "Alpine with release enables range matching", + ecosystem: "Alpine:v3.19", + pkgName: "pkg-alpine", + version: "1.5.0", + want: []string{"VULN-ALPINE-RELEASE"}, + }, + { + name: "Deduplication across multiple rows for same vuln", + ecosystem: "npm", + pkgName: "pkg-multi", + version: "2.5.0", + want: []string{"VULN-MULTI"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var got []string + for match, err := range store.MatchPackages(ctx, tt.ecosystem, tt.pkgName, tt.version, "") { + if err != nil { + t.Fatalf("MatchPackages() error: %v", err) + } + if match.IsMatch { + got = append(got, match.ID) + } + } + if diff := gocmp.Diff(tt.want, got, cmpopts.EquateEmpty()); diff != "" { + t.Errorf("MatchPackages() mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestVulnerabilityStore_PaginationAndBatch(t *testing.T) { + ctx := context.Background() + commitA := []byte("0123456789abcdef0123") + db := &fakeDB{ + commits: []AffectedCommit{ + {CommitSha: commitA, VulnID: "VULN-A"}, + {CommitSha: commitA, VulnID: "VULN-B"}, + {CommitSha: commitA, VulnID: "VULN-C"}, + {CommitSha: commitA, VulnID: "VULN-D"}, + {CommitSha: commitA, VulnID: "VULN-E"}, + }, + affectedVersions: []AffectedVersion{ + { + ID: 1, + VulnID: "VULN-MULTI", + BaseEcosystem: "npm", + Ecosystems: []string{"npm"}, + Name: "pkg-pagemulti", + Versions: []string{"1.0.0"}, + CoarseMin: MinCoarseVersion, + CoarseMax: MaxCoarseVersion, + }, + { + ID: 2, + VulnID: "VULN-MULTI", + BaseEcosystem: "npm", + Ecosystems: []string{"npm"}, + Name: "pkg-pagemulti", + Versions: []string{"1.0.0"}, + CoarseMin: MinCoarseVersion, + CoarseMax: MaxCoarseVersion, + }, + { + ID: 3, + VulnID: "VULN-NEXT", + BaseEcosystem: "npm", + Ecosystems: []string{"npm"}, + Name: "pkg-pagemulti", + Versions: []string{"1.0.0"}, + CoarseMin: MinCoarseVersion, + CoarseMax: MaxCoarseVersion, + }, + }, + } + + store := NewVulnerabilityStore(VulnStoreConfig{DB: db, PageSize: 2}) + + t.Run("MatchCommits pagination across internal pages and user cursors", func(t *testing.T) { + fetch := func(cursor string, limit int) ([]string, string) { + var res []string + var lastCur string + next, stop := iter.Pull2(store.MatchCommits(ctx, commitA, cursor)) + defer stop() + for range limit { + m, err, ok := next() + if !ok { + break + } + if err != nil { + t.Fatalf("MatchCommits error: %v", err) + } + if m.IsMatch { + res = append(res, m.ID) + } + lastCur = m.Cursor() + } + + return res, lastCur + } + + res1, cur1 := fetch("", 3) + if diff := gocmp.Diff([]string{"VULN-A", "VULN-B", "VULN-C"}, res1); diff != "" { + t.Errorf("page 1 mismatch:\n%s", diff) + } + res2, cur2 := fetch(cur1, 3) + if diff := gocmp.Diff([]string{"VULN-D", "VULN-E"}, res2); diff != "" { + t.Errorf("page 2 mismatch:\n%s", diff) + } + res3, _ := fetch(cur2, 3) + if len(res3) != 0 { + t.Errorf("expected empty page 3, got %v", res3) + } + }) + + t.Run("MatchPackages duplicate ID across cursor", func(t *testing.T) { + fetch := func(cursor string, limit int) ([]string, string) { + var res []string + var lastCur string + next, stop := iter.Pull2(store.MatchPackages(ctx, "npm", "pkg-pagemulti", "1.0.0", cursor)) + defer stop() + for { + m, err, ok := next() + if !ok { + break + } + if err != nil { + t.Fatalf("MatchPackages error: %v", err) + } + lastCur = m.Cursor() + if m.IsMatch { + res = append(res, m.ID) + if len(res) >= limit { + break + } + } + } + + return res, lastCur + } + + res1, cur1 := fetch("", 1) + if diff := gocmp.Diff([]string{"VULN-MULTI"}, res1); diff != "" { + t.Errorf("page 1 mismatch:\n%s", diff) + } + res2, _ := fetch(cur1, 1) + if diff := gocmp.Diff([]string{"VULN-NEXT"}, res2); diff != "" { + t.Errorf("page 2 mismatch:\n%s", diff) + } + }) + + t.Run("MatchPackagesBatch and MatchCommitsBatch", func(t *testing.T) { + pkgIters, err := store.MatchPackagesBatch(ctx, []models.PackageQuery{ + {Ecosystem: "npm", Name: "pkg-pagemulti", Version: "1.0.0"}, + {Ecosystem: "npm", Name: "pkg-pagemulti", Version: ""}, + {Ecosystem: "", Name: "pkg-pagemulti", Version: "1.0.0"}, + }) + if err != nil { + t.Fatalf("MatchPackagesBatch error: %v", err) + } + for i, it := range pkgIters { + var got []string + for m, err := range it { + if err != nil { + t.Fatalf("query %d error: %v", i, err) + } + if m.IsMatch { + got = append(got, m.ID) + } + } + if diff := gocmp.Diff([]string{"VULN-MULTI", "VULN-NEXT"}, got); diff != "" { + t.Errorf("query %d mismatch:\n%s", i, diff) + } + } + + commitIters, err := store.MatchCommitsBatch(ctx, []models.CommitQuery{ + {Commit: commitA}, + }) + if err != nil { + t.Fatalf("MatchCommitsBatch error: %v", err) + } + var gotCommits []string + for m, err := range commitIters[0] { + if err != nil { + t.Fatalf("commit batch iter error: %v", err) + } + if m.IsMatch { + gotCommits = append(gotCommits, m.ID) + } + } + if diff := gocmp.Diff([]string{"VULN-A", "VULN-B", "VULN-C", "VULN-D", "VULN-E"}, gotCommits); diff != "" { + t.Errorf("MatchCommitsBatch mismatch:\n%s", diff) + } + }) + + t.Run("MatchPackages versionless and no ecosystem returns error", func(t *testing.T) { + var gotErr error + for _, err := range store.MatchPackages(ctx, "", "pkg-pagemulti", "", "") { + if err != nil { + gotErr = err + break + } + } + if gotErr == nil { + t.Error("expected error when both ecosystem and version are empty, got nil") + } + }) +}