Skip to content

Commit c6d79f0

Browse files
authored
chore: remove Python API code (#5869)
Getting rid of all the Python API implementation, since our shiny new Go implementation works. - Removed the `gcp/api/` directory - Moved related Protos into `proto/` directory - Removed the `googleapis` submodule and vendored the two required protos directly - Moved ESPv2 Dockerfile to `docker` directory - Re-linked terraform symlinks to point to new locations of files - Replaced API tester python script with bash script - Replaced `build_swagger.py` script with bash script
1 parent 6c696c6 commit c6d79f0

57 files changed

Lines changed: 757 additions & 6914 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/generate-protos.yml‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ on:
44
push:
55
branches: [ master ]
66
paths:
7-
- 'gcp/api/v1/osv_service_v1.proto'
7+
- 'proto/v1/osv_service_v1.proto'
88
- 'osv/importfinding.proto'
99
- 'osv/osv-schema/proto/vulnerability.proto'
1010
workflow_dispatch:
@@ -41,9 +41,7 @@ jobs:
4141
run: pip install poetry==2.3.3
4242

4343
- name: Set up poetry
44-
run: |
45-
poetry install
46-
cd gcp/api && poetry install
44+
run: poetry install
4745

4846
- name: Set up Go
4947
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0

‎.gitignore‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ build/lib
1515
hurl-scripts/
1616
temp/*
1717
**/tmp/**
18-
gcp/api/v1/osv/**
1918
.hypothesis
2019
go/api-devserver
2120
go/api

‎.gitmodules‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,3 @@
1-
[submodule "gcp/api/googleapis"]
2-
path = gcp/api/googleapis
3-
url = https://github.com/googleapis/googleapis
41
[submodule "docker/worker/osv-test"]
52
path = gcp/workers/worker/osv-test
63
url = https://github.com/oliverchang/osv-test.git

‎AGENTS.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -250,7 +250,7 @@ Contains deployment setups, workers running in GKE, Cloud Functions, and the use
250250
- **Status**: **Active (Go)**.
251251
- Serves the public OSV gRPC API server (transcoded to HTTP/JSON REST via ESPv2).
252252
- **Deployment Target**: **Google Cloud Run** (managed via Cloud Deploy pipeline `osv-api` deploying to `osv-grpc-backend`).
253-
- *Note*: Fully migrated from Python to Go. The legacy Python implementation remains in `gcp/api/` but is retired.
253+
- *Note*: Fully migrated from Python to Go. Protobuf definitions and descriptor files are located under `proto/v1/`.
254254

255255
### 2. Website (`gcp/website/`)
256256
- **Status**: **Active**.

‎Makefile‎

Lines changed: 17 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -44,17 +44,10 @@ go-tests: ## Run Go services tests
4444
cd go && ./run_tests.sh
4545

4646
api-server-tests: ## Run Go API server integration tests
47-
test -f $(HOME)/.config/gcloud/application_default_credentials.json || (echo "GCP Application Default Credentials not set, try 'gcloud auth application-default login'"; exit 1)
48-
cd go && go build -o ./api ./cmd/api
49-
cd gcp/api && docker build -f Dockerfile.esp -t osv/esp:latest .
50-
cd gcp/api && OSV_USE_GO_BACKEND=1 ./run_tests.sh $(HOME)/.config/gcloud/application_default_credentials.json
51-
cd gcp/api && OSV_USE_GO_BACKEND=1 ./run_tests_e2e.sh $(HOME)/.config/gcloud/application_default_credentials.json
47+
./tools/apitester/run_tests.sh
5248

5349
update-api-snapshots: ## Update API query snapshots
54-
test -f $(HOME)/.config/gcloud/application_default_credentials.json || (echo "GCP Application Default Credentials not set, try 'gcloud auth application-default login'"; exit 1)
55-
cd go && go build -o ./api ./cmd/api
56-
cd gcp/api && docker build -f Dockerfile.esp -t osv/esp:latest .
57-
cd gcp/api && UPDATE_SNAPS=true OSV_USE_GO_BACKEND=1 ./run_tests_e2e.sh $(HOME)/.config/gcloud/application_default_credentials.json
50+
UPDATE_SNAPS=true ./tools/apitester/run_tests.sh
5851

5952
lint: ## Run linters and format checks
6053
GOTOOLCHAIN=auto $(run-cmd) tools/lint_and_format.sh
@@ -63,33 +56,33 @@ build-osv-protos:
6356
cd osv && $(run-cmd) python -m grpc_tools.protoc --python_out=. --mypy_out=. --proto_path=. --proto_path=osv-schema/proto vulnerability.proto importfinding.proto
6457

6558
build-api-protos:
66-
cd gcp/api/v1 && $(run-cmd) python -m grpc_tools.protoc \
59+
cd proto/v1 && protoc \
6760
--include_imports \
6861
--include_source_info \
69-
--proto_path=googleapis \
62+
--proto_path=.. \
7063
--proto_path=. \
71-
--proto_path=osv \
72-
--proto_path=osv/osv-schema/proto \
64+
--proto_path=../../osv \
65+
--proto_path=../../osv/osv-schema/proto \
7366
--descriptor_set_out=api_descriptor.pb \
74-
--python_out=.. \
75-
--grpc_python_out=.. \
76-
--mypy_out=.. \
7767
vulnerability.proto importfinding.proto osv_service_v1.proto
7868
cd osv && protoc \
7969
--proto_path=. \
8070
--go_out=paths=source_relative:../bindings/go/api \
8171
importfinding.proto
82-
cd gcp/api/v1 && protoc \
83-
--proto_path=googleapis \
72+
cd proto/v1 && protoc \
73+
--proto_path=.. \
8474
--proto_path=. \
85-
--proto_path=osv \
86-
--proto_path=osv/osv-schema/proto \
87-
--go_out=paths=source_relative:../../../bindings/go/api \
88-
--go-grpc_out=paths=source_relative:../../../bindings/go/api \
75+
--proto_path=../../osv \
76+
--proto_path=../../osv/osv-schema/proto \
77+
--go_out=paths=source_relative:../../bindings/go/api \
78+
--go-grpc_out=paths=source_relative:../../bindings/go/api \
8979
osv_service_v1.proto
9080

9181
build-protos: build-osv-protos build-api-protos ## Build all protocol buffers
9282

83+
build-swagger: ## Build Swagger/OpenAPI documentation
84+
./docs/build_swagger.sh
85+
9386
build-website-frontend:
9487
cd website/frontend3 && pnpm install && pnpm run build
9588
cd website/blog && hugo --buildFuture -d ../dist/static/blog
@@ -127,26 +120,15 @@ run-go-website-prod: stage-website-assets
127120
# Run with `make run-api-server ARGS=--no-backend` to launch esp without backend.
128121
run-api-server: ## Run local Go API server with ESPv2 proxy
129122
test -f $(HOME)/.config/gcloud/application_default_credentials.json || (echo "GCP Application Default Credentials not set, try 'gcloud auth application-default login'"; exit 1)
130-
docker inspect osv/esp:latest >/dev/null 2>&1 || (cd gcp/api && docker build -f Dockerfile.esp -t osv/esp:latest .)
123+
docker inspect osv/esp:latest >/dev/null 2>&1 || docker build -f docker/esp/Dockerfile -t osv/esp:latest docker/esp
131124
@cd go && go build -o ./api-devserver ./cmd/api-devserver && (GOOGLE_CLOUD_PROJECT=oss-vdb OSV_VULNERABILITIES_BUCKET=osv-vulnerabilities ./api-devserver $(ARGS); EXIT_CODE=$$?; rm -f ./api-devserver; exit $$EXIT_CODE)
132125

133126
# Run the Go developer server orchestrator against the staging/test environment.
134127
run-api-server-test:
135128
test -f $(HOME)/.config/gcloud/application_default_credentials.json || (echo "GCP Application Default Credentials not set, try 'gcloud auth application-default login'"; exit 1)
136-
docker inspect osv/esp:latest >/dev/null 2>&1 || (cd gcp/api && docker build -f Dockerfile.esp -t osv/esp:latest .)
129+
docker inspect osv/esp:latest >/dev/null 2>&1 || docker build -f docker/esp/Dockerfile -t osv/esp:latest docker/esp
137130
@cd go && go build -o ./api-devserver ./cmd/api-devserver && (GOOGLE_CLOUD_PROJECT=oss-vdb-test OSV_VULNERABILITIES_BUCKET=osv-test-vulnerabilities ./api-devserver $(ARGS); EXIT_CODE=$$?; rm -f ./api-devserver; exit $$EXIT_CODE)
138131

139-
# Legacy Python API server targets
140-
run-python-api-server:
141-
test -f $(HOME)/.config/gcloud/application_default_credentials.json || (echo "GCP Application Default Credentials not set, try 'gcloud auth application-default login'"; exit 1)
142-
cd gcp/api && docker build -f Dockerfile.esp -t osv/esp:latest .
143-
cd gcp/api && $(install-cmd) && GOOGLE_CLOUD_PROJECT=oss-vdb OSV_VULNERABILITIES_BUCKET=osv-vulnerabilities $(run-cmd) python test_server.py $(HOME)/.config/gcloud/application_default_credentials.json $(ARGS)
144-
145-
run-python-api-server-test:
146-
test -f $(HOME)/.config/gcloud/application_default_credentials.json || (echo "GCP Application Default Credentials not set, try 'gcloud auth application-default login'"; exit 1)
147-
cd gcp/api && docker build -f Dockerfile.esp -t osv/esp:latest .
148-
cd gcp/api && $(install-cmd) && GOOGLE_CLOUD_PROJECT=oss-vdb-test OSV_VULNERABILITIES_BUCKET=osv-test-vulnerabilities $(run-cmd) python test_server.py $(HOME)/.config/gcloud/application_default_credentials.json $(ARGS)
149-
150132
# TODO: API integration tests.
151133
all-tests: lib-tests worker-tests importer-tests recoverer-tests website-tests vulnfeed-tests bindings-tests go-tests ## Run all tests
152134

‎README.md‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,8 @@ consists of:
3838
|-----------------|------|
3939
| `bindings/` | Language bindings for the [OSV API](https://google.github.io/osv.dev/api/) (currently Go only) |
4040
| `deployment/` | Terraform & Cloud Deploy config files <br /> A few Cloud Build config yamls |
41-
| `docker/` | CI docker files (`ci`, `deployment`, `terraform`) <br /> `worker-base` docker image for `gcp/workers/worker` |
42-
| `docs/` | Jekyll files for https://google.github.io/osv.dev/ <br /> `build_swagger.py` and `tools.go` |
43-
| `gcp/api` | OSV API server files (including files for the local ESP server) <br /> protobuf files in `/v1`|
41+
| `docker/` | CI docker files (`ci`, `deployment`, `terraform`, `esp`) <br /> `worker-base` docker image for `gcp/workers/worker` |
42+
| `docs/` | Jekyll files for https://google.github.io/osv.dev/ <br /> `build_swagger.sh` and `tools.go` |
4443
| `gcp/datastore` | The datastore index file (`index.yaml`) |
4544
| `gcp/functions` | The Cloud Function for publishing PyPI vulnerabilities (maintained, but not developed) |
4645
| `gcp/indexer` | The determine version `indexer` |

‎bindings/go/api/importfinding.pb.go‎

Lines changed: 2 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎bindings/go/api/osv_service_v1.pb.go‎

Lines changed: 2 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎bindings/go/api/osv_service_v1_grpc.pb.go‎

Lines changed: 2 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎cloudbuild.yaml‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,11 +32,6 @@ steps:
3232
args: ['gcloud', 'builds', 'submit', '--region=${LOCATION}', '--config=gcp/website/cloudbuild.yaml', '.']
3333
waitFor: ['init']
3434

35-
- name: 'gcr.io/google.com/cloudsdktool/cloud-sdk:slim'
36-
id: 'api-tests'
37-
args: ['gcloud', 'builds', 'submit', '--region=${LOCATION}', '--config=gcp/api/cloudbuild.yaml', '.']
38-
waitFor: ['init']
39-
4035
- name: 'gcr.io/google.com/cloudsdktool/cloud-sdk:slim'
4136
id: 'vulnfeeds-tests'
4237
args: ['gcloud', 'builds', 'submit', '--region=${LOCATION}', '--config=vulnfeeds/cloudbuild.yaml', '.']

0 commit comments

Comments
 (0)